Compute Library
 22.05
GemmInterleaved< strategy, To, Tr, OutputStage, MergeStep, ForceThreadColumns > Class Template Reference

#include <gemm_interleaved.hpp>

Collaboration diagram for GemmInterleaved< strategy, To, Tr, OutputStage, MergeStep, ForceThreadColumns >:
[legend]

Public Member Functions

 GemmInterleaved (GemmInterleaved &)=delete
 
GemmInterleavedoperator= (GemmInterleaved &)=delete
 
 GemmInterleaved (const GemmArgs &args, const OutputStage &os)
 
 GemmInterleaved (const GemmArgs &args)
 
ndrange_t get_window_size () const override
 
void set_nthreads (int nthreads) override
 
void execute (const ndcoord_t &work_range, const ndcoord_t &, int threadid) override
 Main execute member fucntion. More...
 
size_t get_working_size () const override
 
void set_working_space (void *working_space) override
 
bool B_is_pretransposed () const override
 
bool B_pretranspose_required () const override
 
size_t get_B_pretransposed_array_size () const override
 
void requantize_bias (void *in_buffer, const To *B, const int ldb, const int B_multi_stride) override
 
void pretranspose_B_array (void *in_buffer, const To *B, const int ldb, const int B_multi_stride) override
 
void set_pretransposed_B_data (void *in_buffer) override
 
void set_quantized_bias (const int32_t *bias, size_t bias_multi_stride) override
 
void set_indirect_parameters (size_t string_len, const To *const *const *ptr) override
 
void set_convolution_parameters (ConvolutionParameters parms) override
 
GemmConfig get_config () override
 
- Public Member Functions inherited from GemmCommon< To, Tr >
virtual void set_arrays (const To *A, const int lda, const int A_batch_stride, const int A_multi_stride, const To *B, const int ldb, const int B_multi_stride, Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride, const Tr *bias, const int bias_multi_stride)
 
void set_arrays_generic (const void *A, const int lda, const int A_batch_stride, const int A_multi_stride, const void *B, const int ldb, const int B_multi_stride, void *C, const int ldc, const int C_batch_stride, const int C_multi_stride, const void *bias, const int bias_multi_stride) override
 
void pretranspose_B_array_generic (void *out, const void *in, const int row_stride, const int multi_stride) override
 
void set_indirect_parameters_generic (size_t sz, const void *const *const *ptr) override
 
- Public Member Functions inherited from IGemmCommon
virtual bool supports_dynamic_scheduling () const
 
virtual ~IGemmCommon ()
 

Static Public Member Functions

template<typename perf_type >
static uint64_t estimate_cycles (const GemmArgs &args)
 

Detailed Description

template<typename strategy, typename To, typename Tr, typename OutputStage = Nothing, bool MergeStep = true, bool ForceThreadColumns = false>
class arm_gemm::GemmInterleaved< strategy, To, Tr, OutputStage, MergeStep, ForceThreadColumns >

Definition at line 252 of file gemm_interleaved.hpp.

Constructor & Destructor Documentation

◆ GemmInterleaved() [1/3]

GemmInterleaved ( GemmInterleaved< strategy, To, Tr, OutputStage, MergeStep, ForceThreadColumns > &  )
delete

◆ GemmInterleaved() [2/3]

GemmInterleaved ( const GemmArgs args,
const OutputStage &  os 
)
inline

Definition at line 578 of file gemm_interleaved.hpp.

579  : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
580  _Ksections(args._Ksections), _Ktotal(get_ktotal(args)),
581  _rounded_Ksize(roundup(_Ksize, strategy::k_unroll())),
582  _nbatches(args._nbatches), _nmulti(args._nmulti), _thread_columns(is_thread_columns(args)),
583  _act(args._act), _maxthreads(args._maxthreads), _nthreads(args._maxthreads),
584  _k_block(get_k_block_size(args)), _x_block(get_x_block_size(args)), _Mround(roundup(args._Msize, strategy::out_height())),
585  _os(os) { }
T roundup(const T a, const T b)
Definition: utils.hpp:70

◆ GemmInterleaved() [3/3]

GemmInterleaved ( const GemmArgs args)
inline

Definition at line 588 of file gemm_interleaved.hpp.

589  : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
590  _Ksections(args._Ksections), _Ktotal(get_ktotal(args)),
591  _rounded_Ksize(roundup(_Ksize, strategy::k_unroll())),
592  _nbatches(args._nbatches), _nmulti(args._nmulti), _thread_columns(is_thread_columns(args)),
593  _act(args._act), _maxthreads(args._maxthreads), _nthreads(args._maxthreads),
594  _k_block(get_k_block_size(args)), _x_block(get_x_block_size(args)), _Mround(roundup(args._Msize, strategy::out_height())),
595  _os() { }
T roundup(const T a, const T b)
Definition: utils.hpp:70

Member Function Documentation

◆ B_is_pretransposed()

bool B_is_pretransposed ( ) const
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 912 of file gemm_interleaved.hpp.

912  {
913  return true;
914  }

◆ B_pretranspose_required()

bool B_pretranspose_required ( ) const
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 916 of file gemm_interleaved.hpp.

916  {
917  return (_B_transposed==nullptr);
918  }

◆ estimate_cycles()

static uint64_t estimate_cycles ( const GemmArgs args)
inlinestatic

Definition at line 1035 of file gemm_interleaved.hpp.

References GemmArgs::_ci, GemmArgs::_Ksize, GemmArgs::_maxthreads, GemmArgs::_Msize, GemmArgs::_nbatches, GemmArgs::_nmulti, GemmArgs::_Nsize, arm_gemm::iceildiv(), PerformanceParameters::kernel_macs_cycle, PerformanceParameters::merge_bytes_cycle, PerformanceParameters::prepare_bytes_cycle, and arm_gemm::roundup().

1035  {
1036  unsigned int k_blocks = iceildiv(args._Ksize, get_k_block_size(args));
1037 
1038  const PerformanceParameters &params = strategy::template get_performance_parameters<perf_type>(args._ci);
1039 
1040  uint64_t total_macs = static_cast<uint64_t>(args._nbatches) * args._nmulti * roundup(args._Msize, strategy::out_height()) * roundup(args._Nsize, strategy::out_width()) * get_ktotal(args);
1041  uint64_t prepare_bytes = static_cast<uint64_t>(args._nbatches) * args._nmulti * roundup(args._Msize, strategy::out_height()) * get_ktotal(args) * sizeof(Toi);
1042  uint64_t merge_bytes = static_cast<uint16_t>(args._nbatches) * args._nmulti * k_blocks * args._Msize * roundup(args._Nsize, strategy::out_width()) * sizeof(Tr);
1043 
1044  float mac_cycles = static_cast<float>(total_macs) / params.kernel_macs_cycle;
1045  float prepare_cycles = static_cast<float>(prepare_bytes) / params.prepare_bytes_cycle;
1046  float merge_cycles = static_cast<float>(merge_bytes) / params.merge_bytes_cycle;
1047 
1048  float total_cycles = mac_cycles + prepare_cycles + merge_cycles;
1049 
1050  // We can't thread over multis or width, which makes this a poor
1051  // choice in many threaded cases. Penalize that here.
1052  float parallelism_available = static_cast<float>(iceildiv(args._Msize, strategy::out_height()) * args._nbatches) * 0.9f;
1053 
1054  if (parallelism_available < args._maxthreads) {
1055  total_cycles *= (static_cast<float>(args._maxthreads) / parallelism_available);
1056  }
1057 
1058  return static_cast<uint64_t>(total_cycles);
1059  }
T roundup(const T a, const T b)
Definition: utils.hpp:70
T iceildiv(const T a, const T b)
Definition: utils.hpp:65

◆ execute()

void execute ( const ndcoord_t work_range,
const ndcoord_t thread_locator,
int  threadid 
)
inlineoverridevirtual

Main execute member fucntion.

Parameters
[in]work_rangespecifies the range of work we want to be computed, total range defined by get_window_size()
[in]thread_locatorwhere are we inside of the thread space
[in]threadida unique threadid

Implements IGemmCommon.

Definition at line 620 of file gemm_interleaved.hpp.

References arm_compute::test::validation::batch, arm_compute::mlgo::parser::end(), NDCoordinate< N >::get_position(), NDCoordinate< N >::get_position_end(), arm_gemm::roundup(), arm_compute::test::validation::run(), strategy, and type.

620  {
621 #ifdef CYCLE_PROFILING
622  profiler prof;
623 #endif
624 
625  /* Make sure we've been set up correctly. */
626  assert(_B_transposed);
627  assert(_working_space);
628  int8_t *working_space_bytes = reinterpret_cast<int8_t *>(_working_space);
629 
630  /* Align if needed */
631  intptr_t working_space_v = reinterpret_cast<intptr_t>(_working_space);
632  if (working_space_v & 0x3f) {
633  intptr_t alignment_offset = 0x40 - (working_space_v & 0x3f);
634  working_space_bytes += alignment_offset;
635  }
636 
637  strategy strat(_ci);
638 
639  const auto start = work_range.get_position(0);
640  const auto end = work_range.get_position_end(0);
641 
642  /* Translate 'start' and 'end' into a position within the batches and rows. */
643  const unsigned int window_per_batch = _Mround / strategy::out_height();
644  unsigned int batch_0 = start / window_per_batch;
645  unsigned int batch_end = end / window_per_batch;
646 
647  // In ThreadColumns mode, process work one horizontal strip at a time.
648  // Transpose the block of needed rows at the start, then do all the work on that block.
649  if (_thread_columns) {
650  const auto start_x = work_range.get_position(1) * strategy::out_width();
651  const auto end_x = std::min(work_range.get_position_end(1) * strategy::out_width(), _Nsize);
652 
653  Tri * const c_panel = reinterpret_cast<Tri *>(working_space_bytes + (threadid * get_c_working_size()));
654  Toi * const a_panel = reinterpret_cast<Toi *>(working_space_bytes + (_maxthreads * get_c_working_size()) +
655  (threadid * sizeof(Toi) * get_total_k_depth() * strategy::out_height()));
656 
657  for (unsigned int multi=0; multi<_nmulti; multi++) {
658  for (unsigned int k0=0; k0<_Ktotal; k0+=_k_block) {
659  unsigned int kmax=std::min(k0+_k_block, _Ktotal);
660 
661  unsigned int rounded_width = roundup(_Nsize, strategy::out_width());
662 
663  const bool first_pass = (k0==0);
664  const bool last_pass = (kmax==_Ktotal);
665 
666  // Figure out how many "K" the kernel will actually process.
667  unsigned int kern_k = roundup(kmax - k0, strategy::k_unroll());
668 
669  const Toi *b_ptr = _B_transposed + (rounded_width * _Ktotal * multi) + (k0 * rounded_width) + (start_x * kern_k);
670 
671  unsigned int batch = batch_0;
672  unsigned int start_row = (start - (batch_0 * window_per_batch)) * strategy::out_height();
673 
674  for (unsigned int p=start; p<end; p++) {
675  unsigned int end_row = std::min(start_row + strategy::out_height(), _Msize);
676 
677  // Set up transposed 'A' block
678  {
679 #ifdef CYCLE_PROFILING
680  auto p=prof.ScopedProfiler(PROFILE_PREPA, strategy::out_height() * (kmax-k0) * sizeof(Toi));
681 #endif
682  // See comment above on transform_type<> class: this extracts either 'transforms' or
683  // 'transforms_quantized' as appropriate.
684  typename transform_type<strategy, MergeStep && std::is_same<OutputStage, Requantize32>::value>::type transforms;
685 
686  if (_indirect_buf != nullptr) {
687  transforms.PrepareA_indirect(a_panel,
688  _indirect_buf + (multi * _nbatches * _Ksections) + (batch * _Ksections), _Ksize,
689  _rounded_Ksize, start_row, end_row, k0, kmax, row_sum_multiplier());
690  } else if (_convolver) {
691  transforms.PrepareA_convolution(a_panel,
692  this->_Aptr + (batch * this->_A_batch_stride) + (multi * this->_A_multi_stride),
693  this->_lda, *_convolver, _rounded_Ksize, start_row, end_row, k0, kmax, row_sum_multiplier());
694  } else {
695  transforms.PrepareA(a_panel,
696  this->_Aptr + (batch * this->_A_batch_stride) + (multi * this->_A_multi_stride),
697  this->_lda, start_row, end_row, k0, std::min(kmax, _Ksize), row_sum_multiplier());
698  }
699  }
700 
701  // Perform the kernel and merge step, either separately or together as required.
703  #ifdef CYCLE_PROFILING
704  prof,
705  #endif
706  // Strategy and panel pointers
707  strat, a_panel, b_ptr, c_panel,
708  // Result buffer pointers
709  this->_Cptr + (batch * this->_C_batch_stride) + (multi * this->_C_multi_stride), this->_ldc,
710  // K size, and M/N ranges
711  kern_k, start_row, end_row, start_x, end_x,
712  // Only do bias on the first pass
713  ((first_pass && this->_bias) ? this->_bias + (multi * this->_bias_multi_stride) : nullptr),
714  // Only do activation on the last pass, and accumulation on any non-first pass.
715  (last_pass ? _act : Activation()), !first_pass,
716  // Pass in quantization parameters for requantizing kernels (others will ignore)
717  _os, col_bias + (multi * _Nsize),
718  // Accumulation buffer (not yet implemented on this path)
719  static_cast<Tab *>(nullptr));
720 
721  /* Increment to the next block */
722  start_row += strategy::out_height();
723  if (start_row >= _Msize) {
724  start_row = 0;
725  batch++;
726  }
727  }
728  }
729  }
730  } else {
731  blockwalker current(*this);
732 
733  /* Compute the M values to operate on */
734  unsigned int m_0 = (start - (batch_0 * window_per_batch)) * strategy::out_height();
735  unsigned int m_max = (end - (batch_end * window_per_batch)) * strategy::out_height();
736 
737  // Private buffers. Treat working_space as an array of C buffers
738  // (one per thread) first, followed by the (window-divided) A
739  // buffer.
740  // Set a_panel to the base of the A buffers - compute offsets into it based on M/batches later.
741  Toi * const a_panel = reinterpret_cast<Toi *>(working_space_bytes + (_maxthreads * get_c_working_size()));
742  Tri * const c_panel = reinterpret_cast<Tri *>(working_space_bytes + (threadid * get_c_working_size()));
743 
744  const Toi *b_panel;
745  b_panel = _B_transposed;
746 
747  // newkblock() is always true on the first iteration, so these will be set properly on the first loop.
748 
749  // kern_k tracks the accumulation depth for the CURRENT K block a_panel_stride similarly tracks the total
750  // stride of the A panel (i.e. with 4 added for cases with embedded row sums)
751 
752  // These are distinct from k_block and get_total_k_depth() which are based on the target K block size, and
753  // used for addressing inside a_panel.
754 
755  // In cases where K blocking is in use and the blocks are not all the same size, the (smaller) final block
756  // won't use all the memory allocated.
757  unsigned int kern_k = 0;
758  unsigned int a_panel_stride = 0;
759 
760  for (;!current.done();current.advance()) {
761  if (current.newkblock()) {
762 #ifdef CYCLE_PROFILING
763  auto p=prof.ScopedProfiler(PROFILE_PREPA, (end - start) * strategy::out_height() * (current.kmax()-current.k0()) * sizeof(Toi));
764 #endif
765  // See comment above on transform_type<> class: this extracts either 'transforms' or
766  // 'transforms_quantized' as appropriate.
767  typename transform_type<strategy, MergeStep && std::is_same<OutputStage, Requantize32>::value>::type transforms;
768 
769  for (unsigned int batch = batch_0; batch <= batch_end; batch++) {
770  unsigned int first_m = (batch == batch_0) ? m_0 : 0;
771  unsigned int last_m = (batch == batch_end) ? m_max : _Msize;
772 
773  if (first_m >= last_m)
774  continue;
775 
776  if (_indirect_buf != nullptr) {
777  transforms.PrepareA_indirect(a_panel + ((batch * _Mround + first_m) * get_total_k_depth()),
778  _indirect_buf + (current.multi() * _nbatches * _Ksections) + (batch * _Ksections), _Ksize,
779  _rounded_Ksize, first_m, last_m, current.k0(), current.kmax(), row_sum_multiplier());
780  } else if (_convolver) {
781  transforms.PrepareA_convolution(a_panel + ((batch * _Mround + first_m) * get_total_k_depth()),
782  this->_Aptr + (batch * this->_A_batch_stride) + (current.multi() * this->_A_multi_stride),
783  this->_lda, *_convolver, _rounded_Ksize, first_m, last_m, current.k0(), current.kmax(), row_sum_multiplier());
784  } else {
785  transforms.PrepareA(a_panel + ((batch * _Mround + first_m) * get_total_k_depth()),
786  this->_Aptr + (batch * this->_A_batch_stride) + (current.multi() * this->_A_multi_stride),
787  this->_lda, first_m, last_m, current.k0(), std::min(_Ksize, current.kmax()), row_sum_multiplier());
788  }
789  }
790 
791  // Figure out how many "K" the kernel will actually process.
792  kern_k = roundup(current.kmax() - current.k0(), strategy::k_unroll());
793 
794  // Requantizing GEMMs have the row sums built in to the
795  // transposed data, so the stride between rows is 4 bytes
796  // larger than the (rounded) K value.
797 
798  if(std::is_same<OutputStage, Requantize32>::value) {
799  a_panel_stride = kern_k + (sizeof(int32_t) / sizeof(Toi));
800  } else {
801  a_panel_stride = kern_k;
802  }
803  }
804 
805  /* Do the actual work. */
806  for (unsigned int batch = batch_0; batch <= batch_end; batch++) {
807  unsigned int first_m = (batch == batch_0) ? m_0 : 0;
808  unsigned int last_m = (batch == batch_end) ? m_max : _Msize;
809 
810  const Toi *a_ptr = a_panel + (batch * _Mround + first_m) * get_total_k_depth();
811 
812  if (first_m >= last_m)
813  continue;
814 
815  // For the merge case we need to do this out_height() rows
816  // at a time, as that is the size of our intermediate
817  // buffer. If we are not doing that, we can do all the
818  // relevant rows in one go.
819  unsigned int m_step = MergeStep ? strategy::out_height() : (last_m - first_m);
820 
821  // But in the case where we have an accumulation buffer, we can't do that after all, unless
822  // there is no N blocking.
823  if (_accumulation_buffer && ((current.x0() != 0) || (current.xmax() < _Nsize))) {
824  m_step = strategy::out_height();
825  }
826 
827  for (unsigned int y=first_m; y<last_m; y+=m_step) {
828  unsigned int ymax = std::min(_Msize, y + m_step);
829 
830  const bool first_pass = (current.k0() == 0);
831  const bool last_pass = (current.kmax() == _Ktotal);
832 
833  // Pointer to appropriate part of result array.
834  Tr *result_ptr = this->_Cptr + (batch * this->_C_batch_stride) + (current.multi() * this->_C_multi_stride);
835 
836  // If we are using an accumulation buffer, we don't pass the result buffer to ask the kernel
837  // to write things into the accumulation buffer instead, except on the last pass.
838  if (_accumulation_buffer && !last_pass) {
839  result_ptr = nullptr;
840  }
841 
842  // Perform the kernel and merge step, either separately or together as required.
844  #ifdef CYCLE_PROFILING
845  prof,
846  #endif
847  // Strategy and panel pointers
848  strat, a_ptr, b_panel, c_panel,
849  // Result buffer pointers
850  result_ptr, this->_ldc,
851  // K size, and M/N ranges
852  kern_k, y, ymax, current.x0(), current.xmax(),
853  // Only do bias on the first pass
854  ((first_pass && this->_bias) ? this->_bias + (current.multi() * this->_bias_multi_stride) : nullptr),
855  // Only do activation on the last pass, and accumulation on any non-first pass.
856  (last_pass ? _act : Activation()), !first_pass,
857  // Pass in quantization parameters for requantizing kernels (others will ignore)
858  _os, col_bias + (current.multi() * _Nsize),
859  // Accumulation buffer
860  get_accumulation_buffer(y, current.x0(), batch, current.multi()) );
861 
862  a_ptr += (strategy::out_height() * a_panel_stride);
863  }
864  }
865 
866  b_panel += (roundup(current.xmax() - current.x0(), strategy::out_width()) * kern_k);
867  }
868  }
869  }
T roundup(const T a, const T b)
Definition: utils.hpp:70
arm_compute::ActivationLayerInfo::ActivationFunction Activation
Constant TensorID specifying an equivalent of null tensor.
Definition: Types.h:73
decltype(strategy::transforms) typedef type
void end(TokenStream &in, bool &valid)
Definition: MLGOParser.cpp:290
const StratType * strategy

◆ get_B_pretransposed_array_size()

size_t get_B_pretransposed_array_size ( ) const
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 920 of file gemm_interleaved.hpp.

References arm_gemm::roundup().

920  {
921  unsigned int x_size = roundup(_Nsize, strategy::out_width());
922 
923  return (x_size * _Ktotal * _nmulti * sizeof(Toi)) + get_col_sum_size();
924  }
T roundup(const T a, const T b)
Definition: utils.hpp:70

◆ get_config()

GemmConfig get_config ( )
inlineoverridevirtual

Implements IGemmCommon.

Definition at line 1061 of file gemm_interleaved.hpp.

References GemmConfig::filter, arm_gemm::GEMM_INTERLEAVED, GemmConfig::inner_block_size, GemmConfig::method, and GemmConfig::outer_block_size.

1061  {
1062  GemmConfig c;
1063 
1064  c.method = GemmMethod::GEMM_INTERLEAVED;
1065  c.inner_block_size = _k_block;
1066  c.outer_block_size = _x_block;
1067  c.filter = get_type_name<strategy>();
1068 
1069  return c;
1070  }

◆ get_window_size()

ndrange_t get_window_size ( ) const
inlineoverridevirtual
Returns
an ndrange containing ranges of the compute space which can be broken up and parallelised over

Implements IGemmCommon.

Definition at line 603 of file gemm_interleaved.hpp.

References arm_gemm::iceildiv().

603  {
604  unsigned int row_blocks = (_Mround / strategy::out_height()) * _nbatches;
605 
606  if (_thread_columns) {
607  return { row_blocks, iceildiv(_Nsize, strategy::out_width()) };
608  } else {
609  // _Mround is a multiple of out_height by definition.
610  return { row_blocks };
611  }
612  }
T iceildiv(const T a, const T b)
Definition: utils.hpp:65

◆ get_working_size()

size_t get_working_size ( void  ) const
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 872 of file gemm_interleaved.hpp.

872  {
873  // In all cases, we need one A buffer plus a C buffer per thread, plus an accumulation buffer.
874  size_t size = get_a_working_size() + (get_c_working_size() * _maxthreads) + get_accumulation_buffer_size();
875 
876  size += 128; // Add on two cache lines extra for alignment.
877 
878  return size;
879  }

◆ operator=()

GemmInterleaved& operator= ( GemmInterleaved< strategy, To, Tr, OutputStage, MergeStep, ForceThreadColumns > &  )
delete

◆ pretranspose_B_array()

void pretranspose_B_array ( void *  in_buffer,
const To *  B,
const int  ldb,
const int  B_multi_stride 
)
inlineoverridevirtual

Reimplemented from GemmCommon< To, Tr >.

Definition at line 939 of file gemm_interleaved.hpp.

References arm_gemm::roundup(), and strategy.

939  {
940  requantize_bias(in_buffer, B, ldb, B_multi_stride);
941 
942  // Put the transposed data after the column sums - in non-transposing cases get_col_sum_size() == 0
943  uintptr_t buffer_int = reinterpret_cast<uintptr_t>(in_buffer);
944  Toi *buffer = reinterpret_cast<Toi *>(buffer_int + get_col_sum_size());
945  _B_transposed = buffer;
946 
947  blockwalker current(*this);
948  strategy strat(_ci);
949 
950  do {
951  /* Figure out the size of each block. */
952  unsigned int k_size = (current.kmax() - current.k0());
953 
954  if (_Ksections > 1) {
955  // We need to insert padding at the end of each K section.
956  // The computation needed is a little delicate - the coordinates from the block walker are expressed in
957  // terms of the full, padded, _Ktotal.
958  // But we need to transform each section with reference to the original, unpadded, input, letting the
959  // transform pad each section as needed.
960 
961  // This is needed for computations below.
962  const unsigned int rounded_section_size = roundup(_Ksize, strategy::k_unroll());
963 
964  // The expected output format is also an entire <out_width> columns interleaved, then the next set of
965  // columns, and so on. This means, as we are breaking it up vertically, we have to do it one column at
966  // a time.
967  for (unsigned int x0=current.x0(); x0 < current.xmax(); x0 += strategy::out_width() ) {
968  unsigned int xmax = std::min(x0 + strategy::out_width(), current.xmax());
969 
970  // Track where we are and how much work is left.
971  unsigned int kpos = current.k0();
972  unsigned int kleft = k_size;
973 
974  while (kleft) {
975  // Which section are we in? Based on the rounded-up section size.
976  unsigned int k_section_base = kpos / rounded_section_size;
977  // How far into the section are we?
978  unsigned int k_offset = kpos - (k_section_base * rounded_section_size);
979 
980  // We will either copy the rest of this section, or to the end of the requested length.
981  unsigned int k_length = std::min(_Ksize - k_offset, kleft);
982 
983  strat.transforms.PrepareB(buffer, B + (current.multi() * B_multi_stride), ldb,
984  x0, xmax,
985  (k_section_base * _Ksize) + k_offset, // K starting point - compute row to read based on our section and the true section length.
986  (k_section_base * _Ksize) + k_offset + k_length); // K end point - starting point plus length computed above.
987 
988  // We need to modify our position based on the ROUNDED version of what we just did.
989  unsigned int padded_length = roundup(k_length, strategy::k_unroll());
990 
991  buffer += strategy::out_width() * padded_length;
992 
993  kpos += padded_length;
994  kleft -= padded_length;
995  }
996  }
997  } else {
998  // In the single K section case, can process the whole lot in one go.
999  // Caution: 'blockwalker::kmax()' rounds up, so clamp to valid _Ksize.
1000  strat.transforms.PrepareB(buffer, B + (current.multi() * B_multi_stride), ldb,
1001  current.x0(), current.xmax(), current.k0(), std::min(current.kmax(), _Ksize));
1002  buffer += roundup(current.xmax() - current.x0(), strategy::out_width()) * roundup(current.kmax() - current.k0(), strategy::k_unroll());
1003  }
1004  } while (current.advance());
1005  }
T roundup(const T a, const T b)
Definition: utils.hpp:70
void requantize_bias(void *in_buffer, const To *B, const int ldb, const int B_multi_stride) override
const StratType * strategy

◆ requantize_bias()

void requantize_bias ( void *  in_buffer,
const To *  B,
const int  ldb,
const int  B_multi_stride 
)
inlineoverridevirtual

Reimplemented from GemmCommon< To, Tr >.

Definition at line 926 of file gemm_interleaved.hpp.

References arm_gemm::compute_col_sums().

926  {
927  if (std::is_same<OutputStage, Requantize32>::value) {
928  col_bias = reinterpret_cast<int32_t *>(in_buffer);
929 
930  Requantize32 *qp_ptr = reinterpret_cast<Requantize32 *>(&_os);
931 
932  for (unsigned int i=0; i<_nmulti; i++) {
933  // The input is assumed not to have any padding between sections, so straightforward Ksize * Ksections computation gets the total size.
934  compute_col_sums(*qp_ptr, _Nsize, _Ksize * _Ksections, B + (i * B_multi_stride), ldb, col_bias + (i * _Nsize), _Ksize * _Ksections, i, 0);
935  }
936  }
937  }
void compute_col_sums(const Requantize32 &qp, unsigned int width, unsigned int height, const T *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int multi, unsigned int first_col)

◆ set_convolution_parameters()

void set_convolution_parameters ( ConvolutionParameters  parms)
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 1028 of file gemm_interleaved.hpp.

References ConvolutionParameters::input_channels.

1028  {
1029  assert(parms.input_channels == _Ksize);
1030  _convolver = std::unique_ptr<convolver<To>>(new convolver<To>(parms));
1031  }

◆ set_indirect_parameters()

void set_indirect_parameters ( size_t  string_len,
const To *const *const *  ptr 
)
inlineoverridevirtual

Reimplemented from GemmCommon< To, Tr >.

Definition at line 1023 of file gemm_interleaved.hpp.

1023  {
1024  assert(string_len == _Ksize);
1025  _indirect_buf = ptr;
1026  }

◆ set_nthreads()

void set_nthreads ( int  nthreads)
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 615 of file gemm_interleaved.hpp.

615  {
616  _nthreads = std::min(nthreads, _maxthreads);
617  }

◆ set_pretransposed_B_data()

void set_pretransposed_B_data ( void *  in_buffer)
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 1007 of file gemm_interleaved.hpp.

1007  {
1008  // Put the transposed data after the column sums - in non-transposing cases get_col_sum_size() == 0
1009  uintptr_t buffer_int = reinterpret_cast<uintptr_t>(in_buffer);
1010  _B_transposed = reinterpret_cast<Toi *>(buffer_int + get_col_sum_size());
1011  col_bias = reinterpret_cast<int32_t *>(in_buffer);
1012  }

◆ set_quantized_bias()

void set_quantized_bias ( const int32_t *  bias,
size_t  bias_multi_stride 
)
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 1014 of file gemm_interleaved.hpp.

References Requantize32::bias, bias, and Requantize32::bias_multi_stride.

1014  {
1015  if (std::is_same<OutputStage, Requantize32>::value) {
1016  Requantize32 *qp = reinterpret_cast<Requantize32 *>(&_os);
1017 
1018  qp->bias = bias;
1019  qp->bias_multi_stride = bias_multi_stride;
1020  }
1021  }
const int32_t * bias

◆ set_working_space()

void set_working_space ( void *  working_space)
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 881 of file gemm_interleaved.hpp.

881  {
882  // Make sure everything ends up cache line aligned
883  int8_t *working_space_bytes = reinterpret_cast<int8_t *>(working_space);
884  intptr_t working_space_int = reinterpret_cast<intptr_t>(working_space);
885 
886  size_t diff=0;
887 
888  if (working_space_int & 0x3F) {
889  diff = 0x40 - (working_space_int & 0x3F);
890  }
891 
892  working_space_bytes += diff;
893  working_space_int += diff;
894 
895  // Pretransposed case: just set internal pointer to parameter value.
896  _working_space = reinterpret_cast<void *>(working_space_bytes);
897 
898  // Set up accumulation buffer
899  if (get_accumulation_buffer_size() > 0) {
900  intptr_t acc_buff_int = working_space_int + get_a_working_size() + (get_c_working_size() * _maxthreads);
901  // Make sure the accumulation buffer is aligned (needed if the other blocks are not a multiple of cache line length)
902  if (acc_buff_int & 0x3F) {
903  acc_buff_int += (0x40 - (acc_buff_int & 0x3F));
904  }
905  _accumulation_buffer = reinterpret_cast<Tab *>(acc_buff_int);
906  } else {
907  _accumulation_buffer = nullptr;
908  }
909  }

The documentation for this class was generated from the following file: