Compute Library
 22.11
GemmInterleaved< strategy, To, Tr, OutputStage, MergeStep, FixedFormat, ForceThreadColumns > Class Template Reference

#include <gemm_interleaved.hpp>

Collaboration diagram for GemmInterleaved< strategy, To, Tr, OutputStage, MergeStep, FixedFormat, ForceThreadColumns >:
[legend]

Public Member Functions

 GemmInterleaved (GemmInterleaved &)=delete
 
GemmInterleavedoperator= (GemmInterleaved &)=delete
 
 GemmInterleaved (const GemmArgs &args, const OutputStage &os)
 
 GemmInterleaved (const GemmArgs &args)
 
ndrange_t get_window_size () const override
 
void set_nthreads (int nthreads) override
 
void execute (const ndcoord_t &work_range, const ndcoord_t &, int threadid) override
 Main execute member fucntion. More...
 
size_t get_working_size () const override
 
void set_working_space (void *working_space) override
 
bool B_is_pretransposed () const override
 
bool B_pretranspose_required () const override
 
size_t get_B_pretransposed_array_size () const override
 
void requantize_bias (void *in_buffer, const To *B, const int ldb, const int B_multi_stride) override
 
void pretranspose_B_array (void *in_buffer, const To *B, const int ldb, const int B_multi_stride) override
 
void set_pretransposed_B_data (void *in_buffer) override
 
void set_quantized_bias (const int32_t *bias, size_t bias_multi_stride) override
 
void set_indirect_parameters (size_t string_len, const To *const *const *ptr) override
 
void set_convolution_parameters (ConvolutionParameters parms) override
 
GemmConfig get_config () override
 
- Public Member Functions inherited from GemmCommon< To, Tr >
virtual void set_arrays (const To *A, const int lda, const int A_batch_stride, const int A_multi_stride, const To *B, const int ldb, const int B_multi_stride, Tr *C, const int ldc, const int C_batch_stride, const int C_multi_stride, const Tr *bias, const int bias_multi_stride)
 
void set_arrays_generic (const void *A, const int lda, const int A_batch_stride, const int A_multi_stride, const void *B, const int ldb, const int B_multi_stride, void *C, const int ldc, const int C_batch_stride, const int C_multi_stride, const void *bias, const int bias_multi_stride) override
 
void pretranspose_B_array_generic (void *out, const void *in, const int row_stride, const int multi_stride) override
 
void set_indirect_parameters_generic (size_t sz, const void *const *const *ptr) override
 
- Public Member Functions inherited from IGemmCommon
virtual bool supports_dynamic_scheduling () const
 
virtual ~IGemmCommon ()
 

Static Public Member Functions

template<typename perf_type >
static uint64_t estimate_cycles (const GemmArgs &args)
 

Detailed Description

template<typename strategy, typename To, typename Tr, typename OutputStage = Nothing, bool MergeStep = true, bool FixedFormat = false, bool ForceThreadColumns = false>
class arm_gemm::GemmInterleaved< strategy, To, Tr, OutputStage, MergeStep, FixedFormat, ForceThreadColumns >

Definition at line 324 of file gemm_interleaved.hpp.

Constructor & Destructor Documentation

◆ GemmInterleaved() [1/3]

GemmInterleaved ( GemmInterleaved< strategy, To, Tr, OutputStage, MergeStep, FixedFormat, ForceThreadColumns > &  )
delete

◆ GemmInterleaved() [2/3]

GemmInterleaved ( const GemmArgs args,
const OutputStage &  os 
)
inline

Definition at line 650 of file gemm_interleaved.hpp.

651  : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
652  _Ksections(args._Ksections), _Ktotal(get_ktotal(args)),
653  _rounded_Ksize(roundup(_Ksize, strategy::k_unroll())),
654  _nbatches(args._nbatches), _nmulti(args._nmulti), _thread_columns(is_thread_columns(args)),
655  _act(args._act), _maxthreads(args._maxthreads), _nthreads(args._maxthreads),
656  _k_block(get_k_block_size(args)), _x_block(get_x_block_size(args)), _Mround(roundup(args._Msize, strategy::out_height())),
657  _os(os) { }
T roundup(const T a, const T b)
Definition: utils.hpp:70

◆ GemmInterleaved() [3/3]

GemmInterleaved ( const GemmArgs args)
inline

Definition at line 660 of file gemm_interleaved.hpp.

661  : _ci(args._ci), _Msize(args._Msize), _Nsize(args._Nsize), _Ksize(args._Ksize),
662  _Ksections(args._Ksections), _Ktotal(get_ktotal(args)),
663  _rounded_Ksize(roundup(_Ksize, strategy::k_unroll())),
664  _nbatches(args._nbatches), _nmulti(args._nmulti), _thread_columns(is_thread_columns(args)),
665  _act(args._act), _maxthreads(args._maxthreads), _nthreads(args._maxthreads),
666  _k_block(get_k_block_size(args)), _x_block(get_x_block_size(args)), _Mround(roundup(args._Msize, strategy::out_height())),
667  _os() { }
T roundup(const T a, const T b)
Definition: utils.hpp:70

Member Function Documentation

◆ B_is_pretransposed()

bool B_is_pretransposed ( ) const
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 997 of file gemm_interleaved.hpp.

997  {
998  return (FixedFormat == false);
999  }

◆ B_pretranspose_required()

bool B_pretranspose_required ( ) const
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 1001 of file gemm_interleaved.hpp.

1001  {
1002  return (FixedFormat == false) && (_B_transposed==nullptr);
1003  }

◆ estimate_cycles()

static uint64_t estimate_cycles ( const GemmArgs args)
inlinestatic

Definition at line 1124 of file gemm_interleaved.hpp.

References GemmArgs::_ci, GemmArgs::_Ksize, GemmArgs::_maxthreads, GemmArgs::_Msize, GemmArgs::_nbatches, GemmArgs::_nmulti, GemmArgs::_Nsize, arm_gemm::iceildiv(), PerformanceParameters::kernel_macs_cycle, PerformanceParameters::merge_bytes_cycle, PerformanceParameters::prepare_bytes_cycle, and arm_gemm::roundup().

1124  {
1125  unsigned int k_blocks = iceildiv(args._Ksize, get_k_block_size(args));
1126 
1127  const PerformanceParameters &params = strategy::template get_performance_parameters<perf_type>(args._ci);
1128 
1129  uint64_t total_macs = static_cast<uint64_t>(args._nbatches) * args._nmulti * roundup(args._Msize, strategy::out_height()) * roundup(args._Nsize, strategy::out_width()) * get_ktotal(args);
1130  uint64_t prepare_bytes = static_cast<uint64_t>(args._nbatches) * args._nmulti * roundup(args._Msize, strategy::out_height()) * get_ktotal(args) * sizeof(Toi);
1131  uint64_t merge_bytes = static_cast<uint64_t>(args._nbatches) * args._nmulti * k_blocks * args._Msize * roundup(args._Nsize, strategy::out_width()) * sizeof(Tr);
1132 
1133  float mac_cycles = static_cast<float>(total_macs) / params.kernel_macs_cycle;
1134  float prepare_cycles = static_cast<float>(prepare_bytes) / params.prepare_bytes_cycle;
1135  float merge_cycles = static_cast<float>(merge_bytes) / params.merge_bytes_cycle;
1136 
1137  float total_cycles = mac_cycles + prepare_cycles + merge_cycles;
1138 
1139  // We can't thread over multis or width, which makes this a poor
1140  // choice in many threaded cases. Penalize that here.
1141  float parallelism_available = static_cast<float>(iceildiv(args._Msize, strategy::out_height()) * args._nbatches) * 0.9f;
1142 
1143  if (parallelism_available < args._maxthreads) {
1144  total_cycles *= (static_cast<float>(args._maxthreads) / parallelism_available);
1145  }
1146 
1147  return static_cast<uint64_t>(total_cycles);
1148  }
T roundup(const T a, const T b)
Definition: utils.hpp:70
T iceildiv(const T a, const T b)
Definition: utils.hpp:65

◆ execute()

void execute ( const ndcoord_t work_range,
const ndcoord_t thread_locator,
int  threadid 
)
inlineoverridevirtual

Main execute member fucntion.

Parameters
[in]work_rangespecifies the range of work we want to be computed, total range defined by get_window_size()
[in]thread_locatorwhere are we inside of the thread space
[in]threadida unique threadid

Implements IGemmCommon.

Definition at line 692 of file gemm_interleaved.hpp.

References arm_compute::test::validation::batch, arm_compute::mlgo::parser::end(), NDCoordinate< N >::get_position(), NDCoordinate< N >::get_position_end(), arm_gemm::roundup(), arm_compute::test::validation::run(), strategy, and type.

692  {
693 #ifdef CYCLE_PROFILING
694  profiler prof;
695 #endif
696 
697  /* Make sure we've been set up correctly. */
698  assert(FixedFormat || _B_transposed);
699  assert(_working_space);
700  int8_t *working_space_bytes = reinterpret_cast<int8_t *>(_working_space);
701 
702  /* Align if needed */
703  intptr_t working_space_v = reinterpret_cast<intptr_t>(_working_space);
704  if (working_space_v & 0x3f) {
705  intptr_t alignment_offset = 0x40 - (working_space_v & 0x3f);
706  working_space_bytes += alignment_offset;
707  }
708 
709  strategy strat(_ci);
710 
711  const auto start = work_range.get_position(0);
712  const auto end = work_range.get_position_end(0);
713 
714  /* Translate 'start' and 'end' into a position within the batches and rows. */
715  const unsigned int window_per_batch = _Mround / strategy::out_height();
716  unsigned int batch_0 = start / window_per_batch;
717  unsigned int batch_end = end / window_per_batch;
718 
719  // In ThreadColumns mode, process work one horizontal strip at a time.
720  // Transpose the block of needed rows at the start, then do all the work on that block.
721  if (_thread_columns) {
722  const auto start_x = work_range.get_position(1) * strategy::out_width();
723  const auto end_x = std::min(work_range.get_position_end(1) * strategy::out_width(), _Nsize);
724 
725  Tri * const c_panel = reinterpret_cast<Tri *>(working_space_bytes + (threadid * get_c_working_size()));
726  Toi * const a_panel = reinterpret_cast<Toi *>(working_space_bytes + (_maxthreads * get_c_working_size()) +
727  (threadid * sizeof(Toi) * get_total_k_depth() * strategy::out_height()));
728 
729  for (unsigned int multi=0; multi<_nmulti; multi++) {
730  for (unsigned int k0=0; k0<_Ktotal; k0+=_k_block) {
731  unsigned int kmax=std::min(k0+_k_block, _Ktotal);
732 
733  unsigned int rounded_width = roundup(_Nsize, strategy::out_width());
734 
735  const bool first_pass = (k0==0);
736  const bool last_pass = (kmax==_Ktotal);
737 
738  // Figure out how many "K" the kernel will actually process.
739  unsigned int kern_k = roundup(kmax - k0, strategy::k_unroll());
740 
741  const Toi *b_ptr = FixedFormat ?
742  reinterpret_cast<const Toi *>(this->_Bptr) + (multi * this->_B_multi_stride) +
743  ((start_x / get_stripe_width<strategy, FixedFormat>::get()) * this->_ldb) +
744  (k0 * get_stripe_width<strategy, FixedFormat>::get()) :
745  _B_transposed + (rounded_width * _Ktotal * multi) + (k0 * rounded_width) + (start_x * kern_k);
746 
747  unsigned int batch = batch_0;
748  unsigned int start_row = (start - (batch_0 * window_per_batch)) * strategy::out_height();
749 
750  for (unsigned int p=start; p<end; p++) {
751  unsigned int end_row = std::min(start_row + strategy::out_height(), _Msize);
752 
753  // Set up transposed 'A' block
754  {
755 #ifdef CYCLE_PROFILING
756  auto p=prof.ScopedProfiler(PROFILE_PREPA, strategy::out_height() * (kmax-k0) * sizeof(Toi));
757 #endif
758  // See comment above on transform_type<> class: this extracts either 'transforms' or
759  // 'transforms_quantized' as appropriate.
760  typename transform_type<strategy, MergeStep && std::is_same<OutputStage, Requantize32>::value>::type transforms;
761 
762  if (_indirect_buf != nullptr) {
763  transforms.PrepareA_indirect(a_panel,
764  _indirect_buf + (multi * _nbatches * _Ksections) + (batch * _Ksections), _Ksize,
765  _rounded_Ksize, start_row, end_row, k0, kmax, row_sum_multiplier());
766  } else if (_convolver) {
767  transforms.PrepareA_convolution(a_panel,
768  this->_Aptr + (batch * this->_A_batch_stride) + (multi * this->_A_multi_stride),
769  this->_lda, *_convolver, _rounded_Ksize, start_row, end_row, k0, kmax, row_sum_multiplier());
770  } else {
771  transforms.PrepareA(a_panel,
772  this->_Aptr + (batch * this->_A_batch_stride) + (multi * this->_A_multi_stride),
773  this->_lda, start_row, end_row, k0, std::min(kmax, _Ksize), row_sum_multiplier());
774  }
775  }
776 
777  // Perform the kernel and merge step, either separately or together as required.
779  #ifdef CYCLE_PROFILING
780  prof,
781  #endif
782  // Strategy and panel pointers
783  strat, a_panel, b_ptr, this->_ldb, c_panel,
784  // Result buffer pointers
785  this->_Cptr + (batch * this->_C_batch_stride) + (multi * this->_C_multi_stride), this->_ldc,
786  // K size, and M/N ranges
787  kern_k, start_row, end_row, start_x, end_x,
788  // Only do bias on the first pass
789  ((first_pass && this->_bias) ? this->_bias + (multi * this->_bias_multi_stride) : nullptr),
790  // Only do activation on the last pass, and accumulation on any non-first pass.
791  (last_pass ? _act : Activation()), !first_pass,
792  // Pass in quantization parameters for requantizing kernels (others will ignore)
793  _os, col_bias + (multi * _Nsize),
794  // Accumulation buffer (not yet implemented on this path)
795  static_cast<Tab *>(nullptr));
796 
797  /* Increment to the next block */
798  start_row += strategy::out_height();
799  if (start_row >= _Msize) {
800  start_row = 0;
801  batch++;
802  }
803  }
804  }
805  }
806  } else {
807  blockwalker current(*this);
808 
809  /* Compute the M values to operate on */
810  unsigned int m_0 = (start - (batch_0 * window_per_batch)) * strategy::out_height();
811  unsigned int m_max = (end - (batch_end * window_per_batch)) * strategy::out_height();
812 
813  // Private buffers. Treat working_space as an array of C buffers
814  // (one per thread) first, followed by the (window-divided) A
815  // buffer.
816  // Set a_panel to the base of the A buffers - compute offsets into it based on M/batches later.
817  Toi * const a_panel = reinterpret_cast<Toi *>(working_space_bytes + (_maxthreads * get_c_working_size()));
818  Tri * const c_panel = reinterpret_cast<Tri *>(working_space_bytes + (threadid * get_c_working_size()));
819 
820  const Toi *b_panel;
821  b_panel = _B_transposed;
822 
823  // newkblock() is always true on the first iteration, so these will be set properly on the first loop.
824 
825  // kern_k tracks the accumulation depth for the CURRENT K block a_panel_stride similarly tracks the total
826  // stride of the A panel (i.e. with 4 added for cases with embedded row sums)
827 
828  // These are distinct from k_block and get_total_k_depth() which are based on the target K block size, and
829  // used for addressing inside a_panel.
830 
831  // In cases where K blocking is in use and the blocks are not all the same size, the (smaller) final block
832  // won't use all the memory allocated.
833  unsigned int kern_k = 0;
834  unsigned int a_panel_stride = 0;
835 
836  for (;!current.done();current.advance()) {
837  if (current.newkblock()) {
838 #ifdef CYCLE_PROFILING
839  auto p=prof.ScopedProfiler(PROFILE_PREPA, (end - start) * strategy::out_height() * (current.kmax()-current.k0()) * sizeof(Toi));
840 #endif
841  // See comment above on transform_type<> class: this extracts either 'transforms' or
842  // 'transforms_quantized' as appropriate.
843  typename transform_type<strategy, MergeStep && std::is_same<OutputStage, Requantize32>::value>::type transforms;
844 
845  for (unsigned int batch = batch_0; batch <= batch_end; batch++) {
846  unsigned int first_m = (batch == batch_0) ? m_0 : 0;
847  unsigned int last_m = (batch == batch_end) ? m_max : _Msize;
848 
849  if (first_m >= last_m)
850  continue;
851 
852  if (_indirect_buf != nullptr) {
853  transforms.PrepareA_indirect(a_panel + ((batch * _Mround + first_m) * get_total_k_depth()),
854  _indirect_buf + (current.multi() * _nbatches * _Ksections) + (batch * _Ksections), _Ksize,
855  _rounded_Ksize, first_m, last_m, current.k0(), current.kmax(), row_sum_multiplier());
856  } else if (_convolver) {
857  transforms.PrepareA_convolution(a_panel + ((batch * _Mround + first_m) * get_total_k_depth()),
858  this->_Aptr + (batch * this->_A_batch_stride) + (current.multi() * this->_A_multi_stride),
859  this->_lda, *_convolver, _rounded_Ksize, first_m, last_m, current.k0(), current.kmax(), row_sum_multiplier());
860  } else {
861  transforms.PrepareA(a_panel + ((batch * _Mround + first_m) * get_total_k_depth()),
862  this->_Aptr + (batch * this->_A_batch_stride) + (current.multi() * this->_A_multi_stride),
863  this->_lda, first_m, last_m, current.k0(), std::min(_Ksize, current.kmax()), row_sum_multiplier());
864  }
865  }
866 
867  // Figure out how many "K" the kernel will actually process.
868  kern_k = roundup(current.kmax() - current.k0(), strategy::k_unroll());
869 
870  // Requantizing GEMMs have the row sums built in to the
871  // transposed data, so the stride between rows is 4 bytes
872  // larger than the (rounded) K value.
873 
874  if(std::is_same<OutputStage, Requantize32>::value) {
875  a_panel_stride = kern_k + (sizeof(int32_t) / sizeof(Toi));
876  } else {
877  a_panel_stride = kern_k;
878  }
879  }
880 
881  // For FixedFormat cases, figure out the B pointer. The loop below moves through batches and vertically through the output so this will be the same throughout.
882  if (FixedFormat) {
883  b_panel = reinterpret_cast<const Toi *>(this->_Bptr) + (current.multi() * this->_B_multi_stride) +
884  ((current.x0() / get_stripe_width<strategy, FixedFormat>::get()) * this->_ldb) +
885  (current.k0() * get_stripe_width<strategy, FixedFormat>::get());
886  }
887 
888  /* Do the actual work. */
889  for (unsigned int batch = batch_0; batch <= batch_end; batch++) {
890  unsigned int first_m = (batch == batch_0) ? m_0 : 0;
891  unsigned int last_m = (batch == batch_end) ? m_max : _Msize;
892 
893  const Toi *a_ptr = a_panel + (batch * _Mround + first_m) * get_total_k_depth();
894 
895  if (first_m >= last_m)
896  continue;
897 
898  // For the merge case we need to do this out_height() rows
899  // at a time, as that is the size of our intermediate
900  // buffer. If we are not doing that, we can do all the
901  // relevant rows in one go.
902  unsigned int m_step = MergeStep ? strategy::out_height() : (last_m - first_m);
903 
904  // But in the case where we have an accumulation buffer, we can't do that after all, unless
905  // there is no N blocking.
906  if (_accumulation_buffer && ((current.x0() != 0) || (current.xmax() < _Nsize))) {
907  m_step = strategy::out_height();
908  }
909 
910  for (unsigned int y=first_m; y<last_m; y+=m_step) {
911  unsigned int ymax = std::min(_Msize, y + m_step);
912 
913  const bool first_pass = (current.k0() == 0);
914  const bool last_pass = (current.kmax() == _Ktotal);
915 
916  // Pointer to appropriate part of result array.
917  Tr *result_ptr = this->_Cptr + (batch * this->_C_batch_stride) + (current.multi() * this->_C_multi_stride);
918 
919  // If we are using an accumulation buffer, we don't pass the result buffer to ask the kernel
920  // to write things into the accumulation buffer instead, except on the last pass.
921  if (_accumulation_buffer && !last_pass) {
922  result_ptr = nullptr;
923  }
924 
925  // Perform the kernel and merge step, either separately or together as required.
927  #ifdef CYCLE_PROFILING
928  prof,
929  #endif
930  // Strategy and panel pointers
931  strat, a_ptr, b_panel, this->_ldb, c_panel,
932  // Result buffer pointers
933  result_ptr, this->_ldc,
934  // K size, and M/N ranges
935  kern_k, y, ymax, current.x0(), current.xmax(),
936  // Only do bias on the first pass
937  ((first_pass && this->_bias) ? this->_bias + (current.multi() * this->_bias_multi_stride) : nullptr),
938  // Only do activation on the last pass, and accumulation on any non-first pass.
939  (last_pass ? _act : Activation()), !first_pass,
940  // Pass in quantization parameters for requantizing kernels (others will ignore)
941  _os, col_bias + (current.multi() * _Nsize),
942  // Accumulation buffer
943  get_accumulation_buffer(y, current.x0(), batch, current.multi()) );
944 
945  a_ptr += (strategy::out_height() * a_panel_stride);
946  }
947  }
948 
949  if (FixedFormat == false) {
950  b_panel += (roundup(current.xmax() - current.x0(), strategy::out_width()) * kern_k);
951  }
952  }
953  }
954  }
T roundup(const T a, const T b)
Definition: utils.hpp:70
arm_compute::ActivationLayerInfo::ActivationFunction Activation
Constant TensorID specifying an equivalent of null tensor.
Definition: Types.h:73
decltype(strategy::transforms) typedef type
void end(TokenStream &in, bool &valid)
Definition: MLGOParser.cpp:290
const StratType * strategy

◆ get_B_pretransposed_array_size()

size_t get_B_pretransposed_array_size ( ) const
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 1005 of file gemm_interleaved.hpp.

References arm_gemm::roundup().

1005  {
1006  if (FixedFormat) {
1007  return 0;
1008  }
1009 
1010  unsigned int x_size = roundup(_Nsize, strategy::out_width());
1011 
1012  return (x_size * _Ktotal * _nmulti * sizeof(Toi)) + get_col_sum_size();
1013  }
T roundup(const T a, const T b)
Definition: utils.hpp:70

◆ get_config()

GemmConfig get_config ( )
inlineoverridevirtual

Implements IGemmCommon.

Definition at line 1150 of file gemm_interleaved.hpp.

References GemmConfig::filter, arm_gemm::GEMM_INTERLEAVED, arm_gemm::get_weight_format(), GemmConfig::inner_block_size, GemmConfig::method, GemmConfig::outer_block_size, and GemmConfig::weight_format.

1150  {
1151  GemmConfig c;
1152 
1153  c.method = GemmMethod::GEMM_INTERLEAVED;
1154  c.inner_block_size = _k_block;
1155  c.outer_block_size = _x_block;
1156  c.filter = get_type_name<strategy>();
1157  c.weight_format = get_weight_format(get_kernel_weight_format<strategy, FixedFormat, To>::get(), sizeof(To));
1158 
1159  return c;
1160  }
WeightFormat get_weight_format(const KernelWeightFormat, size_t)
Definition: misc.cpp:40

◆ get_window_size()

ndrange_t get_window_size ( ) const
inlineoverridevirtual
Returns
an ndrange containing ranges of the compute space which can be broken up and parallelised over

Implements IGemmCommon.

Definition at line 675 of file gemm_interleaved.hpp.

References arm_gemm::iceildiv().

675  {
676  unsigned int row_blocks = (_Mround / strategy::out_height()) * _nbatches;
677 
678  if (_thread_columns) {
679  return { row_blocks, iceildiv(_Nsize, strategy::out_width()) };
680  } else {
681  // _Mround is a multiple of out_height by definition.
682  return { row_blocks };
683  }
684  }
T iceildiv(const T a, const T b)
Definition: utils.hpp:65

◆ get_working_size()

size_t get_working_size ( void  ) const
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 957 of file gemm_interleaved.hpp.

957  {
958  // In all cases, we need one A buffer plus a C buffer per thread, plus an accumulation buffer.
959  size_t size = get_a_working_size() + (get_c_working_size() * _maxthreads) + get_accumulation_buffer_size();
960 
961  size += 128; // Add on two cache lines extra for alignment.
962 
963  return size;
964  }

◆ operator=()

GemmInterleaved& operator= ( GemmInterleaved< strategy, To, Tr, OutputStage, MergeStep, FixedFormat, ForceThreadColumns > &  )
delete

◆ pretranspose_B_array()

void pretranspose_B_array ( void *  in_buffer,
const To *  B,
const int  ldb,
const int  B_multi_stride 
)
inlineoverridevirtual

Reimplemented from GemmCommon< To, Tr >.

Definition at line 1028 of file gemm_interleaved.hpp.

References arm_gemm::roundup(), and strategy.

1028  {
1029  requantize_bias(in_buffer, B, ldb, B_multi_stride);
1030 
1031  // Put the transposed data after the column sums - in non-quantized cases get_col_sum_size() == 0
1032  uintptr_t buffer_int = reinterpret_cast<uintptr_t>(in_buffer);
1033  Toi *buffer = reinterpret_cast<Toi *>(buffer_int + get_col_sum_size());
1034  _B_transposed = buffer;
1035 
1036  blockwalker current(*this);
1037  strategy strat(_ci);
1038 
1039  do {
1040  /* Figure out the size of each block. */
1041  unsigned int k_size = (current.kmax() - current.k0());
1042 
1043  if (_Ksections > 1) {
1044  // We need to insert padding at the end of each K section.
1045  // The computation needed is a little delicate - the coordinates from the block walker are expressed in
1046  // terms of the full, padded, _Ktotal.
1047  // But we need to transform each section with reference to the original, unpadded, input, letting the
1048  // transform pad each section as needed.
1049 
1050  // This is needed for computations below.
1051  const unsigned int rounded_section_size = roundup(_Ksize, strategy::k_unroll());
1052 
1053  // The expected output format is also an entire <out_width> columns interleaved, then the next set of
1054  // columns, and so on. This means, as we are breaking it up vertically, we have to do it one column at
1055  // a time.
1056  for (unsigned int x0=current.x0(); x0 < current.xmax(); x0 += strategy::out_width() ) {
1057  unsigned int xmax = std::min(x0 + strategy::out_width(), current.xmax());
1058 
1059  // Track where we are and how much work is left.
1060  unsigned int kpos = current.k0();
1061  unsigned int kleft = k_size;
1062 
1063  while (kleft) {
1064  // Which section are we in? Based on the rounded-up section size.
1065  unsigned int k_section_base = kpos / rounded_section_size;
1066  // How far into the section are we?
1067  unsigned int k_offset = kpos - (k_section_base * rounded_section_size);
1068 
1069  // We will either copy the rest of this section, or to the end of the requested length.
1070  unsigned int k_length = std::min(_Ksize - k_offset, kleft);
1071 
1072  strat.transforms.PrepareB(buffer, B + (current.multi() * B_multi_stride), ldb,
1073  x0, xmax,
1074  (k_section_base * _Ksize) + k_offset, // K starting point - compute row to read based on our section and the true section length.
1075  (k_section_base * _Ksize) + k_offset + k_length); // K end point - starting point plus length computed above.
1076 
1077  // We need to modify our position based on the ROUNDED version of what we just did.
1078  unsigned int padded_length = roundup(k_length, strategy::k_unroll());
1079 
1080  buffer += strategy::out_width() * padded_length;
1081 
1082  kpos += padded_length;
1083  kleft -= padded_length;
1084  }
1085  }
1086  } else {
1087  // In the single K section case, can process the whole lot in one go.
1088  // Caution: 'blockwalker::kmax()' rounds up, so clamp to valid _Ksize.
1089  strat.transforms.PrepareB(buffer, B + (current.multi() * B_multi_stride), ldb,
1090  current.x0(), current.xmax(), current.k0(), std::min(current.kmax(), _Ksize));
1091  buffer += roundup(current.xmax() - current.x0(), strategy::out_width()) * roundup(current.kmax() - current.k0(), strategy::k_unroll());
1092  }
1093  } while (current.advance());
1094  }
T roundup(const T a, const T b)
Definition: utils.hpp:70
void requantize_bias(void *in_buffer, const To *B, const int ldb, const int B_multi_stride) override
const StratType * strategy

◆ requantize_bias()

void requantize_bias ( void *  in_buffer,
const To *  B,
const int  ldb,
const int  B_multi_stride 
)
inlineoverridevirtual

Reimplemented from GemmCommon< To, Tr >.

Definition at line 1015 of file gemm_interleaved.hpp.

References arm_gemm::compute_col_sums().

1015  {
1016  if (std::is_same<OutputStage, Requantize32>::value) {
1017  col_bias = reinterpret_cast<int32_t *>(in_buffer);
1018 
1019  Requantize32 *qp_ptr = reinterpret_cast<Requantize32 *>(&_os);
1020 
1021  for (unsigned int i=0; i<_nmulti; i++) {
1022  // The input is assumed not to have any padding between sections, so straightforward Ksize * Ksections computation gets the total size.
1023  compute_col_sums(*qp_ptr, _Nsize, _Ksize * _Ksections, B + (i * B_multi_stride), ldb, col_bias + (i * _Nsize), _Ksize * _Ksections, i, 0);
1024  }
1025  }
1026  }
void compute_col_sums(const Requantize32 &qp, unsigned int width, unsigned int height, const T *input, unsigned int in_stride, int32_t *col_bias, unsigned int depth, unsigned int multi, unsigned int first_col)

◆ set_convolution_parameters()

void set_convolution_parameters ( ConvolutionParameters  parms)
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 1117 of file gemm_interleaved.hpp.

References ConvolutionParameters::input_channels.

1117  {
1118  assert(parms.input_channels == _Ksize);
1119  _convolver = std::unique_ptr<convolver<To>>(new convolver<To>(parms));
1120  }

◆ set_indirect_parameters()

void set_indirect_parameters ( size_t  string_len,
const To *const *const *  ptr 
)
inlineoverridevirtual

Reimplemented from GemmCommon< To, Tr >.

Definition at line 1112 of file gemm_interleaved.hpp.

1112  {
1113  assert(string_len == _Ksize);
1114  _indirect_buf = ptr;
1115  }

◆ set_nthreads()

void set_nthreads ( int  nthreads)
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 687 of file gemm_interleaved.hpp.

687  {
688  _nthreads = std::min(nthreads, _maxthreads);
689  }

◆ set_pretransposed_B_data()

void set_pretransposed_B_data ( void *  in_buffer)
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 1096 of file gemm_interleaved.hpp.

1096  {
1097  // Put the transposed data after the column sums - in non-quantized cases get_col_sum_size() == 0
1098  uintptr_t buffer_int = reinterpret_cast<uintptr_t>(in_buffer);
1099  _B_transposed = reinterpret_cast<Toi *>(buffer_int + get_col_sum_size());
1100  col_bias = reinterpret_cast<int32_t *>(in_buffer);
1101  }

◆ set_quantized_bias()

void set_quantized_bias ( const int32_t *  bias,
size_t  bias_multi_stride 
)
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 1103 of file gemm_interleaved.hpp.

References Requantize32::bias, bias, and Requantize32::bias_multi_stride.

1103  {
1104  if (std::is_same<OutputStage, Requantize32>::value) {
1105  Requantize32 *qp = reinterpret_cast<Requantize32 *>(&_os);
1106 
1107  qp->bias = bias;
1108  qp->bias_multi_stride = bias_multi_stride;
1109  }
1110  }
const int32_t * bias

◆ set_working_space()

void set_working_space ( void *  working_space)
inlineoverridevirtual

Reimplemented from IGemmCommon.

Definition at line 966 of file gemm_interleaved.hpp.

966  {
967  // Make sure everything ends up cache line aligned
968  int8_t *working_space_bytes = reinterpret_cast<int8_t *>(working_space);
969  intptr_t working_space_int = reinterpret_cast<intptr_t>(working_space);
970 
971  size_t diff=0;
972 
973  if (working_space_int & 0x3F) {
974  diff = 0x40 - (working_space_int & 0x3F);
975  }
976 
977  working_space_bytes += diff;
978  working_space_int += diff;
979 
980  // Pretransposed case: just set internal pointer to parameter value.
981  _working_space = reinterpret_cast<void *>(working_space_bytes);
982 
983  // Set up accumulation buffer
984  if (get_accumulation_buffer_size() > 0) {
985  intptr_t acc_buff_int = working_space_int + get_a_working_size() + (get_c_working_size() * _maxthreads);
986  // Make sure the accumulation buffer is aligned (needed if the other blocks are not a multiple of cache line length)
987  if (acc_buff_int & 0x3F) {
988  acc_buff_int += (0x40 - (acc_buff_int & 0x3F));
989  }
990  _accumulation_buffer = reinterpret_cast<Tab *>(acc_buff_int);
991  } else {
992  _accumulation_buffer = nullptr;
993  }
994  }

The documentation for this class was generated from the following file: