ArmNN
 24.08
WorkloadUtils.cpp
Go to the documentation of this file.
1 //
2 // Copyright © 2017-2024 Arm Ltd. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5 
7 
8 #include <armnn/Utils.hpp>
12 
13 #include <fmt/format.h>
14 #include <numeric>
15 
16 namespace armnn
17 {
18 
20  const PermutationVector& permutationVector, void* permuteBuffer)
21 {
22  if (tensor == nullptr)
23  {
24  throw armnn::InvalidArgumentException("WorkloadUtils: PermuteTensor: Null input tensor pointer");
25  }
26  if (permuteBuffer == nullptr)
27  {
28  throw armnn::InvalidArgumentException("WorkloadUtils: PermuteTensor: Null permute buffer pointer");
29  }
30 
31  TensorInfo tensorInfo = tensor->GetTensorInfo();
32 
33  if (permutationVector.GetSize() > 0)
34  {
35  tensorInfo = armnnUtils::Permuted(tensorInfo, permutationVector);
36  armnnUtils::Permute(tensorInfo.GetShape(), permutationVector,
37  tensor->GetConstTensor<void>(), permuteBuffer,
38  GetDataTypeSize(tensorInfo.GetDataType()));
39  }
40  else
41  {
42  ::memcpy(permuteBuffer, tensor->GetConstTensor<void>(), tensorInfo.GetNumBytes());
43  }
44  tensorInfo.SetConstant(true);
45  return ConstTensor(tensorInfo, permuteBuffer);
46 }
47 
48 void ReshapeWeightsForAcl(TensorInfo& weightInfo, DataLayout dataLayout)
49 {
50  // Reshape the weights in-place
51  const TensorShape& weightShape = weightInfo.GetShape();
52  switch (dataLayout)
53  {
54  case DataLayout::NHWC:
55  // The data layout is NHWC, reshape from [ H, W, I, M ] to [ 1, H, W, I * M ]
56  weightInfo.SetShape({ 1,
57  weightShape[0],
58  weightShape[1],
59  weightShape[2] * weightShape[3] });
60  weightInfo.SetShape({ 1,
61  weightShape[0] * weightShape[1],
62  weightShape[2],
63  weightShape[3] });
64  break;
65  case DataLayout::NCHW:
66  default:
67  // The data layout is NCHW, reshape from [ M, I, H, W ] to [ 1, I * M, H, W, ]
68  weightInfo.SetShape({ 1, weightShape[0] * weightShape[1], weightShape[2], weightShape[3] });
69  break;
70  }
71 }
72 
73 template <typename DataType>
74 ConstTensor ReorderWeightChannelsForAcl(const ConstTensor& weightHandle, DataLayout dataLayout, void* permuteBuffer)
75 {
76  DataType* weight = static_cast<DataType*>(permuteBuffer);
77  const TensorShape& weightShape = weightHandle.GetShape();
78  unsigned int multiplier;
79  unsigned int height;
80  unsigned int width;
81  unsigned int inputChannels;
82  switch (dataLayout)
83  {
84  case DataLayout::NHWC: //It actually is [ H, W, I, M ]
85  height = weightShape[0];
86  width = weightShape[1];
87  inputChannels = weightShape[2];
88  multiplier = weightShape[3];
89  break;
90  case DataLayout::NCHW: //It actually is [ M, I, H, W ]
91  default:
92  height = weightShape[2];
93  width = weightShape[3];
94  inputChannels = weightShape[1];
95  multiplier = weightShape[0];
96  break;
97  }
98 
99  std::vector<DataType> weightAclOrder(height*width*inputChannels*multiplier);
100  unsigned int destinationWeightsChannel;
101  unsigned int totalChannels = inputChannels * multiplier;
102  unsigned int channelSize = height * width;
103  unsigned int inputChannel = 0;
104 
105  for (unsigned int originWeightsChannel = 0; originWeightsChannel < totalChannels; originWeightsChannel++)
106  {
107  inputChannel = originWeightsChannel % inputChannels;
108  destinationWeightsChannel = (originWeightsChannel - inputChannel) / inputChannels + multiplier * inputChannel;
109 
110  for (unsigned int i = 0; i < channelSize; i++)
111  {
112  weightAclOrder[i + destinationWeightsChannel * channelSize] =
113  weight[i + originWeightsChannel * channelSize];
114  }
115  }
116 
117  ::memcpy(permuteBuffer, weightAclOrder.data(), weightHandle.GetInfo().GetNumBytes());
118  return ConstTensor(weightHandle.GetInfo(), permuteBuffer);
119 }
120 
121 
123 {
124  // Convert the weight format from ArmNN's [ M, I, H, W ] (does NOT depend on the data layout) to either
125  // [ 1, H, W, I * M ] (if NHWC) or [ 1, I * M, H, W ] (if NCHW), as required by the compute library
126 
127  // 1. Permute the weights if necessary
128  // If the data layout is NCHW no permutation is necessary, as a reshape to [ 1, I * M, H, W ] can be better done
129  // starting from the current shape of [ M, I, H, W ]
130  TensorInfo weightPermutedInfo(weightInfo);
131  if (dataLayout == DataLayout::NHWC)
132  {
133  // The data layout is NHWC, then permute the weights from [ M, I, H, W ] to [ H, W, I, M ]
134  PermutationVector permutationVector{ 3, 2, 0, 1 };
135  weightPermutedInfo = armnnUtils::Permuted(weightInfo, permutationVector);
136  }
137 
138  // 2. Reshape the weights
139  ReshapeWeightsForAcl(weightPermutedInfo, dataLayout);
140 
141  // 3. Return the permuted weight info
142  return weightPermutedInfo;
143 }
144 
145 
146 std::tuple<ConstTensor, unsigned int> Convert1HWOTensorToAcl(const ConstTensorHandle* weightTensor,
147  const TensorInfo& inputInfo,
148  const DataLayout dataLayout,
149  void* permuteBuffer)
150 {
151  TensorInfo weightsInfo = weightTensor->GetTensorInfo();
152  unsigned int depthMultiplier = 1;
153  PermutationVector permutationVector{};
154  if (dataLayout == armnn::DataLayout::NHWC)
155  {
156  // No permutation required. Data layouts are the same.
157 
158  depthMultiplier = weightsInfo.GetShape()[3] / inputInfo.GetShape()[3];
159  }
160  else if (dataLayout == armnn::DataLayout::NCHW)
161  {
162  // [ 1, H, W, I*M] --> [ 1, I * M, H, W ]
163  depthMultiplier = weightsInfo.GetShape()[3] / inputInfo.GetShape()[1];
164  permutationVector = { 0, 2, 3, 1 };
165  }
166  else
167  {
168  throw InvalidArgumentException(fmt::format("Unknown data layout for tensor conversion: {}",
169  GetDataLayoutName(dataLayout)));
170  }
171 
172  ConstTensor weightsPermuted = PermuteTensor(weightTensor, permutationVector, permuteBuffer);
173 
174  return std::make_tuple(weightsPermuted, depthMultiplier);
175 }
176 
177 std::tuple<TensorInfo, unsigned int> Convert1HWOTensorInfoToAcl(const TensorInfo& weightInfo,
178  const TensorInfo& inputInfo,
179  const DataLayout dataLayout)
180 {
181  unsigned int aclDepthMultiplier = 1;
182  TensorInfo weightsPermuted;
183  if (dataLayout == armnn::DataLayout::NHWC)
184  {
185  // No permutation required. Input and weights data layouts are the same.
186  aclDepthMultiplier = weightInfo.GetShape()[3] / inputInfo.GetShape()[3];
187  weightsPermuted = weightInfo;
188  }
189 
190  else if (dataLayout == armnn::DataLayout::NCHW)
191  {
192  // Weights permutation required. Weights [N,H,W,C] and input [N,C,H,W] data layouts are different.
193  // [ 1, H, W, I*M] --> [ 1, I * M, H, W ]
194  aclDepthMultiplier = weightInfo.GetShape()[3] / inputInfo.GetShape()[1];
195  PermutationVector permutationVector{ 0, 2, 3, 1 };
196  weightsPermuted = armnnUtils::Permuted(weightInfo, permutationVector);
197  }
198  else
199  {
200  throw InvalidArgumentException(fmt::format("Unknown data layout for tensor info conversion: {}",
201  GetDataLayoutName(dataLayout)));
202  }
203 
204  return std::make_tuple(weightsPermuted, aclDepthMultiplier);
205 }
206 
207 
208 std::tuple<ConstTensor, unsigned int> Convert1HWOtoMIHW(const ConstTensorHandle* weightTensor,
209  const TensorInfo& inputInfo,
210  const DataLayout& dataLayout,
211  void* permuteBuffer)
212 {
213  TensorInfo weightsInfo = weightTensor->GetTensorInfo();
214 
215  if (weightsInfo.HasPerAxisQuantization())
216  {
217  throw InvalidArgumentException("Can't convert tensor from [1,H,W,Cout] to [M,Cin,H,W] when per channel "
218  "quantization is applied.");
219  }
220 
221  // Reshape weights [ 1, H, W, I*M ] --> [ H, W, I, M ]
222  auto weightsShape = weightsInfo.GetShape();
223  auto channelIndex = armnnUtils::DataLayoutIndexed(dataLayout).GetChannelsIndex();
224  unsigned int depthMultiplier = weightsShape[3] / inputInfo.GetShape()[channelIndex];
225  weightsInfo.SetShape({ weightsShape[1],
226  weightsShape[2],
227  inputInfo.GetShape()[channelIndex],
228  depthMultiplier});
229 
230  // Permute [ H, W, I, M ] --> [ M, I, H, W ]
231  PermutationVector permutationVector = { 2, 3, 1, 0 };
232  ConstTensor weightsPermuted = PermuteTensor(weightTensor, permutationVector, permuteBuffer);
233 
234  return std::make_tuple(weightsPermuted, depthMultiplier);
235 }
236 
238  DataLayout dataLayout,
239  void* permuteBuffer)
240 {
241  if (weightTensor == nullptr)
242  {
243  throw armnn::InvalidArgumentException("WorkloadUtils: PermuteTensor: Null input tensor pointer");
244  }
245  if (permuteBuffer == nullptr)
246  {
247  throw armnn::InvalidArgumentException("WorkloadUtils: PermuteTensor: Null permute buffer pointer");
248  }
249 
250  auto multiplier = weightTensor->GetTensorInfo().GetShape()[0];
251  auto inputChannels = weightTensor->GetTensorInfo().GetShape()[1];
252 
253  // Convert the weight format from ArmNN's [ M, I, H, W ] (does NOT depend on the data layout) to either
254  // [ 1, H, W, I * M ] (if NHWC) or [ 1, I * M, H, W ] (if NCHW), as required by the compute library
255 
256  // 1. Permute the weights if necessary
257  // If the data layout is NCHW no permutation is necessary, as a reshape to [ 1, I * M, H, W ] can be better done
258  // starting from the current shape of [ M, I, H, W ]
259  // If no permutation is necessary, leave the permutation vector empty
260  PermutationVector permutationVector{};
261  if (dataLayout == DataLayout::NHWC)
262  {
263  // The data layout is NHWC, then permute the weights from [ M, I, H, W ] to [ H, W, I, M ]
264  permutationVector = { 3, 2, 0, 1 };
265  }
266  ConstTensor weightPermuted = PermuteTensor(weightTensor, permutationVector, permuteBuffer);
267 
268  // Shuffle the weights data to obtain the channel order needed used by Acl
269  if (multiplier > 1 && inputChannels > 1 && dataLayout == DataLayout::NCHW)
270  {
271  switch (weightPermuted.GetDataType())
272  {
273  case DataType::Float32:
274  weightPermuted = ReorderWeightChannelsForAcl<float>(weightPermuted, dataLayout, permuteBuffer);
275  break;
276  case DataType::Float16:
277  weightPermuted =
278  ReorderWeightChannelsForAcl<half_float::half>(weightPermuted, dataLayout, permuteBuffer);
279  break;
280  case DataType::QAsymmS8:
281  case DataType::QAsymmU8:
282  weightPermuted = ReorderWeightChannelsForAcl<uint8_t>(weightPermuted, dataLayout, permuteBuffer);
283  break;
284  case DataType::QSymmS8:
285  weightPermuted = ReorderWeightChannelsForAcl<int8_t>(weightPermuted, dataLayout, permuteBuffer);
286  break;
287  default:
288  break;
289  }
290  }
291 
292  // 2. Reshape the weights
293  ReshapeWeightsForAcl(weightPermuted.GetInfo(), dataLayout);
294 
295  // 3. Return both the tensor and the allocated storage to ensure that the data stays alive
296  return weightPermuted;
297 }
298 
299 int32_t ConvertMaskToACLFormat(int32_t mask, int32_t numDim)
300 {
301  int32_t reversedMask = 0;
302  for (unsigned int i = 0; i < armnn::numeric_cast<unsigned int>(numDim); ++i)
303  {
304  // Check if bit set in mask for each dimension
305  int32_t bit = (mask & 1 << i) != 0;
306  // Increment the new mask with the bits reversed
307  reversedMask += (bit << std::max(numDim-(armnn::numeric_cast<int>(i)+1), 0));
308  }
309 
310  return reversedMask;
311 }
312 
313 std::map<std::string, unsigned int> CalculateGatherNdKeyIndices(TensorInfo inputInfo0, TensorInfo inputInfo1)
314 {
315  std::vector<unsigned int> paramsShape;
316  for (unsigned int i = 0; i < inputInfo0.GetNumDimensions(); ++i)
317  {
318  paramsShape.push_back(inputInfo0.GetShape()[i]);
319  }
320 
321  std::vector<unsigned int> indicesShape;
322  for (unsigned int i = 0; i < inputInfo1.GetNumDimensions(); ++i)
323  {
324  indicesShape.push_back(inputInfo1.GetShape()[i]);
325  }
326 
327  std::map<std::string, unsigned int> keyIndices;
328 
329  // N: number of batches
330  keyIndices["N"] = 1;
331 
332  // ND: number of dimensions that are sliced from params
333  keyIndices["ND"] = indicesShape.back();
334 
335  // W: number of indices in each batch (all but the last dimension)
336  keyIndices["W"] =
337  static_cast<unsigned int>(std::accumulate(std::begin(indicesShape),
338  std::end(indicesShape) - 1,
339  1,
340  std::multiplies<>() ));
341  // K: range of each index
342  keyIndices["K"] =
343  static_cast<unsigned int>(std::accumulate(std::begin(paramsShape),
344  std::begin(paramsShape) + static_cast<int>(keyIndices["ND"]),
345  1,
346  std::multiplies<>() ));
347  // C: number of channels for each index
348  keyIndices["C"] =
349  static_cast<unsigned int>(std::accumulate(std::begin(paramsShape) + static_cast<int>(keyIndices["ND"]),
350  std::end(paramsShape),
351  1,
352  std::multiplies<>() ));
353 
354  return keyIndices;
355 }
356 
358 {
359  armnn::PermutationVector permutationVector{};
360  switch (rank)
361  {
362  case 2:
363  permutationVector = {1U, 0U};
364  break;
365  case 3:
366  permutationVector = {0U, 2U, 1U};
367  break;
368  case 4:
369  permutationVector = {0U, 1U, 3U, 2U};
370  break;
371  default:
372  throw Exception("Invalid number of dimensions.");
373  }
374  return permutationVector;
375 }
376 
377 std::set<unsigned int> ComputeSplitAxis(const armnn::SplitterDescriptor& desc, const TensorShape& input)
378 {
379  unsigned int numSplit = desc.GetNumViews();
380  unsigned int numDimensions = desc.GetNumDimensions();
381  std::set<unsigned int> splitAxis;
382  if (desc.HasAxis())
383  {
384  splitAxis.insert(armnnUtils::GetUnsignedAxis(desc.GetNumDimensions(), desc.GetAxis()));
385  }
386  else
387  {
388  for (unsigned int i = 0; i < numSplit; ++i)
389  {
390  for (unsigned int dimIdx = 0; dimIdx < numDimensions; ++dimIdx)
391  {
392  if (desc.GetViewSizes(i)[dimIdx] != input[dimIdx])
393  {
394  splitAxis.insert(dimIdx);
395  }
396  }
397  }
398  }
399  return splitAxis;
400 }
401 
402 } // namespace armnn
armnn::ViewsDescriptor
A ViewsDescriptor for the SplitterLayer.
Definition: Descriptors.hpp:244
armnn::Convert1HWOTensorInfoToAcl
std::tuple< TensorInfo, unsigned int > Convert1HWOTensorInfoToAcl(const TensorInfo &weightInfo, const TensorInfo &inputInfo, const DataLayout dataLayout)
Weights for depthwise have a datalayout of [1,H,W,O] = [1,H,W,I*M] This function coverts a TensorInfo...
Definition: WorkloadUtils.cpp:177
armnn::Convert1HWOTensorToAcl
std::tuple< ConstTensor, unsigned int > Convert1HWOTensorToAcl(const ConstTensorHandle *weightTensor, const TensorInfo &inputInfo, const DataLayout dataLayout, void *permuteBuffer)
Weights for depthwise have a datalayout of [1,H,W,O] = [1,H,W,I*M] This function coverts a ConstCpuTe...
Definition: WorkloadUtils.cpp:146
armnn::TensorInfo::GetNumBytes
unsigned int GetNumBytes() const
Definition: Tensor.cpp:427
armnn::DataLayout
DataLayout
Definition: Types.hpp:62
WorkloadUtils.hpp
armnn::DataLayout::NHWC
@ NHWC
armnn::ConstTensorHandle
Definition: TensorHandle.hpp:24
armnnUtils::GetUnsignedAxis
unsigned int GetUnsignedAxis(const unsigned int inputDimension, const int axis)
Definition: TensorUtils.cpp:236
armnn::ViewsDescriptor::HasAxis
bool HasAxis() const
Returns true if an axis has been set.
Definition: Descriptors.cpp:388
armnn::TensorInfo
Definition: Tensor.hpp:152
armnn::PermuteTensor
armnn::ConstTensor PermuteTensor(const ConstTensorHandle *tensor, const PermutationVector &permutationVector, void *permuteBuffer)
Definition: WorkloadUtils.cpp:19
armnn::TensorInfo::GetNumDimensions
unsigned int GetNumDimensions() const
Definition: Tensor.hpp:197
armnnUtils::DataLayoutIndexed
Provides access to the appropriate indexes for Channels, Height and Width based on DataLayout.
Definition: DataLayoutIndexed.hpp:17
armnn::DataType::Float32
@ Float32
armnn::GetDataLayoutName
constexpr const char * GetDataLayoutName(DataLayout dataLayout)
Definition: TypesUtils.hpp:253
armnn::DataType::QAsymmU8
@ QAsymmU8
armnn::ConstTensorHandle::GetTensorInfo
const TensorInfo & GetTensorInfo() const
Definition: TensorHandle.hpp:40
armnn::DataType::QSymmS8
@ QSymmS8
armnnUtils::Permute
void Permute(const armnn::TensorShape &dstShape, const armnn::PermutationVector &mappings, const void *src, void *dst, size_t dataTypeSize)
Definition: Permute.cpp:164
armnnUtils::Permuted
armnn::TensorShape Permuted(const armnn::TensorShape &srcShape, const armnn::PermutationVector &mappings)
Definition: Permute.cpp:125
armnn::TensorInfo::HasPerAxisQuantization
bool HasPerAxisQuantization() const
Definition: Tensor.cpp:446
NumericCast.hpp
TensorUtils.hpp
armnn::BaseTensor::GetDataType
DataType GetDataType() const
Definition: Tensor.hpp:302
armnn::Convert1HWOtoMIHW
std::tuple< ConstTensor, unsigned int > Convert1HWOtoMIHW(const ConstTensorHandle *weightTensor, const TensorInfo &inputInfo, const DataLayout &dataLayout, void *permuteBuffer)
Converts a (weights) tensor from [1, H, W, I*M] = [1, H, W, O] to [M, I, H, W].
Definition: WorkloadUtils.cpp:208
armnn::ViewsDescriptor::GetViewSizes
const uint32_t * GetViewSizes(uint32_t idx) const
Get the view sizes at the int value idx.
Definition: Descriptors.cpp:347
armnn::ConstTensorHandle::GetConstTensor
const T * GetConstTensor() const
Definition: TensorHandle.hpp:28
armnn::TensorShape
Definition: Tensor.hpp:20
armnn::DataType::Float16
@ Float16
armnn::ViewsDescriptor::GetAxis
int32_t GetAxis() const
Get the axis value.
Definition: Descriptors.cpp:382
armnn::ConvertWeightTensorFromArmnnToAcl
armnn::ConstTensor ConvertWeightTensorFromArmnnToAcl(const ConstTensorHandle *weightTensor, DataLayout dataLayout, void *permuteBuffer)
Definition: WorkloadUtils.cpp:237
Utils.hpp
armnn::CalculateGatherNdKeyIndices
std::map< std::string, unsigned int > CalculateGatherNdKeyIndices(TensorInfo inputInfo0, TensorInfo inputInfo1)
Calculates the key index values needed for GatherNd: N, ND, K, W, C (N is always 1)
Definition: WorkloadUtils.cpp:313
armnn::DataType
DataType
Definition: Types.hpp:48
armnn::InvalidArgumentException
Definition: Exceptions.hpp:80
armnn::GetDataTypeSize
constexpr unsigned int GetDataTypeSize(DataType dataType)
Definition: TypesUtils.hpp:182
armnn::BaseTensor::GetShape
const TensorShape & GetShape() const
Definition: Tensor.hpp:299
armnn::PermutationVector
Definition: Types.hpp:314
armnn::ReorderWeightChannelsForAcl
ConstTensor ReorderWeightChannelsForAcl(const ConstTensor &weightHandle, DataLayout dataLayout, void *permuteBuffer)
Definition: WorkloadUtils.cpp:74
armnn::Exception
Base class for all ArmNN exceptions so that users can filter to just those.
Definition: Exceptions.hpp:46
armnn::BaseTensor::GetInfo
const TensorInfo & GetInfo() const
Definition: Tensor.hpp:297
armnn::TensorInfo::GetDataType
DataType GetDataType() const
Definition: Tensor.hpp:200
armnn::DataType::QAsymmS8
@ QAsymmS8
armnn::ReshapeWeightsForAcl
void ReshapeWeightsForAcl(TensorInfo &weightInfo, DataLayout dataLayout)
Definition: WorkloadUtils.cpp:48
armnn::PermutationVector::GetSize
SizeType GetSize() const
Definition: Types.hpp:357
armnn::TensorInfo::GetShape
const TensorShape & GetShape() const
Definition: Tensor.hpp:193
armnn::GeneratePermutationVectorOnLastTwoDimensions
armnn::PermutationVector GeneratePermutationVectorOnLastTwoDimensions(unsigned int rank)
Generates a permutation vector of size rank that permutes the 2 most right dimensions.
Definition: WorkloadUtils.cpp:357
armnn::ViewsDescriptor::GetNumDimensions
uint32_t GetNumDimensions() const
Get the number of dimensions.
Definition: Descriptors.cpp:307
armnn::ConvertMaskToACLFormat
int32_t ConvertMaskToACLFormat(int32_t mask, int32_t numDim)
Definition: WorkloadUtils.cpp:299
armnn::ConvertWeightTensorInfoFromArmnnToAcl
TensorInfo ConvertWeightTensorInfoFromArmnnToAcl(const TensorInfo &weightInfo, DataLayout dataLayout)
Definition: WorkloadUtils.cpp:122
armnn::TensorInfo::SetShape
void SetShape(const TensorShape &newShape)
Definition: Tensor.hpp:195
armnn
Copyright (c) 2021 ARM Limited and Contributors.
Definition: 01_00_quick_start.dox:6
armnnUtils::DataLayoutIndexed::GetChannelsIndex
unsigned int GetChannelsIndex() const
Definition: DataLayoutIndexed.hpp:23
armnn::ComputeSplitAxis
std::set< unsigned int > ComputeSplitAxis(const armnn::SplitterDescriptor &desc, const TensorShape &input)
Calculates the axis values for split operation.
Definition: WorkloadUtils.cpp:377
armnn::ViewsDescriptor::GetNumViews
uint32_t GetNumViews() const
Get the number of views.
Definition: Descriptors.cpp:302
armnn::ConstTensor
A tensor defined by a TensorInfo (shape and data type) and an immutable backing store.
Definition: Tensor.hpp:329
armnn::TensorInfo::SetConstant
void SetConstant(const bool IsConstant=true)
Marks the data corresponding to this tensor info as constant.
Definition: Tensor.cpp:518
DataLayoutIndexed.hpp
armnn::DataLayout::NCHW
@ NCHW