ArmNN
 24.11
ElementwiseBinaryOperator.cpp
Go to the documentation of this file.
1 //
2 // Copyright © 2022-2024 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5 
6 #include "ElementwiseBinaryOperator.hpp"
7 #include "TosaRescaleOperatorUtils.hpp"
8 
9 void AddRescaleOp(const string &inputName,
10  const string &outputName,
11  std::vector<TosaSerializationTensor*>& tensors,
12  const std::vector<const TensorInfo*>& inputs,
13  const std::vector<const TensorInfo*>& outputs,
14  std::vector<TosaSerializationOperator*>& operators)
15 {
16  double scale_alpha = inputs[1]->GetQuantizationScale() / outputs[0]->GetQuantizationScale();
17  int32_t input_zp = inputs[1]->GetQuantizationOffset();
18  int32_t output_zp = outputs[0]->GetQuantizationOffset();
19 
20  TosaSerializationOperator* rescaleOp = nullptr;
21  CreateRescaleTosaOperator(inputName,
22  outputName,
23  scale_alpha,
24  input_zp,
25  output_zp,
26  true,
27  true,
28  &rescaleOp);
29 
30  std::vector<int32_t> inputShape = GetTosaTensorShape(inputs[1]->GetShape());
31  tensors.push_back(new TosaSerializationTensor(outputName,
32  inputShape,
33  DType_INT32, {}));
34  operators.push_back(rescaleOp);
35 }
36 
37 TosaSerializationBasicBlock* ConvertElementwiseBinaryToTosaOperator(const Layer* layer,
38  const LayerType type,
39  const std::vector<const TensorInfo*>& inputs,
40  const std::vector<const TensorInfo*>& outputs,
41  const ElementwiseBinaryDescriptor* descriptor)
42 {
43  std::string input0Name = std::string("input_0");
44  std::string input1Name = std::string("input_1");
45  std::string outputName = std::string("output0_");
46  std::string input0ElemenwiseBinaryName = std::string("intermediate0_") + GetUniqueTosaMappingID();
47  std::string input1ElemenwiseBinaryName = std::string("intermediate0_") + GetUniqueTosaMappingID();
48  std::string blockName;
49 
50  // If a layer is present then the block will be used for execution, so input and output names need to be determined
51  // using the previous and following layers so the graph is connected correctly. For validation this doesn't matter.
52  if(layer != nullptr)
53  {
54  input0Name = GenerateUniqueInputName(layer->GetInputSlot(0));
55  input1Name = GenerateUniqueInputName(layer->GetInputSlot(1));
56  outputName = GenerateUniqueOutputName(*layer);
57  }
58 
59  TosaSerializationOperator* op = nullptr;
60 
61  std::vector<TosaSerializationTensor*> tensors;
62  std::vector<TosaSerializationOperator*> operators;
63  DType inputDType0 = ArmNNToDType(inputs[0]->GetDataType());
64  DType inputDType1 = ArmNNToDType(inputs[1]->GetDataType());
65  DType outputDType0 = ArmNNToDType(outputs[0]->GetDataType());
66  bool isInputInt8 = (inputDType0 == DType_INT8);
67 
68  // Only add input tensors if connected layer is an input layer.
69  // As intermediate or constant tensors will be created separately.
70  // There also can't be duplicate tensor.
71  if(input0Name.find("input_") != std::string::npos)
72  {
73  std::vector<int32_t> inputShape0 = GetTosaTensorShape(inputs[0]->GetShape());
74  tensors.push_back(new TosaSerializationTensor(input0Name, inputShape0, inputDType0, {}));
75  }
76  if(input1Name.find("input_") != std::string::npos)
77  {
78  std::vector<int32_t> inputShape1 = GetTosaTensorShape(inputs[1]->GetShape());
79  tensors.push_back(new TosaSerializationTensor(input1Name, inputShape1, inputDType1, {}));
80  }
81 
82  std::vector<int32_t> outputShape0 = GetTosaTensorShape(outputs[0]->GetShape());
83 
84  // Assign an output name and add to tensors based on the input type
85  // An int8 input for all ops will require the output to be rescaled from int32 to int8
86  std::string outputElemenwiseBinaryName;
87  if (isInputInt8)
88  {
89  outputElemenwiseBinaryName = std::string("intermediate0_") + GetUniqueTosaMappingID();
90  tensors.push_back(new TosaSerializationTensor(outputElemenwiseBinaryName, outputShape0, DType_INT32, {}));
91  }
92  else
93  {
94  tensors.push_back(new TosaSerializationTensor(outputName, outputShape0, outputDType0, {}));
95  }
96 
97  // Add supports DType_INT32 input only, so a rescale is required when input is DType_INT8
98  // MUL op is the only exception which has TOSA int8 support
99  bool isMulDesc = descriptor ? descriptor->m_Operation == BinaryOperation::Mul : false;
100  bool isMulOp = (type == LayerType::Multiplication) || isMulDesc ? true : false;
101  if (isInputInt8 && !isMulOp)
102  {
103  AddRescaleOp(input0Name, input0ElemenwiseBinaryName, tensors, inputs, outputs, operators);
104  AddRescaleOp(input1Name, input1ElemenwiseBinaryName, tensors, inputs, outputs, operators);
105  }
106 
107  std::string& elementwiseInput0Str = isInputInt8 ? input0ElemenwiseBinaryName : input0Name;
108  std::string& elementwiseInput1Str = isInputInt8 ? input1ElemenwiseBinaryName : input1Name;
109  std::string& elementwiseOutputStr = isInputInt8 ? outputElemenwiseBinaryName : outputName;
110 
111  switch(type)
112  {
113  case LayerType::Addition:
114  {
115  op = new TosaSerializationOperator(Op_ADD,
116  Attribute_NONE,
117  nullptr,
118  {input0Name, input1Name},
119  {outputName});
120  blockName = std::string("Op_ADD_block_") + GetUniqueTosaMappingID();
121  break;
122  }
123  case LayerType::ElementwiseBinary:
124  {
125  switch (descriptor->m_Operation)
126  {
127  case BinaryOperation::Add:
128  {
129  op = new TosaSerializationOperator(Op_ADD,
130  Attribute_NONE,
131  nullptr,
132  {elementwiseInput0Str, elementwiseInput1Str},
133  {elementwiseOutputStr});
134  blockName = std::string("Op_ADD_block_") + GetUniqueTosaMappingID();
135  break;
136  }
137  case BinaryOperation::Maximum:
138  {
139  op = new TosaSerializationOperator(Op_MAXIMUM,
140  Attribute_NONE,
141  nullptr,
142  {elementwiseInput0Str, elementwiseInput1Str},
143  {elementwiseOutputStr});
144  blockName = std::string("Op_MAXIMUM_block_") + GetUniqueTosaMappingID();
145  break;
146  }
147  case BinaryOperation::Mul:
148  {
149  int8_t shift = 0;
150  TosaMulAttribute mulAttribute(shift);
151 
152  // Mul supports input DType_INT8 so will not require a rescale before the op.
153  // i.e "input0Name" is used for the input and not intermediate "elementwiseInput0Str"
154  op = new TosaSerializationOperator(Op_MUL,
155  Attribute_MulAttribute,
156  &mulAttribute,
157  {input0Name, input1Name},
158  {elementwiseOutputStr});
159  blockName = std::string("Op_MUL_block_") + GetUniqueTosaMappingID();
160  break;
161  }
162  case BinaryOperation::Sub:
163  {
164  op = new TosaSerializationOperator(Op_SUB,
165  Attribute_NONE,
166  nullptr,
167  {elementwiseInput0Str, elementwiseInput1Str},
168  {elementwiseOutputStr});
169  blockName = std::string("Op_SUB_block_") + GetUniqueTosaMappingID();
170  break;
171  }
172  case BinaryOperation::SqDiff:
173  {
174  throw Exception("TOSA mappings of Squared Difference operator "
175  "implemented under ConvertSquaredDifferenceToTosaOperator().");
176  }
177  default:
178  throw Exception("ConvertElementwiseBinaryToTosaOperator: Unsupported layer type.");
179  }
180  break;
181  }
182  case LayerType::Multiplication:
183  {
184  int32_t shift = 0;
185  TosaMulAttribute mulAttribute(shift);
186  op = new TosaSerializationOperator(Op_MUL,
187  Attribute_MulAttribute,
188  &mulAttribute,
189  {input0Name, input1Name},
190  {outputName});
191  blockName = std::string("Op_MUL_block_") + GetUniqueTosaMappingID();
192  break;
193  }
194  case LayerType::Subtraction:
195  {
196  op = new TosaSerializationOperator(Op_SUB,
197  Attribute_NONE,
198  nullptr,
199  {input0Name, input1Name},
200  {outputName});
201  blockName = std::string("Op_SUB_block_") + GetUniqueTosaMappingID();
202  break;
203  }
204  default:
205  throw Exception("ConvertElementwiseBinaryToTosaOperator: Unsupported layer type.");
206  }
207 
208  operators.push_back(op);
209 
210  // All ElementwiseBinary operators require a rescale of output
211  // from DType_INT32 to DType_INT8 when the input is DType_INT8
212  if (inputDType0 == DType_INT8)
213  {
214  AddRescaleOp(outputElemenwiseBinaryName, outputName, tensors, inputs, outputs, operators);
215  }
216 
217  return new TosaSerializationBasicBlock(blockName, // name
218  mainName, // region name
219  {operators}, // operators
220  tensors, // tensors
221  {input0Name, input1Name}, // inputs
222  {outputName}); // outputs
223 }
224 
225 TosaSerializationBasicBlock* ConvertSquaredDifferenceToTosaOperator(const Layer* layer,
226  const LayerType,
227  const std::vector<const TensorInfo*>& inputs,
228  const std::vector<const TensorInfo*>& outputs,
229  const ElementwiseBinaryDescriptor* descriptor)
230 {
231  if (descriptor->m_Operation != BinaryOperation::SqDiff)
232  {
233  throw Exception("ElementwiseBinaryDescriptor operation must be SqDiff"
234  "in ConvertSquaredDifferenceToTosaOperator().");
235  }
236 
237  std::string input0Name = std::string("input_0");
238  std::string input1Name = std::string("input_1");
239  std::string outputName = std::string("output0_");
240  std::string interElemenwiseBinaryName = std::string("intermediate0_") + GetUniqueTosaMappingID();
241  std::string blockName = std::string("Op_SQDIFF_block_") + GetUniqueTosaMappingID();
242 
243  // If a layer is present then the block will be used for execution, so input and output names need to be determined
244  // using the previous and following layers so the graph is connected correctly. For validation this doesn't matter.
245  if (layer != nullptr)
246  {
247  if (layer->GetInputSlot(0).GetConnectedOutputSlot()->GetOwningLayer().GetType() == LayerType::Reshape ||
248  layer->GetInputSlot(1).GetConnectedOutputSlot()->GetOwningLayer().GetType() == LayerType::Reshape)
249  {
250  interElemenwiseBinaryName = std::string("intermediate1_") + GetUniqueTosaMappingID();
251  }
252 
253  input0Name = GenerateUniqueInputName(layer->GetInputSlot(0));
254  input1Name = GenerateUniqueInputName(layer->GetInputSlot(1));
255  outputName = GenerateUniqueOutputName(*layer);
256  }
257 
258  std::vector<TosaSerializationTensor*> tensors {};
259  std::vector<TosaSerializationOperator*> operators {};
260  DType inputDType0 = ArmNNToDType(inputs[0]->GetDataType());
261  DType inputDType1 = ArmNNToDType(inputs[1]->GetDataType());
262  DType outputDType0 = ArmNNToDType(outputs[0]->GetDataType());
263  bool isInputInt8 = (inputDType0 == DType_INT8);
264 
265  // Only add input tensors if connected layer is an input layer.
266  // As intermediate or constant tensors will be created separately.
267  // There also can't be duplicate tensor.
268  if(input0Name.find("input_") != std::string::npos)
269  {
270  std::vector<int32_t> inputShape0 = GetTosaTensorShape(inputs[0]->GetShape());
271  tensors.push_back(new TosaSerializationTensor(input0Name, inputShape0, inputDType0, {}));
272  }
273  if(input1Name.find("input_") != std::string::npos)
274  {
275  std::vector<int32_t> inputShape1 = GetTosaTensorShape(inputs[1]->GetShape());
276  tensors.push_back(new TosaSerializationTensor(input1Name, inputShape1, inputDType1, {}));
277  }
278 
279  std::vector<int32_t> outputShape0 = GetTosaTensorShape(outputs[0]->GetShape());
280 
281  if (inputDType0 == DType_FP32 ||
282  inputDType0 == DType_FP16 ||
283  inputDType0 == DType_INT32)
284  {
285  operators.push_back(new TosaSerializationOperator(
286  Op_SUB,
287  Attribute_NONE,
288  nullptr,
289  {input0Name, input1Name},
290  {interElemenwiseBinaryName}));
291  tensors.push_back(new TosaSerializationTensor(interElemenwiseBinaryName,
292  outputShape0,
293  outputDType0,
294  {}));
295 
296  int8_t shift = 0;
297  TosaMulAttribute mulAttribute(shift);
298 
299  operators.push_back(new TosaSerializationOperator(
300  Op_MUL,
301  Attribute_MulAttribute,
302  &mulAttribute,
303  {interElemenwiseBinaryName, interElemenwiseBinaryName},
304  {outputName}));
305  }
306  else if (isInputInt8)
307  {
308  std::string rescale0Output0Name = std::string("intermediate0_") + GetUniqueTosaMappingID();
309  std::string rescale0Output1Name = std::string("intermediate1_") + GetUniqueTosaMappingID();
310  std::string rescale1Output0Name = std::string("intermediate2_") + GetUniqueTosaMappingID();
311  std::string rescale1Output1Name = std::string("intermediate3_") + GetUniqueTosaMappingID();
312  std::string mulOutputName = std::string("intermediate4_") + GetUniqueTosaMappingID();
313  interElemenwiseBinaryName = std::string("intermediate5_") + GetUniqueTosaMappingID();
314 
315  // We need to make sure the inputs are rescaled correctly
316  // Following the behaviour defined here lite/kernels/squared_difference.cc
317  double in_x_scale = inputs[0]->GetQuantizationScale();
318  double in_y_scale = inputs[1]->GetQuantizationScale();
319  double result_scale = outputs[0]->GetQuantizationScale();
320  double twice_max_input_scale = 2.0 * std::max(in_x_scale, in_y_scale);
321  const int32_t LEFT_SHIFT = 7;
322  double x_rescale_scale = in_x_scale / twice_max_input_scale;
323  double y_rescale_scale = in_y_scale / twice_max_input_scale;
324  double output_rescale_scale =
325  (twice_max_input_scale * twice_max_input_scale) /
326  ((static_cast<double>(1 << LEFT_SHIFT * 2)) * result_scale);
327 
328  TosaSerializationOperator* xShiftOp = nullptr;
329  CreateRescaleTosaOperator(input0Name,
330  rescale0Output0Name,
331  (1 << LEFT_SHIFT),
332  inputs[0]->GetQuantizationOffset(),
333  0,
334  true,
335  true,
336  &xShiftOp);
337  operators.push_back(xShiftOp);
338  tensors.push_back(new TosaSerializationTensor(rescale0Output0Name,
339  GetTosaTensorShape(inputs[0]->GetShape()),
340  DType_INT32,
341  {}));
342 
343  TosaSerializationOperator* yShiftOp = nullptr;
344  CreateRescaleTosaOperator(input1Name,
345  rescale0Output1Name,
346  (1 << LEFT_SHIFT),
347  inputs[1]->GetQuantizationOffset(),
348  0,
349  true,
350  true,
351  &yShiftOp);
352  operators.push_back(yShiftOp);
353  tensors.push_back(new TosaSerializationTensor(rescale0Output1Name,
354  GetTosaTensorShape(inputs[1]->GetShape()),
355  DType_INT32,
356  {}));
357 
358  TosaSerializationOperator* xScaledOp = nullptr;
359  CreateRescaleTosaOperator(rescale0Output0Name,
360  rescale1Output0Name, //change
361  x_rescale_scale,
362  0,
363  0,
364  true,
365  true,
366  &xScaledOp);
367  operators.push_back(xScaledOp);
368  tensors.push_back(new TosaSerializationTensor(rescale1Output0Name,
369  GetTosaTensorShape(inputs[0]->GetShape()),
370  DType_INT32,
371  {}));
372 
373  TosaSerializationOperator* yScaledOp = nullptr;
374  CreateRescaleTosaOperator(rescale0Output1Name,
375  rescale1Output1Name, //change
376  y_rescale_scale,
377  0,
378  0,
379  true,
380  true,
381  &yScaledOp);
382  operators.push_back(yScaledOp);
383  tensors.push_back(new TosaSerializationTensor(rescale1Output1Name,
384  GetTosaTensorShape(inputs[1]->GetShape()),
385  DType_INT32,
386  {}));
387 
388 
389 
390  operators.push_back(new TosaSerializationOperator(
391  Op_SUB,
392  Attribute_NONE,
393  nullptr,
394  {rescale1Output0Name, rescale1Output1Name},
395  {interElemenwiseBinaryName}));
396  tensors.push_back(new TosaSerializationTensor(interElemenwiseBinaryName,
397  GetTosaTensorShape(outputs[0]->GetShape()),
398  DType_INT32,
399  {}));
400  int8_t shift = 0;
401  TosaMulAttribute mulAttribute(shift);
402 
403  operators.push_back(new TosaSerializationOperator(
404  Op_MUL,
405  Attribute_MulAttribute,
406  &mulAttribute,
407  {interElemenwiseBinaryName, interElemenwiseBinaryName},
408  {mulOutputName}));
409  tensors.push_back(new TosaSerializationTensor(mulOutputName,
410  GetTosaTensorShape(outputs[0]->GetShape()),
411  DType_INT32,
412  {}));
413 
414 
415  TosaSerializationOperator* rescaleOutputOp = nullptr;
416  CreateRescaleTosaOperator(mulOutputName,
417  outputName,
418  output_rescale_scale,
419  0,
420  outputs[0]->GetQuantizationOffset(),
421  true,
422  true,
423  &rescaleOutputOp);
424  operators.push_back(rescaleOutputOp);
425  }
426  else
427  {
428  throw Exception("TOSA spec only supports INT8, INT32, FP16 and FP32 datatypes for SqDiff.");
429  }
430 
431  tensors.push_back(new TosaSerializationTensor(outputName, outputShape0, outputDType0, {}));
432 
433  return new TosaSerializationBasicBlock(blockName, // name
434  mainName, // region name
435  {operators}, // operators
436  tensors, // tensors
437  {input0Name, input1Name}, // inputs
438  {outputName}); // outputs
439 }
440 
ConvertElementwiseBinaryToTosaOperator
TosaSerializationBasicBlock * ConvertElementwiseBinaryToTosaOperator(const Layer *layer, const LayerType type, const std::vector< const TensorInfo * > &inputs, const std::vector< const TensorInfo * > &outputs, const ElementwiseBinaryDescriptor *descriptor)
Definition: ElementwiseBinaryOperator.cpp:37
ElementwiseBinaryOperator.hpp
TosaRescaleOperatorUtils.hpp
GenerateUniqueOutputName
std::string GenerateUniqueOutputName(const Layer &layer, uint32_t layerSlot=0)
Definition: TosaOperatorUtils.hpp:120
armnn::Layer::GetInputSlot
const InputSlot & GetInputSlot(unsigned int index) const override
Get a const input slot handle by slot index.
Definition: Layer.hpp:337
ConvertSquaredDifferenceToTosaOperator
TosaSerializationBasicBlock * ConvertSquaredDifferenceToTosaOperator(const Layer *layer, const LayerType, const std::vector< const TensorInfo * > &inputs, const std::vector< const TensorInfo * > &outputs, const ElementwiseBinaryDescriptor *descriptor)
Definition: ElementwiseBinaryOperator.cpp:225
armnn::Layer
Definition: Layer.hpp:230
armnn::ElementwiseBinaryDescriptor
A ElementwiseBinaryDescriptor for the ElementwiseBinaryLayer.
Definition: Descriptors.hpp:109
mainName
const std::string mainName
Definition: TosaOperatorUtils.hpp:19
armnn::OutputSlot::GetOwningLayer
Layer & GetOwningLayer() const
Definition: Layer.hpp:132
ArmNNToDType
DType ArmNNToDType(const DataType &type)
Definition: TosaOperatorUtils.hpp:22
armnn::Exception
Base class for all ArmNN exceptions so that users can filter to just those.
Definition: Exceptions.hpp:46
armnn::Layer::GetType
LayerType GetType() const override
Returns the armnn::LayerType of this layer.
Definition: Layer.hpp:286
GetTosaTensorShape
std::vector< int32_t > GetTosaTensorShape(const TensorShape &shape)
Definition: TosaOperatorUtils.hpp:79
armnn::ElementwiseBinaryDescriptor::m_Operation
BinaryOperation m_Operation
Specifies the elementwiseBinary operation to execute.
Definition: Descriptors.hpp:125
armnn::InputSlot::GetConnectedOutputSlot
const OutputSlot * GetConnectedOutputSlot() const
Definition: Layer.hpp:56
CreateRescaleTosaOperator
void CreateRescaleTosaOperator(const std::string &inputName, const std::string &outputName, const std::vector< int32_t > &multipliers, const std::vector< int32_t > &shifts, int32_t input_zp, int32_t output_zp, bool double_round, bool scale32, bool per_channel, TosaSerializationOperator **op)
Definition: TosaRescaleOperatorUtils.hpp:10
GenerateUniqueInputName
std::string GenerateUniqueInputName(const armnn::InputSlot &slot)
Definition: TosaOperatorUtils.hpp:109
AddRescaleOp
void AddRescaleOp(const string &inputName, const string &outputName, std::vector< TosaSerializationTensor * > &tensors, const std::vector< const TensorInfo * > &inputs, const std::vector< const TensorInfo * > &outputs, std::vector< TosaSerializationOperator * > &operators)
Definition: ElementwiseBinaryOperator.cpp:9
armnn::LayerType
LayerType
When adding a new layer, adapt also the LastLayer enum value in the enum class LayerType below.
Definition: Types.hpp:496
GetUniqueTosaMappingID
std::string GetUniqueTosaMappingID()
Definition: TosaOperatorUtils.hpp:138