ArmNN
 24.08
armnn_delegate.hpp
Go to the documentation of this file.
1 //
2 // Copyright © 2023-2024 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5 
6 #pragma once
7 
8 #include <DelegateOptions.hpp>
9 #include <Version.hpp>
10 
11 #include <tensorflow/core/public/version.h>
12 #include <tensorflow/lite/c/c_api_opaque.h>
13 
14 #include <tensorflow/lite/acceleration/configuration/delegate_registry.h>
15 #include <tensorflow/lite/core/acceleration/configuration/c/stable_delegate.h>
16 
17 #if TF_MAJOR_VERSION > 2 || (TF_MAJOR_VERSION == 2 && TF_MINOR_VERSION > 5)
18 #define ARMNN_POST_TFLITE_2_5
19 #endif
20 
22 {
23 
25 {
26  DelegateData(const std::vector<armnn::BackendId>& backends)
27  : m_Backends(backends)
28  , m_Network(nullptr, nullptr)
29  {}
30 
31  const std::vector<armnn::BackendId> m_Backends;
33  std::vector<armnn::IOutputSlot*> m_OutputSlotForNode;
34 };
35 
36 /// Forward declaration for functions initializing the ArmNN Delegate
37 ::armnnDelegate::DelegateOptions TfLiteArmnnDelegateOptionsDefault();
38 
39 TfLiteOpaqueDelegate* TfLiteArmnnOpaqueDelegateCreate(armnnDelegate::DelegateOptions options);
40 
41 void TfLiteArmnnOpaqueDelegateDelete(TfLiteOpaqueDelegate* tfLiteDelegate);
42 
43 TfLiteStatus DoPrepare(TfLiteOpaqueContext* context, TfLiteOpaqueDelegate* delegate, void* data);
44 
45 armnnDelegate::DelegateOptions ParseArmNNSettings(const tflite::TFLiteSettings* tflite_settings);
46 
47 /// ArmNN Opaque Delegate
49 {
50  friend class ArmnnSubgraph;
51 public:
52  explicit ArmnnOpaqueDelegate(armnnDelegate::DelegateOptions options);
53 
54  TfLiteIntArray* IdentifyOperatorsToDelegate(TfLiteOpaqueContext* context);
55 
56  TfLiteOpaqueDelegateBuilder* GetDelegateBuilder() { return &m_Builder; }
57 
58  /// Retrieve version in X.Y.Z form
59  static const std::string GetVersion();
60 
61 private:
62  /**
63  * Returns a pointer to the armnn::IRuntime* this will be shared by all armnn_delegates.
64  */
65  armnn::IRuntime* GetRuntime(const armnn::IRuntime::CreationOptions& options)
66  {
67  static armnn::IRuntimePtr instance = armnn::IRuntime::Create(options);
68  /// Instantiated on first use.
69  return instance.get();
70  }
71 
72  TfLiteOpaqueDelegateBuilder m_Builder =
73  {
74  reinterpret_cast<void*>(this), // .data_
75  DoPrepare, // .Prepare
76  nullptr, // .CopyFromBufferHandle
77  nullptr, // .CopyToBufferHandle
78  nullptr, // .FreeBufferHandle
79  kTfLiteDelegateFlagsNone, // .flags
80  };
81 
82  /// ArmNN Runtime pointer
83  armnn::IRuntime* m_Runtime;
84  /// ArmNN Delegate Options
85  armnnDelegate::DelegateOptions m_Options;
86 };
87 
88 static int TfLiteArmnnOpaqueDelegateErrno(TfLiteOpaqueDelegate* delegate) { return 0; }
89 
90 /// In order for the delegate to be loaded by TfLite
91 const TfLiteOpaqueDelegatePlugin* GetArmnnDelegatePluginApi();
92 
93 using tflite::delegates::DelegatePluginInterface;
94 using TfLiteOpaqueDelegatePtr = tflite::delegates::TfLiteDelegatePtr;
95 
96 class ArmnnDelegatePlugin : public DelegatePluginInterface
97 {
98 public:
99  static std::unique_ptr<ArmnnDelegatePlugin> New(const tflite::TFLiteSettings& tfliteSettings)
100  {
101  return std::make_unique<ArmnnDelegatePlugin>(tfliteSettings);
102  }
103 
104  tflite::delegates::TfLiteDelegatePtr Create() override
105  {
106  return tflite::delegates::TfLiteDelegatePtr(TfLiteArmnnOpaqueDelegateCreate(m_delegateOptions),
108  }
109 
110  int GetDelegateErrno(TfLiteOpaqueDelegate* from_delegate) override
111  {
112  return 0;
113  }
114 
115  explicit ArmnnDelegatePlugin(const tflite::TFLiteSettings& tfliteSettings)
116  : m_delegateOptions(ParseArmNNSettings(&tfliteSettings))
117  {}
118 
119 private:
120  armnnDelegate::DelegateOptions m_delegateOptions;
121 };
122 
123 /// ArmnnSubgraph class where parsing the nodes to ArmNN format and creating the ArmNN Graph
125 {
126 public:
127  static ArmnnSubgraph* Create(TfLiteOpaqueContext* tfLiteContext,
128  const TfLiteOpaqueDelegateParams* parameters,
129  const ArmnnOpaqueDelegate* delegate);
130 
131  ~ArmnnSubgraph();
132 
133  TfLiteStatus Prepare(TfLiteOpaqueContext* tfLiteContext);
134 
135  TfLiteStatus Invoke(TfLiteOpaqueContext* tfLiteContext, TfLiteOpaqueNode* tfLiteNode);
136 
137  static TfLiteStatus VisitNode(DelegateData& delegateData,
138  TfLiteOpaqueContext* tfLiteContext,
139  TfLiteRegistrationExternal* tfLiteRegistration,
140  TfLiteOpaqueNode* tfLiteNode,
141  int nodeIndex);
142 private:
144  armnn::IRuntime* runtime,
145  std::vector<armnn::BindingPointInfo>& inputBindings,
146  std::vector<armnn::BindingPointInfo>& outputBindings)
147  : m_NetworkId(networkId)
148  , m_Runtime(runtime)
149  , m_InputBindings(inputBindings)
150  , m_OutputBindings(outputBindings)
151  {}
152  static TfLiteStatus AddInputLayer(DelegateData& delegateData,
153  TfLiteOpaqueContext* tfLiteContext,
154  const TfLiteIntArray* inputs,
155  std::vector<armnn::BindingPointInfo>& inputBindings);
156  static TfLiteStatus AddOutputLayer(DelegateData& delegateData,
157  TfLiteOpaqueContext* tfLiteContext,
158  const TfLiteIntArray* outputs,
159  std::vector<armnn::BindingPointInfo>& outputBindings);
160  /// The Network Id
161  armnn::NetworkId m_NetworkId;
162  /// ArmNN Runtime
163  armnn::IRuntime* m_Runtime;
164  /// Binding information for inputs and outputs
165  std::vector<armnn::BindingPointInfo> m_InputBindings;
166  std::vector<armnn::BindingPointInfo> m_OutputBindings;
167 };
168 
169 } // armnnOpaqueDelegate namespace
armnn::INetworkPtr
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
Definition: INetwork.hpp:339
armnnOpaqueDelegate::ParseArmNNSettings
armnnDelegate::DelegateOptions ParseArmNNSettings(const tflite::TFLiteSettings *tflite_settings)
armnnOpaqueDelegate::TfLiteOpaqueDelegatePtr
tflite::delegates::TfLiteDelegatePtr TfLiteOpaqueDelegatePtr
Definition: armnn_delegate.hpp:94
Version.hpp
armnnOpaqueDelegate::TfLiteArmnnOpaqueDelegateDelete
void TfLiteArmnnOpaqueDelegateDelete(TfLiteOpaqueDelegate *tfLiteDelegate)
armnnOpaqueDelegate::ArmnnSubgraph::VisitNode
static TfLiteStatus VisitNode(DelegateData &delegateData, TfLiteOpaqueContext *tfLiteContext, TfLiteRegistrationExternal *tfLiteRegistration, TfLiteOpaqueNode *tfLiteNode, int nodeIndex)
armnnOpaqueDelegate::ArmnnOpaqueDelegate::GetDelegateBuilder
TfLiteOpaqueDelegateBuilder * GetDelegateBuilder()
Definition: armnn_delegate.hpp:56
armnnOpaqueDelegate::ArmnnDelegatePlugin
Definition: armnn_delegate.hpp:96
armnnOpaqueDelegate::ArmnnOpaqueDelegate::IdentifyOperatorsToDelegate
TfLiteIntArray * IdentifyOperatorsToDelegate(TfLiteOpaqueContext *context)
armnnOpaqueDelegate::ArmnnSubgraph::Prepare
TfLiteStatus Prepare(TfLiteOpaqueContext *tfLiteContext)
armnnOpaqueDelegate::DoPrepare
TfLiteStatus DoPrepare(TfLiteOpaqueContext *context, TfLiteOpaqueDelegate *delegate, void *data)
armnn::IRuntime
Definition: IRuntime.hpp:75
armnnOpaqueDelegate::ArmnnDelegatePlugin::ArmnnDelegatePlugin
ArmnnDelegatePlugin(const tflite::TFLiteSettings &tfliteSettings)
Definition: armnn_delegate.hpp:115
armnnOpaqueDelegate::DelegateData::DelegateData
DelegateData(const std::vector< armnn::BackendId > &backends)
Definition: armnn_delegate.hpp:26
armnn::NetworkId
int NetworkId
Definition: IRuntime.hpp:35
armnnOpaqueDelegate::GetArmnnDelegatePluginApi
const TfLiteOpaqueDelegatePlugin * GetArmnnDelegatePluginApi()
In order for the delegate to be loaded by TfLite.
armnnOpaqueDelegate
Definition: armnn_delegate.hpp:21
armnnOpaqueDelegate::ArmnnOpaqueDelegate
ArmNN Opaque Delegate.
Definition: armnn_delegate.hpp:48
armnnOpaqueDelegate::TfLiteArmnnOpaqueDelegateCreate
TfLiteOpaqueDelegate * TfLiteArmnnOpaqueDelegateCreate(armnnDelegate::DelegateOptions options)
armnn::IRuntimePtr
std::unique_ptr< IRuntime, void(*)(IRuntime *runtime)> IRuntimePtr
Definition: IRuntime.hpp:41
armnnOpaqueDelegate::ArmnnDelegatePlugin::GetDelegateErrno
int GetDelegateErrno(TfLiteOpaqueDelegate *from_delegate) override
Definition: armnn_delegate.hpp:110
armnnOpaqueDelegate::ArmnnSubgraph::Invoke
TfLiteStatus Invoke(TfLiteOpaqueContext *tfLiteContext, TfLiteOpaqueNode *tfLiteNode)
armnnOpaqueDelegate::ArmnnSubgraph::Create
static ArmnnSubgraph * Create(TfLiteOpaqueContext *tfLiteContext, const TfLiteOpaqueDelegateParams *parameters, const ArmnnOpaqueDelegate *delegate)
armnnOpaqueDelegate::DelegateData::m_Network
armnn::INetworkPtr m_Network
Definition: armnn_delegate.hpp:32
armnn::IRuntime::CreationOptions
Definition: IRuntime.hpp:78
armnnOpaqueDelegate::DelegateData
Definition: armnn_delegate.hpp:24
armnn::IRuntime::Create
static IRuntimePtr Create(const CreationOptions &options)
Definition: Runtime.cpp:52
armnnOpaqueDelegate::DelegateData::m_OutputSlotForNode
std::vector< armnn::IOutputSlot * > m_OutputSlotForNode
Definition: armnn_delegate.hpp:33
armnnOpaqueDelegate::TfLiteArmnnDelegateOptionsDefault
::armnnDelegate::DelegateOptions TfLiteArmnnDelegateOptionsDefault()
Forward declaration for functions initializing the ArmNN Delegate.
armnnOpaqueDelegate::ArmnnSubgraph::~ArmnnSubgraph
~ArmnnSubgraph()
armnnOpaqueDelegate::ArmnnOpaqueDelegate::ArmnnOpaqueDelegate
ArmnnOpaqueDelegate(armnnDelegate::DelegateOptions options)
armnnOpaqueDelegate::ArmnnSubgraph
ArmnnSubgraph class where parsing the nodes to ArmNN format and creating the ArmNN Graph.
Definition: armnn_delegate.hpp:124
armnnOpaqueDelegate::ArmnnOpaqueDelegate::GetVersion
static const std::string GetVersion()
Retrieve version in X.Y.Z form.
armnnOpaqueDelegate::DelegateData::m_Backends
const std::vector< armnn::BackendId > m_Backends
Definition: armnn_delegate.hpp:31
armnnOpaqueDelegate::ArmnnDelegatePlugin::Create
tflite::delegates::TfLiteDelegatePtr Create() override
Definition: armnn_delegate.hpp:104
armnnOpaqueDelegate::ArmnnDelegatePlugin::New
static std::unique_ptr< ArmnnDelegatePlugin > New(const tflite::TFLiteSettings &tfliteSettings)
Definition: armnn_delegate.hpp:99