31 #include <common/include/ProfilingGuid.hpp> 33 #include <fmt/format.h> 74 return pNetworkImpl->AddComparisonLayer(comparisonDescriptor, name);
81 return pNetworkImpl->AddConcatLayer(concatDescriptor, name);
90 return pNetworkImpl->AddConvolution2dLayer(convolution2dDescriptor, weights, biases, name);
99 return pNetworkImpl->AddConvolution2dLayer(convolution2dDescriptor, weights, biases, name);
109 return pNetworkImpl->AddConvolution2dLayer(convolution2dDescriptor,
119 return pNetworkImpl->AddDepthToSpaceLayer(depthToSpaceDescriptor, name);
129 return pNetworkImpl->AddDepthwiseConvolution2dLayer(convolution2dDescriptor, weights, biases, name);
139 return pNetworkImpl->AddDepthwiseConvolution2dLayer(convolution2dDescriptor, weights, biases, name);
149 return pNetworkImpl->AddDepthwiseConvolution2dLayer(convolution2dDescriptor, weights,
165 return pNetworkImpl->AddDetectionPostProcessLayer(descriptor, anchors, name);
172 return pNetworkImpl->AddElementwiseUnaryLayer(elementwiseUnaryDescriptor, name);
179 return pNetworkImpl->AddFillLayer(fillDescriptor, name);
185 return pNetworkImpl->AddFullyConnectedLayer(fullyConnectedDescriptor, name);
193 return pNetworkImpl->AddFullyConnectedLayer(fullyConnectedDescriptor,
204 return pNetworkImpl->AddFullyConnectedLayer(fullyConnectedDescriptor, weights, biases, name);
210 return pNetworkImpl->AddPermuteLayer(permuteDescriptor, name);
216 return pNetworkImpl->AddBatchToSpaceNdLayer(batchToSpaceNdDescriptor, name);
222 return pNetworkImpl->AddPooling2dLayer(pooling2dDescriptor, name);
228 return pNetworkImpl->AddActivationLayer(activationDescriptor, name);
234 return pNetworkImpl->AddNormalizationLayer(normalizationDescriptor, name);
239 return pNetworkImpl->AddSliceLayer(sliceDescriptor, name);
244 return pNetworkImpl->AddSoftmaxLayer(softmaxDescriptor, name);
250 return pNetworkImpl->AddSplitterLayer(splitterDescriptor, name);
261 return pNetworkImpl->AddConcatLayer(mergerDescriptor, name);
286 return pNetworkImpl->AddBatchNormalizationLayer(desc, mean, variance, beta, gamma, name);
305 return pNetworkImpl->AddResizeLayer(resizeDescriptor, name);
311 return pNetworkImpl->AddResizeLayer(resizeDescriptor, name);
317 return pNetworkImpl->AddReduceLayer(reduceDescriptor, name);
323 return pNetworkImpl->AddInstanceNormalizationLayer(desc, name);
329 return pNetworkImpl->AddL2NormalizationLayer(desc, name);
335 return pNetworkImpl->AddLogSoftmaxLayer(logSoftmaxDescriptor, name);
347 return pNetworkImpl->AddReshapeLayer(reshapeDescriptor, name);
353 return pNetworkImpl->AddSpaceToBatchNdLayer(spaceToBatchNdDescriptor, name);
359 return pNetworkImpl->AddSpaceToDepthLayer(spaceToDepthDescriptor, name);
375 return pNetworkImpl->AddLstmLayer(descriptor, params, name);
395 return pNetworkImpl->AddMeanLayer(meanDescriptor, name);
412 return pNetworkImpl->AddStridedSliceLayer(stridedSliceDescriptor, name);
438 return pNetworkImpl->AddGatherLayer(gatherDescriptor, name);
462 return pNetworkImpl->AddTransposeConvolution2dLayer(descriptor, weights, biases, name);
468 return pNetworkImpl->AddTransposeLayer(transposeDescriptor, name);
491 return pNetworkImpl->AddQuantizedLstmLayer(params, name);
498 return pNetworkImpl->AddQLstmLayer(descriptor, params, name);
504 return pNetworkImpl->AddLogicalBinaryLayer(descriptor, name);
512 return pNetworkImpl->AddUnidirectionalSequenceLstmLayer(descriptor, params, name);
527 return new INetwork(networkOptions);
541 : pOptimizedNetworkImpl(new
OptimizedNetworkImpl(*other.pOptimizedNetworkImpl.get(), modelOptions)) {}
582 return m_Graph->SerializeToDot(stream);
586 Optional<std::vector<std::string>&> errorMessages)
588 std::stringstream fullErrorMessage;
589 fullErrorMessage <<
"ERROR: " << errorMessage;
593 errorMessages.value().push_back(fullErrorMessage.str());
598 Optional<std::vector<std::string>&> warningMessages)
600 std::stringstream fullWarningMessage;
601 fullWarningMessage <<
"WARNING: " << warningMessage;
605 warningMessages.value().push_back(fullWarningMessage.str());
612 Optional<std::vector<std::string>&> errMessages)
614 std::stringstream failureMsg;
626 bool noErrors =
true;
628 for (
unsigned int i = 0; i < numOutputs; i++) {
634 std::stringstream ss;
636 <<
" (" << layer->
GetNameStr() <<
") is of type" 637 <<
" Quantized 8 bit but its scale parameter has not been set";
645 std::stringstream ss;
646 ss <<
"Quantization parameters for Softmax layer (Scale: " <<
648 ") are incorrect and have been updated to Scale: 0.00390625 and Offset: 0";
659 template <
typename LayerT>
662 LayerT* layer = PolymorphicDowncast<LayerT*>(l);
673 layer->m_Weight->template GetConstTensor<armnn::BFloat16>(), info.
GetNumElements(), newValues.data());
689 const std::vector<BackendId>& availablePreferredBackends,
690 std::string& reasonIfUnsupported,
691 Optional<std::vector<std::string>&> errMessages)
696 auto ReturnError = [&](
const Layer* layer)
712 auto ConstantLayerFromFp16ToFp32 = [](
Layer& layer)
716 ConstantLayer* constantLayer = PolymorphicDowncast<ConstantLayer*>(&layer);
722 std::vector<float> newValues(info.GetNumElements());
726 info.GetNumElements(),
739 bool checkType =
false;
743 auto connectedOutputSlot = inputSlot.GetConnectedOutputSlot();
746 if (connectedOutputSlot->GetNumConnections() == 1)
749 ConstantLayerFromFp16ToFp32(connectedOutputSlot->GetOwningLayer());
755 std::vector<ConvertFp16ToFp32Layer*> convertFp16ToFp32Layers;
758 convertFp16ToFp32Layers =
763 std::vector<ConvertFp32ToFp16Layer*> convertFp32ToFp16Layers;
766 convertFp32ToFp16Layers =
771 auto AssignFirstSupportedBackend = [&](
Layer* layer,
BackendId preferredBackend)
773 bool supportedBackendFound =
false;
774 std::string reasonIfUnsupported;
780 reasonIfUnsupported))
782 supportedBackendFound =
true;
786 for (
const auto& backend : availablePreferredBackends)
789 if (backend == preferredBackend)
797 reasonIfUnsupported))
799 supportedBackendFound =
true;
805 return supportedBackendFound;
810 if (!AssignFirstSupportedBackend(convertLayer, backend))
812 return ReturnError(convertLayer);
818 if (!AssignFirstSupportedBackend(convertLayer, backend))
820 return ReturnError(convertLayer);
834 std::vector<ConvertBf16ToFp32Layer*> convertBf16ToFp32Layers;
837 convertBf16ToFp32Layers =
841 ConvertBf16ToFp32Weight<Convolution2dLayer>(layer);
845 ConvertBf16ToFp32Weight<FullyConnectedLayer>(layer);
850 std::vector<ConvertFp32ToBf16Layer*> convertFp32ToBf16Layers;
853 convertFp32ToBf16Layers =
858 auto AssignFirstSupportedBackend = [&](
Layer* layer,
BackendId preferredBackend)
860 bool supportedBackendFound =
false;
861 std::string reasonIfUnsupported;
867 reasonIfUnsupported))
869 supportedBackendFound =
true;
873 for (
const auto& backend : availablePreferredBackends)
876 if (backend == preferredBackend)
884 reasonIfUnsupported))
886 supportedBackendFound =
true;
892 return supportedBackendFound;
897 if (!AssignFirstSupportedBackend(convertLayer, backend))
899 return ReturnError(convertLayer);
905 if (!AssignFirstSupportedBackend(convertLayer, backend))
907 return ReturnError(convertLayer);
915 std::stringstream warningMsg;
917 <<
" is not supported on requested backend " << layer->
GetBackendId().
Get()
920 <<
" (reason: " << reasonIfUnsupported
921 <<
"), falling back to the next backend.";
937 Optional<std::vector<std::string>&> errMessages)
942 auto ReturnError = [&](
const Layer* layer)
949 if (availablePreferredBackends.empty())
951 std::stringstream failureMsg;
952 failureMsg <<
"No preferred backends are available";
959 for (
auto it = firstLayer; it != lastLayer; ++it)
964 layer->GetInputSlot(0).GetConnectedOutputSlot()->GetTensorInfo().GetDataType();
966 layer->GetOutputSlot(0).GetTensorInfo().GetDataType();
968 std::string reasonIfUnsupported;
978 if (layer->GetBackendHint().has_value() &&
983 layer->GetBackendHint().value(),
986 availablePreferredBackends,
996 for (
const auto& backend : availablePreferredBackends)
998 if (layer->GetBackendHint().has_value() &&
999 layer->GetBackendHint().value() == backend)
1010 availablePreferredBackends,
1011 reasonIfUnsupported,
1046 layer->SetBackendId(cpuBackendId);
1051 return ReturnError(layer);
1062 Optional<std::vector<std::string>&> errMessages)
1080 auto backendFactory = backendRegistry.GetFactory(selectedBackend);
1081 auto backendObjPtr = backendFactory();
1084 backendObjPtr->RegisterTensorHandleFactories(handleFactoryRegistry);
1086 backends[backendObjPtr->GetId()] = std::move(backendObjPtr);
1096 Optional<std::vector<std::string>&> errMessages)
1108 auto backendObjPtr = backends.find(selectedBackend)->second.get();
1115 [&backendObjPtr](
const Layer& layer)
1121 if (subgraphs.empty())
1128 for (
auto& subgraph : subgraphs)
1131 OptimizationViews optimizationViews = backendObjPtr->OptimizeSubgraphView(*subgraph, modelOptions);
1138 SubgraphView& replacementSubgraph = substitution.m_ReplacementSubgraph;
1139 SubgraphView& substitutableSubgraph = substitution.m_SubstitutableSubgraph;
1143 std::for_each(replacementSubgraph.
begin(), replacementSubgraph.
end(), [&selectedBackend](
Layer* l)
1146 l->SetBackendId(selectedBackend);
1152 std::stringstream warningMsg;
1153 warningMsg <<
"Some sub-graph(s) failed to optimized on " << backendObjPtr->GetId() <<
" backend.";
1158 if (!backendObjPtr->GetId().IsCpuRef())
1161 settingsCopy.m_IgnoredBackends.insert(backendObjPtr->GetId());
1168 std::stringstream subgraphMsg;
1169 subgraphMsg <<
"Re-assigning backends to " << failedSubgraph.GetLayers().size()
1170 <<
" layers inside sub-graph " << count++;
1177 if (reassignmentResult.m_Error)
1200 if (srcFactory && dstFactory &&
1226 if (frmBackend == backends.end() ||
1227 !frmBackend->second->SupportsTensorAllocatorAPI())
1234 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
1241 const Layer& connectedLayer = connection->GetOwningLayer();
1243 auto toBackend = backends.find(connectedLayer.
GetBackendId());
1244 ARMNN_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
1246 if (!toBackend->second.get()->SupportsTensorAllocatorAPI())
1252 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
1253 for (
auto&& dst : dstPrefs)
1267 auto it = factoryScores.find(dst);
1268 if (it == factoryScores.end())
1271 factoryScores[dst] = 0;
1280 factoryScores[dst]++;
1283 if (factoryScores[dst] > topScore)
1285 topScore = factoryScores[dst];
1314 if (frmBackend == backends.end() ||
1315 !frmBackend->second->SupportsTensorAllocatorAPI())
1320 bool outputConnection =
false;
1323 const Layer& connectedLayer = connection->GetOwningLayer();
1326 outputConnection =
true;
1334 std::map<ITensorHandleFactory::FactoryId, int> factoryScores;
1335 for (
auto&& pref : srcPrefs)
1340 if (outputConnection)
1343 bool fallbackConnection =
false;
1346 if (inputSlot.GetConnectedOutputSlot()->GetOwningLayer().GetBackendId() != layer.
GetBackendId())
1348 fallbackConnection =
true;
1351 if (fallbackConnection)
1355 if (!factoryCap.empty())
1365 if (!outputConnection)
1369 if (!factoryCap.empty())
1388 auto it = factoryScores.find(pref);
1389 if (it == factoryScores.end())
1392 factoryScores[pref] = 0;
1399 const Layer& connectedLayer = connection->GetOwningLayer();
1401 auto toBackend = backends.find(connectedLayer.
GetBackendId());
1402 ARMNN_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
1404 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
1405 for (
auto&& src : srcPrefs)
1407 if (factoryScores.find(src) == factoryScores.end())
1412 for (
auto&& dst : dstPrefs)
1417 factoryScores[src]++;
1425 int minScore = std::numeric_limits<int>::max();
1426 for (
auto it : factoryScores)
1428 minScore = std::min(minScore, it.second);
1432 std::vector<ITensorHandleFactory::FactoryId> optimalFactories;
1433 for (
auto it : factoryScores)
1435 if (it.second == minScore)
1437 optimalFactories.push_back(it.first);
1442 for (
auto&& srcPref : srcPrefs)
1444 for (
auto&& comp : optimalFactories)
1446 if (comp == srcPref)
1459 const Layer& connectedLayer,
1463 auto toBackend = backends.find(connectedLayer.
GetBackendId());
1464 ARMNN_ASSERT_MSG(toBackend != backends.end(),
"Backend id not found for the connected layer");
1466 auto dstPrefs = toBackend->second.get()->GetHandleFactoryPreferences();
1489 for (
auto&& pref : dstPrefs)
1491 if (pref == srcFactoryId)
1501 for (
auto&& pref : dstPrefs)
1520 if (srcCapability.empty() && dstCapability.empty() && srcFallback.empty() && dstFallback.empty())
1531 for (
auto&& pref : dstPrefs)
1549 Optional<std::vector<std::string>&> errMessages)
1553 optGraph.
ForEachLayer([&backends, ®istry, &result, &errMessages, importEnabled](
Layer* layer)
1584 unsigned int connectionIdx = 0;
1587 const Layer& connectedLayer = connection->GetOwningLayer();
1590 registry, importEnabled);
1597 errMessages.value().emplace_back(
"Could not find valid strategy required for compatibility" 1598 " between backends.");
1614 const std::vector<BackendId>& backendPreferences,
1617 Optional<std::vector<std::string>&> messages)
1619 if (backendPreferences.empty())
1629 std::unique_ptr<Graph> graph = std::make_unique<Graph>(inNetwork.
pNetworkImpl->GetGraph());
1646 using namespace optimizations;
1696 std::stringstream failureMsg;
1697 failureMsg <<
"None of the preferred backends " << backendPreferences
1715 if (assignBackendsResult.
m_Error)
1730 if (backendOptimizationResult.
m_Error)
1746 tensorHandleFactoryRegistry,
1766 auto backendPtr = factoryFun();
1770 auto backendSpecificOptimizations = backendPtr->GetOptimizations();
1773 if (!backendSpecificOptimizations.empty())
1781 bool NetworkImpl::GetShapeInferenceMethod()
1783 if (m_NetworkOptions.size() > 0 && m_NetworkOptions[0].GetBackendId().Get() ==
"ShapeInferenceMethod")
1785 return m_NetworkOptions[0].GetOption(0).GetValue().AsBool();
1791 : m_NetworkOptions(networkOptions),
1792 m_Graph(
std::make_unique<
Graph>(GetShapeInferenceMethod()))
1807 return m_Graph->AddLayer<
InputLayer>(id, name);
1818 return m_Graph->AddLayer<
CastLayer>(name);
1824 return m_Graph->AddLayer<
ComparisonLayer>(comparisonDescriptor, name);
1836 return m_Graph->AddLayer<
FillLayer>(fillDescriptor, name);
1852 unsigned int numInputs = fullyConnectedDescriptor.
GetNumInputs();
1861 weightsInfo.SetConstant();
1877 biasInfo.SetConstant();
1895 if ( fullyConnectedDescriptor.m_BiasEnabled && numInputs == 3 )
1903 else if ( !fullyConnectedDescriptor.m_BiasEnabled && numInputs == 2 )
1906 layer->m_Bias =
nullptr;
1911 "AddFullyConnectedLayer: Value mismatch. When bias is enabled in the " 1912 "descriptor the number of inputs is expected to be 3 otherwise 2. " 1913 "BiasEnabled={}, numInputs={}",
1914 fullyConnectedDescriptor.m_BiasEnabled,
1933 return m_Graph->AddLayer<
ConcatLayer>(concatDescriptor, name);
1946 const auto layer = m_Graph->AddLayer<
Convolution2dLayer>(convolution2dDescriptor, name);
1948 layer->
m_Weight = std::make_shared<ScopedTensorHandle>(weights);
1950 if (convolution2dDescriptor.m_BiasEnabled)
1952 layer->m_Bias = std::make_shared<ScopedTensorHandle>(biases.
value());
1963 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1971 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
1980 return AddConvolution2dLayerImpl(convolution2dDescriptor, weights, optionalBiases, name);
1996 layer->
m_Weight = std::make_shared<ScopedTensorHandle>(weights);
1998 if (convolution2dDescriptor.m_BiasEnabled)
2000 layer->m_Bias = std::make_shared<ScopedTensorHandle>(biases.
value());
2018 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
2027 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
2037 return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, optionalBiases, name);
2045 layer->
m_Anchors = std::make_shared<ScopedTensorHandle>(anchors);
2053 return m_Graph->AddLayer<
PermuteLayer>(permuteDescriptor, name);
2059 return m_Graph->AddLayer<
Pooling2dLayer>(pooling2dDescriptor, name);
2065 return m_Graph->AddLayer<
ActivationLayer>(activationDescriptor, name);
2071 return m_Graph->AddLayer<
ArgMinMaxLayer>(argMinMaxDescriptor, name);
2075 normalizationDescriptor,
2083 return m_Graph->AddLayer<
SliceLayer>(sliceDescriptor, name);
2089 return m_Graph->AddLayer<
SoftmaxLayer>(softmaxDescriptor, name);
2095 return m_Graph->AddLayer<
SplitterLayer>(splitterDescriptor, name);
2143 layer->
m_Mean = std::make_shared<ScopedTensorHandle>(mean);
2144 layer->m_Variance = std::make_shared<ScopedTensorHandle>(variance);
2145 layer->m_Beta = std::make_shared<ScopedTensorHandle>(beta);
2146 layer->m_Gamma = std::make_shared<ScopedTensorHandle>(gamma);
2153 return m_Graph->AddLayer<
RankLayer>(name);
2159 return m_Graph->AddLayer<
ReduceLayer>(reduceDescriptor, name);
2173 return m_Graph->AddLayer<
ResizeLayer>(resizeDescriptor, name);
2178 return m_Graph->AddLayer<
ResizeLayer>(resizeDescriptor, name);
2208 layer->
m_LayerOutput = std::make_shared<ScopedTensorHandle>(input);
2216 return m_Graph->AddLayer<
ReshapeLayer>(reshapeDescriptor, name);
2240 const auto layer = m_Graph->AddLayer<
LstmLayer>(descriptor, name);
2245 layer->m_BasicParameters.m_InputToCellWeights =
2247 layer->m_BasicParameters.m_InputToOutputWeights =
2249 layer->m_BasicParameters.m_RecurrentToForgetWeights =
2251 layer->m_BasicParameters.m_RecurrentToCellWeights =
2253 layer->m_BasicParameters.m_RecurrentToOutputWeights =
2255 layer->m_BasicParameters.m_ForgetGateBias =
2257 layer->m_BasicParameters.m_CellBias =
2258 std::make_shared<ScopedTensorHandle>(*(params.
m_CellBias));
2259 layer->m_BasicParameters.m_OutputGateBias =
2263 if(!descriptor.m_CifgEnabled)
2268 "when CIFG is disabled.");
2273 "AddLstmLayer: Recurrent To Input Weights cannot be NULL " 2274 "when CIFG is disabled.");
2279 "when CIFG is disabled.");
2281 layer->m_CifgParameters.m_InputToInputWeights =
2283 layer->m_CifgParameters.m_RecurrentToInputWeights =
2285 layer->m_CifgParameters.m_InputGateBias =
2290 if(descriptor.m_ProjectionEnabled)
2295 "when projection is enabled.");
2297 layer->m_ProjectionParameters.m_ProjectionWeights =
2301 layer->m_ProjectionParameters.m_ProjectionBias =
2307 if(descriptor.m_PeepholeEnabled)
2309 if(!descriptor.m_CifgEnabled)
2314 "when Peephole is enabled and CIFG disabled.");
2317 layer->m_PeepholeParameters.m_CellToInputWeights =
2324 "when Peephole is enabled.");
2329 "when Peephole is enabled.");
2332 layer->m_PeepholeParameters.m_CellToForgetWeights =
2334 layer->m_PeepholeParameters.m_CellToOutputWeights =
2339 if(descriptor.m_LayerNormEnabled)
2341 if(!descriptor.m_CifgEnabled)
2346 "when layer normalization is enabled and CIFG disabled.");
2348 layer->m_LayerNormParameters.m_InputLayerNormWeights =
2355 "when layer normalization is enabled.");
2360 "when layer normalization is enabled.");
2365 "when layer normalization is enabled.");
2367 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
2369 layer->m_LayerNormParameters.m_CellLayerNormWeights =
2371 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
2389 return m_Graph->AddLayer<
MeanLayer>(meanDescriptor,name);
2394 return m_Graph->AddLayer<
PadLayer>(padDescriptor,name);
2437 return m_Graph->AddLayer<
GatherLayer>(gatherDescriptor, name);
2467 layer->
m_Weight = std::make_shared<ScopedTensorHandle>(weights);
2469 if (descriptor.m_BiasEnabled)
2471 layer->m_Bias = std::make_shared<ScopedTensorHandle>(biases.
value());
2480 return m_Graph->AddLayer<
TransposeLayer>(transposeDescriptor, name);
2486 return m_Graph->AddLayer<
StackLayer>(stackDescriptor, name);
2504 layer->m_QuantizedLstmParameters.m_InputToForgetWeights =
2506 layer->m_QuantizedLstmParameters.m_InputToCellWeights =
2508 layer->m_QuantizedLstmParameters.m_InputToOutputWeights =
2512 layer->m_QuantizedLstmParameters.m_RecurrentToInputWeights =
2514 layer->m_QuantizedLstmParameters.m_RecurrentToForgetWeights =
2516 layer->m_QuantizedLstmParameters.m_RecurrentToCellWeights =
2518 layer->m_QuantizedLstmParameters.m_RecurrentToOutputWeights =
2522 layer->m_QuantizedLstmParameters.m_InputGateBias =
2524 layer->m_QuantizedLstmParameters.m_ForgetGateBias =
2526 layer->m_QuantizedLstmParameters.m_CellBias =
2527 std::make_shared<ScopedTensorHandle>(params.
GetCellBias());
2528 layer->m_QuantizedLstmParameters.m_OutputGateBias =
2538 const auto layer = m_Graph->AddLayer<
QLstmLayer>(descriptor, name);
2543 layer->m_BasicParameters.m_InputToCellWeights =
2545 layer->m_BasicParameters.m_InputToOutputWeights =
2547 layer->m_BasicParameters.m_RecurrentToForgetWeights =
2549 layer->m_BasicParameters.m_RecurrentToCellWeights =
2551 layer->m_BasicParameters.m_RecurrentToOutputWeights =
2553 layer->m_BasicParameters.m_ForgetGateBias =
2555 layer->m_BasicParameters.m_CellBias =
2556 std::make_shared<ScopedTensorHandle>(*(params.
m_CellBias));
2557 layer->m_BasicParameters.m_OutputGateBias =
2561 if(!descriptor.m_CifgEnabled)
2571 "AddQLstmLayer: Recurrent To Input Weights cannot be NULL");
2579 layer->m_CifgParameters.m_InputToInputWeights =
2581 layer->m_CifgParameters.m_RecurrentToInputWeights =
2583 layer->m_CifgParameters.m_InputGateBias =
2588 if(descriptor.m_ProjectionEnabled)
2595 layer->m_ProjectionParameters.m_ProjectionWeights =
2601 layer->m_ProjectionParameters.m_ProjectionBias =
2608 if(descriptor.m_PeepholeEnabled)
2620 if(!descriptor.m_CifgEnabled)
2627 layer->m_PeepholeParameters.m_CellToInputWeights =
2631 layer->m_PeepholeParameters.m_CellToForgetWeights =
2633 layer->m_PeepholeParameters.m_CellToOutputWeights =
2638 if(descriptor.m_LayerNormEnabled)
2655 if(!descriptor.m_CifgEnabled)
2662 layer->m_LayerNormParameters.m_InputLayerNormWeights =
2666 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
2668 layer->m_LayerNormParameters.m_CellLayerNormWeights =
2670 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
2692 layer->m_BasicParameters.m_InputToCellWeights =
2694 layer->m_BasicParameters.m_InputToOutputWeights =
2696 layer->m_BasicParameters.m_RecurrentToForgetWeights =
2698 layer->m_BasicParameters.m_RecurrentToCellWeights =
2700 layer->m_BasicParameters.m_RecurrentToOutputWeights =
2702 layer->m_BasicParameters.m_ForgetGateBias =
2704 layer->m_BasicParameters.m_CellBias =
2705 std::make_shared<ScopedTensorHandle>(*(params.
m_CellBias));
2706 layer->m_BasicParameters.m_OutputGateBias =
2710 if(!descriptor.m_CifgEnabled)
2715 "when CIFG is disabled.");
2720 "AddUnidirectionalSequenceLstmLayer: Recurrent To Input Weights cannot be NULL " 2721 "when CIFG is disabled.");
2726 "when CIFG is disabled.");
2728 layer->m_CifgParameters.m_InputToInputWeights =
2730 layer->m_CifgParameters.m_RecurrentToInputWeights =
2732 layer->m_CifgParameters.m_InputGateBias =
2737 if(descriptor.m_ProjectionEnabled)
2742 "when projection is enabled.");
2744 layer->m_ProjectionParameters.m_ProjectionWeights =
2748 layer->m_ProjectionParameters.m_ProjectionBias =
2754 if(descriptor.m_PeepholeEnabled)
2756 if(!descriptor.m_CifgEnabled)
2761 "cannot be NULL when Peephole is enabled and CIFG disabled.");
2764 layer->m_PeepholeParameters.m_CellToInputWeights =
2771 "when Peephole is enabled.");
2776 "when Peephole is enabled.");
2779 layer->m_PeepholeParameters.m_CellToForgetWeights =
2781 layer->m_PeepholeParameters.m_CellToOutputWeights =
2786 if(descriptor.m_LayerNormEnabled)
2788 if(!descriptor.m_CifgEnabled)
2793 "cannot be NULL when layer normalization is enabled and CIFG disabled.");
2795 layer->m_LayerNormParameters.m_InputLayerNormWeights =
2802 "cannot be NULL when layer normalization is enabled.");
2807 "cannot be NULL when layer normalization is enabled.");
2812 "cannot be NULL when layer normalization is enabled.");
2814 layer->m_LayerNormParameters.m_ForgetLayerNormWeights =
2816 layer->m_LayerNormParameters.m_CellLayerNormWeights =
2818 layer->m_LayerNormParameters.m_OutputLayerNormWeights =
2828 layer->Accept(visitor);
2836 layer->ExecuteStrategy(strategy);
2841 : m_Graph(new
Graph(*other.m_Graph.get()))
2843 , m_ModelOptions(modelOptions)
2853 : m_Graph(
std::move(graph)), m_Guid(profiling::
ProfilingService::GetNextGuid()), m_ModelOptions(modelOptions)
A layer that the constant data can be bound to.
OptimizeForConnection< Layer, PermuteLayer, SquashEqualSiblingsImpl< PermuteLayer > > SquashEqualPermuteSiblings
void ReportError(const std::string &errorMessage, Optional< std::vector< std::string > &> errorMessages)
Iterator begin()
Returns iterator pointing to the beginning of the list. Lowercase for range-based for loops...
IConnectableLayer * AddSubtractionLayer(const char *name=nullptr)
Adds a subtraction layer to the network.
bool m_BiasEnabled
Enable/disable bias.
IConnectableLayer * AddReduceLayer(const ReduceDescriptor &reduceDescriptor, const char *name=nullptr)
ModelOptions m_ModelOptions
IConnectableLayer * AddRsqrtLayer(const char *name=nullptr)
IConnectableLayer * AddActivationLayer(const ActivationDescriptor &activationDescriptor, const char *name=nullptr)
Adds an activation layer to the network.
bool m_HalfPixelCenters
Half Pixel Centers.
bool m_AlignCorners
Aligned corners.
This layer represents a minimum operation.
static const FactoryId DeferredFactoryId
Use the workload factory to create the tensor handle.
This layer represents a split operation.
OptimizationResult AssignBackends(OptimizedNetworkImpl *optNetObjPtr, BackendSettings &backendSettings, Graph::Iterator &firstLayer, Graph::Iterator &lastLayer, Optional< std::vector< std::string > &> errMessages)
IConnectableLayer * AddCastLayer(const char *name=nullptr)
Adds a cast layer to the network.
LstmBasicParameters m_BasicParameters
const std::vector< InputSlot * > & GetConnections() const
FactoryFunction GetFactory(const BackendId &id) const
This layer represents a batch normalization operation.
static void Destroy(INetwork *network)
A ViewsDescriptor for the SplitterLayer.
Interface for a layer that is connectable to other layers via InputSlots and OutputSlots.
OptimizeForConnection< PermuteLayer, PermuteLayer, OptimizeInversePermutesImpl< PermuteLayer > > OptimizeInversePermutes
std::vector< ConvertFp32ToFp16Layer * > InsertConvertFp32ToFp16LayersAfter(Graph &graph, Layer &layer)
bool m_BiasEnabled
Enable/disable bias.
void ExecuteStrategy(IStrategy &strategy) const
void SetEdgeStrategy(unsigned int connectionIndex, EdgeStrategy strategy)
QuantizedLstmParameters m_QuantizedLstmParameters
This layer represents a 2D transpose convolution operation.
virtual Status PrintGraph()
No strategy has been defined. Used internally to verify integrity of optimizations.
std::vector< ConvertFp16ToFp32Layer * > InsertConvertFp16ToFp32LayersBefore(Graph &graph, Layer &layer, bool expectCorrectInputType)
IConnectableLayer * AddResizeLayer(const ResizeDescriptor &resizeDescriptor, const char *name=nullptr)
ShapeInferenceMethod m_shapeInferenceMethod
virtual ~OptimizedNetworkImpl()
A TransposeConvolution2dDescriptor for the TransposeConvolution2dLayer.
const TensorShape & GetShape() const
CPU Execution: Reference C++ kernels.
IConnectableLayer * AddTransposeConvolution2dLayer(const TransposeConvolution2dDescriptor &descriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr)
Adds a 2D transpose convolution layer to the network.
IConnectableLayer * AddQuantizeLayer(const char *name=nullptr)
OptimizeForExclusiveConnection< PadLayer, Convolution2dLayer, pad_fold::FoldPadIntoConvolution2dImpl > FoldPadIntoConvolution2d
Optimizer::Optimizations MakeOptimizations(Args &&... args)
IConnectableLayer * AddQLstmLayer(const QLstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr)
Add a QLstm layer to the network.
IConnectableLayer * AddRankLayer(const char *name=nullptr)
A ReshapeDescriptor for the ReshapeLayer.
OptimizeForConnection< TransposeLayer, TransposeLayer, OptimizeInversePermutesImpl< TransposeLayer > > OptimizeInverseTransposes
IConnectableLayer * AddAdditionLayer(const char *name=nullptr)
OptimizeForExclusiveConnection< PadLayer, DepthwiseConvolution2dLayer, pad_fold::FoldPadIntoDepthwiseConvolution2dImpl > FoldPadIntoDepthwiseConvolution2d
OptimizeForConnection< TransposeLayer, BatchToSpaceNdLayer, PermuteAndBatchToSpaceAsDepthToSpaceImpl< TransposeLayer > > TransposeAndBatchToSpaceAsDepthToSpace
OptimizeForExclusiveConnection< DepthwiseConvolution2dLayer, BatchNormalizationLayer, FuseBatchNorm< DepthwiseConvolution2dLayer, armnn::DataType::Float32 > > FuseBatchNormIntoDepthwiseConvolution2DFloat32
IConnectableLayer * AddConstantLayer(const ConstTensor &input, const char *name=nullptr)
Adds a layer with no inputs and a single output, which always corresponds to the passed in constant t...
#define ARMNN_NO_DEPRECATE_WARN_BEGIN
IConnectableLayer * AddDepthwiseConvolution2dLayer(const DepthwiseConvolution2dDescriptor &convolution2dDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr)
Adds a 2D depthwise convolution layer to the network.
ITensorHandleFactory::FactoryId CalculateSlotOptionForOutput(BackendsMap &backends, OutputSlot &slot, TensorHandleFactoryRegistry ®istry)
A ComparisonDescriptor for the ComparisonLayer.
IConnectableLayer * AddAbsLayer(const char *name=nullptr)
This layer represents a depthwise convolution 2d operation.
static void ConvertBFloat16ToFloat32(const void *srcBFloat16Buffer, size_t numElements, float *dstFloat32Buffer)
bool RequiresCopy(ITensorHandleFactory::FactoryId src, ITensorHandleFactory::FactoryId dst, TensorHandleFactoryRegistry ®istry)
IConnectableLayer * AddPooling2dLayer(const Pooling2dDescriptor &pooling2dDescriptor, const char *name=nullptr)
NetworkImpl(NetworkOptions networkOptions={})
IConnectableLayer * AddStackLayer(const StackDescriptor &descriptor, const char *name=nullptr)
Adds a stack layer to the network.
std::shared_ptr< ConstTensorHandle > m_LayerOutput
uint32_t m_TargetWidth
Target width value.
IConnectableLayer * AddShapeLayer(const char *name=nullptr)
Adds a shape layer to the network.
DataLayout m_DataLayout
The data layout to be used (NCHW, NHWC).
std::vector< BackendOptions > ModelOptions
void Accept(ILayerVisitor &visitor) const
IConnectableLayer * AddMergeLayer(const char *name=nullptr)
IConnectableLayer * AddRsqrtLayer(const char *name=nullptr)
Add Reciprocal of square root layer to the network.
A Convolution2dDescriptor for the Convolution2dLayer.
Layer & GetOwningLayer() const
Source backends tensor data can be exported to destination backend tensor without copy...
IConnectableLayer * AddQuantizeLayer(const char *name=nullptr)
Add a quantize layer to the network.
This layer converts data type Float 16 to Float 32.
int Connect(InputSlot &destination)
IConnectableLayer * AddMinimumLayer(const char *name=nullptr)
Add a Minimum layer to the network.
IConnectableLayer * AddQuantizedLstmLayer(const QuantizedLstmInputParams ¶ms, const char *name=nullptr)
IConnectableLayer * AddL2NormalizationLayer(const L2NormalizationDescriptor &desc, const char *name=nullptr)
Adds an L2 normalization layer to the network.
bool m_BiasEnabled
Enable/disable bias.
IConnectableLayer * AddConstantLayer(const ConstTensor &input, const char *name=nullptr)
IConnectableLayer * AddElementwiseUnaryLayer(const ElementwiseUnaryDescriptor &elementwiseUnaryDescriptor, const char *name=nullptr)
IConnectableLayer * AddMergeLayer(const char *name=nullptr)
Adds a merge layer to the network.
IConnectableLayer * AddPermuteLayer(const PermuteDescriptor &permuteDescriptor, const char *name=nullptr)
Adds a permute layer to the network.
ResizeMethod m_Method
The Interpolation method to use (Bilinear, NearestNeighbor).
IConnectableLayer * AddNormalizationLayer(const NormalizationDescriptor &normalizationDescriptor, const char *name=nullptr)
IConnectableLayer * AddSliceLayer(const SliceDescriptor &sliceDescriptor, const char *name=nullptr)
Adds a slice layer to the network.
static void Pass(Graph &graph, const Optimizations &optimizations)
OptimizeForExclusiveConnection< DepthwiseConvolution2dLayer, BatchNormalizationLayer, FuseBatchNorm< DepthwiseConvolution2dLayer, armnn::DataType::Float16 > > FuseBatchNormIntoDepthwiseConvolution2DFloat16
IConnectableLayer * AddFloorLayer(const char *name=nullptr)
Adds a floor layer to the network.
IConnectableLayer * AddPooling2dLayer(const Pooling2dDescriptor &pooling2dDescriptor, const char *name=nullptr)
Adds a pooling layer to the network.
This layer represents a SpaceToDepth operation.
IConnectableLayer * AddMeanLayer(const MeanDescriptor &meanDescriptor, const char *name=nullptr)
This layer represents a reshape operation.
void ExecuteStrategy(IStrategy &strategy) const
OptimizeForExclusiveConnection< Convolution2dLayer, BatchNormalizationLayer, FuseBatchNorm< Convolution2dLayer, armnn::DataType::Float16 > > FuseBatchNormIntoConvolution2DFloat16
OptimizeForExclusiveConnection< Convolution2dLayer, BatchNormalizationLayer, FuseBatchNorm< Convolution2dLayer, armnn::DataType::Float32 > > FuseBatchNormIntoConvolution2DFloat32
#define ARMNN_LOG(severity)
IConnectableLayer * AddMinimumLayer(const char *name=nullptr)
std::shared_ptr< ConstTensorHandle > m_Weight
A unique pointer to store Weight values.
IConnectableLayer * AddFillLayer(const FillDescriptor &fillDescriptor, const char *name=nullptr)
Add an Fill layer to the network.
Main network class which provides the interface for building up a neural network. ...
This layer represents an activation operation with the specified activation function.
IConnectableLayer * AddSpaceToDepthLayer(const SpaceToDepthDescriptor &spaceToDepthDescriptor, const char *name=nullptr)
BackendRegistry & BackendRegistryInstance()
This layer converts data type BFloat16 to Float32.
LayerT * ConvertBf16ToFp32Weight(Layer *l)
std::vector< BackendOptions > NetworkOptions
std::shared_ptr< ConstTensorHandle > m_Mean
A unique pointer to store Mean values.
A LogicalBinaryDescriptor for the LogicalBinaryLayer.
IConnectableLayer * AddConvolution2dLayer(const Convolution2dDescriptor &convolution2dDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr)
Adds a 2D convolution layer to the network.
This layer represents an unknown operation in the input graph.
OptimizeForConnection< Layer, ReshapeLayer, SquashEqualSiblingsImpl< ReshapeLayer > > SquashEqualReshapeSiblings
IConnectableLayer * AddLogSoftmaxLayer(const LogSoftmaxDescriptor &logSoftmaxDescriptor, const char *name=nullptr)
Adds a log softmax layer to the network.
IConnectableLayer * AddResizeLayer(const ResizeDescriptor &resizeDescriptor, const char *name=nullptr)
Adds a resize layer to the network.
IConnectableLayer * AddActivationLayer(const ActivationDescriptor &activationDescriptor, const char *name=nullptr)
This layer represents a detection postprocess operator.
BackendIdSet m_SupportedBackends
OptimizeForConnection< Layer, TransposeLayer, MoveTransposeUpImpl > MoveTransposeUp
IConnectableLayer * AddBatchToSpaceNdLayer(const BatchToSpaceNdDescriptor &batchToSpaceNdDescriptor, const char *name=nullptr)
Adds a batch to space ND layer to the network.
OptimizationResult ReturnWithError(OptimizationResult res, const Layer *layer, const BackendSettings &backendSettings, Optional< std::vector< std::string > &> errMessages)
Copyright (c) 2021 ARM Limited and Contributors.
This layer represents a pad operation.
This layer represents a LSTM operation.
void IgnoreUnused(Ts &&...)
IConnectableLayer * AddDivisionLayer(const char *name=nullptr)
Adds a division layer to the network.
void SetBackendId(const BackendId &id)
const std::vector< InputSlot > & GetInputSlots() const
bool IsBackendSupported(const BackendId &backend) const
LayerList::const_iterator Iterator
This layer represents a reduction operation.
IConnectableLayer * AddDepthwiseConvolution2dLayer(const DepthwiseConvolution2dDescriptor &convolution2dDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr)
IConnectableLayer * AddLogSoftmaxLayer(const LogSoftmaxDescriptor &logSoftmaxDescriptor, const char *name=nullptr)
A SpaceToDepthDescriptor for the SpaceToDepthLayer.
This layer represents a permutation operation.
unsigned int GetNumOutputSlots() const override
Returns the number of connectable output slots.
This layer represents a SpaceToBatchNd operation.
A BatchToSpaceNdDescriptor for the BatchToSpaceNdLayer.
OptimizeForType< Layer, AddDebugImpl > InsertDebugLayer
virtual Status SerializeToDot(std::ostream &stream) const
IConnectableLayer * AddFullyConnectedLayer(const FullyConnectedDescriptor &fullyConnectedDescriptor, const char *name=nullptr)
Adds a fully connected layer to the network.
OptimizeForConnection< ReshapeLayer, ReshapeLayer, OptimizeConsecutiveReshapesImpl > OptimizeConsecutiveReshapes
IConnectableLayer * AddMeanLayer(const MeanDescriptor &meanDescriptor, const char *name=nullptr)
Add a Mean layer to the network.
Private implementation of INetwork.
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
IConnectableLayer * AddInputLayer(LayerBindingId id, const char *name=nullptr)
Adds an input layer to the network.
This layer represents a elementwiseUnary operation.
constexpr const char * GetDataTypeName(DataType dataType)
IConnectableLayer * AddStridedSliceLayer(const StridedSliceDescriptor &stridedSliceDescriptor, const char *name=nullptr)
Adds a strided slice layer to the network.
A ResizeDescriptor for the ResizeLayer.
A StackDescriptor for the StackLayer.
Destination backend can work directly with tensors on source backend.
virtual std::vector< ITensorHandleFactory::FactoryId > GetHandleFactoryPreferences() const
(Optional) Returns a vector of supported TensorHandleFactory ids in preference order.
IConnectableLayer * AddSpaceToDepthLayer(const SpaceToDepthDescriptor &spaceToDepthDescriptor, const char *name=nullptr)
Adds a space to depth layer to the network.
OptimizeForConnection< ConvertFp16ToFp32Layer, ConvertFp32ToFp16Layer, OptimizeInverseConversionsImpl > OptimizeInverseConversionsFp16
IConnectableLayer * AddSoftmaxLayer(const SoftmaxDescriptor &softmaxDescriptor, const char *name=nullptr)
Adds a softmax layer to the network.
The SubgraphView class represents a subgraph of a Graph.
IConnectableLayer * AddMergerLayer(const MergerDescriptor &mergerDescriptor, const char *name=nullptr)
profiling::ProfilingGuid GetGuid() const
IConnectableLayer * AddFloorLayer(const char *name=nullptr)
A PadDescriptor for the PadLayer.
This layer represents an instance normalization operation.
std::shared_ptr< ConstTensorHandle > m_InputToForgetWeights
A unique pointer to represent 2D weights tensor with dimensions [num_units, inputSize] (QSymmS8)...
IConnectableLayer * AddMultiplicationLayer(const char *name=nullptr)
OptimizeForConnection< PermuteLayer, BatchToSpaceNdLayer, PermuteAndBatchToSpaceAsDepthToSpaceImpl< PermuteLayer > > PermuteAndBatchToSpaceAsDepthToSpace
IConnectableLayer * AddQLstmLayer(const QLstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr)
OptimizeForConnection< Layer, PermuteLayer, MovePermuteUpImpl > MovePermuteUp
This layer represents a Logical Binary operation.
IConnectableLayer * AddStridedSliceLayer(const StridedSliceDescriptor &stridedSliceDescriptor, const char *name=nullptr)
std::unique_ptr< NetworkImpl > pNetworkImpl
A layer user-provided data can be bound to (e.g. inputs, outputs).
IConnectableLayer * AddResizeBilinearLayer(const ResizeBilinearDescriptor &resizeDesc, const char *name=nullptr)
Adds a resize bilinear layer to the network.
friend IOptimizedNetworkPtr Optimize(const INetwork &inNetwork, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options, Optional< std::vector< std::string > &> messages)
Create an optimized version of the network.
void ForEachLayer(Func func) const
virtual std::vector< Capability > GetCapabilities(const IConnectableLayer *layer, const IConnectableLayer *connectedLayer, CapabilityClass capabilityClass)
IConnectableLayer * AddInputLayer(LayerBindingId id, const char *name=nullptr)
This layer dequantizes the input tensor.
ConvertConstants< Float32ToFloat16, IsFloat16Layer > ConvertConstantsFloatToHalf
OptimizeForType< TransposeLayer, TransposeAsReshapeImpl > TransposeAsReshape
This layer represents a Gather operator.
This layer represents a fully connected operation.
An LstmDescriptor for the LstmLayer.
#define ARMNN_NO_DEPRECATE_WARN_END
IConnectableLayer * AddLstmLayer(const LstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr)
Add a Lstm layer to the network.
IConnectableLayer * AddTransposeConvolution2dLayer(const TransposeConvolution2dDescriptor &descriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr)
#define ARMNN_ASSERT_MSG(COND, MSG)
IConnectableLayer * AddRankLayer(const char *name=nullptr)
Adds a rank layer to the network.
This layer represents a QuantizedLstm operation.
This layer represents a log softmax operation.
OptimizationResult ApplyBackendOptimizations(OptimizedNetworkImpl *optNetObjPtr, BackendSettings &backendSettings, BackendsMap &backends, const ModelOptions &modelOptions, Optional< std::vector< std::string > &> errMessages)
IConnectableLayer * AddPreluLayer(const char *name=nullptr)
A L2NormalizationDescriptor for the L2NormalizationLayer.
int32_t GetQuantizationOffset() const
LstmBasicParameters m_BasicParameters
An ArgMinMaxDescriptor for ArgMinMaxLayer.
float GetQuantizationScale() const
DataType GetDataType() const
An OriginsDescriptor for the ConcatLayer.
A ReduceDescriptor for the REDUCE operators.
IConnectableLayer * AddDetectionPostProcessLayer(const DetectionPostProcessDescriptor &descriptor, const ConstTensor &anchors, const char *name=nullptr)
bool has_value() const noexcept
A FullyConnectedDescriptor for the FullyConnectedLayer.
bool m_BiasEnabled
Enable/disable bias.
This layer represents a stack operation.
A tensor defined by a TensorInfo (shape and data type) and an immutable backing store.
const Subgraphs & GetFailedSubgraphs() const
This layer represents a merge operation.
This layer represents a softmax operation.
Validate all output shapes.
IConnectableLayer * AddInstanceNormalizationLayer(const InstanceNormalizationDescriptor &desc, const char *name=nullptr)
Adds an instance normalization layer to the network.
const std::string & GetNameStr() const
LayerType GetType() const override
Returns the armnn::LayerType of this layer.
uint32_t m_TargetWidth
Target width value.
std::vector< ConvertBf16ToFp32Layer * > InsertConvertBf16ToFp32LayersBefore(Graph &graph, Layer &layer, bool expectCorrectInputType)
A GatherDescriptor for the GatherLayer.
std::unique_ptr< OptimizedNetworkImpl > pOptimizedNetworkImpl
IConnectableLayer * AddDivisionLayer(const char *name=nullptr)
This layer represents a BatchToSpaceNd operation.
IConnectableLayer * AddOutputLayer(LayerBindingId id, const char *name=nullptr)
Adds an output layer to the network.
std::vector< SubgraphViewPtr > Subgraphs
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
bool m_HalfPixelCenters
Half Pixel Centers.
IConnectableLayer * AddArgMinMaxLayer(const ArgMinMaxDescriptor &desc, const char *name=nullptr)
IConnectableLayer * AddOutputLayer(LayerBindingId id, const char *name=nullptr)
IConnectableLayer * AddDequantizeLayer(const char *name=nullptr)
void SetQuantizationScale(float scale)
This layer represents a ArgMinMax operation.
IConnectableLayer * AddConcatLayer(const ConcatDescriptor &concatDescriptor, const char *name=nullptr)
#define ARMNN_ASSERT(COND)
A StandInDescriptor for the StandIn layer.
A QLstmDescriptor for the QLstmLayer.
BackendIdVector GetAvailablePreferredBackends() const
Device specific knowledge to be passed to the optimizer.
IConnectableLayer * AddSwitchLayer(const char *name=nullptr)
static bool IsLayerSupported(const BackendId &backendId, const IConnectableLayer &layer, Optional< DataType > dataType, std::string &outReasonIfUnsupported)
IConnectableLayer * AddEqualLayer(const char *name=nullptr)
Add a Equal layer to the network.
static void ConvertFloat16To32(const void *srcFloat16Buffer, size_t numElements, float *dstFloat32Buffer)
IConnectableLayer * AddAbsLayer(const char *name=nullptr)
Add absolute layer to the network.
IConnectableLayer * AddSubtractionLayer(const char *name=nullptr)
IConnectableLayer * AddResizeBilinearLayer(const ResizeBilinearDescriptor &resizeDesc, const char *name=nullptr)
bool Validate(const SubgraphView &originalSubgraph) const
An ActivationDescriptor for the ActivationLayer.
IConnectableLayer * AddLogicalBinaryLayer(const LogicalBinaryDescriptor &logicalBinaryDescriptor, const char *name=nullptr)
std::vector< ConvertFp32ToBf16Layer * > InsertConvertFp32ToBf16LayersAfter(Graph &graph, Layer &layer)
const BackendId & GetBackendId() const
uint32_t m_TargetHeight
Target height value.
This layer represents a floor operation.
void Accept(ILayerVisitor &visitor) const
IConnectableLayer * AddBatchNormalizationLayer(const BatchNormalizationDescriptor &desc, const ConstTensor &mean, const ConstTensor &variance, const ConstTensor &beta, const ConstTensor &gamma, const char *name=nullptr)
Adds a batch normalization layer to the network.
INetwork(NetworkOptions networkOptions={})
uint32_t m_TargetHeight
Target height value.
A SliceDescriptor for the SliceLayer.
IConnectableLayer * AddSpaceToBatchNdLayer(const SpaceToBatchNdDescriptor &spaceToBatchNdDescriptor, const char *name=nullptr)
Adds a space to batch layer to the network.
This layer represents a normalization operation.
virtual MemorySourceFlags GetExportFlags() const
IConnectableLayer * AddStandInLayer(const StandInDescriptor &descriptor, const char *name=nullptr)
This layer represents a pooling 2d operation.
This layer converts data type Float 32 to Float 16.
This layer represents a transpose operation.
IConnectableLayer * AddPermuteLayer(const PermuteDescriptor &permuteDescriptor, const char *name=nullptr)
This layer represents an addition operation.
IConnectableLayer * AddComparisonLayer(const ComparisonDescriptor &comparisonDescriptor, const char *name=nullptr)
Add a Comparison layer to the network.
QLstmBasicParameters m_BasicParameters
void SubstituteSubgraph(SubgraphView &subgraph, IConnectableLayer *substituteLayer)
Substitutes the given sub-graph with either a new layer or a new sub-graph.
IConnectableLayer * AddL2NormalizationLayer(const L2NormalizationDescriptor &desc, const char *name=nullptr)
IConnectableLayer * AddArgMinMaxLayer(const ArgMinMaxDescriptor &desc, const char *name=nullptr)
Adds an ArgMinMax layer to the network.
bool CheckScaleSetOnQuantizedType(Layer *layer, Optional< std::vector< std::string > &> errMessages)
IConnectableLayer * AddSplitterLayer(const ViewsDescriptor &splitterDescriptor, const char *name=nullptr)
Adds a splitter layer to the network.
void SetTensorHandleFactory(const ITensorHandleFactory::FactoryId &id)
A SpaceToBatchNdDescriptor for the SpaceToBatchNdLayer.
OptimizeForType< PermuteLayer, PermuteAsReshapeImpl > PermuteAsReshape
bool IsWarningOnly() const
OptimizeForConnection< Layer, TransposeLayer, SquashEqualSiblingsImpl< TransposeLayer > > SquashEqualTransposeSiblings
This layer represents a QLstm operation.
IConnectableLayer * AddAdditionLayer(const char *name=nullptr)
Adds an addition layer to the network.
const Substitutions & GetSubstitutions() const
BackendIdVector m_PreferredBackends
This layer represents a subtraction operation.
IConnectableLayer * AddUnidirectionalSequenceLstmLayer(const UnidirectionalSequenceLstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr)
Add a UnidirectionalSequenceLstm layer to the network.
IConnectableLayer * AddTransposeLayer(const TransposeDescriptor &transposeDescriptor, const char *name=nullptr)
This layer calculates both true and false outputs for input.
uint32_t GetNumInputs() const
Get the number of views/inputs.
This layer represents a LSTM operation.
IConnectableLayer * AddReshapeLayer(const ReshapeDescriptor &reshapeDescriptor, const char *name=nullptr)
Adds a reshape layer to the network.
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
IConnectableLayer * AddPadLayer(const PadDescriptor &padDescriptor, const char *name=nullptr)
Adds a fully pad layer to the network.
ConvertConstants< Float16ToFloat32, IsFloat32Layer > ConvertConstantsHalfToFloat
bool m_AlignCorners
Aligned corners.
A ElementwiseUnaryDescriptor for the ElementwiseUnaryLayer.
std::shared_ptr< ConstTensorHandle > m_InputToForgetWeights
A unique pointer to represent 2D weights tensor with dimensions [input_size, num_units].
IConnectableLayer * AddUnidirectionalSequenceLstmLayer(const UnidirectionalSequenceLstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr)
static Subgraphs SelectSubgraphs(Graph &graph, const LayerSelectorFunction &selector)
Selects subgraphs from a graph based on the selector function and the algorithm.
IConnectableLayer * AddFullyConnectedLayer(const FullyConnectedDescriptor &fullyConnectedDescriptor, const char *name=nullptr)
IConnectableLayer * AddReshapeLayer(const ReshapeDescriptor &reshapeDescriptor, const char *name=nullptr)
This layer represents a L2 normalization operation.
This layer represents a cast operation.
IConnectableLayer * AddMaximumLayer(const char *name=nullptr)
IConnectableLayer * AddShapeLayer(const char *name=nullptr)
std::shared_ptr< ConstTensorHandle > m_Weight
A unique pointer to store Weight values.
BackendsMap CreateSupportedBackends(TensorHandleFactoryRegistry &handleFactoryRegistry, BackendSettings &backendSettings)
IConnectableLayer * AddCastLayer(const char *name=nullptr)
OptimizeForConnection< ConvertFp32ToFp16Layer, ConvertFp16ToFp32Layer, OptimizeInverseConversionsImpl > OptimizeInverseConversionsFp32
IConnectableLayer * AddStackLayer(const StackDescriptor &stackDescriptor, const char *name=nullptr)
IConnectableLayer * AddSoftmaxLayer(const SoftmaxDescriptor &softmaxDescriptor, const char *name=nullptr)
const std::string & Get() const
IConnectableLayer * AddGreaterLayer(const char *name=nullptr)
IConnectableLayer * AddNormalizationLayer(const NormalizationDescriptor &normalizationDescriptor, const char *name=nullptr)
Adds a normalization layer to the network.
BackendIdSet m_SelectedBackends
OptimizeForExclusiveConnection< PadLayer, Pooling2dLayer, pad_fold::FoldPadIntoPooling2dImpl > FoldPadIntoPooling2d
IConnectableLayer * AddEqualLayer(const char *name=nullptr)
IConnectableLayer * AddFillLayer(const FillDescriptor &fillDescriptor, const char *name=nullptr)
Iterator end()
Returns iterator pointing to the end of the list. Lowercase for range-based for loops.
const Graph & GetGraph() const
OptimizationResult AttemptBackendAssignment(BackendSettings &backendSettings, Graph &graph, Layer *layer, BackendId backend, DataType dataTypeIn, DataType dataTypeOut, const std::vector< BackendId > &availablePreferredBackends, std::string &reasonIfUnsupported, Optional< std::vector< std::string > &> errMessages)
ITensorHandleFactory * GetFactory(ITensorHandleFactory::FactoryId id) const
Find a TensorHandleFactory by Id Returns nullptr if not found.
void SetTensorInfo(const TensorInfo &tensorInfo) override
A MeanDescriptor for the MeanLayer.
This layer represents a division operation.
Status SerializeToDot(std::ostream &stream) const
IConnectableLayer * AddSpaceToBatchNdLayer(const SpaceToBatchNdDescriptor &spaceToBatchNdDescriptor, const char *name=nullptr)
IConnectableLayer * AddQuantizedLstmLayer(const QuantizedLstmInputParams ¶ms, const char *name=nullptr)
Add a QuantizedLstm layer to the network.
This layer represents a strided slice operation.
ITensorHandleFactory::FactoryId CalculateSlotOption(BackendsMap &backends, OutputSlot &outputSlot, TensorHandleFactoryRegistry ®istry, bool importEnabled)
This layer represents a maximum operation.
const OutputSlot & GetOutputSlot(unsigned int index=0) const override
Get the const output slot handle by slot index.
Infer missing output shapes and validate all output shapes.
OptimizeForType< Layer, ConvertFp32NetworkToFp16Impl > Fp32NetworkToFp16Converter
A TransposeDescriptor for the TransposeLayer.
A StridedSliceDescriptor for the StridedSliceLayer.
IConnectableLayer * AddComparisonLayer(const ComparisonDescriptor &comparisonDescriptor, const char *name=nullptr)
OptimizationResult SelectTensorHandleStrategy(Graph &optGraph, BackendsMap &backends, TensorHandleFactoryRegistry ®istry, bool importEnabled, Optional< std::vector< std::string > &> errMessages)
void ReportWarning(const std::string &warningMessage, Optional< std::vector< std::string > &> warningMessages)
This layer represents a convolution 2d operation.
This layer converts data type Float32 to BFloat16.
void SetQuantizationOffset(int32_t offset)
IConnectableLayer * AddSwitchLayer(const char *name=nullptr)
Adds a switch layer to the network.
IConnectableLayer * AddDetectionPostProcessLayer(const DetectionPostProcessDescriptor &descriptor, const ConstTensor &anchors, const char *name=nullptr)
Adds a Detection PostProcess layer to the network.
static INetwork * CreateRaw(NetworkOptions networkOptions={})
IConnectableLayer * AddMergerLayer(const MergerDescriptor &mergerDescriptor, const char *name=nullptr)
Adds a concat layer to the network.
IConnectableLayer * AddMultiplicationLayer(const char *name=nullptr)
Adds a multiplication layer to the network.
IConnectableLayer * AddStandInLayer(const StandInDescriptor &descriptor, const char *name=nullptr)
Add a stand-in layer for a type unknown to the Arm NN framework.
This layer represents a mean operation.
IOptimizedNetwork(const IOptimizedNetwork &other, const ModelOptions &modelOptions)
This layer represents a comparison operation.
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
IConnectableLayer * AddBatchNormalizationLayer(const BatchNormalizationDescriptor &desc, const ConstTensor &mean, const ConstTensor &variance, const ConstTensor &beta, const ConstTensor &gamma, const char *name=nullptr)
IConnectableLayer * AddReduceLayer(const ReduceDescriptor &reduceDescriptor, const char *name=nullptr)
Adds a reduce layer to the network.
IConnectableLayer * AddElementwiseUnaryLayer(const ElementwiseUnaryDescriptor &elementwiseUnaryDescriptor, const char *name=nullptr)
Add an ElementwiseUnary layer to the network.
OptimizeForType< Layer, AddBroadcastReshapeLayerImpl > AddBroadcastReshapeLayer
IConnectableLayer * AddGatherLayer(const char *name=nullptr)
IConnectableLayer * AddDequantizeLayer(const char *name=nullptr)
Adds a Dequantize layer to the network.
A Pooling2dDescriptor for the Pooling2dLayer.
This layer dequantizes the input tensor.
A NormalizationDescriptor for the NormalizationLayer.
IConnectableLayer * AddPreluLayer(const char *name=nullptr)
Adds a PReLU layer to the network.
IConnectableLayer * AddGreaterLayer(const char *name=nullptr)
Add a Greater layer to the network.
OptimizedNetworkImpl(const OptimizedNetworkImpl &other, const ModelOptions &modelOptions)
IConnectableLayer * AddDepthToSpaceLayer(const DepthToSpaceDescriptor &depthToSpaceDescriptor, const char *name=nullptr)
IConnectableLayer * AddMaximumLayer(const char *name=nullptr)
Add a Maximum layer to the network.
DataLayout m_DataLayout
The data layout to be used (NCHW, NHWC).
An InstanceNormalizationDescriptor for InstanceNormalizationLayer.
This layer represents a multiplication operation.
IConnectableLayer * AddConvolution2dLayer(const Convolution2dDescriptor &convolution2dDescriptor, const ConstTensor &weights, const Optional< ConstTensor > &biases, const char *name=nullptr)
std::shared_ptr< ConstTensorHandle > m_Anchors
A unique pointer to store Anchor values.
IConnectableLayer * AddSplitterLayer(const ViewsDescriptor &splitterDescriptor, const char *name=nullptr)
IConnectableLayer * AddBatchToSpaceNdLayer(const BatchToSpaceNdDescriptor &batchToSpaceNdDescriptor, const char *name=nullptr)
A ResizeBilinearDescriptor for the ResizeBilinearLayer.
IConnectableLayer * AddLstmLayer(const LstmDescriptor &descriptor, const LstmInputParams ¶ms, const char *name=nullptr)
IConnectableLayer * AddPadLayer(const PadDescriptor &padDescriptor, const char *name=nullptr)
std::shared_ptr< ConstTensorHandle > m_InputToInputWeights
A unique pointer to represent 2D weights tensor with dimensions [outputSize, inputSize] (QAsymm8)...
const TensorInfo & GetTensorInfo() const override
static INetworkPtr Create(NetworkOptions networkOptions={})
IConnectableLayer * AddLogicalBinaryLayer(const LogicalBinaryDescriptor &descriptor, const char *name=nullptr)
Adds a Logical Binary layer to the network.
IConnectableLayer * AddGatherLayer(const char *name=nullptr)
Add Gather layer to the network.
EdgeStrategy CalculateEdgeStrategy(BackendsMap &backends, ITensorHandleFactory::FactoryId srcFactoryId, const Layer &layer, const Layer &connectedLayer, TensorHandleFactoryRegistry ®istry, bool importEnabled)
static void Destroy(IOptimizedNetwork *network)
IConnectableLayer * AddConcatLayer(const ConcatDescriptor &concatDescriptor, const char *name=nullptr)
Adds a concatenation layer to the network.
virtual MemorySourceFlags GetImportFlags() const
OptimizeForType< Layer, ConvertFp32NetworkToBf16Impl > Fp32NetworkToBf16Converter
A SoftmaxDescriptor for the SoftmaxLayer.
const char * GetLayerTypeAsCString(LayerType type)
virtual bool SupportsMapUnmap() const
void AddCompatibilityLayers(std::map< BackendId, std::unique_ptr< class IBackendInternal >> &backends, TensorHandleFactoryRegistry ®istry)
Modifies the graph in-place, removing edges connecting layers using different compute devices...
std::shared_ptr< ConstTensorHandle > m_Weight
A unique pointer to store weight values.
bool IsCpuRefUsed() const
static const FactoryId LegacyFactoryId
ITensorHandleFactory::FactoryId CalculateSlotOptionForInput(BackendsMap &backends, OutputSlot &slot, TensorHandleFactoryRegistry ®istry, bool importEnabled)
This layer represents a fill operation.
A DepthwiseConvolution2dDescriptor for the DepthwiseConvolution2dLayer.
A FillDescriptor for the FillLayer.
This layer represents a DepthToSpace operation.
A BatchNormalizationDescriptor for the BatchNormalizationLayer.
IConnectableLayer * AddTransposeLayer(const TransposeDescriptor &transposeDescriptor, const char *name=nullptr)
Adds a transpose layer to the network.
unsigned int GetNumElements() const
std::map< BackendId, std::unique_ptr< class IBackendInternal > > BackendsMap
This layer represents a resize operation.
A PermuteDescriptor for the PermuteLayer.
IConnectableLayer * AddSliceLayer(const SliceDescriptor &sliceDescriptor, const char *name=nullptr)
IConnectableLayer * AddDepthToSpaceLayer(const DepthToSpaceDescriptor &depthToSpaceDescriptor, const char *name=nullptr)
Adds a depth to space layer to the network.
LayerType
When adding a new layer, adapt also the LastLayer enum value in the enum class LayerType below...
bool m_ConstantWeights
Enable/disable constant weights and biases.
IConnectableLayer * AddInstanceNormalizationLayer(const InstanceNormalizationDescriptor &desc, const char *name=nullptr)
OptimizeForType< FullyConnectedLayer, RedirectMembersToConstantInputsImpl > RedirectMembersToConstantInputs