IOptimizedNetwork Class Reference

#include <INetwork.hpp>

Public Member Functions
Status	PrintGraph ()
Status	SerializeToDot (std::ostream &stream) const
arm::pipe::ProfilingGuid	GetGuid () const
size_t	GetNumInputs () const
size_t	GetNumOutputs () const
void	ExecuteStrategy (IStrategy &strategy) const
	IOptimizedNetwork (const IOptimizedNetwork &other, const ModelOptions &modelOptions)
	Creates a copy of the IOptimizedNetwork.
	IOptimizedNetwork (std::unique_ptr< Graph > graph)
	IOptimizedNetwork (std::unique_ptr< OptimizedNetworkImpl > impl)
	~IOptimizedNetwork ()
const std::shared_ptr< IProfiler > &	GetProfiler () const

Static Public Member Functions
static void	Destroy (IOptimizedNetwork *network)

Protected Member Functions
	IOptimizedNetwork (std::unique_ptr< Graph > graph, const ModelOptions &modelOptions)

Protected Attributes
std::unique_ptr< OptimizedNetworkImpl >	pOptimizedNetworkImpl

Friends
class	LoadedNetwork
Graph &	GetGraphForTesting (IOptimizedNetwork *optNetPtr)
ModelOptions &	GetModelOptionsForTesting (IOptimizedNetwork *optNetPtr)
IOptimizedNetworkPtr	Optimize (const INetwork &inNetwork, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptionsOpaque &options=OptimizerOptionsOpaque(), Optional< std::vector< std::string > & > messages=EmptyOptional())
	Create an optimized version of the network.
IOptimizedNetworkPtr	Optimize (const Graph &inGraph, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptionsOpaque &options, Optional< std::vector< std::string > & > messages=EmptyOptional())
	Create an optimized version of the network.

Detailed Description

Definition at line 902 of file INetwork.hpp.

Constructor & Destructor Documentation

IOptimizedNetwork() [1/4]

IOptimizedNetwork	(	const IOptimizedNetwork &	other,
		const ModelOptions &	modelOptions )

Creates a copy of the IOptimizedNetwork.

The IOptimizedNetwork will not be reoptimized, the provided ModelOptions will only be used when creating a LoadedNetwork.

Definition at line 692 of file Network.cpp.

    : pOptimizedNetworkImpl(new OptimizedNetworkImpl(*other.pOptimizedNetworkImpl.get(), modelOptions)) {}

References IOptimizedNetwork(), and pOptimizedNetworkImpl.

Referenced by Destroy(), GetGraphForTesting, GetModelOptionsForTesting, IOptimizedNetwork(), LoadedNetwork, and Optimize.

IOptimizedNetwork() [2/4]

IOptimizedNetwork

(

std::unique_ptr< Graph >

graph

)

Definition at line 695 of file Network.cpp.

    : pOptimizedNetworkImpl(new OptimizedNetworkImpl(std::move(graph))) {}

References pOptimizedNetworkImpl.

IOptimizedNetwork() [3/4]

IOptimizedNetwork

(

std::unique_ptr< OptimizedNetworkImpl >

impl

)

Definition at line 698 of file Network.cpp.

    : pOptimizedNetworkImpl(std::move(impl)) {}

References pOptimizedNetworkImpl.

~IOptimizedNetwork()

~IOptimizedNetwork ( )

default

IOptimizedNetwork() [4/4]

IOptimizedNetwork ( std::unique_ptr< Graph > graph, const ModelOptions & modelOptions )

protected

Definition at line 701 of file Network.cpp.

    : pOptimizedNetworkImpl(new OptimizedNetworkImpl(std::move(graph), modelOptions)) {}

References pOptimizedNetworkImpl.

Member Function Documentation

Destroy()

void Destroy ( IOptimizedNetwork * network )

static

Definition at line 706 of file Network.cpp.

{
    delete network;
}

References IOptimizedNetwork().

Referenced by Optimize.

ExecuteStrategy()

void ExecuteStrategy

(

IStrategy &

strategy

)

const

Definition at line 3277 of file Network.cpp.

{
    pOptimizedNetworkImpl->ExecuteStrategy(strategy);
}

References pOptimizedNetworkImpl.

GetGuid()

arm::pipe::ProfilingGuid GetGuid

(

)

const

Definition at line 726 of file Network.cpp.

{
    return pOptimizedNetworkImpl->GetGuid();
}

References pOptimizedNetworkImpl.

GetNumInputs()

size_t GetNumInputs

(

)

const

Definition at line 731 of file Network.cpp.

{
    return pOptimizedNetworkImpl->GetNumInputs();
}

References pOptimizedNetworkImpl.

GetNumOutputs()

size_t GetNumOutputs

(

)

const

Definition at line 736 of file Network.cpp.

{
    return pOptimizedNetworkImpl->GetNumOutputs();
}

References pOptimizedNetworkImpl.

GetProfiler()

const std::shared_ptr< IProfiler > & GetProfiler

(

)

const

Definition at line 721 of file Network.cpp.

{
    return pOptimizedNetworkImpl->GetGraph().GetProfiler();
}

References pOptimizedNetworkImpl.

PrintGraph()

Status PrintGraph

(

)

Definition at line 711 of file Network.cpp.

{
    return pOptimizedNetworkImpl->PrintGraph();
}

References pOptimizedNetworkImpl.

SerializeToDot()

Status SerializeToDot

(

std::ostream &

stream

)

const

Definition at line 716 of file Network.cpp.

{
    return pOptimizedNetworkImpl->SerializeToDot(stream);
}

References pOptimizedNetworkImpl.

Friends And Related Symbol Documentation

GetGraphForTesting

Graph & GetGraphForTesting ( IOptimizedNetwork * optNetPtr )

friend

Definition at line 49 of file TestUtils.cpp.

{
    return optNet->pOptimizedNetworkImpl->GetGraph();
}

References IOptimizedNetwork(), and pOptimizedNetworkImpl.

GetModelOptionsForTesting

ModelOptions & GetModelOptionsForTesting ( IOptimizedNetwork * optNetPtr )

friend

Definition at line 54 of file TestUtils.cpp.

{
    return optNet->pOptimizedNetworkImpl->GetModelOptions();
}

References IOptimizedNetwork(), and pOptimizedNetworkImpl.

LoadedNetwork

friend class LoadedNetwork

friend

Definition at line 927 of file INetwork.hpp.

References IOptimizedNetwork(), and LoadedNetwork.

Referenced by LoadedNetwork.

Optimize [1/2]

IOptimizedNetworkPtr Optimize ( const Graph & inGraph, const std::vector< BackendId > & backendPreferences, const IDeviceSpec & deviceSpec, const OptimizerOptionsOpaque & options, Optional< std::vector< std::string > & > messages = EmptyOptional() )

friend

Create an optimized version of the network.

Parameters

inGraph	Graph to be optimized.
backendPreferences	The choice of the backend ordered by user preferences.
deviceSpec	DeviceSpec object as queried from the runtime. See IRuntime::GetDeviceSpec()
messages	If there are failures or warnings a string describing same will be added to the vector
options	OptimizerOptions object with optimizer configuration options

Returns

An IOptimizedNetworkPtr interface to the optimized network, throws an exception derived from armnn::Exception if process fails.

Definition at line 2026 of file Network.cpp.

{
    ARMNN_LOG(debug) << options.ToString();
 
    // Enable profiling
    auto profiler = inGraph.GetProfiler();
    ProfilerManager::GetInstance().RegisterProfiler(profiler.get());
    profiler->EnableProfiling(options.GetProfilingEnabled());
 
    ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer");
    if (backendPreferences.empty())
    {
        throw InvalidArgumentException("Invoked Optimize with no backends specified");
    }
 
    if (options.GetReduceFp32ToBf16())
    {
        throw InvalidArgumentException("BFloat16 optimization is currently ignored. In order to use Bf16 optimization "
                                       "Please use the FastMathEnabled backend option for CpuAcc or GpuAcc.");
    }
 
    if (options.GetReduceFp32ToFp16() && options.GetReduceFp32ToBf16())
    {
        throw InvalidArgumentException("BFloat16 and Float16 optimization cannot be enabled at the same time.");
    }
 
    // Ensure TensorInfo is set on all output slots of ConstantLayers in the graph
    inGraph.VerifyConstantLayerSetTensorInfo();
 
    std::unique_ptr<Graph> graph = std::make_unique<Graph>(inGraph);
 
    // We need to pass on the information about whether import and export is enabled to the LoadNetwork phase.
    // The mechanism to do that is to add model options to the optimized network.
    armnn::BackendOptions importExport("Global",
                                        {{"ImportEnabled", options.GetImportEnabled()},
                                         {"ExportEnabled", options.GetExportEnabled()}});
    ModelOptions optimizedOptions(options.GetModelOptions());
    optimizedOptions.push_back(importExport);
 
    auto optNet = IOptimizedNetworkPtr(new IOptimizedNetwork(std::move(graph), optimizedOptions),
                                       &IOptimizedNetwork::Destroy);
 
    IOptimizedNetwork* optNetObjPtr = optNet.get();
 
    // Get the optimized graph
    Graph& optGraph = optNetObjPtr->pOptimizedNetworkImpl->GetGraph();
 
    if(options.GetShapeInferenceMethod() == ShapeInferenceMethod::InferAndValidate)
    {
        // Infer the tensor infos for all output slots. Throws an exception on failure
        optGraph.InferTensorInfos();
    }
 
    using namespace optimizations;
    // Substitute Max + Min with Bounded Relu before AddBroadcastReshapeLayer optimisation,
    // as Bounded ReLu needs the constants to be 1D size 1
    Optimizer::Pass(optGraph, MakeOptimizations(MaxMinIntoBoundedRelu()));
 
    // Perform BroadcastToOptimizationLayer before AddBroadcastReshapeLayer optimisation
    Optimizer::Pass(optGraph, MakeOptimizations(BroadcastToOptimizationLayer()));
 
    Optimizer::Pass(optGraph, MakeOptimizations(AddBroadcastReshapeLayer()));
 
    if(options.GetShapeInferenceMethod() == ShapeInferenceMethod::ValidateOnly)
    {
        // Validate the tensor infos for all output slots. Throws an exception on failure
        optGraph.InferTensorInfos();
    }
 
    // Initialize backend settings
    BackendSettings backendSettings(backendPreferences, deviceSpec);
    auto availablePreferredBackends = backendSettings.GetAvailablePreferredBackends();
    if (availablePreferredBackends.empty())
    {
        std::stringstream failureMsg;
        failureMsg << "None of the preferred backends " << backendPreferences
                   << " are supported. Current platform provides " << backendSettings.m_SupportedBackends;
        ReportError(failureMsg.str(), messages);
        throw InvalidArgumentException(failureMsg.str());
    }
 
    // Create a map to temporarily hold initialized backend objects
    TensorHandleFactoryRegistry tensorHandleFactoryRegistry;
    BackendsMap backends = CreateSupportedBackends(tensorHandleFactoryRegistry, backendSettings);
    bool hasFp16 = CheckFp16Support(backends, availablePreferredBackends);
 
    bool reduceFp32ToFp16 = options.GetReduceFp32ToFp16();
    // If fp16 is supported on the backend and fastmath has been enabled and the model is a TfLite converted Fp16
    // model: enable turbo mode optimizations
    if (hasFp16 && CheckFastMathSupport(availablePreferredBackends, optimizedOptions) && IsTfLiteTurboModel(optGraph))
    {
        Optimizer::Pass(optGraph, MakeOptimizations(TurboConvertConstDequantisationLayersToConstLayers()));
        reduceFp32ToFp16 = true;
    }
    else
    {
        Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstDequantisationLayersToConstLayers()));
    }
 
    // Group Constant Layer optimizations together where possible.
    // This is important as:
    // FusePermuteIntoConstantLayer must happen before FoldPadIntoDepthwiseConvolution2d and
    // FuseBatchNormIntoDepthwiseConvolution2D.
    Optimizer::Pass(optGraph, MakeOptimizations(FusePermuteIntoConstLayer()));
    // Perform optimisation passes
    Optimizer::Pass(optGraph, MakeOptimizations(SquashEqualPermuteSiblings(),
                                                SquashEqualTransposeSiblings(),
                                                SquashEqualReshapeSiblings(),
                                                OptimizeInversePermutes(),
                                                OptimizeInverseTransposes(),
                                                MovePermuteUp(),
                                                MoveTransposeUp(),
                                                PermuteAsReshape(),
                                                TransposeAsReshape(),
                                                OptimizeConsecutiveReshapes(),
                                                BroadcastToOptimizationLayer(),
                                                PermuteAndBatchToSpaceAsDepthToSpace(),
                                                TransposeAndBatchToSpaceAsDepthToSpace(),
                                                FuseBatchNormIntoConvolution2DFloat32(),
                                                FuseBatchNormIntoConvolution2DFloat16(),
                                                FuseBatchNormIntoDepthwiseConvolution2DFloat32(),
                                                FuseBatchNormIntoDepthwiseConvolution2DFloat16()));
 
    const std::vector<BackendId> mappedGpuBackends = BackendRegistryInstance().GetMappedGpuBackends();
 
    // All or nothing Gpu backends cannot be used as fallback
    for (auto backend : mappedGpuBackends)
    {
        if (std::count(backendPreferences.begin(), backendPreferences.end(), backend)
            && (backendPreferences[0] != backend) &&
            (backendPreferences[0] != armnn::BackendId("GpuAcc")))
        {
            std::stringstream failureMsg;
            failureMsg << backend << " backend cannot be specified as fallback.";
            ReportError(failureMsg.str(), messages);
            throw InvalidArgumentException(failureMsg.str());
        }
    }
 
    std::vector<BackendId> amendedBackendPreferences = backendPreferences;
    std::unordered_set<BackendId> supportedBackends = armnn::BackendRegistryInstance().GetBackendIds();
    if (amendedBackendPreferences[0] == armnn::BackendId("GpuAcc"))
    {
        // Add mapped Gpu backends if not already there and GpuAcc is first backend requested
        for (auto backend : mappedGpuBackends)
        {
            if (!std::count(amendedBackendPreferences.begin(), amendedBackendPreferences.end(), backend))
            {
                amendedBackendPreferences.insert(amendedBackendPreferences.begin(), backend);
            }
        }
    }
 
    if (reduceFp32ToFp16 && hasFp16)
    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_ReduceFp32ToFp16");
        Optimizer::Pass(optGraph, MakeOptimizations(Fp32NetworkToFp16Converter()));
        Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsFloatToHalf()));
    }
    // Assign an available backend to each layer
    Graph::Iterator firstLayer = optGraph.begin();
    Graph::Iterator lastLayer  = optGraph.end();
    OptimizationResult assignBackendsResult = AssignBackends(optNetObjPtr->pOptimizedNetworkImpl.get(),
                                                             backendSettings,
                                                             firstLayer,
                                                             lastLayer,
                                                             messages);
    if (assignBackendsResult.m_Error)
    {
        // Failed to assign a backend to each layer
        throw InvalidArgumentException("Failed to assign a backend to each layer");
    }
 
    Optimizer::Pass(optGraph, MakeOptimizations(OptimizeInverseConversionsFp16(),
                                                OptimizeInverseConversionsFp32()));
 
    // Apply the backend-specific optimizations
    OptimizationResult backendOptimizationResult = ApplyBackendOptimizations(optNetObjPtr->pOptimizedNetworkImpl.get(),
                                                                             backendSettings,
                                                                             backends,
                                                                             options.GetModelOptions(),
                                                                             messages);
    if (backendOptimizationResult.m_Error)
    {
        // Failed to apply the backend-specific optimizations
        throw InvalidArgumentException("Failed to apply the backend-specific optimizations");
    }
 
    // Convert constants
    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_ConvertConstants");
        Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsFloatToHalf()));
        Optimizer::Pass(optGraph, MakeOptimizations(ConvertConstantsHalfToFloat()));
    }
 
    // This must occur after all topological changes to the graph and any redirection of variables
    // If the debug flag is set, then insert a DebugLayer after each layer
    // Doing this after applying the backend optimizations as they might have changed some layers
    if (options.GetDebugEnabled() && !options.GetDebugToFileEnabled())
    {
        Optimizer::Pass(optGraph, MakeOptimizations(InsertDebugLayer()));
    }
    else if (options.GetDebugToFileEnabled())
    {
        // Setup the output file path
        try
        {
#if !defined(ARMNN_DISABLE_FILESYSTEM)
            auto result = armnnUtils::Filesystem::CreateDirectory("/ArmNNIntermediateLayerOutputs");
            ARMNN_LOG(info) << "Intermediate tensors will be written to: " << result;
#endif
            Optimizer::Pass(optGraph, MakeOptimizations(InsertDebugToFileLayer()));
        }
        catch (const armnn::RuntimeException& e)
        {
            // If we cannot create the output directory then we'll issue a warning and continue.
            ARMNN_LOG(warning) << "Unable to print intermediate layer outputs : " << e.what();
        }
    }
 
    // Calculate the compatibility strategies for tensor handles
    OptimizationResult strategyResult = SelectTensorHandleStrategy(optGraph,
                                                                   backends,
                                                                   tensorHandleFactoryRegistry,
                                                                   options.GetImportEnabled(),
                                                                   options.GetExportEnabled(),
                                                                   messages);
 
    if (strategyResult.m_Error)
    {
        // Failed to apply the backend-specific optimizations
        return IOptimizedNetworkPtr(nullptr, &IOptimizedNetwork::Destroy);
    }
 
    // Based on the tensor handle strategy determined above, insert copy layers where required.
    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Optimizer_AddCompatibilityLayers");
        optGraph.AddCompatibilityLayers(backends, tensorHandleFactoryRegistry);
    }
 
    return optNet;
}

References Graph::AddCompatibilityLayers(), armnn::ApplyBackendOptimizations(), ARMNN_LOG, ARMNN_SCOPED_PROFILING_EVENT, armnn::AssignBackends(), armnn::BackendRegistryInstance(), Graph::begin(), armnn::CheckFastMathSupport(), armnn::CheckFp16Support(), armnnUtils::Filesystem::CreateDirectory(), armnn::CreateSupportedBackends(), armnn::debug, Destroy(), Graph::end(), BackendSettings::GetAvailablePreferredBackends(), BackendRegistry::GetBackendIds(), OptimizerOptionsOpaque::GetDebugEnabled(), OptimizerOptionsOpaque::GetDebugToFileEnabled(), OptimizerOptionsOpaque::GetExportEnabled(), OptimizerOptionsOpaque::GetImportEnabled(), ProfilerManager::GetInstance(), BackendRegistry::GetMappedGpuBackends(), OptimizerOptionsOpaque::GetModelOptions(), Graph::GetProfiler(), OptimizerOptionsOpaque::GetProfilingEnabled(), OptimizerOptionsOpaque::GetReduceFp32ToBf16(), OptimizerOptionsOpaque::GetReduceFp32ToFp16(), OptimizerOptionsOpaque::GetShapeInferenceMethod(), armnn::InferAndValidate, Graph::InferTensorInfos(), armnn::info, IOptimizedNetwork(), armnn::IsTfLiteTurboModel(), OptimizationResult::m_Error, BackendSettings::m_SupportedBackends, armnn::MakeOptimizations(), Optimizer::Pass(), pOptimizedNetworkImpl, ProfilerManager::RegisterProfiler(), armnn::ReportError(), armnn::SelectTensorHandleStrategy(), OptimizerOptionsOpaque::ToString(), armnn::Undefined, armnn::ValidateOnly, Graph::VerifyConstantLayerSetTensorInfo(), armnn::warning, and Exception::what().

Optimize [2/2]

IOptimizedNetworkPtr Optimize ( const INetwork & inNetwork, const std::vector< BackendId > & backendPreferences, const IDeviceSpec & deviceSpec, const OptimizerOptionsOpaque & options = OptimizerOptionsOpaque(), Optional< std::vector< std::string > & > messages = EmptyOptional() )

friend

Create an optimized version of the network.

Parameters

network	INetwork description of the network to be optimized.
backendPreferences	The choice of the backend ordered by user preferences.
deviceSpec	DeviceSpec object as queried from the runtime. See IRuntime::GetDeviceSpec()
messages	If there are failures or warnings a string describing same will be added to the vector
options	OptimizerOptions object with optimizer configuration options

Returns

An IOptimizedNetworkPtr interface to the optimized network, throws an exception derived from armnn::Exception if process fails.

Definition at line 2287 of file Network.cpp.

{
    return Optimize(inNetwork.pNetworkImpl->GetGraph(),
                    backendPreferences,
                    deviceSpec,
                    options,
                    messages);
}

Member Data Documentation

pOptimizedNetworkImpl

std::unique_ptr<OptimizedNetworkImpl> pOptimizedNetworkImpl

protected

Definition at line 944 of file INetwork.hpp.

Referenced by ExecuteStrategy(), GetGraphForTesting, GetGuid(), GetModelOptionsForTesting, GetNumInputs(), GetNumOutputs(), GetProfiler(), IOptimizedNetwork(), IOptimizedNetwork(), IOptimizedNetwork(), IOptimizedNetwork(), Optimize, PrintGraph(), and SerializeToDot().

---

The documentation for this class was generated from the following files:

• include/armnn/INetwork.hpp
• src/armnn/Network.cpp