LoadedNetwork Class Reference

#include <LoadedNetwork.hpp>

Public Types
using	WorkloadQueue = std::vector<std::unique_ptr<IWorkload>>

Public Member Functions
	~LoadedNetwork ()
TensorInfo	GetInputTensorInfo (LayerBindingId layerId) const
TensorInfo	GetOutputTensorInfo (LayerBindingId layerId) const
std::vector< ImportedInputId >	ImportInputs (const InputTensors &inputTensors, MemorySource forceImportMemorySource=MemorySource::Undefined)
std::vector< ImportedOutputId >	ImportOutputs (const OutputTensors &outputTensors, MemorySource forceImportMemorySource=MemorySource::Undefined)
void	ClearImportedInputs (const std::vector< ImportedInputId > inputIds)
void	ClearImportedOutputs (const std::vector< ImportedOutputId > outputIds)
Status	EnqueueWorkload (const InputTensors &inputTensors, const OutputTensors &outputTensors, std::vector< ImportedInputId > preImportedInputIds={}, std::vector< ImportedOutputId > preImportedOutputIds={})
	Single thread execution of the loaded network.
const std::shared_ptr< IProfiler > &	GetProfiler () const
void	FreeWorkingMemory ()
void	RegisterDebugCallback (const DebugCallbackFunction &func)
void	SendNetworkStructure (arm::pipe::IProfilingService &profilingService)
arm::pipe::ProfilingGuid	GetNetworkGuid ()

Static Public Member Functions
static std::unique_ptr< LoadedNetwork >	MakeLoadedNetwork (std::unique_ptr< IOptimizedNetwork > net, std::string &errorMessage, const INetworkProperties &networkProperties, arm::pipe::IProfilingService *profilingService)

Detailed Description

Definition at line 42 of file LoadedNetwork.hpp.

Member Typedef Documentation

WorkloadQueue

using WorkloadQueue = std::vector<std::unique_ptr<IWorkload>>

Definition at line 45 of file LoadedNetwork.hpp.

Constructor & Destructor Documentation

~LoadedNetwork()

~LoadedNetwork ( )

inline

Definition at line 47 of file LoadedNetwork.hpp.

    {
        FreeWorkingMemory();
    }

References FreeWorkingMemory().

Member Function Documentation

ClearImportedInputs()

void ClearImportedInputs

(

const std::vector< ImportedInputId >

inputIds

)

Definition at line 1525 of file LoadedNetwork.cpp.

{
    for (auto id : inputIds)
    {
        if (id > m_PreImportedInputHandles.size())
        {
            throw InvalidArgumentException(fmt::format("ClearImportedInputs::Unknown ImportedInputId: {}", id));
        }
 
        auto& importedTensorHandle = m_PreImportedInputHandles[id].m_TensorHandle;
        if (!importedTensorHandle)
        {
            throw InvalidArgumentException(
                    fmt::format("ClearImportedInputs::ImportedInput with id: {} has already been deleted", id));
        }
        // Call Unimport then destroy the tensorHandle
        importedTensorHandle->Unimport();
        importedTensorHandle = {};
    }
}

ClearImportedOutputs()

void ClearImportedOutputs

(

const std::vector< ImportedOutputId >

outputIds

)

Definition at line 1546 of file LoadedNetwork.cpp.

{
    for (auto id : outputIds)
    {
        if (id > m_PreImportedOutputHandles.size())
        {
            throw InvalidArgumentException(fmt::format("ClearImportedOutputs::Unknown ImportedOutputId: {}", id));
        }
 
        auto& importedTensorHandle = m_PreImportedOutputHandles[id].m_TensorHandle;
        if (!importedTensorHandle)
        {
            throw InvalidArgumentException(
                    fmt::format("ClearImportedOutputs::ImportedOutput with id: {} has already been deleted", id));
        }
        // Call Unimport then destroy the tensorHandle
        importedTensorHandle->Unimport();
        importedTensorHandle = {};
    }
}

EnqueueWorkload()

Status EnqueueWorkload	(	const InputTensors &	inputTensors,
		const OutputTensors &	outputTensors,
		std::vector< ImportedInputId >	preImportedInputIds = {},
		std::vector< ImportedOutputId >	preImportedOutputIds = {} )

Single thread execution of the loaded network.

Definition at line 783 of file LoadedNetwork.cpp.

{
    const Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();
 
    // Walk graph to determine the order of execution.
    if (graph.GetNumLayers() < 2)
    {
        ARMNN_LOG(warning) << "IRuntime::EnqueueWorkload()::Less than two nodes in graph";
        return Status::Failure;
    }
 
    // Data that must be kept alive for the entire execution of the workload.
    WorkloadData workloadData(inputTensors, outputTensors);
 
    // Input tensors can be provided as parameters or pre imported. Either way the number of
    // tensors should match the number of inputs.
    if (graph.GetNumInputs() != (inputTensors.size() + preImportedInputIds.size()))
    {
        throw InvalidArgumentException("Number of inputs provided does not match network.");
    }
 
    // For each input to the network, call EnqueueInput with the data passed by the user.
    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "PrepareInputs");
        m_InputQueue.clear();
        m_InputQueue.reserve(graph.GetNumInputs());
 
        unsigned int inputIndex = 0;
        unsigned int importedInputIdIndex = 0;
        std::sort(preImportedInputIds.begin(), preImportedInputIds.end());
        for (const BindableLayer* inputLayer : graph.GetInputLayers())
        {
            if (importedInputIdIndex < preImportedInputIds.size() &&
                inputIndex == preImportedInputIds[importedInputIdIndex])
            {
                // Only replace tensorhandles if they have not already been replaced
                if (!m_IsInputImported[inputIndex])
                {
                    auto outputTensorHandle = m_PreImportedInputHandles[inputIndex].m_TensorHandle.get();
 
                    for (const auto& workloadInfo: m_InputWorkloadSlotPairs[inputLayer->GetBindingId()])
                    {
                        auto workload = m_WorkloadQueue[workloadInfo.m_WorkloadIndex].get();
                        workload->ReplaceInputTensorHandle(outputTensorHandle, workloadInfo.m_SlotIndex);
                    }
                    m_IsInputImported[inputIndex] = true;
                }
                importedInputIdIndex++;
            }
            else
            {
                if (m_IsInputImported[inputIndex])
                {
                    OutputHandler& handler = const_cast<OutputHandler&>(inputLayer->GetOutputHandler(0));
 
                    for (const auto& workloadInfo: m_InputWorkloadSlotPairs[inputLayer->GetBindingId()])
                    {
                        auto workload = m_WorkloadQueue[workloadInfo.m_WorkloadIndex].get();
                        workload->ReplaceInputTensorHandle(handler.GetData(), workloadInfo.m_SlotIndex);
                    }
 
                    m_IsInputImported[inputIndex] = false;
                }
 
                // InputTensorHandle is not imported yet, process to enqueue input
                const TensorPin& pin = workloadData.GetInputTensorPin(inputLayer->GetBindingId());
                EnqueueInput(*inputLayer, pin.GetTensorHandle(), pin.GetTensorInfo());
            }
            inputIndex++;
        }
    }
    // For each output to the network, call EnqueueOutput with the data passed by the user.
    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "PrepareOutputs");
        m_OutputQueue.clear();
        m_OutputQueue.reserve(graph.GetNumOutputs());
 
        if (preImportedOutputIds.size() > graph.GetNumOutputs())
        {
            throw InvalidArgumentException("Invalid number of preImportedOutputIds");
        }
 
        unsigned int outputIndex = 0;
        unsigned int importedOutputIdIndex = 0;
        std::sort(preImportedOutputIds.begin(), preImportedOutputIds.end());
        for (const BindableLayer* outputLayer : graph.GetOutputLayers())
        {
            if (importedOutputIdIndex < preImportedOutputIds.size() &&
                outputIndex == preImportedOutputIds[importedOutputIdIndex])
            {
                // Only replace tensorhandles if they have not already been replaced
                ITensorHandle* inputTensorHandle = m_PreImportedOutputHandles[outputIndex].m_TensorHandle.get();
 
                if (!m_IsOutputImported[outputIndex])
                {
                    const auto bindingId = outputLayer->GetBindingId();
                    const auto& indices = m_OutputWorkloadSlotPairs[bindingId];
 
                    auto outputWorkload = m_WorkloadQueue[indices.m_OutputSlotIndices.m_WorkloadIndex].get();
 
                    outputWorkload->ReplaceOutputTensorHandle(inputTensorHandle,
                                                              indices.m_OutputSlotIndices.m_SlotIndex);
 
                    for (const auto& workloadInfo: indices.m_InputSlotIndices)
                    {
                        auto inputWorkload = m_WorkloadQueue[workloadInfo.m_WorkloadIndex].get();
                        inputWorkload->ReplaceInputTensorHandle(inputTensorHandle, workloadInfo.m_SlotIndex);
                    }
                    m_IsOutputImported[outputIndex] = true;
                }
 
                if (!inputTensorHandle)
                {
                    throw armnn::NullPointerException("Data should have been allocated.");
                }
 
                MemSyncQueueDescriptor syncDesc;
                syncDesc.m_Inputs.push_back(inputTensorHandle);
                WorkloadInfo info;
                info.m_InputTensorInfos.push_back(outputLayer->GetInputSlot(0).GetTensorInfo());
 
                auto syncWorkload = std::make_unique<SyncMemGenericWorkload>(syncDesc, info);
                if (!syncWorkload)
                {
                    throw armnn::NullPointerException("No sync workload created");
                }
 
                m_OutputQueue.push_back(std::move(syncWorkload));
                importedOutputIdIndex++;
            }
            else
            {
                if (m_IsOutputImported[outputIndex])
                {
                    const auto bindingId = outputLayer->GetBindingId();
                    const auto& indices = m_OutputWorkloadSlotPairs[bindingId];
 
                    auto outputWorkload = m_WorkloadQueue[indices.m_OutputSlotIndices.m_WorkloadIndex].get();
                    const OutputHandler& outputHandler =
                            outputLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetOutputHandler();
 
                    outputWorkload->ReplaceOutputTensorHandle(
                            outputHandler.GetData(), indices.m_OutputSlotIndices.m_SlotIndex);
 
                    for (const auto& workloadInfo: indices.m_InputSlotIndices)
                    {
                        auto inputWorkload = m_WorkloadQueue[workloadInfo.m_WorkloadIndex].get();
                        inputWorkload->ReplaceInputTensorHandle(outputHandler.GetData(), workloadInfo.m_SlotIndex);
                    }
                    m_IsOutputImported[outputIndex] = false;
                }
 
                const TensorPin& pin = workloadData.GetOutputTensorPin(outputLayer->GetBindingId());
                // OutputTensorHandle is not imported yet, process to enqueue Output
                EnqueueOutput(*outputLayer, pin.GetTensorHandle(), pin.GetTensorInfo());
            }
            outputIndex++;
        }
    }
 
    std::unique_ptr<TimelineUtilityMethods> timelineUtils =
                        TimelineUtilityMethods::GetTimelineUtils(*m_ProfilingService);
    ProfilingGuid inferenceGuid = m_ProfilingService->GetNextGuid();
    if (timelineUtils)
    {
        // Add inference timeline trace if profiling is enabled.
        ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
        timelineUtils->CreateTypedEntity(inferenceGuid, LabelsAndEventClasses::INFERENCE_GUID);
        timelineUtils->CreateRelationship(ProfilingRelationshipType::RetentionLink,
                                          networkGuid,
                                          inferenceGuid,
                                          LabelsAndEventClasses::EXECUTION_OF_GUID);
        timelineUtils->RecordEvent(inferenceGuid, LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS);
    }
 
    bool executionSucceeded = true;
 
    {
        if (m_ProfilingService->IsProfilingEnabled())
        {
            m_ProfilingService->IncrementCounterValue(INFERENCES_RUN);
        }
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Execute");
        ARMNN_SCOPED_HEAP_PROFILING("Executing");
        executionSucceeded = Execute(timelineUtils, inferenceGuid);
    }
 
    if (timelineUtils)
    {
        // Add end of life of the inference timeline if profiling is enabled.
        timelineUtils->RecordEvent(inferenceGuid, LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS);
        timelineUtils->Commit();
    }
 
    return executionSucceeded ? Status::Success : Status::Failure;
}

References ARMNN_LOG, ARMNN_SCOPED_HEAP_PROFILING, ARMNN_SCOPED_PROFILING_EVENT, armnn::Failure, OutputHandler::GetData(), Graph::GetInputLayers(), Graph::GetNumInputs(), Graph::GetNumLayers(), Graph::GetNumOutputs(), Graph::GetOutputLayers(), armnn::info, QueueDescriptor::m_Inputs, armnn::Success, armnn::Undefined, and armnn::warning.

Referenced by RuntimeImpl::EnqueueWorkload().

FreeWorkingMemory()

void FreeWorkingMemory

(

)

Definition at line 1197 of file LoadedNetwork.cpp.

{
#if !defined(ARMNN_DISABLE_THREADS)
    std::lock_guard<std::mutex> lockGuard(m_WorkingMemMutex);
#endif
 
    if (!m_IsWorkingMemAllocated)
    {
        return;
    }
 
    if (m_ExternalMemoryManager)
    {
        m_ExternalMemoryManager->Deallocate();
    }
 
    // Informs the memory managers to release memory in its respective memory group
    for (auto&& memoryManager : m_BackendMemoryMangers)
    {
        if (memoryManager)
        {
            memoryManager->Release();
        }
    }
    m_TensorHandleFactoryRegistry.ReleaseMemory();
    m_IsWorkingMemAllocated = false;
}

Referenced by RuntimeImpl::EnqueueWorkload(), and ~LoadedNetwork().

GetInputTensorInfo()

TensorInfo GetInputTensorInfo

(

LayerBindingId

layerId

)

const

Definition at line 626 of file LoadedNetwork.cpp.

{
    for (auto&& inputLayer : m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetInputLayers())
    {
        if (inputLayer->GetNumOutputSlots() != 1)
        {
            throw armnn::GraphValidationException("Input layer should have exactly 1 output slot");
        }
 
        if (inputLayer->GetBindingId() == layerId)
        {
            return inputLayer->GetOutputSlot(0).GetTensorInfo();
        }
    }
 
    throw InvalidArgumentException(fmt::format("No input layer is associated with id {}", layerId));
}

GetNetworkGuid()

ProfilingGuid GetNetworkGuid

(

)

Definition at line 621 of file LoadedNetwork.cpp.

{
    return m_OptimizedNetwork->GetGuid();
}

GetOutputTensorInfo()

TensorInfo GetOutputTensorInfo

(

LayerBindingId

layerId

)

const

Definition at line 644 of file LoadedNetwork.cpp.

{
    for (auto&& outputLayer : m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetOutputLayers())
    {
        if (outputLayer->GetNumInputSlots() != 1)
        {
            throw armnn::GraphValidationException("Output layer should have exactly 1 input slot");
        }
 
        if (!outputLayer->GetInputSlot(0).GetConnection())
        {
            throw armnn::GraphValidationException("Input slot on Output layer must be connected");
        }
 
        if (outputLayer->GetBindingId() == layerId)
        {
            return outputLayer->GetInputSlot(0).GetTensorInfo();
        }
    }
 
    throw InvalidArgumentException(fmt::format("No output layer is associated with id {}", layerId));
}

GetProfiler()

const std::shared_ptr< IProfiler > & GetProfiler ( ) const

inline

Definition at line 76 of file LoadedNetwork.hpp.

{ return m_OptimizedNetwork->GetProfiler(); }

Referenced by RuntimeImpl::EnqueueWorkload().

ImportInputs()

std::vector< ImportedInputId > ImportInputs	(	const InputTensors &	inputTensors,
		MemorySource	forceImportMemorySource = MemorySource::Undefined )

Definition at line 1402 of file LoadedNetwork.cpp.

{
    // Cannot import if import is not enabled and forceImportMemorySource is undefined
    if (forceImportMemorySource == MemorySource::Undefined)
    {
        throw MemoryImportException("ImportInputs: Memory Import failed, NetworkProperties.m_ImportEnabled");
    }
    // The number of pre imported tensors should not exceed the number of inputs.
    if (inputTensors.size() > m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetNumInputs())
    {
        throw MemoryImportException("ImportInputs: The number of tensors provided exceeds the number of inputs.");
    }
 
    std::vector<ImportedInputId> importedInputs;
    Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().TopologicalSort();
    unsigned int inputIndex = 0;
    for (const BindableLayer* inputLayer : graph.GetInputLayers())
    {
        auto outputTensorHandle = m_PreImportedInputHandles[inputIndex].m_TensorHandle.get();
 
        if (!outputTensorHandle)
        {
            inputIndex++;
            continue;
        }
 
        auto layerBindingId = inputLayer->GetBindingId();
        auto it = std::find_if(inputTensors.begin(), inputTensors.end(), [=](const auto& inputTensor)
        {
            return inputTensor.first == layerBindingId;
        });
 
        if (it == inputTensors.end())
        {
            inputIndex++;
            continue;
        }
 
        const auto& inputTensor = *it;
        std::unique_ptr<ITensorHandle> passThroughTensorHandle =
                std::make_unique<ConstPassthroughTensorHandle>(inputTensor.second.GetInfo(),
                                                               inputTensor.second.GetMemoryArea());
 
        try
        {
            if (outputTensorHandle->CanBeImported(passThroughTensorHandle->Map(), forceImportMemorySource)
                && (outputTensorHandle->Import(passThroughTensorHandle->Map(), forceImportMemorySource)))
            {
                importedInputs.push_back(inputIndex);
            }
            passThroughTensorHandle->Unmap();
        }
        catch(const MemoryImportException& exception)
        {
            ARMNN_LOG(error) << "An error occurred attempting to import input_"
                                       << inputIndex << " : " << exception.what();
            passThroughTensorHandle->Unmap();
        }
        inputIndex++;
    }
 
    return importedInputs;
}

References ARMNN_LOG, armnn::error, Graph::GetInputLayers(), Graph::TopologicalSort(), and armnn::Undefined.

ImportOutputs()

std::vector< ImportedOutputId > ImportOutputs	(	const OutputTensors &	outputTensors,
		MemorySource	forceImportMemorySource = MemorySource::Undefined )

Definition at line 1467 of file LoadedNetwork.cpp.

{
    // Cannot import if import is not enabled and forceImportMemorySource is undefined
    if (forceImportMemorySource == MemorySource::Undefined)
    {
        throw MemoryImportException("ImportOutputs: Memory Import failed, NetworkProperties.m_ImportEnabled");
    }
    // If forceImportMemorySource is defined, try import if memory is aligned
    if (outputTensors.size() != m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetNumOutputs())
    {
        throw MemoryImportException("ImportOutputs: Force Import failed, incorrect number of tensors");
    }
    std::vector<ImportedOutputId> importedOutputs;
    Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().TopologicalSort();
 
    unsigned int outputIndex = 0;
    for (const BindableLayer* const outputLayer : graph.GetOutputLayers())
    {
        auto inputTensorHandle = m_PreImportedOutputHandles[outputIndex].m_TensorHandle.get();
        if (!inputTensorHandle)
        {
            outputIndex++;
            continue;
        }
 
        auto layerBindingId = outputLayer->GetBindingId();
        auto it = std::find_if(outputTensors.begin(), outputTensors.end(), [=] (const auto& outputTensor)
        {
            return outputTensor.first == layerBindingId;
        });
 
        if (it == outputTensors.end())
        {
            outputIndex++;
            continue;
        }
 
        const auto outputTensor = *it;
        try
        {
            // Check if the output memory can be imported
            if (inputTensorHandle->CanBeImported(outputTensor.second.GetMemoryArea(), forceImportMemorySource)
                && inputTensorHandle->Import(outputTensor.second.GetMemoryArea(), forceImportMemorySource))
            {
                importedOutputs.push_back(outputIndex);
            }
        }
        catch(const MemoryImportException& exception)
        {
            ARMNN_LOG(error) << "An error occurred attempting to import output_"
                             << outputIndex << " : " << exception.what();
        }
        outputIndex++;
    }
    return importedOutputs;
}

References ARMNN_LOG, armnn::error, Graph::GetOutputLayers(), Graph::TopologicalSort(), and armnn::Undefined.

MakeLoadedNetwork()

std::unique_ptr< LoadedNetwork > MakeLoadedNetwork ( std::unique_ptr< IOptimizedNetwork > net, std::string & errorMessage, const INetworkProperties & networkProperties, arm::pipe::IProfilingService * profilingService )

static

Definition at line 170 of file LoadedNetwork.cpp.

{
    std::unique_ptr<LoadedNetwork> loadedNetwork;
 
    auto Fail = [&](const std::exception& error) -> std::unique_ptr<LoadedNetwork>
    {
        errorMessage = ToErrorMessage("An error occurred when preparing the network workloads: ", error);
        ARMNN_LOG(error) << errorMessage;
 
        return std::unique_ptr<LoadedNetwork>();
    };
 
    try
    {
        loadedNetwork.reset(new LoadedNetwork(std::move(net), networkProperties, profilingService));
    }
    catch (const armnn::RuntimeException& error)
    {
        return Fail(error);
    }
    catch (const armnn::Exception& error)
    {
        return Fail(error);
    }
    catch (const std::runtime_error& error)
    {
        return Fail(error);
    }
 
    return loadedNetwork;
}

References ARMNN_LOG, armnn::error, and MakeLoadedNetwork().

Referenced by RuntimeImpl::LoadNetwork(), and MakeLoadedNetwork().

RegisterDebugCallback()

void RegisterDebugCallback

(

const DebugCallbackFunction &

func

)

Definition at line 1567 of file LoadedNetwork.cpp.

{
    for (auto&& workloadPtr: m_WorkloadQueue)
    {
        workloadPtr.get()->RegisterDebugCallback(func);
    }
}

Referenced by RuntimeImpl::RegisterDebugCallback().

SendNetworkStructure()

void SendNetworkStructure

(

arm::pipe::IProfilingService &

profilingService

)

Definition at line 583 of file LoadedNetwork.cpp.

{
    ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "LoadNetwork_SendNetworkStructure");
    Graph& order = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().TopologicalSort();
    ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
 
    std::unique_ptr<TimelineUtilityMethods> timelineUtils =
        TimelineUtilityMethods::GetTimelineUtils(profilingService);
 
    timelineUtils->CreateTypedEntity(networkGuid, LabelsAndEventClasses::NETWORK_GUID);
 
    for (auto&& layer : order)
    {
        // Add layer to the post-optimisation network structure
        AddLayerStructure(timelineUtils, *layer, networkGuid);
        switch (layer->GetType())
        {
            case LayerType::Input:
            case LayerType::Output:
            {
                // Inputs and outputs are treated in a special way - see EnqueueInput() and EnqueueOutput().
                break;
            }
            default:
            {
                for (auto& workload : m_WorkloadQueue)
                {
                    // Add workload to the post-optimisation network structure
                    AddWorkloadStructure(timelineUtils, workload, *layer);
                }
            break;
            }
        }
    }
    // Commit to send the post-optimisation network structure
    timelineUtils->Commit();
}

References ARMNN_SCOPED_PROFILING_EVENT, armnn::Input, armnn::Output, Graph::TopologicalSort(), and armnn::Undefined.

---

The documentation for this class was generated from the following files:

• src/armnn/LoadedNetwork.hpp
• src/armnn/LoadedNetwork.cpp