NeonTensorHandle.hpp

Go to the documentation of this file.

//
// Copyright © 2017-2024 Arm Ltd and Contributors. All rights reserved.
// SPDX-License-Identifier: MIT
//
 
#pragma once
 
#include <BFloat16.hpp>
#include <Half.hpp>
 
#include <aclCommon/ArmComputeTensorHandle.hpp>
#include <aclCommon/ArmComputeTensorUtils.hpp>
#include <armnn/Exceptions.hpp>
#include <armnn/utility/PolymorphicDowncast.hpp>
 
#include <arm_compute/runtime/MemoryGroup.h>
#include <arm_compute/runtime/IMemoryGroup.h>
#include <arm_compute/runtime/Tensor.h>
#include <arm_compute/runtime/SubTensor.h>
#include <arm_compute/core/TensorShape.h>
#include <arm_compute/core/Coordinates.h>
#include "armnn/TypesUtils.hpp"
 
namespace armnn
{
class NeonTensorHandleDecorator;
 
class NeonTensorHandle : public IAclTensorHandle
{
public:
    NeonTensorHandle(const TensorInfo& tensorInfo)
                     : m_ImportFlags(static_cast<MemorySourceFlags>(MemorySource::Malloc)),
                       m_Imported(false),
                       m_IsImportEnabled(false),
                       m_TypeAlignment(GetDataTypeSize(tensorInfo.GetDataType()))
    {
        armnn::armcomputetensorutils::BuildArmComputeTensor(m_Tensor, tensorInfo);
    }
 
    NeonTensorHandle(const TensorInfo& tensorInfo,
                     DataLayout dataLayout,
                     MemorySourceFlags importFlags = static_cast<MemorySourceFlags>(MemorySource::Malloc))
                     : m_ImportFlags(importFlags),
                       m_Imported(false),
                       m_IsImportEnabled(false),
                       m_TypeAlignment(GetDataTypeSize(tensorInfo.GetDataType()))
 
 
    {
        armnn::armcomputetensorutils::BuildArmComputeTensor(m_Tensor, tensorInfo, dataLayout);
    }
 
    arm_compute::ITensor& GetTensor() override { return m_Tensor; }
    arm_compute::ITensor const& GetTensor() const override { return m_Tensor; }
 
    virtual void Allocate() override
    {
        // If we have enabled Importing, don't Allocate the tensor
        if (!m_IsImportEnabled)
        {
            armnn::armcomputetensorutils::InitialiseArmComputeTensorEmpty(m_Tensor);
        }
    };
 
    virtual void Manage() override
    {
        // If we have enabled Importing, don't manage the tensor
        if (!m_IsImportEnabled)
        {
            ARMNN_THROW_INVALIDARG_MSG_IF_FALSE(m_MemoryGroup, "arm_compute::MemoryGroup is null.");
            m_MemoryGroup->manage(&m_Tensor);
        }
    }
 
    virtual ITensorHandle* GetParent() const override { return nullptr; }
 
    virtual arm_compute::DataType GetDataType() const override
    {
        return m_Tensor.info()->data_type();
    }
 
    virtual void SetMemoryGroup(const std::shared_ptr<arm_compute::IMemoryGroup>& memoryGroup) override
    {
        m_MemoryGroup = PolymorphicPointerDowncast<arm_compute::MemoryGroup>(memoryGroup);
    }
 
    virtual const void* Map(bool /* blocking = true */) const override
    {
        return static_cast<const void*>(m_Tensor.buffer() + m_Tensor.info()->offset_first_element_in_bytes());
    }
 
    virtual void Unmap() const override {}
 
    TensorShape GetStrides() const override
    {
        return armcomputetensorutils::GetStrides(m_Tensor.info()->strides_in_bytes());
    }
 
    TensorShape GetShape() const override
    {
        return armcomputetensorutils::GetShape(m_Tensor.info()->tensor_shape());
    }
 
    void SetImportFlags(MemorySourceFlags importFlags)
    {
        m_ImportFlags = importFlags;
    }
 
    MemorySourceFlags GetImportFlags() const override
    {
        return m_ImportFlags;
    }
 
    void SetImportEnabledFlag(bool importEnabledFlag)
    {
        m_IsImportEnabled = importEnabledFlag;
    }
 
    bool CanBeImported(void* memory, MemorySource source) override
    {
        if (source != MemorySource::Malloc || reinterpret_cast<uintptr_t>(memory) % m_TypeAlignment)
        {
            return false;
        }
        return true;
    }
 
    virtual bool Import(void* memory, MemorySource source) override
    {
        if (m_ImportFlags&  static_cast<MemorySourceFlags>(source))
        {
            if (source == MemorySource::Malloc && m_IsImportEnabled)
            {
                if (!CanBeImported(memory, source))
                {
                    throw MemoryImportException("NeonTensorHandle::Import Attempting to import unaligned memory");
                }
 
                // m_Tensor not yet Allocated
                if (!m_Imported && !m_Tensor.buffer())
                {
                    arm_compute::Status status = m_Tensor.allocator()->import_memory(memory);
                    // Use the overloaded bool operator of Status to check if it worked, if not throw an exception
                    // with the Status error message
                    m_Imported = bool(status);
                    if (!m_Imported)
                    {
                        throw MemoryImportException(status.error_description());
                    }
                    return m_Imported;
                }
 
                // m_Tensor.buffer() initially allocated with Allocate().
                if (!m_Imported && m_Tensor.buffer())
                {
                    throw MemoryImportException(
                        "NeonTensorHandle::Import Attempting to import on an already allocated tensor");
                }
 
                // m_Tensor.buffer() previously imported.
                if (m_Imported)
                {
                    arm_compute::Status status = m_Tensor.allocator()->import_memory(memory);
                    // Use the overloaded bool operator of Status to check if it worked, if not throw an exception
                    // with the Status error message
                    m_Imported = bool(status);
                    if (!m_Imported)
                    {
                        throw MemoryImportException(status.error_description());
                    }
                    return m_Imported;
                }
            }
            else
            {
                throw MemoryImportException("NeonTensorHandle::Import is disabled");
            }
        }
        else
        {
            throw MemoryImportException("NeonTensorHandle::Incorrect import flag");
        }
        return false;
    }
 
    virtual std::shared_ptr<ITensorHandle> DecorateTensorHandle(const TensorInfo& tensorInfo) override;
 
private:
    // Only used for testing
    void CopyOutTo(void* memory) const override
    {
        switch (this->GetDataType())
        {
            case arm_compute::DataType::F32:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<float*>(memory));
                break;
            case arm_compute::DataType::U8:
            case arm_compute::DataType::QASYMM8:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<uint8_t*>(memory));
                break;
            case arm_compute::DataType::QSYMM8:
            case arm_compute::DataType::QASYMM8_SIGNED:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<int8_t*>(memory));
                break;
            case arm_compute::DataType::BFLOAT16:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<armnn::BFloat16*>(memory));
                break;
            case arm_compute::DataType::F16:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<armnn::Half*>(memory));
                break;
            case arm_compute::DataType::S16:
            case arm_compute::DataType::QSYMM16:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<int16_t*>(memory));
                break;
            case arm_compute::DataType::S32:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<int32_t*>(memory));
                break;
            case arm_compute::DataType::S64:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<int64_t*>(memory));
                break;
            default:
            {
                throw armnn::UnimplementedException();
            }
        }
    }
 
    // Only used for testing
    void CopyInFrom(const void* memory) override
    {
        switch (this->GetDataType())
        {
            case arm_compute::DataType::F32:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const float*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::U8:
            case arm_compute::DataType::QASYMM8:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const uint8_t*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::QSYMM8:
            case arm_compute::DataType::QASYMM8_SIGNED:
            case arm_compute::DataType::QSYMM8_PER_CHANNEL:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const int8_t*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::BFLOAT16:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const armnn::BFloat16*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::F16:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const armnn::Half*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::S16:
            case arm_compute::DataType::QSYMM16:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const int16_t*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::S32:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const int32_t*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::S64:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const int64_t*>(memory),
                                                                 this->GetTensor());
                break;
            default:
            {
                throw armnn::UnimplementedException();
            }
        }
    }
 
    arm_compute::Tensor m_Tensor;
    std::shared_ptr<arm_compute::MemoryGroup> m_MemoryGroup;
    MemorySourceFlags m_ImportFlags;
    bool m_Imported;
    bool m_IsImportEnabled;
    const uintptr_t m_TypeAlignment;
    std::vector<std::shared_ptr<NeonTensorHandleDecorator>> m_Decorated;
};
 
class NeonSubTensorHandle : public IAclTensorHandle
{
public:
    NeonSubTensorHandle(IAclTensorHandle* parent,
                        const arm_compute::TensorShape& shape,
                        const arm_compute::Coordinates& coords)
     : m_Tensor(&parent->GetTensor(), shape, coords, true)
    {
        parentHandle = parent;
    }
 
    arm_compute::ITensor& GetTensor() override { return m_Tensor; }
    arm_compute::ITensor const& GetTensor() const override { return m_Tensor; }
 
    virtual void Allocate() override {}
    virtual void Manage() override {}
 
    virtual ITensorHandle* GetParent() const override { return parentHandle; }
 
    virtual arm_compute::DataType GetDataType() const override
    {
        return m_Tensor.info()->data_type();
    }
 
    virtual void SetMemoryGroup(const std::shared_ptr<arm_compute::IMemoryGroup>&) override {}
 
    virtual const void* Map(bool /* blocking = true */) const override
    {
        return static_cast<const void*>(m_Tensor.buffer() + m_Tensor.info()->offset_first_element_in_bytes());
    }
    virtual void Unmap() const override {}
 
    TensorShape GetStrides() const override
    {
        return armcomputetensorutils::GetStrides(m_Tensor.info()->strides_in_bytes());
    }
 
    TensorShape GetShape() const override
    {
        return armcomputetensorutils::GetShape(m_Tensor.info()->tensor_shape());
    }
 
    virtual std::shared_ptr<ITensorHandle> DecorateTensorHandle(const TensorInfo&) override
    {
        return nullptr;
    };
 
private:
    // Only used for testing
    void CopyOutTo(void* memory) const override
    {
        switch (this->GetDataType())
        {
            case arm_compute::DataType::F32:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<float*>(memory));
                break;
            case arm_compute::DataType::U8:
            case arm_compute::DataType::QASYMM8:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<uint8_t*>(memory));
                break;
            case arm_compute::DataType::QSYMM8:
            case arm_compute::DataType::QASYMM8_SIGNED:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<int8_t*>(memory));
                break;
            case arm_compute::DataType::S16:
            case arm_compute::DataType::QSYMM16:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<int16_t*>(memory));
                break;
            case arm_compute::DataType::S32:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<int32_t*>(memory));
                break;
            default:
            {
                throw armnn::UnimplementedException();
            }
        }
    }
 
    // Only used for testing
    void CopyInFrom(const void* memory) override
    {
        switch (this->GetDataType())
        {
            case arm_compute::DataType::F32:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const float*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::U8:
            case arm_compute::DataType::QASYMM8:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const uint8_t*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::QSYMM8:
            case arm_compute::DataType::QASYMM8_SIGNED:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const int8_t*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::S16:
            case arm_compute::DataType::QSYMM16:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const int16_t*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::S32:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const int32_t*>(memory),
                                                                 this->GetTensor());
                break;
            default:
            {
                throw armnn::UnimplementedException();
            }
        }
    }
 
    arm_compute::SubTensor m_Tensor;
    ITensorHandle* parentHandle = nullptr;
};
 
/// NeonTensorDecorator wraps an existing Neon tensor allowing us to override the TensorInfo for it
class NeonTensorDecorator : public arm_compute::ITensor
{
public:
    NeonTensorDecorator();
 
    NeonTensorDecorator(arm_compute::ITensor* original, const TensorInfo& info);
 
    ~NeonTensorDecorator() = default;
 
    NeonTensorDecorator(const NeonTensorDecorator&) = delete;
 
    NeonTensorDecorator& operator=(const NeonTensorDecorator&) = delete;
 
    NeonTensorDecorator(NeonTensorDecorator&&) = default;
 
    NeonTensorDecorator& operator=(NeonTensorDecorator&&) = default;
 
    // Inherited methods overridden:
    arm_compute::ITensorInfo* info() const override;
 
    arm_compute::ITensorInfo* info() override;
 
    uint8_t* buffer() const override;
 
private:
    arm_compute::ITensor*           m_Original;
    mutable arm_compute::TensorInfo m_TensorInfo;
};
 
class NeonTensorHandleDecorator : public IAclTensorHandle
{
public:
    NeonTensorHandleDecorator(IAclTensorHandle* parent, const TensorInfo& info)
    : m_Tensor(&parent->GetTensor(), info)
    {
        parentHandle = parent;
    }
 
    arm_compute::ITensor& GetTensor() override { return m_Tensor; }
    arm_compute::ITensor const& GetTensor() const override { return m_Tensor; }
 
    virtual void Allocate() override {}
    virtual void Manage() override {}
 
    virtual ITensorHandle* GetParent() const override { return nullptr; }
 
    virtual arm_compute::DataType GetDataType() const override
    {
        return m_Tensor.info()->data_type();
    }
 
    virtual void SetMemoryGroup(const std::shared_ptr<arm_compute::IMemoryGroup>&) override {}
 
    virtual const void* Map(bool /* blocking = true */) const override
    {
        return static_cast<const void*>(m_Tensor.buffer() + m_Tensor.info()->offset_first_element_in_bytes());
    }
    virtual void Unmap() const override {}
 
    TensorShape GetStrides() const override
    {
        return armcomputetensorutils::GetStrides(m_Tensor.info()->strides_in_bytes());
    }
 
    TensorShape GetShape() const override
    {
        return armcomputetensorutils::GetShape(m_Tensor.info()->tensor_shape());
    }
 
    virtual std::shared_ptr<ITensorHandle> DecorateTensorHandle(const TensorInfo&) override
    {
        return nullptr;
    };
 
private:
    // Only used for testing
    void CopyOutTo(void* memory) const override
    {
        switch (this->GetDataType())
        {
            case arm_compute::DataType::F32:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<float*>(memory));
                break;
            case arm_compute::DataType::U8:
            case arm_compute::DataType::QASYMM8:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<uint8_t*>(memory));
                break;
            case arm_compute::DataType::QSYMM8:
            case arm_compute::DataType::QASYMM8_SIGNED:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<int8_t*>(memory));
                break;
            case arm_compute::DataType::S16:
            case arm_compute::DataType::QSYMM16:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<int16_t*>(memory));
                break;
            case arm_compute::DataType::S32:
                armcomputetensorutils::CopyArmComputeITensorData(this->GetTensor(),
                                                                 static_cast<int32_t*>(memory));
                break;
            default:
            {
                throw armnn::UnimplementedException();
            }
        }
    }
 
    // Only used for testing
    void CopyInFrom(const void* memory) override
    {
        switch (this->GetDataType())
        {
            case arm_compute::DataType::F32:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const float*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::U8:
            case arm_compute::DataType::QASYMM8:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const uint8_t*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::QSYMM8:
            case arm_compute::DataType::QASYMM8_SIGNED:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const int8_t*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::S16:
            case arm_compute::DataType::QSYMM16:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const int16_t*>(memory),
                                                                 this->GetTensor());
                break;
            case arm_compute::DataType::S32:
                armcomputetensorutils::CopyArmComputeITensorData(static_cast<const int32_t*>(memory),
                                                                 this->GetTensor());
                break;
            default:
            {
                throw armnn::UnimplementedException();
            }
        }
    }
 
    NeonTensorDecorator m_Tensor;
    ITensorHandle* parentHandle = nullptr;
};
 
 
} // namespace armnn