neuralnetworks/1.0/vts/functional/GeneratedTestHarness.cpp

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "Event.h"
#include "TestHarness.h"
#include "VtsHalNeuralnetworksV1_0TargetTest.h"

#include <android-base/logging.h>
#include <android/hidl/memory/1.0/IMemory.h>
#include <hidlmemory/mapping.h>

namespace android {
namespace hardware {
namespace neuralnetworks {
namespace V1_0 {
namespace vts {
namespace functional {
// allocator helper
hidl_memory allocateSharedMemory(int64_t size, const std::string& type = "ashmem");

namespace generated_tests {
using ::android::hardware::neuralnetworks::V1_0::implementation::Event;
using ::generated_tests::filter;
using ::generated_tests::for_all;
using ::generated_tests::for_each;
using ::generated_tests::resize_accordingly;
using ::generated_tests::MixedTyped;
using ::generated_tests::MixedTypedExampleType;
using ::generated_tests::Float32Operands;
using ::generated_tests::Int32Operands;
using ::generated_tests::Quant8Operands;
// Top level driver for models and examples generated by test_generator.py
// Test driver for those generated from ml/nn/runtime/test/spec
void Execute(const sp<IDevice>& device, std::function<Model(void)> create_model,
             std::function<bool(int)> is_ignored,
             const std::vector<MixedTypedExampleType>& examples) {
    Model model = create_model();
    sp<IPreparedModel> preparedModel;
    sp<Event> preparationEvent = new Event();
    ASSERT_NE(nullptr, preparationEvent.get());
    Return<void> prepareRet = device->prepareModel(
        model, preparationEvent, [&](ErrorStatus status, const sp<IPreparedModel>& prepared) {
            EXPECT_EQ(ErrorStatus::NONE, status);
            preparedModel = prepared;
        });
    ASSERT_TRUE(prepareRet.isOk());
    ASSERT_NE(nullptr, preparedModel.get());
    Event::Status preparationStatus = preparationEvent->wait();
    EXPECT_EQ(Event::Status::SUCCESS, preparationStatus);

    const uint32_t INPUT = 0;
    const uint32_t OUTPUT = 1;

    int example_no = 1;
    for (auto& example : examples) {
        SCOPED_TRACE(example_no++);

        const MixedTyped& inputs = example.first;
        const MixedTyped& golden = example.second;

        std::vector<RequestArgument> inputs_info, outputs_info;
        uint32_t inputSize = 0, outputSize = 0;

        // This function only partially specifies the metadata (vector of RequestArguments).
        // The contents are copied over below.
        for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) {
            if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1);
            RequestArgument arg = {
                .location = {.poolIndex = INPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
                .dimensions = {},
            };
            inputs_info[index] = arg;
            inputSize += s;
        });
        // Compute offset for inputs 1 and so on
        {
            size_t offset = 0;
            for (auto& i : inputs_info) {
                i.location.offset = offset;
                offset += i.location.length;
            }
        }

        MixedTyped test;  // holding test results

        // Go through all outputs, initialize RequestArgument descriptors
        resize_accordingly<float>(golden, test);
        resize_accordingly<int32_t>(golden, test);
        resize_accordingly<uint8_t>(golden, test);
        for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) {
            if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1);
            RequestArgument arg = {
                .location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)},
                .dimensions = {},
            };
            outputs_info[index] = arg;
            outputSize += s;
        });
        // Compute offset for outputs 1 and so on
        {
            size_t offset = 0;
            for (auto& i : outputs_info) {
                i.location.offset = offset;
                offset += i.location.length;
            }
        }
        std::vector<hidl_memory> pools = {allocateSharedMemory(inputSize),
                                          allocateSharedMemory(outputSize)};
        ASSERT_NE(0ull, pools[INPUT].size());
        ASSERT_NE(0ull, pools[OUTPUT].size());

        // load data
        sp<IMemory> inputMemory = mapMemory(pools[INPUT]);
        sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]);
        ASSERT_NE(nullptr, inputMemory.get());
        ASSERT_NE(nullptr, outputMemory.get());
        char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer()));
        char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer()));
        ASSERT_NE(nullptr, inputPtr);
        ASSERT_NE(nullptr, outputPtr);
        inputMemory->update();
        outputMemory->update();

        // Go through all inputs, copy the values
        for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) {
            char* begin = (char*)p;
            char* end = begin + s;
            // TODO: handle more than one input
            std::copy(begin, end, inputPtr + inputs_info[index].location.offset);
        });

        inputMemory->commit();
        outputMemory->commit();
        // execute request
        sp<Event> executionEvent = new Event();
        ASSERT_NE(nullptr, executionEvent.get());
        Return<ErrorStatus> executeStatus = preparedModel->execute(
            {.inputs = inputs_info, .outputs = outputs_info, .pools = pools}, executionEvent);
        ASSERT_TRUE(executeStatus.isOk());
        EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executeStatus));
        Event::Status eventStatus = executionEvent->wait();
        EXPECT_EQ(Event::Status::SUCCESS, eventStatus);

        // validate results
        outputMemory->read();
#define COPY_BACK(ty)                                                              \
    for_each<ty>(test, [&outputs_info, outputPtr](int index, std::vector<ty>& m) { \
        RequestArgument& i = outputs_info[index];                                  \
        ASSERT_EQ(m.size(), i.location.length / sizeof(ty));                       \
        char* begin = outputPtr + i.location.offset;                               \
        memcpy(m.data(), begin, i.location.length);                                \
    });
        COPY_BACK(float);
        COPY_BACK(int32_t);
        COPY_BACK(uint8_t);
#undef COPY_BACK
        outputMemory->commit();
        // Filter out don't cares
        MixedTyped filtered_golden;
        MixedTyped filtered_test;
        filter<float>(golden, &filtered_golden, is_ignored);
        filter<float>(test, &filtered_test, is_ignored);
        filter<int32_t>(golden, &filtered_golden, is_ignored);
        filter<int32_t>(test, &filtered_test, is_ignored);
        filter<uint8_t>(golden, &filtered_golden, is_ignored);
        filter<uint8_t>(test, &filtered_test, is_ignored);

        // We want "close-enough" results for float
        for_each<float>(filtered_golden, [&filtered_test](int index, auto& golden_float) {
            auto& test_float_operands = std::get<Float32Operands>(filtered_test);
            auto& test_float = test_float_operands[index];
            for (unsigned int i = 0; i < golden_float.size(); i++) {
                SCOPED_TRACE(i);
                EXPECT_NEAR(golden_float[i], test_float[i], 1.e-5);
            }
        });
        EXPECT_EQ(std::get<Int32Operands>(filtered_golden), std::get<Int32Operands>(filtered_test));
        EXPECT_EQ(std::get<Quant8Operands>(filtered_golden),
                  std::get<Quant8Operands>(filtered_test));
    }
}

}  // namespace generated_tests

}  // namespace functional
}  // namespace vts
}  // namespace V1_0
}  // namespace neuralnetworks
}  // namespace hardware
}  // namespace android