1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "neuralnetworks_hidl_hal_test"
18
19 #include "VtsHalNeuralnetworks.h"
20 #include <android-base/logging.h>
21 #include <hidl/ServiceManagement.h>
22 #include <string>
23 #include <utility>
24 #include "1.0/Callbacks.h"
25 #include "1.0/Utils.h"
26 #include "GeneratedTestHarness.h"
27 #include "TestHarness.h"
28
29 namespace android::hardware::neuralnetworks::V1_2::vts::functional {
30
31 using implementation::PreparedModelCallback;
32 using HidlToken = hidl_array<uint8_t, static_cast<uint32_t>(Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;
33 using V1_0::ErrorStatus;
34 using V1_0::Request;
35 using V1_1::ExecutionPreference;
36
37 // internal helper function
createPreparedModel(const sp<IDevice> & device,const Model & model,sp<IPreparedModel> * preparedModel)38 void createPreparedModel(const sp<IDevice>& device, const Model& model,
39 sp<IPreparedModel>* preparedModel) {
40 ASSERT_NE(nullptr, preparedModel);
41 *preparedModel = nullptr;
42
43 // see if service can handle model
44 bool fullySupportsModel = false;
45 const Return<void> supportedCall = device->getSupportedOperations_1_2(
46 model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
47 ASSERT_EQ(ErrorStatus::NONE, status);
48 ASSERT_NE(0ul, supported.size());
49 fullySupportsModel = std::all_of(supported.begin(), supported.end(),
50 [](bool valid) { return valid; });
51 });
52 ASSERT_TRUE(supportedCall.isOk());
53
54 // launch prepare model
55 const sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
56 const Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_2(
57 model, ExecutionPreference::FAST_SINGLE_ANSWER, hidl_vec<hidl_handle>(),
58 hidl_vec<hidl_handle>(), HidlToken(), preparedModelCallback);
59 ASSERT_TRUE(prepareLaunchStatus.isOk());
60 ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));
61
62 // retrieve prepared model
63 preparedModelCallback->wait();
64 const ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
65 *preparedModel = getPreparedModel_1_2(preparedModelCallback);
66
67 // The getSupportedOperations_1_2 call returns a list of operations that are
68 // guaranteed not to fail if prepareModel_1_2 is called, and
69 // 'fullySupportsModel' is true i.f.f. the entire model is guaranteed.
70 // If a driver has any doubt that it can prepare an operation, it must
71 // return false. So here, if a driver isn't sure if it can support an
72 // operation, but reports that it successfully prepared the model, the test
73 // can continue.
74 if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
75 ASSERT_EQ(nullptr, preparedModel->get());
76 LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot prepare "
77 "model that it does not support.";
78 std::cout << "[ ] Early termination of test because vendor service cannot "
79 "prepare model that it does not support."
80 << std::endl;
81 GTEST_SKIP();
82 }
83 ASSERT_EQ(ErrorStatus::NONE, prepareReturnStatus);
84 ASSERT_NE(nullptr, preparedModel->get());
85 }
86
SetUp()87 void NeuralnetworksHidlTest::SetUp() {
88 testing::TestWithParam<NeuralnetworksHidlTestParam>::SetUp();
89 ASSERT_NE(kDevice, nullptr);
90 }
91
makeNamedDevice(const std::string & name)92 static NamedDevice makeNamedDevice(const std::string& name) {
93 return {name, IDevice::getService(name)};
94 }
95
getNamedDevicesImpl()96 static std::vector<NamedDevice> getNamedDevicesImpl() {
97 // Retrieves the name of all service instances that implement IDevice,
98 // including any Lazy HAL instances.
99 const std::vector<std::string> names = hardware::getAllHalInstanceNames(IDevice::descriptor);
100
101 // Get a handle to each device and pair it with its name.
102 std::vector<NamedDevice> namedDevices;
103 namedDevices.reserve(names.size());
104 std::transform(names.begin(), names.end(), std::back_inserter(namedDevices), makeNamedDevice);
105 return namedDevices;
106 }
107
getNamedDevices()108 const std::vector<NamedDevice>& getNamedDevices() {
109 const static std::vector<NamedDevice> devices = getNamedDevicesImpl();
110 return devices;
111 }
112
printNeuralnetworksHidlTest(const testing::TestParamInfo<NeuralnetworksHidlTestParam> & info)113 std::string printNeuralnetworksHidlTest(
114 const testing::TestParamInfo<NeuralnetworksHidlTestParam>& info) {
115 return gtestCompliantName(getName(info.param));
116 }
117
118 INSTANTIATE_DEVICE_TEST(NeuralnetworksHidlTest);
119
120 // Forward declaration from ValidateModel.cpp
121 void validateModel(const sp<IDevice>& device, const Model& model);
122 // Forward declaration from ValidateRequest.cpp
123 void validateRequest(const sp<IPreparedModel>& preparedModel, const V1_0::Request& request);
124 // Forward declaration from ValidateRequest.cpp
125 void validateRequestFailure(const sp<IPreparedModel>& preparedModel, const V1_0::Request& request);
126 // Forward declaration from ValidateBurst.cpp
127 void validateBurst(const sp<IPreparedModel>& preparedModel, const V1_0::Request& request);
128
validateEverything(const sp<IDevice> & device,const Model & model,const Request & request)129 void validateEverything(const sp<IDevice>& device, const Model& model, const Request& request) {
130 validateModel(device, model);
131
132 // Create IPreparedModel.
133 sp<IPreparedModel> preparedModel;
134 createPreparedModel(device, model, &preparedModel);
135 if (preparedModel == nullptr) return;
136
137 validateRequest(preparedModel, request);
138 validateBurst(preparedModel, request);
139 }
140
validateFailure(const sp<IDevice> & device,const Model & model,const Request & request)141 void validateFailure(const sp<IDevice>& device, const Model& model, const Request& request) {
142 // TODO: Should this always succeed?
143 // What if the invalid input is part of the model (i.e., a parameter).
144 validateModel(device, model);
145
146 // Create IPreparedModel.
147 sp<IPreparedModel> preparedModel;
148 createPreparedModel(device, model, &preparedModel);
149 if (preparedModel == nullptr) return;
150
151 validateRequestFailure(preparedModel, request);
152 }
153
TEST_P(ValidationTest,Test)154 TEST_P(ValidationTest, Test) {
155 const Model model = createModel(kTestModel);
156 ExecutionContext context;
157 const Request request = context.createRequest(kTestModel);
158 if (kTestModel.expectFailure) {
159 validateFailure(kDevice, model, request);
160 } else {
161 validateEverything(kDevice, model, request);
162 }
163 }
164
__anoneeeb8a370302(const std::string& testName) 165 INSTANTIATE_GENERATED_TEST(ValidationTest, [](const std::string& testName) {
166 // Skip validation for the "inputs_as_internal" and "all_tensors_as_inputs"
167 // generated tests.
168 return testName.find("inputs_as_internal") == std::string::npos &&
169 testName.find("all_tensors_as_inputs") == std::string::npos;
170 });
171
getPreparedModel_1_2(const sp<implementation::PreparedModelCallback> & callback)172 sp<IPreparedModel> getPreparedModel_1_2(const sp<implementation::PreparedModelCallback>& callback) {
173 sp<V1_0::IPreparedModel> preparedModelV1_0 = callback->getPreparedModel();
174 return IPreparedModel::castFrom(preparedModelV1_0).withDefault(nullptr);
175 }
176
177 } // namespace android::hardware::neuralnetworks::V1_2::vts::functional
178