1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.suite;
17 
18 import static org.junit.Assert.assertEquals;
19 import static org.junit.Assert.assertNull;
20 import static org.junit.Assert.assertTrue;
21 import static org.junit.Assert.fail;
22 import static org.mockito.Mockito.mock;
23 
24 import com.android.tradefed.build.BuildInfo;
25 import com.android.tradefed.build.IBuildInfo;
26 import com.android.tradefed.build.IDeviceBuildInfo;
27 import com.android.tradefed.config.Configuration;
28 import com.android.tradefed.config.ConfigurationDef;
29 import com.android.tradefed.config.ConfigurationException;
30 import com.android.tradefed.config.IConfiguration;
31 import com.android.tradefed.config.OptionSetter;
32 import com.android.tradefed.device.DeviceNotAvailableException;
33 import com.android.tradefed.device.ITestDevice;
34 import com.android.tradefed.invoker.IInvocationContext;
35 import com.android.tradefed.invoker.InvocationContext;
36 import com.android.tradefed.invoker.TestInformation;
37 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
38 import com.android.tradefed.result.ILogSaver;
39 import com.android.tradefed.result.ITestInvocationListener;
40 import com.android.tradefed.testtype.Abi;
41 import com.android.tradefed.testtype.IAbi;
42 import com.android.tradefed.testtype.IRemoteTest;
43 import com.android.tradefed.testtype.StubTest;
44 import com.android.tradefed.util.AbiUtils;
45 import com.android.tradefed.util.FileUtil;
46 import com.android.tradefed.util.ZipUtil;
47 
48 import org.easymock.EasyMock;
49 import org.junit.Before;
50 import org.junit.Test;
51 import org.junit.runner.RunWith;
52 import org.junit.runners.JUnit4;
53 
54 import java.io.File;
55 import java.util.ArrayList;
56 import java.util.Collection;
57 import java.util.HashMap;
58 import java.util.LinkedHashMap;
59 import java.util.LinkedHashSet;
60 import java.util.List;
61 import java.util.Set;
62 
63 /**
64  * Unit tests for {@link TfSuiteRunner}.
65  */
66 @RunWith(JUnit4.class)
67 public class TfSuiteRunnerTest {
68 
69     private static final String TEST_CONFIG =
70             "<configuration description=\"Runs a stub tests part of some suite\">\n"
71                     + "    <option name=\"test-suite-tag\" value=\"example-suite\" />\n"
72                     + "    <test class=\"com.android.tradefed.testtype.StubTest\" />\n"
73                     + "</configuration>";
74 
75     private TfSuiteRunner mRunner;
76     private IConfiguration mStubMainConfiguration;
77     private ILogSaver mMockLogSaver;
78     private TestInformation mTestInfo;
79 
80     @Before
setUp()81     public void setUp() {
82         mRunner = new TestTfSuiteRunner();
83         mMockLogSaver = EasyMock.createMock(ILogSaver.class);
84         mStubMainConfiguration = new Configuration("stub", "stub");
85         mStubMainConfiguration.setLogSaver(mMockLogSaver);
86         mRunner.setConfiguration(mStubMainConfiguration);
87 
88         IInvocationContext context = new InvocationContext();
89         context.addAllocatedDevice(ConfigurationDef.DEFAULT_DEVICE_NAME, null);
90         context.addDeviceBuildInfo(ConfigurationDef.DEFAULT_DEVICE_NAME, new BuildInfo());
91         mTestInfo = TestInformation.newBuilder().setInvocationContext(context).build();
92     }
93 
94     /**
95      * Test TfSuiteRunner that hardcodes the abis to avoid failures related to running the tests
96      * against a particular abi build of tradefed.
97      */
98     public static class TestTfSuiteRunner extends TfSuiteRunner {
99         @Override
getAbis(ITestDevice device)100         public Set<IAbi> getAbis(ITestDevice device) throws DeviceNotAvailableException {
101             Set<IAbi> abis = new LinkedHashSet<>();
102             abis.add(new Abi("arm64-v8a", AbiUtils.getBitness("arm64-v8a")));
103             abis.add(new Abi("armeabi-v7a", AbiUtils.getBitness("armeabi-v7a")));
104             return abis;
105         }
106     }
107 
108     /**
109      * Test for {@link TfSuiteRunner#loadTests()} implementation, for basic example configurations.
110      */
111     @Test
testLoadTests()112     public void testLoadTests() throws Exception {
113         OptionSetter setter = new OptionSetter(mRunner);
114         setter.setOptionValue("suite-config-prefix", "suite");
115         setter.setOptionValue("run-suite-tag", "example-suite");
116         LinkedHashMap <String, IConfiguration> configMap = mRunner.loadTests();
117         assertEquals(4, configMap.size());
118         assertTrue(configMap.containsKey("arm64-v8a suite/stub1"));
119         assertTrue(configMap.containsKey("armeabi-v7a suite/stub1"));
120         assertTrue(configMap.containsKey("arm64-v8a suite/stub2"));
121         assertTrue(configMap.containsKey("armeabi-v7a suite/stub2"));
122     }
123 
124     /**
125      * Test for {@link TfSuiteRunner#loadTests()} implementation, only stub1.xml is part of this
126      * suite.
127      */
128     @Test
testLoadTests_suite2()129     public void testLoadTests_suite2() throws Exception {
130         OptionSetter setter = new OptionSetter(mRunner);
131         setter.setOptionValue("suite-config-prefix", "suite");
132         setter.setOptionValue("run-suite-tag", "example-suite2");
133         LinkedHashMap <String, IConfiguration> configMap = mRunner.loadTests();
134         assertEquals(2, configMap.size());
135         assertTrue(configMap.containsKey("arm64-v8a suite/stub1"));
136         assertTrue(configMap.containsKey("armeabi-v7a suite/stub1"));
137     }
138 
139     /** Test that when splitting, the instance of the implementation is used. */
140     @Test
testSplit()141     public void testSplit() throws Exception {
142         OptionSetter setter = new OptionSetter(mRunner);
143         setter.setOptionValue("suite-config-prefix", "suite");
144         setter.setOptionValue("run-suite-tag", "example-suite");
145         Collection<IRemoteTest> tests = mRunner.split(2, mTestInfo);
146         assertEquals(4, tests.size());
147         for (IRemoteTest test : tests) {
148             assertTrue(test instanceof TfSuiteRunner);
149         }
150     }
151 
152     /**
153      * Test that when {@link TfSuiteRunner} run-suite-tag is not set we cannot shard since there is
154      * no configuration.
155      */
156     @Test
testSplit_nothingToLoad()157     public void testSplit_nothingToLoad() throws Exception {
158         OptionSetter setter = new OptionSetter(mRunner);
159         setter.setOptionValue("suite-config-prefix", "doesnotexists");
160         setter.setOptionValue("run-suite-tag", "doesnotexists");
161         assertNull(mRunner.split(2));
162     }
163 
164     /**
165      * Attempt to load a suite from a suite, but the sub-suite does not have a default run-suite-tag
166      * so it cannot run anything.
167      */
168     @Test
testLoadSuite_noSubConfigs()169     public void testLoadSuite_noSubConfigs() throws ConfigurationException {
170         OptionSetter setter = new OptionSetter(mRunner);
171         setter.setOptionValue("suite-config-prefix", "suite");
172         setter.setOptionValue("run-suite-tag", "test-empty");
173         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
174         assertEquals(0, configMap.size());
175     }
176 
177     /**
178      * Attempt to load a suite from a suite, the sub-suite has a default run-suite-tag that will be
179      * loaded.
180      */
181     @Test
testLoadSuite()182     public void testLoadSuite() throws ConfigurationException {
183         OptionSetter setter = new OptionSetter(mRunner);
184         setter.setOptionValue("suite-config-prefix", "suite");
185         setter.setOptionValue("run-suite-tag", "test-sub-suite");
186         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
187         assertEquals(6, configMap.size());
188         // 4 test configs loaded from the sub-suite
189         assertTrue(configMap.containsKey("arm64-v8a suite/stub1"));
190         assertTrue(configMap.containsKey("armeabi-v7a suite/stub1"));
191         assertTrue(configMap.containsKey("arm64-v8a suite/stub2"));
192         assertTrue(configMap.containsKey("armeabi-v7a suite/stub2"));
193         // 2 config from the left over <test> that was not a suite.
194         assertTrue(configMap.containsKey("arm64-v8a suite/sub-suite"));
195         assertTrue(configMap.containsKey("armeabi-v7a suite/sub-suite"));
196         IConfiguration config = configMap.get("arm64-v8a suite/sub-suite");
197         // assert that the TfSuiteRunner was removed from the config, only the stubTest remains
198         assertTrue(config.getTests().size() == 1);
199         assertTrue(config.getTests().get(0) instanceof StubTest);
200     }
201 
202     /**
203      * In case of cycle include of sub-suite configuration. We throw an exception to prevent any
204      * weird runs.
205      */
206     @Test
testLoadSuite_cycle()207     public void testLoadSuite_cycle() throws ConfigurationException {
208         OptionSetter setter = new OptionSetter(mRunner);
209         setter.setOptionValue("suite-config-prefix", "suite");
210         setter.setOptionValue("run-suite-tag", "test-cycle-a");
211         try {
212             mRunner.loadTests();
213             fail("Should have thrown an exception.");
214         } catch (RuntimeException expected) {
215             // expected
216         }
217     }
218 
219     /**
220      * Test for {@link TfSuiteRunner#run(TestInformation, ITestInvocationListener)} when loading
221      * another suite.
222      */
223     @Test
testLoadTests_suite()224     public void testLoadTests_suite() throws Exception {
225         OptionSetter setter = new OptionSetter(mRunner);
226         setter.setOptionValue("suite-config-prefix", "suite");
227         setter.setOptionValue("run-suite-tag", "example-suite3");
228         ITestInvocationListener listener = EasyMock.createMock(ITestInvocationListener.class);
229         mRunner.setDevice(mock(ITestDevice.class));
230         mRunner.setBuild(mock(IBuildInfo.class));
231         mRunner.setSystemStatusChecker(new ArrayList<>());
232         IInvocationContext context = new InvocationContext();
233         context.addAllocatedDevice(ConfigurationDef.DEFAULT_DEVICE_NAME, mock(ITestDevice.class));
234         mRunner.setInvocationContext(context);
235         TestInformation testInfo =
236                 TestInformation.newBuilder().setInvocationContext(context).build();
237         // runs the expanded suite
238         listener.testModuleStarted(EasyMock.anyObject());
239         listener.testRunStarted(
240                 EasyMock.eq("arm64-v8a suite/stub1"),
241                 EasyMock.eq(0),
242                 EasyMock.eq(0),
243                 EasyMock.anyLong());
244         listener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
245         listener.testModuleEnded();
246         listener.testModuleStarted(EasyMock.anyObject());
247         listener.testRunStarted(
248                 EasyMock.eq("armeabi-v7a suite/stub1"),
249                 EasyMock.eq(0),
250                 EasyMock.eq(0),
251                 EasyMock.anyLong());
252         listener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
253         listener.testModuleEnded();
254         EasyMock.replay(listener);
255         mRunner.run(testInfo, listener);
256         EasyMock.verify(listener);
257     }
258 
259     /**
260      * Test for {@link TfSuiteRunner#run(TestInformation, ITestInvocationListener)} when loading
261      * test configs from additional-tests-zip.
262      */
263     @Test
testLoadTests_additionalTestsZip()264     public void testLoadTests_additionalTestsZip() throws Exception {
265         File tmpDir = null;
266         File deviceTestDir = null;
267         File additionalTestsZipFile = null;
268         try {
269             tmpDir = FileUtil.createTempDir("test");
270             // tests directory for the build.
271             deviceTestDir = FileUtil.createTempDir("build-info-test-dir");
272 
273             File zipDir = FileUtil.getFileForPath(tmpDir, "suite");
274             FileUtil.mkdirsRWX(zipDir);
275 
276             // Create 2 test configs inside a zip.
277             File testConfig = new File(zipDir, "test1.config");
278             FileUtil.writeToFile(TEST_CONFIG, testConfig);
279             File testConfig2 = new File(zipDir, "test2.config");
280             FileUtil.writeToFile(TEST_CONFIG, testConfig2);
281             additionalTestsZipFile = ZipUtil.createZip(zipDir);
282 
283             OptionSetter setter = new OptionSetter(mRunner);
284             setter.setOptionValue("suite-config-prefix", "suite");
285             setter.setOptionValue("run-suite-tag", "example-suite");
286             setter.setOptionValue("additional-tests-zip", additionalTestsZipFile.getAbsolutePath());
287 
288             IDeviceBuildInfo deviceBuildInfo = EasyMock.createMock(IDeviceBuildInfo.class);
289             EasyMock.expect(deviceBuildInfo.getTestsDir()).andReturn(deviceTestDir);
290             mRunner.setBuild(deviceBuildInfo);
291 
292             EasyMock.replay(deviceBuildInfo);
293             LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
294             assertEquals(8, configMap.size());
295             // The keySet should be stable and always ensure the same order of files.
296             List<String> keyList = new ArrayList<>(configMap.keySet());
297             // test1 and test2 name was sanitized to look like the included configs.
298             assertEquals("arm64-v8a suite/test1", keyList.get(0));
299             assertEquals("armeabi-v7a suite/test1", keyList.get(1));
300             assertEquals("arm64-v8a suite/test2", keyList.get(2));
301             assertEquals("armeabi-v7a suite/test2", keyList.get(3));
302             assertEquals("arm64-v8a suite/stub1", keyList.get(4));
303             assertEquals("armeabi-v7a suite/stub1", keyList.get(5));
304             assertEquals("arm64-v8a suite/stub2", keyList.get(6));
305             assertEquals("armeabi-v7a suite/stub2", keyList.get(7));
306             EasyMock.verify(deviceBuildInfo);
307         } finally {
308             FileUtil.recursiveDelete(deviceTestDir);
309             FileUtil.recursiveDelete(tmpDir);
310             FileUtil.recursiveDelete(additionalTestsZipFile);
311         }
312     }
313 
314     /**
315      * Test for {@link TfSuiteRunner#loadTests()} that when a test config supports IAbiReceiver,
316      * multiple instances of the config are queued up.
317      */
318     @Test
testLoadTestsForMultiAbi()319     public void testLoadTestsForMultiAbi() throws Exception {
320         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
321         mRunner.setDevice(mockDevice);
322         OptionSetter setter = new OptionSetter(mRunner);
323         setter.setOptionValue("suite-config-prefix", "suite");
324         setter.setOptionValue("run-suite-tag", "example-suite-abi");
325         EasyMock.replay(mockDevice);
326         LinkedHashMap<String, IConfiguration> configMap = mRunner.loadTests();
327         assertEquals(2, configMap.size());
328         assertTrue(configMap.containsKey("arm64-v8a suite/stubAbi"));
329         assertTrue(configMap.containsKey("armeabi-v7a suite/stubAbi"));
330         EasyMock.verify(mockDevice);
331     }
332 }
333