1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.suite.retry;
17 
18 import static org.junit.Assert.assertNull;
19 
20 import com.android.annotations.VisibleForTesting;
21 import com.android.tradefed.config.ConfigurationException;
22 import com.android.tradefed.config.ConfigurationFactory;
23 import com.android.tradefed.config.IConfiguration;
24 import com.android.tradefed.config.IConfigurationFactory;
25 import com.android.tradefed.config.IConfigurationReceiver;
26 import com.android.tradefed.config.Option;
27 import com.android.tradefed.config.Option.Importance;
28 import com.android.tradefed.device.DeviceNotAvailableException;
29 import com.android.tradefed.device.IDeviceSelection;
30 import com.android.tradefed.invoker.IRescheduler;
31 import com.android.tradefed.invoker.TestInformation;
32 import com.android.tradefed.log.FileLogger;
33 import com.android.tradefed.log.ILeveledLogOutput;
34 import com.android.tradefed.log.LogUtil.CLog;
35 import com.android.tradefed.result.CollectingTestListener;
36 import com.android.tradefed.result.ITestInvocationListener;
37 import com.android.tradefed.result.TestDescription;
38 import com.android.tradefed.result.TestResult;
39 import com.android.tradefed.result.TestRunResult;
40 import com.android.tradefed.result.TextResultReporter;
41 import com.android.tradefed.testtype.IRemoteTest;
42 import com.android.tradefed.testtype.suite.BaseTestSuite;
43 import com.android.tradefed.testtype.suite.ITestSuite;
44 import com.android.tradefed.testtype.suite.SuiteTestFilter;
45 import com.android.tradefed.util.AbiUtils;
46 import com.android.tradefed.util.QuotationAwareTokenizer;
47 
48 import com.google.inject.Inject;
49 
50 import java.util.ArrayList;
51 import java.util.HashSet;
52 import java.util.LinkedHashMap;
53 import java.util.LinkedHashSet;
54 import java.util.List;
55 import java.util.Map;
56 import java.util.Map.Entry;
57 import java.util.Set;
58 
59 /**
60  * A special runner that allows to reschedule a previous run tests that failed or where not
61  * executed.
62  */
63 public final class RetryRescheduler implements IRemoteTest, IConfigurationReceiver {
64 
65     /** The types of the tests that can be retried. */
66     public enum RetryType {
67         FAILED,
68         NOT_EXECUTED,
69     }
70 
71     @Option(
72             name = "retry-type",
73             description =
74                     "used to retry tests of a certain status. Possible values include \"failed\" "
75                             + "and \"not_executed\".")
76     private RetryType mRetryType = null;
77 
78     @Option(
79         name = BaseTestSuite.MODULE_OPTION,
80         shortName = BaseTestSuite.MODULE_OPTION_SHORT_NAME,
81         description = "the test module to run. Only works for configuration in the tests dir."
82     )
83     private String mModuleName = null;
84 
85     /**
86      * It's possible to add extra exclusion from the rerun. But these tests will not change their
87      * state.
88      */
89     @Option(
90         name = BaseTestSuite.EXCLUDE_FILTER_OPTION,
91         description = "the exclude module filters to apply.",
92         importance = Importance.ALWAYS
93     )
94     private Set<String> mExcludeFilters = new HashSet<>();
95 
96     // Carry some options from suites that are convenient and don't impact the tests selection.
97     @Option(
98         name = ITestSuite.REBOOT_BEFORE_TEST,
99         description = "Reboot the device before the test suite starts."
100     )
101     private boolean mRebootBeforeTest = false;
102 
103     public static final String PREVIOUS_LOADER_NAME = "previous_loader";
104 
105     private IConfiguration mConfiguration;
106     private IRescheduler mRescheduler;
107 
108     private IConfigurationFactory mFactory;
109 
110     private IConfiguration mRescheduledConfiguration;
111 
112     @Override
run( TestInformation testInfo , ITestInvocationListener listener )113     public void run(
114             TestInformation testInfo /* do not use - should be null */,
115             ITestInvocationListener listener /* do not use - should be null */)
116             throws DeviceNotAvailableException {
117         assertNull(testInfo);
118         assertNull(listener);
119 
120         // Get the re-loader for previous results
121         Object loader = mConfiguration.getConfigurationObject(PREVIOUS_LOADER_NAME);
122         if (loader == null) {
123             throw new RuntimeException(
124                     String.format(
125                             "An <object> of type %s was expected in the retry.",
126                             PREVIOUS_LOADER_NAME));
127         }
128         if (!(loader instanceof ITestSuiteResultLoader)) {
129             throw new RuntimeException(
130                     String.format(
131                             "%s should be implementing %s",
132                             loader.getClass().getCanonicalName(),
133                             ITestSuiteResultLoader.class.getCanonicalName()));
134         }
135 
136         ITestSuiteResultLoader previousLoader = (ITestSuiteResultLoader) loader;
137         // First init the reloader.
138         previousLoader.init();
139         // Then get the command line of the previous run
140         String commandLine = previousLoader.getCommandLine();
141         IConfiguration originalConfig;
142         try {
143             originalConfig =
144                     getFactory()
145                             .createConfigurationFromArgs(
146                                     QuotationAwareTokenizer.tokenizeLine(commandLine));
147             // Transfer the sharding options from the original command.
148             originalConfig
149                     .getCommandOptions()
150                     .setShardCount(mConfiguration.getCommandOptions().getShardCount());
151             originalConfig
152                     .getCommandOptions()
153                     .setShardIndex(mConfiguration.getCommandOptions().getShardIndex());
154             IDeviceSelection requirements = mConfiguration.getDeviceRequirements();
155             // It should be safe to use the current requirements against the old config because
156             // There will be more checks like fingerprint if it was supposed to run.
157             originalConfig.setDeviceRequirements(requirements);
158 
159             // Transfer log level from retry to subconfig
160             ILeveledLogOutput originalLogger = originalConfig.getLogOutput();
161             originalLogger.setLogLevel(mConfiguration.getLogOutput().getLogLevel());
162             if (originalLogger instanceof FileLogger) {
163                 ((FileLogger) originalLogger)
164                         .setLogLevelDisplay(mConfiguration.getLogOutput().getLogLevel());
165             }
166 
167             handleExtraResultReporter(originalConfig, mConfiguration);
168         } catch (ConfigurationException e) {
169             throw new RuntimeException(e);
170         }
171         // Get previous results
172         CollectingTestListener collectedTests = previousLoader.loadPreviousResults();
173         previousLoader.cleanUp();
174 
175         // Appropriately update the configuration
176         IRemoteTest test = originalConfig.getTests().get(0);
177         if (!(test instanceof BaseTestSuite)) {
178             throw new RuntimeException(
179                     "RetryScheduler only works for BaseTestSuite implementations");
180         }
181         BaseTestSuite suite = (BaseTestSuite) test;
182         ResultsPlayer replayer = new ResultsPlayer();
183         updateRunner(suite, collectedTests, replayer);
184         collectedTests = null;
185         updateConfiguration(originalConfig, replayer);
186         // Do the customization of the configuration for specialized use cases.
187         customizeConfig(previousLoader, originalConfig);
188 
189         if (mRebootBeforeTest) {
190             suite.enableRebootBeforeTest();
191         }
192 
193         mRescheduledConfiguration = originalConfig;
194 
195         if (mRescheduler != null) {
196             // At the end, reschedule if requested
197             boolean res = mRescheduler.scheduleConfig(originalConfig);
198             if (!res) {
199                 CLog.e("Something went wrong, failed to kick off the retry run.");
200             }
201         }
202     }
203 
204     @Inject
setRescheduler(IRescheduler rescheduler)205     public void setRescheduler(IRescheduler rescheduler) {
206         mRescheduler = rescheduler;
207     }
208 
209     @Override
setConfiguration(IConfiguration configuration)210     public void setConfiguration(IConfiguration configuration) {
211         mConfiguration = configuration;
212     }
213 
getFactory()214     private IConfigurationFactory getFactory() {
215         if (mFactory != null) {
216             return mFactory;
217         }
218         return ConfigurationFactory.getInstance();
219     }
220 
221     @VisibleForTesting
setConfigurationFactory(IConfigurationFactory factory)222     void setConfigurationFactory(IConfigurationFactory factory) {
223         mFactory = factory;
224     }
225 
226     /** Returns the {@link IConfiguration} that should be retried. */
getRetryConfiguration()227     public final IConfiguration getRetryConfiguration() {
228         return mRescheduledConfiguration;
229     }
230 
231     /**
232      * Update the configuration to be ready for re-run.
233      *
234      * @param suite The {@link BaseTestSuite} that will be re-run.
235      * @param results The results of the previous run.
236      * @param replayer The {@link ResultsPlayer} that will replay the non-retried use cases.
237      */
updateRunner( BaseTestSuite suite, CollectingTestListener results, ResultsPlayer replayer)238     private void updateRunner(
239             BaseTestSuite suite, CollectingTestListener results, ResultsPlayer replayer) {
240         List<RetryType> types = new ArrayList<>();
241         if (mRetryType == null) {
242             types.add(RetryType.FAILED);
243             types.add(RetryType.NOT_EXECUTED);
244         } else {
245             types.add(mRetryType);
246         }
247 
248         // Expand the --module option in case no abi is specified.
249         Set<String> expandedModuleOption = new HashSet<>();
250         if (mModuleName != null) {
251             SuiteTestFilter moduleFilter = SuiteTestFilter.createFrom(mModuleName);
252             expandedModuleOption.add(mModuleName);
253             if (moduleFilter.getAbi() == null) {
254                 Set<String> abis = AbiUtils.getAbisSupportedByCompatibility();
255                 for (String abi : abis) {
256                     SuiteTestFilter namingFilter =
257                             new SuiteTestFilter(
258                                     abi, moduleFilter.getName(), moduleFilter.getTest());
259                     expandedModuleOption.add(namingFilter.toString());
260                 }
261             }
262         }
263 
264         // Expand the exclude-filter in case no abi is specified.
265         Set<String> extendedExcludeRetryFilters = new HashSet<>();
266         for (String excludeFilter : mExcludeFilters) {
267             SuiteTestFilter suiteFilter = SuiteTestFilter.createFrom(excludeFilter);
268             // Keep the current exclude-filter
269             extendedExcludeRetryFilters.add(excludeFilter);
270             if (suiteFilter.getAbi() == null) {
271                 // If no abi is specified, exclude them all.
272                 Set<String> abis = AbiUtils.getAbisSupportedByCompatibility();
273                 for (String abi : abis) {
274                     SuiteTestFilter namingFilter =
275                             new SuiteTestFilter(abi, suiteFilter.getName(), suiteFilter.getTest());
276                     extendedExcludeRetryFilters.add(namingFilter.toString());
277                 }
278             }
279         }
280 
281         // Prepare exclusion filters
282         for (TestRunResult moduleResult : results.getMergedTestRunResults()) {
283             // If the module is explicitly excluded from retries, preserve the original results.
284             if (!extendedExcludeRetryFilters.contains(moduleResult.getName())
285                     && (expandedModuleOption.isEmpty()
286                             || expandedModuleOption.contains(moduleResult.getName()))
287                     && RetryResultHelper.shouldRunModule(moduleResult, types)) {
288                 if (types.contains(RetryType.NOT_EXECUTED)) {
289                     // Clear the run failure since we are attempting to rerun all non-executed
290                     moduleResult.resetRunFailure();
291                 }
292 
293                 Map<TestDescription, TestResult> parameterizedMethods = new LinkedHashMap<>();
294 
295                 for (Entry<TestDescription, TestResult> result :
296                         moduleResult.getTestResults().entrySet()) {
297                     // Put aside all parameterized methods
298                     if (isParameterized(result.getKey())) {
299                         parameterizedMethods.put(result.getKey(), result.getValue());
300                         continue;
301                     }
302                     if (!RetryResultHelper.shouldRunTest(result.getValue(), types)) {
303                         addExcludeToConfig(suite, moduleResult, result.getKey().toString());
304                         replayer.addToReplay(
305                                 results.getModuleContextForRunResult(moduleResult.getName()),
306                                 moduleResult,
307                                 result);
308                     }
309                 }
310 
311                 // Handle parameterized methods
312                 for (Entry<String, Map<TestDescription, TestResult>> subMap :
313                         sortMethodToClass(parameterizedMethods).entrySet()) {
314                     boolean shouldNotrerunAnything =
315                             subMap.getValue()
316                                     .entrySet()
317                                     .stream()
318                                     .noneMatch(
319                                             (v) ->
320                                                     RetryResultHelper.shouldRunTest(
321                                                                     v.getValue(), types)
322                                                             == true);
323                     // If None of the base method need to be rerun exclude it
324                     if (shouldNotrerunAnything) {
325                         // Exclude the base method
326                         addExcludeToConfig(suite, moduleResult, subMap.getKey());
327                         // Replay all test cases
328                         for (Entry<TestDescription, TestResult> result :
329                                 subMap.getValue().entrySet()) {
330                             replayer.addToReplay(
331                                     results.getModuleContextForRunResult(moduleResult.getName()),
332                                     moduleResult,
333                                     result);
334                         }
335                     }
336                 }
337             } else {
338                 // Exclude the module completely - it will keep its current status
339                 addExcludeToConfig(suite, moduleResult, null);
340                 replayer.addToReplay(
341                         results.getModuleContextForRunResult(moduleResult.getName()),
342                         moduleResult,
343                         null);
344             }
345         }
346     }
347 
348     /** Update the configuration to put the replayer before all the actual real tests. */
updateConfiguration(IConfiguration config, ResultsPlayer replayer)349     private void updateConfiguration(IConfiguration config, ResultsPlayer replayer) {
350         List<IRemoteTest> tests = config.getTests();
351         List<IRemoteTest> newList = new ArrayList<>();
352         // Add the replayer first to replay all the tests cases first.
353         newList.add(replayer);
354         newList.addAll(tests);
355         config.setTests(newList);
356     }
357 
358     /** Allow the specialized loader to customize the config before re-running it. */
customizeConfig(ITestSuiteResultLoader loader, IConfiguration originalConfig)359     private void customizeConfig(ITestSuiteResultLoader loader, IConfiguration originalConfig) {
360         loader.customizeConfiguration(originalConfig);
361     }
362 
363     /** Add the filter to the suite. */
addExcludeToConfig( BaseTestSuite suite, TestRunResult moduleResult, String testDescription)364     private void addExcludeToConfig(
365             BaseTestSuite suite, TestRunResult moduleResult, String testDescription) {
366         String filter = moduleResult.getName();
367         if (testDescription != null) {
368             filter = String.format("%s %s", filter, testDescription);
369         }
370         SuiteTestFilter testFilter = SuiteTestFilter.createFrom(filter);
371         Set<String> excludeFilter = new LinkedHashSet<>();
372         excludeFilter.add(testFilter.toString());
373         suite.setExcludeFilter(excludeFilter);
374     }
375 
376     /** Returns True if a test case is a parameterized one. */
isParameterized(TestDescription description)377     private boolean isParameterized(TestDescription description) {
378         return !description.getTestName().equals(description.getTestNameWithoutParams());
379     }
380 
sortMethodToClass( Map<TestDescription, TestResult> paramMethods)381     private Map<String, Map<TestDescription, TestResult>> sortMethodToClass(
382             Map<TestDescription, TestResult> paramMethods) {
383         Map<String, Map<TestDescription, TestResult>> returnMap = new LinkedHashMap<>();
384         for (Entry<TestDescription, TestResult> entry : paramMethods.entrySet()) {
385             String noParamName =
386                     String.format(
387                             "%s#%s",
388                             entry.getKey().getClassName(),
389                             entry.getKey().getTestNameWithoutParams());
390             Map<TestDescription, TestResult> forClass = returnMap.get(noParamName);
391             if (forClass == null) {
392                 forClass = new LinkedHashMap<>();
393                 returnMap.put(noParamName, forClass);
394             }
395             forClass.put(entry.getKey(), entry.getValue());
396         }
397         return returnMap;
398     }
399 
400     /**
401      * Fetch additional result_reporter from the retry configuration and add them to the original
402      * command. This is the only allowed modification of the original command: add more result
403      * end-points.
404      */
handleExtraResultReporter( IConfiguration originalConfig, IConfiguration retryConfig)405     private void handleExtraResultReporter(
406             IConfiguration originalConfig, IConfiguration retryConfig) {
407         // Since we always have 1 default reporter, avoid carrying it for no reason. Only carry
408         // reporters if some actual ones were specified.
409         if (retryConfig.getTestInvocationListeners().size() == 1
410                 && (mConfiguration.getTestInvocationListeners().get(0)
411                         instanceof TextResultReporter)) {
412             return;
413         }
414         List<ITestInvocationListener> listeners = originalConfig.getTestInvocationListeners();
415         listeners.addAll(retryConfig.getTestInvocationListeners());
416         originalConfig.setTestInvocationListeners(listeners);
417     }
418 }
419