1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.media.tests; 18 19 import com.android.ddmlib.IDevice; 20 import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; 21 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.device.DeviceNotAvailableException; 24 import com.android.tradefed.device.ITestDevice; 25 import com.android.tradefed.log.LogUtil.CLog; 26 import com.android.tradefed.result.CollectingTestListener; 27 import com.android.tradefed.result.FileInputStreamSource; 28 import com.android.tradefed.result.ITestInvocationListener; 29 import com.android.tradefed.result.InputStreamSource; 30 import com.android.tradefed.result.LogDataType; 31 import com.android.tradefed.testtype.IDeviceTest; 32 import com.android.tradefed.testtype.IRemoteTest; 33 import com.android.tradefed.util.FileUtil; 34 import com.android.tradefed.util.RegexTrie; 35 import com.android.tradefed.util.StreamUtil; 36 import com.android.tradefed.util.proto.TfMetricProtoUtil; 37 38 import junit.framework.TestCase; 39 40 import org.junit.Assert; 41 42 import java.io.ByteArrayInputStream; 43 import java.io.File; 44 import java.io.FileInputStream; 45 import java.io.IOException; 46 import java.io.InputStream; 47 import java.util.ArrayList; 48 import java.util.Arrays; 49 import java.util.HashMap; 50 import java.util.List; 51 import java.util.ListIterator; 52 import java.util.Map; 53 import java.util.concurrent.TimeUnit; 54 55 /** 56 * Runs the Camera latency testcases. 57 * FIXME: more details 58 * <p/> 59 * Note that this test will not run properly unless /sdcard is mounted and writable. 60 */ 61 public class CameraLatencyTest implements IDeviceTest, IRemoteTest { 62 ITestDevice mTestDevice = null; 63 64 // Constants for running the tests 65 private static final String TEST_PACKAGE_NAME = "com.google.android.camera.tests"; 66 67 private final String mOutputPath = "mediaStressOut.txt"; 68 69 //Max timeout for the test - 30 mins 70 private static final int MAX_TEST_TIMEOUT = 30 * 60 * 1000; 71 72 /** 73 * Stores the test cases that we should consider running. 74 * 75 * <p>This currently consists of "startup" and "latency" 76 */ 77 private List<TestInfo> mTestCases = new ArrayList<>(); 78 79 // Options for the running the gCam test 80 @Option(name = "gCam", description = "Run gCam startup test") 81 private boolean mGcam = false; 82 83 84 /** 85 * A struct that contains useful info about the tests to run 86 */ 87 static class TestInfo { 88 public String mTestName = null; 89 public String mClassName = null; 90 public String mTestMetricsName = null; 91 public RegexTrie<String> mPatternMap = new RegexTrie<>(); 92 93 @Override toString()94 public String toString() { 95 return String.format("TestInfo: name(%s) class(%s) metric(%s) patterns(%s)", mTestName, 96 mClassName, mTestMetricsName, mPatternMap); 97 } 98 } 99 100 /** 101 * Set up the configurations for the test cases we want to run 102 */ testInfoSetup()103 private void testInfoSetup() { 104 // Startup tests 105 TestInfo t = new TestInfo(); 106 107 if (mGcam) { 108 t.mTestName = "testLaunchCamera"; 109 t.mClassName = "com.android.camera.stress.CameraStartUp"; 110 t.mTestMetricsName = "GCameraStartup"; 111 RegexTrie<String> map = t.mPatternMap; 112 map = t.mPatternMap; 113 map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)"); 114 map.put("CameraStartup", "^Camera average startup time: (\\d+) ms"); 115 mTestCases.add(t); 116 } else { 117 t.mTestName = "startup"; 118 t.mClassName = "com.android.camera.stress.CameraStartUp"; 119 t.mTestMetricsName = "CameraVideoRecorderStartup"; 120 RegexTrie<String> map = t.mPatternMap; 121 map = t.mPatternMap; 122 map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)"); 123 map.put("CameraStartup", "^Camera average startup time: (\\d+) ms"); 124 map.put("FirstVideoStartup", "^First Video Startup: (\\d+)"); 125 map.put("VideoStartup", "^Video average startup time: (\\d+) ms"); 126 mTestCases.add(t); 127 128 // Latency tests 129 t = new TestInfo(); 130 t.mTestName = "latency"; 131 t.mClassName = "com.android.camera.stress.CameraLatency"; 132 t.mTestMetricsName = "CameraLatency"; 133 map = t.mPatternMap; 134 map.put("AutoFocus", "^Avg AutoFocus = (\\d+)"); 135 map.put("ShutterLag", "^Avg mShutterLag = (\\d+)"); 136 map.put("Preview", "^Avg mShutterToPictureDisplayedTime = (\\d+)"); 137 map.put("RawPictureGeneration", "^Avg mPictureDisplayedToJpegCallbackTime = (\\d+)"); 138 map.put("GenTimeDiffOverJPEGAndRaw", "^Avg mJpegCallbackFinishTime = (\\d+)"); 139 map.put("FirstPreviewTime", "^Avg FirstPreviewTime = (\\d+)"); 140 mTestCases.add(t); 141 } 142 143 } 144 145 @Override run(ITestInvocationListener listener)146 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 147 Assert.assertNotNull(mTestDevice); 148 testInfoSetup(); 149 for (TestInfo test : mTestCases) { 150 cleanTmpFiles(); 151 executeTest(test, listener); 152 logOutputFile(test, listener); 153 } 154 155 cleanTmpFiles(); 156 } 157 executeTest(TestInfo test, ITestInvocationListener listener)158 private void executeTest(TestInfo test, ITestInvocationListener listener) 159 throws DeviceNotAvailableException { 160 IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME, 161 mTestDevice.getIDevice()); 162 CollectingTestListener auxListener = new CollectingTestListener(); 163 164 runner.setClassName(test.mClassName); 165 runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS); 166 if (mGcam) { 167 runner.setMethodName(test.mClassName, test.mTestName); 168 } 169 mTestDevice.runInstrumentationTests(runner, listener, auxListener); 170 171 // Grab a bugreport if warranted 172 if (auxListener.hasFailedTests()) { 173 CLog.i("Grabbing bugreport after test '%s' finished with %d failures.", 174 test.mTestName, auxListener.getNumAllFailedTests()); 175 InputStreamSource bugreport = mTestDevice.getBugreport(); 176 listener.testLog(String.format("bugreport-%s.txt", test.mTestName), 177 LogDataType.BUGREPORT, bugreport); 178 bugreport.close(); 179 } 180 } 181 182 /** 183 * Clean up temp files from test runs 184 * <p /> 185 * Note that all photos on the test device will be removed 186 */ cleanTmpFiles()187 private void cleanTmpFiles() throws DeviceNotAvailableException { 188 String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); 189 //TODO: Remove the DCIM folder when the bug is fixed. 190 mTestDevice.executeShellCommand(String.format("rm %s/DCIM/Camera/*", extStore)); 191 mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath)); 192 } 193 194 /** 195 * Pull the output file from the device, add it to the logs, and also parse out the relevant 196 * test metrics and report them. 197 */ logOutputFile(TestInfo test, ITestInvocationListener listener)198 private void logOutputFile(TestInfo test, ITestInvocationListener listener) 199 throws DeviceNotAvailableException { 200 File outputFile = null; 201 InputStreamSource outputSource = null; 202 try { 203 outputFile = mTestDevice.pullFileFromExternal(mOutputPath); 204 205 if (outputFile == null) { 206 return; 207 } 208 209 // Upload a verbatim copy of the output file 210 CLog.d("Sending %d byte file %s into the logosphere!", outputFile.length(), outputFile); 211 outputSource = new FileInputStreamSource(outputFile); 212 listener.testLog(String.format("output-%s.txt", test.mTestName), LogDataType.TEXT, 213 outputSource); 214 215 // Parse the output file to upload aggregated metrics 216 parseOutputFile(test, new FileInputStream(outputFile), listener); 217 } catch (IOException e) { 218 CLog.e("IOException while reading or parsing output file"); 219 CLog.e(e); 220 } finally { 221 FileUtil.deleteFile(outputFile); 222 StreamUtil.cancel(outputSource); 223 } 224 } 225 226 /** 227 * Parse the relevant metrics from the Instrumentation test output file 228 */ parseOutputFile(TestInfo test, InputStream dataStream, ITestInvocationListener listener)229 private void parseOutputFile(TestInfo test, InputStream dataStream, 230 ITestInvocationListener listener) { 231 Map<String, String> runMetrics = new HashMap<>(); 232 233 // try to parse it 234 String contents; 235 try { 236 contents = StreamUtil.getStringFromStream(dataStream); 237 } catch (IOException e) { 238 CLog.e("Got IOException during %s test processing", test.mTestName); 239 CLog.e(e); 240 return; 241 } 242 243 List<String> lines = Arrays.asList(contents.split("\n")); 244 ListIterator<String> lineIter = lines.listIterator(); 245 String line; 246 while (lineIter.hasNext()) { 247 line = lineIter.next(); 248 List<List<String>> capture = new ArrayList<>(1); 249 String key = test.mPatternMap.retrieve(capture, line); 250 if (key != null) { 251 CLog.d("Got %s key '%s' and captures '%s'", test.mTestName, key, 252 capture.toString()); 253 } else if (line.isEmpty()) { 254 // ignore 255 continue; 256 } else { 257 CLog.d("Got unmatched line: %s", line); 258 continue; 259 } 260 261 runMetrics.put(key, capture.get(0).get(0)); 262 } 263 264 reportMetrics(listener, test, runMetrics); 265 } 266 267 /** 268 * Report run metrics by creating an empty test run to stick them in 269 * <p /> 270 * Exposed for unit testing 271 */ reportMetrics(ITestInvocationListener listener, TestInfo test, Map<String, String> metrics)272 void reportMetrics(ITestInvocationListener listener, TestInfo test, 273 Map<String, String> metrics) { 274 // Create an empty testRun to report the parsed runMetrics 275 CLog.d("About to report metrics for %s: %s", test.mTestMetricsName, metrics); 276 listener.testRunStarted(test.mTestMetricsName, 0); 277 listener.testRunEnded(0, TfMetricProtoUtil.upgradeConvert(metrics)); 278 } 279 280 @Override setDevice(ITestDevice device)281 public void setDevice(ITestDevice device) { 282 mTestDevice = device; 283 } 284 285 @Override getDevice()286 public ITestDevice getDevice() { 287 return mTestDevice; 288 } 289 290 /** 291 * A meta-test to ensure that bits of the CameraLatencyTest are working properly 292 */ 293 public static class MetaTest extends TestCase { 294 private CameraLatencyTest mTestInstance = null; 295 296 private TestInfo mTestInfo = null; 297 298 private TestInfo mReportedTestInfo = null; 299 private Map<String, String> mReportedMetrics = null; 300 join(String... pieces)301 private static String join(String... pieces) { 302 StringBuilder sb = new StringBuilder(); 303 for (String piece : pieces) { 304 sb.append(piece); 305 sb.append("\n"); 306 } 307 return sb.toString(); 308 } 309 310 @Override setUp()311 public void setUp() throws Exception { 312 mTestInstance = new CameraLatencyTest() { 313 @Override 314 void reportMetrics(ITestInvocationListener l, TestInfo test, 315 Map<String, String> metrics) { 316 mReportedTestInfo = test; 317 mReportedMetrics = metrics; 318 } 319 }; 320 321 // Startup tests 322 mTestInfo = new TestInfo(); 323 TestInfo t = mTestInfo; // convenience alias 324 t.mTestName = "startup"; 325 t.mClassName = "com.android.camera.stress.CameraStartUp"; 326 t.mTestMetricsName = "camera_video_recorder_startup"; 327 RegexTrie<String> map = t.mPatternMap; 328 map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)"); 329 map.put("CameraStartup", "^Camera average startup time: (\\d+) ms"); 330 map.put("FirstVideoStartup", "^First Video Startup: (\\d+)"); 331 map.put("VideoStartup", "^Video average startup time: (\\d+) ms"); 332 map.put("FirstPreviewTime", "^Avg FirstPreviewTime = (\\d+)"); 333 } 334 335 /** 336 * Make sure that parsing works in the expected case 337 */ testParse()338 public void testParse() throws Exception { 339 String output = join( 340 "First Camera Startup: 1421", /* "FirstCameraStartup" key */ 341 "Camerastartup time: ", 342 "Number of loop: 19", 343 "Individual Camera Startup Time = 1920 ,1929 ,1924 ,1871 ,1840 ,1892 ,1813 " + 344 ",1927 ,1856 ,1929 ,1911 ,1873 ,1381 ,1888 ,2405 ,1926 ,1840 ,2502 " + 345 ",2357 ,", 346 "", 347 "Camera average startup time: 1946 ms", /* "CameraStartup" key */ 348 "", 349 "First Video Startup: 2176", /* "FirstVideoStartup" key */ 350 "Camera Latency : ", 351 "Number of loop: 20", 352 "Avg AutoFocus = 2304", 353 "Avg mShutterLag = 403", 354 "Avg mShutterToPictureDisplayedTime = 369", 355 "Avg mPictureDisplayedToJpegCallbackTime = 50", 356 "Avg mJpegCallbackFinishTime = 1679", 357 "Avg FirstPreviewTime = 1340"); 358 359 InputStream iStream = new ByteArrayInputStream(output.getBytes()); 360 mTestInstance.parseOutputFile(mTestInfo, iStream, null); 361 assertEquals(mTestInfo, mReportedTestInfo); 362 assertNotNull(mReportedMetrics); 363 assertEquals(4, mReportedMetrics.size()); 364 assertEquals("1946", mReportedMetrics.get("CameraStartup")); 365 assertEquals("2176", mReportedMetrics.get("FirstVideoStartup")); 366 assertEquals("1421", mReportedMetrics.get("FirstCameraStartup")); 367 assertEquals("1340", mReportedMetrics.get("FirstPreviewTime")); 368 } 369 } 370 } 371