/* * Copyright (C) 2011 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.media.tests; import com.android.ddmlib.IDevice; import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; import com.android.tradefed.config.Option; import com.android.tradefed.device.DeviceNotAvailableException; import com.android.tradefed.device.ITestDevice; import com.android.tradefed.log.LogUtil.CLog; import com.android.tradefed.result.CollectingTestListener; import com.android.tradefed.result.FileInputStreamSource; import com.android.tradefed.result.ITestInvocationListener; import com.android.tradefed.result.InputStreamSource; import com.android.tradefed.result.LogDataType; import com.android.tradefed.testtype.IDeviceTest; import com.android.tradefed.testtype.IRemoteTest; import com.android.tradefed.util.FileUtil; import com.android.tradefed.util.RegexTrie; import com.android.tradefed.util.StreamUtil; import com.android.tradefed.util.proto.TfMetricProtoUtil; import junit.framework.TestCase; import org.junit.Assert; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.ListIterator; import java.util.Map; import java.util.concurrent.TimeUnit; /** * Runs the Camera latency testcases. * FIXME: more details *

* Note that this test will not run properly unless /sdcard is mounted and writable. */ public class CameraLatencyTest implements IDeviceTest, IRemoteTest { ITestDevice mTestDevice = null; // Constants for running the tests private static final String TEST_PACKAGE_NAME = "com.google.android.camera.tests"; private final String mOutputPath = "mediaStressOut.txt"; //Max timeout for the test - 30 mins private static final int MAX_TEST_TIMEOUT = 30 * 60 * 1000; /** * Stores the test cases that we should consider running. * *

This currently consists of "startup" and "latency" */ private List mTestCases = new ArrayList<>(); // Options for the running the gCam test @Option(name = "gCam", description = "Run gCam startup test") private boolean mGcam = false; /** * A struct that contains useful info about the tests to run */ static class TestInfo { public String mTestName = null; public String mClassName = null; public String mTestMetricsName = null; public RegexTrie mPatternMap = new RegexTrie<>(); @Override public String toString() { return String.format("TestInfo: name(%s) class(%s) metric(%s) patterns(%s)", mTestName, mClassName, mTestMetricsName, mPatternMap); } } /** * Set up the configurations for the test cases we want to run */ private void testInfoSetup() { // Startup tests TestInfo t = new TestInfo(); if (mGcam) { t.mTestName = "testLaunchCamera"; t.mClassName = "com.android.camera.stress.CameraStartUp"; t.mTestMetricsName = "GCameraStartup"; RegexTrie map = t.mPatternMap; map = t.mPatternMap; map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)"); map.put("CameraStartup", "^Camera average startup time: (\\d+) ms"); mTestCases.add(t); } else { t.mTestName = "startup"; t.mClassName = "com.android.camera.stress.CameraStartUp"; t.mTestMetricsName = "CameraVideoRecorderStartup"; RegexTrie map = t.mPatternMap; map = t.mPatternMap; map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)"); map.put("CameraStartup", "^Camera average startup time: (\\d+) ms"); map.put("FirstVideoStartup", "^First Video Startup: (\\d+)"); map.put("VideoStartup", "^Video average startup time: (\\d+) ms"); mTestCases.add(t); // Latency tests t = new TestInfo(); t.mTestName = "latency"; t.mClassName = "com.android.camera.stress.CameraLatency"; t.mTestMetricsName = "CameraLatency"; map = t.mPatternMap; map.put("AutoFocus", "^Avg AutoFocus = (\\d+)"); map.put("ShutterLag", "^Avg mShutterLag = (\\d+)"); map.put("Preview", "^Avg mShutterToPictureDisplayedTime = (\\d+)"); map.put("RawPictureGeneration", "^Avg mPictureDisplayedToJpegCallbackTime = (\\d+)"); map.put("GenTimeDiffOverJPEGAndRaw", "^Avg mJpegCallbackFinishTime = (\\d+)"); map.put("FirstPreviewTime", "^Avg FirstPreviewTime = (\\d+)"); mTestCases.add(t); } } @Override public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { Assert.assertNotNull(mTestDevice); testInfoSetup(); for (TestInfo test : mTestCases) { cleanTmpFiles(); executeTest(test, listener); logOutputFile(test, listener); } cleanTmpFiles(); } private void executeTest(TestInfo test, ITestInvocationListener listener) throws DeviceNotAvailableException { IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME, mTestDevice.getIDevice()); CollectingTestListener auxListener = new CollectingTestListener(); runner.setClassName(test.mClassName); runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS); if (mGcam) { runner.setMethodName(test.mClassName, test.mTestName); } mTestDevice.runInstrumentationTests(runner, listener, auxListener); // Grab a bugreport if warranted if (auxListener.hasFailedTests()) { CLog.i("Grabbing bugreport after test '%s' finished with %d failures.", test.mTestName, auxListener.getNumAllFailedTests()); InputStreamSource bugreport = mTestDevice.getBugreport(); listener.testLog(String.format("bugreport-%s.txt", test.mTestName), LogDataType.BUGREPORT, bugreport); bugreport.close(); } } /** * Clean up temp files from test runs *

* Note that all photos on the test device will be removed */ private void cleanTmpFiles() throws DeviceNotAvailableException { String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); //TODO: Remove the DCIM folder when the bug is fixed. mTestDevice.executeShellCommand(String.format("rm %s/DCIM/Camera/*", extStore)); mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath)); } /** * Pull the output file from the device, add it to the logs, and also parse out the relevant * test metrics and report them. */ private void logOutputFile(TestInfo test, ITestInvocationListener listener) throws DeviceNotAvailableException { File outputFile = null; InputStreamSource outputSource = null; try { outputFile = mTestDevice.pullFileFromExternal(mOutputPath); if (outputFile == null) { return; } // Upload a verbatim copy of the output file CLog.d("Sending %d byte file %s into the logosphere!", outputFile.length(), outputFile); outputSource = new FileInputStreamSource(outputFile); listener.testLog(String.format("output-%s.txt", test.mTestName), LogDataType.TEXT, outputSource); // Parse the output file to upload aggregated metrics parseOutputFile(test, new FileInputStream(outputFile), listener); } catch (IOException e) { CLog.e("IOException while reading or parsing output file"); CLog.e(e); } finally { FileUtil.deleteFile(outputFile); StreamUtil.cancel(outputSource); } } /** * Parse the relevant metrics from the Instrumentation test output file */ private void parseOutputFile(TestInfo test, InputStream dataStream, ITestInvocationListener listener) { Map runMetrics = new HashMap<>(); // try to parse it String contents; try { contents = StreamUtil.getStringFromStream(dataStream); } catch (IOException e) { CLog.e("Got IOException during %s test processing", test.mTestName); CLog.e(e); return; } List lines = Arrays.asList(contents.split("\n")); ListIterator lineIter = lines.listIterator(); String line; while (lineIter.hasNext()) { line = lineIter.next(); List> capture = new ArrayList<>(1); String key = test.mPatternMap.retrieve(capture, line); if (key != null) { CLog.d("Got %s key '%s' and captures '%s'", test.mTestName, key, capture.toString()); } else if (line.isEmpty()) { // ignore continue; } else { CLog.d("Got unmatched line: %s", line); continue; } runMetrics.put(key, capture.get(0).get(0)); } reportMetrics(listener, test, runMetrics); } /** * Report run metrics by creating an empty test run to stick them in *

* Exposed for unit testing */ void reportMetrics(ITestInvocationListener listener, TestInfo test, Map metrics) { // Create an empty testRun to report the parsed runMetrics CLog.d("About to report metrics for %s: %s", test.mTestMetricsName, metrics); listener.testRunStarted(test.mTestMetricsName, 0); listener.testRunEnded(0, TfMetricProtoUtil.upgradeConvert(metrics)); } @Override public void setDevice(ITestDevice device) { mTestDevice = device; } @Override public ITestDevice getDevice() { return mTestDevice; } /** * A meta-test to ensure that bits of the CameraLatencyTest are working properly */ public static class MetaTest extends TestCase { private CameraLatencyTest mTestInstance = null; private TestInfo mTestInfo = null; private TestInfo mReportedTestInfo = null; private Map mReportedMetrics = null; private static String join(String... pieces) { StringBuilder sb = new StringBuilder(); for (String piece : pieces) { sb.append(piece); sb.append("\n"); } return sb.toString(); } @Override public void setUp() throws Exception { mTestInstance = new CameraLatencyTest() { @Override void reportMetrics(ITestInvocationListener l, TestInfo test, Map metrics) { mReportedTestInfo = test; mReportedMetrics = metrics; } }; // Startup tests mTestInfo = new TestInfo(); TestInfo t = mTestInfo; // convenience alias t.mTestName = "startup"; t.mClassName = "com.android.camera.stress.CameraStartUp"; t.mTestMetricsName = "camera_video_recorder_startup"; RegexTrie map = t.mPatternMap; map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)"); map.put("CameraStartup", "^Camera average startup time: (\\d+) ms"); map.put("FirstVideoStartup", "^First Video Startup: (\\d+)"); map.put("VideoStartup", "^Video average startup time: (\\d+) ms"); map.put("FirstPreviewTime", "^Avg FirstPreviewTime = (\\d+)"); } /** * Make sure that parsing works in the expected case */ public void testParse() throws Exception { String output = join( "First Camera Startup: 1421", /* "FirstCameraStartup" key */ "Camerastartup time: ", "Number of loop: 19", "Individual Camera Startup Time = 1920 ,1929 ,1924 ,1871 ,1840 ,1892 ,1813 " + ",1927 ,1856 ,1929 ,1911 ,1873 ,1381 ,1888 ,2405 ,1926 ,1840 ,2502 " + ",2357 ,", "", "Camera average startup time: 1946 ms", /* "CameraStartup" key */ "", "First Video Startup: 2176", /* "FirstVideoStartup" key */ "Camera Latency : ", "Number of loop: 20", "Avg AutoFocus = 2304", "Avg mShutterLag = 403", "Avg mShutterToPictureDisplayedTime = 369", "Avg mPictureDisplayedToJpegCallbackTime = 50", "Avg mJpegCallbackFinishTime = 1679", "Avg FirstPreviewTime = 1340"); InputStream iStream = new ByteArrayInputStream(output.getBytes()); mTestInstance.parseOutputFile(mTestInfo, iStream, null); assertEquals(mTestInfo, mReportedTestInfo); assertNotNull(mReportedMetrics); assertEquals(4, mReportedMetrics.size()); assertEquals("1946", mReportedMetrics.get("CameraStartup")); assertEquals("2176", mReportedMetrics.get("FirstVideoStartup")); assertEquals("1421", mReportedMetrics.get("FirstCameraStartup")); assertEquals("1340", mReportedMetrics.get("FirstPreviewTime")); } } }