1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.tradefed.result;
18 
19 import com.android.ddmlib.Log;
20 import com.android.ddmlib.Log.LogLevel;
21 import com.android.ddmlib.testrunner.TestResult.TestStatus;
22 import com.android.tradefed.config.Option;
23 import com.android.tradefed.config.OptionClass;
24 import com.android.tradefed.util.FileUtil;
25 import com.android.tradefed.util.StreamUtil;
26 
27 import com.google.common.annotations.VisibleForTesting;
28 
29 import org.kxml2.io.KXmlSerializer;
30 
31 import java.io.BufferedOutputStream;
32 import java.io.File;
33 import java.io.FileOutputStream;
34 import java.io.IOException;
35 import java.io.OutputStream;
36 import java.text.SimpleDateFormat;
37 import java.util.Date;
38 import java.util.Locale;
39 import java.util.Map;
40 import java.util.TimeZone;
41 
42 /**
43  * MetricsXMLResultReporter writes test metrics and run metrics to an XML file in a folder specified
44  * by metrics-folder parameter at the invocationEnded phase of the test. The XML file will be piped
45  * into an algorithm to detect regression.
46  *
47  * <p>All k-v paris in run metrics map will be formatted into: <runmetric name="name" value="value"
48  * /> and placed under <testsuite/> tag
49  *
50  * <p>All k-v paris in run metrics map will be formatted into: <testmetric name="name" value="value"
51  * /> and placed under <testcase/> tag, a tag nested under <testsuite/>.
52  *
53  * <p>A sample XML format: <testsuite name="suite" tests="1" failures="0" time="10"
54  * timestamp="2017-01-01T01:00:00"> <runmetric name="sample" value="1.0" /> <testcase
55  * testname="test" classname="classname" time="2"> <testmetric name="sample" value="1.0" />
56  * </testcase> </testsuite>
57  */
58 @OptionClass(alias = "metricsreporter")
59 public class MetricsXMLResultReporter extends CollectingTestListener {
60 
61     private static final String TAG = "MetricsXMLResultReporter";
62     private static final String METRICS_PREFIX = "metrics-";
63     private static final String TAG_TESTSUITE = "testsuite";
64     private static final String TAG_TESTCASE = "testcase";
65     private static final String TAG_RUN_METRIC = "runmetric";
66     private static final String TAG_TEST_METRIC = "testmetric";
67     private static final String ATTR_NAME = "name";
68     private static final String ATTR_VALUE = "value";
69     private static final String ATTR_TESTNAME = "testname";
70     private static final String ATTR_TIME = "time";
71     private static final String ATTR_FAILURES = "failures";
72     private static final String ATTR_TESTS = "tests";
73     private static final String ATTR_CLASSNAME = "classname";
74     private static final String ATTR_TIMESTAMP = "timestamp";
75     /** the XML namespace */
76     private static final String NS = null;
77 
78     @Option(name = "metrics-folder", description = "The folder to save metrics files")
79     private File mFolder;
80 
81     private File mLog;
82 
83     @Override
invocationEnded(long elapsedTime)84     public void invocationEnded(long elapsedTime) {
85         super.invocationEnded(elapsedTime);
86         if (mFolder == null) {
87             Log.w(TAG, "metrics-folder not specified, unable to record metrics");
88             return;
89         }
90         generateResults(elapsedTime);
91     }
92 
generateResults(long elapsedTime)93     private void generateResults(long elapsedTime) {
94         String timestamp = getTimeStamp();
95         OutputStream os = null;
96 
97         try {
98             os = createOutputStream();
99             if (os == null) {
100                 return;
101             }
102             KXmlSerializer serializer = new KXmlSerializer();
103             serializer.setOutput(os, "UTF-8");
104             serializer.startDocument("UTF-8", null);
105             serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
106             printRunResults(serializer, timestamp, elapsedTime);
107             serializer.endDocument();
108             if (mLog != null) {
109                 String msg =
110                         String.format(
111                                 Locale.US,
112                                 "XML metrics report generated at %s. "
113                                         + "Total tests %d, Failed %d",
114                                 mLog.getPath(),
115                                 getNumTotalTests(),
116                                 getNumAllFailedTests());
117                 Log.logAndDisplay(LogLevel.INFO, TAG, msg);
118             }
119         } catch (IOException e) {
120             Log.e(TAG, "Failed to generate XML metric report");
121             throw new RuntimeException(e);
122         } finally {
123             StreamUtil.close(os);
124         }
125     }
126 
printRunResults(KXmlSerializer serializer, String timestamp, long elapsedTime)127     private void printRunResults(KXmlSerializer serializer, String timestamp, long elapsedTime)
128             throws IOException {
129         serializer.startTag(NS, TAG_TESTSUITE);
130         serializer.attribute(NS, ATTR_NAME, getInvocationContext().getTestTag());
131         serializer.attribute(NS, ATTR_TESTS, Integer.toString(getNumTotalTests()));
132         serializer.attribute(NS, ATTR_FAILURES, Integer.toString(getNumAllFailedTests()));
133         serializer.attribute(NS, ATTR_TIME, Long.toString(elapsedTime));
134         serializer.attribute(NS, ATTR_TIMESTAMP, timestamp);
135 
136         for (TestRunResult runResult : getMergedTestRunResults()) {
137             printRunMetrics(serializer, runResult.getRunMetrics());
138             Map<TestDescription, TestResult> testResults = runResult.getTestResults();
139             for (TestDescription test : testResults.keySet()) {
140                 printTestResults(serializer, test, testResults.get(test));
141             }
142         }
143 
144         serializer.endTag(NS, TAG_TESTSUITE);
145     }
146 
printTestResults( KXmlSerializer serializer, TestDescription testId, TestResult testResult)147     private void printTestResults(
148             KXmlSerializer serializer, TestDescription testId, TestResult testResult)
149             throws IOException {
150         serializer.startTag(NS, TAG_TESTCASE);
151         serializer.attribute(NS, ATTR_TESTNAME, testId.getTestName());
152         serializer.attribute(NS, ATTR_CLASSNAME, testId.getClassName());
153         long elapsedTime = testResult.getEndTime() - testResult.getStartTime();
154         serializer.attribute(NS, ATTR_TIME, Long.toString(elapsedTime));
155 
156         printTestMetrics(serializer, testResult.getMetrics());
157 
158         if (!TestStatus.PASSED.equals(testResult.getStatus())) {
159             String result = testResult.getStatus().name();
160             serializer.startTag(NS, result);
161             String stackText = sanitize(testResult.getStackTrace());
162             serializer.text(stackText);
163             serializer.endTag(NS, result);
164         }
165 
166         serializer.endTag(NS, TAG_TESTCASE);
167     }
168 
printRunMetrics(KXmlSerializer serializer, Map<String, String> metrics)169     private void printRunMetrics(KXmlSerializer serializer, Map<String, String> metrics)
170             throws IOException {
171         for (String key : metrics.keySet()) {
172             serializer.startTag(NS, TAG_RUN_METRIC);
173             serializer.attribute(NS, ATTR_NAME, key);
174             serializer.attribute(NS, ATTR_VALUE, metrics.get(key));
175             serializer.endTag(NS, TAG_RUN_METRIC);
176         }
177     }
178 
printTestMetrics(KXmlSerializer serializer, Map<String, String> metrics)179     private void printTestMetrics(KXmlSerializer serializer, Map<String, String> metrics)
180             throws IOException {
181         for (String key : metrics.keySet()) {
182             serializer.startTag(NS, TAG_TEST_METRIC);
183             serializer.attribute(NS, ATTR_NAME, key);
184             serializer.attribute(NS, ATTR_VALUE, metrics.get(key));
185             serializer.endTag(NS, TAG_TEST_METRIC);
186         }
187     }
188 
189     @VisibleForTesting
createOutputStream()190     public OutputStream createOutputStream() throws IOException {
191         if (!mFolder.exists() && !mFolder.mkdirs()) {
192             throw new IOException(String.format("Unable to create metrics directory: %s", mFolder));
193         }
194         mLog = FileUtil.createTempFile(METRICS_PREFIX, ".xml", mFolder);
195         return new BufferedOutputStream(new FileOutputStream(mLog));
196     }
197 
198     /** Returns the text in a format that is safe for use in an XML document. */
sanitize(String text)199     private String sanitize(String text) {
200         return text.replace("\0", "<\\0>");
201     }
202 
203     /** Return the current timestamp as a {@link String}. */
204     @VisibleForTesting
getTimeStamp()205     public String getTimeStamp() {
206         SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
207         dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
208         dateFormat.setLenient(true);
209         return dateFormat.format(new Date());
210     }
211 }
212