1#!/usr/bin/python
2"""Compare failed tests in CTS/VTS test_result.xml.
3
4Given two test_result.xml's (A and B), this script lists all failed tests in A,
5and shows result of the same test in B.
6"""
7
8import argparse
9import collections
10import csv
11import xml.etree.ElementTree as ET
12
13
14PASS = 'pass'
15FAIL = 'fail'
16NO_DATA = 'no_data'
17
18ATTRS_TO_SHOW = ['Result::Build.build_model',
19                 'Result::Build.build_id',
20                 'Result.suite_name',
21                 'Result.suite_plan',
22                 'Result.suite_build_number',
23                 'Result.start_display',
24                 'Result::Build.build_abis_32',
25                 'Result::Build.build_abis_64',]
26
27
28def parse_attrib_path(attrib_path):
29  first_dot = attrib_path.index('.')
30  tags = attrib_path[:first_dot].split('::')
31  attr_name = attrib_path[first_dot+1:]
32  return tags, attr_name
33
34
35def get_test_info(root):
36  """Get test info from test_result.xml."""
37
38  test_info = collections.OrderedDict()
39
40  for attrib_path in ATTRS_TO_SHOW:
41    tags, attr_name = parse_attrib_path(attrib_path)
42    node = root
43
44    while True:
45      tags = tags[1:]
46      if tags:
47        node = node.find(tags[0])
48      else:
49        break
50
51    test_info[attr_name] = node.attrib[attr_name]
52
53  return test_info
54
55
56def print_test_infos(test_result_a, test_result_b):
57  """Print test infomation of both results in table format."""
58
59  info_a = test_result_a['info']
60  info_b = test_result_b['info']
61
62  max_key_len = max([len(k) for k in info_a])
63  max_value_a_len = max([len(info_a[k]) for k in info_a])
64  max_value_b_len = max([len(info_b[k]) for k in info_b])
65  table_len = (max_key_len + 2 + max_value_a_len + 2 + max_value_b_len)
66
67  line_format = '{:{}}  {:{}}  {}'
68
69  print '=' * table_len
70
71  for key in info_a:
72    print line_format.format(key, max_key_len,
73                             info_a[key], max_value_a_len,
74                             info_b[key])
75
76  print '=' * table_len
77  print
78
79
80def get_result(test_result, module_name, testcase_name, test_name):
81  """Get result of specifc module, testcase and test name."""
82
83  modules = test_result['modules']
84  if module_name not in modules:
85    return NO_DATA
86
87  testcases = modules[module_name]
88  if testcase_name not in testcases:
89    return NO_DATA
90
91  tests = testcases[testcase_name]
92  if test_name not in tests:
93    return NO_DATA
94
95  return ', '.join([x + ': ' + y for x, y in tests[test_name].items()])
96
97
98def read_test_result_xml(test_result_path):
99  """Given the path to a test_result.xml, read that into a ordered dict."""
100
101  tree = ET.parse(test_result_path)
102  root = tree.getroot()
103
104  test_result = collections.OrderedDict()
105  test_result['info'] = get_test_info(root)
106
107  modules = collections.OrderedDict()
108  test_result['modules'] = modules
109
110  for module in root.iter('Module'):
111    abi = module.attrib['abi']
112
113    module_name = module.attrib['name']
114
115    if not module_name in modules:
116      modules[module_name] = collections.OrderedDict()
117
118    testcases = modules[module_name]
119
120    for testcase in module.iter('TestCase'):
121      testcase_name = testcase.attrib['name']
122
123      if not testcase_name in testcases:
124        testcases[testcase_name] = collections.OrderedDict()
125
126      tests = testcases[testcase_name]
127
128      for test in testcase.iter('Test'):
129        test_name = test.attrib['name']
130
131        if not test_name in tests:
132          tests[test_name] = collections.OrderedDict()
133
134        if abi in tests[test_name]:
135          print '[WARNING] duplicated test:', test_name
136
137        tests[test_name][abi] = test.attrib['result']
138
139  return test_result
140
141
142def compare_failed_tests(test_result_a, test_result_b, csvfile):
143  """Do the comparison.
144
145  Given two test result dicts (A and B), list all failed test in A and display
146  result of the same test in B.
147
148  Args:
149    test_result_a: the dict returned from read_test_result(test_result_a.xml)
150    test_result_b: the dict returned from read_test_result(test_result_b.xml)
151    csvfile: a opened file
152
153  Returns:
154    string: diff report, summary
155  """
156
157  writer = csv.writer(csvfile)
158  writer.writerow(['module', 'testcase', 'test', 'result in B'])
159
160  summary = ''
161
162  modules = test_result_a['modules']
163
164  for module_name, testcases in modules.iteritems():
165    module_sub_summary = ''
166
167    for testcase_name, tests in testcases.iteritems():
168      testcase_sub_summary = ''
169
170      for test_name, result in tests.iteritems():
171        if FAIL in result.values():
172          result_b = get_result(
173              test_result_b, module_name, testcase_name, test_name)
174
175          testcase_sub_summary += '    ' + test_name + ': ' + result_b + '\n'
176          writer.writerow([module_name, testcase_name, test_name, result_b])
177
178      if testcase_sub_summary:
179        module_sub_summary = '  ' + testcase_name + '\n' + testcase_sub_summary
180
181    if module_sub_summary:
182      summary += module_name + '\n' + module_sub_summary + '\n'
183
184  return summary
185
186
187def main():
188  parser = argparse.ArgumentParser()
189
190  parser.add_argument('test_result_a', help='path to first test_result.xml')
191  parser.add_argument('test_result_b', help='path to second test_result.xml')
192  parser.add_argument('--csv', default='diff.csv', help='path to csv output')
193
194  args = parser.parse_args()
195
196  test_result_a = read_test_result_xml(args.test_result_a)
197  test_result_b = read_test_result_xml(args.test_result_b)
198
199  print_test_infos(test_result_a, test_result_b)
200
201  with open(args.csv, 'w') as csvfile:
202    summary = compare_failed_tests(test_result_a, test_result_b, csvfile)
203
204    print summary
205
206
207if __name__ == '__main__':
208  main()
209