1#!/usr/bin/env python3
2#
3# Copyright 2019, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Unittests for event_handler."""
18
19# pylint: disable=line-too-long
20
21import unittest
22
23from importlib import reload
24from unittest import mock
25
26from test_runners import atest_tf_test_runner as atf_tr
27from test_runners import event_handler as e_h
28from test_runners import test_runner_base
29
30
31EVENTS_NORMAL = [
32    ('TEST_MODULE_STARTED', {
33        'moduleContextFileName':'serial-util1146216{974}2772610436.ser',
34        'moduleName':'someTestModule'}),
35    ('TEST_RUN_STARTED', {'testCount': 2, 'runName': 'com.android.UnitTests'}),
36    ('TEST_STARTED', {'start_time':52, 'className':'someClassName',
37                      'testName':'someTestName'}),
38    ('TEST_ENDED', {'end_time':1048, 'className':'someClassName',
39                    'testName':'someTestName'}),
40    ('TEST_STARTED', {'start_time':48, 'className':'someClassName2',
41                      'testName':'someTestName2'}),
42    ('TEST_FAILED', {'className':'someClassName2', 'testName':'someTestName2',
43                     'trace': 'someTrace'}),
44    ('TEST_ENDED', {'end_time':9876450, 'className':'someClassName2',
45                    'testName':'someTestName2'}),
46    ('TEST_RUN_ENDED', {}),
47    ('TEST_MODULE_ENDED', {'foo': 'bar'}),
48]
49
50EVENTS_RUN_FAILURE = [
51    ('TEST_MODULE_STARTED', {
52        'moduleContextFileName': 'serial-util11462169742772610436.ser',
53        'moduleName': 'someTestModule'}),
54    ('TEST_RUN_STARTED', {'testCount': 2, 'runName': 'com.android.UnitTests'}),
55    ('TEST_STARTED', {'start_time':10, 'className': 'someClassName',
56                      'testName':'someTestName'}),
57    ('TEST_RUN_FAILED', {'reason': 'someRunFailureReason'})
58]
59
60
61EVENTS_INVOCATION_FAILURE = [
62    ('TEST_RUN_STARTED', {'testCount': None, 'runName': 'com.android.UnitTests'}),
63    ('INVOCATION_FAILED', {'cause': 'someInvocationFailureReason'})
64]
65
66EVENTS_MISSING_TEST_RUN_STARTED_EVENT = [
67    ('TEST_STARTED', {'start_time':52, 'className':'someClassName',
68                      'testName':'someTestName'}),
69    ('TEST_ENDED', {'end_time':1048, 'className':'someClassName',
70                    'testName':'someTestName'}),
71]
72
73EVENTS_NOT_BALANCED_BEFORE_RAISE = [
74    ('TEST_MODULE_STARTED', {
75        'moduleContextFileName':'serial-util1146216{974}2772610436.ser',
76        'moduleName':'someTestModule'}),
77    ('TEST_RUN_STARTED', {'testCount': 2, 'runName': 'com.android.UnitTests'}),
78    ('TEST_STARTED', {'start_time':10, 'className':'someClassName',
79                      'testName':'someTestName'}),
80    ('TEST_ENDED', {'end_time':18, 'className':'someClassName',
81                    'testName':'someTestName'}),
82    ('TEST_STARTED', {'start_time':19, 'className':'someClassName',
83                      'testName':'someTestName'}),
84    ('TEST_FAILED', {'className':'someClassName2', 'testName':'someTestName2',
85                     'trace': 'someTrace'}),
86]
87
88EVENTS_IGNORE = [
89    ('TEST_MODULE_STARTED', {
90        'moduleContextFileName':'serial-util1146216{974}2772610436.ser',
91        'moduleName':'someTestModule'}),
92    ('TEST_RUN_STARTED', {'testCount': 2, 'runName': 'com.android.UnitTests'}),
93    ('TEST_STARTED', {'start_time':8, 'className':'someClassName',
94                      'testName':'someTestName'}),
95    ('TEST_ENDED', {'end_time':18, 'className':'someClassName',
96                    'testName':'someTestName'}),
97    ('TEST_STARTED', {'start_time':28, 'className':'someClassName2',
98                      'testName':'someTestName2'}),
99    ('TEST_IGNORED', {'className':'someClassName2', 'testName':'someTestName2',
100                      'trace': 'someTrace'}),
101    ('TEST_ENDED', {'end_time':90, 'className':'someClassName2',
102                    'testName':'someTestName2'}),
103    ('TEST_RUN_ENDED', {}),
104    ('TEST_MODULE_ENDED', {'foo': 'bar'}),
105]
106
107EVENTS_WITH_PERF_INFO = [
108    ('TEST_MODULE_STARTED', {
109        'moduleContextFileName':'serial-util1146216{974}2772610436.ser',
110        'moduleName':'someTestModule'}),
111    ('TEST_RUN_STARTED', {'testCount': 2, 'runName': 'com.android.UnitTests'}),
112    ('TEST_STARTED', {'start_time':52, 'className':'someClassName',
113                      'testName':'someTestName'}),
114    ('TEST_ENDED', {'end_time':1048, 'className':'someClassName',
115                    'testName':'someTestName'}),
116    ('TEST_STARTED', {'start_time':48, 'className':'someClassName2',
117                      'testName':'someTestName2'}),
118    ('TEST_FAILED', {'className':'someClassName2', 'testName':'someTestName2',
119                     'trace': 'someTrace'}),
120    ('TEST_ENDED', {'end_time':9876450, 'className':'someClassName2',
121                    'testName':'someTestName2', 'cpu_time':'1234.1234(ns)',
122                    'real_time':'5678.5678(ns)', 'iterations':'6666'}),
123    ('TEST_STARTED', {'start_time':10, 'className':'someClassName3',
124                      'testName':'someTestName3'}),
125    ('TEST_ENDED', {'end_time':70, 'className':'someClassName3',
126                    'testName':'someTestName3', 'additional_info_min':'102773',
127                    'additional_info_mean':'105973', 'additional_info_median':'103778'}),
128    ('TEST_RUN_ENDED', {}),
129    ('TEST_MODULE_ENDED', {'foo': 'bar'}),
130]
131
132class EventHandlerUnittests(unittest.TestCase):
133    """Unit tests for event_handler.py"""
134
135    def setUp(self):
136        reload(e_h)
137        self.mock_reporter = mock.Mock()
138        self.fake_eh = e_h.EventHandler(self.mock_reporter,
139                                        atf_tr.AtestTradefedTestRunner.NAME)
140
141    def tearDown(self):
142        mock.patch.stopall()
143
144    def test_process_event_normal_results(self):
145        """Test process_event method for normal test results."""
146        for name, data in EVENTS_NORMAL:
147            self.fake_eh.process_event(name, data)
148        call1 = mock.call(test_runner_base.TestResult(
149            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
150            group_name='someTestModule',
151            test_name='someClassName#someTestName',
152            status=test_runner_base.PASSED_STATUS,
153            details=None,
154            test_count=1,
155            test_time='(996ms)',
156            runner_total=None,
157            group_total=2,
158            additional_info={},
159            test_run_name='com.android.UnitTests'
160        ))
161        call2 = mock.call(test_runner_base.TestResult(
162            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
163            group_name='someTestModule',
164            test_name='someClassName2#someTestName2',
165            status=test_runner_base.FAILED_STATUS,
166            details='someTrace',
167            test_count=2,
168            test_time='(2h44m36.402s)',
169            runner_total=None,
170            group_total=2,
171            additional_info={},
172            test_run_name='com.android.UnitTests'
173        ))
174        self.mock_reporter.process_test_result.assert_has_calls([call1, call2])
175
176    def test_process_event_run_failure(self):
177        """Test process_event method run failure."""
178        for name, data in EVENTS_RUN_FAILURE:
179            self.fake_eh.process_event(name, data)
180        call = mock.call(test_runner_base.TestResult(
181            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
182            group_name='someTestModule',
183            test_name='someClassName#someTestName',
184            status=test_runner_base.ERROR_STATUS,
185            details='someRunFailureReason',
186            test_count=1,
187            test_time='',
188            runner_total=None,
189            group_total=2,
190            additional_info={},
191            test_run_name='com.android.UnitTests'
192        ))
193        self.mock_reporter.process_test_result.assert_has_calls([call])
194
195    def test_process_event_invocation_failure(self):
196        """Test process_event method with invocation failure."""
197        for name, data in EVENTS_INVOCATION_FAILURE:
198            self.fake_eh.process_event(name, data)
199        call = mock.call(test_runner_base.TestResult(
200            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
201            group_name=None,
202            test_name=None,
203            status=test_runner_base.ERROR_STATUS,
204            details='someInvocationFailureReason',
205            test_count=0,
206            test_time='',
207            runner_total=None,
208            group_total=None,
209            additional_info={},
210            test_run_name='com.android.UnitTests'
211        ))
212        self.mock_reporter.process_test_result.assert_has_calls([call])
213
214    def test_process_event_missing_test_run_started_event(self):
215        """Test process_event method for normal test results."""
216        for name, data in EVENTS_MISSING_TEST_RUN_STARTED_EVENT:
217            self.fake_eh.process_event(name, data)
218        call = mock.call(test_runner_base.TestResult(
219            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
220            group_name=None,
221            test_name='someClassName#someTestName',
222            status=test_runner_base.PASSED_STATUS,
223            details=None,
224            test_count=1,
225            test_time='(996ms)',
226            runner_total=None,
227            group_total=None,
228            additional_info={},
229            test_run_name=None
230        ))
231        self.mock_reporter.process_test_result.assert_has_calls([call])
232
233    # pylint: disable=protected-access
234    def test_process_event_not_balanced(self):
235        """Test process_event method with start/end event name not balanced."""
236        for name, data in EVENTS_NOT_BALANCED_BEFORE_RAISE:
237            self.fake_eh.process_event(name, data)
238        call = mock.call(test_runner_base.TestResult(
239            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
240            group_name='someTestModule',
241            test_name='someClassName#someTestName',
242            status=test_runner_base.PASSED_STATUS,
243            details=None,
244            test_count=1,
245            test_time='(8ms)',
246            runner_total=None,
247            group_total=2,
248            additional_info={},
249            test_run_name='com.android.UnitTests'
250        ))
251        self.mock_reporter.process_test_result.assert_has_calls([call])
252        # Event pair: TEST_STARTED -> TEST_RUN_ENDED
253        # It should raise TradeFedExitError in _check_events_are_balanced()
254        name = 'TEST_RUN_ENDED'
255        data = {}
256        self.assertRaises(e_h.EventHandleError,
257                          self.fake_eh._check_events_are_balanced,
258                          name, self.mock_reporter)
259        # Event pair: TEST_RUN_STARTED -> TEST_MODULE_ENDED
260        # It should raise TradeFedExitError in _check_events_are_balanced()
261        name = 'TEST_MODULE_ENDED'
262        data = {'foo': 'bar'}
263        self.assertRaises(e_h.EventHandleError,
264                          self.fake_eh._check_events_are_balanced,
265                          name, self.mock_reporter)
266
267    def test_process_event_ignore(self):
268        """Test _process_event method for normal test results."""
269        for name, data in EVENTS_IGNORE:
270            self.fake_eh.process_event(name, data)
271        call1 = mock.call(test_runner_base.TestResult(
272            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
273            group_name='someTestModule',
274            test_name='someClassName#someTestName',
275            status=test_runner_base.PASSED_STATUS,
276            details=None,
277            test_count=1,
278            test_time='(10ms)',
279            runner_total=None,
280            group_total=2,
281            additional_info={},
282            test_run_name='com.android.UnitTests'
283        ))
284        call2 = mock.call(test_runner_base.TestResult(
285            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
286            group_name='someTestModule',
287            test_name='someClassName2#someTestName2',
288            status=test_runner_base.IGNORED_STATUS,
289            details=None,
290            test_count=2,
291            test_time='(62ms)',
292            runner_total=None,
293            group_total=2,
294            additional_info={},
295            test_run_name='com.android.UnitTests'
296        ))
297        self.mock_reporter.process_test_result.assert_has_calls([call1, call2])
298
299    def test_process_event_with_additional_info(self):
300        """Test process_event method with perf information."""
301        for name, data in EVENTS_WITH_PERF_INFO:
302            self.fake_eh.process_event(name, data)
303        call1 = mock.call(test_runner_base.TestResult(
304            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
305            group_name='someTestModule',
306            test_name='someClassName#someTestName',
307            status=test_runner_base.PASSED_STATUS,
308            details=None,
309            test_count=1,
310            test_time='(996ms)',
311            runner_total=None,
312            group_total=2,
313            additional_info={},
314            test_run_name='com.android.UnitTests'
315        ))
316
317        test_additional_info = {'cpu_time':'1234.1234(ns)', 'real_time':'5678.5678(ns)',
318                                'iterations':'6666'}
319        call2 = mock.call(test_runner_base.TestResult(
320            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
321            group_name='someTestModule',
322            test_name='someClassName2#someTestName2',
323            status=test_runner_base.FAILED_STATUS,
324            details='someTrace',
325            test_count=2,
326            test_time='(2h44m36.402s)',
327            runner_total=None,
328            group_total=2,
329            additional_info=test_additional_info,
330            test_run_name='com.android.UnitTests'
331        ))
332
333        test_additional_info2 = {'additional_info_min':'102773',
334                                 'additional_info_mean':'105973',
335                                 'additional_info_median':'103778'}
336        call3 = mock.call(test_runner_base.TestResult(
337            runner_name=atf_tr.AtestTradefedTestRunner.NAME,
338            group_name='someTestModule',
339            test_name='someClassName3#someTestName3',
340            status=test_runner_base.PASSED_STATUS,
341            details=None,
342            test_count=3,
343            test_time='(60ms)',
344            runner_total=None,
345            group_total=2,
346            additional_info=test_additional_info2,
347            test_run_name='com.android.UnitTests'
348        ))
349        self.mock_reporter.process_test_result.assert_has_calls([call1, call2, call3])
350
351if __name__ == '__main__':
352    unittest.main()
353