1#!/usr/bin/env python3
2#
3#   Copyright 2016 - The Android Open Source Project
4#
5#   Licensed under the Apache License, Version 2.0 (the "License");
6#   you may not use this file except in compliance with the License.
7#   You may obtain a copy of the License at
8#
9#       http://www.apache.org/licenses/LICENSE-2.0
10#
11#   Unless required by applicable law or agreed to in writing, software
12#   distributed under the License is distributed on an "AS IS" BASIS,
13#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14#   See the License for the specific language governing permissions and
15#   limitations under the License.
16
17import shutil
18import tempfile
19import unittest
20
21import mock
22import mock_controller
23
24from acts import asserts
25from acts import base_test
26from acts import error
27from acts import signals
28
29from mobly import base_test as mobly_base_test
30import mobly.config_parser as mobly_config_parser
31
32MSG_EXPECTED_EXCEPTION = 'This is an expected exception.'
33MSG_EXPECTED_TEST_FAILURE = 'This is an expected test failure.'
34MSG_UNEXPECTED_EXCEPTION = 'Unexpected exception!'
35
36MOCK_EXTRA = {'key': 'value', 'answer_to_everything': 42}
37
38
39def never_call():
40    raise Exception(MSG_UNEXPECTED_EXCEPTION)
41
42
43class SomeError(Exception):
44    """A custom exception class used for tests in this module."""
45
46
47class ActsBaseClassTest(unittest.TestCase):
48    def setUp(self):
49        self.tmp_dir = tempfile.mkdtemp()
50        self.tb_key = 'testbed_configs'
51        self.test_run_config = mobly_config_parser.TestRunConfig()
52        self.test_run_config.testbed_name = 'SampleTestBed'
53        self.test_run_config.controller_configs = {
54            self.tb_key: {
55                'name': self.test_run_config.testbed_name,
56            },
57        }
58        self.test_run_config.log_path = self.tmp_dir
59        self.test_run_config.user_params = {'some_param': 'hahaha'}
60        self.test_run_config.summary_writer = mock.MagicMock()
61        self.mock_test_name = 'test_something'
62
63    def tearDown(self):
64        shutil.rmtree(self.tmp_dir)
65
66    def test_current_test_case_name(self):
67        class MockBaseTest(base_test.BaseTestClass):
68            def test_func(self):
69                asserts.assert_true(
70                    self.current_test_name == 'test_func',
71                    'Got unexpected test name %s.' % self.current_test_name)
72
73        bt_cls = MockBaseTest(self.test_run_config)
74        bt_cls.run(test_names=['test_func'])
75        actual_record = bt_cls.results.passed[0]
76        self.assertEqual(actual_record.test_name, 'test_func')
77        self.assertIsNone(actual_record.details)
78        self.assertIsNone(actual_record.extras)
79
80    def test_self_tests_list(self):
81        class MockBaseTest(base_test.BaseTestClass):
82            def __init__(self, controllers):
83                super(MockBaseTest, self).__init__(controllers)
84                self.tests = ('test_something', )
85
86            def test_something(self):
87                pass
88
89            def test_never(self):
90                # This should not execute it's not on default test list.
91                never_call()
92
93        bt_cls = MockBaseTest(self.test_run_config)
94        bt_cls.run()
95        actual_record = bt_cls.results.passed[0]
96        self.assertEqual(actual_record.test_name, 'test_something')
97
98    def test_self_tests_list_fail_by_convention(self):
99        class MockBaseTest(base_test.BaseTestClass):
100            def __init__(self, controllers):
101                super(MockBaseTest, self).__init__(controllers)
102                self.tests = ('not_a_test_something', )
103
104            def not_a_test_something(self):
105                pass
106
107            def test_never(self):
108                # This should not execute it's not on default test list.
109                never_call()
110
111        bt_cls = MockBaseTest(self.test_run_config)
112        expected_msg = ('Test case name not_a_test_something does not follow '
113                        'naming convention test_\*, abort.')
114        with self.assertRaisesRegex(base_test.Error, expected_msg):
115            bt_cls.run()
116
117    def test_cli_test_selection_match_self_tests_list(self):
118        class MockBaseTest(base_test.BaseTestClass):
119            def __init__(self, controllers):
120                super(MockBaseTest, self).__init__(controllers)
121                self.tests = ('test_star1', 'test_star2', 'test_question_mark',
122                              'test_char_seq', 'test_no_match')
123
124            def test_star1(self):
125                pass
126
127            def test_star2(self):
128                pass
129
130            def test_question_mark(self):
131                pass
132
133            def test_char_seq(self):
134                pass
135
136            def test_no_match(self):
137                # This should not execute because it does not match any regex
138                # in the cmd line input.
139                never_call()
140
141        bt_cls = MockBaseTest(self.test_run_config)
142        test_names = [
143            'test_st*r1', 'test_*2', 'test_?uestion_mark', 'test_c[fghi]ar_seq'
144        ]
145        bt_cls.run(test_names=test_names)
146        passed_names = [p.test_name for p in bt_cls.results.passed]
147        self.assertEqual(len(passed_names), len(test_names))
148        for test in [
149                'test_star1', 'test_star2', 'test_question_mark',
150                'test_char_seq'
151        ]:
152            self.assertIn(test, passed_names)
153
154    def test_default_execution_of_all_tests(self):
155        class MockBaseTest(base_test.BaseTestClass):
156            def test_something(self):
157                pass
158
159            def not_a_test(self):
160                # This should not execute its name doesn't follow test case
161                # naming convention.
162                never_call()
163
164        bt_cls = MockBaseTest(self.test_run_config)
165        bt_cls.run()
166        actual_record = bt_cls.results.passed[0]
167        self.assertEqual(actual_record.test_name, 'test_something')
168
169    def test_missing_requested_test_func(self):
170        class MockBaseTest(base_test.BaseTestClass):
171            def __init__(self, controllers):
172                super(MockBaseTest, self).__init__(controllers)
173                self.tests = ('test_something', )
174
175        bt_cls = MockBaseTest(self.test_run_config)
176        bt_cls.run()
177        self.assertFalse(bt_cls.results.executed)
178        self.assertTrue(bt_cls.results.skipped)
179
180    def test_setup_class_fail_by_exception(self):
181        call_check = mock.MagicMock()
182
183        class MockBaseTest(base_test.BaseTestClass):
184            def setup_class(self):
185                raise Exception(MSG_EXPECTED_EXCEPTION)
186
187            def test_something(self):
188                # This should not execute because setup_class failed.
189                never_call()
190
191            def on_skip(self, test_name, begin_time):
192                call_check('haha')
193
194        bt_cls = MockBaseTest(self.test_run_config)
195        bt_cls.run()
196        actual_record = bt_cls.results.error[0]
197        self.assertEqual(actual_record.test_name, 'test_something')
198        expected_summary = {
199            'Error': 1,
200            'Executed': 1,
201            'Failed': 0,
202            'Passed': 0,
203            'Requested': 1,
204            'Skipped': 0
205        }
206        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
207        call_check.assert_called_once_with('haha')
208
209    def test_setup_test_fail_by_exception(self):
210        class MockBaseTest(base_test.BaseTestClass):
211            def setup_test(self):
212                raise Exception(MSG_EXPECTED_EXCEPTION)
213
214            def test_something(self):
215                # This should not execute because setup_test failed.
216                never_call()
217
218        bt_cls = MockBaseTest(self.test_run_config)
219        bt_cls.run(test_names=['test_something'])
220        actual_record = bt_cls.results.error[0]
221        self.assertEqual(actual_record.test_name, self.mock_test_name)
222        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
223        self.assertIsNone(actual_record.extras)
224        expected_summary = {
225            'Error': 1,
226            'Executed': 1,
227            'Failed': 0,
228            'Passed': 0,
229            'Requested': 1,
230            'Skipped': 0
231        }
232        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
233
234    def test_setup_test_fail_by_test_signal(self):
235        class MockBaseTest(base_test.BaseTestClass):
236            def setup_test(self):
237                raise signals.TestFailure(MSG_EXPECTED_EXCEPTION)
238
239            def test_something(self):
240                # This should not execute because setup_test failed.
241                never_call()
242
243        bt_cls = MockBaseTest(self.test_run_config)
244        bt_cls.run(test_names=['test_something'])
245        actual_record = bt_cls.results.failed[0]
246        self.assertEqual(actual_record.test_name, self.mock_test_name)
247        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
248        self.assertIsNone(actual_record.extras)
249        expected_summary = {
250            'Error': 0,
251            'Executed': 1,
252            'Failed': 1,
253            'Passed': 0,
254            'Requested': 1,
255            'Skipped': 0
256        }
257        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
258
259    def test_setup_test_fail_by_return_False(self):
260        class MockBaseTest(base_test.BaseTestClass):
261            def setup_test(self):
262                return False
263
264            def test_something(self):
265                # This should not execute because setup_test failed.
266                never_call()
267
268        bt_cls = MockBaseTest(self.test_run_config)
269        bt_cls.run(test_names=['test_something'])
270        actual_record = bt_cls.results.failed[0]
271        expected_msg = 'Setup for %s failed.' % self.mock_test_name
272        self.assertEqual(actual_record.test_name, self.mock_test_name)
273        self.assertEqual(actual_record.details, expected_msg)
274        self.assertIsNone(actual_record.extras)
275        expected_summary = {
276            'Error': 0,
277            'Executed': 1,
278            'Failed': 1,
279            'Passed': 0,
280            'Requested': 1,
281            'Skipped': 0
282        }
283        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
284
285    def test_run_fail_by_ActsError_(self):
286        class MockBaseTest(base_test.BaseTestClass):
287            def __init__(self, controllers):
288                super(MockBaseTest, self).__init__(controllers)
289
290            def test_something(self):
291                raise error.ActsError()
292
293        bt_cls = MockBaseTest(self.test_run_config)
294        bt_cls.run(test_names=['test_something'])
295        expected_summary = {
296            'Error': 1,
297            'Executed': 1,
298            'Failed': 0,
299            'Passed': 0,
300            'Requested': 1,
301            'Skipped': 0
302        }
303        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
304
305    def test_teardown_test_assert_fail(self):
306        class MockBaseTest(base_test.BaseTestClass):
307            def teardown_test(self):
308                asserts.assert_true(False, MSG_EXPECTED_EXCEPTION)
309
310            def test_something(self):
311                pass
312
313        bt_cls = MockBaseTest(self.test_run_config)
314        bt_cls.run()
315        actual_record = bt_cls.results.error[0]
316        self.assertEqual(actual_record.test_name, self.mock_test_name)
317        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
318        self.assertIsNone(actual_record.extras)
319        expected_summary = {
320            'Error': 1,
321            'Executed': 1,
322            'Failed': 0,
323            'Passed': 0,
324            'Requested': 1,
325            'Skipped': 0
326        }
327        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
328
329    def test_teardown_test_raise_exception(self):
330        class MockBaseTest(base_test.BaseTestClass):
331            def teardown_test(self):
332                raise Exception(MSG_EXPECTED_EXCEPTION)
333
334            def test_something(self):
335                pass
336
337        bt_cls = MockBaseTest(self.test_run_config)
338        bt_cls.run()
339        actual_record = bt_cls.results.error[0]
340        self.assertEqual(actual_record.test_name, self.mock_test_name)
341        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
342        self.assertIsNone(actual_record.extras)
343        expected_summary = {
344            'Error': 1,
345            'Executed': 1,
346            'Failed': 0,
347            'Passed': 0,
348            'Requested': 1,
349            'Skipped': 0
350        }
351        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
352
353    def test_teardown_test_executed_if_test_pass(self):
354        my_mock = mock.MagicMock()
355
356        class MockBaseTest(base_test.BaseTestClass):
357            def teardown_test(self):
358                my_mock('teardown_test')
359
360            def test_something(self):
361                pass
362
363        bt_cls = MockBaseTest(self.test_run_config)
364        bt_cls.run()
365        actual_record = bt_cls.results.passed[0]
366        my_mock.assert_called_once_with('teardown_test')
367        self.assertEqual(actual_record.test_name, self.mock_test_name)
368        self.assertIsNone(actual_record.details)
369        self.assertIsNone(actual_record.extras)
370        expected_summary = {
371            'Error': 0,
372            'Executed': 1,
373            'Failed': 0,
374            'Passed': 1,
375            'Requested': 1,
376            'Skipped': 0
377        }
378        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
379
380    def test_teardown_test_executed_if_setup_test_fails(self):
381        my_mock = mock.MagicMock()
382
383        class MockBaseTest(base_test.BaseTestClass):
384            def setup_test(self):
385                raise Exception(MSG_EXPECTED_EXCEPTION)
386
387            def teardown_test(self):
388                my_mock('teardown_test')
389
390            def test_something(self):
391                pass
392
393        bt_cls = MockBaseTest(self.test_run_config)
394        bt_cls.run()
395        actual_record = bt_cls.results.error[0]
396        my_mock.assert_called_once_with('teardown_test')
397        self.assertEqual(actual_record.test_name, self.mock_test_name)
398        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
399        self.assertIsNone(actual_record.extras)
400        expected_summary = {
401            'Error': 1,
402            'Executed': 1,
403            'Failed': 0,
404            'Passed': 0,
405            'Requested': 1,
406            'Skipped': 0
407        }
408        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
409
410    def test_teardown_test_executed_if_test_fails(self):
411        my_mock = mock.MagicMock()
412
413        class MockBaseTest(base_test.BaseTestClass):
414            def teardown_test(self):
415                my_mock('teardown_test')
416
417            def test_something(self):
418                raise Exception(MSG_EXPECTED_EXCEPTION)
419
420        bt_cls = MockBaseTest(self.test_run_config)
421        bt_cls.run()
422        actual_record = bt_cls.results.error[0]
423        my_mock.assert_called_once_with('teardown_test')
424        self.assertEqual(actual_record.test_name, self.mock_test_name)
425        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
426        self.assertIsNone(actual_record.extras)
427        expected_summary = {
428            'Error': 1,
429            'Executed': 1,
430            'Failed': 0,
431            'Passed': 0,
432            'Requested': 1,
433            'Skipped': 0
434        }
435        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
436
437    def test_on_exception_executed_if_teardown_test_fails(self):
438        my_mock = mock.MagicMock()
439
440        class MockBaseTest(base_test.BaseTestClass):
441            def on_exception(self, test_name, begin_time):
442                my_mock('on_exception')
443
444            def teardown_test(self):
445                raise Exception(MSG_EXPECTED_EXCEPTION)
446
447            def test_something(self):
448                pass
449
450        bt_cls = MockBaseTest(self.test_run_config)
451        bt_cls.run()
452        my_mock.assert_called_once_with('on_exception')
453        actual_record = bt_cls.results.error[0]
454        self.assertEqual(actual_record.test_name, self.mock_test_name)
455        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
456        self.assertIsNone(actual_record.extras)
457        expected_summary = {
458            'Error': 1,
459            'Executed': 1,
460            'Failed': 0,
461            'Passed': 0,
462            'Requested': 1,
463            'Skipped': 0
464        }
465        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
466
467    def test_on_fail_executed_if_test_fails(self):
468        my_mock = mock.MagicMock()
469
470        class MockBaseTest(base_test.BaseTestClass):
471            def on_fail(self, test_name, begin_time):
472                my_mock('on_fail')
473
474            def test_something(self):
475                asserts.assert_true(False, MSG_EXPECTED_EXCEPTION)
476
477        bt_cls = MockBaseTest(self.test_run_config)
478        bt_cls.run()
479        my_mock.assert_called_once_with('on_fail')
480        actual_record = bt_cls.results.failed[0]
481        self.assertEqual(actual_record.test_name, self.mock_test_name)
482        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
483        self.assertIsNone(actual_record.extras)
484        expected_summary = {
485            'Error': 0,
486            'Executed': 1,
487            'Failed': 1,
488            'Passed': 0,
489            'Requested': 1,
490            'Skipped': 0
491        }
492        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
493
494    def test_on_fail_executed_if_test_setup_fails_by_exception(self):
495        my_mock = mock.MagicMock()
496
497        class MockBaseTest(base_test.BaseTestClass):
498            def setup_test(self):
499                raise Exception(MSG_EXPECTED_EXCEPTION)
500
501            def on_fail(self, test_name, begin_time):
502                my_mock('on_fail')
503
504            def test_something(self):
505                pass
506
507        bt_cls = MockBaseTest(self.test_run_config)
508        bt_cls.run()
509        my_mock.assert_called_once_with('on_fail')
510        actual_record = bt_cls.results.error[0]
511        self.assertEqual(actual_record.test_name, self.mock_test_name)
512        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
513        self.assertIsNone(actual_record.extras)
514        expected_summary = {
515            'Error': 1,
516            'Executed': 1,
517            'Failed': 0,
518            'Passed': 0,
519            'Requested': 1,
520            'Skipped': 0
521        }
522        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
523
524    def test_on_fail_executed_if_test_setup_fails_by_return_False(self):
525        my_mock = mock.MagicMock()
526
527        class MockBaseTest(base_test.BaseTestClass):
528            def setup_test(self):
529                return False
530
531            def on_fail(self, test_name, begin_time):
532                my_mock('on_fail')
533
534            def test_something(self):
535                pass
536
537        bt_cls = MockBaseTest(self.test_run_config)
538        bt_cls.run()
539        my_mock.assert_called_once_with('on_fail')
540        actual_record = bt_cls.results.failed[0]
541        self.assertEqual(actual_record.test_name, self.mock_test_name)
542        self.assertEqual(actual_record.details,
543                         'Setup for test_something failed.')
544        self.assertIsNone(actual_record.extras)
545        expected_summary = {
546            'Error': 0,
547            'Executed': 1,
548            'Failed': 1,
549            'Passed': 0,
550            'Requested': 1,
551            'Skipped': 0
552        }
553        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
554
555    def test_failure_to_call_procedure_function_is_recorded(self):
556        class MockBaseTest(base_test.BaseTestClass):
557            # Wrong method signature; will raise exception
558            def on_pass(self):
559                pass
560
561            def test_something(self):
562                asserts.explicit_pass(MSG_EXPECTED_EXCEPTION)
563
564        bt_cls = MockBaseTest(self.test_run_config)
565        bt_cls.run()
566        actual_record = bt_cls.results.error[0]
567        self.assertIn('_on_pass', actual_record.extra_errors)
568        self.assertEqual(actual_record.test_name, self.mock_test_name)
569        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
570        self.assertIsNone(actual_record.extras)
571        expected_summary = {
572            'Error': 1,
573            'Executed': 1,
574            'Failed': 0,
575            'Passed': 0,
576            'Requested': 1,
577            'Skipped': 0
578        }
579        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
580
581    def test_failure_in_procedure_functions_is_recorded(self):
582        expected_msg = 'Something failed in on_pass.'
583
584        class MockBaseTest(base_test.BaseTestClass):
585            def on_pass(self, test_name, begin_time):
586                raise Exception(expected_msg)
587
588            def test_something(self):
589                asserts.explicit_pass(MSG_EXPECTED_EXCEPTION)
590
591        bt_cls = MockBaseTest(self.test_run_config)
592        bt_cls.run()
593        actual_record = bt_cls.results.error[0]
594        self.assertEqual(actual_record.test_name, self.mock_test_name)
595        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
596        self.assertIsNone(actual_record.extras)
597        expected_summary = {
598            'Error': 1,
599            'Executed': 1,
600            'Failed': 0,
601            'Passed': 0,
602            'Requested': 1,
603            'Skipped': 0
604        }
605        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
606
607    def test_both_teardown_and_test_body_raise_exceptions(self):
608        class MockBaseTest(base_test.BaseTestClass):
609            def teardown_test(self):
610                asserts.assert_true(False, MSG_EXPECTED_EXCEPTION)
611
612            def test_something(self):
613                raise Exception('Test Body Exception.')
614
615        bt_cls = MockBaseTest(self.test_run_config)
616        bt_cls.run()
617        actual_record = bt_cls.results.error[0]
618        self.assertEqual(actual_record.test_name, self.mock_test_name)
619        self.assertEqual(actual_record.details, 'Test Body Exception.')
620        self.assertIsNone(actual_record.extras)
621        self.assertEqual(actual_record.extra_errors['teardown_test'].details,
622                         'This is an expected exception.')
623        expected_summary = {
624            'Error': 1,
625            'Executed': 1,
626            'Failed': 0,
627            'Passed': 0,
628            'Requested': 1,
629            'Skipped': 0
630        }
631        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
632
633    def test_explicit_pass_but_teardown_test_raises_an_exception(self):
634        """Test record result should be marked as UNKNOWN as opposed to PASS.
635        """
636        class MockBaseTest(base_test.BaseTestClass):
637            def teardown_test(self):
638                asserts.assert_true(False, MSG_EXPECTED_EXCEPTION)
639
640            def test_something(self):
641                asserts.explicit_pass('Test Passed!')
642
643        bt_cls = MockBaseTest(self.test_run_config)
644        bt_cls.run()
645        actual_record = bt_cls.results.error[0]
646        self.assertEqual(actual_record.test_name, self.mock_test_name)
647        self.assertEqual(actual_record.details, 'Test Passed!')
648        self.assertIsNone(actual_record.extras)
649        self.assertEqual(actual_record.extra_errors['teardown_test'].details,
650                         'This is an expected exception.')
651        expected_summary = {
652            'Error': 1,
653            'Executed': 1,
654            'Failed': 0,
655            'Passed': 0,
656            'Requested': 1,
657            'Skipped': 0
658        }
659        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
660
661    def test_on_pass_raise_exception(self):
662        class MockBaseTest(base_test.BaseTestClass):
663            def on_pass(self, test_name, begin_time):
664                raise Exception(MSG_EXPECTED_EXCEPTION)
665
666            def test_something(self):
667                asserts.explicit_pass(MSG_EXPECTED_EXCEPTION,
668                                      extras=MOCK_EXTRA)
669
670        bt_cls = MockBaseTest(self.test_run_config)
671        bt_cls.run()
672        actual_record = bt_cls.results.error[0]
673        self.assertEqual(actual_record.test_name, self.mock_test_name)
674        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
675        self.assertEqual(actual_record.extras, MOCK_EXTRA)
676        expected_summary = {
677            'Error': 1,
678            'Executed': 1,
679            'Failed': 0,
680            'Passed': 0,
681            'Requested': 1,
682            'Skipped': 0
683        }
684        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
685
686    def test_on_fail_raise_exception(self):
687        class MockBaseTest(base_test.BaseTestClass):
688            def on_fail(self, test_name, begin_time):
689                raise Exception(MSG_EXPECTED_EXCEPTION)
690
691            def test_something(self):
692                asserts.fail(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA)
693
694        bt_cls = MockBaseTest(self.test_run_config)
695        bt_cls.run()
696        actual_record = bt_cls.results.failed[0]
697        self.assertEqual(bt_cls.results.error, [])
698        self.assertEqual(actual_record.test_name, self.mock_test_name)
699        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
700        self.assertEqual(actual_record.extras, MOCK_EXTRA)
701        expected_summary = {
702            'Error': 0,
703            'Executed': 1,
704            'Failed': 1,
705            'Passed': 0,
706            'Requested': 1,
707            'Skipped': 0
708        }
709        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
710
711    def test_abort_class(self):
712        class MockBaseTest(base_test.BaseTestClass):
713            def test_1(self):
714                pass
715
716            def test_2(self):
717                asserts.abort_class(MSG_EXPECTED_EXCEPTION)
718                never_call()
719
720            def test_3(self):
721                never_call()
722
723        bt_cls = MockBaseTest(self.test_run_config)
724        bt_cls.run(test_names=['test_1', 'test_2', 'test_3'])
725        self.assertEqual(bt_cls.results.passed[0].test_name, 'test_1')
726        self.assertEqual(bt_cls.results.failed[0].details,
727                         MSG_EXPECTED_EXCEPTION)
728        expected_summary = {
729            'Error': 0,
730            'Executed': 2,
731            'Failed': 1,
732            'Passed': 1,
733            'Requested': 3,
734            'Skipped': 0
735        }
736        self.assertEqual(bt_cls.results.summary_dict(), expected_summary)
737
738    def test_uncaught_exception(self):
739        class MockBaseTest(base_test.BaseTestClass):
740            def test_func(self):
741                raise Exception(MSG_EXPECTED_EXCEPTION)
742                never_call()
743
744        bt_cls = MockBaseTest(self.test_run_config)
745        bt_cls.run(test_names=['test_func'])
746        actual_record = bt_cls.results.error[0]
747        self.assertEqual(actual_record.test_name, 'test_func')
748        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
749        self.assertIsNone(actual_record.extras)
750
751    def test_fail(self):
752        class MockBaseTest(base_test.BaseTestClass):
753            def test_func(self):
754                asserts.fail(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA)
755                never_call()
756
757        bt_cls = MockBaseTest(self.test_run_config)
758        bt_cls.run(test_names=['test_func'])
759        actual_record = bt_cls.results.failed[0]
760        self.assertEqual(actual_record.test_name, 'test_func')
761        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
762        self.assertEqual(actual_record.extras, MOCK_EXTRA)
763
764    def test_assert_true(self):
765        class MockBaseTest(base_test.BaseTestClass):
766            def test_func(self):
767                asserts.assert_true(False,
768                                    MSG_EXPECTED_EXCEPTION,
769                                    extras=MOCK_EXTRA)
770                never_call()
771
772        bt_cls = MockBaseTest(self.test_run_config)
773        bt_cls.run(test_names=['test_func'])
774        actual_record = bt_cls.results.failed[0]
775        self.assertEqual(actual_record.test_name, 'test_func')
776        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
777        self.assertEqual(actual_record.extras, MOCK_EXTRA)
778
779    def test_assert_equal_pass(self):
780        class MockBaseTest(base_test.BaseTestClass):
781            def test_func(self):
782                asserts.assert_equal(1, 1, extras=MOCK_EXTRA)
783
784        bt_cls = MockBaseTest(self.test_run_config)
785        bt_cls.run()
786        actual_record = bt_cls.results.passed[0]
787        self.assertEqual(actual_record.test_name, 'test_func')
788        self.assertIsNone(actual_record.details)
789        self.assertIsNone(actual_record.extras)
790
791    def test_assert_equal_fail(self):
792        class MockBaseTest(base_test.BaseTestClass):
793            def test_func(self):
794                asserts.assert_equal(1, 2, extras=MOCK_EXTRA)
795
796        bt_cls = MockBaseTest(self.test_run_config)
797        bt_cls.run()
798        actual_record = bt_cls.results.failed[0]
799        self.assertEqual(actual_record.test_name, 'test_func')
800        self.assertIn('1 != 2', actual_record.details)
801        self.assertEqual(actual_record.extras, MOCK_EXTRA)
802
803    def test_assert_equal_fail_with_msg(self):
804        class MockBaseTest(base_test.BaseTestClass):
805            def test_func(self):
806                asserts.assert_equal(1,
807                                     2,
808                                     msg=MSG_EXPECTED_EXCEPTION,
809                                     extras=MOCK_EXTRA)
810
811        bt_cls = MockBaseTest(self.test_run_config)
812        bt_cls.run()
813        actual_record = bt_cls.results.failed[0]
814        self.assertEqual(actual_record.test_name, 'test_func')
815        expected_msg = '1 != 2 ' + MSG_EXPECTED_EXCEPTION
816        self.assertIn(expected_msg, actual_record.details)
817        self.assertEqual(actual_record.extras, MOCK_EXTRA)
818
819    def test_assert_raises_pass(self):
820        class MockBaseTest(base_test.BaseTestClass):
821            def test_func(self):
822                with asserts.assert_raises(SomeError, extras=MOCK_EXTRA):
823                    raise SomeError(MSG_EXPECTED_EXCEPTION)
824
825        bt_cls = MockBaseTest(self.test_run_config)
826        bt_cls.run()
827        actual_record = bt_cls.results.passed[0]
828        self.assertEqual(actual_record.test_name, 'test_func')
829        self.assertIsNone(actual_record.details)
830        self.assertIsNone(actual_record.extras)
831
832    def test_assert_raises_regex_pass(self):
833        class MockBaseTest(base_test.BaseTestClass):
834            def test_func(self):
835                with asserts.assert_raises_regex(
836                        SomeError,
837                        expected_regex=MSG_EXPECTED_EXCEPTION,
838                        extras=MOCK_EXTRA):
839                    raise SomeError(MSG_EXPECTED_EXCEPTION)
840
841        bt_cls = MockBaseTest(self.test_run_config)
842        bt_cls.run()
843        actual_record = bt_cls.results.passed[0]
844        self.assertEqual(actual_record.test_name, 'test_func')
845        self.assertIsNone(actual_record.details)
846        self.assertIsNone(actual_record.extras)
847
848    def test_assert_raises_fail_with_noop(self):
849        class MockBaseTest(base_test.BaseTestClass):
850            def test_func(self):
851                with asserts.assert_raises_regex(
852                        SomeError,
853                        expected_regex=MSG_EXPECTED_EXCEPTION,
854                        extras=MOCK_EXTRA):
855                    pass
856
857        bt_cls = MockBaseTest(self.test_run_config)
858        bt_cls.run()
859        actual_record = bt_cls.results.failed[0]
860        self.assertEqual(actual_record.test_name, 'test_func')
861        self.assertEqual(actual_record.details, 'SomeError not raised')
862        self.assertEqual(actual_record.extras, MOCK_EXTRA)
863
864    def test_assert_raises_fail_with_wrong_regex(self):
865        wrong_msg = 'ha'
866
867        class MockBaseTest(base_test.BaseTestClass):
868            def test_func(self):
869                with asserts.assert_raises_regex(
870                        SomeError,
871                        expected_regex=MSG_EXPECTED_EXCEPTION,
872                        extras=MOCK_EXTRA):
873                    raise SomeError(wrong_msg)
874
875        bt_cls = MockBaseTest(self.test_run_config)
876        bt_cls.run()
877        actual_record = bt_cls.results.failed[0]
878        self.assertEqual(actual_record.test_name, 'test_func')
879        expected_details = ('"This is an expected exception." does not match '
880                            '"%s"') % wrong_msg
881        self.assertEqual(actual_record.details, expected_details)
882        self.assertEqual(actual_record.extras, MOCK_EXTRA)
883
884    def test_assert_raises_fail_with_wrong_error(self):
885        class MockBaseTest(base_test.BaseTestClass):
886            def test_func(self):
887                with asserts.assert_raises_regex(
888                        SomeError,
889                        expected_regex=MSG_EXPECTED_EXCEPTION,
890                        extras=MOCK_EXTRA):
891                    raise AttributeError(MSG_UNEXPECTED_EXCEPTION)
892
893        bt_cls = MockBaseTest(self.test_run_config)
894        bt_cls.run()
895        actual_record = bt_cls.results.error[0]
896        self.assertEqual(actual_record.test_name, 'test_func')
897        self.assertEqual(actual_record.details, MSG_UNEXPECTED_EXCEPTION)
898        self.assertIsNone(actual_record.extras)
899
900    def test_explicit_pass(self):
901        class MockBaseTest(base_test.BaseTestClass):
902            def test_func(self):
903                asserts.explicit_pass(MSG_EXPECTED_EXCEPTION,
904                                      extras=MOCK_EXTRA)
905                never_call()
906
907        bt_cls = MockBaseTest(self.test_run_config)
908        bt_cls.run(test_names=['test_func'])
909        actual_record = bt_cls.results.passed[0]
910        self.assertEqual(actual_record.test_name, 'test_func')
911        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
912        self.assertEqual(actual_record.extras, MOCK_EXTRA)
913
914    def test_implicit_pass(self):
915        class MockBaseTest(base_test.BaseTestClass):
916            def test_func(self):
917                pass
918
919        bt_cls = MockBaseTest(self.test_run_config)
920        bt_cls.run(test_names=['test_func'])
921        actual_record = bt_cls.results.passed[0]
922        self.assertEqual(actual_record.test_name, 'test_func')
923        self.assertIsNone(actual_record.details)
924        self.assertIsNone(actual_record.extras)
925
926    def test_skip(self):
927        class MockBaseTest(base_test.BaseTestClass):
928            def test_func(self):
929                asserts.skip(MSG_EXPECTED_EXCEPTION, extras=MOCK_EXTRA)
930                never_call()
931
932        bt_cls = MockBaseTest(self.test_run_config)
933        bt_cls.run(test_names=['test_func'])
934        actual_record = bt_cls.results.skipped[0]
935        self.assertEqual(actual_record.test_name, 'test_func')
936        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
937        self.assertEqual(actual_record.extras, MOCK_EXTRA)
938
939    def test_skip_if(self):
940        class MockBaseTest(base_test.BaseTestClass):
941            def test_func(self):
942                asserts.skip_if(False, MSG_UNEXPECTED_EXCEPTION)
943                asserts.skip_if(True,
944                                MSG_EXPECTED_EXCEPTION,
945                                extras=MOCK_EXTRA)
946                never_call()
947
948        bt_cls = MockBaseTest(self.test_run_config)
949        bt_cls.run(test_names=['test_func'])
950        actual_record = bt_cls.results.skipped[0]
951        self.assertEqual(actual_record.test_name, 'test_func')
952        self.assertEqual(actual_record.details, MSG_EXPECTED_EXCEPTION)
953        self.assertEqual(actual_record.extras, MOCK_EXTRA)
954
955    def test_unpack_userparams_required(self):
956        """Missing a required param should raise an error."""
957        required = ['some_param']
958        bc = base_test.BaseTestClass(self.test_run_config)
959        bc.unpack_userparams(required)
960        expected_value = self.test_run_config.user_params['some_param']
961        self.assertEqual(bc.some_param, expected_value)
962
963    def test_unpack_userparams_required_missing(self):
964        """Missing a required param should raise an error."""
965        required = ['something']
966        bc = base_test.BaseTestClass(self.test_run_config)
967        expected_msg = ('Missing required user param "%s" in test '
968                        'configuration.') % required[0]
969        with self.assertRaises(mobly_base_test.Error, msg=expected_msg):
970            bc.unpack_userparams(required)
971
972    def test_unpack_userparams_optional(self):
973        """If an optional param is specified, the value should be what's in the
974        config.
975        """
976        opt = ['some_param']
977        bc = base_test.BaseTestClass(self.test_run_config)
978        bc.unpack_userparams(opt_param_names=opt)
979        expected_value = self.test_run_config.user_params['some_param']
980        self.assertEqual(bc.some_param, expected_value)
981
982    def test_unpack_userparams_optional_with_default(self):
983        """If an optional param is specified with a default value, and the
984        param is not in the config, the value should be the default value.
985        """
986        bc = base_test.BaseTestClass(self.test_run_config)
987        bc.unpack_userparams(optional_thing='whatever')
988        self.assertEqual(bc.optional_thing, 'whatever')
989
990    def test_unpack_userparams_default_overwrite_by_optional_param_list(self):
991        """If an optional param is specified in kwargs, and the param is in the
992        config, the value should be the one in the config.
993        """
994        bc = base_test.BaseTestClass(self.test_run_config)
995        bc.unpack_userparams(some_param='whatever')
996        expected_value = self.test_run_config.user_params['some_param']
997        self.assertEqual(bc.some_param, expected_value)
998
999    def test_unpack_userparams_default_overwrite_by_required_param_list(self):
1000        """If an optional param is specified in kwargs, the param is in the
1001        required param list, and the param is not specified in the config, the
1002        param's alue should be the default value and there should be no error
1003        thrown.
1004        """
1005        bc = base_test.BaseTestClass(self.test_run_config)
1006        bc.unpack_userparams(req_param_names=['a_kwarg_param'],
1007                             a_kwarg_param='whatever')
1008        self.assertEqual(bc.a_kwarg_param, 'whatever')
1009
1010    def test_unpack_userparams_optional_missing(self):
1011        """Missing an optional param should not raise an error."""
1012        opt = ['something']
1013        bc = base_test.BaseTestClass(self.test_run_config)
1014        bc.unpack_userparams(opt_param_names=opt)
1015
1016    def test_unpack_userparams_basic(self):
1017        """Required and optional params are unpacked properly."""
1018        required = ['something']
1019        optional = ['something_else']
1020        configs = self.test_run_config.copy()
1021        configs.user_params['something'] = 42
1022        configs.user_params['something_else'] = 53
1023        bc = base_test.BaseTestClass(configs)
1024        bc.unpack_userparams(req_param_names=required,
1025                             opt_param_names=optional)
1026        self.assertEqual(bc.something, 42)
1027        self.assertEqual(bc.something_else, 53)
1028
1029    def test_unpack_userparams_default_overwrite(self):
1030        default_arg_val = 'haha'
1031        actual_arg_val = 'wawa'
1032        arg_name = 'arg1'
1033        configs = self.test_run_config.copy()
1034        configs.user_params[arg_name] = actual_arg_val
1035        bc = base_test.BaseTestClass(configs)
1036        bc.unpack_userparams(opt_param_names=[arg_name], arg1=default_arg_val)
1037        self.assertEqual(bc.arg1, actual_arg_val)
1038
1039    def test_unpack_userparams_default_None(self):
1040        bc = base_test.BaseTestClass(self.test_run_config)
1041        bc.unpack_userparams(arg1='haha')
1042        self.assertEqual(bc.arg1, 'haha')
1043
1044    def test_register_controller_no_config(self):
1045        base_cls = base_test.BaseTestClass(self.test_run_config)
1046        with self.assertRaisesRegexp(signals.ControllerError,
1047                                     'No corresponding config found for'):
1048            base_cls.register_controller(mock_controller)
1049
1050    def test_register_optional_controller_no_config(self):
1051        base_cls = base_test.BaseTestClass(self.test_run_config)
1052        self.assertIsNone(
1053            base_cls.register_controller(mock_controller, required=False))
1054
1055    def test_register_controller_third_party_dup_register(self):
1056        """Verifies correctness of registration, internal tally of controllers
1057        objects, and the right error happen when a controller module is
1058        registered twice.
1059        """
1060        mock_test_config = self.test_run_config.copy()
1061        mock_ctrlr_config_name = mock_controller.ACTS_CONTROLLER_CONFIG_NAME
1062        mock_test_config.controller_configs[mock_ctrlr_config_name] = [
1063            'magic1', 'magic2'
1064        ]
1065        base_cls = base_test.BaseTestClass(mock_test_config)
1066        base_cls.register_controller(mock_controller)
1067        registered_name = 'mock_controller'
1068        controller_objects = base_cls._controller_manager._controller_objects
1069        self.assertTrue(registered_name in controller_objects)
1070        mock_ctrlrs = controller_objects[registered_name]
1071        self.assertEqual(mock_ctrlrs[0].magic, 'magic1')
1072        self.assertEqual(mock_ctrlrs[1].magic, 'magic2')
1073        expected_msg = 'Controller module .* has already been registered.'
1074        with self.assertRaisesRegexp(signals.ControllerError, expected_msg):
1075            base_cls.register_controller(mock_controller)
1076
1077    def test_register_optional_controller_third_party_dup_register(self):
1078        """Verifies correctness of registration, internal tally of controllers
1079        objects, and the right error happen when an optional controller module
1080        is registered twice.
1081        """
1082        mock_test_config = self.test_run_config.copy()
1083        mock_ctrlr_config_name = mock_controller.ACTS_CONTROLLER_CONFIG_NAME
1084        mock_test_config.controller_configs[mock_ctrlr_config_name] = [
1085            'magic1', 'magic2'
1086        ]
1087        base_cls = base_test.BaseTestClass(mock_test_config)
1088        base_cls.register_controller(mock_controller, required=False)
1089        expected_msg = 'Controller module .* has already been registered.'
1090        with self.assertRaisesRegexp(signals.ControllerError, expected_msg):
1091            base_cls.register_controller(mock_controller, required=False)
1092
1093    def test_register_controller_builtin_dup_register(self):
1094        """Same as test_register_controller_third_party_dup_register, except
1095        this is for a builtin controller module.
1096        """
1097        mock_test_config = self.test_run_config.copy()
1098        mock_ctrlr_config_name = mock_controller.ACTS_CONTROLLER_CONFIG_NAME
1099        mock_ref_name = 'haha'
1100        setattr(mock_controller, 'ACTS_CONTROLLER_REFERENCE_NAME',
1101                mock_ref_name)
1102        try:
1103            mock_ctrlr_ref_name = mock_controller.ACTS_CONTROLLER_REFERENCE_NAME
1104            mock_test_config.controller_configs[mock_ctrlr_config_name] = [
1105                'magic1', 'magic2'
1106            ]
1107            base_cls = base_test.BaseTestClass(mock_test_config)
1108            base_cls.register_controller(mock_controller, builtin=True)
1109            self.assertTrue(hasattr(base_cls, mock_ref_name))
1110            self.assertTrue(mock_controller.__name__ in
1111                            base_cls._controller_manager._controller_objects)
1112            mock_ctrlrs = getattr(base_cls, mock_ctrlr_ref_name)
1113            self.assertEqual(mock_ctrlrs[0].magic, 'magic1')
1114            self.assertEqual(mock_ctrlrs[1].magic, 'magic2')
1115            expected_msg = 'Controller module .* has already been registered.'
1116            with self.assertRaisesRegexp(signals.ControllerError,
1117                                         expected_msg):
1118                base_cls.register_controller(mock_controller, builtin=True)
1119        finally:
1120            delattr(mock_controller, 'ACTS_CONTROLLER_REFERENCE_NAME')
1121
1122    def test_register_controller_no_get_info(self):
1123        mock_test_config = self.test_run_config.copy()
1124        mock_ctrlr_config_name = mock_controller.ACTS_CONTROLLER_CONFIG_NAME
1125        mock_ref_name = 'haha'
1126        get_info = getattr(mock_controller, 'get_info')
1127        delattr(mock_controller, 'get_info')
1128        try:
1129            mock_test_config.controller_configs[mock_ctrlr_config_name] = [
1130                'magic1', 'magic2'
1131            ]
1132            base_cls = base_test.BaseTestClass(mock_test_config)
1133            base_cls.register_controller(mock_controller)
1134            self.assertEqual(base_cls.results.controller_info, [])
1135        finally:
1136            setattr(mock_controller, 'get_info', get_info)
1137
1138    def test_register_controller_return_value(self):
1139        mock_test_config = self.test_run_config.copy()
1140        mock_ctrlr_config_name = mock_controller.ACTS_CONTROLLER_CONFIG_NAME
1141        mock_test_config.controller_configs[mock_ctrlr_config_name] = [
1142            'magic1', 'magic2'
1143        ]
1144        base_cls = base_test.BaseTestClass(mock_test_config)
1145        magic_devices = base_cls.register_controller(mock_controller)
1146        self.assertEqual(magic_devices[0].magic, 'magic1')
1147        self.assertEqual(magic_devices[1].magic, 'magic2')
1148
1149
1150if __name__ == '__main__':
1151    unittest.main()
1152