1#!/usr/bin/env python3
2#
3#   Copyright 2016 - The Android Open Source Project
4#
5#   Licensed under the Apache License, Version 2.0 (the "License");
6#   you may not use this file except in compliance with the License.
7#   You may obtain a copy of the License at
8#
9#       http://www.apache.org/licenses/LICENSE-2.0
10#
11#   Unless required by applicable law or agreed to in writing, software
12#   distributed under the License is distributed on an "AS IS" BASIS,
13#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14#   See the License for the specific language governing permissions and
15#   limitations under the License.
16import shutil
17import tempfile
18import unittest
19import warnings
20from unittest import TestCase
21
22from mobly.config_parser import TestRunConfig
23
24from acts.base_test import BaseTestClass
25from acts.metrics.loggers.blackbox import BlackboxMetricLogger
26from acts.test_runner import TestRunner
27from mock import Mock
28from mock import patch
29
30COMPILE_PROTO = 'acts.metrics.logger.MetricLogger._compile_proto'
31GET_CONTEXT_FOR_EVENT = 'acts.metrics.logger.get_context_for_event'
32PROTO_METRIC_PUBLISHER = 'acts.metrics.logger.ProtoMetricPublisher'
33
34
35class BlackboxMetricLoggerTest(TestCase):
36    """Unit tests for BlackboxMetricLogger."""
37
38    TEST_METRIC_NAME = "metric_name"
39    TEST_FILE_NAME = "blackbox_metric_name"
40
41    def setUp(self):
42        self.proto_module = Mock()
43        self.event = Mock()
44        self.context = Mock()
45        self.publisher = Mock()
46        self._get_blackbox_identifier = lambda: str(id(self.context))
47
48    @patch(COMPILE_PROTO)
49    def test_default_init_attributes(self, compile_proto):
50        metric_name = Mock()
51        compile_proto.return_value = self.proto_module
52
53        logger = BlackboxMetricLogger(metric_name)
54
55        self.assertEqual(logger.metric_name, metric_name)
56        self.assertEqual(logger.proto_module, self.proto_module)
57        self.assertIsNone(logger.metric_key)
58
59    @patch(COMPILE_PROTO)
60    def test_init_with_params(self, compile_proto):
61        metric_name = Mock()
62        metric_key = Mock()
63
64        logger = BlackboxMetricLogger(metric_name, metric_key=metric_key)
65
66        self.assertEqual(logger.metric_key, metric_key)
67
68    @patch(PROTO_METRIC_PUBLISHER)
69    @patch(GET_CONTEXT_FOR_EVENT)
70    @patch(COMPILE_PROTO)
71    def test_init_with_event(self, compile_proto, get_context, publisher_cls):
72        metric_name = Mock()
73
74        logger = BlackboxMetricLogger(metric_name, event=self.event)
75
76        self.assertIsNotNone(logger.context)
77        self.assertIsNotNone(logger.publisher)
78
79    @patch(COMPILE_PROTO)
80    def test_end_populates_result(self, compile_proto):
81        result = Mock()
82        compile_proto.return_value = self.proto_module
83        self.proto_module.ActsBlackboxMetricResult.return_value = result
84
85        logger = BlackboxMetricLogger(self.TEST_METRIC_NAME)
86        logger.context = self.context
87        logger.publisher = self.publisher
88        logger.context.identifier = 'Class.test'
89        logger.metric_value = 'foo'
90
91        logger.end(self.event)
92
93        self.assertEqual(result.test_identifier, 'Class#test')
94        self.assertEqual(result.metric_key,
95                         '%s.%s' % ('Class#test', self.TEST_METRIC_NAME))
96        self.assertEqual(result.metric_value, logger.metric_value)
97
98    @patch(COMPILE_PROTO)
99    def test_end_uses_metric_value_on_metric_value_not_none(
100        self, compile_proto):
101        result = Mock()
102        expected_result = Mock()
103        compile_proto.return_value = self.proto_module
104        self.proto_module.ActsBlackboxMetricResult.return_value = result
105
106        logger = BlackboxMetricLogger(self.TEST_METRIC_NAME)
107        logger.context = self.context
108        logger.context.identifier = 'Class.test'
109        logger.publisher = self.publisher
110        logger.metric_value = expected_result
111        logger.end(self.event)
112
113        self.assertEqual(result.metric_value, expected_result)
114
115    @patch(COMPILE_PROTO)
116    def test_end_uses_custom_metric_key(self, compile_proto):
117        result = Mock()
118        compile_proto.return_value = self.proto_module
119        self.proto_module.ActsBlackboxMetricResult.return_value = result
120        metric_key = 'metric_key'
121
122        logger = BlackboxMetricLogger(self.TEST_METRIC_NAME,
123                                      metric_key=metric_key)
124        logger.context = self.context
125        logger.publisher = self.publisher
126        logger._get_blackbox_identifier = self._get_blackbox_identifier
127        logger.metric_value = 'foo'
128
129        logger.end(self.event)
130
131        expected_metric_key = '%s.%s' % (metric_key, self.TEST_METRIC_NAME)
132        self.assertEqual(result.metric_key, expected_metric_key)
133
134    @patch('acts.metrics.loggers.blackbox.ProtoMetric')
135    @patch(COMPILE_PROTO)
136    def test_end_does_publish(self, compile_proto, proto_metric_cls):
137        result = Mock()
138        compile_proto.return_value = self.proto_module
139        self.proto_module.ActsBlackboxMetricResult.return_value = result
140        metric_key = 'metric_key'
141
142        logger = BlackboxMetricLogger(self.TEST_METRIC_NAME,
143                                      metric_key=metric_key)
144        logger.context = self.context
145        logger.publisher = self.publisher
146        logger._get_blackbox_identifier = self._get_blackbox_identifier
147        logger.metric_value = 'foo'
148
149        logger.end(self.event)
150
151        proto_metric_cls.assert_called_once_with(name=self.TEST_FILE_NAME,
152                                                 data=result)
153        self.publisher.publish.assert_called_once_with(
154            [proto_metric_cls.return_value])
155
156
157class _BaseTestClassWithCleanup(BaseTestClass):
158    """Subclass of ACTS base test that generates a temp directory for
159    proto compiler output and cleans up upon exit.
160    """
161    def __init__(self, controllers):
162        super().__init__(controllers)
163        self.proto_dir = tempfile.mkdtemp()
164
165    def __del__(self):
166        shutil.rmtree(self.proto_dir)
167
168
169class BlackboxMetricLoggerIntegrationTest(TestCase):
170    """Integration tests for BlackboxMetricLogger."""
171    def setUp(self):
172        warnings.simplefilter('ignore', ResourceWarning)
173
174    @patch('acts.test_runner.sys')
175    @patch('acts.test_runner.utils')
176    @patch('acts.test_runner.importlib')
177    def run_acts_test(self, test_class, importlib, utils, sys):
178        test_run_config = TestRunConfig()
179        test_run_config.testbed_name = 'SampleTestBed'
180        test_run_config.log_path = tempfile.mkdtemp()
181        test_run_config.controller_configs = {'testpaths': ['./']}
182
183        mock_module = Mock()
184        setattr(mock_module, test_class.__name__, test_class)
185        utils.find_files.return_value = [(None, None, None)]
186        importlib.import_module.return_value = mock_module
187        runner = TestRunner(test_run_config, [(
188            test_class.__name__,
189            None,
190        )])
191
192        runner.run()
193        runner.stop()
194        shutil.rmtree(test_run_config.log_path)
195        return runner
196
197    @patch('acts.metrics.logger.ProtoMetricPublisher')
198    def test_test_case_metric(self, publisher_cls):
199        result = 5.0
200
201        class MyTest(_BaseTestClassWithCleanup):
202            def __init__(self, controllers):
203                super().__init__(controllers)
204                self.tests = ('test_case', )
205                self.metric = BlackboxMetricLogger.for_test_case(
206                    'my_metric', compiler_out=self.proto_dir)
207
208            def test_case(self):
209                self.metric.metric_value = result
210
211        self.run_acts_test(MyTest)
212
213        args_list = publisher_cls().publish.call_args_list
214        self.assertEqual(len(args_list), 1)
215        metric = self.__get_only_arg(args_list[0])[0]
216        self.assertEqual(metric.name, 'blackbox_my_metric')
217        self.assertEqual(metric.data.test_identifier, 'MyTest#test_case')
218        self.assertEqual(metric.data.metric_key, 'MyTest#test_case.my_metric')
219        self.assertEqual(metric.data.metric_value, result)
220
221    @patch('acts.metrics.logger.ProtoMetricPublisher')
222    def test_multiple_test_case_metrics(self, publisher_cls):
223        result = 5.0
224
225        class MyTest(_BaseTestClassWithCleanup):
226            def __init__(self, controllers):
227                super().__init__(controllers)
228                self.tests = ('test_case', )
229                self.metric_1 = (BlackboxMetricLogger.for_test_case(
230                    'my_metric_1', compiler_out=self.proto_dir))
231                self.metric_2 = (BlackboxMetricLogger.for_test_case(
232                    'my_metric_2', compiler_out=self.proto_dir))
233
234            def test_case(self):
235                self.metric_1.metric_value = result
236                self.metric_2.metric_value = result
237
238        self.run_acts_test(MyTest)
239
240        args_list = publisher_cls().publish.call_args_list
241        self.assertEqual(len(args_list), 2)
242        metrics = [self.__get_only_arg(args)[0] for args in args_list]
243        self.assertEqual({metric.name
244                          for metric in metrics},
245                         {'blackbox_my_metric_1', 'blackbox_my_metric_2'})
246        self.assertEqual({metric.data.test_identifier
247                          for metric in metrics}, {'MyTest#test_case'})
248        self.assertEqual(
249            {metric.data.metric_key
250             for metric in metrics},
251            {'MyTest#test_case.my_metric_1', 'MyTest#test_case.my_metric_2'})
252        self.assertEqual({metric.data.metric_value
253                          for metric in metrics}, {result})
254
255    @patch('acts.metrics.logger.ProtoMetricPublisher')
256    def test_test_case_metric_with_custom_key(self, publisher_cls):
257        result = 5.0
258
259        class MyTest(_BaseTestClassWithCleanup):
260            def __init__(self, controllers):
261                super().__init__(controllers)
262                self.tests = ('test_case', )
263                self.metrics = BlackboxMetricLogger.for_test_case(
264                    'my_metric',
265                    metric_key='my_metric_key',
266                    compiler_out=self.proto_dir)
267
268            def test_case(self):
269                self.metrics.metric_value = result
270
271        self.run_acts_test(MyTest)
272
273        args_list = publisher_cls().publish.call_args_list
274        self.assertEqual(len(args_list), 1)
275        metric = self.__get_only_arg(args_list[0])[0]
276        self.assertEqual(metric.data.metric_key, 'my_metric_key.my_metric')
277
278    @patch('acts.metrics.logger.ProtoMetricPublisher')
279    def test_test_class_metric(self, publisher_cls):
280        publisher_cls().publish = Mock()
281        result_1 = 5.0
282        result_2 = 8.0
283
284        class MyTest(_BaseTestClassWithCleanup):
285            def __init__(self, controllers):
286                super().__init__(controllers)
287                self.tests = (
288                    'test_case_1',
289                    'test_case_2',
290                )
291                self.metric = BlackboxMetricLogger.for_test_class(
292                    'my_metric', compiler_out=self.proto_dir)
293
294            def setup_class(self):
295                self.metric.metric_value = 0
296
297            def test_case_1(self):
298                self.metric.metric_value += result_1
299
300            def test_case_2(self):
301                self.metric.metric_value += result_2
302
303        self.run_acts_test(MyTest)
304
305        args_list = publisher_cls().publish.call_args_list
306        self.assertEqual(len(args_list), 1)
307        metric = self.__get_only_arg(args_list[0])[0]
308        self.assertEqual(metric.data.metric_value, result_1 + result_2)
309        self.assertEqual(metric.data.test_identifier, MyTest.__name__)
310
311    def __get_only_arg(self, call_args):
312        self.assertEqual(len(call_args[0]) + len(call_args[1]), 1)
313        if len(call_args[0]) == 1:
314            return call_args[0][0]
315        return next(iter(call_args[1].values()))
316
317
318if __name__ == '__main__':
319    unittest.main()
320