1#!/usr/bin/env python3
2#
3# Copyright (C) 2016 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#   http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import abc
18import argparse
19import filecmp
20import os
21import shlex
22import shutil
23import subprocess
24import sys
25
26from glob import glob
27from subprocess import DEVNULL
28from tempfile import mkdtemp
29
30sys.path.append(os.path.dirname(os.path.dirname(
31    os.path.realpath(__file__))))
32
33from common.common import RetCode
34from common.common import CommandListToCommandString
35from common.common import FatalError
36from common.common import GetEnvVariableOrError
37from common.common import RunCommand
38from common.common import RunCommandForOutput
39from common.common import DeviceTestEnv
40
41# Return codes supported by bisection bug search.
42BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT)
43
44
45def GetExecutionModeRunner(dexer, debug_info, device, mode):
46  """Returns a runner for the given execution mode.
47
48  Args:
49    dexer: string, defines dexer
50    debug_info: boolean, if True include debugging info
51    device: string, target device serial number (or None)
52    mode: string, execution mode
53  Returns:
54    TestRunner with given execution mode
55  Raises:
56    FatalError: error for unknown execution mode
57  """
58  if mode == 'ri':
59    return TestRunnerRIOnHost(debug_info)
60  if mode == 'hint':
61    return TestRunnerArtIntOnHost(dexer, debug_info)
62  if mode == 'hopt':
63    return TestRunnerArtOptOnHost(dexer, debug_info)
64  if mode == 'tint':
65    return TestRunnerArtIntOnTarget(dexer, debug_info, device)
66  if mode == 'topt':
67    return TestRunnerArtOptOnTarget(dexer, debug_info, device)
68  raise FatalError('Unknown execution mode')
69
70
71#
72# Execution mode classes.
73#
74
75
76class TestRunner(object):
77  """Abstraction for running a test in a particular execution mode."""
78  __meta_class__ = abc.ABCMeta
79
80  @abc.abstractproperty
81  def description(self):
82    """Returns a description string of the execution mode."""
83
84  @abc.abstractproperty
85  def id(self):
86    """Returns a short string that uniquely identifies the execution mode."""
87
88  @property
89  def output_file(self):
90    return self.id + '_out.txt'
91
92  @abc.abstractmethod
93  def GetBisectionSearchArgs(self):
94    """Get arguments to pass to bisection search tool.
95
96    Returns:
97      list of strings - arguments for bisection search tool, or None if
98      runner is not bisectable
99    """
100
101  @abc.abstractmethod
102  def CompileAndRunTest(self):
103    """Compile and run the generated test.
104
105    Ensures that the current Test.java in the temporary directory is compiled
106    and executed under the current execution mode. On success, transfers the
107    generated output to the file self.output_file in the temporary directory.
108
109    Most nonzero return codes are assumed non-divergent, since systems may
110    exit in different ways. This is enforced by normalizing return codes.
111
112    Returns:
113      normalized return code
114    """
115
116
117class TestRunnerWithHostCompilation(TestRunner):
118  """Abstract test runner that supports compilation on host."""
119
120  def  __init__(self, dexer, debug_info):
121    """Constructor for the runner with host compilation.
122
123    Args:
124      dexer: string, defines dexer
125      debug_info: boolean, if True include debugging info
126    """
127    self._dexer = dexer
128    self._debug_info = debug_info
129
130  def CompileOnHost(self):
131    if self._dexer == 'dx' or self._dexer == 'd8':
132      dbg = '-g' if self._debug_info else '-g:none'
133      if RunCommand(['javac', '--release=8', dbg, 'Test.java'],
134                    out=None, err=None, timeout=30) == RetCode.SUCCESS:
135        dx = 'dx' if self._dexer == 'dx' else 'd8-compat-dx'
136        retc = RunCommand([dx, '--dex', '--output=classes.dex'] + glob('*.class'),
137                          out=None, err='dxerr.txt', timeout=30)
138      else:
139        retc = RetCode.NOTCOMPILED
140    else:
141      raise FatalError('Unknown dexer: ' + self._dexer)
142    return retc
143
144
145class TestRunnerRIOnHost(TestRunner):
146  """Concrete test runner of the reference implementation on host."""
147
148  def  __init__(self, debug_info):
149    """Constructor for the runner with host compilation.
150
151    Args:
152      debug_info: boolean, if True include debugging info
153    """
154    self._debug_info = debug_info
155
156  @property
157  def description(self):
158    return 'RI on host'
159
160  @property
161  def id(self):
162    return 'RI'
163
164  def CompileAndRunTest(self):
165    dbg = '-g' if self._debug_info else '-g:none'
166    if RunCommand(['javac', '--release=8', dbg, 'Test.java'],
167                  out=None, err=None, timeout=30) == RetCode.SUCCESS:
168      retc = RunCommand(['java', 'Test'], self.output_file, err=None)
169    else:
170      retc = RetCode.NOTCOMPILED
171    return retc
172
173  def GetBisectionSearchArgs(self):
174    return None
175
176
177class TestRunnerArtOnHost(TestRunnerWithHostCompilation):
178  """Abstract test runner of Art on host."""
179
180  def  __init__(self, dexer, debug_info, extra_args=None):
181    """Constructor for the Art on host tester.
182
183    Args:
184      dexer: string, defines dexer
185      debug_info: boolean, if True include debugging info
186      extra_args: list of strings, extra arguments for dalvikvm
187    """
188    super().__init__(dexer, debug_info)
189    self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex']
190    if extra_args is not None:
191      self._art_cmd += extra_args
192    self._art_cmd.append('Test')
193
194  def CompileAndRunTest(self):
195    if self.CompileOnHost() == RetCode.SUCCESS:
196      retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt')
197    else:
198      retc = RetCode.NOTCOMPILED
199    return retc
200
201
202class TestRunnerArtIntOnHost(TestRunnerArtOnHost):
203  """Concrete test runner of interpreter mode Art on host."""
204
205  def  __init__(self, dexer, debug_info):
206    """Constructor for the Art on host tester (interpreter).
207
208    Args:
209      dexer: string, defines dexer
210      debug_info: boolean, if True include debugging info
211   """
212    super().__init__(dexer, debug_info, ['-Xint'])
213
214  @property
215  def description(self):
216    return 'Art interpreter on host'
217
218  @property
219  def id(self):
220    return 'HInt'
221
222  def GetBisectionSearchArgs(self):
223    return None
224
225
226class TestRunnerArtOptOnHost(TestRunnerArtOnHost):
227  """Concrete test runner of optimizing compiler mode Art on host."""
228
229  def  __init__(self, dexer, debug_info):
230    """Constructor for the Art on host tester (optimizing).
231
232    Args:
233      dexer: string, defines dexer
234      debug_info: boolean, if True include debugging info
235   """
236    super().__init__(dexer, debug_info, None)
237
238  @property
239  def description(self):
240    return 'Art optimizing on host'
241
242  @property
243  def id(self):
244    return 'HOpt'
245
246  def GetBisectionSearchArgs(self):
247    cmd_str = CommandListToCommandString(
248        self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:])
249    return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
250
251
252class TestRunnerArtOnTarget(TestRunnerWithHostCompilation):
253  """Abstract test runner of Art on target."""
254
255  def  __init__(self, dexer, debug_info, device, extra_args=None):
256    """Constructor for the Art on target tester.
257
258    Args:
259      dexer: string, defines dexer
260      debug_info: boolean, if True include debugging info
261      device: string, target device serial number (or None)
262      extra_args: list of strings, extra arguments for dalvikvm
263    """
264    super().__init__(dexer, debug_info)
265    self._test_env = DeviceTestEnv('jfuzz_', specific_device=device)
266    self._dalvik_cmd = ['dalvikvm']
267    if extra_args is not None:
268      self._dalvik_cmd += extra_args
269    self._device = device
270    self._device_classpath = None
271
272  def CompileAndRunTest(self):
273    if self.CompileOnHost() == RetCode.SUCCESS:
274      self._device_classpath = self._test_env.PushClasspath('classes.dex')
275      cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']
276      (output, retc) = self._test_env.RunCommand(
277          cmd, {'ANDROID_LOG_TAGS': '*:s'})
278      with open(self.output_file, 'w') as run_out:
279        run_out.write(output)
280    else:
281      retc = RetCode.NOTCOMPILED
282    return retc
283
284  def GetBisectionSearchArgs(self):
285    cmd_str = CommandListToCommandString(
286        self._dalvik_cmd + ['-cp',self._device_classpath, 'Test'])
287    cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
288    if self._device:
289      cmd += ['--device-serial', self._device]
290    else:
291      cmd.append('--device')
292    return cmd
293
294
295class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget):
296  """Concrete test runner of interpreter mode Art on target."""
297
298  def  __init__(self, dexer, debug_info, device):
299    """Constructor for the Art on target tester (interpreter).
300
301    Args:
302      dexer: string, defines dexer
303      debug_info: boolean, if True include debugging info
304      device: string, target device serial number (or None)
305    """
306    super().__init__(dexer, debug_info, device, ['-Xint'])
307
308  @property
309  def description(self):
310    return 'Art interpreter on target'
311
312  @property
313  def id(self):
314    return 'TInt'
315
316  def GetBisectionSearchArgs(self):
317    return None
318
319
320class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget):
321  """Concrete test runner of optimizing compiler mode Art on target."""
322
323  def  __init__(self, dexer, debug_info, device):
324    """Constructor for the Art on target tester (optimizing).
325
326    Args:
327      dexer: string, defines dexer
328      debug_info: boolean, if True include debugging info
329      device: string, target device serial number (or None)
330    """
331    super().__init__(dexer, debug_info, device, None)
332
333  @property
334  def description(self):
335    return 'Art optimizing on target'
336
337  @property
338  def id(self):
339    return 'TOpt'
340
341  def GetBisectionSearchArgs(self):
342    cmd_str = CommandListToCommandString(
343        self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'])
344    cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
345    if self._device:
346      cmd += ['--device-serial', self._device]
347    else:
348      cmd.append('--device')
349    return cmd
350
351
352#
353# Tester class.
354#
355
356
357class JFuzzTester(object):
358  """Tester that runs JFuzz many times and report divergences."""
359
360  def  __init__(self, num_tests, device, mode1, mode2, jfuzz_args,
361                report_script, true_divergence_only, dexer, debug_info):
362    """Constructor for the tester.
363
364    Args:
365      num_tests: int, number of tests to run
366      device: string, target device serial number (or None)
367      mode1: string, execution mode for first runner
368      mode2: string, execution mode for second runner
369      jfuzz_args: list of strings, additional arguments for jfuzz
370      report_script: string, path to script called for each divergence
371      true_divergence_only: boolean, if True don't bisect timeout divergences
372      dexer: string, defines dexer
373      debug_info: boolean, if True include debugging info
374    """
375    self._num_tests = num_tests
376    self._device = device
377    self._runner1 = GetExecutionModeRunner(dexer, debug_info, device, mode1)
378    self._runner2 = GetExecutionModeRunner(dexer, debug_info, device, mode2)
379    self._jfuzz_args = jfuzz_args
380    self._report_script = report_script
381    self._true_divergence_only = true_divergence_only
382    self._dexer = dexer
383    self._debug_info = debug_info
384    self._save_dir = None
385    self._results_dir = None
386    self._jfuzz_dir = None
387    # Statistics.
388    self._test = 0
389    self._num_success = 0
390    self._num_not_compiled = 0
391    self._num_not_run = 0
392    self._num_timed_out = 0
393    self._num_divergences = 0
394
395  def __enter__(self):
396    """On entry, enters new temp directory after saving current directory.
397
398    Raises:
399      FatalError: error when temp directory cannot be constructed
400    """
401    self._save_dir = os.getcwd()
402    self._results_dir = mkdtemp(dir='/tmp/')
403    self._jfuzz_dir = mkdtemp(dir=self._results_dir)
404    if self._results_dir is None or self._jfuzz_dir is None:
405      raise FatalError('Cannot obtain temp directory')
406    os.chdir(self._jfuzz_dir)
407    return self
408
409  def __exit__(self, etype, evalue, etraceback):
410    """On exit, re-enters previously saved current directory and cleans up."""
411    os.chdir(self._save_dir)
412    shutil.rmtree(self._jfuzz_dir)
413    if self._num_divergences == 0:
414      shutil.rmtree(self._results_dir)
415
416  def Run(self):
417    """Runs JFuzz many times and report divergences."""
418    print()
419    print('**\n**** JFuzz Testing\n**')
420    print()
421    print('#Tests    :', self._num_tests)
422    print('Device    :', self._device)
423    print('Directory :', self._results_dir)
424    print('Exec-mode1:', self._runner1.description)
425    print('Exec-mode2:', self._runner2.description)
426    print('Dexer     :', self._dexer)
427    print('Debug-info:', self._debug_info)
428    print()
429    self.ShowStats()
430    for self._test in range(1, self._num_tests + 1):
431      self.RunJFuzzTest()
432      self.ShowStats()
433    if self._num_divergences == 0:
434      print('\n\nsuccess (no divergences)\n')
435    else:
436      print('\n\nfailure (divergences)\n')
437
438  def ShowStats(self):
439    """Shows current statistics (on same line) while tester is running."""
440    print('\rTests:', self._test,
441          'Success:', self._num_success,
442          'Not-compiled:', self._num_not_compiled,
443          'Not-run:', self._num_not_run,
444          'Timed-out:', self._num_timed_out,
445          'Divergences:', self._num_divergences,
446          end='')
447    sys.stdout.flush()
448
449  def RunJFuzzTest(self):
450    """Runs a single JFuzz test, comparing two execution modes."""
451    self.ConstructTest()
452    retc1 = self._runner1.CompileAndRunTest()
453    retc2 = self._runner2.CompileAndRunTest()
454    self.CheckForDivergence(retc1, retc2)
455    self.CleanupTest()
456
457  def ConstructTest(self):
458    """Use JFuzz to generate next Test.java test.
459
460    Raises:
461      FatalError: error when jfuzz fails
462    """
463    if (RunCommand(['jfuzz'] + self._jfuzz_args, out='Test.java', err=None)
464          != RetCode.SUCCESS):
465      raise FatalError('Unexpected error while running JFuzz')
466
467  def CheckForDivergence(self, retc1, retc2):
468    """Checks for divergences and updates statistics.
469
470    Args:
471      retc1: int, normalized return code of first runner
472      retc2: int, normalized return code of second runner
473    """
474    if retc1 == retc2:
475      # No divergence in return code.
476      if retc1 == RetCode.SUCCESS:
477        # Both compilations and runs were successful, inspect generated output.
478        runner1_out = self._runner1.output_file
479        runner2_out = self._runner2.output_file
480        if not filecmp.cmp(runner1_out, runner2_out, shallow=False):
481          # Divergence in output.
482          self.ReportDivergence(retc1, retc2, is_output_divergence=True)
483        else:
484          # No divergence in output.
485          self._num_success += 1
486      elif retc1 == RetCode.TIMEOUT:
487        self._num_timed_out += 1
488      elif retc1 == RetCode.NOTCOMPILED:
489        self._num_not_compiled += 1
490      else:
491        self._num_not_run += 1
492    else:
493      # Divergence in return code.
494      if self._true_divergence_only:
495        # When only true divergences are requested, any divergence in return
496        # code where one is a time out is treated as a regular time out.
497        if RetCode.TIMEOUT in (retc1, retc2):
498          self._num_timed_out += 1
499          return
500        # When only true divergences are requested, a runtime crash in just
501        # the RI is treated as if not run at all.
502        if retc1 == RetCode.ERROR and retc2 == RetCode.SUCCESS:
503          if self._runner1.GetBisectionSearchArgs() is None:
504            self._num_not_run += 1
505            return
506      self.ReportDivergence(retc1, retc2, is_output_divergence=False)
507
508  def GetCurrentDivergenceDir(self):
509    return self._results_dir + '/divergence' + str(self._num_divergences)
510
511  def ReportDivergence(self, retc1, retc2, is_output_divergence):
512    """Reports and saves a divergence."""
513    self._num_divergences += 1
514    print('\n' + str(self._num_divergences), end='')
515    if is_output_divergence:
516      print(' divergence in output')
517    else:
518      print(' divergence in return code: ' + retc1.name + ' vs. ' +
519            retc2.name)
520    # Save.
521    ddir = self.GetCurrentDivergenceDir()
522    os.mkdir(ddir)
523    for f in glob('*.txt') + ['Test.java']:
524      shutil.copy(f, ddir)
525    # Maybe run bisection bug search.
526    if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES:
527      self.MaybeBisectDivergence(retc1, retc2, is_output_divergence)
528    # Call reporting script.
529    if self._report_script:
530      self.RunReportScript(retc1, retc2, is_output_divergence)
531
532  def RunReportScript(self, retc1, retc2, is_output_divergence):
533    """Runs report script."""
534    try:
535      title = "Divergence between {0} and {1} (found with fuzz testing)".format(
536          self._runner1.description, self._runner2.description)
537      # Prepare divergence comment.
538      jfuzz_cmd_and_version = subprocess.check_output(
539          ['grep', '-o', 'jfuzz.*', 'Test.java'], universal_newlines=True)
540      (jfuzz_cmd_str, jfuzz_ver) = jfuzz_cmd_and_version.split('(')
541      # Strip right parenthesis and new line.
542      jfuzz_ver = jfuzz_ver[:-2]
543      jfuzz_args = ['\'-{0}\''.format(arg)
544                    for arg in jfuzz_cmd_str.strip().split(' -')][1:]
545      wrapped_args = ['--jfuzz_arg={0}'.format(opt) for opt in jfuzz_args]
546      repro_cmd_str = (os.path.basename(__file__) +
547                       ' --num_tests=1 --dexer=' + self._dexer +
548                       (' --debug_info ' if self._debug_info else ' ') +
549                       ' '.join(wrapped_args))
550      comment = 'jfuzz {0}\nReproduce test:\n{1}\nReproduce divergence:\n{2}\n'.format(
551          jfuzz_ver, jfuzz_cmd_str, repro_cmd_str)
552      if is_output_divergence:
553        (output, _, _) = RunCommandForOutput(
554            ['diff', self._runner1.output_file, self._runner2.output_file],
555            None, subprocess.PIPE, subprocess.STDOUT)
556        comment += 'Diff:\n' + output
557      else:
558        comment += '{0} vs {1}\n'.format(retc1, retc2)
559      # Prepare report script command.
560      script_cmd = [self._report_script, title, comment]
561      ddir = self.GetCurrentDivergenceDir()
562      bisection_out_files = glob(ddir + '/*_bisection_out.txt')
563      if bisection_out_files:
564        script_cmd += ['--bisection_out', bisection_out_files[0]]
565      subprocess.check_call(script_cmd, stdout=DEVNULL, stderr=DEVNULL)
566    except subprocess.CalledProcessError as err:
567      print('Failed to run report script.\n', err)
568
569  def RunBisectionSearch(self, args, expected_retcode, expected_output,
570                         runner_id):
571    ddir = self.GetCurrentDivergenceDir()
572    outfile_path = ddir + '/' + runner_id + '_bisection_out.txt'
573    logfile_path = ddir + '/' + runner_id + '_bisection_log.txt'
574    errfile_path = ddir + '/' + runner_id + '_bisection_err.txt'
575    args = list(args) + ['--logfile', logfile_path, '--cleanup']
576    args += ['--expected-retcode', expected_retcode.name]
577    if expected_output:
578      args += ['--expected-output', expected_output]
579    bisection_search_path = os.path.join(
580        GetEnvVariableOrError('ANDROID_BUILD_TOP'),
581        'art/tools/bisection_search/bisection_search.py')
582    if RunCommand([bisection_search_path] + args, out=outfile_path,
583                  err=errfile_path, timeout=300) == RetCode.TIMEOUT:
584      print('Bisection search TIMEOUT')
585
586  def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence):
587    bisection_args1 = self._runner1.GetBisectionSearchArgs()
588    bisection_args2 = self._runner2.GetBisectionSearchArgs()
589    if is_output_divergence:
590      maybe_output1 = self._runner1.output_file
591      maybe_output2 = self._runner2.output_file
592    else:
593      maybe_output1 = maybe_output2 = None
594    if bisection_args1 is not None:
595      self.RunBisectionSearch(bisection_args1, retc2, maybe_output2,
596                              self._runner1.id)
597    if bisection_args2 is not None:
598      self.RunBisectionSearch(bisection_args2, retc1, maybe_output1,
599                              self._runner2.id)
600
601  def CleanupTest(self):
602    """Cleans up after a single test run."""
603    for file_name in os.listdir(self._jfuzz_dir):
604      file_path = os.path.join(self._jfuzz_dir, file_name)
605      if os.path.isfile(file_path):
606        os.unlink(file_path)
607      elif os.path.isdir(file_path):
608        shutil.rmtree(file_path)
609
610
611def main():
612  # Handle arguments.
613  parser = argparse.ArgumentParser()
614  parser.add_argument('--num_tests', default=10000, type=int,
615                      help='number of tests to run')
616  parser.add_argument('--device', help='target device serial number')
617  parser.add_argument('--mode1', default='ri',
618                      help='execution mode 1 (default: ri)')
619  parser.add_argument('--mode2', default='hopt',
620                      help='execution mode 2 (default: hopt)')
621  parser.add_argument('--report_script',
622                      help='script called for each divergence')
623  parser.add_argument('--jfuzz_arg', default=[], dest='jfuzz_args',
624                      action='append',
625                      help='argument for jfuzz')
626  parser.add_argument('--true_divergence', default=False, action='store_true',
627                      help='do not bisect timeout divergences')
628  parser.add_argument('--dexer', default='dx', type=str,
629                      help='defines dexer as dx or d8 (default: dx)')
630  parser.add_argument('--debug_info', default=False, action='store_true',
631                      help='include debugging info')
632  args = parser.parse_args()
633  if args.mode1 == args.mode2:
634    raise FatalError('Identical execution modes given')
635  # Run the JFuzz tester.
636  with JFuzzTester(args.num_tests,
637                   args.device,
638                   args.mode1, args.mode2,
639                   args.jfuzz_args,
640                   args.report_script,
641                   args.true_divergence,
642                   args.dexer,
643                   args.debug_info) as fuzzer:
644    fuzzer.Run()
645
646if __name__ == '__main__':
647  main()
648