1#!/bin/bash
2#
3# Copyright (C) 2014 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17# Exit on errors.
18set -e
19
20if [ ! -d libcore ]; then
21  echo "Script needs to be run at the root of the android tree"
22  exit 1
23fi
24
25# "Root" (actually "system") directory on device (in the case of
26# target testing).
27android_root=${ART_TEST_ANDROID_ROOT:-/system}
28
29function classes_jar_path {
30  local var="$1"
31  local suffix="jar"
32  if [ -z "$ANDROID_PRODUCT_OUT" ] ; then
33    local java_libraries=out/target/common/obj/JAVA_LIBRARIES
34  else
35    local java_libraries=${ANDROID_PRODUCT_OUT}/../../common/obj/JAVA_LIBRARIES
36  fi
37  echo "${java_libraries}/${var}_intermediates/classes.${suffix}"
38}
39
40function cparg {
41  for var
42  do
43    printf -- "--classpath $(classes_jar_path "$var") ";
44  done
45}
46
47function boot_classpath_arg {
48  local dir="$1"
49  local suffix="$2"
50  shift 2
51  printf -- "--vm-arg -Xbootclasspath"
52  for var
53  do
54    printf -- ":${dir}/${var}${suffix}.jar";
55  done
56}
57
58function usage {
59  local me=$(basename "${BASH_SOURCE[0]}")
60  (
61    cat << EOF
62  Usage: ${me} --mode=<mode> [options] [-- <package_to_test> ...]
63
64  Run libcore tests using the vogar testing tool.
65
66  Required parameters:
67    --mode=device|host|jvm Specify where tests should be run.
68
69  Optional parameters:
70    --debug                Use debug version of ART (device|host only).
71    --dry-run              Print vogar command-line, but do not run.
72    --no-getrandom         Ignore failures from getrandom() (for kernel < 3.17).
73    --no-jit               Disable JIT (device|host only).
74    --Xgc:gcstress         Enable GC stress configuration (device|host only).
75
76  The script passes unrecognized options to the command-line created for vogar.
77
78  The script runs a hardcoded list of libcore test packages by default. The user
79  may run a subset of packages by appending '--' followed by a list of package
80  names.
81
82  Examples:
83
84    1. Run full test suite on host:
85      ${me} --mode=host
86
87    2. Run full test suite on device:
88      ${me} --mode=device
89
90    3. Run tests only from the libcore.java.lang package on device:
91      ${me} --mode=device -- libcore.java.lang
92EOF
93  ) | sed -e 's/^  //' >&2 # Strip leading whitespace from heredoc.
94}
95
96# Packages that currently work correctly with the expectation files.
97working_packages=("libcore.android.system"
98                  "libcore.build"
99                  "libcore.dalvik.system"
100                  "libcore.java.awt"
101                  "libcore.java.lang"
102                  "libcore.java.math"
103                  "libcore.java.text"
104                  "libcore.java.util"
105                  "libcore.javax.crypto"
106                  "libcore.javax.net"
107                  "libcore.javax.security"
108                  "libcore.javax.sql"
109                  "libcore.javax.xml"
110                  "libcore.libcore.internal"
111                  "libcore.libcore.io"
112                  "libcore.libcore.net"
113                  "libcore.libcore.reflect"
114                  "libcore.libcore.util"
115                  "libcore.libcore.timezone"
116                  "libcore.sun.invoke"
117                  "libcore.sun.net"
118                  "libcore.sun.misc"
119                  "libcore.sun.security"
120                  "libcore.sun.util"
121                  "libcore.xml"
122                  "org.apache.harmony.annotation"
123                  "org.apache.harmony.crypto"
124                  "org.apache.harmony.luni"
125                  "org.apache.harmony.nio"
126                  "org.apache.harmony.regex"
127                  "org.apache.harmony.testframework"
128                  "org.apache.harmony.tests.java.io"
129                  "org.apache.harmony.tests.java.lang"
130                  "org.apache.harmony.tests.java.math"
131                  "org.apache.harmony.tests.java.util"
132                  "org.apache.harmony.tests.java.text"
133                  "org.apache.harmony.tests.javax.security"
134                  "tests.java.lang.String"
135                  "jsr166")
136
137# List of packages we could run, but don't have rights to revert
138# changes in case of failures.
139# "org.apache.harmony.security"
140
141#
142# Setup environment for running tests.
143#
144source build/envsetup.sh >&/dev/null # for get_build_var, setpaths
145setpaths # include platform prebuilt java, javac, etc in $PATH.
146
147# Note: This must start with the CORE_IMG_JARS in Android.common_path.mk
148# because that's what we use for compiling the core.art image.
149# It may contain additional modules from TEST_CORE_JARS.
150BOOT_CLASSPATH_JARS="core-oj core-libart core-icu4j okhttp bouncycastle apache-xml conscrypt"
151
152DEPS="core-tests jsr166-tests mockito-target"
153
154for lib in $DEPS
155do
156  if [[ ! -f "$(classes_jar_path "$lib")" ]]; then
157    echo "${lib} is missing. Before running, you must run art/tools/buildbot-build.sh"
158    exit 1
159  fi
160done
161
162#
163# Defaults affected by command-line parsing
164#
165
166# Use JIT compiling by default.
167use_jit=true
168
169gcstress=false
170debug=false
171dry_run=false
172
173# Run tests that use the getrandom() syscall? (Requires Linux 3.17+).
174getrandom=true
175
176# Execution mode specifies where to run tests (device|host|jvm).
177execution_mode=""
178
179# Default expectations file.
180expectations="--expectations art/tools/libcore_failures.txt"
181
182vogar_args=""
183while [ -n "$1" ]; do
184  case "$1" in
185    --mode=device)
186      # Use --mode=device_testdex not --mode=device for buildbot-build.sh.
187      # See commit 191cae33c7c24e for more details.
188      vogar_args="$vogar_args --mode=device_testdex"
189      vogar_args="$vogar_args --vm-arg -Ximage:/data/art-test/core.art"
190      vogar_args="$vogar_args $(boot_classpath_arg /system/framework -testdex $BOOT_CLASSPATH_JARS)"
191      execution_mode="device"
192      ;;
193    --mode=host)
194      # We explicitly give a wrong path for the image, to ensure vogar
195      # will create a boot image with the default compiler. Note that
196      # giving an existing image on host does not work because of
197      # classpath/resources differences when compiling the boot image.
198      vogar_args="$vogar_args $1 --vm-arg -Ximage:/non/existent/vogar.art"
199      execution_mode="host"
200      ;;
201    --mode=jvm)
202      vogar_args="$vogar_args $1"
203      execution_mode="jvm"
204      ;;
205    --no-getrandom)
206      getrandom=false
207      ;;
208    --no-jit)
209      use_jit=false
210      ;;
211    --debug)
212      vogar_args="$vogar_args --vm-arg -XXlib:libartd.so --vm-arg -XX:SlowDebug=true"
213      debug=true
214      ;;
215    -Xgc:gcstress)
216      vogar_args="$vogar_args $1"
217      gcstress=true
218      ;;
219    --dry-run)
220      dry_run=true
221      ;;
222    --)
223      shift
224      # Assume remaining elements are packages to test.
225      user_packages=("$@")
226      break
227      ;;
228    --help)
229      usage
230      exit 1
231      ;;
232    *)
233      vogar_args="$vogar_args $1"
234      ;;
235  esac
236  shift
237done
238
239if [ -z "$execution_mode" ]; then
240  usage
241  exit 1
242fi
243
244# Default timeout, gets overridden on device under gcstress.
245timeout_secs=480
246
247if [ $execution_mode = "device" ]; then
248  # Honor environment variable ART_TEST_CHROOT.
249  if [[ -n "$ART_TEST_CHROOT" ]]; then
250    # Set Vogar's `--chroot` option.
251    vogar_args="$vogar_args --chroot $ART_TEST_CHROOT"
252    vogar_args="$vogar_args --device-dir=/tmp"
253  else
254    # When not using a chroot on device, set Vogar's work directory to
255    # /data/local/tmp.
256    vogar_args="$vogar_args --device-dir=/data/local/tmp"
257  fi
258  vogar_args="$vogar_args --vm-command=$android_root/bin/art"
259
260  # Increase the timeout, as vogar cannot set individual test
261  # timeout when being asked to run packages, and some tests go above
262  # the default timeout.
263  if $gcstress; then
264    if $debug; then
265      timeout_secs=1440
266    else
267      timeout_secs=900
268    fi
269  fi
270fi  # $execution_mode = "device"
271
272if [ $execution_mode = "device" -o $execution_mode = "host" ]; then
273  # Add timeout to vogar command-line.
274  vogar_args="$vogar_args --timeout $timeout_secs"
275
276  # set the toolchain to use.
277  vogar_args="$vogar_args --toolchain d8 --language CUR"
278
279  # JIT settings.
280  if $use_jit; then
281    vogar_args="$vogar_args --vm-arg -Xcompiler-option --vm-arg --compiler-filter=quicken"
282  fi
283  vogar_args="$vogar_args --vm-arg -Xusejit:$use_jit"
284
285  # gcstress may lead to timeouts, so we need dedicated expectations files for it.
286  if $gcstress; then
287    expectations="$expectations --expectations art/tools/libcore_gcstress_failures.txt"
288    if $debug; then
289      expectations="$expectations --expectations art/tools/libcore_gcstress_debug_failures.txt"
290    fi
291  else
292    # We only run this package when user has not specified packages
293    # to run and not under gcstress as it can cause timeouts. See
294    # b/78228743.
295    working_packages+=("libcore.libcore.icu")
296  fi
297
298  if $getrandom; then :; else
299    # Ignore failures in tests that use the system calls not supported
300    # on fugu (Nexus Player, kernel version Linux 3.10).
301    expectations="$expectations --expectations art/tools/libcore_fugu_failures.txt"
302  fi
303fi
304
305if [ ! -t 1 ] ; then
306  # Suppress color codes if not attached to a terminal
307  vogar_args="$vogar_args --no-color"
308fi
309
310# Override working_packages if user provided specific packages to
311# test.
312if [[ ${#user_packages[@]} != 0 ]] ; then
313  working_packages=("${user_packages[@]}")
314fi
315
316# Run the tests using vogar.
317echo "Running tests for the following test packages:"
318echo ${working_packages[@]} | tr " " "\n"
319
320cmd="vogar $vogar_args $expectations $(cparg $DEPS) ${working_packages[@]}"
321echo "Running $cmd"
322$dry_run || eval $cmd
323