1#!/usr/bin/env python3
2
3import gzip
4import os
5import subprocess
6import sys
7import tempfile
8import collections
9
10
11SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
12
13try:
14    AOSP_DIR = os.environ['ANDROID_BUILD_TOP']
15except KeyError:
16    print('error: ANDROID_BUILD_TOP environment variable is not set.',
17          file=sys.stderr)
18    sys.exit(1)
19
20BUILTIN_HEADERS_DIR = (
21    os.path.join(AOSP_DIR, 'bionic', 'libc', 'include'),
22    os.path.join(AOSP_DIR, 'external', 'libcxx', 'include'),
23    os.path.join(AOSP_DIR, 'prebuilts', 'clang-tools', 'linux-x86',
24                 'clang-headers'),
25)
26
27EXPORTED_HEADERS_DIR = (
28    os.path.join(AOSP_DIR, 'development', 'vndk', 'tools', 'header-checker',
29                 'tests'),
30)
31
32SO_EXT = '.so'
33SOURCE_ABI_DUMP_EXT_END = '.lsdump'
34SOURCE_ABI_DUMP_EXT = SO_EXT + SOURCE_ABI_DUMP_EXT_END
35COMPRESSED_SOURCE_ABI_DUMP_EXT = SOURCE_ABI_DUMP_EXT + '.gz'
36VENDOR_SUFFIX = '.vendor'
37
38DEFAULT_CPPFLAGS = ['-x', 'c++', '-std=c++11']
39DEFAULT_CFLAGS = ['-std=gnu99']
40DEFAULT_HEADER_FLAGS = ["-dump-function-declarations"]
41DEFAULT_FORMAT = 'ProtobufTextFormat'
42
43
44class Target(object):
45    def __init__(self, is_2nd, product):
46        extra = '_2ND' if is_2nd else ''
47        build_vars_to_fetch = ['TARGET_ARCH',
48                               'TARGET{}_ARCH'.format(extra),
49                               'TARGET{}_ARCH_VARIANT'.format(extra),
50                               'TARGET{}_CPU_VARIANT'.format(extra)]
51        build_vars = get_build_vars_for_product(build_vars_to_fetch, product)
52        self.primary_arch = build_vars[0]
53        assert self.primary_arch != ''
54        self.arch = build_vars[1]
55        self.arch_variant = build_vars[2]
56        self.cpu_variant = build_vars[3]
57
58    def get_arch_str(self):
59        """Return a string that represents the architecture and the
60        architecture variant.
61
62        If TARGET_ARCH == TARGET_ARCH_VARIANT, soong makes targetArchVariant
63        empty. This is the case for aosp_x86_64.
64        """
65        if not self.arch_variant or self.arch_variant == self.arch:
66            arch_variant = ''
67        else:
68            arch_variant = '_' + self.arch_variant
69
70        return self.arch + arch_variant
71
72    def get_arch_cpu_str(self):
73        """Return a string that represents the architecture, the architecture
74        variant, and the CPU variant."""
75        if not self.cpu_variant or self.cpu_variant == 'generic':
76            cpu_variant = ''
77        else:
78            cpu_variant = '_' + self.cpu_variant
79
80        return self.get_arch_str() + cpu_variant
81
82
83def copy_reference_dump(lib_path, reference_dump_dir, compress):
84    reference_dump_path = os.path.join(
85        reference_dump_dir, os.path.basename(lib_path))
86    if compress:
87        reference_dump_path += '.gz'
88    os.makedirs(os.path.dirname(reference_dump_path), exist_ok=True)
89    output_content = read_output_content(lib_path, AOSP_DIR)
90    if compress:
91        with gzip.open(reference_dump_path, 'wb') as f:
92            f.write(bytes(output_content, 'utf-8'))
93    else:
94        with open(reference_dump_path, 'wb') as f:
95            f.write(bytes(output_content, 'utf-8'))
96    print('Created abi dump at', reference_dump_path)
97    return reference_dump_path
98
99
100def read_output_content(output_path, replace_str):
101    with open(output_path, 'r') as f:
102        return f.read().replace(replace_str, '')
103
104
105def run_header_abi_dumper(input_path, cflags=tuple(),
106                          export_include_dirs=EXPORTED_HEADERS_DIR,
107                          flags=tuple()):
108    """Run header-abi-dumper to dump ABI from `input_path` and return the
109    output."""
110    with tempfile.TemporaryDirectory() as tmp:
111        output_path = os.path.join(tmp, os.path.basename(input_path)) + '.dump'
112        run_header_abi_dumper_on_file(input_path, output_path,
113                                      export_include_dirs, cflags, flags)
114        return read_output_content(output_path, AOSP_DIR)
115
116
117def run_header_abi_dumper_on_file(input_path, output_path,
118                                  export_include_dirs=tuple(), cflags=tuple(),
119                                  flags=tuple()):
120    """Run header-abi-dumper to dump ABI from `input_path` and the output is
121    written to `output_path`."""
122    input_ext = os.path.splitext(input_path)[1]
123    cmd = ['header-abi-dumper', '-o', output_path, input_path]
124    for dir in export_include_dirs:
125        cmd += ['-I', dir]
126    cmd += flags
127    if '-output-format' not in flags:
128        cmd += ['-output-format', DEFAULT_FORMAT]
129    if input_ext == ".h":
130        cmd += DEFAULT_HEADER_FLAGS
131    cmd += ['--']
132    cmd += cflags
133    if input_ext in ('.cpp', '.cc', '.h'):
134        cmd += DEFAULT_CPPFLAGS
135    else:
136        cmd += DEFAULT_CFLAGS
137
138    for dir in BUILTIN_HEADERS_DIR:
139        cmd += ['-isystem', dir]
140    # The export include dirs imply local include dirs.
141    for dir in export_include_dirs:
142        cmd += ['-I', dir]
143    subprocess.check_call(cmd)
144
145
146def run_header_abi_linker(output_path, inputs, version_script, api, arch,
147                          flags=tuple()):
148    """Link inputs, taking version_script into account"""
149    cmd = ['header-abi-linker', '-o', output_path, '-v', version_script,
150           '-api', api, '-arch', arch]
151    cmd += flags
152    if '-input-format' not in flags:
153        cmd += ['-input-format', DEFAULT_FORMAT]
154    if '-output-format' not in flags:
155        cmd += ['-output-format', DEFAULT_FORMAT]
156    cmd += inputs
157    subprocess.check_call(cmd)
158    return read_output_content(output_path, AOSP_DIR)
159
160
161def make_targets(product, variant, targets):
162    make_cmd = ['build/soong/soong_ui.bash', '--make-mode', '-j',
163                'TARGET_PRODUCT=' + product, 'TARGET_BUILD_VARIANT=' + variant]
164    make_cmd += targets
165    subprocess.check_call(make_cmd, cwd=AOSP_DIR)
166
167
168def make_tree(product, variant):
169    """Build all lsdump files."""
170    return make_targets(product, variant, ['findlsdumps'])
171
172
173def make_libraries(product, variant, vndk_version, targets, libs):
174    """Build lsdump files for specific libs."""
175    lsdump_paths = read_lsdump_paths(product, variant, vndk_version, targets,
176                                     build=True)
177    make_target_paths = []
178    for name in libs:
179        if not (name in lsdump_paths and lsdump_paths[name]):
180            raise KeyError('Cannot find lsdump for %s.' % name)
181        for tag_path_dict in lsdump_paths[name].values():
182            make_target_paths.extend(tag_path_dict.values())
183    make_targets(product, variant, make_target_paths)
184
185
186def get_lsdump_paths_file_path(product, variant):
187    """Get the path to lsdump_paths.txt."""
188    product_out = get_build_vars_for_product(
189        ['PRODUCT_OUT'], product, variant)[0]
190    return os.path.join(product_out, 'lsdump_paths.txt')
191
192
193def _is_sanitizer_variation(variation):
194    """Check whether the variation is introduced by a sanitizer."""
195    return variation in {'asan', 'hwasan', 'tsan', 'intOverflow', 'cfi', 'scs'}
196
197
198def _get_module_variant_dir_name(tag, vndk_version, arch_cpu_str):
199    """Return the module variant directory name.
200
201    For example, android_x86_shared, android_vendor.R_arm_armv7-a-neon_shared.
202    """
203    if tag in ('LLNDK', 'NDK', 'PLATFORM'):
204        return 'android_%s_shared' % arch_cpu_str
205    if tag.startswith('VNDK'):
206        return 'android_vendor.%s_%s_shared' % (vndk_version, arch_cpu_str)
207    raise ValueError(tag + ' is not a known tag.')
208
209
210def _read_lsdump_paths(lsdump_paths_file_path, vndk_version, targets):
211    """Read lsdump paths from lsdump_paths.txt for each libname and variant.
212
213    This function returns a dictionary, {lib_name: {arch_cpu: {tag: path}}}.
214    For example,
215    {
216      "libc": {
217        "x86_x86_64": {
218          "NDK": "path/to/libc.so.lsdump"
219        }
220      }
221    }
222    """
223    lsdump_paths = collections.defaultdict(
224        lambda: collections.defaultdict(dict))
225    suffixes = collections.defaultdict(dict)
226
227    with open(lsdump_paths_file_path, 'r') as lsdump_paths_file:
228        for line in lsdump_paths_file:
229            tag, path = (x.strip() for x in line.split(':', 1))
230            if not path:
231                continue
232            dirname, filename = os.path.split(path)
233            if not filename.endswith(SOURCE_ABI_DUMP_EXT):
234                continue
235            libname = filename[:-len(SOURCE_ABI_DUMP_EXT)]
236            if not libname:
237                continue
238            variant = os.path.basename(dirname)
239            if not variant:
240                continue
241            for target in targets:
242                arch_cpu = target.get_arch_cpu_str()
243                prefix = _get_module_variant_dir_name(tag, vndk_version,
244                                                      arch_cpu)
245                if not variant.startswith(prefix):
246                    continue
247                new_suffix = variant[len(prefix):]
248                # Skip if the suffix contains APEX variations.
249                new_variations = [x for x in new_suffix.split('_') if x]
250                if new_variations and not all(_is_sanitizer_variation(x)
251                                              for x in new_variations):
252                    continue
253                old_suffix = suffixes[libname].get(arch_cpu)
254                if not old_suffix or new_suffix > old_suffix:
255                    lsdump_paths[libname][arch_cpu][tag] = path
256                    suffixes[libname][arch_cpu] = new_suffix
257    return lsdump_paths
258
259
260def read_lsdump_paths(product, variant, vndk_version, targets, build=True):
261    """Build lsdump_paths.txt and read the paths."""
262    lsdump_paths_file_path = get_lsdump_paths_file_path(product, variant)
263    if build:
264        make_targets(product, variant, [lsdump_paths_file_path])
265    lsdump_paths_file_abspath = os.path.join(AOSP_DIR, lsdump_paths_file_path)
266    return _read_lsdump_paths(lsdump_paths_file_abspath, vndk_version,
267                              targets)
268
269
270def find_lib_lsdumps(lsdump_paths, libs, target):
271    """Find the lsdump corresponding to libs for the given target.
272
273    This function returns a list of (tag, absolute_path).
274    For example,
275    [
276      (
277        "NDK",
278        "/path/to/libc.so.lsdump"
279      )
280    ]
281    """
282    arch_cpu = target.get_arch_cpu_str()
283    result = []
284    if libs:
285        for lib_name in libs:
286            if not (lib_name in lsdump_paths and
287                    arch_cpu in lsdump_paths[lib_name]):
288                raise KeyError('Cannot find lsdump for %s, %s.' %
289                               (lib_name, arch_cpu))
290            result.extend(lsdump_paths[lib_name][arch_cpu].items())
291    else:
292        for arch_tag_path_dict in lsdump_paths.values():
293            result.extend(arch_tag_path_dict[arch_cpu].items())
294    return [(tag, os.path.join(AOSP_DIR, path)) for tag, path in result]
295
296
297def run_abi_diff(old_test_dump_path, new_test_dump_path, arch, lib_name,
298                 flags=tuple()):
299    abi_diff_cmd = ['header-abi-diff', '-new', new_test_dump_path, '-old',
300                    old_test_dump_path, '-arch', arch, '-lib', lib_name]
301    with tempfile.TemporaryDirectory() as tmp:
302        output_name = os.path.join(tmp, lib_name) + '.abidiff'
303        abi_diff_cmd += ['-o', output_name]
304        abi_diff_cmd += flags
305        if '-input-format-old' not in flags:
306            abi_diff_cmd += ['-input-format-old', DEFAULT_FORMAT]
307        if '-input-format-new' not in flags:
308            abi_diff_cmd += ['-input-format-new', DEFAULT_FORMAT]
309        try:
310            subprocess.check_call(abi_diff_cmd)
311        except subprocess.CalledProcessError as err:
312            return err.returncode
313
314    return 0
315
316
317def get_build_vars_for_product(names, product=None, variant=None):
318    """ Get build system variable for the launched target."""
319
320    if product is None and 'ANDROID_PRODUCT_OUT' not in os.environ:
321        return None
322
323    env = os.environ.copy()
324    if product:
325        env['TARGET_PRODUCT'] = product
326    if variant:
327        env['TARGET_BUILD_VARIANT'] = variant
328    cmd = [
329        os.path.join('build', 'soong', 'soong_ui.bash'),
330        '--dumpvars-mode', '-vars', ' '.join(names),
331    ]
332
333    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
334                            stderr=subprocess.PIPE, cwd=AOSP_DIR, env=env)
335    out, err = proc.communicate()
336
337    if proc.returncode != 0:
338        print("error: %s" % err.decode('utf-8'), file=sys.stderr)
339        return None
340
341    build_vars = out.decode('utf-8').strip().splitlines()
342
343    build_vars_list = []
344    for build_var in build_vars:
345        value = build_var.partition('=')[2]
346        build_vars_list.append(value.replace('\'', ''))
347    return build_vars_list
348