1#!/usr/bin/env python
2#
3# Copyright (C) 2019 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""Call cargo -v, parse its output, and generate Android.bp.
17
18Usage: Run this script in a crate workspace root directory.
19The Cargo.toml file should work at least for the host platform.
20
21(1) Without other flags, "cargo2android.py --run"
22    calls cargo clean, calls cargo build -v, and generates Android.bp.
23    The cargo build only generates crates for the host,
24    without test crates.
25
26(2) To build crates for both host and device in Android.bp, use the
27    --device flag, for example:
28    cargo2android.py --run --device
29
30    This is equivalent to using the --cargo flag to add extra builds:
31    cargo2android.py --run
32      --cargo "build"
33      --cargo "build --target x86_64-unknown-linux-gnu"
34
35    On MacOS, use x86_64-apple-darwin as target triple.
36    Here the host target triple is used as a fake cross compilation target.
37    If the crate's Cargo.toml and environment configuration works for an
38    Android target, use that target triple as the cargo build flag.
39
40(3) To build default and test crates, for host and device, use both
41    --device and --tests flags:
42    cargo2android.py --run --device --tests
43
44    This is equivalent to using the --cargo flag to add extra builds:
45    cargo2android.py --run
46      --cargo "build"
47      --cargo "build --tests"
48      --cargo "build --target x86_64-unknown-linux-gnu"
49      --cargo "build --tests --target x86_64-unknown-linux-gnu"
50
51Since Android rust builds by default treat all warnings as errors,
52if there are rustc warning messages, this script will add
53deny_warnings:false to the owner crate module in Android.bp.
54"""
55
56from __future__ import print_function
57
58import argparse
59import os
60import os.path
61import re
62
63RENAME_MAP = {
64    # This map includes all changes to the default rust library module
65    # names to resolve name conflicts or avoid confusion.
66    'libbacktrace': 'libbacktrace_rust',
67    'libgcc': 'libgcc_rust',
68    'liblog': 'liblog_rust',
69    'libsync': 'libsync_rust',
70    'libx86_64': 'libx86_64_rust',
71}
72
73# Header added to all generated Android.bp files.
74ANDROID_BP_HEADER = '// This file is generated by cargo2android.py.\n'
75
76CARGO_OUT = 'cargo.out'  # Name of file to keep cargo build -v output.
77
78TARGET_TMP = 'target.tmp'  # Name of temporary output directory.
79
80# Message to be displayed when this script is called without the --run flag.
81DRY_RUN_NOTE = (
82    'Dry-run: This script uses ./' + TARGET_TMP + ' for output directory,\n' +
83    'runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n' +
84    'and writes to Android.bp in the current and subdirectories.\n\n' +
85    'To do do all of the above, use the --run flag.\n' +
86    'See --help for other flags, and more usage notes in this script.\n')
87
88# Cargo -v output of a call to rustc.
89RUSTC_PAT = re.compile('^ +Running `rustc (.*)`$')
90
91# Cargo -vv output of a call to rustc could be split into multiple lines.
92# Assume that the first line will contain some CARGO_* env definition.
93RUSTC_VV_PAT = re.compile('^ +Running `.*CARGO_.*=.*$')
94# The combined -vv output rustc command line pattern.
95RUSTC_VV_CMD_ARGS = re.compile('^ *Running `.*CARGO_.*=.* rustc (.*)`$')
96
97# Cargo -vv output of a "cc" or "ar" command; all in one line.
98CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$')
99# Some package, such as ring-0.13.5, has pattern '... running "cc"'.
100
101# Rustc output of file location path pattern for a warning message.
102WARNING_FILE_PAT = re.compile('^ *--> ([^:]*):[0-9]+')
103
104# Rust package name with suffix -d1.d2.d3.
105VERSION_SUFFIX_PAT = re.compile(r'^(.*)-[0-9]+\.[0-9]+\.[0-9]+$')
106
107
108def altered_name(name):
109  return RENAME_MAP[name] if (name in RENAME_MAP) else name
110
111
112def is_build_crate_name(name):
113  # We added special prefix to build script crate names.
114  return name.startswith('build_script_')
115
116
117def is_dependent_file_path(path):
118  # Absolute or dependent '.../' paths are not main files of this crate.
119  return path.startswith('/') or path.startswith('.../')
120
121
122def get_module_name(crate):  # to sort crates in a list
123  return crate.module_name
124
125
126def pkg2crate_name(s):
127  return s.replace('-', '_').replace('.', '_')
128
129
130def file_base_name(path):
131  return os.path.splitext(os.path.basename(path))[0]
132
133
134def test_base_name(path):
135  return pkg2crate_name(file_base_name(path))
136
137
138def unquote(s):  # remove quotes around str
139  if s and len(s) > 1 and s[0] == '"' and s[-1] == '"':
140    return s[1:-1]
141  return s
142
143
144def remove_version_suffix(s):  # remove -d1.d2.d3 suffix
145  if VERSION_SUFFIX_PAT.match(s):
146    return VERSION_SUFFIX_PAT.match(s).group(1)
147  return s
148
149
150def short_out_name(pkg, s):  # replace /.../pkg-*/out/* with .../out/*
151  return re.sub('^/.*/' + pkg + '-[0-9a-f]*/out/', '.../out/', s)
152
153
154def escape_quotes(s):  # replace '"' with '\\"'
155  return s.replace('"', '\\"')
156
157
158class Crate(object):
159  """Information of a Rust crate to collect/emit for an Android.bp module."""
160
161  def __init__(self, runner, outf_name):
162    # Remembered global runner and its members.
163    self.runner = runner
164    self.debug = runner.args.debug
165    self.cargo_dir = ''  # directory of my Cargo.toml
166    self.outf_name = outf_name  # path to Android.bp
167    self.outf = None  # open file handle of outf_name during dump*
168    # Variants/results that could be merged from multiple rustc lines.
169    self.host_supported = False
170    self.device_supported = False
171    self.has_warning = False
172    # Android module properties derived from rustc parameters.
173    self.module_name = ''  # unique in Android build system
174    self.module_type = ''  # rust_{binary,library,test}[_host] etc.
175    self.root_pkg = ''  # parent package name of a sub/test packge, from -L
176    self.srcs = list()  # main_src or merged multiple source files
177    self.stem = ''  # real base name of output file
178    # Kept parsed status
179    self.errors = ''  # all errors found during parsing
180    self.line_num = 1  # runner told input source line number
181    self.line = ''  # original rustc command line parameters
182    # Parameters collected from rustc command line.
183    self.crate_name = ''  # follows --crate-name
184    self.main_src = ''  # follows crate_name parameter, shortened
185    self.crate_type = ''  # bin|lib|test (see --test flag)
186    self.cfgs = list()  # follows --cfg, without feature= prefix
187    self.features = list()  # follows --cfg, name in 'feature="..."'
188    self.codegens = list()  # follows -C, some ignored
189    self.externs = list()  # follows --extern
190    self.core_externs = list()  # first part of self.externs elements
191    self.static_libs = list()  # e.g.  -l static=host_cpuid
192    self.shared_libs = list()  # e.g.  -l dylib=wayland-client, -l z
193    self.cap_lints = ''  # follows --cap-lints
194    self.emit_list = ''  # e.g., --emit=dep-info,metadata,link
195    self.edition = '2015'  # rustc default, e.g., --edition=2018
196    self.target = ''  # follows --target
197
198  def write(self, s):
199    # convenient way to output one line at a time with EOL.
200    self.outf.write(s + '\n')
201
202  def same_flags(self, other):
203    # host_supported, device_supported, has_warning are not compared but merged
204    # target is not compared, to merge different target/host modules
205    # externs is not compared; only core_externs is compared
206    return (not self.errors and not other.errors and
207            self.edition == other.edition and
208            self.cap_lints == other.cap_lints and
209            self.emit_list == other.emit_list and
210            self.core_externs == other.core_externs and
211            self.codegens == other.codegens and
212            self.features == other.features and
213            self.static_libs == other.static_libs and
214            self.shared_libs == other.shared_libs and self.cfgs == other.cfgs)
215
216  def merge_host_device(self, other):
217    """Returns true if attributes are the same except host/device support."""
218    return (self.crate_name == other.crate_name and
219            self.crate_type == other.crate_type and
220            self.main_src == other.main_src and self.stem == other.stem and
221            self.root_pkg == other.root_pkg and not self.skip_crate() and
222            self.same_flags(other))
223
224  def merge_test(self, other):
225    """Returns true if self and other are tests of same root_pkg."""
226    # Before merger, each test has its own crate_name.
227    # A merged test uses its source file base name as output file name,
228    # so a test is mergeable only if its base name equals to its crate name.
229    return (self.crate_type == other.crate_type and
230            self.crate_type == 'test' and self.root_pkg == other.root_pkg and
231            not self.skip_crate() and
232            other.crate_name == test_base_name(other.main_src) and
233            (len(self.srcs) > 1 or
234             (self.crate_name == test_base_name(self.main_src)) and
235             self.host_supported == other.host_supported and
236             self.device_supported == other.device_supported) and
237            self.same_flags(other))
238
239  def merge(self, other, outf_name):
240    """Try to merge crate into self."""
241    should_merge_host_device = self.merge_host_device(other)
242    should_merge_test = False
243    if not should_merge_host_device:
244      should_merge_test = self.merge_test(other)
245    # A for-device test crate can be merged with its for-host version,
246    # or merged with a different test for the same host or device.
247    # Since we run cargo once for each device or host, test crates for the
248    # first device or host will be merged first. Then test crates for a
249    # different device or host should be allowed to be merged into a
250    # previously merged one, maybe for a different device or host.
251    if should_merge_host_device or should_merge_test:
252      self.runner.init_bp_file(outf_name)
253      with open(outf_name, 'a') as outf:  # to write debug info
254        self.outf = outf
255        other.outf = outf
256        self.do_merge(other, should_merge_test)
257      return True
258    return False
259
260  def do_merge(self, other, should_merge_test):
261    """Merge attributes of other to self."""
262    if self.debug:
263      self.write('\n// Before merge definition (1):')
264      self.dump_debug_info()
265      self.write('\n// Before merge definition (2):')
266      other.dump_debug_info()
267    # Merge properties of other to self.
268    self.host_supported = self.host_supported or other.host_supported
269    self.device_supported = self.device_supported or other.device_supported
270    self.has_warning = self.has_warning or other.has_warning
271    if not self.target:  # okay to keep only the first target triple
272      self.target = other.target
273    # decide_module_type sets up default self.stem,
274    # which can be changed if self is a merged test module.
275    self.decide_module_type()
276    if should_merge_test:
277      self.srcs.append(other.main_src)
278      # use a short unique name as the merged module name.
279      prefix = self.root_pkg + '_tests'
280      self.module_name = self.runner.claim_module_name(prefix, self, 0)
281      self.stem = self.module_name
282      # This normalized root_pkg name although might be the same
283      # as other module's crate_name, it is not actually used for
284      # output file name. A merged test module always have multiple
285      # source files and each source file base name is used as
286      # its output file name.
287      self.crate_name = pkg2crate_name(self.root_pkg)
288    if self.debug:
289      self.write('\n// After merge definition (1):')
290      self.dump_debug_info()
291
292  def find_cargo_dir(self):
293    """Deepest directory with Cargo.toml and contains the main_src."""
294    if not is_dependent_file_path(self.main_src):
295      dir_name = os.path.dirname(self.main_src)
296      while dir_name:
297        if os.path.exists(dir_name + '/Cargo.toml'):
298          self.cargo_dir = dir_name
299          return
300        dir_name = os.path.dirname(dir_name)
301
302  def parse(self, line_num, line):
303    """Find important rustc arguments to convert to Android.bp properties."""
304    self.line_num = line_num
305    self.line = line
306    args = line.split()  # Loop through every argument of rustc.
307    i = 0
308    while i < len(args):
309      arg = args[i]
310      if arg == '--crate-name':
311        self.crate_name = args[i + 1]
312        i += 2
313        # shorten imported crate main source path
314        self.main_src = re.sub('^/[^ ]*/registry/src/', '.../', args[i])
315        self.main_src = re.sub('^.../github.com-[0-9a-f]*/', '.../',
316                               self.main_src)
317        self.find_cargo_dir()
318        if self.cargo_dir and not self.runner.args.onefile:
319          # Write to Android.bp in the subdirectory with Cargo.toml.
320          self.outf_name = self.cargo_dir + '/Android.bp'
321          self.main_src = self.main_src[len(self.cargo_dir) + 1:]
322      elif arg == '--crate-type':
323        i += 1
324        if self.crate_type:
325          self.errors += '  ERROR: multiple --crate-type '
326          self.errors += self.crate_type + ' ' + args[i] + '\n'
327          # TODO(chh): handle multiple types, e.g. lexical-core-0.4.6 has
328          #   crate-type = ["lib", "staticlib", "cdylib"]
329          # output: debug/liblexical_core.{a,so,rlib}
330          # cargo calls rustc with multiple --crate-type flags.
331          # rustc can accept:
332          #   --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro]
333          #   Comma separated list of types of crates for the compiler to emit
334        self.crate_type = args[i]
335      elif arg == '--test':
336        # only --test or --crate-type should appear once
337        if self.crate_type:
338          self.errors += ('  ERROR: found both --test and --crate-type ' +
339                          self.crate_type + '\n')
340        else:
341          self.crate_type = 'test'
342      elif arg == '--target':
343        i += 1
344        self.target = args[i]
345      elif arg == '--cfg':
346        i += 1
347        if args[i].startswith('\'feature='):
348          self.features.append(unquote(args[i].replace('\'feature=', '')[:-1]))
349        else:
350          self.cfgs.append(args[i])
351      elif arg == '--extern':
352        i += 1
353        extern_names = re.sub('=/[^ ]*/deps/', ' = ', args[i])
354        self.externs.append(extern_names)
355        self.core_externs.append(re.sub(' = .*', '', extern_names))
356      elif arg == '-C':  # codegen options
357        i += 1
358        # ignore options not used in Android
359        if not (args[i].startswith('debuginfo=') or
360                args[i].startswith('extra-filename=') or
361                args[i].startswith('incremental=') or
362                args[i].startswith('metadata=')):
363          self.codegens.append(args[i])
364      elif arg == '--cap-lints':
365        i += 1
366        self.cap_lints = args[i]
367      elif arg == '-L':
368        i += 1
369        if args[i].startswith('dependency=') and args[i].endswith('/deps'):
370          if '/' + TARGET_TMP + '/' in args[i]:
371            self.root_pkg = re.sub(
372                '^.*/', '', re.sub('/' + TARGET_TMP + '/.*/deps$', '', args[i]))
373          else:
374            self.root_pkg = re.sub('^.*/', '',
375                                   re.sub('/[^/]+/[^/]+/deps$', '', args[i]))
376          self.root_pkg = remove_version_suffix(self.root_pkg)
377      elif arg == '-l':
378        i += 1
379        if args[i].startswith('static='):
380          self.static_libs.append(re.sub('static=', '', args[i]))
381        elif args[i].startswith('dylib='):
382          self.shared_libs.append(re.sub('dylib=', '', args[i]))
383        else:
384          self.shared_libs.append(args[i])
385      elif arg == '--out-dir' or arg == '--color':  # ignored
386        i += 1
387      elif arg.startswith('--error-format=') or arg.startswith('--json='):
388        _ = arg  # ignored
389      elif arg.startswith('--emit='):
390        self.emit_list = arg.replace('--emit=', '')
391      elif arg.startswith('--edition='):
392        self.edition = arg.replace('--edition=', '')
393      else:
394        self.errors += 'ERROR: unknown ' + arg + '\n'
395      i += 1
396    if not self.crate_name:
397      self.errors += 'ERROR: missing --crate-name\n'
398    if not self.main_src:
399      self.errors += 'ERROR: missing main source file\n'
400    else:
401      self.srcs.append(self.main_src)
402    if not self.crate_type:
403      # Treat "--cfg test" as "--test"
404      if 'test' in self.cfgs:
405        self.crate_type = 'test'
406      else:
407        self.errors += 'ERROR: missing --crate-type\n'
408    if not self.root_pkg:
409      self.root_pkg = self.crate_name
410    if self.target:
411      self.device_supported = True
412    self.host_supported = True  # assume host supported for all builds
413    self.cfgs = sorted(set(self.cfgs))
414    self.features = sorted(set(self.features))
415    self.codegens = sorted(set(self.codegens))
416    self.externs = sorted(set(self.externs))
417    self.core_externs = sorted(set(self.core_externs))
418    self.static_libs = sorted(set(self.static_libs))
419    self.shared_libs = sorted(set(self.shared_libs))
420    self.decide_module_type()
421    self.module_name = altered_name(self.stem)
422    return self
423
424  def dump_line(self):
425    self.write('\n// Line ' + str(self.line_num) + ' ' + self.line)
426
427  def feature_list(self):
428    """Return a string of main_src + "feature_list"."""
429    pkg = self.main_src
430    if pkg.startswith('.../'):  # keep only the main package name
431      pkg = re.sub('/.*', '', pkg[4:])
432    if not self.features:
433      return pkg
434    return pkg + ' "' + ','.join(self.features) + '"'
435
436  def dump_skip_crate(self, kind):
437    if self.debug:
438      self.write('\n// IGNORED: ' + kind + ' ' + self.main_src)
439    return self
440
441  def skip_crate(self):
442    """Return crate_name or a message if this crate should be skipped."""
443    if is_build_crate_name(self.crate_name):
444      return self.crate_name
445    if is_dependent_file_path(self.main_src):
446      return 'dependent crate'
447    return ''
448
449  def dump(self):
450    """Dump all error/debug/module code to the output .bp file."""
451    self.runner.init_bp_file(self.outf_name)
452    with open(self.outf_name, 'a') as outf:
453      self.outf = outf
454      if self.errors:
455        self.dump_line()
456        self.write(self.errors)
457      elif self.skip_crate():
458        self.dump_skip_crate(self.skip_crate())
459      else:
460        if self.debug:
461          self.dump_debug_info()
462        self.dump_android_module()
463
464  def dump_debug_info(self):
465    """Dump parsed data, when cargo2android is called with --debug."""
466
467    def dump(name, value):
468      self.write('//%12s = %s' % (name, value))
469
470    def opt_dump(name, value):
471      if value:
472        dump(name, value)
473
474    def dump_list(fmt, values):
475      for v in values:
476        self.write(fmt % v)
477
478    self.dump_line()
479    dump('module_name', self.module_name)
480    dump('crate_name', self.crate_name)
481    dump('crate_type', self.crate_type)
482    dump('main_src', self.main_src)
483    dump('has_warning', self.has_warning)
484    dump('for_host', self.host_supported)
485    dump('for_device', self.device_supported)
486    dump('module_type', self.module_type)
487    opt_dump('target', self.target)
488    opt_dump('edition', self.edition)
489    opt_dump('emit_list', self.emit_list)
490    opt_dump('cap_lints', self.cap_lints)
491    dump_list('//         cfg = %s', self.cfgs)
492    dump_list('//         cfg = \'feature "%s"\'', self.features)
493    # TODO(chh): escape quotes in self.features, but not in other dump_list
494    dump_list('//     codegen = %s', self.codegens)
495    dump_list('//     externs = %s', self.externs)
496    dump_list('//   -l static = %s', self.static_libs)
497    dump_list('//  -l (dylib) = %s', self.shared_libs)
498
499  def dump_android_module(self):
500    """Dump one Android module definition."""
501    if not self.module_type:
502      self.write('\nERROR: unknown crate_type ' + self.crate_type)
503      return
504    self.write('\n' + self.module_type + ' {')
505    self.dump_android_core_properties()
506    if self.edition:
507      self.write('    edition: "' + self.edition + '",')
508    self.dump_android_property_list('features', '"%s"', self.features)
509    cfg_fmt = '"--cfg %s"'
510    if self.cap_lints:
511      allowed = '"--cap-lints ' + self.cap_lints + '"'
512      if not self.cfgs:
513        self.write('    flags: [' + allowed + '],')
514      else:
515        self.write('    flags: [\n       ' + allowed + ',')
516        self.dump_android_property_list_items(cfg_fmt, self.cfgs)
517        self.write('    ],')
518    else:
519      self.dump_android_property_list('flags', cfg_fmt, self.cfgs)
520    if self.externs:
521      self.dump_android_externs()
522    self.dump_android_property_list('static_libs', '"lib%s"', self.static_libs)
523    self.dump_android_property_list('shared_libs', '"lib%s"', self.shared_libs)
524    self.write('}')
525
526  def test_module_name(self):
527    """Return a unique name for a test module."""
528    # root_pkg+'_tests_'+(crate_name|source_file_path)
529    suffix = self.crate_name
530    if not suffix:
531      suffix = re.sub('/', '_', re.sub('.rs$', '', self.main_src))
532    return self.root_pkg + '_tests_' + suffix
533
534  def decide_module_type(self):
535    """Decide which Android module type to use."""
536    host = '' if self.device_supported else '_host'
537    if self.crate_type == 'bin':  # rust_binary[_host]
538      self.module_type = 'rust_binary' + host
539      self.stem = self.crate_name
540    elif self.crate_type == 'lib':  # rust_library[_host]_rlib
541      self.module_type = 'rust_library' + host + '_rlib'
542      self.stem = 'lib' + self.crate_name
543    elif self.crate_type == 'cdylib':  # rust_library[_host]_dylib
544      # TODO(chh): complete and test cdylib module type
545      self.module_type = 'rust_library' + host + '_dylib'
546      self.stem = 'lib' + self.crate_name + '.so'
547    elif self.crate_type == 'test':  # rust_test[_host]
548      self.module_type = 'rust_test' + host
549      self.stem = self.test_module_name()
550      # self.stem will be changed after merging with other tests.
551      # self.stem is NOT used for final test binary name.
552      # rust_test uses each source file base name as its output file name,
553      # unless crate_name is specified by user in Cargo.toml.
554    elif self.crate_type == 'proc-macro':  # rust_proc_macro
555      self.module_type = 'rust_proc_macro'
556      self.stem = 'lib' + self.crate_name
557    else:  # unknown module type, rust_prebuilt_dylib? rust_library[_host]?
558      self.module_type = ''
559      self.stem = ''
560
561  def dump_android_property_list_items(self, fmt, values):
562    for v in values:
563      # fmt has quotes, so we need escape_quotes(v)
564      self.write('        ' + (fmt % escape_quotes(v)) + ',')
565
566  def dump_android_property_list(self, name, fmt, values):
567    if values:
568      self.write('    ' + name + ': [')
569      self.dump_android_property_list_items(fmt, values)
570      self.write('    ],')
571
572  def dump_android_core_properties(self):
573    """Dump the module header, name, stem, etc."""
574    self.write('    name: "' + self.module_name + '",')
575    if self.stem != self.module_name:
576      self.write('    stem: "' + self.stem + '",')
577    if self.has_warning and not self.cap_lints:
578      self.write('    deny_warnings: false,')
579    if self.host_supported and self.device_supported:
580      self.write('    host_supported: true,')
581    self.write('    crate_name: "' + self.crate_name + '",')
582    if len(self.srcs) > 1:
583      self.srcs = sorted(set(self.srcs))
584      self.dump_android_property_list('srcs', '"%s"', self.srcs)
585    else:
586      self.write('    srcs: ["' + self.main_src + '"],')
587    if self.crate_type == 'test':
588      # self.root_pkg can have multiple test modules, with different *_tests[n]
589      # names, but their executables can all be installed under the same _tests
590      # directory. When built from Cargo.toml, all tests should have different
591      # file or crate names.
592      self.write('    relative_install_path: "' + self.root_pkg + '_tests",')
593      self.write('    test_suites: ["general-tests"],')
594      self.write('    auto_gen_config: true,')
595
596  def dump_android_externs(self):
597    """Dump the dependent rlibs and dylibs property."""
598    so_libs = list()
599    rust_libs = ''
600    deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$')
601    for lib in self.externs:
602      # normal value of lib: "libc = liblibc-*.rlib"
603      # strange case in rand crate:  "getrandom_package = libgetrandom-*.rlib"
604      # we should use "libgetrandom", not "lib" + "getrandom_package"
605      groups = deps_libname.match(lib)
606      if groups is not None:
607        lib_name = groups.group(1)
608      else:
609        lib_name = re.sub(' .*$', '', lib)
610      if lib.endswith('.rlib') or lib.endswith('.rmeta'):
611        # On MacOS .rmeta is used when Linux uses .rlib or .rmeta.
612        rust_libs += '        "' + altered_name('lib' + lib_name) + '",\n'
613      elif lib.endswith('.so'):
614        so_libs.append(lib_name)
615      else:
616        rust_libs += '        // ERROR: unknown type of lib ' + lib_name + '\n'
617    if rust_libs:
618      self.write('    rlibs: [\n' + rust_libs + '    ],')
619    # Are all dependent .so files proc_macros?
620    # TODO(chh): Separate proc_macros and dylib.
621    self.dump_android_property_list('proc_macros', '"lib%s"', so_libs)
622
623
624class ARObject(object):
625  """Information of an "ar" link command."""
626
627  def __init__(self, runner, outf_name):
628    # Remembered global runner and its members.
629    self.runner = runner
630    self.pkg = ''
631    self.outf_name = outf_name  # path to Android.bp
632    # "ar" arguments
633    self.line_num = 1
634    self.line = ''
635    self.flags = ''  # e.g. "crs"
636    self.lib = ''  # e.g. "/.../out/lib*.a"
637    self.objs = list()  # e.g. "/.../out/.../*.o"
638
639  def parse(self, pkg, line_num, args_line):
640    """Collect ar obj/lib file names."""
641    self.pkg = pkg
642    self.line_num = line_num
643    self.line = args_line
644    args = args_line.split()
645    num_args = len(args)
646    if num_args < 3:
647      print('ERROR: "ar" command has too few arguments', args_line)
648    else:
649      self.flags = unquote(args[0])
650      self.lib = unquote(args[1])
651      self.objs = sorted(set(map(unquote, args[2:])))
652    return self
653
654  def write(self, s):
655    self.outf.write(s + '\n')
656
657  def dump_debug_info(self):
658    self.write('\n// Line ' + str(self.line_num) + ' "ar" ' + self.line)
659    self.write('// ar_object for %12s' % self.pkg)
660    self.write('//   flags = %s' % self.flags)
661    self.write('//     lib = %s' % short_out_name(self.pkg, self.lib))
662    for o in self.objs:
663      self.write('//     obj = %s' % short_out_name(self.pkg, o))
664
665  def dump_android_lib(self):
666    """Write cc_library_static into Android.bp."""
667    self.write('\ncc_library_static {')
668    self.write('    name: "' + file_base_name(self.lib) + '",')
669    self.write('    host_supported: true,')
670    if self.flags != 'crs':
671      self.write('    // ar flags = %s' % self.flags)
672    if self.pkg not in self.runner.pkg_obj2cc:
673      self.write('    ERROR: cannot find source files.\n}')
674      return
675    self.write('    srcs: [')
676    obj2cc = self.runner.pkg_obj2cc[self.pkg]
677    # Note: wflags are ignored.
678    dflags = list()
679    fflags = list()
680    for obj in self.objs:
681      self.write('        "' + short_out_name(self.pkg, obj2cc[obj].src) + '",')
682      # TODO(chh): union of dflags and flags of all obj
683      # Now, just a temporary hack that uses the last obj's flags
684      dflags = obj2cc[obj].dflags
685      fflags = obj2cc[obj].fflags
686    self.write('    ],')
687    self.write('    cflags: [')
688    self.write('        "-O3",')  # TODO(chh): is this default correct?
689    self.write('        "-Wno-error",')
690    for x in fflags:
691      self.write('        "-f' + x + '",')
692    for x in dflags:
693      self.write('        "-D' + x + '",')
694    self.write('    ],')
695    self.write('}')
696
697  def dump(self):
698    """Dump error/debug/module info to the output .bp file."""
699    self.runner.init_bp_file(self.outf_name)
700    with open(self.outf_name, 'a') as outf:
701      self.outf = outf
702      if self.runner.args.debug:
703        self.dump_debug_info()
704      self.dump_android_lib()
705
706
707class CCObject(object):
708  """Information of a "cc" compilation command."""
709
710  def __init__(self, runner, outf_name):
711    # Remembered global runner and its members.
712    self.runner = runner
713    self.pkg = ''
714    self.outf_name = outf_name  # path to Android.bp
715    # "cc" arguments
716    self.line_num = 1
717    self.line = ''
718    self.src = ''
719    self.obj = ''
720    self.dflags = list()  # -D flags
721    self.fflags = list()  # -f flags
722    self.iflags = list()  # -I flags
723    self.wflags = list()  # -W flags
724    self.other_args = list()
725
726  def parse(self, pkg, line_num, args_line):
727    """Collect cc compilation flags and src/out file names."""
728    self.pkg = pkg
729    self.line_num = line_num
730    self.line = args_line
731    args = args_line.split()
732    i = 0
733    while i < len(args):
734      arg = args[i]
735      if arg == '"-c"':
736        i += 1
737        if args[i].startswith('"-o'):
738          # ring-0.13.5 dumps: ... "-c" "-o/.../*.o" ".../*.c"
739          self.obj = unquote(args[i])[2:]
740          i += 1
741          self.src = unquote(args[i])
742        else:
743          self.src = unquote(args[i])
744      elif arg == '"-o"':
745        i += 1
746        self.obj = unquote(args[i])
747      elif arg == '"-I"':
748        i += 1
749        self.iflags.append(unquote(args[i]))
750      elif arg.startswith('"-D'):
751        self.dflags.append(unquote(args[i])[2:])
752      elif arg.startswith('"-f'):
753        self.fflags.append(unquote(args[i])[2:])
754      elif arg.startswith('"-W'):
755        self.wflags.append(unquote(args[i])[2:])
756      elif not (arg.startswith('"-O') or arg == '"-m64"' or arg == '"-g"' or
757                arg == '"-g3"'):
758        # ignore -O -m64 -g
759        self.other_args.append(unquote(args[i]))
760      i += 1
761    self.dflags = sorted(set(self.dflags))
762    self.fflags = sorted(set(self.fflags))
763    # self.wflags is not sorted because some are order sensitive
764    # and we ignore them anyway.
765    if self.pkg not in self.runner.pkg_obj2cc:
766      self.runner.pkg_obj2cc[self.pkg] = {}
767    self.runner.pkg_obj2cc[self.pkg][self.obj] = self
768    return self
769
770  def write(self, s):
771    self.outf.write(s + '\n')
772
773  def dump_debug_flags(self, name, flags):
774    self.write('//  ' + name + ':')
775    for f in flags:
776      self.write('//    %s' % f)
777
778  def dump(self):
779    """Dump only error/debug info to the output .bp file."""
780    if not self.runner.args.debug:
781      return
782    self.runner.init_bp_file(self.outf_name)
783    with open(self.outf_name, 'a') as outf:
784      self.outf = outf
785      self.write('\n// Line ' + str(self.line_num) + ' "cc" ' + self.line)
786      self.write('// cc_object for %12s' % self.pkg)
787      self.write('//    src = %s' % short_out_name(self.pkg, self.src))
788      self.write('//    obj = %s' % short_out_name(self.pkg, self.obj))
789      self.dump_debug_flags('-I flags', self.iflags)
790      self.dump_debug_flags('-D flags', self.dflags)
791      self.dump_debug_flags('-f flags', self.fflags)
792      self.dump_debug_flags('-W flags', self.wflags)
793      if self.other_args:
794        self.dump_debug_flags('other args', self.other_args)
795
796
797class Runner(object):
798  """Main class to parse cargo -v output and print Android module definitions."""
799
800  def __init__(self, args):
801    self.bp_files = set()  # Remember all output Android.bp files.
802    self.root_pkg = ''  # name of package in ./Cargo.toml
803    # Saved flags, modes, and data.
804    self.args = args
805    self.dry_run = not args.run
806    self.skip_cargo = args.skipcargo
807    # All cc/ar objects, crates, dependencies, and warning files
808    self.cc_objects = list()
809    self.pkg_obj2cc = {}
810    # pkg_obj2cc[cc_object[i].pkg][cc_objects[i].obj] = cc_objects[i]
811    self.ar_objects = list()
812    self.crates = list()
813    self.dependencies = list()  # dependent and build script crates
814    self.warning_files = set()
815    # Keep a unique mapping from (module name) to crate
816    self.name_owners = {}
817    # Default action is cargo clean, followed by build or user given actions.
818    if args.cargo:
819      self.cargo = ['clean'] + args.cargo
820    else:
821      self.cargo = ['clean', 'build']
822      default_target = '--target x86_64-unknown-linux-gnu'
823      if args.device:
824        self.cargo.append('build ' + default_target)
825        if args.tests:
826          self.cargo.append('build --tests')
827          self.cargo.append('build --tests ' + default_target)
828      elif args.tests:
829        self.cargo.append('build --tests')
830
831  def init_bp_file(self, name):
832    if name not in self.bp_files:
833      self.bp_files.add(name)
834      with open(name, 'w') as outf:
835        outf.write(ANDROID_BP_HEADER)
836
837  def claim_module_name(self, prefix, owner, counter):
838    """Return prefix if not owned yet, otherwise, prefix+str(counter)."""
839    while True:
840      name = prefix
841      if counter > 0:
842        name += str(counter)
843      if name not in self.name_owners:
844        self.name_owners[name] = owner
845        return name
846      if owner == self.name_owners[name]:
847        return name
848      counter += 1
849
850  def find_root_pkg(self):
851    """Read name of [package] in ./Cargo.toml."""
852    if not os.path.exists('./Cargo.toml'):
853      return
854    with open('./Cargo.toml', 'r') as inf:
855      pkg_section = re.compile(r'^ *\[package\]')
856      name = re.compile('^ *name *= * "([^"]*)"')
857      in_pkg = False
858      for line in inf:
859        if in_pkg:
860          if name.match(line):
861            self.root_pkg = name.match(line).group(1)
862            break
863        else:
864          in_pkg = pkg_section.match(line) is not None
865
866  def run_cargo(self):
867    """Calls cargo -v and save its output to ./cargo.out."""
868    if self.skip_cargo:
869      return self
870    cargo = './Cargo.toml'
871    if not os.access(cargo, os.R_OK):
872      print('ERROR: Cannot find or read', cargo)
873      return self
874    if not self.dry_run and os.path.exists('cargo.out'):
875      os.remove('cargo.out')
876    cmd_tail = ' --target-dir ' + TARGET_TMP + ' >> cargo.out 2>&1'
877    for c in self.cargo:
878      features = ''
879      if self.args.features and c != 'clean':
880        features = ' --features ' + self.args.features
881      cmd = 'cargo -vv ' if self.args.vv else 'cargo -v '
882      cmd += c + features + cmd_tail
883      if self.args.rustflags and c != 'clean':
884        cmd = 'RUSTFLAGS="' + self.args.rustflags + '" ' + cmd
885      if self.dry_run:
886        print('Dry-run skip:', cmd)
887      else:
888        if self.args.verbose:
889          print('Running:', cmd)
890        with open('cargo.out', 'a') as cargo_out:
891          cargo_out.write('### Running: ' + cmd + '\n')
892        os.system(cmd)
893    return self
894
895  def dump_dependencies(self):
896    """Append dependencies and their features to Android.bp."""
897    if not self.dependencies:
898      return
899    dependent_list = list()
900    for c in self.dependencies:
901      dependent_list.append(c.feature_list())
902    sorted_dependencies = sorted(set(dependent_list))
903    self.init_bp_file('Android.bp')
904    with open('Android.bp', 'a') as outf:
905      outf.write('\n// dependent_library ["feature_list"]\n')
906      for s in sorted_dependencies:
907        outf.write('//   ' + s + '\n')
908
909  def dump_pkg_obj2cc(self):
910    """Dump debug info of the pkg_obj2cc map."""
911    if not self.args.debug:
912      return
913    self.init_bp_file('Android.bp')
914    with open('Android.bp', 'a') as outf:
915      sorted_pkgs = sorted(self.pkg_obj2cc.keys())
916      for pkg in sorted_pkgs:
917        if not self.pkg_obj2cc[pkg]:
918          continue
919        outf.write('\n// obj => src for %s\n' % pkg)
920        obj2cc = self.pkg_obj2cc[pkg]
921        for obj in sorted(obj2cc.keys()):
922          outf.write('//  ' + short_out_name(pkg, obj) + ' => ' +
923                     short_out_name(pkg, obj2cc[obj].src) + '\n')
924
925  def gen_bp(self):
926    """Parse cargo.out and generate Android.bp files."""
927    if self.dry_run:
928      print('Dry-run skip: read', CARGO_OUT, 'write Android.bp')
929    elif os.path.exists(CARGO_OUT):
930      self.find_root_pkg()
931      with open(CARGO_OUT, 'r') as cargo_out:
932        self.parse(cargo_out, 'Android.bp')
933        self.crates.sort(key=get_module_name)
934        for obj in self.cc_objects:
935          obj.dump()
936        self.dump_pkg_obj2cc()
937        for crate in self.crates:
938          crate.dump()
939        dumped_libs = set()
940        for lib in self.ar_objects:
941          if lib.pkg == self.root_pkg:
942            lib_name = file_base_name(lib.lib)
943            if lib_name not in dumped_libs:
944              dumped_libs.add(lib_name)
945              lib.dump()
946        if self.args.dependencies and self.dependencies:
947          self.dump_dependencies()
948    return self
949
950  def add_ar_object(self, obj):
951    self.ar_objects.append(obj)
952
953  def add_cc_object(self, obj):
954    self.cc_objects.append(obj)
955
956  def add_crate(self, crate):
957    """Merge crate with someone in crates, or append to it. Return crates."""
958    if crate.skip_crate():
959      if self.args.debug:  # include debug info of all crates
960        self.crates.append(crate)
961      if self.args.dependencies:  # include only dependent crates
962        if (is_dependent_file_path(crate.main_src) and
963            not is_build_crate_name(crate.crate_name)):
964          self.dependencies.append(crate)
965    else:
966      for c in self.crates:
967        if c.merge(crate, 'Android.bp'):
968          return
969      self.crates.append(crate)
970
971  def find_warning_owners(self):
972    """For each warning file, find its owner crate."""
973    missing_owner = False
974    for f in self.warning_files:
975      cargo_dir = ''  # find lowest crate, with longest path
976      owner = None  # owner crate of this warning
977      for c in self.crates:
978        if (f.startswith(c.cargo_dir + '/') and
979            len(cargo_dir) < len(c.cargo_dir)):
980          cargo_dir = c.cargo_dir
981          owner = c
982      if owner:
983        owner.has_warning = True
984      else:
985        missing_owner = True
986    if missing_owner and os.path.exists('Cargo.toml'):
987      # owner is the root cargo, with empty cargo_dir
988      for c in self.crates:
989        if not c.cargo_dir:
990          c.has_warning = True
991
992  def rustc_command(self, n, rustc_line, line, outf_name):
993    """Process a rustc command line from cargo -vv output."""
994    # cargo build -vv output can have multiple lines for a rustc command
995    # due to '\n' in strings for environment variables.
996    # strip removes leading spaces and '\n' at the end
997    new_rustc = (rustc_line.strip() + line) if rustc_line else line
998    # Use an heuristic to detect the completions of a multi-line command.
999    # This might fail for some very rare case, but easy to fix manually.
1000    if not line.endswith('`\n') or (new_rustc.count('`') % 2) != 0:
1001      return new_rustc
1002    if RUSTC_VV_CMD_ARGS.match(new_rustc):
1003      args = RUSTC_VV_CMD_ARGS.match(new_rustc).group(1)
1004      self.add_crate(Crate(self, outf_name).parse(n, args))
1005    else:
1006      self.assert_empty_vv_line(new_rustc)
1007    return ''
1008
1009  def cc_ar_command(self, n, groups, outf_name):
1010    pkg = groups.group(1)
1011    line = groups.group(3)
1012    if groups.group(2) == 'cc':
1013      self.add_cc_object(CCObject(self, outf_name).parse(pkg, n, line))
1014    else:
1015      self.add_ar_object(ARObject(self, outf_name).parse(pkg, n, line))
1016
1017  def assert_empty_vv_line(self, line):
1018    if line:  # report error if line is not empty
1019      self.init_bp_file('Android.bp')
1020      with open('Android.bp', 'a') as outf:
1021        outf.write('ERROR -vv line: ', line)
1022    return ''
1023
1024  def parse(self, inf, outf_name):
1025    """Parse rustc and warning messages in inf, return a list of Crates."""
1026    n = 0  # line number
1027    prev_warning = False  # true if the previous line was warning: ...
1028    rustc_line = ''  # previous line(s) matching RUSTC_VV_PAT
1029    for line in inf:
1030      n += 1
1031      if line.startswith('warning: '):
1032        prev_warning = True
1033        rustc_line = self.assert_empty_vv_line(rustc_line)
1034        continue
1035      new_rustc = ''
1036      if RUSTC_PAT.match(line):
1037        args_line = RUSTC_PAT.match(line).group(1)
1038        self.add_crate(Crate(self, outf_name).parse(n, args_line))
1039        self.assert_empty_vv_line(rustc_line)
1040      elif rustc_line or RUSTC_VV_PAT.match(line):
1041        new_rustc = self.rustc_command(n, rustc_line, line, outf_name)
1042      elif CC_AR_VV_PAT.match(line):
1043        self.cc_ar_command(n, CC_AR_VV_PAT.match(line), outf_name)
1044      elif prev_warning and WARNING_FILE_PAT.match(line):
1045        self.assert_empty_vv_line(rustc_line)
1046        fpath = WARNING_FILE_PAT.match(line).group(1)
1047        if fpath[0] != '/':  # ignore absolute path
1048          self.warning_files.add(fpath)
1049      prev_warning = False
1050      rustc_line = new_rustc
1051    self.find_warning_owners()
1052
1053
1054def parse_args():
1055  """Parse main arguments."""
1056  parser = argparse.ArgumentParser('cargo2android')
1057  parser.add_argument(
1058      '--cargo',
1059      action='append',
1060      metavar='args_string',
1061      help=('extra cargo build -v args in a string, ' +
1062            'each --cargo flag calls cargo build -v once'))
1063  parser.add_argument(
1064      '--debug',
1065      action='store_true',
1066      default=False,
1067      help='dump debug info into Android.bp')
1068  parser.add_argument(
1069      '--dependencies',
1070      action='store_true',
1071      default=False,
1072      help='dump debug info of dependent crates')
1073  parser.add_argument(
1074      '--device',
1075      action='store_true',
1076      default=False,
1077      help='run cargo also for a default device target')
1078  parser.add_argument(
1079      '--features', type=str, help='passing features to cargo build')
1080  parser.add_argument(
1081      '--onefile',
1082      action='store_true',
1083      default=False,
1084      help=('output all into one ./Android.bp, default will generate ' +
1085            'one Android.bp per Cargo.toml in subdirectories'))
1086  parser.add_argument(
1087      '--run',
1088      action='store_true',
1089      default=False,
1090      help='run it, default is dry-run')
1091  parser.add_argument('--rustflags', type=str, help='passing flags to rustc')
1092  parser.add_argument(
1093      '--skipcargo',
1094      action='store_true',
1095      default=False,
1096      help='skip cargo command, parse cargo.out, and generate Android.bp')
1097  parser.add_argument(
1098      '--tests',
1099      action='store_true',
1100      default=False,
1101      help='run cargo build --tests after normal build')
1102  parser.add_argument(
1103      '--verbose',
1104      action='store_true',
1105      default=False,
1106      help='echo executed commands')
1107  parser.add_argument(
1108      '--vv',
1109      action='store_true',
1110      default=False,
1111      help='run cargo with -vv instead of default -v')
1112  return parser.parse_args()
1113
1114
1115def main():
1116  args = parse_args()
1117  if not args.run:  # default is dry-run
1118    print(DRY_RUN_NOTE)
1119  Runner(args).run_cargo().gen_bp()
1120
1121
1122if __name__ == '__main__':
1123  main()
1124