1#!/usr/bin/env python3 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""ModuleData information.""" 18 19from __future__ import absolute_import 20 21import glob 22import logging 23import os 24import re 25 26from aidegen import constant 27from aidegen.lib import common_util 28from aidegen.lib import module_info 29from aidegen.lib import project_config 30 31# Parse package name from the package declaration line of a java. 32# Group matches "foo.bar" of line "package foo.bar;" or "package foo.bar" 33_PACKAGE_RE = re.compile(r'\s*package\s+(?P<package>[^(;|\s)]+)\s*', re.I) 34_ANDROID_SUPPORT_PATH_KEYWORD = 'prebuilts/sdk/current/' 35 36# File extensions 37_JAVA_EXT = '.java' 38_KOTLIN_EXT = '.kt' 39_SRCJAR_EXT = '.srcjar' 40_TARGET_FILES = [_JAVA_EXT, _KOTLIN_EXT] 41_JARJAR_RULES_FILE = 'jarjar-rules.txt' 42_KEY_JARJAR_RULES = 'jarjar_rules' 43_NAME_AAPT2 = 'aapt2' 44_TARGET_R_SRCJAR = 'R.srcjar' 45_TARGET_AAPT2_SRCJAR = _NAME_AAPT2 + _SRCJAR_EXT 46_TARGET_BUILD_FILES = [_TARGET_AAPT2_SRCJAR, _TARGET_R_SRCJAR] 47_IGNORE_DIRS = [ 48 # The java files under this directory have to be ignored because it will 49 # cause duplicated classes by libcore/ojluni/src/main/java. 50 'libcore/ojluni/src/lambda/java' 51] 52_ANDROID = 'android' 53_REPACKAGES = 'repackaged' 54_FRAMEWORK_SRCJARS_PATH = os.path.join(constant.FRAMEWORK_PATH, 55 constant.FRAMEWORK_SRCJARS) 56 57 58class ModuleData: 59 """ModuleData class. 60 61 Attributes: 62 All following relative paths stand for the path relative to the android 63 repo root. 64 65 module_path: A string of the relative path to the module. 66 src_dirs: A list to keep the unique source folder relative paths. 67 test_dirs: A list to keep the unique test folder relative paths. 68 jar_files: A list to keep the unique jar file relative paths. 69 r_java_paths: A list to keep the R folder paths to use in Eclipse. 70 srcjar_paths: A list to keep the srcjar source root paths to use in 71 IntelliJ. 72 dep_paths: A list to keep the dependency modules' path. 73 referenced_by_jar: A boolean to check if the module is referenced by a 74 jar file. 75 build_targets: A set to keep the unique build target jar or srcjar file 76 relative paths which are ready to be rebuld. 77 missing_jars: A set to keep the jar file relative paths if it doesn't 78 exist. 79 specific_soong_path: A string of the relative path to the module's 80 intermediates folder under out/. 81 """ 82 83 def __init__(self, module_name, module_data, depth): 84 """Initialize ModuleData. 85 86 Args: 87 module_name: Name of the module. 88 module_data: A dictionary holding a module information. 89 depth: An integer shows the depth of module dependency referenced by 90 source. Zero means the max module depth. 91 For example: 92 { 93 'class': ['APPS'], 94 'path': ['path/to/the/module'], 95 'depth': 0, 96 'dependencies': ['bouncycastle', 'ims-common'], 97 'srcs': [ 98 'path/to/the/module/src/com/android/test.java', 99 'path/to/the/module/src/com/google/test.java', 100 'out/soong/.intermediates/path/to/the/module/test/src/ 101 com/android/test.srcjar' 102 ], 103 'installed': ['out/target/product/generic_x86_64/ 104 system/framework/framework.jar'], 105 'jars': ['settings.jar'], 106 'jarjar_rules': ['jarjar-rules.txt'] 107 } 108 """ 109 assert module_name, 'Module name can\'t be null.' 110 assert module_data, 'Module data of %s can\'t be null.' % module_name 111 self.module_name = module_name 112 self.module_data = module_data 113 self._init_module_path() 114 self._init_module_depth(depth) 115 self.src_dirs = [] 116 self.test_dirs = [] 117 self.jar_files = [] 118 self.r_java_paths = [] 119 self.srcjar_paths = [] 120 self.dep_paths = [] 121 self.referenced_by_jar = False 122 self.build_targets = set() 123 self.missing_jars = set() 124 self.specific_soong_path = os.path.join( 125 'out/soong/.intermediates', self.module_path, self.module_name) 126 127 def _is_app_module(self): 128 """Check if the current module's class is APPS""" 129 return self._check_key('class') and 'APPS' in self.module_data['class'] 130 131 def _is_target_module(self): 132 """Check if the current module is a target module. 133 134 A target module is the target project or a module under the 135 target project and it's module depth is 0. 136 For example: aidegen Settings framework 137 The target projects are Settings and framework so they are also 138 target modules. And the dependent module SettingsUnitTests's path 139 is packages/apps/Settings/tests/unit so it also a target module. 140 """ 141 return self.module_depth == 0 142 143 def _collect_r_srcs_paths(self): 144 """Collect the source folder of R.java. 145 146 Check if the path of aapt2.srcjar or R.srcjar exists, these are both the 147 values of key "srcjars" in module_data. If neither of the cases exists, 148 build it onto an intermediates directory. 149 150 For IntelliJ, we can set the srcjar file as a source root for 151 dependency. For Eclipse, we still use the R folder as dependencies until 152 we figure out how to set srcjar file as dependency. 153 # TODO(b/135594800): Set aapt2.srcjar or R.srcjar as a dependency in 154 Eclipse. 155 """ 156 if (self._is_app_module() and self._is_target_module() 157 and self._check_key(constant.KEY_SRCJARS)): 158 for srcjar in self.module_data[constant.KEY_SRCJARS]: 159 if not os.path.exists(common_util.get_abs_path(srcjar)): 160 self.build_targets.add(srcjar) 161 self._collect_srcjar_path(srcjar) 162 r_dir = self._get_r_dir(srcjar) 163 if r_dir and r_dir not in self.r_java_paths: 164 self.r_java_paths.append(r_dir) 165 166 def _collect_srcjar_path(self, srcjar): 167 """Collect the source folders from a srcjar path. 168 169 Set the aapt2.srcjar or R.srcjar as source root: 170 Case aapt2.srcjar: 171 The source path string is 172 out/.../Bluetooth_intermediates/aapt2.srcjar. 173 Case R.srcjar: 174 The source path string is out/soong/.../gen/android/R.srcjar. 175 176 Args: 177 srcjar: A file path string relative to ANDROID_BUILD_TOP, the build 178 target of the module to generate R.java. 179 """ 180 if (os.path.basename(srcjar) in _TARGET_BUILD_FILES 181 and srcjar not in self.srcjar_paths): 182 self.srcjar_paths.append(srcjar) 183 184 def _collect_all_srcjar_paths(self): 185 """Collect all srcjar files of target module as source folders. 186 187 Since the aidl files are built to *.java and collected in the 188 aidl.srcjar file by the build system. AIDEGen needs to collect these 189 aidl.srcjar files as the source root folders in IntelliJ. Furthermore, 190 AIDEGen collects all *.srcjar files for other cases to fulfil the same 191 purpose. 192 """ 193 if self._is_target_module() and self._check_key(constant.KEY_SRCJARS): 194 for srcjar in self.module_data[constant.KEY_SRCJARS]: 195 if not os.path.exists(common_util.get_abs_path(srcjar)): 196 self.build_targets.add(srcjar) 197 if srcjar not in self.srcjar_paths: 198 self.srcjar_paths.append(srcjar) 199 200 @staticmethod 201 def _get_r_dir(srcjar): 202 """Get the source folder of R.java for Eclipse. 203 204 Get the folder contains the R.java of aapt2.srcjar or R.srcjar: 205 Case aapt2.srcjar: 206 If the relative path of the aapt2.srcjar is a/b/aapt2.srcjar, the 207 source root of the R.java is a/b/aapt2 208 Case R.srcjar: 209 If the relative path of the R.srcjar is a/b/android/R.srcjar, the 210 source root of the R.java is a/b/aapt2/R 211 212 Args: 213 srcjar: A file path string, the build target of the module to 214 generate R.java. 215 216 Returns: 217 A relative source folder path string, and return None if the target 218 file name is not aapt2.srcjar or R.srcjar. 219 """ 220 target_folder, target_file = os.path.split(srcjar) 221 base_dirname = os.path.basename(target_folder) 222 if target_file == _TARGET_AAPT2_SRCJAR: 223 return os.path.join(target_folder, _NAME_AAPT2) 224 if target_file == _TARGET_R_SRCJAR and base_dirname == _ANDROID: 225 return os.path.join(os.path.dirname(target_folder), 226 _NAME_AAPT2, 'R') 227 return None 228 229 def _init_module_path(self): 230 """Inintialize self.module_path.""" 231 self.module_path = (self.module_data[constant.KEY_PATH][0] 232 if self._check_key(constant.KEY_PATH) else '') 233 234 def _init_module_depth(self, depth): 235 """Initialize module depth's settings. 236 237 Set the module's depth from module info when user have -d parameter. 238 Set the -d value from user input, default to 0. 239 240 Args: 241 depth: the depth to be set. 242 """ 243 self.module_depth = (int(self.module_data[constant.KEY_DEPTH]) 244 if depth else 0) 245 self.depth_by_source = depth 246 247 def _is_android_supported_module(self): 248 """Determine if this is an Android supported module.""" 249 return common_util.is_source_under_relative_path( 250 self.module_path, _ANDROID_SUPPORT_PATH_KEYWORD) 251 252 def _check_jarjar_rules_exist(self): 253 """Check if jarjar rules exist.""" 254 return (_KEY_JARJAR_RULES in self.module_data and 255 self.module_data[_KEY_JARJAR_RULES][0] == _JARJAR_RULES_FILE) 256 257 def _check_jars_exist(self): 258 """Check if jars exist.""" 259 return self._check_key(constant.KEY_JARS) 260 261 def _check_classes_jar_exist(self): 262 """Check if classes_jar exist.""" 263 return self._check_key(constant.KEY_CLASSES_JAR) 264 265 def _collect_srcs_paths(self): 266 """Collect source folder paths in src_dirs from module_data['srcs'].""" 267 if self._check_key(constant.KEY_SRCS): 268 scanned_dirs = set() 269 for src_item in self.module_data[constant.KEY_SRCS]: 270 src_dir = None 271 src_item = os.path.relpath(src_item) 272 if common_util.is_target(src_item, _TARGET_FILES): 273 # Only scan one java file in each source directories. 274 src_item_dir = os.path.dirname(src_item) 275 if src_item_dir not in scanned_dirs: 276 scanned_dirs.add(src_item_dir) 277 src_dir = self._get_source_folder(src_item) 278 else: 279 # To record what files except java and kt in the srcs. 280 logging.debug('%s is not in parsing scope.', src_item) 281 if src_dir: 282 self._add_to_source_or_test_dirs( 283 self._switch_repackaged(src_dir)) 284 285 def _check_key(self, key): 286 """Check if key is in self.module_data and not empty. 287 288 Args: 289 key: the key to be checked. 290 """ 291 return key in self.module_data and self.module_data[key] 292 293 def _add_to_source_or_test_dirs(self, src_dir): 294 """Add folder to source or test directories. 295 296 Args: 297 src_dir: the directory to be added. 298 """ 299 if (src_dir not in _IGNORE_DIRS and src_dir not in self.src_dirs 300 and src_dir not in self.test_dirs): 301 if self._is_test_module(src_dir): 302 self.test_dirs.append(src_dir) 303 else: 304 self.src_dirs.append(src_dir) 305 306 @staticmethod 307 def _is_test_module(src_dir): 308 """Check if the module path is a test module path. 309 310 Args: 311 src_dir: the directory to be checked. 312 313 Returns: 314 True if module path is a test module path, otherwise False. 315 """ 316 return constant.KEY_TESTS in src_dir.split(os.sep) 317 318 def _get_source_folder(self, java_file): 319 """Parsing a java to get the package name to filter out source path. 320 321 Args: 322 java_file: A string, the java file with relative path. 323 e.g. path/to/the/java/file.java 324 325 Returns: 326 source_folder: A string of path to source folder(e.g. src/main/java) 327 or none when it failed to get package name. 328 """ 329 abs_java_path = common_util.get_abs_path(java_file) 330 if os.path.exists(abs_java_path): 331 package_name = self._get_package_name(abs_java_path) 332 if package_name: 333 return self._parse_source_path(java_file, package_name) 334 return None 335 336 @staticmethod 337 def _parse_source_path(java_file, package_name): 338 """Parse the source path by filter out the package name. 339 340 Case 1: 341 java file: a/b/c/d/e.java 342 package name: c.d 343 The source folder is a/b. 344 345 Case 2: 346 java file: a/b/c.d/e.java 347 package name: c.d 348 The source folder is a/b. 349 350 Case 3: 351 java file: a/b/c/d/e.java 352 package name: x.y 353 The source folder is a/b/c/d. 354 355 Case 4: 356 java file: a/b/c.d/e/c/d/f.java 357 package name: c.d 358 The source folder is a/b/c.d/e. 359 360 Case 5: 361 java file: a/b/c.d/e/c.d/e/f.java 362 package name: c.d.e 363 The source folder is a/b/c.d/e. 364 365 Args: 366 java_file: A string of the java file relative path. 367 package_name: A string of the java file's package name. 368 369 Returns: 370 A string, the source folder path. 371 """ 372 java_file_name = os.path.basename(java_file) 373 pattern = r'%s/%s$' % (package_name, java_file_name) 374 search_result = re.search(pattern, java_file) 375 if search_result: 376 return java_file[:search_result.start()].strip(os.sep) 377 return os.path.dirname(java_file) 378 379 @staticmethod 380 def _switch_repackaged(src_dir): 381 """Changes the directory to repackaged if it does exist. 382 383 Args: 384 src_dir: a string of relative path. 385 386 Returns: 387 The source folder under repackaged if it exists, otherwise the 388 original one. 389 """ 390 root_path = common_util.get_android_root_dir() 391 dir_list = src_dir.split(os.sep) 392 for i in range(1, len(dir_list)): 393 tmp_dir = dir_list.copy() 394 tmp_dir.insert(i, _REPACKAGES) 395 real_path = os.path.join(root_path, os.path.join(*tmp_dir)) 396 if os.path.exists(real_path): 397 return os.path.relpath(real_path, root_path) 398 return src_dir 399 400 @staticmethod 401 def _get_package_name(abs_java_path): 402 """Get the package name by parsing a java file. 403 404 Args: 405 abs_java_path: A string of the java file with absolute path. 406 e.g. /root/path/to/the/java/file.java 407 408 Returns: 409 package_name: A string of package name. 410 """ 411 package_name = None 412 with open(abs_java_path, encoding='utf8') as data: 413 for line in data.read().splitlines(): 414 match = _PACKAGE_RE.match(line) 415 if match: 416 package_name = match.group('package') 417 break 418 return package_name 419 420 def _append_jar_file(self, jar_path): 421 """Append a path to the jar file into self.jar_files if it's exists. 422 423 Args: 424 jar_path: A path supposed to be a jar file. 425 426 Returns: 427 Boolean: True if jar_path is an existing jar file. 428 """ 429 if common_util.is_target(jar_path, constant.TARGET_LIBS): 430 self.referenced_by_jar = True 431 if os.path.isfile(common_util.get_abs_path(jar_path)): 432 if jar_path not in self.jar_files: 433 self.jar_files.append(jar_path) 434 else: 435 self.missing_jars.add(jar_path) 436 return True 437 return False 438 439 def _append_classes_jar(self): 440 """Append the jar file as dependency for prebuilt modules.""" 441 for jar in self.module_data[constant.KEY_CLASSES_JAR]: 442 if self._append_jar_file(jar): 443 break 444 445 def _append_jar_from_installed(self, specific_dir=None): 446 """Append a jar file's path to the list of jar_files with matching 447 path_prefix. 448 449 There might be more than one jar in "installed" parameter and only the 450 first jar file is returned. If specific_dir is set, the jar file must be 451 under the specific directory or its sub-directory. 452 453 Args: 454 specific_dir: A string of path. 455 """ 456 if self._check_key(constant.KEY_INSTALLED): 457 for jar in self.module_data[constant.KEY_INSTALLED]: 458 if specific_dir and not jar.startswith(specific_dir): 459 continue 460 if self._append_jar_file(jar): 461 break 462 463 def _set_jars_jarfile(self): 464 """Append prebuilt jars of module into self.jar_files. 465 466 Some modules' sources are prebuilt jar files instead of source java 467 files. The jar files can be imported into IntelliJ as a dependency 468 directly. There is only jar file name in self.module_data['jars'], it 469 has to be combined with self.module_data['path'] to append into 470 self.jar_files. 471 Once the file doesn't exist, it's not assumed to be a prebuilt jar so 472 that we can ignore it. 473 # TODO(b/141959125): Collect the correct prebuilt jar files by jdeps.go. 474 475 For example: 476 'asm-6.0': { 477 'jars': [ 478 'asm-6.0.jar' 479 ], 480 'path': [ 481 'prebuilts/misc/common/asm' 482 ], 483 }, 484 Path to the jar file is prebuilts/misc/common/asm/asm-6.0.jar. 485 """ 486 if self._check_key(constant.KEY_JARS): 487 for jar_name in self.module_data[constant.KEY_JARS]: 488 if self._check_key(constant.KEY_INSTALLED): 489 self._append_jar_from_installed() 490 else: 491 jar_path = os.path.join(self.module_path, jar_name) 492 jar_abs = common_util.get_abs_path(jar_path) 493 if not os.path.isfile(jar_abs) and jar_name.endswith( 494 'prebuilt.jar'): 495 rel_path = self._get_jar_path_from_prebuilts(jar_name) 496 if rel_path: 497 jar_path = rel_path 498 if os.path.exists(common_util.get_abs_path(jar_path)): 499 self._append_jar_file(jar_path) 500 501 @staticmethod 502 def _get_jar_path_from_prebuilts(jar_name): 503 """Get prebuilt jar file from prebuilts folder. 504 505 If the prebuilt jar file we get from method _set_jars_jarfile() does not 506 exist, we should search the prebuilt jar file in prebuilts folder. 507 For example: 508 'platformprotos': { 509 'jars': [ 510 'platformprotos-prebuilt.jar' 511 ], 512 'path': [ 513 'frameworks/base' 514 ], 515 }, 516 We get an incorrect path: 'frameworks/base/platformprotos-prebuilt.jar' 517 If the file does not exist, we should search the file name from 518 prebuilts folder. If we can get the correct path from 'prebuilts', we 519 can replace it with the incorrect path. 520 521 Args: 522 jar_name: The prebuilt jar file name. 523 524 Return: 525 A relative prebuilt jar file path if found, otherwise None. 526 """ 527 rel_path = '' 528 search = os.sep.join( 529 [common_util.get_android_root_dir(), 'prebuilts/**', jar_name]) 530 results = glob.glob(search, recursive=True) 531 if results: 532 jar_abs = results[0] 533 rel_path = os.path.relpath( 534 jar_abs, common_util.get_android_root_dir()) 535 return rel_path 536 537 def _collect_specific_jars(self): 538 """Collect specific types of jar files.""" 539 if self._is_android_supported_module(): 540 self._append_jar_from_installed() 541 elif self._check_jarjar_rules_exist(): 542 self._append_jar_from_installed(self.specific_soong_path) 543 elif self._check_jars_exist(): 544 self._set_jars_jarfile() 545 546 def _collect_classes_jars(self): 547 """Collect classes jar files.""" 548 # If there is no source/tests folder of the module, reference the 549 # module by jar. 550 if not self.src_dirs and not self.test_dirs: 551 # Add the classes.jar from the classes_jar attribute as 552 # dependency if it exists. If the classes.jar doesn't exist, 553 # find the jar file from the installed attribute and add the jar 554 # as dependency. 555 if self._check_classes_jar_exist(): 556 self._append_classes_jar() 557 else: 558 self._append_jar_from_installed() 559 560 def _collect_srcs_and_r_srcs_paths(self): 561 """Collect source and R source folder paths for the module.""" 562 self._collect_specific_jars() 563 self._collect_srcs_paths() 564 self._collect_classes_jars() 565 self._collect_r_srcs_paths() 566 self._collect_all_srcjar_paths() 567 568 def _collect_missing_jars(self): 569 """Collect missing jar files to rebuild them.""" 570 if self.referenced_by_jar and self.missing_jars: 571 self.build_targets |= self.missing_jars 572 573 def _collect_dep_paths(self): 574 """Collects the path of dependency modules.""" 575 config = project_config.ProjectConfig.get_instance() 576 modules_info = config.atest_module_info 577 self.dep_paths = [] 578 if self.module_path != constant.FRAMEWORK_PATH: 579 self.dep_paths.append(constant.FRAMEWORK_PATH) 580 self.dep_paths.append(_FRAMEWORK_SRCJARS_PATH) 581 if self.module_path != constant.LIBCORE_PATH: 582 self.dep_paths.append(constant.LIBCORE_PATH) 583 for module in self.module_data.get(constant.KEY_DEPENDENCIES, []): 584 for path in modules_info.get_paths(module): 585 if path not in self.dep_paths and path != self.module_path: 586 self.dep_paths.append(path) 587 588 def locate_sources_path(self): 589 """Locate source folders' paths or jar files.""" 590 # Check if users need to reference source according to source depth. 591 if not self.module_depth <= self.depth_by_source: 592 self._append_jar_from_installed(self.specific_soong_path) 593 else: 594 self._collect_srcs_and_r_srcs_paths() 595 self._collect_missing_jars() 596 597 598class EclipseModuleData(ModuleData): 599 """Deal with modules data for Eclipse 600 601 Only project target modules use source folder type and the other ones use 602 jar as their source. We'll combine both to establish the whole project's 603 dependencies. If the source folder used to build dependency jar file exists 604 in Android, we should provide the jar file path as <linkedResource> item in 605 source data. 606 """ 607 608 def __init__(self, module_name, module_data, project_relpath): 609 """Initialize EclipseModuleData. 610 611 Only project target modules apply source folder type, so set the depth 612 of module referenced by source to 0. 613 614 Args: 615 module_name: String type, name of the module. 616 module_data: A dictionary contains a module information. 617 project_relpath: A string stands for the project's relative path. 618 """ 619 super().__init__(module_name, module_data, depth=0) 620 related = module_info.AidegenModuleInfo.is_project_path_relative_module( 621 module_data, project_relpath) 622 self.is_project = related 623 624 def locate_sources_path(self): 625 """Locate source folders' paths or jar files. 626 627 Only collect source folders for the project modules and collect jar 628 files for the other dependent modules. 629 """ 630 if self.is_project: 631 self._locate_project_source_path() 632 else: 633 self._locate_jar_path() 634 self._collect_classes_jars() 635 self._collect_missing_jars() 636 637 def _add_to_source_or_test_dirs(self, src_dir): 638 """Add a folder to source list if it is not in ignored directories. 639 640 Override the parent method since the tests folder has no difference 641 with source folder in Eclipse. 642 643 Args: 644 src_dir: a string of relative path to the Android root. 645 """ 646 if src_dir not in _IGNORE_DIRS and src_dir not in self.src_dirs: 647 self.src_dirs.append(src_dir) 648 649 def _locate_project_source_path(self): 650 """Locate the source folder paths of the project module. 651 652 A project module is the target modules or paths that users key in 653 aidegen command. Collecting the source folders is necessary for 654 developers to edit code. And also collect the central R folder for the 655 dependency of resources. 656 """ 657 self._collect_srcs_paths() 658 self._collect_r_srcs_paths() 659 660 def _locate_jar_path(self): 661 """Locate the jar path of the module. 662 663 Use jar files for dependency modules for Eclipse. Collect the jar file 664 path with different cases. 665 """ 666 if self._check_jarjar_rules_exist(): 667 self._append_jar_from_installed(self.specific_soong_path) 668 elif self._check_jars_exist(): 669 self._set_jars_jarfile() 670 elif self._check_classes_jar_exist(): 671 self._append_classes_jar() 672 else: 673 self._append_jar_from_installed() 674