1#!/usr/bin/env python
2#
3# Copyright (C) 2017 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import argparse
19import glob
20import logging
21import os
22import shutil
23import subprocess
24import tempfile
25import xml.etree.ElementTree as xml_tree
26
27import utils
28
29
30class GPLChecker(object):
31    """Checks that all GPL projects in a VNDK snapshot have released sources.
32
33    Makes sure that the current source tree have the sources for all GPL
34    prebuilt libraries in a specified VNDK snapshot version.
35    """
36    MANIFEST_XML = utils.MANIFEST_FILE_NAME
37    MODULE_PATHS_TXT = utils.MODULE_PATHS_FILE_NAME
38
39    def __init__(self, install_dir, android_build_top, temp_artifact_dir,
40                 remote_git):
41        """GPLChecker constructor.
42
43        Args:
44          install_dir: string, absolute path to the prebuilts/vndk/v{version}
45            directory where the build files will be generated.
46          android_build_top: string, absolute path to ANDROID_BUILD_TOP
47          temp_artifact_dir: string, temp directory to hold build artifacts
48            fetched from Android Build server.
49          remote_git: string, remote name to fetch and check if the revision of
50            VNDK snapshot is included in the source if it is not in the current
51            git repository.
52        """
53        self._android_build_top = android_build_top
54        self._install_dir = install_dir
55        self._remote_git = remote_git
56        self._manifest_file = os.path.join(temp_artifact_dir,
57                                           self.MANIFEST_XML)
58        self._notice_files_dir = os.path.join(install_dir,
59                                              utils.NOTICE_FILES_DIR_PATH)
60
61        if not os.path.isfile(self._manifest_file):
62            raise RuntimeError(
63                '{manifest} not found at {manifest_file}'.format(
64                    manifest=self.MANIFEST_XML,
65                    manifest_file=self._manifest_file))
66
67    def _parse_module_paths(self):
68        """Parses the module_paths.txt files into a dictionary,
69
70        Returns:
71          module_paths: dict, e.g. {libfoo.so: some/path/here}
72        """
73        module_paths = dict()
74        for file in utils.find(self._install_dir, [self.MODULE_PATHS_TXT]):
75            file_path = os.path.join(self._install_dir, file)
76            with open(file_path, 'r') as f:
77                for line in f.read().strip().split('\n'):
78                    paths = line.split(' ')
79                    if len(paths) > 1:
80                        if paths[0] not in module_paths:
81                            module_paths[paths[0]] = paths[1]
82        return module_paths
83
84    def _parse_manifest(self):
85        """Parses manifest.xml file and returns list of 'project' tags."""
86
87        root = xml_tree.parse(self._manifest_file).getroot()
88        return root.findall('project')
89
90    def _get_revision(self, module_path, manifest_projects):
91        """Returns revision value recorded in manifest.xml for given project.
92
93        Args:
94          module_path: string, project path relative to ANDROID_BUILD_TOP
95          manifest_projects: list of xml_tree.Element, list of 'project' tags
96        """
97        revision = None
98        for project in manifest_projects:
99            path = project.get('path')
100            if module_path.startswith(path):
101                revision = project.get('revision')
102                break
103        return revision
104
105    def _check_revision_exists(self, revision, git_project_path):
106        """Checks whether a revision is found in a git project of current tree.
107
108        Args:
109          revision: string, revision value recorded in manifest.xml
110          git_project_path: string, path relative to ANDROID_BUILD_TOP
111        """
112        path = utils.join_realpath(self._android_build_top, git_project_path)
113
114        def _check_rev_list(revision):
115            """Checks whether revision is reachable from HEAD of git project."""
116
117            logging.info('Checking if revision {rev} exists in {proj}'.format(
118                rev=revision, proj=git_project_path))
119            try:
120                cmd = [
121                    'git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision)
122                ]
123                output = utils.check_output(cmd).strip()
124            except subprocess.CalledProcessError as error:
125                logging.error('Error: {}'.format(error))
126                return False
127            else:
128                if output:
129                    logging.debug(
130                        '{proj} does not have the following revisions: {rev}'.
131                        format(proj=git_project_path, rev=output))
132                    return False
133                else:
134                    logging.info(
135                        'Found revision {rev} in project {proj}'.format(
136                            rev=revision, proj=git_project_path))
137            return True
138
139        if not _check_rev_list(revision):
140            # VNDK snapshots built from a *-release branch will have merge
141            # CLs in the manifest because the *-dev branch is merged to the
142            # *-release branch periodically. In order to extract the
143            # revision relevant to the source of the git_project_path,
144            # we fetch the *-release branch and get the revision of the
145            # parent commit with FETCH_HEAD^2.
146            logging.info(
147                'Checking if the parent of revision {rev} exists in {proj}'.
148                format(rev=revision, proj=git_project_path))
149            try:
150                cmd = ['git', '-C', path, 'fetch', self._remote_git, revision]
151                utils.check_call(cmd)
152                cmd = ['git', '-C', path, 'rev-parse', 'FETCH_HEAD^2']
153                parent_revision = utils.check_output(cmd).strip()
154            except subprocess.CalledProcessError as error:
155                logging.error(
156                    'Failed to get parent of revision {rev} from "{remote}": '
157                    '{err}'.format(
158                        rev=revision, remote=self._remote_git, err=error))
159                logging.error('Try --remote to manually set remote name')
160                raise
161            else:
162                if not _check_rev_list(parent_revision):
163                    return False
164
165        return True
166
167    def check_gpl_projects(self):
168        """Checks that all GPL projects have released sources.
169
170        Raises:
171          ValueError: There are GPL projects with unreleased sources.
172        """
173        logging.info('Starting license check for GPL projects...')
174
175        notice_files = glob.glob('{}/*'.format(self._notice_files_dir))
176        if len(notice_files) == 0:
177            raise RuntimeError('No license files found in {}'.format(
178                self._notice_files_dir))
179
180        gpl_projects = []
181        pattern = 'GENERAL PUBLIC LICENSE'
182        for notice_file_path in notice_files:
183            with open(notice_file_path, 'r') as notice_file:
184                if pattern in notice_file.read():
185                    lib_name = os.path.splitext(
186                        os.path.basename(notice_file_path))[0]
187                    gpl_projects.append(lib_name)
188
189        if not gpl_projects:
190            logging.info('No GPL projects found.')
191            return
192
193        logging.info('GPL projects found: {}'.format(', '.join(gpl_projects)))
194
195        module_paths = self._parse_module_paths()
196        manifest_projects = self._parse_manifest()
197        released_projects = []
198        unreleased_projects = []
199
200        for lib in gpl_projects:
201            if lib in module_paths:
202                module_path = module_paths[lib]
203                revision = self._get_revision(module_path, manifest_projects)
204                if not revision:
205                    raise RuntimeError(
206                        'No project found for {path} in {manifest}'.format(
207                            path=module_path, manifest=self.MANIFEST_XML))
208                revision_exists = self._check_revision_exists(
209                    revision, module_path)
210                if not revision_exists:
211                    unreleased_projects.append((lib, module_path))
212                else:
213                    released_projects.append((lib, module_path))
214            else:
215                raise RuntimeError(
216                    'No module path was found for {lib} in {module_paths}'.
217                    format(lib=lib, module_paths=self.MODULE_PATHS_TXT))
218
219        if released_projects:
220            logging.info('Released GPL projects: {}'.format(released_projects))
221
222        if unreleased_projects:
223            raise ValueError(
224                ('FAIL: The following GPL projects have NOT been released in '
225                 'current tree: {}'.format(unreleased_projects)))
226
227        logging.info('PASS: All GPL projects have source in current tree.')
228
229
230def get_args():
231    parser = argparse.ArgumentParser()
232    parser.add_argument(
233        'vndk_version',
234        type=int,
235        help='VNDK snapshot version to check, e.g. "27".')
236    parser.add_argument('-b', '--branch', help='Branch to pull manifest from.')
237    parser.add_argument('--build', help='Build number to pull manifest from.')
238    parser.add_argument(
239        '--remote',
240        default='aosp',
241        help=('Remote name to fetch and check if the revision of VNDK snapshot '
242              'is included in the source to conform GPL license. default=aosp'))
243    parser.add_argument(
244        '-v',
245        '--verbose',
246        action='count',
247        default=0,
248        help='Increase output verbosity, e.g. "-v", "-vv".')
249    return parser.parse_args()
250
251
252def main():
253    """For local testing purposes.
254
255    Note: VNDK snapshot must be already installed under
256      prebuilts/vndk/v{version}.
257    """
258    ANDROID_BUILD_TOP = utils.get_android_build_top()
259    PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP,
260                                             'prebuilts/vndk')
261
262    args = get_args()
263    vndk_version = args.vndk_version
264    install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version))
265    remote = args.remote
266    if not os.path.isdir(install_dir):
267        raise ValueError(
268            'Please provide valid VNDK version. {} does not exist.'
269            .format(install_dir))
270    utils.set_logging_config(args.verbose)
271
272    temp_artifact_dir = tempfile.mkdtemp()
273    os.chdir(temp_artifact_dir)
274    manifest_pattern = 'manifest_{}.xml'.format(args.build)
275    manifest_dest = os.path.join(temp_artifact_dir, utils.MANIFEST_FILE_NAME)
276    logging.info('Fetching {file} from {branch} (bid: {build})'.format(
277        file=manifest_pattern, branch=args.branch, build=args.build))
278    utils.fetch_artifact(args.branch, args.build, manifest_pattern,
279                         manifest_dest)
280
281    license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP,
282                                 temp_artifact_dir, remote)
283    try:
284        license_checker.check_gpl_projects()
285    except ValueError as error:
286        logging.error('Error: {}'.format(error))
287        raise
288    finally:
289        logging.info(
290            'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir))
291        shutil.rmtree(temp_artifact_dir)
292
293    logging.info('Done.')
294
295
296if __name__ == '__main__':
297    main()
298