| #!/usr/bin/env -S python -u |
| # |
| # Copyright (C) 2022 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Analyze bootclasspath_fragment usage.""" |
| import argparse |
| import dataclasses |
| import json |
| import logging |
| import os |
| import re |
| import shutil |
| import subprocess |
| import tempfile |
| import textwrap |
| import typing |
| import sys |
| |
| _STUB_FLAGS_FILE = "out/soong/hiddenapi/hiddenapi-stub-flags.txt" |
| |
| _FLAGS_FILE = "out/soong/hiddenapi/hiddenapi-flags.csv" |
| |
| _INCONSISTENT_FLAGS = "ERROR: Hidden API flags are inconsistent:" |
| |
| |
| class BuildOperation: |
| |
| def __init__(self, popen): |
| self.popen = popen |
| self.returncode = None |
| |
| def lines(self): |
| """Return an iterator over the lines output by the build operation. |
| |
| The lines have had any trailing white space, including the newline |
| stripped. |
| """ |
| return newline_stripping_iter(self.popen.stdout.readline) |
| |
| def wait(self, *args, **kwargs): |
| self.popen.wait(*args, **kwargs) |
| self.returncode = self.popen.returncode |
| |
| |
| @dataclasses.dataclass() |
| class FlagDiffs: |
| """Encapsulates differences in flags reported by the build""" |
| |
| # Map from member signature to the (module flags, monolithic flags) |
| diffs: typing.Dict[str, typing.Tuple[str, str]] |
| |
| |
| @dataclasses.dataclass() |
| class ModuleInfo: |
| """Provides access to the generated module-info.json file. |
| |
| This is used to find the location of the file within which specific modules |
| are defined. |
| """ |
| |
| modules: typing.Dict[str, typing.Dict[str, typing.Any]] |
| |
| @staticmethod |
| def load(filename): |
| with open(filename, "r", encoding="utf8") as f: |
| j = json.load(f) |
| return ModuleInfo(j) |
| |
| def _module(self, module_name): |
| """Find module by name in module-info.json file""" |
| if module_name in self.modules: |
| return self.modules[module_name] |
| |
| raise Exception(f"Module {module_name} could not be found") |
| |
| def module_path(self, module_name): |
| module = self._module(module_name) |
| # The "path" is actually a list of paths, one for each class of module |
| # but as the modules are all created from bp files if a module does |
| # create multiple classes of make modules they should all have the same |
| # path. |
| paths = module["path"] |
| unique_paths = set(paths) |
| if len(unique_paths) != 1: |
| raise Exception(f"Expected module '{module_name}' to have a " |
| f"single unique path but found {unique_paths}") |
| return paths[0] |
| |
| |
| def extract_indent(line): |
| return re.match(r"([ \t]*)", line).group(1) |
| |
| |
| _SPECIAL_PLACEHOLDER: str = "SPECIAL_PLACEHOLDER" |
| |
| |
| @dataclasses.dataclass |
| class BpModifyRunner: |
| |
| bpmodify_path: str |
| |
| def add_values_to_property(self, property_name, values, module_name, |
| bp_file): |
| cmd = [ |
| self.bpmodify_path, "-a", values, "-property", property_name, "-m", |
| module_name, "-w", bp_file, bp_file |
| ] |
| |
| logging.debug(" ".join(cmd)) |
| subprocess.run( |
| cmd, |
| stderr=subprocess.STDOUT, |
| stdout=log_stream_for_subprocess(), |
| check=True) |
| |
| |
| @dataclasses.dataclass |
| class FileChange: |
| path: str |
| |
| description: str |
| |
| def __lt__(self, other): |
| return self.path < other.path |
| |
| |
| @dataclasses.dataclass |
| class HiddenApiPropertyChange: |
| |
| property_name: str |
| |
| values: typing.List[str] |
| |
| property_comment: str = "" |
| |
| def snippet(self, indent): |
| snippet = "\n" |
| snippet += format_comment_as_text(self.property_comment, indent) |
| snippet += f"{indent}{self.property_name}: [" |
| if self.values: |
| snippet += "\n" |
| for value in self.values: |
| snippet += f'{indent} "{value}",\n' |
| snippet += f"{indent}" |
| snippet += "],\n" |
| return snippet |
| |
| def fix_bp_file(self, bcpf_bp_file, bcpf, bpmodify_runner: BpModifyRunner): |
| # Add an additional placeholder value to identify the modification that |
| # bpmodify makes. |
| bpmodify_values = [_SPECIAL_PLACEHOLDER] |
| bpmodify_values.extend(self.values) |
| |
| packages = ",".join(bpmodify_values) |
| bpmodify_runner.add_values_to_property( |
| f"hidden_api.{self.property_name}", packages, bcpf, bcpf_bp_file) |
| |
| with open(bcpf_bp_file, "r", encoding="utf8") as tio: |
| lines = tio.readlines() |
| lines = [line.rstrip("\n") for line in lines] |
| |
| if self.fixup_bpmodify_changes(bcpf_bp_file, lines): |
| with open(bcpf_bp_file, "w", encoding="utf8") as tio: |
| for line in lines: |
| print(line, file=tio) |
| |
| def fixup_bpmodify_changes(self, bcpf_bp_file, lines): |
| # Find the line containing the placeholder that has been inserted. |
| place_holder_index = -1 |
| for i, line in enumerate(lines): |
| if _SPECIAL_PLACEHOLDER in line: |
| place_holder_index = i |
| break |
| if place_holder_index == -1: |
| logging.debug("Could not find %s in %s", _SPECIAL_PLACEHOLDER, |
| bcpf_bp_file) |
| return False |
| |
| # Remove the place holder. Do this before inserting the comment as that |
| # would change the location of the place holder in the list. |
| place_holder_line = lines[place_holder_index] |
| if place_holder_line.endswith("],"): |
| place_holder_line = place_holder_line.replace( |
| f'"{_SPECIAL_PLACEHOLDER}"', "") |
| lines[place_holder_index] = place_holder_line |
| else: |
| del lines[place_holder_index] |
| |
| # Scan forward to the end of the property block to remove a blank line |
| # that bpmodify inserts. |
| end_property_array_index = -1 |
| for i in range(place_holder_index, len(lines)): |
| line = lines[i] |
| if line.endswith("],"): |
| end_property_array_index = i |
| break |
| if end_property_array_index == -1: |
| logging.debug("Could not find end of property array in %s", |
| bcpf_bp_file) |
| return False |
| |
| # If bdmodify inserted a blank line afterwards then remove it. |
| if (not lines[end_property_array_index + 1] and |
| lines[end_property_array_index + 2].endswith("},")): |
| del lines[end_property_array_index + 1] |
| |
| # Scan back to find the preceding property line. |
| property_line_index = -1 |
| for i in range(place_holder_index, 0, -1): |
| line = lines[i] |
| if line.lstrip().startswith(f"{self.property_name}: ["): |
| property_line_index = i |
| break |
| if property_line_index == -1: |
| logging.debug("Could not find property line in %s", bcpf_bp_file) |
| return False |
| |
| # Only insert a comment if the property does not already have a comment. |
| line_preceding_property = lines[(property_line_index - 1)] |
| if (self.property_comment and |
| not re.match("([ \t]+)// ", line_preceding_property)): |
| # Extract the indent from the property line and use it to format the |
| # comment. |
| indent = extract_indent(lines[property_line_index]) |
| comment_lines = format_comment_as_lines(self.property_comment, |
| indent) |
| |
| # If the line before the comment is not blank then insert an extra |
| # blank line at the beginning of the comment. |
| if line_preceding_property: |
| comment_lines.insert(0, "") |
| |
| # Insert the comment before the property. |
| lines[property_line_index:property_line_index] = comment_lines |
| return True |
| |
| |
| @dataclasses.dataclass() |
| class Result: |
| """Encapsulates the result of the analysis.""" |
| |
| # The diffs in the flags. |
| diffs: typing.Optional[FlagDiffs] = None |
| |
| # The bootclasspath_fragment hidden API properties changes. |
| property_changes: typing.List[HiddenApiPropertyChange] = dataclasses.field( |
| default_factory=list) |
| |
| # The list of file changes. |
| file_changes: typing.List[FileChange] = dataclasses.field( |
| default_factory=list) |
| |
| |
| @dataclasses.dataclass() |
| class BcpfAnalyzer: |
| # Path to this tool. |
| tool_path: str |
| |
| # Directory pointed to by ANDROID_BUILD_OUT |
| top_dir: str |
| |
| # Directory pointed to by OUT_DIR of {top_dir}/out if that is not set. |
| out_dir: str |
| |
| # Directory pointed to by ANDROID_PRODUCT_OUT. |
| product_out_dir: str |
| |
| # The name of the bootclasspath_fragment module. |
| bcpf: str |
| |
| # The name of the apex module containing {bcpf}, only used for |
| # informational purposes. |
| apex: str |
| |
| # The name of the sdk module containing {bcpf}, only used for |
| # informational purposes. |
| sdk: str |
| |
| # If true then this will attempt to automatically fix any issues that are |
| # found. |
| fix: bool = False |
| |
| # All the signatures, loaded from all-flags.csv, initialized by |
| # load_all_flags(). |
| _signatures: typing.Set[str] = dataclasses.field(default_factory=set) |
| |
| # All the classes, loaded from all-flags.csv, initialized by |
| # load_all_flags(). |
| _classes: typing.Set[str] = dataclasses.field(default_factory=set) |
| |
| # Information loaded from module-info.json, initialized by |
| # load_module_info(). |
| module_info: ModuleInfo = None |
| |
| @staticmethod |
| def reformat_report_test(text): |
| return re.sub(r"(.)\n([^\s])", r"\1 \2", text) |
| |
| def report(self, text, **kwargs): |
| # Concatenate lines that are not separated by a blank line together to |
| # eliminate formatting applied to the supplied text to adhere to python |
| # line length limitations. |
| text = self.reformat_report_test(text) |
| logging.info("%s", text, **kwargs) |
| |
| def run_command(self, cmd, *args, **kwargs): |
| cmd_line = " ".join(cmd) |
| logging.debug("Running %s", cmd_line) |
| subprocess.run( |
| cmd, |
| *args, |
| check=True, |
| cwd=self.top_dir, |
| stderr=subprocess.STDOUT, |
| stdout=log_stream_for_subprocess(), |
| text=True, |
| **kwargs) |
| |
| @property |
| def signatures(self): |
| if not self._signatures: |
| raise Exception("signatures has not been initialized") |
| return self._signatures |
| |
| @property |
| def classes(self): |
| if not self._classes: |
| raise Exception("classes has not been initialized") |
| return self._classes |
| |
| def load_all_flags(self): |
| all_flags = self.find_bootclasspath_fragment_output_file( |
| "all-flags.csv") |
| |
| # Extract the set of signatures and a separate set of classes produced |
| # by the bootclasspath_fragment. |
| with open(all_flags, "r", encoding="utf8") as f: |
| for line in newline_stripping_iter(f.readline): |
| signature = self.line_to_signature(line) |
| self._signatures.add(signature) |
| class_name = self.signature_to_class(signature) |
| self._classes.add(class_name) |
| |
| def load_module_info(self): |
| module_info_file = os.path.join(self.product_out_dir, |
| "module-info.json") |
| self.report(f""" |
| Making sure that {module_info_file} is up to date. |
| """) |
| output = self.build_file_read_output(module_info_file) |
| lines = output.lines() |
| for line in lines: |
| logging.debug("%s", line) |
| output.wait(timeout=10) |
| if output.returncode: |
| raise Exception(f"Error building {module_info_file}") |
| abs_module_info_file = os.path.join(self.top_dir, module_info_file) |
| self.module_info = ModuleInfo.load(abs_module_info_file) |
| |
| @staticmethod |
| def line_to_signature(line): |
| return line.split(",")[0] |
| |
| @staticmethod |
| def signature_to_class(signature): |
| return signature.split(";->")[0] |
| |
| @staticmethod |
| def to_parent_package(pkg_or_class): |
| return pkg_or_class.rsplit("/", 1)[0] |
| |
| def module_path(self, module_name): |
| return self.module_info.module_path(module_name) |
| |
| def module_out_dir(self, module_name): |
| module_path = self.module_path(module_name) |
| return os.path.join(self.out_dir, "soong/.intermediates", module_path, |
| module_name) |
| |
| def find_bootclasspath_fragment_output_file(self, basename): |
| # Find the output file of the bootclasspath_fragment with the specified |
| # base name. |
| found_file = "" |
| bcpf_out_dir = self.module_out_dir(self.bcpf) |
| for (dirpath, _, filenames) in os.walk(bcpf_out_dir): |
| for f in filenames: |
| if f == basename: |
| found_file = os.path.join(dirpath, f) |
| break |
| if not found_file: |
| raise Exception(f"Could not find {basename} in {bcpf_out_dir}") |
| return found_file |
| |
| def analyze(self): |
| """Analyze a bootclasspath_fragment module. |
| |
| Provides help in resolving any existing issues and provides |
| optimizations that can be applied. |
| """ |
| self.report(f"Analyzing bootclasspath_fragment module {self.bcpf}") |
| self.report(f""" |
| Run this tool to help initialize a bootclasspath_fragment module. Before you |
| start make sure that: |
| |
| 1. The current checkout is up to date. |
| |
| 2. The environment has been initialized using lunch, e.g. |
| lunch aosp_arm64-userdebug |
| |
| 3. You have added a bootclasspath_fragment module to the appropriate Android.bp |
| file. Something like this: |
| |
| bootclasspath_fragment {{ |
| name: "{self.bcpf}", |
| contents: [ |
| "...", |
| ], |
| |
| // The bootclasspath_fragments that provide APIs on which this depends. |
| fragments: [ |
| {{ |
| apex: "com.android.art", |
| module: "art-bootclasspath-fragment", |
| }}, |
| ], |
| }} |
| |
| 4. You have added it to the platform_bootclasspath module in |
| frameworks/base/boot/Android.bp. Something like this: |
| |
| platform_bootclasspath {{ |
| name: "platform-bootclasspath", |
| fragments: [ |
| ... |
| {{ |
| apex: "{self.apex}", |
| module: "{self.bcpf}", |
| }}, |
| ], |
| }} |
| |
| 5. You have added an sdk module. Something like this: |
| |
| sdk {{ |
| name: "{self.sdk}", |
| bootclasspath_fragments: ["{self.bcpf}"], |
| }} |
| """) |
| |
| # Make sure that the module-info.json file is up to date. |
| self.load_module_info() |
| |
| self.report(""" |
| Cleaning potentially stale files. |
| """) |
| # Remove the out/soong/hiddenapi files. |
| shutil.rmtree(f"{self.out_dir}/soong/hiddenapi", ignore_errors=True) |
| |
| # Remove any bootclasspath_fragment output files. |
| shutil.rmtree(self.module_out_dir(self.bcpf), ignore_errors=True) |
| |
| self.build_monolithic_stubs_flags() |
| |
| result = Result() |
| |
| self.build_monolithic_flags(result) |
| |
| # If there were any changes that need to be made to the Android.bp |
| # file then either apply or report them. |
| if result.property_changes: |
| bcpf_dir = self.module_info.module_path(self.bcpf) |
| bcpf_bp_file = os.path.join(self.top_dir, bcpf_dir, "Android.bp") |
| if self.fix: |
| tool_dir = os.path.dirname(self.tool_path) |
| bpmodify_path = os.path.join(tool_dir, "bpmodify") |
| bpmodify_runner = BpModifyRunner(bpmodify_path) |
| for property_change in result.property_changes: |
| property_change.fix_bp_file(bcpf_bp_file, self.bcpf, |
| bpmodify_runner) |
| |
| result.file_changes.append( |
| self.new_file_change( |
| bcpf_bp_file, |
| f"Updated hidden_api properties of '{self.bcpf}'")) |
| |
| else: |
| hiddenapi_snippet = "" |
| for property_change in result.property_changes: |
| hiddenapi_snippet += property_change.snippet(" ") |
| |
| # Remove leading and trailing blank lines. |
| hiddenapi_snippet = hiddenapi_snippet.strip("\n") |
| |
| result.file_changes.append( |
| self.new_file_change( |
| bcpf_bp_file, f""" |
| Add the following snippet into the {self.bcpf} bootclasspath_fragment module |
| in the {bcpf_dir}/Android.bp file. If the hidden_api block already exists then |
| merge these properties into it. |
| |
| hidden_api: {{ |
| {hiddenapi_snippet} |
| }}, |
| """)) |
| |
| if result.file_changes: |
| if self.fix: |
| file_change_message = """ |
| The following files were modified by this script:""" |
| else: |
| file_change_message = """ |
| The following modifications need to be made:""" |
| |
| self.report(f""" |
| {file_change_message}""") |
| result.file_changes.sort() |
| for file_change in result.file_changes: |
| self.report(f""" |
| {file_change.path} |
| {file_change.description} |
| """.lstrip("\n")) |
| |
| if not self.fix: |
| self.report(""" |
| Run the command again with the --fix option to automatically make the above |
| changes. |
| """.lstrip()) |
| |
| def new_file_change(self, file, description): |
| return FileChange( |
| path=os.path.relpath(file, self.top_dir), description=description) |
| |
| def check_inconsistent_flag_lines(self, significant, module_line, |
| monolithic_line, separator_line): |
| if not (module_line.startswith("< ") and |
| monolithic_line.startswith("> ") and not separator_line): |
| # Something went wrong. |
| self.report(f"""Invalid build output detected: |
| module_line: "{module_line}" |
| monolithic_line: "{monolithic_line}" |
| separator_line: "{separator_line}" |
| """) |
| sys.exit(1) |
| |
| if significant: |
| logging.debug("%s", module_line) |
| logging.debug("%s", monolithic_line) |
| logging.debug("%s", separator_line) |
| |
| def scan_inconsistent_flags_report(self, lines): |
| """Scans a hidden API flags report |
| |
| The hidden API inconsistent flags report which looks something like |
| this. |
| |
| < out/soong/.intermediates/.../filtered-stub-flags.csv |
| > out/soong/hiddenapi/hiddenapi-stub-flags.txt |
| |
| < Landroid/compat/Compatibility;->clearOverrides()V |
| > Landroid/compat/Compatibility;->clearOverrides()V,core-platform-api |
| |
| """ |
| |
| # The basic format of an entry in the inconsistent flags report is: |
| # <module specific flag> |
| # <monolithic flag> |
| # <separator> |
| # |
| # Wrap the lines iterator in an iterator which returns a tuple |
| # consisting of the three separate lines. |
| triples = zip(lines, lines, lines) |
| |
| module_line, monolithic_line, separator_line = next(triples) |
| significant = False |
| bcpf_dir = self.module_info.module_path(self.bcpf) |
| if os.path.join(bcpf_dir, self.bcpf) in module_line: |
| # These errors are related to the bcpf being analyzed so |
| # keep them. |
| significant = True |
| else: |
| self.report(f"Filtering out errors related to {module_line}") |
| |
| self.check_inconsistent_flag_lines(significant, module_line, |
| monolithic_line, separator_line) |
| |
| diffs = {} |
| for module_line, monolithic_line, separator_line in triples: |
| self.check_inconsistent_flag_lines(significant, module_line, |
| monolithic_line, "") |
| |
| module_parts = module_line.removeprefix("< ").split(",") |
| module_signature = module_parts[0] |
| module_flags = module_parts[1:] |
| |
| monolithic_parts = monolithic_line.removeprefix("> ").split(",") |
| monolithic_signature = monolithic_parts[0] |
| monolithic_flags = monolithic_parts[1:] |
| |
| if module_signature != monolithic_signature: |
| # Something went wrong. |
| self.report(f"""Inconsistent signatures detected: |
| module_signature: "{module_signature}" |
| monolithic_signature: "{monolithic_signature}" |
| """) |
| sys.exit(1) |
| |
| diffs[module_signature] = (module_flags, monolithic_flags) |
| |
| if separator_line: |
| # If the separator line is not blank then it is the end of the |
| # current report, and possibly the start of another. |
| return separator_line, diffs |
| |
| return "", diffs |
| |
| def build_file_read_output(self, filename): |
| # Make sure the filename is relative to top if possible as the build |
| # may be using relative paths as the target. |
| rel_filename = filename.removeprefix(self.top_dir) |
| cmd = ["build/soong/soong_ui.bash", "--make-mode", rel_filename] |
| cmd_line = " ".join(cmd) |
| logging.debug("%s", cmd_line) |
| # pylint: disable=consider-using-with |
| output = subprocess.Popen( |
| cmd, |
| cwd=self.top_dir, |
| stderr=subprocess.STDOUT, |
| stdout=subprocess.PIPE, |
| text=True, |
| ) |
| return BuildOperation(popen=output) |
| |
| def build_hiddenapi_flags(self, filename): |
| output = self.build_file_read_output(filename) |
| |
| lines = output.lines() |
| diffs = None |
| for line in lines: |
| logging.debug("%s", line) |
| while line == _INCONSISTENT_FLAGS: |
| line, diffs = self.scan_inconsistent_flags_report(lines) |
| |
| output.wait(timeout=10) |
| if output.returncode != 0: |
| logging.debug("Command failed with %s", output.returncode) |
| else: |
| logging.debug("Command succeeded") |
| |
| return diffs |
| |
| def build_monolithic_stubs_flags(self): |
| self.report(f""" |
| Attempting to build {_STUB_FLAGS_FILE} to verify that the |
| bootclasspath_fragment has the correct API stubs available... |
| """) |
| |
| # Build the hiddenapi-stubs-flags.txt file. |
| diffs = self.build_hiddenapi_flags(_STUB_FLAGS_FILE) |
| if diffs: |
| self.report(f""" |
| There is a discrepancy between the stub API derived flags created by the |
| bootclasspath_fragment and the platform_bootclasspath. See preceding error |
| messages to see which flags are inconsistent. The inconsistencies can occur for |
| a couple of reasons: |
| |
| If you are building against prebuilts of the Android SDK, e.g. by using |
| TARGET_BUILD_APPS then the prebuilt versions of the APIs this |
| bootclasspath_fragment depends upon are out of date and need updating. See |
| go/update-prebuilts for help. |
| |
| Otherwise, this is happening because there are some stub APIs that are either |
| provided by or used by the contents of the bootclasspath_fragment but which are |
| not available to it. There are 4 ways to handle this: |
| |
| 1. A java_sdk_library in the contents property will automatically make its stub |
| APIs available to the bootclasspath_fragment so nothing needs to be done. |
| |
| 2. If the API provided by the bootclasspath_fragment is created by an api_only |
| java_sdk_library (or a java_library that compiles files generated by a |
| separate droidstubs module then it cannot be added to the contents and |
| instead must be added to the api.stubs property, e.g. |
| |
| bootclasspath_fragment {{ |
| name: "{self.bcpf}", |
| ... |
| api: {{ |
| stubs: ["$MODULE-api-only"]," |
| }}, |
| }} |
| |
| 3. If the contents use APIs provided by another bootclasspath_fragment then |
| it needs to be added to the fragments property, e.g. |
| |
| bootclasspath_fragment {{ |
| name: "{self.bcpf}", |
| ... |
| // The bootclasspath_fragments that provide APIs on which this depends. |
| fragments: [ |
| ... |
| {{ |
| apex: "com.android.other", |
| module: "com.android.other-bootclasspath-fragment", |
| }}, |
| ], |
| }} |
| |
| 4. If the contents use APIs from a module that is not part of another |
| bootclasspath_fragment then it must be added to the additional_stubs |
| property, e.g. |
| |
| bootclasspath_fragment {{ |
| name: "{self.bcpf}", |
| ... |
| additional_stubs: ["android-non-updatable"], |
| }} |
| |
| Like the api.stubs property these are typically java_sdk_library modules but |
| can be java_library too. |
| |
| Note: The "android-non-updatable" is treated as if it was a java_sdk_library |
| which it is not at the moment but will be in future. |
| """) |
| |
| return diffs |
| |
| def build_monolithic_flags(self, result): |
| self.report(f""" |
| Attempting to build {_FLAGS_FILE} to verify that the |
| bootclasspath_fragment has the correct hidden API flags... |
| """) |
| |
| # Build the hiddenapi-flags.csv file and extract any differences in |
| # the flags between this bootclasspath_fragment and the monolithic |
| # files. |
| result.diffs = self.build_hiddenapi_flags(_FLAGS_FILE) |
| |
| # Load information from the bootclasspath_fragment's all-flags.csv file. |
| self.load_all_flags() |
| |
| if result.diffs: |
| self.report(f""" |
| There is a discrepancy between the hidden API flags created by the |
| bootclasspath_fragment and the platform_bootclasspath. See preceding error |
| messages to see which flags are inconsistent. The inconsistencies can occur for |
| a couple of reasons: |
| |
| If you are building against prebuilts of this bootclasspath_fragment then the |
| prebuilt version of the sdk snapshot (specifically the hidden API flag files) |
| are inconsistent with the prebuilt version of the apex {self.apex}. Please |
| ensure that they are both updated from the same build. |
| |
| 1. There are custom hidden API flags specified in the one of the files in |
| frameworks/base/boot/hiddenapi which apply to the bootclasspath_fragment but |
| which are not supplied to the bootclasspath_fragment module. |
| |
| 2. The bootclasspath_fragment specifies invalid "package_prefixes" or |
| "split_packages" properties that match packages and classes that it does not |
| provide. |
| |
| """) |
| |
| # Check to see if there are any hiddenapi related properties that |
| # need to be added to the |
| self.report(""" |
| Checking custom hidden API flags.... |
| """) |
| self.check_frameworks_base_boot_hidden_api_files(result) |
| |
| def report_hidden_api_flag_file_changes(self, result, property_name, |
| flags_file, rel_bcpf_flags_file, |
| bcpf_flags_file): |
| matched_signatures = set() |
| # Open the flags file to read the flags from. |
| with open(flags_file, "r", encoding="utf8") as f: |
| for signature in newline_stripping_iter(f.readline): |
| if signature in self.signatures: |
| # The signature is provided by the bootclasspath_fragment so |
| # it will need to be moved to the bootclasspath_fragment |
| # specific file. |
| matched_signatures.add(signature) |
| |
| # If the bootclasspath_fragment specific flags file is not empty |
| # then it contains flags. That could either be new flags just moved |
| # from frameworks/base or previous contents of the file. In either |
| # case the file must not be removed. |
| if matched_signatures: |
| insert = textwrap.indent("\n".join(matched_signatures), |
| " ") |
| result.file_changes.append( |
| self.new_file_change( |
| flags_file, f"""Remove the following entries: |
| {insert} |
| """)) |
| |
| result.file_changes.append( |
| self.new_file_change( |
| bcpf_flags_file, f"""Add the following entries: |
| {insert} |
| """)) |
| |
| result.property_changes.append( |
| HiddenApiPropertyChange( |
| property_name=property_name, |
| values=[rel_bcpf_flags_file], |
| )) |
| |
| def fix_hidden_api_flag_files(self, result, property_name, flags_file, |
| rel_bcpf_flags_file, bcpf_flags_file): |
| # Read the file in frameworks/base/boot/hiddenapi/<file> copy any |
| # flags that relate to the bootclasspath_fragment into a local |
| # file in the hiddenapi subdirectory. |
| tmp_flags_file = flags_file + ".tmp" |
| |
| # Make sure the directory containing the bootclasspath_fragment specific |
| # hidden api flags exists. |
| os.makedirs(os.path.dirname(bcpf_flags_file), exist_ok=True) |
| |
| bcpf_flags_file_exists = os.path.exists(bcpf_flags_file) |
| |
| matched_signatures = set() |
| # Open the flags file to read the flags from. |
| with open(flags_file, "r", encoding="utf8") as f: |
| # Open a temporary file to write the flags (minus any removed |
| # flags). |
| with open(tmp_flags_file, "w", encoding="utf8") as t: |
| # Open the bootclasspath_fragment file for append just in |
| # case it already exists. |
| with open(bcpf_flags_file, "a", encoding="utf8") as b: |
| for line in iter(f.readline, ""): |
| signature = line.rstrip() |
| if signature in self.signatures: |
| # The signature is provided by the |
| # bootclasspath_fragment so write it to the new |
| # bootclasspath_fragment specific file. |
| print(line, file=b, end="") |
| matched_signatures.add(signature) |
| else: |
| # The signature is NOT provided by the |
| # bootclasspath_fragment. Copy it to the new |
| # monolithic file. |
| print(line, file=t, end="") |
| |
| # If the bootclasspath_fragment specific flags file is not empty |
| # then it contains flags. That could either be new flags just moved |
| # from frameworks/base or previous contents of the file. In either |
| # case the file must not be removed. |
| if matched_signatures: |
| # There are custom flags related to the bootclasspath_fragment |
| # so replace the frameworks/base/boot/hiddenapi file with the |
| # file that does not contain those flags. |
| shutil.move(tmp_flags_file, flags_file) |
| |
| result.file_changes.append( |
| self.new_file_change(flags_file, |
| f"Removed '{self.bcpf}' specific entries")) |
| |
| result.property_changes.append( |
| HiddenApiPropertyChange( |
| property_name=property_name, |
| values=[rel_bcpf_flags_file], |
| )) |
| |
| # Make sure that the files are sorted. |
| self.run_command([ |
| "tools/platform-compat/hiddenapi/sort_api.sh", |
| bcpf_flags_file, |
| ]) |
| |
| if bcpf_flags_file_exists: |
| desc = f"Added '{self.bcpf}' specific entries" |
| else: |
| desc = f"Created with '{self.bcpf}' specific entries" |
| result.file_changes.append( |
| self.new_file_change(bcpf_flags_file, desc)) |
| else: |
| # There are no custom flags related to the |
| # bootclasspath_fragment so clean up the working files. |
| os.remove(tmp_flags_file) |
| if not bcpf_flags_file_exists: |
| os.remove(bcpf_flags_file) |
| |
| def check_frameworks_base_boot_hidden_api_files(self, result): |
| hiddenapi_dir = os.path.join(self.top_dir, |
| "frameworks/base/boot/hiddenapi") |
| for basename in sorted(os.listdir(hiddenapi_dir)): |
| if not (basename.startswith("hiddenapi-") and |
| basename.endswith(".txt")): |
| continue |
| |
| flags_file = os.path.join(hiddenapi_dir, basename) |
| |
| logging.debug("Checking %s for flags related to %s", flags_file, |
| self.bcpf) |
| |
| # Map the file name in frameworks/base/boot/hiddenapi into a |
| # slightly more meaningful name for use by the |
| # bootclasspath_fragment. |
| if basename == "hiddenapi-max-target-o.txt": |
| basename = "hiddenapi-max-target-o-low-priority.txt" |
| elif basename == "hiddenapi-max-target-r-loprio.txt": |
| basename = "hiddenapi-max-target-r-low-priority.txt" |
| |
| property_name = basename.removeprefix("hiddenapi-") |
| property_name = property_name.removesuffix(".txt") |
| property_name = property_name.replace("-", "_") |
| |
| rel_bcpf_flags_file = f"hiddenapi/{basename}" |
| bcpf_dir = self.module_info.module_path(self.bcpf) |
| bcpf_flags_file = os.path.join(self.top_dir, bcpf_dir, |
| rel_bcpf_flags_file) |
| |
| if self.fix: |
| self.fix_hidden_api_flag_files(result, property_name, |
| flags_file, rel_bcpf_flags_file, |
| bcpf_flags_file) |
| else: |
| self.report_hidden_api_flag_file_changes( |
| result, property_name, flags_file, rel_bcpf_flags_file, |
| bcpf_flags_file) |
| |
| |
| def newline_stripping_iter(iterator): |
| """Return an iterator over the iterator that strips trailing white space.""" |
| lines = iter(iterator, "") |
| lines = (line.rstrip() for line in lines) |
| return lines |
| |
| |
| def format_comment_as_text(text, indent): |
| return "".join( |
| [f"{line}\n" for line in format_comment_as_lines(text, indent)]) |
| |
| |
| def format_comment_as_lines(text, indent): |
| lines = textwrap.wrap(text.strip("\n"), width=77 - len(indent)) |
| lines = [f"{indent}// {line}" for line in lines] |
| return lines |
| |
| |
| def log_stream_for_subprocess(): |
| stream = subprocess.DEVNULL |
| for handler in logging.root.handlers: |
| if handler.level == logging.DEBUG: |
| if isinstance(handler, logging.StreamHandler): |
| stream = handler.stream |
| return stream |
| |
| |
| def main(argv): |
| args_parser = argparse.ArgumentParser( |
| description="Analyze a bootclasspath_fragment module.") |
| args_parser.add_argument( |
| "--bcpf", |
| help="The bootclasspath_fragment module to analyze", |
| required=True, |
| ) |
| args_parser.add_argument( |
| "--apex", |
| help="The apex module to which the bootclasspath_fragment belongs. It " |
| "is not strictly necessary at the moment but providing it will " |
| "allow this script to give more useful messages and it may be" |
| "required in future.", |
| default="SPECIFY-APEX-OPTION") |
| args_parser.add_argument( |
| "--sdk", |
| help="The sdk module to which the bootclasspath_fragment belongs. It " |
| "is not strictly necessary at the moment but providing it will " |
| "allow this script to give more useful messages and it may be" |
| "required in future.", |
| default="SPECIFY-SDK-OPTION") |
| args_parser.add_argument( |
| "--fix", |
| help="Attempt to fix any issues found automatically.", |
| action="store_true", |
| default=False) |
| args = args_parser.parse_args(argv[1:]) |
| top_dir = os.environ["ANDROID_BUILD_TOP"] + "/" |
| out_dir = os.environ.get("OUT_DIR", os.path.join(top_dir, "out")) |
| product_out_dir = os.environ.get("ANDROID_PRODUCT_OUT", top_dir) |
| # Make product_out_dir relative to the top so it can be used as part of a |
| # build target. |
| product_out_dir = product_out_dir.removeprefix(top_dir) |
| log_fd, abs_log_file = tempfile.mkstemp( |
| suffix="_analyze_bcpf.log", text=True) |
| |
| with os.fdopen(log_fd, "w") as log_file: |
| # Set up debug logging to the log file. |
| logging.basicConfig( |
| level=logging.DEBUG, |
| format="%(levelname)-8s %(message)s", |
| stream=log_file) |
| |
| # define a Handler which writes INFO messages or higher to the |
| # sys.stdout with just the message. |
| console = logging.StreamHandler() |
| console.setLevel(logging.INFO) |
| console.setFormatter(logging.Formatter("%(message)s")) |
| # add the handler to the root logger |
| logging.getLogger("").addHandler(console) |
| |
| print(f"Writing log to {abs_log_file}") |
| try: |
| analyzer = BcpfAnalyzer( |
| tool_path=argv[0], |
| top_dir=top_dir, |
| out_dir=out_dir, |
| product_out_dir=product_out_dir, |
| bcpf=args.bcpf, |
| apex=args.apex, |
| sdk=args.sdk, |
| fix=args.fix, |
| ) |
| analyzer.analyze() |
| finally: |
| print(f"Log written to {abs_log_file}") |
| |
| |
| if __name__ == "__main__": |
| main(sys.argv) |