Add analyze_bcpf.py script to help manage bootclasspath_fragments
The analyze_bcpf.py script can be used by bootclasspath_fragment owners
to help setup their fragment and also to debug issues that arise during
use.
It is run by specifying the name of the bootclasspath_fragment module
in the --bcpf option. If specified the --apex and --sdk options will
allow the tool to produce more helpful messages.
The script currently:
* Builds out/soong/hiddenapi/hiddenapi-stub-flags.txt and checks any
inconsistencies between those flags and the flags generated for the
bootclasspath_fragment.
* Builds out/soong/hiddenapi/hiddenapi-flags.csv and checks any
inconsistencies between those flags and the flags generated for the
bootclasspath_fragment. It will describe the changes that need to
be made in order to fix any inconsistencies in the flags.
Bug: 202154151
Test: m analyze_bcpf && analyze_bcpf --bcpf art-bootclasspath-fragment
atest --host analyze_bcpf_test
Change-Id: I824fd9d527d246dfe56725c073d89a0b0e72d25a
diff --git a/scripts/hiddenapi/Android.bp b/scripts/hiddenapi/Android.bp
index 8a47c5d..a6a368d 100644
--- a/scripts/hiddenapi/Android.bp
+++ b/scripts/hiddenapi/Android.bp
@@ -19,6 +19,48 @@
}
python_binary_host {
+ name: "analyze_bcpf",
+ main: "analyze_bcpf.py",
+ srcs: ["analyze_bcpf.py"],
+ libs: [
+ "signature_trie",
+ ],
+ version: {
+ py2: {
+ enabled: false,
+ },
+ py3: {
+ enabled: true,
+ embedded_launcher: true,
+ },
+ },
+}
+
+python_test_host {
+ name: "analyze_bcpf_test",
+ main: "analyze_bcpf_test.py",
+ srcs: [
+ "analyze_bcpf.py",
+ "analyze_bcpf_test.py",
+ ],
+ libs: [
+ "signature_trie",
+ ],
+ version: {
+ py2: {
+ enabled: false,
+ },
+ py3: {
+ enabled: true,
+ embedded_launcher: true,
+ },
+ },
+ test_options: {
+ unit_test: true,
+ },
+}
+
+python_binary_host {
name: "merge_csv",
main: "merge_csv.py",
srcs: ["merge_csv.py"],
diff --git a/scripts/hiddenapi/analyze_bcpf.py b/scripts/hiddenapi/analyze_bcpf.py
new file mode 100644
index 0000000..c648fe8
--- /dev/null
+++ b/scripts/hiddenapi/analyze_bcpf.py
@@ -0,0 +1,794 @@
+#!/usr/bin/env -S python -u
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Analyze bootclasspath_fragment usage."""
+import argparse
+import dataclasses
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import textwrap
+import typing
+import sys
+
+_STUB_FLAGS_FILE = "out/soong/hiddenapi/hiddenapi-stub-flags.txt"
+
+_FLAGS_FILE = "out/soong/hiddenapi/hiddenapi-flags.csv"
+
+_INCONSISTENT_FLAGS = "ERROR: Hidden API flags are inconsistent:"
+
+
+class BuildOperation:
+
+ def __init__(self, popen):
+ self.popen = popen
+ self.returncode = None
+
+ def lines(self):
+ """Return an iterator over the lines output by the build operation.
+
+ The lines have had any trailing white space, including the newline
+ stripped.
+ """
+ return newline_stripping_iter(self.popen.stdout.readline)
+
+ def wait(self, *args, **kwargs):
+ self.popen.wait(*args, **kwargs)
+ self.returncode = self.popen.returncode
+
+
+@dataclasses.dataclass()
+class FlagDiffs:
+ """Encapsulates differences in flags reported by the build"""
+
+ # Map from member signature to the (module flags, monolithic flags)
+ diffs: typing.Dict[str, typing.Tuple[str, str]]
+
+
+@dataclasses.dataclass()
+class ModuleInfo:
+ """Provides access to the generated module-info.json file.
+
+ This is used to find the location of the file within which specific modules
+ are defined.
+ """
+
+ modules: typing.Dict[str, typing.Dict[str, typing.Any]]
+
+ @staticmethod
+ def load(filename):
+ with open(filename, "r", encoding="utf8") as f:
+ j = json.load(f)
+ return ModuleInfo(j)
+
+ def _module(self, module_name):
+ """Find module by name in module-info.json file"""
+ if module_name in self.modules:
+ return self.modules[module_name]
+
+ raise Exception(f"Module {module_name} could not be found")
+
+ def module_path(self, module_name):
+ module = self._module(module_name)
+ # The "path" is actually a list of paths, one for each class of module
+ # but as the modules are all created from bp files if a module does
+ # create multiple classes of make modules they should all have the same
+ # path.
+ paths = module["path"]
+ unique_paths = set(paths)
+ if len(unique_paths) != 1:
+ raise Exception(f"Expected module '{module_name}' to have a "
+ f"single unique path but found {unique_paths}")
+ return paths[0]
+
+
+@dataclasses.dataclass
+class FileChange:
+ path: str
+
+ description: str
+
+ def __lt__(self, other):
+ return self.path < other.path
+
+
+@dataclasses.dataclass
+class HiddenApiPropertyChange:
+
+ property_name: str
+
+ values: typing.List[str]
+
+ property_comment: str = ""
+
+ def snippet(self, indent):
+ snippet = "\n"
+ snippet += format_comment_as_text(self.property_comment, indent)
+ snippet += f"{indent}{self.property_name}: ["
+ if self.values:
+ snippet += "\n"
+ for value in self.values:
+ snippet += f'{indent} "{value}",\n'
+ snippet += f"{indent}"
+ snippet += "],\n"
+ return snippet
+
+
+@dataclasses.dataclass()
+class Result:
+ """Encapsulates the result of the analysis."""
+
+ # The diffs in the flags.
+ diffs: typing.Optional[FlagDiffs] = None
+
+ # The bootclasspath_fragment hidden API properties changes.
+ property_changes: typing.List[HiddenApiPropertyChange] = dataclasses.field(
+ default_factory=list)
+
+ # The list of file changes.
+ file_changes: typing.List[FileChange] = dataclasses.field(
+ default_factory=list)
+
+
+@dataclasses.dataclass()
+class BcpfAnalyzer:
+ # Directory pointed to by ANDROID_BUILD_OUT
+ top_dir: str
+
+ # Directory pointed to by OUT_DIR of {top_dir}/out if that is not set.
+ out_dir: str
+
+ # Directory pointed to by ANDROID_PRODUCT_OUT.
+ product_out_dir: str
+
+ # The name of the bootclasspath_fragment module.
+ bcpf: str
+
+ # The name of the apex module containing {bcpf}, only used for
+ # informational purposes.
+ apex: str
+
+ # The name of the sdk module containing {bcpf}, only used for
+ # informational purposes.
+ sdk: str
+
+ # All the signatures, loaded from all-flags.csv, initialized by
+ # load_all_flags().
+ _signatures: typing.Set[str] = dataclasses.field(default_factory=set)
+
+ # All the classes, loaded from all-flags.csv, initialized by
+ # load_all_flags().
+ _classes: typing.Set[str] = dataclasses.field(default_factory=set)
+
+ # Information loaded from module-info.json, initialized by
+ # load_module_info().
+ module_info: ModuleInfo = None
+
+ @staticmethod
+ def reformat_report_test(text):
+ return re.sub(r"(.)\n([^\s])", r"\1 \2", text)
+
+ def report(self, text, **kwargs):
+ # Concatenate lines that are not separated by a blank line together to
+ # eliminate formatting applied to the supplied text to adhere to python
+ # line length limitations.
+ text = self.reformat_report_test(text)
+ logging.info("%s", text, **kwargs)
+
+ def run_command(self, cmd, *args, **kwargs):
+ cmd_line = " ".join(cmd)
+ logging.debug("Running %s", cmd_line)
+ subprocess.run(
+ cmd,
+ *args,
+ check=True,
+ cwd=self.top_dir,
+ stderr=subprocess.STDOUT,
+ stdout=log_stream_for_subprocess(),
+ text=True,
+ **kwargs)
+
+ @property
+ def signatures(self):
+ if not self._signatures:
+ raise Exception("signatures has not been initialized")
+ return self._signatures
+
+ @property
+ def classes(self):
+ if not self._classes:
+ raise Exception("classes has not been initialized")
+ return self._classes
+
+ def load_all_flags(self):
+ all_flags = self.find_bootclasspath_fragment_output_file(
+ "all-flags.csv")
+
+ # Extract the set of signatures and a separate set of classes produced
+ # by the bootclasspath_fragment.
+ with open(all_flags, "r", encoding="utf8") as f:
+ for line in newline_stripping_iter(f.readline):
+ signature = self.line_to_signature(line)
+ self._signatures.add(signature)
+ class_name = self.signature_to_class(signature)
+ self._classes.add(class_name)
+
+ def load_module_info(self):
+ module_info_file = os.path.join(self.product_out_dir,
+ "module-info.json")
+ self.report(f"""
+Making sure that {module_info_file} is up to date.
+""")
+ output = self.build_file_read_output(module_info_file)
+ lines = output.lines()
+ for line in lines:
+ logging.debug("%s", line)
+ output.wait(timeout=10)
+ if output.returncode:
+ raise Exception(f"Error building {module_info_file}")
+ abs_module_info_file = os.path.join(self.top_dir, module_info_file)
+ self.module_info = ModuleInfo.load(abs_module_info_file)
+
+ @staticmethod
+ def line_to_signature(line):
+ return line.split(",")[0]
+
+ @staticmethod
+ def signature_to_class(signature):
+ return signature.split(";->")[0]
+
+ @staticmethod
+ def to_parent_package(pkg_or_class):
+ return pkg_or_class.rsplit("/", 1)[0]
+
+ def module_path(self, module_name):
+ return self.module_info.module_path(module_name)
+
+ def module_out_dir(self, module_name):
+ module_path = self.module_path(module_name)
+ return os.path.join(self.out_dir, "soong/.intermediates", module_path,
+ module_name)
+
+ def find_bootclasspath_fragment_output_file(self, basename):
+ # Find the output file of the bootclasspath_fragment with the specified
+ # base name.
+ found_file = ""
+ bcpf_out_dir = self.module_out_dir(self.bcpf)
+ for (dirpath, _, filenames) in os.walk(bcpf_out_dir):
+ for f in filenames:
+ if f == basename:
+ found_file = os.path.join(dirpath, f)
+ break
+ if not found_file:
+ raise Exception(f"Could not find {basename} in {bcpf_out_dir}")
+ return found_file
+
+ def analyze(self):
+ """Analyze a bootclasspath_fragment module.
+
+ Provides help in resolving any existing issues and provides
+ optimizations that can be applied.
+ """
+ self.report(f"Analyzing bootclasspath_fragment module {self.bcpf}")
+ self.report(f"""
+Run this tool to help initialize a bootclasspath_fragment module. Before you
+start make sure that:
+
+1. The current checkout is up to date.
+
+2. The environment has been initialized using lunch, e.g.
+ lunch aosp_arm64-userdebug
+
+3. You have added a bootclasspath_fragment module to the appropriate Android.bp
+file. Something like this:
+
+ bootclasspath_fragment {{
+ name: "{self.bcpf}",
+ contents: [
+ "...",
+ ],
+
+ // The bootclasspath_fragments that provide APIs on which this depends.
+ fragments: [
+ {{
+ apex: "com.android.art",
+ module: "art-bootclasspath-fragment",
+ }},
+ ],
+ }}
+
+4. You have added it to the platform_bootclasspath module in
+frameworks/base/boot/Android.bp. Something like this:
+
+ platform_bootclasspath {{
+ name: "platform-bootclasspath",
+ fragments: [
+ ...
+ {{
+ apex: "{self.apex}",
+ module: "{self.bcpf}",
+ }},
+ ],
+ }}
+
+5. You have added an sdk module. Something like this:
+
+ sdk {{
+ name: "{self.sdk}",
+ bootclasspath_fragments: ["{self.bcpf}"],
+ }}
+""")
+
+ # Make sure that the module-info.json file is up to date.
+ self.load_module_info()
+
+ self.report("""
+Cleaning potentially stale files.
+""")
+ # Remove the out/soong/hiddenapi files.
+ shutil.rmtree(f"{self.out_dir}/soong/hiddenapi", ignore_errors=True)
+
+ # Remove any bootclasspath_fragment output files.
+ shutil.rmtree(self.module_out_dir(self.bcpf), ignore_errors=True)
+
+ self.build_monolithic_stubs_flags()
+
+ result = Result()
+
+ self.build_monolithic_flags(result)
+
+ # If there were any changes that need to be made to the Android.bp
+ # file then report them.
+ if result.property_changes:
+ bcpf_dir = self.module_info.module_path(self.bcpf)
+ bcpf_bp_file = os.path.join(self.top_dir, bcpf_dir, "Android.bp")
+ hiddenapi_snippet = ""
+ for property_change in result.property_changes:
+ hiddenapi_snippet += property_change.snippet(" ")
+
+ # Remove leading and trailing blank lines.
+ hiddenapi_snippet = hiddenapi_snippet.strip("\n")
+
+ result.file_changes.append(
+ self.new_file_change(
+ bcpf_bp_file, f"""
+Add the following snippet into the {self.bcpf} bootclasspath_fragment module
+in the {bcpf_dir}/Android.bp file. If the hidden_api block already exists then
+merge these properties into it.
+
+ hidden_api: {{
+{hiddenapi_snippet}
+ }},
+"""))
+
+ if result.file_changes:
+ self.report("""
+The following modifications need to be made:""")
+ result.file_changes.sort()
+ for file_change in result.file_changes:
+ self.report(f"""
+ {file_change.path}
+ {file_change.description}
+""".lstrip("\n"))
+
+ def new_file_change(self, file, description):
+ return FileChange(
+ path=os.path.relpath(file, self.top_dir), description=description)
+
+ def check_inconsistent_flag_lines(self, significant, module_line,
+ monolithic_line, separator_line):
+ if not (module_line.startswith("< ") and
+ monolithic_line.startswith("> ") and not separator_line):
+ # Something went wrong.
+ self.report(f"""Invalid build output detected:
+ module_line: "{module_line}"
+ monolithic_line: "{monolithic_line}"
+ separator_line: "{separator_line}"
+""")
+ sys.exit(1)
+
+ if significant:
+ logging.debug("%s", module_line)
+ logging.debug("%s", monolithic_line)
+ logging.debug("%s", separator_line)
+
+ def scan_inconsistent_flags_report(self, lines):
+ """Scans a hidden API flags report
+
+ The hidden API inconsistent flags report which looks something like
+ this.
+
+ < out/soong/.intermediates/.../filtered-stub-flags.csv
+ > out/soong/hiddenapi/hiddenapi-stub-flags.txt
+
+ < Landroid/compat/Compatibility;->clearOverrides()V
+ > Landroid/compat/Compatibility;->clearOverrides()V,core-platform-api
+
+ """
+
+ # The basic format of an entry in the inconsistent flags report is:
+ # <module specific flag>
+ # <monolithic flag>
+ # <separator>
+ #
+ # Wrap the lines iterator in an iterator which returns a tuple
+ # consisting of the three separate lines.
+ triples = zip(lines, lines, lines)
+
+ module_line, monolithic_line, separator_line = next(triples)
+ significant = False
+ bcpf_dir = self.module_info.module_path(self.bcpf)
+ if os.path.join(bcpf_dir, self.bcpf) in module_line:
+ # These errors are related to the bcpf being analyzed so
+ # keep them.
+ significant = True
+ else:
+ self.report(f"Filtering out errors related to {module_line}")
+
+ self.check_inconsistent_flag_lines(significant, module_line,
+ monolithic_line, separator_line)
+
+ diffs = {}
+ for module_line, monolithic_line, separator_line in triples:
+ self.check_inconsistent_flag_lines(significant, module_line,
+ monolithic_line, "")
+
+ module_parts = module_line.removeprefix("< ").split(",")
+ module_signature = module_parts[0]
+ module_flags = module_parts[1:]
+
+ monolithic_parts = monolithic_line.removeprefix("> ").split(",")
+ monolithic_signature = monolithic_parts[0]
+ monolithic_flags = monolithic_parts[1:]
+
+ if module_signature != monolithic_signature:
+ # Something went wrong.
+ self.report(f"""Inconsistent signatures detected:
+ module_signature: "{module_signature}"
+ monolithic_signature: "{monolithic_signature}"
+""")
+ sys.exit(1)
+
+ diffs[module_signature] = (module_flags, monolithic_flags)
+
+ if separator_line:
+ # If the separator line is not blank then it is the end of the
+ # current report, and possibly the start of another.
+ return separator_line, diffs
+
+ return "", diffs
+
+ def build_file_read_output(self, filename):
+ # Make sure the filename is relative to top if possible as the build
+ # may be using relative paths as the target.
+ rel_filename = filename.removeprefix(self.top_dir)
+ cmd = ["build/soong/soong_ui.bash", "--make-mode", rel_filename]
+ cmd_line = " ".join(cmd)
+ logging.debug("%s", cmd_line)
+ # pylint: disable=consider-using-with
+ output = subprocess.Popen(
+ cmd,
+ cwd=self.top_dir,
+ stderr=subprocess.STDOUT,
+ stdout=subprocess.PIPE,
+ text=True,
+ )
+ return BuildOperation(popen=output)
+
+ def build_hiddenapi_flags(self, filename):
+ output = self.build_file_read_output(filename)
+
+ lines = output.lines()
+ diffs = None
+ for line in lines:
+ logging.debug("%s", line)
+ while line == _INCONSISTENT_FLAGS:
+ line, diffs = self.scan_inconsistent_flags_report(lines)
+
+ output.wait(timeout=10)
+ if output.returncode != 0:
+ logging.debug("Command failed with %s", output.returncode)
+ else:
+ logging.debug("Command succeeded")
+
+ return diffs
+
+ def build_monolithic_stubs_flags(self):
+ self.report(f"""
+Attempting to build {_STUB_FLAGS_FILE} to verify that the
+bootclasspath_fragment has the correct API stubs available...
+""")
+
+ # Build the hiddenapi-stubs-flags.txt file.
+ diffs = self.build_hiddenapi_flags(_STUB_FLAGS_FILE)
+ if diffs:
+ self.report(f"""
+There is a discrepancy between the stub API derived flags created by the
+bootclasspath_fragment and the platform_bootclasspath. See preceding error
+messages to see which flags are inconsistent. The inconsistencies can occur for
+a couple of reasons:
+
+If you are building against prebuilts of the Android SDK, e.g. by using
+TARGET_BUILD_APPS then the prebuilt versions of the APIs this
+bootclasspath_fragment depends upon are out of date and need updating. See
+go/update-prebuilts for help.
+
+Otherwise, this is happening because there are some stub APIs that are either
+provided by or used by the contents of the bootclasspath_fragment but which are
+not available to it. There are 4 ways to handle this:
+
+1. A java_sdk_library in the contents property will automatically make its stub
+ APIs available to the bootclasspath_fragment so nothing needs to be done.
+
+2. If the API provided by the bootclasspath_fragment is created by an api_only
+ java_sdk_library (or a java_library that compiles files generated by a
+ separate droidstubs module then it cannot be added to the contents and
+ instead must be added to the api.stubs property, e.g.
+
+ bootclasspath_fragment {{
+ name: "{self.bcpf}",
+ ...
+ api: {{
+ stubs: ["$MODULE-api-only"],"
+ }},
+ }}
+
+3. If the contents use APIs provided by another bootclasspath_fragment then
+ it needs to be added to the fragments property, e.g.
+
+ bootclasspath_fragment {{
+ name: "{self.bcpf}",
+ ...
+ // The bootclasspath_fragments that provide APIs on which this depends.
+ fragments: [
+ ...
+ {{
+ apex: "com.android.other",
+ module: "com.android.other-bootclasspath-fragment",
+ }},
+ ],
+ }}
+
+4. If the contents use APIs from a module that is not part of another
+ bootclasspath_fragment then it must be added to the additional_stubs
+ property, e.g.
+
+ bootclasspath_fragment {{
+ name: "{self.bcpf}",
+ ...
+ additional_stubs: ["android-non-updatable"],
+ }}
+
+ Like the api.stubs property these are typically java_sdk_library modules but
+ can be java_library too.
+
+ Note: The "android-non-updatable" is treated as if it was a java_sdk_library
+ which it is not at the moment but will be in future.
+""")
+
+ return diffs
+
+ def build_monolithic_flags(self, result):
+ self.report(f"""
+Attempting to build {_FLAGS_FILE} to verify that the
+bootclasspath_fragment has the correct hidden API flags...
+""")
+
+ # Build the hiddenapi-flags.csv file and extract any differences in
+ # the flags between this bootclasspath_fragment and the monolithic
+ # files.
+ result.diffs = self.build_hiddenapi_flags(_FLAGS_FILE)
+
+ # Load information from the bootclasspath_fragment's all-flags.csv file.
+ self.load_all_flags()
+
+ if result.diffs:
+ self.report(f"""
+There is a discrepancy between the hidden API flags created by the
+bootclasspath_fragment and the platform_bootclasspath. See preceding error
+messages to see which flags are inconsistent. The inconsistencies can occur for
+a couple of reasons:
+
+If you are building against prebuilts of this bootclasspath_fragment then the
+prebuilt version of the sdk snapshot (specifically the hidden API flag files)
+are inconsistent with the prebuilt version of the apex {self.apex}. Please
+ensure that they are both updated from the same build.
+
+1. There are custom hidden API flags specified in the one of the files in
+ frameworks/base/boot/hiddenapi which apply to the bootclasspath_fragment but
+ which are not supplied to the bootclasspath_fragment module.
+
+2. The bootclasspath_fragment specifies invalid "package_prefixes" or
+ "split_packages" properties that match packages and classes that it does not
+ provide.
+
+""")
+
+ # Check to see if there are any hiddenapi related properties that
+ # need to be added to the
+ self.report("""
+Checking custom hidden API flags....
+""")
+ self.check_frameworks_base_boot_hidden_api_files(result)
+
+ def report_hidden_api_flag_file_changes(self, result, property_name,
+ flags_file, rel_bcpf_flags_file,
+ bcpf_flags_file):
+ matched_signatures = set()
+ # Open the flags file to read the flags from.
+ with open(flags_file, "r", encoding="utf8") as f:
+ for signature in newline_stripping_iter(f.readline):
+ if signature in self.signatures:
+ # The signature is provided by the bootclasspath_fragment so
+ # it will need to be moved to the bootclasspath_fragment
+ # specific file.
+ matched_signatures.add(signature)
+
+ # If the bootclasspath_fragment specific flags file is not empty
+ # then it contains flags. That could either be new flags just moved
+ # from frameworks/base or previous contents of the file. In either
+ # case the file must not be removed.
+ if matched_signatures:
+ insert = textwrap.indent("\n".join(matched_signatures),
+ " ")
+ result.file_changes.append(
+ self.new_file_change(
+ flags_file, f"""Remove the following entries:
+{insert}
+"""))
+
+ result.file_changes.append(
+ self.new_file_change(
+ bcpf_flags_file, f"""Add the following entries:
+{insert}
+"""))
+
+ result.property_changes.append(
+ HiddenApiPropertyChange(
+ property_name=property_name,
+ values=[rel_bcpf_flags_file],
+ ))
+
+ def check_frameworks_base_boot_hidden_api_files(self, result):
+ hiddenapi_dir = os.path.join(self.top_dir,
+ "frameworks/base/boot/hiddenapi")
+ for basename in sorted(os.listdir(hiddenapi_dir)):
+ if not (basename.startswith("hiddenapi-") and
+ basename.endswith(".txt")):
+ continue
+
+ flags_file = os.path.join(hiddenapi_dir, basename)
+
+ logging.debug("Checking %s for flags related to %s", flags_file,
+ self.bcpf)
+
+ # Map the file name in frameworks/base/boot/hiddenapi into a
+ # slightly more meaningful name for use by the
+ # bootclasspath_fragment.
+ if basename == "hiddenapi-max-target-o.txt":
+ basename = "hiddenapi-max-target-o-low-priority.txt"
+ elif basename == "hiddenapi-max-target-r-loprio.txt":
+ basename = "hiddenapi-max-target-r-low-priority.txt"
+
+ property_name = basename.removeprefix("hiddenapi-")
+ property_name = property_name.removesuffix(".txt")
+ property_name = property_name.replace("-", "_")
+
+ rel_bcpf_flags_file = f"hiddenapi/{basename}"
+ bcpf_dir = self.module_info.module_path(self.bcpf)
+ bcpf_flags_file = os.path.join(self.top_dir, bcpf_dir,
+ rel_bcpf_flags_file)
+
+ self.report_hidden_api_flag_file_changes(result, property_name,
+ flags_file,
+ rel_bcpf_flags_file,
+ bcpf_flags_file)
+
+
+def newline_stripping_iter(iterator):
+ """Return an iterator over the iterator that strips trailing white space."""
+ lines = iter(iterator, "")
+ lines = (line.rstrip() for line in lines)
+ return lines
+
+
+def format_comment_as_text(text, indent):
+ return "".join(
+ [f"{line}\n" for line in format_comment_as_lines(text, indent)])
+
+
+def format_comment_as_lines(text, indent):
+ lines = textwrap.wrap(text.strip("\n"), width=77 - len(indent))
+ lines = [f"{indent}// {line}" for line in lines]
+ return lines
+
+
+def log_stream_for_subprocess():
+ stream = subprocess.DEVNULL
+ for handler in logging.root.handlers:
+ if handler.level == logging.DEBUG:
+ if isinstance(handler, logging.StreamHandler):
+ stream = handler.stream
+ return stream
+
+
+def main(argv):
+ args_parser = argparse.ArgumentParser(
+ description="Analyze a bootclasspath_fragment module.")
+ args_parser.add_argument(
+ "--bcpf",
+ help="The bootclasspath_fragment module to analyze",
+ required=True,
+ )
+ args_parser.add_argument(
+ "--apex",
+ help="The apex module to which the bootclasspath_fragment belongs. It "
+ "is not strictly necessary at the moment but providing it will "
+ "allow this script to give more useful messages and it may be"
+ "required in future.",
+ default="SPECIFY-APEX-OPTION")
+ args_parser.add_argument(
+ "--sdk",
+ help="The sdk module to which the bootclasspath_fragment belongs. It "
+ "is not strictly necessary at the moment but providing it will "
+ "allow this script to give more useful messages and it may be"
+ "required in future.",
+ default="SPECIFY-SDK-OPTION")
+ args = args_parser.parse_args(argv[1:])
+ top_dir = os.environ["ANDROID_BUILD_TOP"] + "/"
+ out_dir = os.environ.get("OUT_DIR", os.path.join(top_dir, "out"))
+ product_out_dir = os.environ.get("ANDROID_PRODUCT_OUT", top_dir)
+ # Make product_out_dir relative to the top so it can be used as part of a
+ # build target.
+ product_out_dir = product_out_dir.removeprefix(top_dir)
+ log_fd, abs_log_file = tempfile.mkstemp(
+ suffix="_analyze_bcpf.log", text=True)
+
+ with os.fdopen(log_fd, "w") as log_file:
+ # Set up debug logging to the log file.
+ logging.basicConfig(
+ level=logging.DEBUG,
+ format="%(levelname)-8s %(message)s",
+ stream=log_file)
+
+ # define a Handler which writes INFO messages or higher to the
+ # sys.stdout with just the message.
+ console = logging.StreamHandler()
+ console.setLevel(logging.INFO)
+ console.setFormatter(logging.Formatter("%(message)s"))
+ # add the handler to the root logger
+ logging.getLogger("").addHandler(console)
+
+ print(f"Writing log to {abs_log_file}")
+ try:
+ analyzer = BcpfAnalyzer(
+ top_dir=top_dir,
+ out_dir=out_dir,
+ product_out_dir=product_out_dir,
+ bcpf=args.bcpf,
+ apex=args.apex,
+ sdk=args.sdk,
+ )
+ analyzer.analyze()
+ finally:
+ print(f"Log written to {abs_log_file}")
+
+
+if __name__ == "__main__":
+ main(sys.argv)
diff --git a/scripts/hiddenapi/analyze_bcpf_test.py b/scripts/hiddenapi/analyze_bcpf_test.py
new file mode 100644
index 0000000..8e34671
--- /dev/null
+++ b/scripts/hiddenapi/analyze_bcpf_test.py
@@ -0,0 +1,331 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for analyzing bootclasspath_fragment modules."""
+import os.path
+import shutil
+import tempfile
+import unittest
+import unittest.mock
+
+import analyze_bcpf as ab
+
+_FRAMEWORK_HIDDENAPI = "frameworks/base/boot/hiddenapi"
+_MAX_TARGET_O = f"{_FRAMEWORK_HIDDENAPI}/hiddenapi-max-target-o.txt"
+_MAX_TARGET_P = f"{_FRAMEWORK_HIDDENAPI}/hiddenapi-max-target-p.txt"
+_MAX_TARGET_Q = f"{_FRAMEWORK_HIDDENAPI}/hiddenapi-max-target-q.txt"
+_MAX_TARGET_R = f"{_FRAMEWORK_HIDDENAPI}/hiddenapi-max-target-r-loprio.txt"
+
+_MULTI_LINE_COMMENT = """
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut arcu justo,
+bibendum eu malesuada vel, fringilla in odio. Etiam gravida ultricies sem
+tincidunt luctus.""".replace("\n", " ").strip()
+
+
+class FakeBuildOperation(ab.BuildOperation):
+
+ def __init__(self, lines, return_code):
+ ab.BuildOperation.__init__(self, None)
+ self._lines = lines
+ self.returncode = return_code
+
+ def lines(self):
+ return iter(self._lines)
+
+ def wait(self, *args, **kwargs):
+ return
+
+
+class TestAnalyzeBcpf(unittest.TestCase):
+
+ def setUp(self):
+ # Create a temporary directory
+ self.test_dir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ # Remove the directory after the test
+ shutil.rmtree(self.test_dir)
+
+ @staticmethod
+ def write_abs_file(abs_path, contents):
+ os.makedirs(os.path.dirname(abs_path), exist_ok=True)
+ with open(abs_path, "w", encoding="utf8") as f:
+ print(contents.removeprefix("\n"), file=f, end="")
+
+ def populate_fs(self, fs):
+ for path, contents in fs.items():
+ abs_path = os.path.join(self.test_dir, path)
+ self.write_abs_file(abs_path, contents)
+
+ def create_analyzer_for_test(self,
+ fs=None,
+ bcpf="bcpf",
+ apex="apex",
+ sdk="sdk"):
+ if fs:
+ self.populate_fs(fs)
+
+ top_dir = self.test_dir
+ out_dir = os.path.join(self.test_dir, "out")
+ product_out_dir = "out/product"
+
+ bcpf_dir = f"{bcpf}-dir"
+ modules = {bcpf: {"path": [bcpf_dir]}}
+ module_info = ab.ModuleInfo(modules)
+
+ analyzer = ab.BcpfAnalyzer(
+ top_dir=top_dir,
+ out_dir=out_dir,
+ product_out_dir=product_out_dir,
+ bcpf=bcpf,
+ apex=apex,
+ sdk=sdk,
+ module_info=module_info,
+ )
+ analyzer.load_all_flags()
+ return analyzer
+
+ def test_reformat_report_text(self):
+ lines = """
+99. An item in a numbered list
+that traverses multiple lines.
+
+ An indented example
+ that should not be reformatted.
+"""
+ reformatted = ab.BcpfAnalyzer.reformat_report_test(lines)
+ self.assertEqual(
+ """
+99. An item in a numbered list that traverses multiple lines.
+
+ An indented example
+ that should not be reformatted.
+""", reformatted)
+
+ def test_build_flags(self):
+ lines = """
+ERROR: Hidden API flags are inconsistent:
+< out/soong/.intermediates/bcpf-dir/bcpf-dir/filtered-flags.csv
+> out/soong/hiddenapi/hiddenapi-flags.csv
+
+< Lacme/test/Class;-><init>()V,blocked
+> Lacme/test/Class;-><init>()V,max-target-o
+
+< Lacme/test/Other;->getThing()Z,blocked
+> Lacme/test/Other;->getThing()Z,max-target-p
+
+< Lacme/test/Widget;-><init()V,blocked
+> Lacme/test/Widget;-><init()V,max-target-q
+
+< Lacme/test/Gadget;->NAME:Ljava/lang/String;,blocked
+> Lacme/test/Gadget;->NAME:Ljava/lang/String;,lo-prio,max-target-r
+16:37:32 ninja failed with: exit status 1
+""".strip().splitlines()
+ operation = FakeBuildOperation(lines=lines, return_code=1)
+
+ fs = {
+ _MAX_TARGET_O:
+ """
+Lacme/items/Magnet;->size:I
+Lacme/test/Class;-><init>()V
+""",
+ _MAX_TARGET_P:
+ """
+Lacme/items/Rocket;->size:I
+Lacme/test/Other;->getThing()Z
+""",
+ _MAX_TARGET_Q:
+ """
+Lacme/items/Rock;->size:I
+Lacme/test/Widget;-><init()V
+""",
+ _MAX_TARGET_R:
+ """
+Lacme/items/Lever;->size:I
+Lacme/test/Gadget;->NAME:Ljava/lang/String;
+""",
+ "bcpf-dir/hiddenapi/hiddenapi-max-target-p.txt":
+ """
+Lacme/old/Class;->getWidget()Lacme/test/Widget;
+""",
+ "out/soong/.intermediates/bcpf-dir/bcpf/all-flags.csv":
+ """
+Lacme/test/Gadget;->NAME:Ljava/lang/String;,blocked
+Lacme/test/Widget;-><init()V,blocked
+Lacme/test/Class;-><init>()V,blocked
+Lacme/test/Other;->getThing()Z,blocked
+""",
+ }
+
+ analyzer = self.create_analyzer_for_test(fs)
+
+ # Override the build_file_read_output() method to just return a fake
+ # build operation.
+ analyzer.build_file_read_output = unittest.mock.Mock(
+ return_value=operation)
+
+ # Override the run_command() method to do nothing.
+ analyzer.run_command = unittest.mock.Mock()
+
+ result = ab.Result()
+
+ analyzer.build_monolithic_flags(result)
+ expected_diffs = {
+ "Lacme/test/Gadget;->NAME:Ljava/lang/String;":
+ (["blocked"], ["lo-prio", "max-target-r"]),
+ "Lacme/test/Widget;-><init()V": (["blocked"], ["max-target-q"]),
+ "Lacme/test/Class;-><init>()V": (["blocked"], ["max-target-o"]),
+ "Lacme/test/Other;->getThing()Z": (["blocked"], ["max-target-p"])
+ }
+ self.assertEqual(expected_diffs, result.diffs, msg="flag differences")
+
+ expected_property_changes = [
+ ab.HiddenApiPropertyChange(
+ property_name="max_target_o_low_priority",
+ values=["hiddenapi/hiddenapi-max-target-o-low-priority.txt"],
+ property_comment=""),
+ ab.HiddenApiPropertyChange(
+ property_name="max_target_p",
+ values=["hiddenapi/hiddenapi-max-target-p.txt"],
+ property_comment=""),
+ ab.HiddenApiPropertyChange(
+ property_name="max_target_q",
+ values=["hiddenapi/hiddenapi-max-target-q.txt"],
+ property_comment=""),
+ ab.HiddenApiPropertyChange(
+ property_name="max_target_r_low_priority",
+ values=["hiddenapi/hiddenapi-max-target-r-low-priority.txt"],
+ property_comment=""),
+ ]
+ self.assertEqual(
+ expected_property_changes,
+ result.property_changes,
+ msg="property changes")
+
+ expected_file_changes = [
+ ab.FileChange(
+ path="bcpf-dir/hiddenapi/"
+ "hiddenapi-max-target-o-low-priority.txt",
+ description="""Add the following entries:
+ Lacme/test/Class;-><init>()V
+""",
+ ),
+ ab.FileChange(
+ path="bcpf-dir/hiddenapi/hiddenapi-max-target-p.txt",
+ description="""Add the following entries:
+ Lacme/test/Other;->getThing()Z
+""",
+ ),
+ ab.FileChange(
+ path="bcpf-dir/hiddenapi/hiddenapi-max-target-q.txt",
+ description="""Add the following entries:
+ Lacme/test/Widget;-><init()V
+"""),
+ ab.FileChange(
+ path="bcpf-dir/hiddenapi/"
+ "hiddenapi-max-target-r-low-priority.txt",
+ description="""Add the following entries:
+ Lacme/test/Gadget;->NAME:Ljava/lang/String;
+"""),
+ ab.FileChange(
+ path="frameworks/base/boot/hiddenapi/"
+ "hiddenapi-max-target-o.txt",
+ description="""Remove the following entries:
+ Lacme/test/Class;-><init>()V
+"""),
+ ab.FileChange(
+ path="frameworks/base/boot/hiddenapi/"
+ "hiddenapi-max-target-p.txt",
+ description="""Remove the following entries:
+ Lacme/test/Other;->getThing()Z
+"""),
+ ab.FileChange(
+ path="frameworks/base/boot/hiddenapi/"
+ "hiddenapi-max-target-q.txt",
+ description="""Remove the following entries:
+ Lacme/test/Widget;-><init()V
+"""),
+ ab.FileChange(
+ path="frameworks/base/boot/hiddenapi/"
+ "hiddenapi-max-target-r-loprio.txt",
+ description="""Remove the following entries:
+ Lacme/test/Gadget;->NAME:Ljava/lang/String;
+""")
+ ]
+ result.file_changes.sort()
+ self.assertEqual(
+ expected_file_changes, result.file_changes, msg="file_changes")
+
+
+class TestHiddenApiPropertyChange(unittest.TestCase):
+
+ def setUp(self):
+ # Create a temporary directory
+ self.test_dir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ # Remove the directory after the test
+ shutil.rmtree(self.test_dir)
+
+ def check_change_snippet(self, change, expected):
+ snippet = change.snippet(" ")
+ self.assertEqual(expected, snippet)
+
+ def test_change_property_with_value_no_comment(self):
+ change = ab.HiddenApiPropertyChange(
+ property_name="split_packages",
+ values=["android.provider"],
+ )
+
+ self.check_change_snippet(
+ change, """
+ split_packages: [
+ "android.provider",
+ ],
+""")
+
+ def test_change_property_with_value_and_comment(self):
+ change = ab.HiddenApiPropertyChange(
+ property_name="split_packages",
+ values=["android.provider"],
+ property_comment=_MULTI_LINE_COMMENT,
+ )
+
+ self.check_change_snippet(
+ change, """
+ // Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut arcu
+ // justo, bibendum eu malesuada vel, fringilla in odio. Etiam gravida
+ // ultricies sem tincidunt luctus.
+ split_packages: [
+ "android.provider",
+ ],
+""")
+
+ def test_change_without_value_or_empty_comment(self):
+ change = ab.HiddenApiPropertyChange(
+ property_name="split_packages",
+ values=[],
+ property_comment="Single line comment.",
+ )
+
+ self.check_change_snippet(
+ change, """
+ // Single line comment.
+ split_packages: [],
+""")
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=3)