Merge "Add a new Soong config variable EMMA_INSTRUMENT" into main
diff --git a/Changes.md b/Changes.md
index fc15e60..9f2449c 100644
--- a/Changes.md
+++ b/Changes.md
@@ -43,14 +43,9 @@
The path set when running builds now makes the `python` executable point to python 3,
whereas on previous versions it pointed to python 2. If you still have python 2 scripts,
you can change the shebang line to use `python2` explicitly. This only applies for
-scripts run directly from makefiles, or from soong genrules. This behavior can be
-temporarily overridden by setting the `BUILD_BROKEN_PYTHON_IS_PYTHON2` environment
-variable to `true`. It's only an environment variable and not a product config variable
-because product config sometimes calls python code.
+scripts run directly from makefiles, or from soong genrules.
-In addition, `python_*` soong modules no longer allow python 2. This can be temporarily
-overridden by setting the `BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES` product configuration
-variable to `true`.
+In addition, `python_*` soong modules no longer allow python 2.
Python 2 is slated for complete removal in V.
diff --git a/ci/Android.bp b/ci/Android.bp
index 22c4851..6d4ac35 100644
--- a/ci/Android.bp
+++ b/ci/Android.bp
@@ -71,11 +71,36 @@
},
}
+python_test_host {
+ name: "optimized_targets_test",
+ main: "optimized_targets_test.py",
+ pkg_path: "testdata",
+ srcs: [
+ "optimized_targets_test.py",
+ ],
+ libs: [
+ "build_test_suites",
+ "pyfakefs",
+ ],
+ test_options: {
+ unit_test: true,
+ },
+ data: [
+ ":py3-cmd",
+ ],
+ version: {
+ py3: {
+ embedded_launcher: true,
+ },
+ },
+}
+
python_library_host {
name: "build_test_suites",
srcs: [
"build_test_suites.py",
"optimized_targets.py",
+ "test_mapping_module_retriever.py",
"build_context.py",
],
}
diff --git a/ci/build_metadata b/ci/build_metadata
new file mode 100755
index 0000000..8136702
--- /dev/null
+++ b/ci/build_metadata
@@ -0,0 +1,25 @@
+#/bin/bash
+
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+export TARGET_PRODUCT=aosp_arm64
+export TARGET_RELEASE=trunk_staging
+export TARGET_BUILD_VARIANT=eng
+
+build/soong/bin/m dist \
+ code_metadata
+
diff --git a/ci/build_test_suites.py b/ci/build_test_suites.py
index 402880c..933e43e 100644
--- a/ci/build_test_suites.py
+++ b/ci/build_test_suites.py
@@ -20,10 +20,8 @@
import logging
import os
import pathlib
-import re
import subprocess
import sys
-from typing import Callable
from build_context import BuildContext
import optimized_targets
@@ -70,7 +68,7 @@
return BuildPlan(set(self.args.extra_targets), set())
build_targets = set()
- packaging_functions = set()
+ packaging_commands = []
for target in self.args.extra_targets:
if self._unused_target_exclusion_enabled(
target
@@ -86,9 +84,9 @@
target, self.build_context, self.args
)
build_targets.update(target_optimizer.get_build_targets())
- packaging_functions.add(target_optimizer.package_outputs)
+ packaging_commands.extend(target_optimizer.get_package_outputs_commands())
- return BuildPlan(build_targets, packaging_functions)
+ return BuildPlan(build_targets, packaging_commands)
def _unused_target_exclusion_enabled(self, target: str) -> bool:
return (
@@ -100,7 +98,7 @@
@dataclass(frozen=True)
class BuildPlan:
build_targets: set[str]
- packaging_functions: set[Callable[..., None]]
+ packaging_commands: list[list[str]]
def build_test_suites(argv: list[str]) -> int:
@@ -182,8 +180,11 @@
except subprocess.CalledProcessError as e:
raise BuildFailureError(e.returncode) from e
- for packaging_function in build_plan.packaging_functions:
- packaging_function()
+ for packaging_command in build_plan.packaging_commands:
+ try:
+ run_command(packaging_command)
+ except subprocess.CalledProcessError as e:
+ raise BuildFailureError(e.returncode) from e
def get_top() -> pathlib.Path:
diff --git a/ci/build_test_suites_test.py b/ci/build_test_suites_test.py
index f3ff6f4..fd06a3a 100644
--- a/ci/build_test_suites_test.py
+++ b/ci/build_test_suites_test.py
@@ -241,17 +241,17 @@
class TestOptimizedBuildTarget(optimized_targets.OptimizedBuildTarget):
def __init__(
- self, target, build_context, args, output_targets, packaging_outputs
+ self, target, build_context, args, output_targets, packaging_commands
):
super().__init__(target, build_context, args)
self.output_targets = output_targets
- self.packaging_outputs = packaging_outputs
+ self.packaging_commands = packaging_commands
def get_build_targets_impl(self):
return self.output_targets
- def package_outputs_impl(self):
- self.packaging_outputs.add(f'packaging {" ".join(self.output_targets)}')
+ def get_package_outputs_commands_impl(self):
+ return self.packaging_commands
def get_enabled_flag(self):
return f'{self.target}_enabled'
@@ -276,7 +276,7 @@
build_plan = build_planner.create_build_plan()
- self.assertEqual(len(build_plan.packaging_functions), 0)
+ self.assertEqual(len(build_plan.packaging_commands), 0)
def test_build_optimization_on_optimizes_target(self):
build_targets = {'target_1', 'target_2'}
@@ -294,20 +294,19 @@
def test_build_optimization_on_packages_target(self):
build_targets = {'target_1', 'target_2'}
- packaging_outputs = set()
+ optimized_target_name = self.get_optimized_target_name('target_1')
+ packaging_commands = [[f'packaging {optimized_target_name}']]
build_planner = self.create_build_planner(
build_targets=build_targets,
build_context=self.create_build_context(
enabled_build_features=[{'name': self.get_target_flag('target_1')}]
),
- packaging_outputs=packaging_outputs,
+ packaging_commands=packaging_commands,
)
build_plan = build_planner.create_build_plan()
- self.run_packaging_functions(build_plan)
- optimized_target_name = self.get_optimized_target_name('target_1')
- self.assertIn(f'packaging {optimized_target_name}', packaging_outputs)
+ self.assertIn([f'packaging {optimized_target_name}'], build_plan.packaging_commands)
def test_individual_build_optimization_off_doesnt_optimize(self):
build_targets = {'target_1', 'target_2'}
@@ -321,16 +320,15 @@
def test_individual_build_optimization_off_doesnt_package(self):
build_targets = {'target_1', 'target_2'}
- packaging_outputs = set()
+ packaging_commands = [['packaging command']]
build_planner = self.create_build_planner(
build_targets=build_targets,
- packaging_outputs=packaging_outputs,
+ packaging_commands=packaging_commands,
)
build_plan = build_planner.create_build_plan()
- self.run_packaging_functions(build_plan)
- self.assertFalse(packaging_outputs)
+ self.assertFalse(build_plan.packaging_commands)
def test_target_output_used_target_built(self):
build_target = 'test_target'
@@ -408,7 +406,7 @@
target_optimizations: dict[
str, optimized_targets.OptimizedBuildTarget
] = None,
- packaging_outputs: set[str] = set(),
+ packaging_commands: list[list[str]] = [],
) -> build_test_suites.BuildPlanner:
if not build_context:
build_context = self.create_build_context()
@@ -418,7 +416,7 @@
target_optimizations = self.create_target_optimizations(
build_context,
build_targets,
- packaging_outputs,
+ packaging_commands,
)
return build_test_suites.BuildPlanner(
build_context, args, target_optimizations
@@ -450,14 +448,14 @@
self,
build_context: BuildContext,
build_targets: set[str],
- packaging_outputs: set[str] = set(),
+ packaging_commands: list[list[str]] = [],
):
target_optimizations = dict()
for target in build_targets:
target_optimizations[target] = functools.partial(
self.TestOptimizedBuildTarget,
output_targets={self.get_optimized_target_name(target)},
- packaging_outputs=packaging_outputs,
+ packaging_commands=packaging_commands,
)
return target_optimizations
@@ -468,10 +466,6 @@
def get_optimized_target_name(self, target: str):
return f'{target}_optimized'
- def run_packaging_functions(self, build_plan: build_test_suites.BuildPlan):
- for packaging_function in build_plan.packaging_functions:
- packaging_function()
-
def get_test_context(self, target: str):
return {
'testInfos': [
diff --git a/ci/buildbot.py b/ci/buildbot.py
new file mode 100644
index 0000000..97097be
--- /dev/null
+++ b/ci/buildbot.py
@@ -0,0 +1,43 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for interacting with buildbot, with a simulation in a local environment"""
+
+import os
+import sys
+
+# Check that the script is running from the root of the tree. Prevents subtle
+# errors later, and CI always runs from the root of the tree.
+if not os.path.exists("build/make/ci/buildbot.py"):
+ raise Exception("CI script must be run from the root of the tree instead of: "
+ + os.getcwd())
+
+# Check that we are using the hermetic interpreter
+if "prebuilts/build-tools/" not in sys.executable:
+ raise Exception("CI script must be run using the hermetic interpreter from "
+ + "prebuilts/build-tools instead of: " + sys.executable)
+
+
+def OutDir():
+ "Get the out directory. Will create it if needed."
+ result = os.environ.get("OUT_DIR", "out")
+ os.makedirs(result, exist_ok=True)
+ return result
+
+def DistDir():
+ "Get the dist directory. Will create it if needed."
+ result = os.environ.get("DIST_DIR", os.path.join(OutDir(), "dist"))
+ os.makedirs(result, exist_ok=True)
+ return result
+
diff --git a/ci/dump_product_config b/ci/dump_product_config
new file mode 100755
index 0000000..77b51dd
--- /dev/null
+++ b/ci/dump_product_config
@@ -0,0 +1,353 @@
+#!prebuilts/build-tools/linux-x86/bin/py3-cmd -B
+
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script to collect all of the make variables from all product config combos.
+
+This script must be run from the root of the source tree.
+
+See GetArgs() below or run dump_product_config for more information.
+"""
+
+import argparse
+import asyncio
+import contextlib
+import csv
+import dataclasses
+import json
+import multiprocessing
+import os
+import subprocess
+import sys
+import time
+from typing import List, Dict, Tuple, Optional
+
+import buildbot
+
+# We have some BIG variables
+csv.field_size_limit(sys.maxsize)
+
+
+class DataclassJSONEncoder(json.JSONEncoder):
+ """JSONEncoder for our custom types."""
+ def default(self, o):
+ if dataclasses.is_dataclass(o):
+ return dataclasses.asdict(o)
+ return super().default(o)
+
+
+def GetProducts():
+ """Get the all of the available TARGET_PRODUCT values."""
+ try:
+ stdout = subprocess.check_output(["build/soong/bin/list_products"], text=True)
+ except subprocess.CalledProcessError:
+ sys.exit(1)
+ return [s.strip() for s in stdout.splitlines() if s.strip()]
+
+
+def GetReleases(product):
+ """For a given product, get the release configs available to it."""
+ if True:
+ # Hard code the list
+ mainline_products = [
+ "module_arm",
+ "module_x86",
+ "module_arm64",
+ "module_riscv64",
+ "module_x86_64",
+ "module_arm64only",
+ "module_x86_64only",
+ ]
+ if product in mainline_products:
+ return ["trunk_staging", "trunk", "mainline"]
+ else:
+ return ["trunk_staging", "trunk", "next"]
+ else:
+ # Get it from the build system
+ try:
+ stdout = subprocess.check_output(["build/soong/bin/list_releases", product], text=True)
+ except subprocess.CalledProcessError:
+ sys.exit(1)
+ return [s.strip() for s in stdout.splitlines() if s.strip()]
+
+
+def GenerateAllLunchTargets():
+ """Generate the full list of lunch targets."""
+ for product in GetProducts():
+ for release in GetReleases(product):
+ for variant in ["user", "userdebug", "eng"]:
+ yield (product, release, variant)
+
+
+async def ParallelExec(parallelism, tasks):
+ '''
+ ParallelExec takes a parallelism number, and an iterator of tasks to run.
+ Then it will run all the tasks, but a maximum of parallelism will be run at
+ any given time. The tasks must be async functions that accept one argument,
+ which will be an integer id of the worker that they're running on.
+ '''
+ tasks = iter(tasks)
+
+ overall_start = time.monotonic()
+ # lists so they can be modified from the inner function
+ total_duration = [0]
+ count = [0]
+ async def dispatch(worker):
+ while True:
+ try:
+ task = next(tasks)
+ item_start = time.monotonic()
+ await task(worker)
+ now = time.monotonic()
+ item_duration = now - item_start
+ count[0] += 1
+ total_duration[0] += item_duration
+ sys.stderr.write(f"Timing: Items processed: {count[0]}, Wall time: {now-overall_start:0.1f} sec, Throughput: {(now-overall_start)/count[0]:0.3f} sec per item, Average duration: {total_duration[0]/count[0]:0.1f} sec\n")
+ except StopIteration:
+ return
+
+ await asyncio.gather(*[dispatch(worker) for worker in range(parallelism)])
+
+
+async def DumpProductConfigs(out, generator, out_dir):
+ """Collects all of the product config data and store it in file."""
+ # Write the outer json list by hand so we can stream it
+ out.write("[")
+ try:
+ first_result = [True] # a list so it can be modified from the inner function
+ def run(lunch):
+ async def curried(worker):
+ sys.stderr.write(f"running: {'-'.join(lunch)}\n")
+ result = await DumpOneProductConfig(lunch, os.path.join(out_dir, f"lunchable_{worker}"))
+ if first_result[0]:
+ out.write("\n")
+ first_result[0] = False
+ else:
+ out.write(",\n")
+ result.dumpToFile(out)
+ sys.stderr.write(f"finished: {'-'.join(lunch)}\n")
+ return curried
+
+ await ParallelExec(multiprocessing.cpu_count(), (run(lunch) for lunch in generator))
+ finally:
+ # Close the json regardless of how we exit
+ out.write("\n]\n")
+
+
+@dataclasses.dataclass(frozen=True)
+class Variable:
+ """A variable name, value and where it was set."""
+ name: str
+ value: str
+ location: str
+
+
+@dataclasses.dataclass(frozen=True)
+class ProductResult:
+ product: str
+ release: str
+ variant: str
+ board_includes: List[str]
+ product_includes: Dict[str, List[str]]
+ product_graph: List[Tuple[str, str]]
+ board_vars: List[Variable]
+ product_vars: List[Variable]
+
+ def dumpToFile(self, f):
+ json.dump(self, f, sort_keys=True, indent=2, cls=DataclassJSONEncoder)
+
+
+@dataclasses.dataclass(frozen=True)
+class ProductError:
+ product: str
+ release: str
+ variant: str
+ error: str
+
+ def dumpToFile(self, f):
+ json.dump(self, f, sort_keys=True, indent=2, cls=DataclassJSONEncoder)
+
+
+def NormalizeInheritGraph(lists):
+ """Flatten the inheritance graph to a simple list for easier querying."""
+ result = set()
+ for item in lists:
+ for i in range(len(item)):
+ result.add((item[i+1] if i < len(item)-1 else "", item[i]))
+ return sorted(list(result))
+
+
+def ParseDump(lunch, filename) -> ProductResult:
+ """Parses the csv and returns a tuple of the data."""
+ def diff(initial, final):
+ return [after for after in final.values() if
+ initial.get(after.name, Variable(after.name, "", "<unset>")).value != after.value]
+ product_initial = {}
+ product_final = {}
+ board_initial = {}
+ board_final = {}
+ inherit_product = [] # The stack of inherit-product calls
+ product_includes = {} # Other files included by each of the properly imported files
+ board_includes = [] # Files included by boardconfig
+ with open(filename) as f:
+ phase = ""
+ for line in csv.reader(f):
+ if line[0] == "phase":
+ phase = line[1]
+ elif line[0] == "val":
+ # TOOD: We should skip these somewhere else.
+ if line[3].startswith("_ALL_RELEASE_FLAGS"):
+ continue
+ if line[3].startswith("PRODUCTS."):
+ continue
+ if phase == "PRODUCTS":
+ if line[2] == "initial":
+ product_initial[line[3]] = Variable(line[3], line[4], line[5])
+ if phase == "PRODUCT-EXPAND":
+ if line[2] == "final":
+ product_final[line[3]] = Variable(line[3], line[4], line[5])
+ if phase == "BOARD":
+ if line[2] == "initial":
+ board_initial[line[3]] = Variable(line[3], line[4], line[5])
+ if line[2] == "final":
+ board_final[line[3]] = Variable(line[3], line[4], line[5])
+ elif line[0] == "imported":
+ imports = [s.strip() for s in line[1].split()]
+ if imports:
+ inherit_product.append(imports)
+ inc = [s.strip() for s in line[2].split()]
+ for f in inc:
+ product_includes.setdefault(imports[0], []).append(f)
+ elif line[0] == "board_config_files":
+ board_includes += [s.strip() for s in line[1].split()]
+ return ProductResult(
+ product = lunch[0],
+ release = lunch[1],
+ variant = lunch[2],
+ product_vars = diff(product_initial, product_final),
+ board_vars = diff(board_initial, board_final),
+ product_graph = NormalizeInheritGraph(inherit_product),
+ product_includes = product_includes,
+ board_includes = board_includes
+ )
+
+
+async def DumpOneProductConfig(lunch, out_dir) -> ProductResult | ProductError:
+ """Print a single config's lunch info to stdout."""
+ product, release, variant = lunch
+
+ dumpconfig_file = os.path.join(out_dir, f"{product}-{release}-{variant}.csv")
+
+ # Run get_build_var to bootstrap soong_ui for this target
+ env = dict(os.environ)
+ env["TARGET_PRODUCT"] = product
+ env["TARGET_RELEASE"] = release
+ env["TARGET_BUILD_VARIANT"] = variant
+ env["OUT_DIR"] = out_dir
+ process = await asyncio.create_subprocess_exec(
+ "build/soong/bin/get_build_var",
+ "TARGET_PRODUCT",
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ env=env
+ )
+ stdout, _ = await process.communicate()
+ stdout = stdout.decode()
+
+ if process.returncode != 0:
+ return ProductError(
+ product = product,
+ release = release,
+ variant = variant,
+ error = stdout
+ )
+ else:
+ # Run kati to extract the data
+ process = await asyncio.create_subprocess_exec(
+ "prebuilts/build-tools/linux-x86/bin/ckati",
+ "-f",
+ "build/make/core/dumpconfig.mk",
+ f"TARGET_PRODUCT={product}",
+ f"TARGET_RELEASE={release}",
+ f"TARGET_BUILD_VARIANT={variant}",
+ f"DUMPCONFIG_FILE={dumpconfig_file}",
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ env=env
+ )
+ stdout, _ = await process.communicate()
+ if process.returncode != 0:
+ stdout = stdout.decode()
+ return ProductError(
+ product = product,
+ release = release,
+ variant = variant,
+ error = stdout
+ )
+ else:
+ # Parse and record the output
+ return ParseDump(lunch, dumpconfig_file)
+
+
+def GetArgs():
+ """Parse command line arguments."""
+ parser = argparse.ArgumentParser(
+ description="Collect all of the make variables from product config.",
+ epilog="NOTE: This script must be run from the root of the source tree.")
+ parser.add_argument("--lunch", nargs="*")
+ parser.add_argument("--dist", action="store_true")
+
+ return parser.parse_args()
+
+
+async def main():
+ args = GetArgs()
+
+ out_dir = buildbot.OutDir()
+
+ if args.dist:
+ cm = open(os.path.join(buildbot.DistDir(), "all_product_config.json"), "w")
+ else:
+ cm = contextlib.nullcontext(sys.stdout)
+
+
+ with cm as out:
+ if args.lunch:
+ lunches = [lunch.split("-") for lunch in args.lunch]
+ fail = False
+ for i in range(len(lunches)):
+ if len(lunches[i]) != 3:
+ sys.stderr.write(f"Malformed lunch targets: {args.lunch[i]}\n")
+ fail = True
+ if fail:
+ sys.exit(1)
+ if len(lunches) == 1:
+ result = await DumpOneProductConfig(lunches[0], out_dir)
+ result.dumpToFile(out)
+ out.write("\n")
+ else:
+ await DumpProductConfigs(out, lunches, out_dir)
+ else:
+ # All configs mode. This will exec single config mode in parallel
+ # for each lunch combo. Write output to $DIST_DIR.
+ await DumpProductConfigs(out, GenerateAllLunchTargets(), out_dir)
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
+
+
+# vim: set syntax=python ts=4 sw=4 sts=4:
+
diff --git a/ci/optimized_targets.py b/ci/optimized_targets.py
index 116d6f8..9143cbf 100644
--- a/ci/optimized_targets.py
+++ b/ci/optimized_targets.py
@@ -16,8 +16,13 @@
from abc import ABC
import argparse
import functools
-from typing import Self
from build_context import BuildContext
+import json
+import logging
+import os
+from typing import Self
+
+import test_mapping_module_retriever
class OptimizedBuildTarget(ABC):
@@ -41,17 +46,23 @@
def get_build_targets(self) -> set[str]:
features = self.build_context.enabled_build_features
if self.get_enabled_flag() in features:
- return self.get_build_targets_impl()
+ self.modules_to_build = self.get_build_targets_impl()
+ return self.modules_to_build
+
+ self.modules_to_build = {self.target}
return {self.target}
- def package_outputs(self):
+ def get_package_outputs_commands(self) -> list[list[str]]:
features = self.build_context.enabled_build_features
if self.get_enabled_flag() in features:
- return self.package_outputs_impl()
+ return self.get_package_outputs_commands_impl()
- def package_outputs_impl(self):
+ return []
+
+ def get_package_outputs_commands_impl(self) -> list[list[str]]:
raise NotImplementedError(
- f'package_outputs_impl not implemented in {type(self).__name__}'
+ 'get_package_outputs_commands_impl not implemented in'
+ f' {type(self).__name__}'
)
def get_enabled_flag(self):
@@ -78,8 +89,33 @@
def get_build_targets(self):
return {self.target}
- def package_outputs(self):
- pass
+ def get_package_outputs_commands(self):
+ return []
+
+
+class ChangeInfo:
+
+ def __init__(self, change_info_file_path):
+ try:
+ with open(change_info_file_path) as change_info_file:
+ change_info_contents = json.load(change_info_file)
+ except json.decoder.JSONDecodeError:
+ logging.error(f'Failed to load CHANGE_INFO: {change_info_file_path}')
+ raise
+
+ self._change_info_contents = change_info_contents
+
+ def find_changed_files(self) -> set[str]:
+ changed_files = set()
+
+ for change in self._change_info_contents['changes']:
+ project_path = change.get('projectPath') + '/'
+
+ for revision in change.get('revisions'):
+ for file_info in revision.get('fileInfos'):
+ changed_files.add(project_path + file_info.get('path'))
+
+ return changed_files
class GeneralTestsOptimizer(OptimizedBuildTarget):
@@ -94,8 +130,55 @@
normally built.
"""
+ # List of modules that are always required to be in general-tests.zip.
+ _REQUIRED_MODULES = frozenset(
+ ['cts-tradefed', 'vts-tradefed', 'compatibility-host-util']
+ )
+
+ def get_build_targets_impl(self) -> set[str]:
+ change_info_file_path = os.environ.get('CHANGE_INFO')
+ if not change_info_file_path:
+ logging.info(
+ 'No CHANGE_INFO env var found, general-tests optimization disabled.'
+ )
+ return {'general-tests'}
+
+ test_infos = self.build_context.test_infos
+ test_mapping_test_groups = set()
+ for test_info in test_infos:
+ is_test_mapping = test_info.is_test_mapping
+ current_test_mapping_test_groups = test_info.test_mapping_test_groups
+ uses_general_tests = test_info.build_target_used('general-tests')
+
+ if uses_general_tests and not is_test_mapping:
+ logging.info(
+ 'Test uses general-tests.zip but is not test-mapping, general-tests'
+ ' optimization disabled.'
+ )
+ return {'general-tests'}
+
+ if is_test_mapping:
+ test_mapping_test_groups.update(current_test_mapping_test_groups)
+
+ change_info = ChangeInfo(change_info_file_path)
+ changed_files = change_info.find_changed_files()
+
+ test_mappings = test_mapping_module_retriever.GetTestMappings(
+ changed_files, set()
+ )
+
+ modules_to_build = set(self._REQUIRED_MODULES)
+
+ modules_to_build.update(
+ test_mapping_module_retriever.FindAffectedModules(
+ test_mappings, changed_files, test_mapping_test_groups
+ )
+ )
+
+ return modules_to_build
+
def get_enabled_flag(self):
- return 'general-tests-optimized'
+ return 'general_tests_optimized'
@classmethod
def get_optimized_targets(cls) -> dict[str, OptimizedBuildTarget]:
diff --git a/ci/optimized_targets_test.py b/ci/optimized_targets_test.py
new file mode 100644
index 0000000..919c193
--- /dev/null
+++ b/ci/optimized_targets_test.py
@@ -0,0 +1,206 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for optimized_targets.py"""
+
+import json
+import logging
+import os
+import pathlib
+import re
+import unittest
+from unittest import mock
+import optimized_targets
+from build_context import BuildContext
+from pyfakefs import fake_filesystem_unittest
+
+
+class GeneralTestsOptimizerTest(fake_filesystem_unittest.TestCase):
+
+ def setUp(self):
+ self.setUpPyfakefs()
+
+ os_environ_patcher = mock.patch.dict('os.environ', {})
+ self.addCleanup(os_environ_patcher.stop)
+ self.mock_os_environ = os_environ_patcher.start()
+
+ self._setup_working_build_env()
+ self._write_change_info_file()
+ test_mapping_dir = pathlib.Path('/project/path/file/path')
+ test_mapping_dir.mkdir(parents=True)
+ self._write_test_mapping_file()
+
+ def _setup_working_build_env(self):
+ self.change_info_file = pathlib.Path('/tmp/change_info')
+
+ self.mock_os_environ.update({
+ 'CHANGE_INFO': str(self.change_info_file),
+ })
+
+ def test_general_tests_optimized(self):
+ optimizer = self._create_general_tests_optimizer()
+
+ build_targets = optimizer.get_build_targets()
+
+ expected_build_targets = set(
+ optimized_targets.GeneralTestsOptimizer._REQUIRED_MODULES
+ )
+ expected_build_targets.add('test_mapping_module')
+
+ self.assertSetEqual(build_targets, expected_build_targets)
+
+ def test_no_change_info_no_optimization(self):
+ del os.environ['CHANGE_INFO']
+
+ optimizer = self._create_general_tests_optimizer()
+
+ build_targets = optimizer.get_build_targets()
+
+ self.assertSetEqual(build_targets, {'general-tests'})
+
+ def test_mapping_groups_unused_module_not_built(self):
+ test_context = self._create_test_context()
+ test_context['testInfos'][0]['extraOptions'] = [
+ {
+ 'key': 'additional-files-filter',
+ 'values': ['general-tests.zip'],
+ },
+ {
+ 'key': 'test-mapping-test-group',
+ 'values': ['unused-test-mapping-group'],
+ },
+ ]
+ optimizer = self._create_general_tests_optimizer(
+ build_context=self._create_build_context(test_context=test_context)
+ )
+
+ build_targets = optimizer.get_build_targets()
+
+ expected_build_targets = set(
+ optimized_targets.GeneralTestsOptimizer._REQUIRED_MODULES
+ )
+ self.assertSetEqual(build_targets, expected_build_targets)
+
+ def test_general_tests_used_by_non_test_mapping_test_no_optimization(self):
+ test_context = self._create_test_context()
+ test_context['testInfos'][0]['extraOptions'] = [{
+ 'key': 'additional-files-filter',
+ 'values': ['general-tests.zip'],
+ }]
+ optimizer = self._create_general_tests_optimizer(
+ build_context=self._create_build_context(test_context=test_context)
+ )
+
+ build_targets = optimizer.get_build_targets()
+
+ self.assertSetEqual(build_targets, {'general-tests'})
+
+ def test_malformed_change_info_raises(self):
+ with open(self.change_info_file, 'w') as f:
+ f.write('not change info')
+
+ optimizer = self._create_general_tests_optimizer()
+
+ with self.assertRaises(json.decoder.JSONDecodeError):
+ build_targets = optimizer.get_build_targets()
+
+ def test_malformed_test_mapping_raises(self):
+ with open('/project/path/file/path/TEST_MAPPING', 'w') as f:
+ f.write('not test mapping')
+
+ optimizer = self._create_general_tests_optimizer()
+
+ with self.assertRaises(json.decoder.JSONDecodeError):
+ build_targets = optimizer.get_build_targets()
+
+ def _write_change_info_file(self):
+ change_info_contents = {
+ 'changes': [{
+ 'projectPath': '/project/path',
+ 'revisions': [{
+ 'fileInfos': [{
+ 'path': 'file/path/file_name',
+ }],
+ }],
+ }]
+ }
+
+ with open(self.change_info_file, 'w') as f:
+ json.dump(change_info_contents, f)
+
+ def _write_test_mapping_file(self):
+ test_mapping_contents = {
+ 'test-mapping-group': [
+ {
+ 'name': 'test_mapping_module',
+ },
+ ],
+ }
+
+ with open('/project/path/file/path/TEST_MAPPING', 'w') as f:
+ json.dump(test_mapping_contents, f)
+
+ def _create_general_tests_optimizer(
+ self, build_context: BuildContext = None
+ ):
+ if not build_context:
+ build_context = self._create_build_context()
+ return optimized_targets.GeneralTestsOptimizer(
+ 'general-tests', build_context, None
+ )
+
+ def _create_build_context(
+ self,
+ general_tests_optimized: bool = True,
+ test_context: dict[str, any] = None,
+ ) -> BuildContext:
+ if not test_context:
+ test_context = self._create_test_context()
+ build_context_dict = {}
+ build_context_dict['enabledBuildFeatures'] = [{'name': 'optimized_build'}]
+ if general_tests_optimized:
+ build_context_dict['enabledBuildFeatures'].append({'name': 'general_tests_optimized'})
+ build_context_dict['testContext'] = test_context
+ return BuildContext(build_context_dict)
+
+ def _create_test_context(self):
+ return {
+ 'testInfos': [
+ {
+ 'name': 'atp_test',
+ 'target': 'test_target',
+ 'branch': 'branch',
+ 'extraOptions': [
+ {
+ 'key': 'additional-files-filter',
+ 'values': ['general-tests.zip'],
+ },
+ {
+ 'key': 'test-mapping-test-group',
+ 'values': ['test-mapping-group'],
+ },
+ ],
+ 'command': '/tf/command',
+ 'extraBuildTargets': [
+ 'extra_build_target',
+ ],
+ },
+ ],
+ }
+
+
+if __name__ == '__main__':
+ # Setup logging to be silent so unit tests can pass through TF.
+ logging.disable(logging.ERROR)
+ unittest.main()
diff --git a/ci/test_mapping_module_retriever.py b/ci/test_mapping_module_retriever.py
index d2c13c0..c93cdd5 100644
--- a/ci/test_mapping_module_retriever.py
+++ b/ci/test_mapping_module_retriever.py
@@ -17,11 +17,13 @@
modules are needed to build for the given list of changed files.
TODO(lucafarsi): Deduplicate from artifact_helper.py
"""
+# TODO(lucafarsi): Share this logic with the original logic in
+# test_mapping_test_retriever.py
-from typing import Any, Dict, Set, Text
import json
import os
import re
+from typing import Any
# Regex to extra test name from the path of test config file.
TEST_NAME_REGEX = r'(?:^|.*/)([^/]+)\.config'
@@ -39,7 +41,7 @@
_COMMENTS_RE = re.compile(r'(\"(?:[^\"\\]|\\.)*\"|(?=//))(?://.*)?')
-def FilterComments(test_mapping_file: Text) -> Text:
+def FilterComments(test_mapping_file: str) -> str:
"""Remove comments in TEST_MAPPING file to valid format.
Only '//' is regarded as comments.
@@ -52,8 +54,8 @@
"""
return re.sub(_COMMENTS_RE, r'\1', test_mapping_file)
-def GetTestMappings(paths: Set[Text],
- checked_paths: Set[Text]) -> Dict[Text, Dict[Text, Any]]:
+def GetTestMappings(paths: set[str],
+ checked_paths: set[str]) -> dict[str, dict[str, Any]]:
"""Get the affected TEST_MAPPING files.
TEST_MAPPING files in source code are packaged into a build artifact
@@ -123,3 +125,68 @@
pass
return test_mappings
+
+
+def FindAffectedModules(
+ test_mappings: dict[str, Any],
+ changed_files: set[str],
+ test_mapping_test_groups: set[str],
+) -> set[str]:
+ """Find affected test modules.
+
+ Find the affected set of test modules that would run in a test mapping run based on the given test mappings, changed files, and test mapping test group.
+
+ Args:
+ test_mappings: A set of test mappings returned by GetTestMappings in the following format:
+ {
+ 'test_mapping_file_path': {
+ 'group_name' : [
+ 'name': 'module_name',
+ ],
+ }
+ }
+ changed_files: A set of files changed for the given run.
+ test_mapping_test_groups: A set of test mapping test groups that are being considered for the given run.
+
+ Returns:
+ A set of test module names which would run for a test mapping test run with the given parameters.
+ """
+
+ modules = set()
+
+ for test_mapping in test_mappings.values():
+ for group_name, group in test_mapping.items():
+ # If a module is not in any of the test mapping groups being tested skip
+ # it.
+ if group_name not in test_mapping_test_groups:
+ continue
+
+ for entry in group:
+ module_name = entry.get('name')
+
+ if not module_name:
+ continue
+
+ file_patterns = entry.get('file_patterns')
+ if not file_patterns:
+ modules.add(module_name)
+ continue
+
+ if matches_file_patterns(file_patterns, changed_files):
+ modules.add(module_name)
+
+ return modules
+
+def MatchesFilePatterns(
+ file_patterns: list[set], changed_files: set[str]
+) -> bool:
+ """Checks if any of the changed files match any of the file patterns.
+
+ Args:
+ file_patterns: A list of file patterns to match against.
+ changed_files: A set of files to check against the file patterns.
+
+ Returns:
+ True if any of the changed files match any of the file patterns.
+ """
+ return any(re.search(pattern, "|".join(changed_files)) for pattern in file_patterns)
diff --git a/cogsetup.sh b/cogsetup.sh
index ef1485d..5c64a06 100644
--- a/cogsetup.sh
+++ b/cogsetup.sh
@@ -57,7 +57,7 @@
fi
function repo {
if [[ "${PWD}" == /google/cog/* ]]; then
- echo "\e[01;31mERROR:\e[0mrepo command is disallowed within Cog workspaces."
+ echo -e "\e[01;31mERROR:\e[0mrepo command is disallowed within Cog workspaces."
return 1
fi
${ORIG_REPO_PATH} "$@"
diff --git a/core/Makefile b/core/Makefile
index e57c1d3..f376014 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -6462,8 +6462,11 @@
$(INSTALLED_RAMDISK_TARGET) \
$(INSTALLED_DTBIMAGE_TARGET) \
$(INSTALLED_2NDBOOTLOADER_TARGET) \
+ $(INSTALLED_VENDOR_KERNEL_RAMDISK_TARGET) \
$(BUILT_RAMDISK_16K_TARGET) \
$(BUILT_KERNEL_16K_TARGET) \
+ $(BUILT_BOOTIMAGE_16K_TARGET) \
+ $(INSTALLED_DTBOIMAGE_16KB_TARGET) \
$(BOARD_PREBUILT_DTBOIMAGE) \
$(BOARD_PREBUILT_RECOVERY_DTBOIMAGE) \
$(BOARD_RECOVERY_ACPIO) \
@@ -6812,14 +6815,22 @@
$(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES
$(hide) cp $(INSTALLED_DTBOIMAGE_TARGET) $(zip_root)/PREBUILT_IMAGES/
endif # BOARD_PREBUILT_DTBOIMAGE
-ifdef BUILT_KERNEL_16K_TARGET
+ifdef BOARD_KERNEL_PATH_16K
$(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES
$(hide) cp $(BUILT_KERNEL_16K_TARGET) $(zip_root)/PREBUILT_IMAGES/
-endif # BUILT_KERNEL_16K_TARGET
-ifdef BUILT_RAMDISK_16K_TARGET
+endif # BOARD_KERNEL_PATH_16K
+ifdef BOARD_KERNEL_MODULES_16K
$(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES
$(hide) cp $(BUILT_RAMDISK_16K_TARGET) $(zip_root)/PREBUILT_IMAGES/
-endif # BUILT_RAMDISK_16K_TARGET
+endif # BOARD_KERNEL_MODULES_16K
+ifdef BUILT_BOOTIMAGE_16K_TARGET
+ $(hide) mkdir -p $(zip_root)/IMAGES
+ $(hide) cp $(BUILT_BOOTIMAGE_16K_TARGET) $(zip_root)/IMAGES/
+endif # BUILT_BOOTIMAGE_16K_TARGET
+ifdef INSTALLED_DTBOIMAGE_16KB_TARGET
+ $(hide) mkdir -p $(zip_root)/IMAGES
+ $(hide) cp $(INSTALLED_DTBOIMAGE_16KB_TARGET) $(zip_root)/IMAGES/
+endif # INSTALLED_DTBOIMAGE_16KB_TARGET
ifeq ($(BOARD_USES_PVMFWIMAGE),true)
$(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES
$(hide) cp $(INSTALLED_PVMFWIMAGE_TARGET) $(zip_root)/PREBUILT_IMAGES/
diff --git a/core/board_config.mk b/core/board_config.mk
index d3f0493..5606964 100644
--- a/core/board_config.mk
+++ b/core/board_config.mk
@@ -237,6 +237,7 @@
.KATI_READONLY := TARGET_DEVICE_DIR
endif
+$(call dump-phase-start,BOARD,,,, build/make/core/board_config.mk)
ifndef RBC_PRODUCT_CONFIG
include $(board_config_mk)
else
@@ -261,6 +262,7 @@
include $(OUT_DIR)/rbc/rbc_board_config_results.mk
endif
+$(call dump-phase-end, build/make/core/board_config.mk)
ifneq (,$(and $(TARGET_ARCH),$(TARGET_ARCH_SUITE)))
$(error $(board_config_mk) erroneously sets both TARGET_ARCH and TARGET_ARCH_SUITE)
diff --git a/core/config.mk b/core/config.mk
index 76ad361..192c8b2 100644
--- a/core/config.mk
+++ b/core/config.mk
@@ -173,6 +173,7 @@
$(KATI_obsolete_var BUILDING_PVMFW_IMAGE,BUILDING_PVMFW_IMAGE is no longer used)
$(KATI_obsolete_var BOARD_BUILD_SYSTEM_ROOT_IMAGE)
$(KATI_obsolete_var FS_GET_STATS)
+$(KATI_obsolete_var BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES)
# Used to force goals to build. Only use for conditionally defined goals.
.PHONY: FORCE
@@ -363,8 +364,7 @@
# configs, generally for cross-cutting features.
# Build broken variables that should be treated as booleans
-_build_broken_bool_vars := \
- BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES \
+_build_broken_bool_vars :=
# Build broken variables that should be treated as lists
_build_broken_list_vars := \
@@ -811,6 +811,12 @@
BOARD_PROPERTY_OVERRIDES_SPLIT_ENABLED ?= true
endif
+ifneq ($(call math_gt_or_eq,$(PRODUCT_SHIPPING_API_LEVEL),36),)
+ ifneq ($(NEED_AIDL_NDK_PLATFORM_BACKEND),)
+ $(error Must not set NEED_AIDL_NDK_PLATFORM_BACKEND, but it is set to: $(NEED_AIDL_NDK_PLATFORM_BACKEND). Support will be removed.)
+ endif
+endif
+
# Set BOARD_SYSTEMSDK_VERSIONS to the latest SystemSDK version starting from P-launching
# devices if unset.
ifndef BOARD_SYSTEMSDK_VERSIONS
@@ -1201,6 +1207,11 @@
APPS_DEFAULT_VERSION_NAME := $(PLATFORM_VERSION)
+# Add BUILD_NUMBER to apps if PRODUCT_BUILD_APPS_WITH_BUILD_NUMBER is defined.
+ifeq ($(PRODUCT_BUILD_APPS_WITH_BUILD_NUMBER),true)
+ APPS_DEFAULT_VERSION_NAME := $(PLATFORM_VERSION)-$(BUILD_NUMBER_FROM_FILE)
+endif
+
# ANDROID_WARNING_ALLOWED_PROJECTS is generated by build/soong.
define find_warning_allowed_projects
$(filter $(ANDROID_WARNING_ALLOWED_PROJECTS),$(1)/)
diff --git a/core/dumpconfig.mk b/core/dumpconfig.mk
index 640fe10..eb4c822 100644
--- a/core/dumpconfig.mk
+++ b/core/dumpconfig.mk
@@ -56,7 +56,7 @@
# Escape quotation marks for CSV, and wraps in quotation marks.
define escape-for-csv
-"$(subst ","",$1)"
+"$(subst ","",$(subst $(newline), ,$1))"
endef
# Args:
@@ -68,7 +68,7 @@
# Args:
# $(1): include stack
define dump-import-done
-$(eval $(file >> $(DUMPCONFIG_FILE),imported,$(strip $(1))))
+$(eval $(file >> $(DUMPCONFIG_FILE),imported,$(strip $(1)),$(filter-out $(1),$(MAKEFILE_LIST))))
endef
# Args:
diff --git a/core/main.mk b/core/main.mk
index 5c280da..8d0b465 100644
--- a/core/main.mk
+++ b/core/main.mk
@@ -305,6 +305,9 @@
$(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] finishing legacy Make module parsing ...)
+# Create necessary directories and symlinks in the root filesystem
+include system/core/rootdir/create_root_structure.mk
+
# -------------------------------------------------------------------
# All module makefiles have been included at this point.
# -------------------------------------------------------------------
diff --git a/core/product.mk b/core/product.mk
index 8d86d92..4c23e5d 100644
--- a/core/product.mk
+++ b/core/product.mk
@@ -493,6 +493,9 @@
# by this flag.
_product_single_value_vars += PRODUCT_NOT_DEBUGGABLE_IN_USERDEBUG
+# If set, the default value of the versionName of apps will include the build number.
+_product_single_value_vars += PRODUCT_BUILD_APPS_WITH_BUILD_NUMBER
+
# If set, build would generate system image from Soong-defined module.
_product_single_value_vars += PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE
diff --git a/core/soong_android_app_set.mk b/core/soong_android_app_set.mk
index ec3d8c8..d97980d 100644
--- a/core/soong_android_app_set.mk
+++ b/core/soong_android_app_set.mk
@@ -9,10 +9,6 @@
LOCAL_BUILT_MODULE_STEM := package.apk
LOCAL_INSTALLED_MODULE_STEM := $(notdir $(LOCAL_PREBUILT_MODULE_FILE))
-# Use the Soong output as the checkbuild target instead of LOCAL_BUILT_MODULE
-# to avoid checkbuilds making an extra copy of every module.
-LOCAL_CHECKED_MODULE := $(LOCAL_PREBUILT_MODULE_FILE)
-
#######################################
include $(BUILD_SYSTEM)/base_rules.mk
#######################################
diff --git a/core/soong_app_prebuilt.mk b/core/soong_app_prebuilt.mk
index 3aa244c..df1cf2d 100644
--- a/core/soong_app_prebuilt.mk
+++ b/core/soong_app_prebuilt.mk
@@ -29,16 +29,6 @@
full_classes_header_jar := $(intermediates.COMMON)/classes-header.jar
-# Use the Soong output as the checkbuild target instead of LOCAL_BUILT_MODULE
-# to avoid checkbuilds making an extra copy of every module.
-LOCAL_CHECKED_MODULE := $(LOCAL_PREBUILT_MODULE_FILE)
-LOCAL_ADDITIONAL_CHECKED_MODULE += $(LOCAL_SOONG_CLASSES_JAR)
-LOCAL_ADDITIONAL_CHECKED_MODULE += $(LOCAL_SOONG_HEADER_JAR)
-LOCAL_ADDITIONAL_CHECKED_MODULE += $(LOCAL_FULL_MANIFEST_FILE)
-LOCAL_ADDITIONAL_CHECKED_MODULE += $(LOCAL_SOONG_DEXPREOPT_CONFIG)
-LOCAL_ADDITIONAL_CHECKED_MODULE += $(LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE)
-LOCAL_ADDITIONAL_CHECKED_MODULE += $(LOCAL_SOONG_DEX_JAR)
-
#######################################
include $(BUILD_SYSTEM)/base_rules.mk
#######################################
diff --git a/core/soong_cc_rust_prebuilt.mk b/core/soong_cc_rust_prebuilt.mk
index a1c6478..da60832 100644
--- a/core/soong_cc_rust_prebuilt.mk
+++ b/core/soong_cc_rust_prebuilt.mk
@@ -38,10 +38,6 @@
endif
endif
-# Use the Soong output as the checkbuild target instead of LOCAL_BUILT_MODULE
-# to avoid checkbuilds making an extra copy of every module.
-LOCAL_CHECKED_MODULE := $(LOCAL_PREBUILT_MODULE_FILE)
-
my_check_same_vndk_variants :=
same_vndk_variants_stamp :=
ifeq ($(LOCAL_CHECK_SAME_VNDK_VARIANTS),true)
@@ -61,7 +57,7 @@
# Note that because `checkbuild` doesn't check LOCAL_BUILT_MODULE for soong-built modules adding
# the timestamp to LOCAL_BUILT_MODULE isn't enough. It is skipped when the vendor variant
# isn't used at all and it may break in the downstream trees.
- LOCAL_ADDITIONAL_CHECKED_MODULE := $(same_vndk_variants_stamp)
+ LOCAL_ADDITIONAL_CHECKED_MODULE += $(same_vndk_variants_stamp)
endif
#######################################
diff --git a/core/soong_config.mk b/core/soong_config.mk
index 2e7cd9f..1e6388a 100644
--- a/core/soong_config.mk
+++ b/core/soong_config.mk
@@ -236,6 +236,12 @@
$(call add_json_list, TargetFSConfigGen, $(TARGET_FS_CONFIG_GEN))
+# Although USE_SOONG_DEFINED_SYSTEM_IMAGE determines whether to use the system image specified by
+# PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE, PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE is still used to compare
+# installed files between make and soong, regardless of the USE_SOONG_DEFINED_SYSTEM_IMAGE setting.
+$(call add_json_bool, UseSoongSystemImage, $(filter true,$(USE_SOONG_DEFINED_SYSTEM_IMAGE)))
+$(call add_json_str, ProductSoongDefinedSystemImage, $(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE))
+
$(call add_json_map, VendorVars)
$(foreach namespace,$(sort $(SOONG_CONFIG_NAMESPACES)),\
$(call add_json_map, $(namespace))\
@@ -286,7 +292,6 @@
$(call add_json_bool, GenruleSandboxing, $(if $(GENRULE_SANDBOXING),$(filter true,$(GENRULE_SANDBOXING)),$(if $(filter true,$(BUILD_BROKEN_GENRULE_SANDBOXING)),,true)))
$(call add_json_bool, BuildBrokenEnforceSyspropOwner, $(filter true,$(BUILD_BROKEN_ENFORCE_SYSPROP_OWNER)))
$(call add_json_bool, BuildBrokenTrebleSyspropNeverallow, $(filter true,$(BUILD_BROKEN_TREBLE_SYSPROP_NEVERALLOW)))
-$(call add_json_bool, BuildBrokenUsesSoongPython2Modules, $(filter true,$(BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES)))
$(call add_json_bool, BuildBrokenVendorPropertyNamespace, $(filter true,$(BUILD_BROKEN_VENDOR_PROPERTY_NAMESPACE)))
$(call add_json_bool, BuildBrokenIncorrectPartitionImages, $(filter true,$(BUILD_BROKEN_INCORRECT_PARTITION_IMAGES)))
$(call add_json_list, BuildBrokenInputDirModules, $(BUILD_BROKEN_INPUT_DIR_MODULES))
@@ -357,6 +362,11 @@
$(call add_json_str, EnableUffdGc, $(_config_enable_uffd_gc))
_config_enable_uffd_gc :=
+$(call add_json_list, DeviceFrameworkCompatibilityMatrixFile, $(DEVICE_FRAMEWORK_COMPATIBILITY_MATRIX_FILE))
+$(call add_json_list, DeviceProductCompatibilityMatrixFile, $(DEVICE_PRODUCT_COMPATIBILITY_MATRIX_FILE))
+$(call add_json_list, BoardAvbSystemAddHashtreeFooterArgs, $(BOARD_AVB_SYSTEM_ADD_HASHTREE_FOOTER_ARGS))
+$(call add_json_bool, BoardAvbEnable, $(filter true,$(BOARD_AVB_ENABLE)))
+
$(call json_end)
$(file >$(SOONG_VARIABLES).tmp,$(json_contents))
diff --git a/core/soong_java_prebuilt.mk b/core/soong_java_prebuilt.mk
index f74bb6d..8c3882f 100644
--- a/core/soong_java_prebuilt.mk
+++ b/core/soong_java_prebuilt.mk
@@ -21,19 +21,6 @@
full_classes_header_jar := $(intermediates.COMMON)/classes-header.jar
common_javalib.jar := $(intermediates.COMMON)/javalib.jar
-ifdef LOCAL_SOONG_AAR
- LOCAL_ADDITIONAL_CHECKED_MODULE += $(LOCAL_SOONG_AAR)
-endif
-
-# Use the Soong output as the checkbuild target instead of LOCAL_BUILT_MODULE
-# to avoid checkbuilds making an extra copy of every module.
-LOCAL_CHECKED_MODULE := $(LOCAL_PREBUILT_MODULE_FILE)
-LOCAL_ADDITIONAL_CHECKED_MODULE += $(LOCAL_SOONG_HEADER_JAR)
-LOCAL_ADDITIONAL_CHECKED_MODULE += $(LOCAL_FULL_MANIFEST_FILE)
-LOCAL_ADDITIONAL_CHECKED_MODULE += $(LOCAL_SOONG_DEXPREOPT_CONFIG)
-LOCAL_ADDITIONAL_CHECKED_MODULE += $(LOCAL_SOONG_RESOURCE_EXPORT_PACKAGE)
-LOCAL_ADDITIONAL_CHECKED_MODULE += $(LOCAL_SOONG_DEX_JAR)
-
#######################################
include $(BUILD_SYSTEM)/base_rules.mk
#######################################
diff --git a/core/tasks/meta-lic.mk b/core/tasks/meta-lic.mk
index 24adfc8..620b1e2 100644
--- a/core/tasks/meta-lic.mk
+++ b/core/tasks/meta-lic.mk
@@ -225,3 +225,6 @@
# Moved here from hardware/libhardware_legacy/Android.mk
$(eval $(call declare-1p-copy-files,hardware/libhardware_legacy,))
+
+# Moved here from system/core/rootdir/Android.mk
+$(eval $(call declare-1p-copy-files,system/core/rootdir,))
diff --git a/core/tasks/sts-sdk.mk b/core/tasks/sts-sdk.mk
index b8ce5bf..4abbc29 100644
--- a/core/tasks/sts-sdk.mk
+++ b/core/tasks/sts-sdk.mk
@@ -28,8 +28,7 @@
rm -f $@ $(STS_SDK_ZIP)_filtered
$(ZIP2ZIP) -i $(STS_SDK_ZIP) -o $(STS_SDK_ZIP)_filtered \
-x android-sts-sdk/tools/sts-tradefed-tests.jar \
- 'android-sts-sdk/tools/*:plugin/src/main/resources/sts-tradefed-tools/' \
- 'android-sts-sdk/jdk/**/*:plugin/src/main/resources/jdk/'
+ 'android-sts-sdk/tools/*:sts-sdk/src/main/resources/sts-tradefed-tools/'
$(MERGE_ZIPS) $@ $(STS_SDK_ZIP)_filtered $(STS_SDK_PLUGIN_SKEL)
rm -f $(STS_SDK_ZIP)_filtered
diff --git a/core/tasks/tools/compatibility.mk b/core/tasks/tools/compatibility.mk
index 86c23f8..9189c2d 100644
--- a/core/tasks/tools/compatibility.mk
+++ b/core/tasks/tools/compatibility.mk
@@ -29,7 +29,9 @@
special_mts_test_suites :=
special_mts_test_suites += mcts
special_mts_test_suites += $(mts_modules)
-ifneq ($(filter $(special_mts_test_suites),$(subst -, ,$(test_suite_name))),)
+ifneq ($(filter $(special_mts_test_suites),$(patsubst mcts-%,%,$(test_suite_name))),)
+ test_suite_subdir := android-mts
+else ifneq ($(filter $(special_mts_test_suites),$(patsubst mts-%,%,$(test_suite_name))),)
test_suite_subdir := android-mts
else
test_suite_subdir := android-$(test_suite_name)
diff --git a/core/tasks/tradefed-tests-list.mk b/core/tasks/tradefed-tests-list.mk
index 61bf136..47c360d 100644
--- a/core/tasks/tradefed-tests-list.mk
+++ b/core/tasks/tradefed-tests-list.mk
@@ -15,6 +15,11 @@
# List all TradeFed tests from COMPATIBILITY.tradefed_tests_dir
.PHONY: tradefed-tests-list
+COMPATIBILITY.tradefed_tests_dir := \
+ $(COMPATIBILITY.tradefed_tests_dir) \
+ tools/tradefederation/core/res/config \
+ tools/tradefederation/core/javatests/res/config
+
tradefed_tests :=
$(foreach dir, $(COMPATIBILITY.tradefed_tests_dir), \
$(eval tradefed_tests += $(shell find $(dir) -type f -name "*.xml")))
diff --git a/target/product/base_product.mk b/target/product/base_product.mk
index 92fc420..acfc653 100644
--- a/target/product/base_product.mk
+++ b/target/product/base_product.mk
@@ -26,3 +26,7 @@
product_manifest.xml \
selinux_policy_product \
product-build.prop \
+
+# Packages included only for eng or userdebug builds, previously debug tagged
+PRODUCT_PACKAGES_DEBUG += \
+ adb_keys \
diff --git a/target/product/base_system.mk b/target/product/base_system.mk
index 5b54051..d806c06 100644
--- a/target/product/base_system.mk
+++ b/target/product/base_system.mk
@@ -83,7 +83,6 @@
CtsShimPrivPrebuilt \
debuggerd\
device_config \
- DeviceDiagnostics \
dmctl \
dnsmasq \
dmesgd \
@@ -466,7 +465,6 @@
# Packages included only for eng or userdebug builds, previously debug tagged
PRODUCT_PACKAGES_DEBUG := \
- adb_keys \
adevice_fingerprint \
arping \
dmuserd \
@@ -534,3 +532,4 @@
$(call soong_config_set, bionic, large_system_property_node, $(RELEASE_LARGE_SYSTEM_PROPERTY_NODE))
$(call soong_config_set, Aconfig, read_from_new_storage, $(RELEASE_READ_FROM_NEW_STORAGE))
+$(call soong_config_set, SettingsLib, legacy_avatar_picker_app_enabled, $(if $(RELEASE_AVATAR_PICKER_APP),,true))
diff --git a/target/product/go_defaults.mk b/target/product/go_defaults.mk
index 4627fde..c928530 100644
--- a/target/product/go_defaults.mk
+++ b/target/product/go_defaults.mk
@@ -17,7 +17,8 @@
# Inherit common Android Go defaults.
$(call inherit-product, build/make/target/product/go_defaults_common.mk)
-PRODUCT_RELEASE_CONFIG_MAPS += $(wildcard vendor/google_shared/build/release/go_devices/release_config_map.textproto)
+# Product config map to toggle between sources and prebuilts of required mainline modules
+PRODUCT_RELEASE_CONFIG_MAPS += $(wildcard vendor/google_shared/build/release/gms_mainline_go/required/release_config_map.textproto)
# Add the system properties.
TARGET_SYSTEM_PROP += \
diff --git a/target/product/handheld_system.mk b/target/product/handheld_system.mk
index 3f3bd01..546bbe7 100644
--- a/target/product/handheld_system.mk
+++ b/target/product/handheld_system.mk
@@ -46,6 +46,7 @@
CertInstaller \
CredentialManager \
DeviceAsWebcam \
+ DeviceDiagnostics \
DocumentsUI \
DownloadProviderUi \
EasterEgg \
diff --git a/target/product/media_system_ext.mk b/target/product/media_system_ext.mk
index 2e20af3..30dd2e2 100644
--- a/target/product/media_system_ext.mk
+++ b/target/product/media_system_ext.mk
@@ -23,3 +23,6 @@
# /system_ext packages
PRODUCT_PACKAGES += \
vndk_apex_snapshot_package \
+
+# Window Extensions
+$(call inherit-product, $(SRC_TARGET_DIR)/product/window_extensions_base.mk)
diff --git a/target/product/security/Android.mk b/target/product/security/Android.mk
index 91b272c..138e5bb 100644
--- a/target/product/security/Android.mk
+++ b/target/product/security/Android.mk
@@ -10,7 +10,7 @@
LOCAL_LICENSE_CONDITIONS := notice
LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE
LOCAL_MODULE_CLASS := ETC
- LOCAL_MODULE_PATH := $(TARGET_ROOT_OUT)
+ LOCAL_MODULE_PATH := $(TARGET_OUT_PRODUCT_ETC)/security
LOCAL_PREBUILT_MODULE_FILE := $(PRODUCT_ADB_KEYS)
include $(BUILD_PREBUILT)
endif
diff --git a/target/product/window_extensions.mk b/target/product/window_extensions.mk
index 5f5431f..d27a613 100644
--- a/target/product/window_extensions.mk
+++ b/target/product/window_extensions.mk
@@ -14,11 +14,14 @@
# limitations under the License.
#
-# /system_ext packages
-PRODUCT_PACKAGES += \
- androidx.window.extensions \
- androidx.window.sidecar
-
-# properties
+# Extension of window_extensions_base.mk to enable the activity embedding
+# feature for all apps by default. All large screen devices must inherit
+# this in build. Optional for other form factors.
+#
+# Indicated whether the Activity Embedding feature should be guarded by
+# Android 15 to avoid app compat impact.
+# If true (or not set), the feature is only enabled for apps with target
+# SDK of Android 15 or above.
+# If false, the feature is enabled for all apps.
PRODUCT_PRODUCT_PROPERTIES += \
- persist.wm.extensions.enabled=true
+ persist.wm.extensions.activity_embedding_guard_with_android_15=false
diff --git a/target/product/window_extensions_base.mk b/target/product/window_extensions_base.mk
new file mode 100644
index 0000000..ee0e5e7
--- /dev/null
+++ b/target/product/window_extensions_base.mk
@@ -0,0 +1,33 @@
+#
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# The base version of window_extensions.mk to be included on all non-wearable
+# devices. Devices that don't support multi-window can choose to drop this.
+#
+# Note: by default the Activity Embedding feature is guarded by app's
+# targetSDK on Android 15 to avoid app compat impact.
+#
+# Large screen devices must inherit window_extensions.mk to enable the
+# Activity Embedding feature for all apps.
+
+# /system_ext packages
+PRODUCT_PACKAGES += \
+ androidx.window.extensions \
+ androidx.window.sidecar
+
+# properties
+PRODUCT_PRODUCT_PROPERTIES += \
+ persist.wm.extensions.enabled=true
diff --git a/tools/aconfig/aconfig/src/codegen/rust.rs b/tools/aconfig/aconfig/src/codegen/rust.rs
index 1292e0a..6f3f7bf 100644
--- a/tools/aconfig/aconfig/src/codegen/rust.rs
+++ b/tools/aconfig/aconfig/src/codegen/rust.rs
@@ -113,6 +113,7 @@
use aconfig_storage_read_api::{Mmap, AconfigStorageError, StorageFileType, PackageReadContext, get_mapped_storage_file, get_boolean_flag_value, get_package_read_context};
use std::path::Path;
use std::io::Write;
+use std::sync::LazyLock;
use log::{log, LevelFilter, Level};
static STORAGE_MIGRATION_MARKER_FILE: &str =
@@ -122,32 +123,29 @@
/// flag provider
pub struct FlagProvider;
-lazy_static::lazy_static! {
/// flag value cache for disabled_rw
- static ref CACHED_disabled_rw: bool = flags_rust::GetServerConfigurableFlag(
+ static CACHED_disabled_rw: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag(
"aconfig_flags.aconfig_test",
"com.android.aconfig.test.disabled_rw",
- "false") == "true";
+ "false") == "true");
/// flag value cache for disabled_rw_exported
- static ref CACHED_disabled_rw_exported: bool = flags_rust::GetServerConfigurableFlag(
+ static CACHED_disabled_rw_exported: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag(
"aconfig_flags.aconfig_test",
"com.android.aconfig.test.disabled_rw_exported",
- "false") == "true";
+ "false") == "true");
/// flag value cache for disabled_rw_in_other_namespace
- static ref CACHED_disabled_rw_in_other_namespace: bool = flags_rust::GetServerConfigurableFlag(
+ static CACHED_disabled_rw_in_other_namespace: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag(
"aconfig_flags.other_namespace",
"com.android.aconfig.test.disabled_rw_in_other_namespace",
- "false") == "true";
+ "false") == "true");
/// flag value cache for enabled_rw
- static ref CACHED_enabled_rw: bool = flags_rust::GetServerConfigurableFlag(
+ static CACHED_enabled_rw: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag(
"aconfig_flags.aconfig_test",
"com.android.aconfig.test.enabled_rw",
- "true") == "true";
-
-}
+ "true") == "true");
impl FlagProvider {
/// query flag disabled_ro
@@ -259,6 +257,7 @@
use aconfig_storage_read_api::{Mmap, AconfigStorageError, StorageFileType, PackageReadContext, get_mapped_storage_file, get_boolean_flag_value, get_package_read_context};
use std::path::Path;
use std::io::Write;
+use std::sync::LazyLock;
use log::{log, LevelFilter, Level};
static STORAGE_MIGRATION_MARKER_FILE: &str =
@@ -268,282 +267,276 @@
/// flag provider
pub struct FlagProvider;
-lazy_static::lazy_static! {
+static PACKAGE_OFFSET: LazyLock<Result<Option<u32>, AconfigStorageError>> = LazyLock::new(|| unsafe {
+ get_mapped_storage_file("system", StorageFileType::PackageMap)
+ .and_then(|package_map| get_package_read_context(&package_map, "com.android.aconfig.test"))
+ .map(|context| context.map(|c| c.boolean_start_index))
+});
- static ref PACKAGE_OFFSET: Result<Option<u32>, AconfigStorageError> = unsafe {
- get_mapped_storage_file("system", StorageFileType::PackageMap)
- .and_then(|package_map| get_package_read_context(&package_map, "com.android.aconfig.test"))
- .map(|context| context.map(|c| c.boolean_start_index))
- };
+static FLAG_VAL_MAP: LazyLock<Result<Mmap, AconfigStorageError>> = LazyLock::new(|| unsafe {
+ get_mapped_storage_file("system", StorageFileType::FlagVal)
+});
- static ref FLAG_VAL_MAP: Result<Mmap, AconfigStorageError> = unsafe {
- get_mapped_storage_file("system", StorageFileType::FlagVal)
- };
- /// flag value cache for disabled_rw
+/// flag value cache for disabled_rw
+static CACHED_disabled_rw: LazyLock<bool> = LazyLock::new(|| {
+ let result = flags_rust::GetServerConfigurableFlag(
+ "aconfig_flags.aconfig_test",
+ "com.android.aconfig.test.disabled_rw",
+ "false") == "true";
- static ref CACHED_disabled_rw: bool = {
- let result = flags_rust::GetServerConfigurableFlag(
- "aconfig_flags.aconfig_test",
- "com.android.aconfig.test.disabled_rw",
- "false") == "true";
+ let use_new_storage_value = flags_rust::GetServerConfigurableFlag(
+ "aconfig_flags.core_experiments_team_internal",
+ "com.android.providers.settings.use_new_storage_value",
+ "false") == "true";
- let use_new_storage_value = flags_rust::GetServerConfigurableFlag(
- "aconfig_flags.core_experiments_team_internal",
- "com.android.providers.settings.use_new_storage_value",
- "false") == "true";
+ if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() {
+ // This will be called multiple times. Subsequent calls after the first are noops.
+ logger::init(
+ logger::Config::default()
+ .with_tag_on_device(MIGRATION_LOG_TAG)
+ .with_max_level(LevelFilter::Info));
- if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() {
- // This will be called multiple times. Subsequent calls after the first are noops.
- logger::init(
- logger::Config::default()
- .with_tag_on_device(MIGRATION_LOG_TAG)
- .with_max_level(LevelFilter::Info));
+ let aconfig_storage_result = FLAG_VAL_MAP
+ .as_ref()
+ .map_err(|err| format!("failed to get flag val map: {err}"))
+ .and_then(|flag_val_map| {
+ PACKAGE_OFFSET
+ .as_ref()
+ .map_err(|err| format!("failed to get package read offset: {err}"))
+ .and_then(|package_offset| {
+ match package_offset {
+ Some(offset) => {
+ get_boolean_flag_value(&flag_val_map, offset + 1)
+ .map_err(|err| format!("failed to get flag: {err}"))
+ },
+ None => Err("no context found for package 'com.android.aconfig.test'".to_string())
+ }
+ })
+ });
- let aconfig_storage_result = FLAG_VAL_MAP
- .as_ref()
- .map_err(|err| format!("failed to get flag val map: {err}"))
- .and_then(|flag_val_map| {
- PACKAGE_OFFSET
- .as_ref()
- .map_err(|err| format!("failed to get package read offset: {err}"))
- .and_then(|package_offset| {
- match package_offset {
- Some(offset) => {
- get_boolean_flag_value(&flag_val_map, offset + 1)
- .map_err(|err| format!("failed to get flag: {err}"))
- },
- None => Err("no context found for package 'com.android.aconfig.test'".to_string())
- }
- })
- });
-
- match aconfig_storage_result {
- Ok(storage_result) if storage_result == result => {
- if use_new_storage_value {
- return storage_result;
- } else {
- return result;
- }
- },
- Ok(storage_result) => {
- log!(Level::Error, "AconfigTestMission1: error: mismatch for flag 'disabled_rw'. Legacy storage was {result}, new storage was {storage_result}");
- if use_new_storage_value {
- return storage_result;
- } else {
- return result;
- }
- },
- Err(err) => {
- log!(Level::Error, "AconfigTestMission1: error: {err}");
- if use_new_storage_value {
- panic!("failed to read flag value: {err}");
- }
+ match aconfig_storage_result {
+ Ok(storage_result) if storage_result == result => {
+ if use_new_storage_value {
+ return storage_result;
+ } else {
+ return result;
+ }
+ },
+ Ok(storage_result) => {
+ log!(Level::Error, "AconfigTestMission1: error: mismatch for flag 'disabled_rw'. Legacy storage was {result}, new storage was {storage_result}");
+ if use_new_storage_value {
+ return storage_result;
+ } else {
+ return result;
+ }
+ },
+ Err(err) => {
+ log!(Level::Error, "AconfigTestMission1: error: {err}");
+ if use_new_storage_value {
+ panic!("failed to read flag value: {err}");
}
}
}
+ }
- result
- };
+ result
+});
- /// flag value cache for disabled_rw_exported
+/// flag value cache for disabled_rw_exported
+static CACHED_disabled_rw_exported: LazyLock<bool> = LazyLock::new(|| {
+ let result = flags_rust::GetServerConfigurableFlag(
+ "aconfig_flags.aconfig_test",
+ "com.android.aconfig.test.disabled_rw_exported",
+ "false") == "true";
- static ref CACHED_disabled_rw_exported: bool = {
- let result = flags_rust::GetServerConfigurableFlag(
- "aconfig_flags.aconfig_test",
- "com.android.aconfig.test.disabled_rw_exported",
- "false") == "true";
+ let use_new_storage_value = flags_rust::GetServerConfigurableFlag(
+ "aconfig_flags.core_experiments_team_internal",
+ "com.android.providers.settings.use_new_storage_value",
+ "false") == "true";
- let use_new_storage_value = flags_rust::GetServerConfigurableFlag(
- "aconfig_flags.core_experiments_team_internal",
- "com.android.providers.settings.use_new_storage_value",
- "false") == "true";
+ if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() {
+ // This will be called multiple times. Subsequent calls after the first are noops.
+ logger::init(
+ logger::Config::default()
+ .with_tag_on_device(MIGRATION_LOG_TAG)
+ .with_max_level(LevelFilter::Info));
- if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() {
- // This will be called multiple times. Subsequent calls after the first are noops.
- logger::init(
- logger::Config::default()
- .with_tag_on_device(MIGRATION_LOG_TAG)
- .with_max_level(LevelFilter::Info));
+ let aconfig_storage_result = FLAG_VAL_MAP
+ .as_ref()
+ .map_err(|err| format!("failed to get flag val map: {err}"))
+ .and_then(|flag_val_map| {
+ PACKAGE_OFFSET
+ .as_ref()
+ .map_err(|err| format!("failed to get package read offset: {err}"))
+ .and_then(|package_offset| {
+ match package_offset {
+ Some(offset) => {
+ get_boolean_flag_value(&flag_val_map, offset + 2)
+ .map_err(|err| format!("failed to get flag: {err}"))
+ },
+ None => Err("no context found for package 'com.android.aconfig.test'".to_string())
+ }
+ })
+ });
- let aconfig_storage_result = FLAG_VAL_MAP
- .as_ref()
- .map_err(|err| format!("failed to get flag val map: {err}"))
- .and_then(|flag_val_map| {
- PACKAGE_OFFSET
- .as_ref()
- .map_err(|err| format!("failed to get package read offset: {err}"))
- .and_then(|package_offset| {
- match package_offset {
- Some(offset) => {
- get_boolean_flag_value(&flag_val_map, offset + 2)
- .map_err(|err| format!("failed to get flag: {err}"))
- },
- None => Err("no context found for package 'com.android.aconfig.test'".to_string())
- }
- })
- });
-
- match aconfig_storage_result {
- Ok(storage_result) if storage_result == result => {
- if use_new_storage_value {
- return storage_result;
- } else {
- return result;
- }
- },
- Ok(storage_result) => {
- log!(Level::Error, "AconfigTestMission1: error: mismatch for flag 'disabled_rw_exported'. Legacy storage was {result}, new storage was {storage_result}");
- if use_new_storage_value {
- return storage_result;
- } else {
- return result;
- }
- },
- Err(err) => {
- log!(Level::Error, "AconfigTestMission1: error: {err}");
- if use_new_storage_value {
- panic!("failed to read flag value: {err}");
- }
+ match aconfig_storage_result {
+ Ok(storage_result) if storage_result == result => {
+ if use_new_storage_value {
+ return storage_result;
+ } else {
+ return result;
+ }
+ },
+ Ok(storage_result) => {
+ log!(Level::Error, "AconfigTestMission1: error: mismatch for flag 'disabled_rw_exported'. Legacy storage was {result}, new storage was {storage_result}");
+ if use_new_storage_value {
+ return storage_result;
+ } else {
+ return result;
+ }
+ },
+ Err(err) => {
+ log!(Level::Error, "AconfigTestMission1: error: {err}");
+ if use_new_storage_value {
+ panic!("failed to read flag value: {err}");
}
}
}
+ }
- result
- };
+ result
+});
- /// flag value cache for disabled_rw_in_other_namespace
+/// flag value cache for disabled_rw_in_other_namespace
+static CACHED_disabled_rw_in_other_namespace: LazyLock<bool> = LazyLock::new(|| {
+ let result = flags_rust::GetServerConfigurableFlag(
+ "aconfig_flags.other_namespace",
+ "com.android.aconfig.test.disabled_rw_in_other_namespace",
+ "false") == "true";
- static ref CACHED_disabled_rw_in_other_namespace: bool = {
- let result = flags_rust::GetServerConfigurableFlag(
- "aconfig_flags.other_namespace",
- "com.android.aconfig.test.disabled_rw_in_other_namespace",
- "false") == "true";
+ let use_new_storage_value = flags_rust::GetServerConfigurableFlag(
+ "aconfig_flags.core_experiments_team_internal",
+ "com.android.providers.settings.use_new_storage_value",
+ "false") == "true";
- let use_new_storage_value = flags_rust::GetServerConfigurableFlag(
- "aconfig_flags.core_experiments_team_internal",
- "com.android.providers.settings.use_new_storage_value",
- "false") == "true";
+ if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() {
+ // This will be called multiple times. Subsequent calls after the first are noops.
+ logger::init(
+ logger::Config::default()
+ .with_tag_on_device(MIGRATION_LOG_TAG)
+ .with_max_level(LevelFilter::Info));
- if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() {
- // This will be called multiple times. Subsequent calls after the first are noops.
- logger::init(
- logger::Config::default()
- .with_tag_on_device(MIGRATION_LOG_TAG)
- .with_max_level(LevelFilter::Info));
+ let aconfig_storage_result = FLAG_VAL_MAP
+ .as_ref()
+ .map_err(|err| format!("failed to get flag val map: {err}"))
+ .and_then(|flag_val_map| {
+ PACKAGE_OFFSET
+ .as_ref()
+ .map_err(|err| format!("failed to get package read offset: {err}"))
+ .and_then(|package_offset| {
+ match package_offset {
+ Some(offset) => {
+ get_boolean_flag_value(&flag_val_map, offset + 3)
+ .map_err(|err| format!("failed to get flag: {err}"))
+ },
+ None => Err("no context found for package 'com.android.aconfig.test'".to_string())
+ }
+ })
+ });
- let aconfig_storage_result = FLAG_VAL_MAP
- .as_ref()
- .map_err(|err| format!("failed to get flag val map: {err}"))
- .and_then(|flag_val_map| {
- PACKAGE_OFFSET
- .as_ref()
- .map_err(|err| format!("failed to get package read offset: {err}"))
- .and_then(|package_offset| {
- match package_offset {
- Some(offset) => {
- get_boolean_flag_value(&flag_val_map, offset + 3)
- .map_err(|err| format!("failed to get flag: {err}"))
- },
- None => Err("no context found for package 'com.android.aconfig.test'".to_string())
- }
- })
- });
-
- match aconfig_storage_result {
- Ok(storage_result) if storage_result == result => {
- if use_new_storage_value {
- return storage_result;
- } else {
- return result;
- }
- },
- Ok(storage_result) => {
- log!(Level::Error, "AconfigTestMission1: error: mismatch for flag 'disabled_rw_in_other_namespace'. Legacy storage was {result}, new storage was {storage_result}");
- if use_new_storage_value {
- return storage_result;
- } else {
- return result;
- }
- },
- Err(err) => {
- log!(Level::Error, "AconfigTestMission1: error: {err}");
- if use_new_storage_value {
- panic!("failed to read flag value: {err}");
- }
+ match aconfig_storage_result {
+ Ok(storage_result) if storage_result == result => {
+ if use_new_storage_value {
+ return storage_result;
+ } else {
+ return result;
+ }
+ },
+ Ok(storage_result) => {
+ log!(Level::Error, "AconfigTestMission1: error: mismatch for flag 'disabled_rw_in_other_namespace'. Legacy storage was {result}, new storage was {storage_result}");
+ if use_new_storage_value {
+ return storage_result;
+ } else {
+ return result;
+ }
+ },
+ Err(err) => {
+ log!(Level::Error, "AconfigTestMission1: error: {err}");
+ if use_new_storage_value {
+ panic!("failed to read flag value: {err}");
}
}
}
+ }
- result
- };
+ result
+});
- /// flag value cache for enabled_rw
- static ref CACHED_enabled_rw: bool = {
- let result = flags_rust::GetServerConfigurableFlag(
- "aconfig_flags.aconfig_test",
- "com.android.aconfig.test.enabled_rw",
- "true") == "true";
+/// flag value cache for enabled_rw
+static CACHED_enabled_rw: LazyLock<bool> = LazyLock::new(|| {
+ let result = flags_rust::GetServerConfigurableFlag(
+ "aconfig_flags.aconfig_test",
+ "com.android.aconfig.test.enabled_rw",
+ "true") == "true";
- let use_new_storage_value = flags_rust::GetServerConfigurableFlag(
- "aconfig_flags.core_experiments_team_internal",
- "com.android.providers.settings.use_new_storage_value",
- "false") == "true";
+ let use_new_storage_value = flags_rust::GetServerConfigurableFlag(
+ "aconfig_flags.core_experiments_team_internal",
+ "com.android.providers.settings.use_new_storage_value",
+ "false") == "true";
- if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() {
- // This will be called multiple times. Subsequent calls after the first are noops.
- logger::init(
- logger::Config::default()
- .with_tag_on_device(MIGRATION_LOG_TAG)
- .with_max_level(LevelFilter::Info));
+ if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() {
+ // This will be called multiple times. Subsequent calls after the first are noops.
+ logger::init(
+ logger::Config::default()
+ .with_tag_on_device(MIGRATION_LOG_TAG)
+ .with_max_level(LevelFilter::Info));
- let aconfig_storage_result = FLAG_VAL_MAP
- .as_ref()
- .map_err(|err| format!("failed to get flag val map: {err}"))
- .and_then(|flag_val_map| {
- PACKAGE_OFFSET
- .as_ref()
- .map_err(|err| format!("failed to get package read offset: {err}"))
- .and_then(|package_offset| {
- match package_offset {
- Some(offset) => {
- get_boolean_flag_value(&flag_val_map, offset + 8)
- .map_err(|err| format!("failed to get flag: {err}"))
- },
- None => Err("no context found for package 'com.android.aconfig.test'".to_string())
- }
- })
- });
+ let aconfig_storage_result = FLAG_VAL_MAP
+ .as_ref()
+ .map_err(|err| format!("failed to get flag val map: {err}"))
+ .and_then(|flag_val_map| {
+ PACKAGE_OFFSET
+ .as_ref()
+ .map_err(|err| format!("failed to get package read offset: {err}"))
+ .and_then(|package_offset| {
+ match package_offset {
+ Some(offset) => {
+ get_boolean_flag_value(&flag_val_map, offset + 8)
+ .map_err(|err| format!("failed to get flag: {err}"))
+ },
+ None => Err("no context found for package 'com.android.aconfig.test'".to_string())
+ }
+ })
+ });
- match aconfig_storage_result {
- Ok(storage_result) if storage_result == result => {
- if use_new_storage_value {
- return storage_result;
- } else {
- return result;
- }
- },
- Ok(storage_result) => {
- log!(Level::Error, "AconfigTestMission1: error: mismatch for flag 'enabled_rw'. Legacy storage was {result}, new storage was {storage_result}");
- if use_new_storage_value {
- return storage_result;
- } else {
- return result;
- }
- },
- Err(err) => {
- log!(Level::Error, "AconfigTestMission1: error: {err}");
- if use_new_storage_value {
- panic!("failed to read flag value: {err}");
- }
+ match aconfig_storage_result {
+ Ok(storage_result) if storage_result == result => {
+ if use_new_storage_value {
+ return storage_result;
+ } else {
+ return result;
+ }
+ },
+ Ok(storage_result) => {
+ log!(Level::Error, "AconfigTestMission1: error: mismatch for flag 'enabled_rw'. Legacy storage was {result}, new storage was {storage_result}");
+ if use_new_storage_value {
+ return storage_result;
+ } else {
+ return result;
+ }
+ },
+ Err(err) => {
+ log!(Level::Error, "AconfigTestMission1: error: {err}");
+ if use_new_storage_value {
+ panic!("failed to read flag value: {err}");
}
}
}
+ }
- result
- };
-
-}
+ result
+});
impl FlagProvider {
@@ -1207,6 +1200,7 @@
use aconfig_storage_read_api::{Mmap, AconfigStorageError, StorageFileType, PackageReadContext, get_mapped_storage_file, get_boolean_flag_value, get_package_read_context};
use std::path::Path;
use std::io::Write;
+use std::sync::LazyLock;
use log::{log, LevelFilter, Level};
static STORAGE_MIGRATION_MARKER_FILE: &str =
@@ -1216,26 +1210,23 @@
/// flag provider
pub struct FlagProvider;
-lazy_static::lazy_static! {
/// flag value cache for disabled_rw_exported
- static ref CACHED_disabled_rw_exported: bool = flags_rust::GetServerConfigurableFlag(
+ static CACHED_disabled_rw_exported: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag(
"aconfig_flags.aconfig_test",
"com.android.aconfig.test.disabled_rw_exported",
- "false") == "true";
+ "false") == "true");
/// flag value cache for enabled_fixed_ro_exported
- static ref CACHED_enabled_fixed_ro_exported: bool = flags_rust::GetServerConfigurableFlag(
+ static CACHED_enabled_fixed_ro_exported: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag(
"aconfig_flags.aconfig_test",
"com.android.aconfig.test.enabled_fixed_ro_exported",
- "false") == "true";
+ "false") == "true");
/// flag value cache for enabled_ro_exported
- static ref CACHED_enabled_ro_exported: bool = flags_rust::GetServerConfigurableFlag(
+ static CACHED_enabled_ro_exported: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag(
"aconfig_flags.aconfig_test",
"com.android.aconfig.test.enabled_ro_exported",
- "false") == "true";
-
-}
+ "false") == "true");
impl FlagProvider {
/// query flag disabled_rw_exported
@@ -1281,6 +1272,7 @@
use aconfig_storage_read_api::{Mmap, AconfigStorageError, StorageFileType, PackageReadContext, get_mapped_storage_file, get_boolean_flag_value, get_package_read_context};
use std::path::Path;
use std::io::Write;
+use std::sync::LazyLock;
use log::{log, LevelFilter, Level};
static STORAGE_MIGRATION_MARKER_FILE: &str =
diff --git a/tools/aconfig/aconfig/src/commands.rs b/tools/aconfig/aconfig/src/commands.rs
index 59f0662..1a14f64 100644
--- a/tools/aconfig/aconfig/src/commands.rs
+++ b/tools/aconfig/aconfig/src/commands.rs
@@ -17,7 +17,8 @@
use anyhow::{bail, ensure, Context, Result};
use itertools::Itertools;
use protobuf::Message;
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
+use std::hash::Hasher;
use std::io::Read;
use std::path::PathBuf;
@@ -31,6 +32,7 @@
ParsedFlagExt, ProtoFlagMetadata, ProtoFlagPermission, ProtoFlagState, ProtoParsedFlag,
ProtoParsedFlags, ProtoTracepoint,
};
+use aconfig_storage_file::sip_hasher13::SipHasher13;
use aconfig_storage_file::StorageFileType;
pub struct Input {
@@ -410,12 +412,43 @@
Ok(flag_ids)
}
+#[allow(dead_code)] // TODO: b/316357686 - Use fingerprint in codegen to
+ // protect hardcoded offset reads.
+pub fn compute_flag_offsets_fingerprint(flags_map: &HashMap<String, u16>) -> Result<u64> {
+ let mut hasher = SipHasher13::new();
+
+ // Need to sort to ensure the data is added to the hasher in the same order
+ // each run.
+ let sorted_map: BTreeMap<&String, &u16> = flags_map.iter().collect();
+
+ for (flag, offset) in sorted_map {
+ // See https://docs.rs/siphasher/latest/siphasher/#note for use of write
+ // over write_i16. Similarly, use to_be_bytes rather than to_ne_bytes to
+ // ensure consistency.
+ hasher.write(flag.as_bytes());
+ hasher.write(&offset.to_be_bytes());
+ }
+ Ok(hasher.finish())
+}
+
#[cfg(test)]
mod tests {
use super::*;
use aconfig_protos::ProtoFlagPurpose;
#[test]
+ fn test_offset_fingerprint() {
+ let parsed_flags = crate::test::parse_test_flags();
+ let package = find_unique_package(&parsed_flags.parsed_flag).unwrap().to_string();
+ let flag_ids = assign_flag_ids(&package, parsed_flags.parsed_flag.iter()).unwrap();
+ let expected_fingerprint = 10709892481002252132u64;
+
+ let hash_result = compute_flag_offsets_fingerprint(&flag_ids);
+
+ assert_eq!(hash_result.unwrap(), expected_fingerprint);
+ }
+
+ #[test]
fn test_parse_flags() {
let parsed_flags = crate::test::parse_test_flags(); // calls parse_flags
aconfig_protos::parsed_flags::verify_fields(&parsed_flags).unwrap();
diff --git a/tools/aconfig/aconfig/templates/rust.template b/tools/aconfig/aconfig/templates/rust.template
index 77a9984..ea1c600 100644
--- a/tools/aconfig/aconfig/templates/rust.template
+++ b/tools/aconfig/aconfig/templates/rust.template
@@ -2,6 +2,7 @@
use aconfig_storage_read_api::\{Mmap, AconfigStorageError, StorageFileType, PackageReadContext, get_mapped_storage_file, get_boolean_flag_value, get_package_read_context};
use std::path::Path;
use std::io::Write;
+use std::sync::LazyLock;
use log::\{log, LevelFilter, Level};
static STORAGE_MIGRATION_MARKER_FILE: &str =
@@ -12,95 +13,93 @@
pub struct FlagProvider;
{{ if has_readwrite- }}
-lazy_static::lazy_static! \{
- {{ if allow_instrumentation }}
- static ref PACKAGE_OFFSET: Result<Option<u32>, AconfigStorageError> = unsafe \{
- get_mapped_storage_file("{container}", StorageFileType::PackageMap)
- .and_then(|package_map| get_package_read_context(&package_map, "{package}"))
- .map(|context| context.map(|c| c.boolean_start_index))
- };
+{{ if allow_instrumentation }}
+static PACKAGE_OFFSET: LazyLock<Result<Option<u32>, AconfigStorageError>> = LazyLock::new(|| unsafe \{
+ get_mapped_storage_file("{container}", StorageFileType::PackageMap)
+ .and_then(|package_map| get_package_read_context(&package_map, "{package}"))
+ .map(|context| context.map(|c| c.boolean_start_index))
+});
- static ref FLAG_VAL_MAP: Result<Mmap, AconfigStorageError> = unsafe \{
- get_mapped_storage_file("{container}", StorageFileType::FlagVal)
- };
- {{ -endif }}
-
+static FLAG_VAL_MAP: LazyLock<Result<Mmap, AconfigStorageError>> = LazyLock::new(|| unsafe \{
+ get_mapped_storage_file("{container}", StorageFileType::FlagVal)
+});
+{{ -endif }}
{{ -for flag in template_flags }}
- {{ -if flag.readwrite }}
- /// flag value cache for {flag.name}
- {{ if allow_instrumentation }}
- static ref CACHED_{flag.name}: bool = \{
- let result = flags_rust::GetServerConfigurableFlag(
- "aconfig_flags.{flag.device_config_namespace}",
- "{flag.device_config_flag}",
- "{flag.default_value}") == "true";
- let use_new_storage_value = flags_rust::GetServerConfigurableFlag(
- "aconfig_flags.core_experiments_team_internal",
- "com.android.providers.settings.use_new_storage_value",
- "false") == "true";
-
- if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() \{
- // This will be called multiple times. Subsequent calls after the first are noops.
- logger::init(
- logger::Config::default()
- .with_tag_on_device(MIGRATION_LOG_TAG)
- .with_max_level(LevelFilter::Info));
-
- let aconfig_storage_result = FLAG_VAL_MAP
- .as_ref()
- .map_err(|err| format!("failed to get flag val map: \{err}"))
- .and_then(|flag_val_map| \{
- PACKAGE_OFFSET
- .as_ref()
- .map_err(|err| format!("failed to get package read offset: \{err}"))
- .and_then(|package_offset| \{
- match package_offset \{
- Some(offset) => \{
- get_boolean_flag_value(&flag_val_map, offset + {flag.flag_offset})
- .map_err(|err| format!("failed to get flag: \{err}"))
- },
- None => Err("no context found for package '{package}'".to_string())
- }
- })
- });
-
- match aconfig_storage_result \{
- Ok(storage_result) if storage_result == result => \{
- if use_new_storage_value \{
- return storage_result;
- } else \{
- return result;
- }
- },
- Ok(storage_result) => \{
- log!(Level::Error, "AconfigTestMission1: error: mismatch for flag '{flag.name}'. Legacy storage was \{result}, new storage was \{storage_result}");
- if use_new_storage_value \{
- return storage_result;
- } else \{
- return result;
- }
- },
- Err(err) => \{
- log!(Level::Error, "AconfigTestMission1: error: \{err}");
- if use_new_storage_value \{
- panic!("failed to read flag value: \{err}");
- }
- }
- }
- }
-
- result
- };
- {{ else }}
- static ref CACHED_{flag.name}: bool = flags_rust::GetServerConfigurableFlag(
+{{ -if flag.readwrite }}
+/// flag value cache for {flag.name}
+{{ if allow_instrumentation }}
+static CACHED_{flag.name}: LazyLock<bool> = LazyLock::new(|| \{
+ let result = flags_rust::GetServerConfigurableFlag(
"aconfig_flags.{flag.device_config_namespace}",
"{flag.device_config_flag}",
"{flag.default_value}") == "true";
- {{ endif }}
- {{ -endif }}
+
+ let use_new_storage_value = flags_rust::GetServerConfigurableFlag(
+ "aconfig_flags.core_experiments_team_internal",
+ "com.android.providers.settings.use_new_storage_value",
+ "false") == "true";
+
+ if Path::new(STORAGE_MIGRATION_MARKER_FILE).exists() \{
+ // This will be called multiple times. Subsequent calls after the first are noops.
+ logger::init(
+ logger::Config::default()
+ .with_tag_on_device(MIGRATION_LOG_TAG)
+ .with_max_level(LevelFilter::Info));
+
+ let aconfig_storage_result = FLAG_VAL_MAP
+ .as_ref()
+ .map_err(|err| format!("failed to get flag val map: \{err}"))
+ .and_then(|flag_val_map| \{
+ PACKAGE_OFFSET
+ .as_ref()
+ .map_err(|err| format!("failed to get package read offset: \{err}"))
+ .and_then(|package_offset| \{
+ match package_offset \{
+ Some(offset) => \{
+ get_boolean_flag_value(&flag_val_map, offset + {flag.flag_offset})
+ .map_err(|err| format!("failed to get flag: \{err}"))
+ },
+ None => Err("no context found for package '{package}'".to_string())
+ }
+ })
+ });
+
+ match aconfig_storage_result \{
+ Ok(storage_result) if storage_result == result => \{
+ if use_new_storage_value \{
+ return storage_result;
+ } else \{
+ return result;
+ }
+ },
+ Ok(storage_result) => \{
+ log!(Level::Error, "AconfigTestMission1: error: mismatch for flag '{flag.name}'. Legacy storage was \{result}, new storage was \{storage_result}");
+ if use_new_storage_value \{
+ return storage_result;
+ } else \{
+ return result;
+ }
+ },
+ Err(err) => \{
+ log!(Level::Error, "AconfigTestMission1: error: \{err}");
+ if use_new_storage_value \{
+ panic!("failed to read flag value: \{err}");
+ }
+ }
+ }
+ }
+
+ result
+ });
+{{ else }}
+static CACHED_{flag.name}: LazyLock<bool> = LazyLock::new(|| flags_rust::GetServerConfigurableFlag(
+ "aconfig_flags.{flag.device_config_namespace}",
+ "{flag.device_config_flag}",
+ "{flag.default_value}") == "true");
+{{ endif }}
+{{ -endif }}
{{ -endfor }}
-}
{{ -endif }}
impl FlagProvider \{
diff --git a/tools/aconfig/aconfig_flags/Android.bp b/tools/aconfig/aconfig_flags/Android.bp
new file mode 100644
index 0000000..e327ced
--- /dev/null
+++ b/tools/aconfig/aconfig_flags/Android.bp
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+rust_library {
+ name: "libaconfig_flags",
+ crate_name: "aconfig_flags",
+ srcs: [
+ "src/lib.rs",
+ ],
+ rustlibs: [
+ "libaconfig_flags_rust",
+ ],
+ host_supported: true,
+}
+
+aconfig_declarations {
+ name: "aconfig_flags",
+ package: "com.android.aconfig.flags",
+ container: "system",
+ srcs: ["flags.aconfig"],
+}
+
+rust_aconfig_library {
+ name: "libaconfig_flags_rust",
+ crate_name: "aconfig_flags_rust",
+ aconfig_declarations: "aconfig_flags",
+ host_supported: true,
+}
+
+cc_aconfig_library {
+ name: "libaconfig_flags_cc",
+ aconfig_declarations: "aconfig_flags",
+}
diff --git a/tools/aconfig/aconfig_flags/Cargo.toml b/tools/aconfig/aconfig_flags/Cargo.toml
new file mode 100644
index 0000000..6eb9f14
--- /dev/null
+++ b/tools/aconfig/aconfig_flags/Cargo.toml
@@ -0,0 +1,10 @@
+[package]
+name = "aconfig_flags"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+default = ["cargo"]
+cargo = []
+
+[dependencies]
\ No newline at end of file
diff --git a/tools/aconfig/aconfig_flags/flags.aconfig b/tools/aconfig/aconfig_flags/flags.aconfig
new file mode 100644
index 0000000..db8b1b7
--- /dev/null
+++ b/tools/aconfig/aconfig_flags/flags.aconfig
@@ -0,0 +1,9 @@
+package: "com.android.aconfig.flags"
+container: "system"
+
+flag {
+ name: "enable_only_new_storage"
+ namespace: "core_experiments_team_internal"
+ bug: "312235596"
+ description: "When enabled, aconfig flags are read from the new aconfig storage only."
+}
diff --git a/tools/aconfig/aconfig_flags/src/lib.rs b/tools/aconfig/aconfig_flags/src/lib.rs
new file mode 100644
index 0000000..a607efb
--- /dev/null
+++ b/tools/aconfig/aconfig_flags/src/lib.rs
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//! `aconfig_flags` is a crate for reading aconfig flags from Rust
+// When building with the Android tool-chain
+//
+// - the flag functions will read from aconfig_flags_inner
+// - the feature "cargo" will be disabled
+//
+// When building with cargo
+//
+// - the flag functions will all return some trivial value, like true
+// - the feature "cargo" will be enabled
+//
+// This module hides these differences from the rest of aconfig.
+
+/// Module used when building with the Android tool-chain
+#[cfg(not(feature = "cargo"))]
+pub mod auto_generated {
+ /// Returns the value for the enable_only_new_storage flag.
+ pub fn enable_only_new_storage() -> bool {
+ aconfig_flags_rust::enable_only_new_storage()
+ }
+}
+
+/// Module used when building with cargo
+#[cfg(feature = "cargo")]
+pub mod auto_generated {
+ /// Returns a placeholder value for the enable_only_new_storage flag.
+ pub fn enable_only_new_storage() -> bool {
+ // Used only to enable typechecking and testing with cargo
+ true
+ }
+}
diff --git a/tools/aconfig/aconfig_storage_file/src/flag_table.rs b/tools/aconfig/aconfig_storage_file/src/flag_table.rs
index 64b90ea..660edac 100644
--- a/tools/aconfig/aconfig_storage_file/src/flag_table.rs
+++ b/tools/aconfig/aconfig_storage_file/src/flag_table.rs
@@ -150,7 +150,7 @@
/// Calculate node bucket index
pub fn find_bucket_index(package_id: u32, flag_name: &str, num_buckets: u32) -> u32 {
let full_flag_name = package_id.to_string() + "/" + flag_name;
- get_bucket_index(&full_flag_name, num_buckets)
+ get_bucket_index(full_flag_name.as_bytes(), num_buckets)
}
}
diff --git a/tools/aconfig/aconfig_storage_file/src/lib.rs b/tools/aconfig/aconfig_storage_file/src/lib.rs
index 26e9c1a..b6367ff 100644
--- a/tools/aconfig/aconfig_storage_file/src/lib.rs
+++ b/tools/aconfig/aconfig_storage_file/src/lib.rs
@@ -37,19 +37,20 @@
pub mod flag_value;
pub mod package_table;
pub mod protos;
+pub mod sip_hasher13;
pub mod test_utils;
use anyhow::anyhow;
use std::cmp::Ordering;
-use std::collections::hash_map::DefaultHasher;
use std::fs::File;
-use std::hash::{Hash, Hasher};
+use std::hash::Hasher;
use std::io::Read;
pub use crate::flag_info::{FlagInfoBit, FlagInfoHeader, FlagInfoList, FlagInfoNode};
pub use crate::flag_table::{FlagTable, FlagTableHeader, FlagTableNode};
pub use crate::flag_value::{FlagValueHeader, FlagValueList};
pub use crate::package_table::{PackageTable, PackageTableHeader, PackageTableNode};
+pub use crate::sip_hasher13::SipHasher13;
use crate::AconfigStorageError::{
BytesParseFail, HashTableSizeLimit, InvalidFlagValueType, InvalidStoredFlagType,
@@ -211,10 +212,12 @@
}
/// Get the corresponding bucket index given the key and number of buckets
-pub(crate) fn get_bucket_index<T: Hash>(val: &T, num_buckets: u32) -> u32 {
- let mut s = DefaultHasher::new();
- val.hash(&mut s);
- (s.finish() % num_buckets as u64) as u32
+pub(crate) fn get_bucket_index(val: &[u8], num_buckets: u32) -> u32 {
+ let mut s = SipHasher13::new();
+ s.write(val);
+ s.write_u8(0xff);
+ let ret = (s.finish() % num_buckets as u64) as u32;
+ ret
}
/// Read and parse bytes as u8
diff --git a/tools/aconfig/aconfig_storage_file/src/package_table.rs b/tools/aconfig/aconfig_storage_file/src/package_table.rs
index b734972..007f86e 100644
--- a/tools/aconfig/aconfig_storage_file/src/package_table.rs
+++ b/tools/aconfig/aconfig_storage_file/src/package_table.rs
@@ -146,7 +146,7 @@
/// construction side (aconfig binary) and consumption side (flag read lib)
/// use the same method of hashing
pub fn find_bucket_index(package: &str, num_buckets: u32) -> u32 {
- get_bucket_index(&package, num_buckets)
+ get_bucket_index(package.as_bytes(), num_buckets)
}
}
diff --git a/tools/aconfig/aconfig_storage_file/src/sip_hasher13.rs b/tools/aconfig/aconfig_storage_file/src/sip_hasher13.rs
new file mode 100644
index 0000000..9be3175
--- /dev/null
+++ b/tools/aconfig/aconfig_storage_file/src/sip_hasher13.rs
@@ -0,0 +1,327 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//! An implementation of SipHash13
+
+use std::cmp;
+use std::mem;
+use std::ptr;
+use std::slice;
+
+use std::hash::Hasher;
+
+/// An implementation of SipHash 2-4.
+///
+#[derive(Debug, Clone, Default)]
+pub struct SipHasher13 {
+ k0: u64,
+ k1: u64,
+ length: usize, // how many bytes we've processed
+ state: State, // hash State
+ tail: u64, // unprocessed bytes le
+ ntail: usize, // how many bytes in tail are valid
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+#[repr(C)]
+struct State {
+ // v0, v2 and v1, v3 show up in pairs in the algorithm,
+ // and simd implementations of SipHash will use vectors
+ // of v02 and v13. By placing them in this order in the struct,
+ // the compiler can pick up on just a few simd optimizations by itself.
+ v0: u64,
+ v2: u64,
+ v1: u64,
+ v3: u64,
+}
+
+macro_rules! compress {
+ ($state:expr) => {{
+ compress!($state.v0, $state.v1, $state.v2, $state.v3)
+ }};
+ ($v0:expr, $v1:expr, $v2:expr, $v3:expr) => {{
+ $v0 = $v0.wrapping_add($v1);
+ $v1 = $v1.rotate_left(13);
+ $v1 ^= $v0;
+ $v0 = $v0.rotate_left(32);
+ $v2 = $v2.wrapping_add($v3);
+ $v3 = $v3.rotate_left(16);
+ $v3 ^= $v2;
+ $v0 = $v0.wrapping_add($v3);
+ $v3 = $v3.rotate_left(21);
+ $v3 ^= $v0;
+ $v2 = $v2.wrapping_add($v1);
+ $v1 = $v1.rotate_left(17);
+ $v1 ^= $v2;
+ $v2 = $v2.rotate_left(32);
+ }};
+}
+
+/// Load an integer of the desired type from a byte stream, in LE order. Uses
+/// `copy_nonoverlapping` to let the compiler generate the most efficient way
+/// to load it from a possibly unaligned address.
+///
+/// Unsafe because: unchecked indexing at i..i+size_of(int_ty)
+macro_rules! load_int_le {
+ ($buf:expr, $i:expr, $int_ty:ident) => {{
+ debug_assert!($i + mem::size_of::<$int_ty>() <= $buf.len());
+ let mut data = 0 as $int_ty;
+ ptr::copy_nonoverlapping(
+ $buf.get_unchecked($i),
+ &mut data as *mut _ as *mut u8,
+ mem::size_of::<$int_ty>(),
+ );
+ data.to_le()
+ }};
+}
+
+/// Load an u64 using up to 7 bytes of a byte slice.
+///
+/// Unsafe because: unchecked indexing at start..start+len
+#[inline]
+unsafe fn u8to64_le(buf: &[u8], start: usize, len: usize) -> u64 {
+ debug_assert!(len < 8);
+ let mut i = 0; // current byte index (from LSB) in the output u64
+ let mut out = 0;
+ if i + 3 < len {
+ out = load_int_le!(buf, start + i, u32) as u64;
+ i += 4;
+ }
+ if i + 1 < len {
+ out |= (load_int_le!(buf, start + i, u16) as u64) << (i * 8);
+ i += 2
+ }
+ if i < len {
+ out |= (*buf.get_unchecked(start + i) as u64) << (i * 8);
+ i += 1;
+ }
+ debug_assert_eq!(i, len);
+ out
+}
+
+impl SipHasher13 {
+ /// Creates a new `SipHasher13` with the two initial keys set to 0.
+ #[inline]
+ pub fn new() -> SipHasher13 {
+ SipHasher13::new_with_keys(0, 0)
+ }
+
+ /// Creates a `SipHasher13` that is keyed off the provided keys.
+ #[inline]
+ pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher13 {
+ let mut sip_hasher = SipHasher13 {
+ k0: key0,
+ k1: key1,
+ length: 0,
+ state: State { v0: 0, v1: 0, v2: 0, v3: 0 },
+ tail: 0,
+ ntail: 0,
+ };
+ sip_hasher.reset();
+ sip_hasher
+ }
+
+ #[inline]
+ fn c_rounds(state: &mut State) {
+ compress!(state);
+ }
+
+ #[inline]
+ fn d_rounds(state: &mut State) {
+ compress!(state);
+ compress!(state);
+ compress!(state);
+ }
+
+ #[inline]
+ fn reset(&mut self) {
+ self.length = 0;
+ self.state.v0 = self.k0 ^ 0x736f6d6570736575;
+ self.state.v1 = self.k1 ^ 0x646f72616e646f6d;
+ self.state.v2 = self.k0 ^ 0x6c7967656e657261;
+ self.state.v3 = self.k1 ^ 0x7465646279746573;
+ self.ntail = 0;
+ }
+
+ // Specialized write function that is only valid for buffers with len <= 8.
+ // It's used to force inlining of write_u8 and write_usize, those would normally be inlined
+ // except for composite types (that includes slices and str hashing because of delimiter).
+ // Without this extra push the compiler is very reluctant to inline delimiter writes,
+ // degrading performance substantially for the most common use cases.
+ #[inline]
+ fn short_write(&mut self, msg: &[u8]) {
+ debug_assert!(msg.len() <= 8);
+ let length = msg.len();
+ self.length += length;
+
+ let needed = 8 - self.ntail;
+ let fill = cmp::min(length, needed);
+ if fill == 8 {
+ // safe to call since msg hasn't been loaded
+ self.tail = unsafe { load_int_le!(msg, 0, u64) };
+ } else {
+ // safe to call since msg hasn't been loaded, and fill <= msg.len()
+ self.tail |= unsafe { u8to64_le(msg, 0, fill) } << (8 * self.ntail);
+ if length < needed {
+ self.ntail += length;
+ return;
+ }
+ }
+ self.state.v3 ^= self.tail;
+ Self::c_rounds(&mut self.state);
+ self.state.v0 ^= self.tail;
+
+ // Buffered tail is now flushed, process new input.
+ self.ntail = length - needed;
+ // safe to call since number of `needed` bytes has been loaded
+ // and self.ntail + needed == msg.len()
+ self.tail = unsafe { u8to64_le(msg, needed, self.ntail) };
+ }
+}
+
+impl Hasher for SipHasher13 {
+ // see short_write comment for explanation
+ #[inline]
+ fn write_usize(&mut self, i: usize) {
+ // safe to call, since convert the pointer to u8
+ let bytes = unsafe {
+ slice::from_raw_parts(&i as *const usize as *const u8, mem::size_of::<usize>())
+ };
+ self.short_write(bytes);
+ }
+
+ // see short_write comment for explanation
+ #[inline]
+ fn write_u8(&mut self, i: u8) {
+ self.short_write(&[i]);
+ }
+
+ #[inline]
+ fn write(&mut self, msg: &[u8]) {
+ let length = msg.len();
+ self.length += length;
+
+ let mut needed = 0;
+
+ // loading unprocessed byte from last write
+ if self.ntail != 0 {
+ needed = 8 - self.ntail;
+ // safe to call, since msg hasn't been processed
+ // and cmp::min(length, needed) < 8
+ self.tail |= unsafe { u8to64_le(msg, 0, cmp::min(length, needed)) } << 8 * self.ntail;
+ if length < needed {
+ self.ntail += length;
+ return;
+ } else {
+ self.state.v3 ^= self.tail;
+ Self::c_rounds(&mut self.state);
+ self.state.v0 ^= self.tail;
+ self.ntail = 0;
+ }
+ }
+
+ // Buffered tail is now flushed, process new input.
+ let len = length - needed;
+ let left = len & 0x7;
+
+ let mut i = needed;
+ while i < len - left {
+ // safe to call since if i < len - left, it means msg has at least 1 byte to load
+ let mi = unsafe { load_int_le!(msg, i, u64) };
+
+ self.state.v3 ^= mi;
+ Self::c_rounds(&mut self.state);
+ self.state.v0 ^= mi;
+
+ i += 8;
+ }
+
+ // safe to call since if left == 0, since this call will load nothing
+ // if left > 0, it means there are number of `left` bytes in msg
+ self.tail = unsafe { u8to64_le(msg, i, left) };
+ self.ntail = left;
+ }
+
+ #[inline]
+ fn finish(&self) -> u64 {
+ let mut state = self.state;
+
+ let b: u64 = ((self.length as u64 & 0xff) << 56) | self.tail;
+
+ state.v3 ^= b;
+ Self::c_rounds(&mut state);
+ state.v0 ^= b;
+
+ state.v2 ^= 0xff;
+ Self::d_rounds(&mut state);
+
+ state.v0 ^ state.v1 ^ state.v2 ^ state.v3
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ use std::hash::{Hash, Hasher};
+ use std::string::String;
+
+ #[test]
+ // this test point locks down the value list serialization
+ fn test_sip_hash13_string_hash() {
+ let mut sip_hash13 = SipHasher13::new();
+ let test_str1 = String::from("com.google.android.test");
+ test_str1.hash(&mut sip_hash13);
+ assert_eq!(17898838669067067585, sip_hash13.finish());
+
+ let test_str2 = String::from("adfadfadf adfafadadf 1231241241");
+ test_str2.hash(&mut sip_hash13);
+ assert_eq!(13543518987672889310, sip_hash13.finish());
+ }
+
+ #[test]
+ fn test_sip_hash13_write() {
+ let mut sip_hash13 = SipHasher13::new();
+ let test_str1 = String::from("com.google.android.test");
+ sip_hash13.write(test_str1.as_bytes());
+ sip_hash13.write_u8(0xff);
+ assert_eq!(17898838669067067585, sip_hash13.finish());
+
+ let mut sip_hash132 = SipHasher13::new();
+ let test_str1 = String::from("com.google.android.test");
+ sip_hash132.write(test_str1.as_bytes());
+ assert_eq!(9685440969685209025, sip_hash132.finish());
+ sip_hash132.write(test_str1.as_bytes());
+ assert_eq!(6719694176662736568, sip_hash132.finish());
+
+ let mut sip_hash133 = SipHasher13::new();
+ let test_str2 = String::from("abcdefg");
+ test_str2.hash(&mut sip_hash133);
+ assert_eq!(2492161047327640297, sip_hash133.finish());
+
+ let mut sip_hash134 = SipHasher13::new();
+ let test_str3 = String::from("abcdefgh");
+ test_str3.hash(&mut sip_hash134);
+ assert_eq!(6689927370435554326, sip_hash134.finish());
+ }
+
+ #[test]
+ fn test_sip_hash13_write_short() {
+ let mut sip_hash13 = SipHasher13::new();
+ sip_hash13.write_u8(0x61);
+ assert_eq!(4644417185603328019, sip_hash13.finish());
+ }
+}
diff --git a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FlagTable.java b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FlagTable.java
index e85fdee..9838a7c 100644
--- a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FlagTable.java
+++ b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/FlagTable.java
@@ -16,41 +16,50 @@
package android.aconfig.storage;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
import java.util.Objects;
public class FlagTable {
private Header mHeader;
- private Map<String, Node> mNodeMap;
+ private ByteBufferReader mReader;
public static FlagTable fromBytes(ByteBuffer bytes) {
FlagTable flagTable = new FlagTable();
- ByteBufferReader reader = new ByteBufferReader(bytes);
- Header header = Header.fromBytes(reader);
- flagTable.mHeader = header;
- flagTable.mNodeMap = new HashMap(TableUtils.getTableSize(header.mNumFlags));
- reader.position(header.mNodeOffset);
- for (int i = 0; i < header.mNumFlags; i++) {
- Node node = Node.fromBytes(reader);
- flagTable.mNodeMap.put(makeKey(node.mPackageId, node.mFlagName), node);
- }
+ flagTable.mReader = new ByteBufferReader(bytes);
+ flagTable.mHeader = Header.fromBytes(flagTable.mReader);
+
return flagTable;
}
public Node get(int packageId, String flagName) {
- return mNodeMap.get(makeKey(packageId, flagName));
+ int numBuckets = (mHeader.mNodeOffset - mHeader.mBucketOffset) / 4;
+ int bucketIndex = TableUtils.getBucketIndex(makeKey(packageId, flagName), numBuckets);
+
+ mReader.position(mHeader.mBucketOffset + bucketIndex * 4);
+ int nodeIndex = mReader.readInt();
+
+ while (nodeIndex != -1) {
+ mReader.position(nodeIndex);
+ Node node = Node.fromBytes(mReader);
+ if (Objects.equals(flagName, node.mFlagName) && packageId == node.mPackageId) {
+ return node;
+ }
+ nodeIndex = node.mNextOffset;
+ }
+
+ throw new AconfigStorageException("get cannot find flag: " + flagName);
}
public Header getHeader() {
return mHeader;
}
- private static String makeKey(int packageId, String flagName) {
+ private static byte[] makeKey(int packageId, String flagName) {
StringBuilder ret = new StringBuilder();
- return ret.append(packageId).append('/').append(flagName).toString();
+ return ret.append(packageId).append('/').append(flagName).toString().getBytes(UTF_8);
}
public static class Header {
diff --git a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/PackageTable.java b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/PackageTable.java
index 7ef947d..773b882 100644
--- a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/PackageTable.java
+++ b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/PackageTable.java
@@ -16,6 +16,8 @@
package android.aconfig.storage;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.nio.ByteBuffer;
import java.util.Objects;
@@ -33,13 +35,22 @@
}
public Node get(String packageName) {
- mReader.position(mHeader.mNodeOffset);
- for (int i = 0; i < mHeader.mNumPackages; i++) {
+
+ int numBuckets = (mHeader.mNodeOffset - mHeader.mBucketOffset) / 4;
+ int bucketIndex = TableUtils.getBucketIndex(packageName.getBytes(UTF_8), numBuckets);
+
+ mReader.position(mHeader.mBucketOffset + bucketIndex * 4);
+ int nodeIndex = mReader.readInt();
+
+ while (nodeIndex != -1) {
+ mReader.position(nodeIndex);
Node node = Node.fromBytes(mReader);
- if (Objects.equals(node.mPackageName, packageName)) {
+ if (Objects.equals(packageName, node.mPackageName)) {
return node;
}
+ nodeIndex = node.mNextOffset;
}
+
throw new AconfigStorageException("get cannot find package: " + packageName);
}
diff --git a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/SipHasher13.java b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/SipHasher13.java
new file mode 100644
index 0000000..64714ee
--- /dev/null
+++ b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/SipHasher13.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.aconfig.storage;
+
+public class SipHasher13 {
+ static class State {
+ private long v0;
+ private long v2;
+ private long v1;
+ private long v3;
+
+ public State(long k0, long k1) {
+ v0 = k0 ^ 0x736f6d6570736575L;
+ v1 = k1 ^ 0x646f72616e646f6dL;
+ v2 = k0 ^ 0x6c7967656e657261L;
+ v3 = k1 ^ 0x7465646279746573L;
+ }
+
+ public void compress(long m) {
+ v3 ^= m;
+ cRounds();
+ v0 ^= m;
+ }
+
+ public long finish() {
+ v2 ^= 0xff;
+ dRounds();
+ return v0 ^ v1 ^ v2 ^ v3;
+ }
+
+ private void cRounds() {
+ v0 += v1;
+ v1 = Long.rotateLeft(v1, 13);
+ v1 ^= v0;
+ v0 = Long.rotateLeft(v0, 32);
+ v2 += v3;
+ v3 = Long.rotateLeft(v3, 16);
+ v3 ^= v2;
+ v0 += v3;
+ v3 = Long.rotateLeft(v3, 21);
+ v3 ^= v0;
+ v2 += v1;
+ v1 = Long.rotateLeft(v1, 17);
+ v1 ^= v2;
+ v2 = Long.rotateLeft(v2, 32);
+ }
+
+ private void dRounds() {
+ for (int i = 0; i < 3; i++) {
+ v0 += v1;
+ v1 = Long.rotateLeft(v1, 13);
+ v1 ^= v0;
+ v0 = Long.rotateLeft(v0, 32);
+ v2 += v3;
+ v3 = Long.rotateLeft(v3, 16);
+ v3 ^= v2;
+ v0 += v3;
+ v3 = Long.rotateLeft(v3, 21);
+ v3 ^= v0;
+ v2 += v1;
+ v1 = Long.rotateLeft(v1, 17);
+ v1 ^= v2;
+ v2 = Long.rotateLeft(v2, 32);
+ }
+ }
+ }
+
+ public static long hash(byte[] data) {
+ State state = new State(0, 0);
+ int len = data.length;
+ int left = len & 0x7;
+ int index = 0;
+
+ while (index < len - left) {
+ long mi = loadLe(data, index, 8);
+ index += 8;
+ state.compress(mi);
+ }
+
+ // padding the end with 0xff to be consistent with rust
+ long m = (0xffL << (left * 8)) | loadLe(data, index, left);
+ if (left == 0x7) {
+ // compress the m w-2
+ state.compress(m);
+ m = 0L;
+ }
+ // len adds 1 since padded 0xff
+ m |= (((len + 1) & 0xffL) << 56);
+ state.compress(m);
+
+ return state.finish();
+ }
+
+ private static long loadLe(byte[] data, int offset, int size) {
+ long m = 0;
+ for (int i = 0; i < size; i++) {
+ m |= (data[i + offset] & 0xffL) << (i * 8);
+ }
+ return m;
+ }
+}
diff --git a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/TableUtils.java b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/TableUtils.java
index 714b53b..81168f5 100644
--- a/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/TableUtils.java
+++ b/tools/aconfig/aconfig_storage_file/srcs/android/aconfig/storage/TableUtils.java
@@ -58,4 +58,9 @@
}
throw new AconfigStorageException("Number of items in a hash table exceeds limit");
}
+
+ public static int getBucketIndex(byte[] val, int numBuckets) {
+ long hashVal = SipHasher13.hash(val);
+ return (int) Long.remainderUnsigned(hashVal, numBuckets);
+ }
}
diff --git a/tools/aconfig/aconfig_storage_file/tests/Android.bp b/tools/aconfig/aconfig_storage_file/tests/Android.bp
index e2e225d..12e4aca 100644
--- a/tools/aconfig/aconfig_storage_file/tests/Android.bp
+++ b/tools/aconfig/aconfig_storage_file/tests/Android.bp
@@ -30,9 +30,10 @@
static_libs: [
"androidx.test.runner",
"junit",
+ "aconfig_storage_file_java",
],
test_config: "AndroidStorageJaveTest.xml",
- certificate: "platform",
+ sdk_version: "test_current",
data: [
"package.map",
"flag.map",
@@ -42,4 +43,5 @@
test_suites: [
"general-tests",
],
+ jarjar_rules: "jarjar.txt",
}
diff --git a/tools/aconfig/aconfig_storage_file/tests/jarjar.txt b/tools/aconfig/aconfig_storage_file/tests/jarjar.txt
new file mode 100644
index 0000000..a6c17fa
--- /dev/null
+++ b/tools/aconfig/aconfig_storage_file/tests/jarjar.txt
@@ -0,0 +1,15 @@
+rule android.aconfig.storage.AconfigStorageException android.aconfig.storage.test.AconfigStorageException
+rule android.aconfig.storage.FlagTable android.aconfig.storage.test.FlagTable
+rule android.aconfig.storage.PackageTable android.aconfig.storage.test.PackageTable
+rule android.aconfig.storage.ByteBufferReader android.aconfig.storage.test.ByteBufferReader
+rule android.aconfig.storage.FlagType android.aconfig.storage.test.FlagType
+rule android.aconfig.storage.SipHasher13 android.aconfig.storage.test.SipHasher13
+rule android.aconfig.storage.FileType android.aconfig.storage.test.FileType
+rule android.aconfig.storage.FlagValueList android.aconfig.storage.test.FlagValueList
+rule android.aconfig.storage.TableUtils android.aconfig.storage.test.TableUtils
+
+
+rule android.aconfig.storage.FlagTable$* android.aconfig.storage.test.FlagTable$@1
+rule android.aconfig.storage.PackageTable$* android.aconfig.storage.test.PackageTable$@1
+rule android.aconfig.storage.FlagValueList$* android.aconfig.storage.test.FlagValueList@1
+rule android.aconfig.storage.SipHasher13$* android.aconfig.storage.test.SipHasher13@1
diff --git a/tools/aconfig/aconfig_storage_file/tests/srcs/SipHasher13Test.java b/tools/aconfig/aconfig_storage_file/tests/srcs/SipHasher13Test.java
new file mode 100644
index 0000000..10620d2
--- /dev/null
+++ b/tools/aconfig/aconfig_storage_file/tests/srcs/SipHasher13Test.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.aconfig.storage.test;
+
+import static org.junit.Assert.assertEquals;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import android.aconfig.storage.SipHasher13;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class SipHasher13Test {
+ @Test
+ public void testSipHash_hashString() throws Exception {
+ String testStr = "com.google.android.test";
+ long result = SipHasher13.hash(testStr.getBytes(UTF_8));
+ assertEquals(0xF86572EFF9C4A0C1L, result);
+
+ testStr = "abcdefg";
+ result = SipHasher13.hash(testStr.getBytes(UTF_8));
+ assertEquals(0x2295EF44BD078AE9L, result);
+
+ testStr = "abcdefgh";
+ result = SipHasher13.hash(testStr.getBytes(UTF_8));
+ assertEquals(0x5CD7657FA7F96C16L, result);
+ }
+}
diff --git a/tools/aconfig/aconfig_storage_read_api/Android.bp b/tools/aconfig/aconfig_storage_read_api/Android.bp
index 9e950a6..f96b223 100644
--- a/tools/aconfig/aconfig_storage_read_api/Android.bp
+++ b/tools/aconfig/aconfig_storage_read_api/Android.bp
@@ -147,6 +147,7 @@
crate_name: "aconfig_storage_read_api_rust_jni",
srcs: ["srcs/lib.rs"],
rustlibs: [
+ "libaconfig_storage_file",
"libaconfig_storage_read_api",
"libanyhow",
"libjni",
diff --git a/tools/aconfig/aconfig_storage_read_api/srcs/android/aconfig/storage/AconfigStorageReadAPI.java b/tools/aconfig/aconfig_storage_read_api/srcs/android/aconfig/storage/AconfigStorageReadAPI.java
index 406ff24..850c2b8 100644
--- a/tools/aconfig/aconfig_storage_read_api/srcs/android/aconfig/storage/AconfigStorageReadAPI.java
+++ b/tools/aconfig/aconfig_storage_read_api/srcs/android/aconfig/storage/AconfigStorageReadAPI.java
@@ -16,18 +16,14 @@
package android.aconfig.storage;
+import dalvik.annotation.optimization.FastNative;
+
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
-import java.nio.channels.FileChannel.MapMode;
-
-import android.aconfig.storage.PackageReadContext;
-import android.aconfig.storage.FlagReadContext;
-
-import dalvik.annotation.optimization.FastNative;
public class AconfigStorageReadAPI {
@@ -50,9 +46,8 @@
}
// Map a storage file given container and file type
- public static MappedByteBuffer getMappedFile(
- String container,
- StorageFileType type) throws IOException{
+ public static MappedByteBuffer getMappedFile(String container, StorageFileType type)
+ throws IOException {
switch (type) {
case PACKAGE_MAP:
return mapStorageFile(STORAGEDIR + "/maps/" + container + ".package.map");
@@ -73,14 +68,14 @@
// @throws IOException if the passed in file is not a valid package map file
@FastNative
private static native ByteBuffer getPackageReadContextImpl(
- ByteBuffer mappedFile, String packageName) throws IOException;
+ ByteBuffer mappedFile, String packageName) throws IOException;
// API to get package read context
// @param mappedFile: memory mapped package map file
// @param packageName: package name
// @throws IOException if the passed in file is not a valid package map file
- static public PackageReadContext getPackageReadContext (
- ByteBuffer mappedFile, String packageName) throws IOException {
+ public static PackageReadContext getPackageReadContext(
+ ByteBuffer mappedFile, String packageName) throws IOException {
ByteBuffer buffer = getPackageReadContextImpl(mappedFile, packageName);
buffer.order(ByteOrder.LITTLE_ENDIAN);
return new PackageReadContext(buffer.getInt(), buffer.getInt(4));
@@ -94,7 +89,7 @@
// @throws IOException if the passed in file is not a valid flag map file
@FastNative
private static native ByteBuffer getFlagReadContextImpl(
- ByteBuffer mappedFile, int packageId, String flagName) throws IOException;
+ ByteBuffer mappedFile, int packageId, String flagName) throws IOException;
// API to get flag read context
// @param mappedFile: memory mapped flag map file
@@ -103,7 +98,7 @@
// @param flagName: flag name
// @throws IOException if the passed in file is not a valid flag map file
public static FlagReadContext getFlagReadContext(
- ByteBuffer mappedFile, int packageId, String flagName) throws IOException {
+ ByteBuffer mappedFile, int packageId, String flagName) throws IOException {
ByteBuffer buffer = getFlagReadContextImpl(mappedFile, packageId, flagName);
buffer.order(ByteOrder.LITTLE_ENDIAN);
return new FlagReadContext(buffer.getInt(), buffer.getInt(4));
@@ -115,8 +110,11 @@
// @throws IOException if the passed in file is not a valid flag value file or the
// flag index went over the file boundary.
@FastNative
- public static native boolean getBooleanFlagValue(
- ByteBuffer mappedFile, int flagIndex) throws IOException;
+ public static native boolean getBooleanFlagValue(ByteBuffer mappedFile, int flagIndex)
+ throws IOException;
+
+ @FastNative
+ public static native long hash(String packageName) throws IOException;
static {
System.loadLibrary("aconfig_storage_read_api_rust_jni");
diff --git a/tools/aconfig/aconfig_storage_read_api/srcs/lib.rs b/tools/aconfig/aconfig_storage_read_api/srcs/lib.rs
index 304a059..f5f12bb 100644
--- a/tools/aconfig/aconfig_storage_read_api/srcs/lib.rs
+++ b/tools/aconfig/aconfig_storage_read_api/srcs/lib.rs
@@ -1,5 +1,6 @@
//! aconfig storage read api java rust interlop
+use aconfig_storage_file::SipHasher13;
use aconfig_storage_read_api::flag_table_query::find_flag_read_context;
use aconfig_storage_read_api::flag_value_query::find_boolean_flag_value;
use aconfig_storage_read_api::package_table_query::find_package_read_context;
@@ -7,8 +8,9 @@
use anyhow::Result;
use jni::objects::{JByteBuffer, JClass, JString};
-use jni::sys::{jboolean, jint};
+use jni::sys::{jboolean, jint, jlong};
use jni::JNIEnv;
+use std::hash::Hasher;
/// Call rust find package read context
fn get_package_read_context_java(
@@ -158,3 +160,30 @@
}
}
}
+
+/// Get flag value JNI
+#[no_mangle]
+#[allow(unused)]
+pub extern "system" fn Java_android_aconfig_storage_AconfigStorageReadAPI_hash<'local>(
+ mut env: JNIEnv<'local>,
+ class: JClass<'local>,
+ package_name: JString<'local>,
+) -> jlong {
+ match siphasher13_hash(&mut env, package_name) {
+ Ok(value) => value as jlong,
+ Err(errmsg) => {
+ env.throw(("java/io/IOException", errmsg.to_string())).expect("failed to throw");
+ 0i64
+ }
+ }
+}
+
+fn siphasher13_hash(env: &mut JNIEnv, package_name: JString) -> Result<u64> {
+ // SAFETY:
+ // The safety here is ensured as the flag name is guaranteed to be a java string
+ let flag_name: String = unsafe { env.get_string_unchecked(&package_name)?.into() };
+ let mut s = SipHasher13::new();
+ s.write(flag_name.as_bytes());
+ s.write_u8(0xff);
+ Ok(s.finish())
+}
diff --git a/tools/aconfig/aconfig_storage_read_api/tests/java/AconfigStorageReadAPITest.java b/tools/aconfig/aconfig_storage_read_api/tests/java/AconfigStorageReadAPITest.java
index a26b257..191741e 100644
--- a/tools/aconfig/aconfig_storage_read_api/tests/java/AconfigStorageReadAPITest.java
+++ b/tools/aconfig/aconfig_storage_read_api/tests/java/AconfigStorageReadAPITest.java
@@ -16,28 +16,29 @@
package android.aconfig.storage.test;
-import java.io.IOException;
-import java.nio.MappedByteBuffer;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+
+import android.aconfig.DeviceProtos;
+import android.aconfig.nano.Aconfig.parsed_flag;
+import android.aconfig.storage.AconfigStorageReadAPI;
+import android.aconfig.storage.FlagReadContext;
+import android.aconfig.storage.FlagReadContext.StoredFlagType;
+import android.aconfig.storage.PackageReadContext;
+import android.aconfig.storage.SipHasher13;
+import android.aconfig.storage.StorageInternalReader;
+
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-import android.aconfig.storage.AconfigStorageReadAPI;
-import android.aconfig.storage.PackageReadContext;
-import android.aconfig.storage.FlagReadContext;
-import android.aconfig.storage.FlagReadContext.StoredFlagType;
+import java.io.IOException;
+import java.nio.MappedByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
@RunWith(JUnit4.class)
-public class AconfigStorageReadAPITest{
+public class AconfigStorageReadAPITest {
private String mStorageDir = "/data/local/tmp/aconfig_java_api_test";
@@ -45,26 +46,29 @@
public void testPackageContextQuery() {
MappedByteBuffer packageMap = null;
try {
- packageMap = AconfigStorageReadAPI.mapStorageFile(
- mStorageDir + "/maps/mockup.package.map");
- } catch(IOException ex){
+ packageMap =
+ AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/maps/mockup.package.map");
+ } catch (IOException ex) {
assertTrue(ex.toString(), false);
}
assertTrue(packageMap != null);
try {
- PackageReadContext context = AconfigStorageReadAPI.getPackageReadContext(
- packageMap, "com.android.aconfig.storage.test_1");
+ PackageReadContext context =
+ AconfigStorageReadAPI.getPackageReadContext(
+ packageMap, "com.android.aconfig.storage.test_1");
assertEquals(context.mPackageId, 0);
assertEquals(context.mBooleanStartIndex, 0);
- context = AconfigStorageReadAPI.getPackageReadContext(
- packageMap, "com.android.aconfig.storage.test_2");
+ context =
+ AconfigStorageReadAPI.getPackageReadContext(
+ packageMap, "com.android.aconfig.storage.test_2");
assertEquals(context.mPackageId, 1);
assertEquals(context.mBooleanStartIndex, 3);
- context = AconfigStorageReadAPI.getPackageReadContext(
- packageMap, "com.android.aconfig.storage.test_4");
+ context =
+ AconfigStorageReadAPI.getPackageReadContext(
+ packageMap, "com.android.aconfig.storage.test_4");
assertEquals(context.mPackageId, 2);
assertEquals(context.mBooleanStartIndex, 6);
} catch (IOException ex) {
@@ -76,19 +80,19 @@
public void testNonExistPackageContextQuery() {
MappedByteBuffer packageMap = null;
try {
- packageMap = AconfigStorageReadAPI.mapStorageFile(
- mStorageDir + "/maps/mockup.package.map");
- } catch(IOException ex){
+ packageMap =
+ AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/maps/mockup.package.map");
+ } catch (IOException ex) {
assertTrue(ex.toString(), false);
}
assertTrue(packageMap != null);
try {
- PackageReadContext context = AconfigStorageReadAPI.getPackageReadContext(
- packageMap, "unknown");
+ PackageReadContext context =
+ AconfigStorageReadAPI.getPackageReadContext(packageMap, "unknown");
assertEquals(context.mPackageId, -1);
assertEquals(context.mBooleanStartIndex, -1);
- } catch(IOException ex){
+ } catch (IOException ex) {
assertTrue(ex.toString(), false);
}
}
@@ -97,12 +101,11 @@
public void testFlagContextQuery() {
MappedByteBuffer flagMap = null;
try {
- flagMap = AconfigStorageReadAPI.mapStorageFile(
- mStorageDir + "/maps/mockup.flag.map");
- } catch(IOException ex){
+ flagMap = AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/maps/mockup.flag.map");
+ } catch (IOException ex) {
assertTrue(ex.toString(), false);
}
- assertTrue(flagMap!= null);
+ assertTrue(flagMap != null);
class Baseline {
public int mPackageId;
@@ -110,10 +113,8 @@
public StoredFlagType mFlagType;
public int mFlagIndex;
- public Baseline(int packageId,
- String flagName,
- StoredFlagType flagType,
- int flagIndex) {
+ public Baseline(
+ int packageId, String flagName, StoredFlagType flagType, int flagIndex) {
mPackageId = packageId;
mFlagName = flagName;
mFlagType = flagType;
@@ -133,8 +134,9 @@
try {
for (Baseline baseline : baselines) {
- FlagReadContext context = AconfigStorageReadAPI.getFlagReadContext(
- flagMap, baseline.mPackageId, baseline.mFlagName);
+ FlagReadContext context =
+ AconfigStorageReadAPI.getFlagReadContext(
+ flagMap, baseline.mPackageId, baseline.mFlagName);
assertEquals(context.mFlagType, baseline.mFlagType);
assertEquals(context.mFlagIndex, baseline.mFlagIndex);
}
@@ -147,21 +149,19 @@
public void testNonExistFlagContextQuery() {
MappedByteBuffer flagMap = null;
try {
- flagMap = AconfigStorageReadAPI.mapStorageFile(
- mStorageDir + "/maps/mockup.flag.map");
- } catch(IOException ex){
+ flagMap = AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/maps/mockup.flag.map");
+ } catch (IOException ex) {
assertTrue(ex.toString(), false);
}
- assertTrue(flagMap!= null);
+ assertTrue(flagMap != null);
try {
- FlagReadContext context = AconfigStorageReadAPI.getFlagReadContext(
- flagMap, 0, "unknown");
+ FlagReadContext context =
+ AconfigStorageReadAPI.getFlagReadContext(flagMap, 0, "unknown");
assertEquals(context.mFlagType, null);
assertEquals(context.mFlagIndex, -1);
- context = AconfigStorageReadAPI.getFlagReadContext(
- flagMap, 3, "enabled_ro");
+ context = AconfigStorageReadAPI.getFlagReadContext(flagMap, 3, "enabled_ro");
assertEquals(context.mFlagType, null);
assertEquals(context.mFlagIndex, -1);
} catch (IOException ex) {
@@ -173,12 +173,11 @@
public void testBooleanFlagValueQuery() {
MappedByteBuffer flagVal = null;
try {
- flagVal = AconfigStorageReadAPI.mapStorageFile(
- mStorageDir + "/boot/mockup.val");
+ flagVal = AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/boot/mockup.val");
} catch (IOException ex) {
assertTrue(ex.toString(), false);
}
- assertTrue(flagVal!= null);
+ assertTrue(flagVal != null);
boolean[] baselines = {false, true, true, false, true, true, true, true};
for (int i = 0; i < 8; ++i) {
@@ -195,12 +194,11 @@
public void testInvalidBooleanFlagValueQuery() {
MappedByteBuffer flagVal = null;
try {
- flagVal = AconfigStorageReadAPI.mapStorageFile(
- mStorageDir + "/boot/mockup.val");
+ flagVal = AconfigStorageReadAPI.mapStorageFile(mStorageDir + "/boot/mockup.val");
} catch (IOException ex) {
assertTrue(ex.toString(), false);
}
- assertTrue(flagVal!= null);
+ assertTrue(flagVal != null);
try {
Boolean value = AconfigStorageReadAPI.getBooleanFlagValue(flagVal, 9);
@@ -210,4 +208,63 @@
assertTrue(ex.toString(), ex.toString().contains(expectedErrmsg));
}
}
- }
+
+ @Test
+ public void testRustJavaEqualHash() throws IOException {
+ List<parsed_flag> flags = DeviceProtos.loadAndParseFlagProtos();
+ for (parsed_flag flag : flags) {
+ String packageName = flag.package_;
+ String flagName = flag.name;
+ long rHash = AconfigStorageReadAPI.hash(packageName);
+ long jHash = SipHasher13.hash(packageName.getBytes());
+ assertEquals(rHash, jHash);
+
+ String fullFlagName = packageName + "/" + flagName;
+ rHash = AconfigStorageReadAPI.hash(fullFlagName);
+ jHash = SipHasher13.hash(fullFlagName.getBytes());
+ assertEquals(rHash, jHash);
+ }
+ }
+
+ @Test
+ public void testRustJavaEqualFlag() throws IOException {
+ List<parsed_flag> flags = DeviceProtos.loadAndParseFlagProtos();
+
+ String mapPath = "/metadata/aconfig/maps/";
+ String flagsPath = "/metadata/aconfig/boot/";
+
+ for (parsed_flag flag : flags) {
+
+ String container = flag.container;
+ String packageName = flag.package_;
+ String flagName = flag.name;
+ String fullFlagName = packageName + "/" + flagName;
+
+ MappedByteBuffer packageMap =
+ AconfigStorageReadAPI.mapStorageFile(mapPath + container + ".package.map");
+ MappedByteBuffer flagMap =
+ AconfigStorageReadAPI.mapStorageFile(mapPath + container + ".flag.map");
+ MappedByteBuffer flagValList =
+ AconfigStorageReadAPI.mapStorageFile(flagsPath + container + ".val");
+
+ PackageReadContext packageContext =
+ AconfigStorageReadAPI.getPackageReadContext(packageMap, packageName);
+
+ FlagReadContext flagContext =
+ AconfigStorageReadAPI.getFlagReadContext(
+ flagMap, packageContext.mPackageId, flagName);
+
+ boolean rVal =
+ AconfigStorageReadAPI.getBooleanFlagValue(
+ flagValList,
+ packageContext.mBooleanStartIndex + flagContext.mFlagIndex);
+
+ StorageInternalReader reader = new StorageInternalReader(container, packageName);
+ boolean jVal = reader.getBooleanFlagValue(flagContext.mFlagIndex);
+
+ long rHash = AconfigStorageReadAPI.hash(packageName);
+ long jHash = SipHasher13.hash(packageName.getBytes());
+ assertEquals(rVal, jVal);
+ }
+ }
+}
diff --git a/tools/aconfig/aconfig_storage_read_api/tests/java/Android.bp b/tools/aconfig/aconfig_storage_read_api/tests/java/Android.bp
index 11b3824..3d4e9ad 100644
--- a/tools/aconfig/aconfig_storage_read_api/tests/java/Android.bp
+++ b/tools/aconfig/aconfig_storage_read_api/tests/java/Android.bp
@@ -2,6 +2,8 @@
name: "aconfig_storage_read_api.test.java",
srcs: ["./**/*.java"],
static_libs: [
+ "aconfig_device_paths_java",
+ "aconfig_storage_file_java",
"aconfig_storage_reader_java",
"androidx.test.rules",
"libaconfig_storage_read_api_java",
diff --git a/tools/aconfig/aflags/Android.bp b/tools/aconfig/aflags/Android.bp
index c48585a..2040cc6 100644
--- a/tools/aconfig/aflags/Android.bp
+++ b/tools/aconfig/aflags/Android.bp
@@ -10,6 +10,7 @@
srcs: ["src/main.rs"],
rustlibs: [
"libaconfig_device_paths",
+ "libaconfig_flags",
"libaconfig_protos",
"libaconfigd_protos",
"libaconfig_storage_read_api",
@@ -24,6 +25,7 @@
rust_binary {
name: "aflags",
+ host_supported: true,
defaults: ["aflags.defaults"],
}
diff --git a/tools/aconfig/aflags/Cargo.toml b/tools/aconfig/aflags/Cargo.toml
index 7dc3436..7efce6d 100644
--- a/tools/aconfig/aflags/Cargo.toml
+++ b/tools/aconfig/aflags/Cargo.toml
@@ -15,3 +15,4 @@
aconfig_storage_read_api = { version = "0.1.0", path = "../aconfig_storage_read_api" }
clap = {version = "4.5.2" }
aconfig_device_paths = { version = "0.1.0", path = "../aconfig_device_paths" }
+aconfig_flags = { version = "0.1.0", path = "../aconfig_flags" }
\ No newline at end of file
diff --git a/tools/aconfig/aflags/src/aconfig_storage_source.rs b/tools/aconfig/aflags/src/aconfig_storage_source.rs
index a2c6012..b2fd3c9 100644
--- a/tools/aconfig/aflags/src/aconfig_storage_source.rs
+++ b/tools/aconfig/aflags/src/aconfig_storage_source.rs
@@ -62,7 +62,7 @@
permission,
value_picked_from,
staged_value,
- container: msg.container.ok_or(anyhow!("missing container"))?,
+ container: "-".to_string(),
// TODO: remove once DeviceConfig is not in the CLI.
namespace: "-".to_string(),
diff --git a/tools/aconfig/aflags/src/main.rs b/tools/aconfig/aflags/src/main.rs
index d8912a9..a726cc0 100644
--- a/tools/aconfig/aflags/src/main.rs
+++ b/tools/aconfig/aflags/src/main.rs
@@ -164,10 +164,6 @@
enum Command {
/// List all aconfig flags on this device.
List {
- /// Read from the new flag storage.
- #[clap(long)]
- use_new_storage: bool,
-
/// Optionally filter by container name.
#[clap(short = 'c', long = "container")]
container: Option<String>,
@@ -184,6 +180,9 @@
/// <package>.<flag_name>
qualified_name: String,
},
+
+ /// Display which flag storage backs aconfig flags.
+ WhichBacking,
}
struct PaddingInfo {
@@ -282,21 +281,31 @@
Ok(result)
}
+fn display_which_backing() -> String {
+ if aconfig_flags::auto_generated::enable_only_new_storage() {
+ "aconfig_storage".to_string()
+ } else {
+ "device_config".to_string()
+ }
+}
+
fn main() -> Result<()> {
ensure!(nix::unistd::Uid::current().is_root(), "must be root");
let cli = Cli::parse();
let output = match cli.command {
- Command::List { use_new_storage: true, container } => {
- list(FlagSourceType::AconfigStorage, container)
- .map_err(|err| anyhow!("storage may not be enabled: {err}"))
- .map(Some)
- }
- Command::List { use_new_storage: false, container } => {
- list(FlagSourceType::DeviceConfig, container).map(Some)
+ Command::List { container } => {
+ if aconfig_flags::auto_generated::enable_only_new_storage() {
+ list(FlagSourceType::AconfigStorage, container)
+ .map_err(|err| anyhow!("storage may not be enabled: {err}"))
+ .map(Some)
+ } else {
+ list(FlagSourceType::DeviceConfig, container).map(Some)
+ }
}
Command::Enable { qualified_name } => set_flag(&qualified_name, "true").map(|_| None),
Command::Disable { qualified_name } => set_flag(&qualified_name, "false").map(|_| None),
+ Command::WhichBacking => Ok(Some(display_which_backing())),
};
match output {
Ok(Some(text)) => println!("{text}"),
diff --git a/tools/filelistdiff/Android.bp b/tools/filelistdiff/Android.bp
index 632ada3..ab766d6 100644
--- a/tools/filelistdiff/Android.bp
+++ b/tools/filelistdiff/Android.bp
@@ -24,4 +24,4 @@
prebuilt_etc_host {
name: "system_image_diff_allowlist",
src: "allowlist",
-}
+}
\ No newline at end of file
diff --git a/tools/filelistdiff/allowlist b/tools/filelistdiff/allowlist
index abe35bf..c4a464d 100644
--- a/tools/filelistdiff/allowlist
+++ b/tools/filelistdiff/allowlist
@@ -1,27 +1,5 @@
# Known diffs only in the KATI system image
etc/NOTICE.xml.gz
-etc/compatconfig/TeleService-platform-compat-config.xml
-etc/compatconfig/calendar-provider-compat-config.xml
-etc/compatconfig/contacts-provider-platform-compat-config.xml
-etc/compatconfig/documents-ui-compat-config.xml
-etc/compatconfig/framework-location-compat-config.xml
-etc/compatconfig/framework-platform-compat-config.xml
-etc/compatconfig/icu4j-platform-compat-config.xml
-etc/compatconfig/services-platform-compat-config.xml
-etc/permissions/android.software.credentials.xml
-etc/permissions/android.software.preview_sdk.xml
-etc/permissions/android.software.webview.xml
-etc/permissions/android.software.window_magnification.xml
-etc/permissions/com.android.adservices.sdksandbox.xml
-etc/security/otacerts.zip
-etc/vintf/compatibility_matrix.202404.xml
-etc/vintf/compatibility_matrix.202504.xml
-etc/vintf/compatibility_matrix.5.xml
-etc/vintf/compatibility_matrix.6.xml
-etc/vintf/compatibility_matrix.7.xml
-etc/vintf/compatibility_matrix.8.xml
-etc/vintf/compatibility_matrix.device.xml
-etc/vintf/manifest.xml
framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex
framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex.fsv_meta
framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex
@@ -56,4 +34,16 @@
# Known diffs only in the Soong system image
lib/libhidcommand_jni.so
-lib/libuinputcommand_jni.so
\ No newline at end of file
+lib/libuinputcommand_jni.so
+
+# Known diffs in internal source
+bin/uprobestats
+etc/aconfig/flag.map
+etc/aconfig/flag.val
+etc/aconfig/package.map
+etc/bpf/uprobestats/BitmapAllocation.o
+etc/bpf/uprobestats/GenericInstrumentation.o
+etc/init/UprobeStats.rc
+lib/libuprobestats_client.so
+lib64/libuprobestats_client.so
+priv-app/DeviceDiagnostics/DeviceDiagnostics.apk
\ No newline at end of file
diff --git a/tools/finalization/README.md b/tools/finalization/README.md
index d0aed69..5e2aecd 100644
--- a/tools/finalization/README.md
+++ b/tools/finalization/README.md
@@ -19,3 +19,8 @@
## Utility:
[Full cleanup](./cleanup.sh). Remove all local changes and switch each project into head-less state. This is the best state to sync/rebase/finalize the branch.
+
+## Dry run:
+[Full cleanup](./dryrun-cleanup.sh). Remove all local changes and switch each project into head-less state. Also removes "DryRun" branches.
+[SDK](./dryrun-step-1.sh). Perform SDK finalization and upload the CLs to Gerrit.
+[SDK and REL](./dryrun-step-1-and-2.sh). Perform SDK finalization, plus all necessary changes to switch configuration to REL, and upload the CLs to Gerrit.
\ No newline at end of file
diff --git a/tools/finalization/build-step-1-and-m.sh b/tools/finalization/build-step-1-and-m.sh
index 0e7129f..88bb347 100755
--- a/tools/finalization/build-step-1-and-m.sh
+++ b/tools/finalization/build-step-1-and-m.sh
@@ -9,10 +9,9 @@
local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
# This command tests:
- # The release state for AIDL.
# ABI difference between user and userdebug builds.
# Resource/SDK finalization.
- AIDL_FROZEN_REL=true $m
+ $m
}
finalize_main_step1_and_m
diff --git a/tools/finalization/cleanup.sh b/tools/finalization/cleanup.sh
index cd87b1d..e2a0592 100755
--- a/tools/finalization/cleanup.sh
+++ b/tools/finalization/cleanup.sh
@@ -14,8 +14,8 @@
repo forall -c '\
git checkout . ; git revert --abort ; git clean -fdx ;\
- git checkout @ ; git branch fina-step1 -D ; git reset --hard; \
- repo start fina-step1 ; git checkout @ ; git b fina-step1 -D ;'
+ git checkout @ --detach ; git branch fina-step1 -D ; git reset --hard; \
+ repo start fina-step1 ; git checkout @ --detach ; git b fina-step1 -D ;'
}
finalize_revert_local_changes_main
diff --git a/tools/finalization/command-line-options.sh b/tools/finalization/command-line-options.sh
new file mode 100644
index 0000000..d9397c2
--- /dev/null
+++ b/tools/finalization/command-line-options.sh
@@ -0,0 +1,8 @@
+ARGV=$(getopt --options '' --long dry-run -- "$@")
+eval set -- "$ARGV"
+while true; do
+ case "$1" in
+ --dry-run) repo_upload_dry_run_arg="--dry-run"; repo_branch="finalization-dry-run"; shift ;;
+ *) break
+ esac
+done
diff --git a/tools/finalization/dryrun-cleanup.sh b/tools/finalization/dryrun-cleanup.sh
new file mode 100755
index 0000000..ddaffae
--- /dev/null
+++ b/tools/finalization/dryrun-cleanup.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# Brings local repository to a remote head state. Also removes all dryrun branches.
+
+# set -ex
+
+function finalize_revert_local_changes_main() {
+ local top="$(dirname "$0")"/../../../..
+ source $top/build/make/tools/finalization/environment.sh
+
+ local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
+
+ # remove the out folder
+ $m clobber
+
+ repo selfupdate
+
+ repo forall -c '\
+ git checkout . ; git revert --abort ; git clean -fdx ;\
+ git checkout @ --detach ; git branch fina-step1 -D ; git reset --hard; \
+ repo start fina-step1 ; git checkout @ --detach ; git b fina-step1 -D ; \
+ git b $FINAL_PLATFORM_CODENAME-SDK-Finalization-DryRun -D; \
+ git b $FINAL_PLATFORM_CODENAME-SDK-Finalization-DryRun-Rel -D; '
+}
+
+finalize_revert_local_changes_main
diff --git a/tools/finalization/dryrun-step-1-and-2.sh b/tools/finalization/dryrun-step-1-and-2.sh
new file mode 100755
index 0000000..f883bca
--- /dev/null
+++ b/tools/finalization/dryrun-step-1-and-2.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# Script to perform 1st and 2nd step of Android Finalization, create CLs and upload to Gerrit.
+
+function commit_step_2_changes() {
+ repo forall -c '\
+ if [[ $(git status --short) ]]; then
+ repo start "$FINAL_PLATFORM_CODENAME-SDK-Finalization-DryRun-Rel" ;
+ git add -A . ;
+ git commit -m "$FINAL_PLATFORM_CODENAME/$FINAL_PLATFORM_SDK_VERSION is now REL" \
+ -m "Ignore-AOSP-First: $FINAL_PLATFORM_CODENAME Finalization
+Bug: $FINAL_BUG_ID
+Test: build";
+
+ repo upload --cbr --no-verify -o nokeycheck -t -y . ;
+ fi'
+}
+
+function finalize_step_2_main() {
+ local top="$(dirname "$0")"/../../../..
+ source $top/build/make/tools/finalization/environment.sh
+
+ source $top/build/make/tools/finalization/finalize-sdk-resources.sh
+
+ source $top/build/make/tools/finalization/localonly-steps.sh
+
+ source $top/build/make/tools/finalization/finalize-sdk-rel.sh
+
+ # move all changes to finalization branch/topic and upload to gerrit
+ commit_step_2_changes
+
+ # build to confirm everything is OK
+ local m_next="$top/build/soong/soong_ui.bash --make-mode TARGET_RELEASE=next TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
+ $m_next
+
+ local m_fina="$top/build/soong/soong_ui.bash --make-mode TARGET_RELEASE=fina_2 TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
+ $m_fina
+}
+
+finalize_step_2_main
diff --git a/tools/finalization/dryrun-step-1.sh b/tools/finalization/dryrun-step-1.sh
new file mode 100755
index 0000000..0f2bc63
--- /dev/null
+++ b/tools/finalization/dryrun-step-1.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Script to perform a dry run of step 1 of Android Finalization, create CLs and upload to Gerrit.
+
+function commit_step_1_changes() {
+ repo forall -c '\
+ if [[ $(git status --short) ]]; then
+ repo start "$FINAL_PLATFORM_CODENAME-SDK-Finalization-DryRun" ;
+ git add -A . ;
+ git commit -m "$FINAL_PLATFORM_CODENAME is now $FINAL_PLATFORM_SDK_VERSION" \
+ -m "Ignore-AOSP-First: $FINAL_PLATFORM_CODENAME Finalization
+Bug: $FINAL_BUG_ID
+Test: build";
+
+ repo upload --cbr --no-verify -o nokeycheck -t -y . ;
+ fi'
+}
+
+function finalize_step_1_main() {
+ local top="$(dirname "$0")"/../../../..
+ source $top/build/make/tools/finalization/environment.sh
+
+ source $top/build/make/tools/finalization/finalize-sdk-resources.sh
+
+ # move all changes to finalization branch/topic and upload to gerrit
+ commit_step_1_changes
+
+ # build to confirm everything is OK
+ local m_next="$top/build/soong/soong_ui.bash --make-mode TARGET_RELEASE=next TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
+ $m_next
+
+ local m_fina="$top/build/soong/soong_ui.bash --make-mode TARGET_RELEASE=fina_1 TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
+ $m_fina
+}
+
+finalize_step_1_main
diff --git a/tools/finalization/environment.sh b/tools/finalization/environment.sh
index 7961e8b..f68a68b 100755
--- a/tools/finalization/environment.sh
+++ b/tools/finalization/environment.sh
@@ -15,14 +15,14 @@
# We might or might not fix this in future, but for now let's keep it +1.
export FINAL_PLATFORM_SDK_VERSION='35'
# Feel free to randomize once in a while to detect buggy version detection code.
-export FINAL_MAINLINE_EXTENSION='58'
+export FINAL_MAINLINE_EXTENSION='13'
# Options:
# 'unfinalized' - branch is in development state,
# 'vintf' - VINTF is finalized
# 'sdk' - VINTF and SDK/API are finalized
# 'rel' - branch is finalized, switched to REL
-export FINAL_STATE='vintf'
+export FINAL_STATE='rel'
export BUILD_FROM_SOURCE_STUB=true
diff --git a/tools/finalization/finalize-sdk-rel.sh b/tools/finalization/finalize-sdk-rel.sh
index 59fe28c..c49f974 100755
--- a/tools/finalization/finalize-sdk-rel.sh
+++ b/tools/finalization/finalize-sdk-rel.sh
@@ -8,12 +8,6 @@
fi
}
-function revert_resources_sdk_int_fix() {
- if grep -q 'public static final int RESOURCES_SDK_INT = SDK_INT;' "$top/frameworks/base/core/java/android/os/Build.java" ; then
- patch --strip=1 --no-backup-if-mismatch --directory="$top/frameworks/base" --input=../../build/make/tools/finalization/frameworks_base.revert_resource_sdk_int.diff
- fi
-}
-
function apply_prerelease_sdk_hack() {
if ! grep -q 'STOPSHIP: hack for the pre-release SDK' "$top/frameworks/base/core/java/android/content/pm/parsing/FrameworkParsingPackageUtils.java" ; then
patch --strip=1 --no-backup-if-mismatch --directory="$top/frameworks/base" --input=../../build/make/tools/finalization/frameworks_base.apply_hack.diff
@@ -30,25 +24,18 @@
# let the apps built with pre-release SDK parse
apply_prerelease_sdk_hack
- # in REL mode, resources would correctly set the resources_sdk_int, no fix required
- revert_resources_sdk_int_fix
-
# cts
- echo "$FINAL_PLATFORM_VERSION" > "$top/cts/tests/tests/os/assets/platform_versions.txt"
+ if ! grep -q "${FINAL_PLATFORM_VERSION}" "$top/cts/tests/tests/os/assets/platform_versions.txt" ; then
+ echo ${FINAL_PLATFORM_VERSION} >> "$top/cts/tests/tests/os/assets/platform_versions.txt"
+ fi
if [ "$FINAL_PLATFORM_CODENAME" != "$CURRENT_PLATFORM_CODENAME" ]; then
echo "$CURRENT_PLATFORM_CODENAME" >> "./cts/tests/tests/os/assets/platform_versions.txt"
fi
git -C "$top/cts" mv hostsidetests/theme/assets/${FINAL_PLATFORM_CODENAME} hostsidetests/theme/assets/${FINAL_PLATFORM_SDK_VERSION}
# prebuilts/abi-dumps/platform
- mkdir -p "$top/prebuilts/abi-dumps/platform/$FINAL_PLATFORM_SDK_VERSION"
- cp -r "$top/prebuilts/abi-dumps/platform/current/64/" "$top/prebuilts/abi-dumps/platform/$FINAL_PLATFORM_SDK_VERSION/"
-
- # TODO(b/309880485)
- # uncomment and update
- # prebuilts/abi-dumps/ndk
- #mkdir -p "$top/prebuilts/abi-dumps/ndk/$FINAL_PLATFORM_SDK_VERSION"
- #cp -r "$top/prebuilts/abi-dumps/ndk/current/64/" "$top/prebuilts/abi-dumps/ndk/$FINAL_PLATFORM_SDK_VERSION/"
+ "$top/build/soong/soong_ui.bash" --make-mode TARGET_RELEASE=next TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug create_reference_dumps
+ ANDROID_BUILD_TOP="$top" "$top/out/host/linux-x86/bin/create_reference_dumps" -release next --build-variant userdebug --lib-variant APEX
}
finalize_sdk_rel
diff --git a/tools/finalization/finalize-sdk-resources.sh b/tools/finalization/finalize-sdk-resources.sh
index 596f803..10266ed 100755
--- a/tools/finalization/finalize-sdk-resources.sh
+++ b/tools/finalization/finalize-sdk-resources.sh
@@ -9,13 +9,6 @@
fi
}
-function apply_resources_sdk_int_fix() {
- if ! grep -q 'public static final int RESOURCES_SDK_INT = SDK_INT;' "$top/frameworks/base/core/java/android/os/Build.java" ; then
- local base_git_root="$(readlink -f $top/frameworks/base)"
- patch --strip=1 --no-backup-if-mismatch --directory="$base_git_root" --input=../../build/make/tools/finalization/frameworks_base.apply_resource_sdk_int.diff
- fi
-}
-
function finalize_bionic_ndk() {
# Adding __ANDROID_API_<>__.
# If this hasn't done then it's not used and not really needed. Still, let's check and add this.
@@ -41,7 +34,8 @@
echo " /** Checks if the device is running on a release version of Android $FINAL_PLATFORM_CODENAME or newer */
@ChecksSdkIntAtLeast(api = $FINAL_PLATFORM_SDK_VERSION /* BUILD_VERSION_CODES.$FINAL_PLATFORM_CODENAME */)
public static boolean isAtLeast${FINAL_PLATFORM_CODENAME:0:1}() {
- return SDK_INT >= $FINAL_PLATFORM_SDK_VERSION;
+ return SDK_INT >= $FINAL_PLATFORM_SDK_VERSION ||
+ (SDK_INT == $(($FINAL_PLATFORM_SDK_VERSION - 1)) && isAtLeastPreReleaseCodename(\"$FINAL_PLATFORM_CODENAME\"));
}" > "$tmpfile"
local javaFuncRegex='\/\*\*[^{]*isAtLeast'"${shortCodename}"'() {[^{}]*}'
@@ -55,7 +49,11 @@
d}' $javaSdkLevel
echo "// Checks if the device is running on release version of Android ${FINAL_PLATFORM_CODENAME:0:1} or newer.
-inline bool IsAtLeast${FINAL_PLATFORM_CODENAME:0:1}() { return android_get_device_api_level() >= $FINAL_PLATFORM_SDK_VERSION; }" > "$tmpfile"
+inline bool IsAtLeast${FINAL_PLATFORM_CODENAME:0:1}() {
+ return android_get_device_api_level() >= $FINAL_PLATFORM_SDK_VERSION ||
+ (android_get_device_api_level() == $(($FINAL_PLATFORM_SDK_VERSION - 1)) &&
+ detail::IsAtLeastPreReleaseCodename(\"$FINAL_PLATFORM_CODENAME\"));
+}" > "$tmpfile"
local cppFuncRegex='\/\/[^{]*IsAtLeast'"${shortCodename}"'() {[^{}]*}'
local cppFuncReplace="N;N;N;N;N;N; s/$cppFuncRegex/$methodPlaceholder/; /$cppFuncRegex/!{P;D};"
@@ -123,14 +121,18 @@
sed -i -e 's/Pkg\.Revision.*/Pkg\.Revision=${PLATFORM_SDK_VERSION}.0.0/g' $build_tools_source
# build/soong
- local codename_version="\"${FINAL_PLATFORM_CODENAME}\": ${FINAL_PLATFORM_SDK_VERSION}"
+ local codename_version="\"${FINAL_PLATFORM_CODENAME}\": ${FINAL_PLATFORM_SDK_VERSION}"
if ! grep -q "$codename_version" "$top/build/soong/android/api_levels.go" ; then
sed -i -e "/:.*$((${FINAL_PLATFORM_SDK_VERSION}-1)),/a \\\t\t$codename_version," "$top/build/soong/android/api_levels.go"
fi
# cts
- echo ${FINAL_PLATFORM_VERSION} > "$top/cts/tests/tests/os/assets/platform_releases.txt"
- sed -i -e "s/EXPECTED_SDK = $((${FINAL_PLATFORM_SDK_VERSION}-1))/EXPECTED_SDK = ${FINAL_PLATFORM_SDK_VERSION}/g" "$top/cts/tests/tests/os/src/android/os/cts/BuildVersionTest.java"
+ if ! grep -q "${FINAL_PLATFORM_VERSION}" "$top/cts/tests/tests/os/assets/platform_releases.txt" ; then
+ echo ${FINAL_PLATFORM_VERSION} >> "$top/cts/tests/tests/os/assets/platform_releases.txt"
+ fi
+ if ! grep -q "$((${FINAL_PLATFORM_SDK_VERSION}-1)), ${FINAL_PLATFORM_VERSION}" "$top/cts/tests/tests/os/src/android/os/cts/BuildVersionTest.java" ; then
+ sed -i -e "s/.*EXPECTED_SDKS = List.of(.*$((${FINAL_PLATFORM_SDK_VERSION}-1))/&, $FINAL_PLATFORM_SDK_VERSION/" "$top/cts/tests/tests/os/src/android/os/cts/BuildVersionTest.java"
+ fi
# libcore
sed -i "s%$SDK_CODENAME%$SDK_VERSION%g" "$top/libcore/dalvik/src/main/java/dalvik/annotation/compat/VersionCodes.java"
@@ -153,7 +155,6 @@
# frameworks/base
sed -i "s%$SDK_CODENAME%$SDK_VERSION%g" "$top/frameworks/base/core/java/android/os/Build.java"
- apply_resources_sdk_int_fix
sed -i -e "/=.*$((${FINAL_PLATFORM_SDK_VERSION}-1)),/a \\ SDK_${FINAL_PLATFORM_CODENAME_JAVA} = ${FINAL_PLATFORM_SDK_VERSION}," "$top/frameworks/base/tools/aapt/SdkConstants.h"
sed -i -e "/=.*$((${FINAL_PLATFORM_SDK_VERSION}-1)),/a \\ SDK_${FINAL_PLATFORM_CODENAME_JAVA} = ${FINAL_PLATFORM_SDK_VERSION}," "$top/frameworks/base/tools/aapt2/SdkConstants.h"
diff --git a/tools/finalization/finalize-vintf-resources.sh b/tools/finalization/finalize-vintf-resources.sh
index a55d8e1..d532b25 100755
--- a/tools/finalization/finalize-vintf-resources.sh
+++ b/tools/finalization/finalize-vintf-resources.sh
@@ -16,8 +16,6 @@
export TARGET_RELEASE=fina_0
export TARGET_PRODUCT=aosp_arm64
- # TODO(b/314010764): finalize LL_NDK
-
# system/sepolicy
"$top/system/sepolicy/tools/finalize-vintf-resources.sh" "$top" "$FINAL_BOARD_API_LEVEL"
@@ -25,7 +23,11 @@
# pre-finalization build target (trunk)
local aidl_m="$top/build/soong/soong_ui.bash --make-mode"
- AIDL_TRANSITIVE_FREEZE=true $aidl_m aidl-freeze-api
+ AIDL_TRANSITIVE_FREEZE=true $aidl_m aidl-freeze-api create_reference_dumps
+
+ # Generate LLNDK ABI dumps
+ # This command depends on ANDROID_BUILD_TOP
+ "$ANDROID_HOST_OUT/bin/create_reference_dumps" -release "$TARGET_RELEASE" --build-variant "$TARGET_BUILD_VARIANT" --lib-variant LLNDK
}
function create_new_compat_matrix_and_kernel_configs() {
diff --git a/tools/finalization/frameworks_base.apply_resource_sdk_int.diff b/tools/finalization/frameworks_base.apply_resource_sdk_int.diff
deleted file mode 100644
index f0576d0..0000000
--- a/tools/finalization/frameworks_base.apply_resource_sdk_int.diff
+++ /dev/null
@@ -1,24 +0,0 @@
-From cdb47fc90b8d6860ec1dc5efada1f9ccd471618b Mon Sep 17 00:00:00 2001
-From: Alex Buynytskyy <alexbuy@google.com>
-Date: Tue, 11 Apr 2023 22:12:44 +0000
-Subject: [PATCH] Don't force +1 for resource resolution.
-
-Bug: 277674088
-Fixes: 277674088
-Test: boots, no crashes
-Change-Id: I17e743a0f1cf6f98fddd40c358dea5a8b9cc7723
----
-
-diff --git a/core/java/android/os/Build.java b/core/java/android/os/Build.java
-index eb47170..4d3e92b 100755
---- a/core/java/android/os/Build.java
-+++ b/core/java/android/os/Build.java
-@@ -493,7 +493,7 @@
- * @hide
- */
- @TestApi
-- public static final int RESOURCES_SDK_INT = SDK_INT + ACTIVE_CODENAMES.length;
-+ public static final int RESOURCES_SDK_INT = SDK_INT;
-
- /**
- * The current lowest supported value of app target SDK. Applications targeting
diff --git a/tools/finalization/frameworks_base.revert_resource_sdk_int.diff b/tools/finalization/frameworks_base.revert_resource_sdk_int.diff
deleted file mode 100644
index 2ade499..0000000
--- a/tools/finalization/frameworks_base.revert_resource_sdk_int.diff
+++ /dev/null
@@ -1,27 +0,0 @@
-From c7e460bb19071d867cd7ca04282ce42694f4f358 Mon Sep 17 00:00:00 2001
-From: Alex Buynytskyy <alexbuy@google.com>
-Date: Wed, 12 Apr 2023 01:06:26 +0000
-Subject: [PATCH] Revert "Don't force +1 for resource resolution."
-
-It's not required for master.
-
-This reverts commit f1cb683988f81579a76ddbf9993848a4a06dd28c.
-
-Bug: 277674088
-Test: boots, no crashes
-Change-Id: Ia1692548f26496fdc6f1e4f0557213c7996d6823
----
-
-diff --git a/core/java/android/os/Build.java b/core/java/android/os/Build.java
-index 4d3e92b..eb47170 100755
---- a/core/java/android/os/Build.java
-+++ b/core/java/android/os/Build.java
-@@ -493,7 +493,7 @@
- * @hide
- */
- @TestApi
-- public static final int RESOURCES_SDK_INT = SDK_INT;
-+ public static final int RESOURCES_SDK_INT = SDK_INT + ACTIVE_CODENAMES.length;
-
- /**
- * The current lowest supported value of app target SDK. Applications targeting
diff --git a/tools/finalization/localonly-steps.sh b/tools/finalization/localonly-steps.sh
index bebd563..94ee368 100755
--- a/tools/finalization/localonly-steps.sh
+++ b/tools/finalization/localonly-steps.sh
@@ -10,14 +10,15 @@
local m="$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=aosp_arm64 TARGET_RELEASE=fina_1 TARGET_BUILD_VARIANT=userdebug DIST_DIR=out/dist"
# adb keys
- $m adb
- LOGNAME=android-eng HOSTNAME=google.com "$top/out/host/linux-x86/bin/adb" keygen "$top/vendor/google/security/adb/${FINAL_PLATFORM_VERSION}.adb_key"
+ # The keys are already generated for Android 15. Keeping the command (commented out) for future reference.
+ # $m adb
+ # LOGNAME=android-eng HOSTNAME=google.com "$top/out/host/linux-x86/bin/adb" keygen "$top/vendor/google/security/adb/${FINAL_PLATFORM_VERSION}.adb_key"
# Build Platform SDKs.
$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=sdk TARGET_RELEASE=fina_1 TARGET_BUILD_VARIANT=userdebug sdk dist sdk_repo DIST_DIR=out/dist
# Build Modules SDKs.
- TARGET_RELEASE=fina_1 TARGET_BUILD_VARIANT=userdebug UNBUNDLED_BUILD_SDKS_FROM_SOURCE=true DIST_DIR=out/dist "$top/vendor/google/build/mainline_modules_sdks.sh" --build-release=latest
+ TARGET_RELEASE=fina_1 TARGET_BUILD_VARIANT=userdebug UNBUNDLED_BUILD_SDKS_FROM_SOURCE=true DIST_DIR=out/dist "$top/vendor/google/build/mainline_modules_sdks.sh" --build-release=next
# Update prebuilts.
"$top/prebuilts/build-tools/path/linux-x86/python3" -W ignore::DeprecationWarning "$top/prebuilts/sdk/update_prebuilts.py" --local_mode -f ${FINAL_PLATFORM_SDK_VERSION} -e ${FINAL_MAINLINE_EXTENSION} --bug 1 1
diff --git a/tools/finalization/step-0.sh b/tools/finalization/step-0.sh
index e61c644..2087f6e 100755
--- a/tools/finalization/step-0.sh
+++ b/tools/finalization/step-0.sh
@@ -9,19 +9,21 @@
set +e
repo forall -c '\
if [[ $(git status --short) ]]; then
- repo start "VINTF-$FINAL_BOARD_API_LEVEL-Finalization" ;
+ repo start "'$repo_branch'" ;
git add -A . ;
git commit -m "Vendor API level $FINAL_BOARD_API_LEVEL is now frozen" \
-m "Ignore-AOSP-First: VINTF $FINAL_BOARD_API_LEVEL Finalization
Bug: $FINAL_BUG_ID
Test: build";
- repo upload --cbr --no-verify -o nokeycheck -t -y . ;
+ repo upload '"$repo_upload_dry_run_arg"' --cbr --no-verify -o nokeycheck -t -y . ;
fi'
}
function finalize_step_0_main() {
local top="$(dirname "$0")"/../../../..
source $top/build/make/tools/finalization/environment.sh
+ local repo_branch="VINTF-$FINAL_BOARD_API_LEVEL-Finalization"
+ source $top/build/make/tools/finalization/command-line-options.sh
local m="$top/build/soong/soong_ui.bash --make-mode TARGET_RELEASE=next TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
@@ -34,4 +36,4 @@
AIDL_FROZEN_REL=true $m
}
-finalize_step_0_main
+finalize_step_0_main $@
diff --git a/tools/finalization/step-1.sh b/tools/finalization/step-1.sh
index 0e483d5..736d641 100755
--- a/tools/finalization/step-1.sh
+++ b/tools/finalization/step-1.sh
@@ -7,21 +7,21 @@
set +e
repo forall -c '\
if [[ $(git status --short) ]]; then
- repo start "$FINAL_PLATFORM_CODENAME-SDK-Finalization" ;
+ repo start "'$repo_branch'" ;
git add -A . ;
git commit -m "$FINAL_PLATFORM_CODENAME is now $FINAL_PLATFORM_SDK_VERSION and extension version $FINAL_MAINLINE_EXTENSION" \
-m "Ignore-AOSP-First: $FINAL_PLATFORM_CODENAME Finalization
Bug: $FINAL_BUG_ID
Test: build";
- repo upload --cbr --no-verify -o nokeycheck -t -y . ;
+ repo upload '"$repo_upload_dry_run_arg"' --cbr --no-verify -o nokeycheck -t -y . ;
fi'
}
function finalize_step_1_main() {
local top="$(dirname "$0")"/../../../..
source $top/build/make/tools/finalization/environment.sh
-
- local m="$top/build/soong/soong_ui.bash --make-mode TARGET_RELEASE=next TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
+ local repo_branch="$FINAL_PLATFORM_CODENAME-SDK-Finalization"
+ source $top/build/make/tools/finalization/command-line-options.sh
source $top/build/make/tools/finalization/finalize-sdk-resources.sh
@@ -29,7 +29,11 @@
commit_step_1_changes
# build to confirm everything is OK
- AIDL_FROZEN_REL=true $m
+ local m_next="$top/build/soong/soong_ui.bash --make-mode TARGET_RELEASE=next TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
+ $m_next
+
+ local m_fina="$top/build/soong/soong_ui.bash --make-mode TARGET_RELEASE=fina_1 TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
+ $m_fina
}
-finalize_step_1_main
+finalize_step_1_main $@
diff --git a/tools/finalization/step-2.sh b/tools/finalization/step-2.sh
index 356cad0..52e3887 100755
--- a/tools/finalization/step-2.sh
+++ b/tools/finalization/step-2.sh
@@ -4,22 +4,22 @@
function commit_step_2_changes() {
repo forall -c '\
if [[ $(git status --short) ]]; then
- repo start "$FINAL_PLATFORM_CODENAME-SDK-Finalization-Rel" ;
+ repo start "'$repo_branch'" ;
git add -A . ;
git commit -m "$FINAL_PLATFORM_CODENAME/$FINAL_PLATFORM_SDK_VERSION is now REL" \
-m "Ignore-AOSP-First: $FINAL_PLATFORM_CODENAME Finalization
Bug: $FINAL_BUG_ID
Test: build";
- repo upload --cbr --no-verify -o nokeycheck -t -y . ;
+ repo upload '"$repo_upload_dry_run_arg"' --cbr --no-verify -o nokeycheck -t -y . ;
fi'
}
function finalize_step_2_main() {
local top="$(dirname "$0")"/../../../..
source $top/build/make/tools/finalization/environment.sh
-
- local m="$top/build/soong/soong_ui.bash --make-mode TARGET_RELEASE=next TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
+ local repo_branch="$FINAL_PLATFORM_CODENAME-SDK-Finalization-Rel"
+ source $top/build/make/tools/finalization/command-line-options.sh
# prebuilts etc
source $top/build/make/tools/finalization/finalize-sdk-rel.sh
@@ -28,7 +28,11 @@
commit_step_2_changes
# build to confirm everything is OK
- AIDL_FROZEN_REL=true $m
+ local m_next="$top/build/soong/soong_ui.bash --make-mode TARGET_RELEASE=next TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
+ $m_next
+
+ local m_fina="$top/build/soong/soong_ui.bash --make-mode TARGET_RELEASE=fina_2 TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug"
+ $m_fina
}
-finalize_step_2_main
+finalize_step_2_main $@
diff --git a/tools/releasetools/Android.bp b/tools/releasetools/Android.bp
index cf7e2ae..8c71044 100644
--- a/tools/releasetools/Android.bp
+++ b/tools/releasetools/Android.bp
@@ -368,6 +368,9 @@
libs: [
"ota_utils_lib",
],
+ required: [
+ "signapk",
+ ],
}
python_binary_host {
@@ -555,6 +558,7 @@
"releasetools_common",
"ota_metadata_proto",
"ota_utils_lib",
+ "update_payload",
],
}
diff --git a/tools/releasetools/add_img_to_target_files.py b/tools/releasetools/add_img_to_target_files.py
index b39a82c..c25ff27 100644
--- a/tools/releasetools/add_img_to_target_files.py
+++ b/tools/releasetools/add_img_to_target_files.py
@@ -464,6 +464,7 @@
dtbo_prebuilt_path = os.path.join(
OPTIONS.input_tmp, "PREBUILT_IMAGES", "dtbo.img")
assert os.path.exists(dtbo_prebuilt_path)
+ os.makedirs(os.path.dirname(img.name), exist_ok=True)
shutil.copy(dtbo_prebuilt_path, img.name)
# AVB-sign the image as needed.
diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py
index edd4366..f04dfb7 100644
--- a/tools/releasetools/common.py
+++ b/tools/releasetools/common.py
@@ -3238,7 +3238,9 @@
return t
def WriteToDir(self, d):
- with open(os.path.join(d, self.name), "wb") as fp:
+ output_path = os.path.join(d, self.name)
+ os.makedirs(os.path.dirname(output_path), exist_ok=True)
+ with open(output_path, "wb") as fp:
fp.write(self.data)
def AddToZip(self, z, compression=None):
diff --git a/tools/releasetools/sign_target_files_apks.py b/tools/releasetools/sign_target_files_apks.py
index b485440..0254f37 100755
--- a/tools/releasetools/sign_target_files_apks.py
+++ b/tools/releasetools/sign_target_files_apks.py
@@ -190,10 +190,10 @@
import add_img_to_target_files
import ota_from_raw_img
-import ota_utils
import apex_utils
import common
import payload_signer
+import update_payload
from payload_signer import SignOtaPackage, PAYLOAD_BIN
@@ -223,6 +223,7 @@
OPTIONS.allow_gsi_debug_sepolicy = False
OPTIONS.override_apk_keys = None
OPTIONS.override_apex_keys = None
+OPTIONS.input_tmp = None
AVB_FOOTER_ARGS_BY_PARTITION = {
@@ -581,18 +582,34 @@
filename.endswith("/prop.default")
+def GetOtaSigningArgs():
+ args = []
+ if OPTIONS.package_key:
+ args.extend(["--package_key", OPTIONS.package_key])
+ if OPTIONS.payload_signer:
+ args.extend(["--payload_signer=" + OPTIONS.payload_signer])
+ if OPTIONS.payload_signer_args:
+ args.extend(["--payload_signer_args=" + OPTIONS.payload_signer_args])
+ if OPTIONS.search_path:
+ args.extend(["--search_path", OPTIONS.search_path])
+ if OPTIONS.payload_signer_maximum_signature_size:
+ args.extend(["--payload_signer_maximum_signature_size",
+ OPTIONS.payload_signer_maximum_signature_size])
+ if OPTIONS.private_key_suffix:
+ args.extend(["--private_key_suffix", OPTIONS.private_key_suffix])
+ return args
+
+
def RegenerateKernelPartitions(input_tf_zip: zipfile.ZipFile, output_tf_zip: zipfile.ZipFile, misc_info):
"""Re-generate boot and dtbo partitions using new signing configuration"""
+ files_to_unzip = [
+ "PREBUILT_IMAGES/*", "BOOTABLE_IMAGES/*.img", "*/boot_16k.img", "*/dtbo_16k.img"]
if OPTIONS.input_tmp is None:
- OPTIONS.input_tmp = common.UnzipTemp(input_tf_zip.filename, [
- "*/boot.img", "*/dtbo.img"])
+ OPTIONS.input_tmp = common.UnzipTemp(input_tf_zip.filename, files_to_unzip)
else:
- common.UnzipToDir(input_tf_zip, OPTIONS.input_tmp, [
- "*/boot.img", "*/dtbo.img"])
+ common.UnzipToDir(input_tf_zip.filename, OPTIONS.input_tmp, files_to_unzip)
unzip_dir = OPTIONS.input_tmp
- image_dir = os.path.join(unzip_dir, "IMAGES")
- shutil.rmtree(image_dir)
- os.makedirs(image_dir, exist_ok=True)
+ os.makedirs(os.path.join(unzip_dir, "IMAGES"), exist_ok=True)
boot_image = common.GetBootableImage(
"IMAGES/boot.img", "boot.img", unzip_dir, "BOOT", misc_info)
@@ -601,37 +618,64 @@
boot_image = os.path.join(unzip_dir, boot_image.name)
common.ZipWrite(output_tf_zip, boot_image, "IMAGES/boot.img",
compress_type=zipfile.ZIP_STORED)
- add_img_to_target_files.AddDtbo(output_tf_zip)
+ if misc_info.get("has_dtbo") == "true":
+ add_img_to_target_files.AddDtbo(output_tf_zip)
return unzip_dir
-def RegenerateBootOTA(input_tf_zip: zipfile.ZipFile, output_tf_zip: zipfile.ZipFile, misc_info, filename, input_ota):
- if filename not in ["VENDOR/boot_otas/boot_ota_4k.zip", "SYSTEM/boot_otas/boot_ota_4k.zip"]:
- # We only need to re-generate 4K boot OTA, for other OTA packages
- # simply copy as is
- with input_tf_zip.open(filename, "r") as in_fp:
- shutil.copyfileobj(in_fp, input_ota)
- input_ota.flush()
+def RegenerateBootOTA(input_tf_zip: zipfile.ZipFile, filename, input_ota):
+ with input_tf_zip.open(filename, "r") as in_fp:
+ payload = update_payload.Payload(in_fp)
+ is_incremental = any([part.HasField('old_partition_info')
+ for part in payload.manifest.partitions])
+ is_boot_ota = filename.startswith(
+ "VENDOR/boot_otas/") or filename.startswith("SYSTEM/boot_otas/")
+ if not is_boot_ota:
return
- timestamp = misc_info["build.prop"].GetProp(
- "ro.system.build.date.utc")
- unzip_dir = RegenerateKernelPartitions(
- input_tf_zip, output_tf_zip, misc_info)
- signed_boot_image = os.path.join(unzip_dir, "IMAGES/boot.img")
- signed_dtbo_image = os.path.join(unzip_dir, "IMAGES/dtbo.img")
+ is_4k_boot_ota = filename in [
+ "VENDOR/boot_otas/boot_ota_4k.zip", "SYSTEM/boot_otas/boot_ota_4k.zip"]
+ # Only 4K boot image is re-generated, so if 16K boot ota isn't incremental,
+ # we do not need to re-generate
+ if not is_4k_boot_ota and not is_incremental:
+ return
+ timestamp = str(payload.manifest.max_timestamp)
+ partitions = [part.partition_name for part in payload.manifest.partitions]
+ unzip_dir = OPTIONS.input_tmp
+ signed_boot_image = os.path.join(unzip_dir, "IMAGES", "boot.img")
if not os.path.exists(signed_boot_image):
logger.warn("Need to re-generate boot OTA {} but failed to get signed boot image. 16K dev option will be impacted, after rolling back to 4K user would need to sideload/flash their device to continue receiving OTAs.")
return
- logger.info(
- "Re-generating boot OTA {} with timestamp {}".format(filename, timestamp))
- args = ["ota_from_raw_img", "--package_key", OPTIONS.package_key,
+ signed_dtbo_image = os.path.join(unzip_dir, "IMAGES", "dtbo.img")
+ if "dtbo" in partitions and not os.path.exists(signed_dtbo_image):
+ raise ValueError(
+ "Boot OTA {} has dtbo partition, but no dtbo image found in target files.".format(filename))
+ if is_incremental:
+ signed_16k_boot_image = os.path.join(
+ unzip_dir, "IMAGES", "boot_16k.img")
+ signed_16k_dtbo_image = os.path.join(
+ unzip_dir, "IMAGES", "dtbo_16k.img")
+ if is_4k_boot_ota:
+ if os.path.exists(signed_16k_boot_image):
+ signed_boot_image = signed_16k_boot_image + ":" + signed_boot_image
+ if os.path.exists(signed_16k_dtbo_image):
+ signed_dtbo_image = signed_16k_dtbo_image + ":" + signed_dtbo_image
+ else:
+ if os.path.exists(signed_16k_boot_image):
+ signed_boot_image += ":" + signed_16k_boot_image
+ if os.path.exists(signed_16k_dtbo_image):
+ signed_dtbo_image += ":" + signed_16k_dtbo_image
+
+ args = ["ota_from_raw_img",
"--max_timestamp", timestamp, "--output", input_ota.name]
- if os.path.exists(signed_dtbo_image):
+ args.extend(GetOtaSigningArgs())
+ if "dtbo" in partitions:
args.extend(["--partition_name", "boot,dtbo",
signed_boot_image, signed_dtbo_image])
else:
args.extend(["--partition_name", "boot", signed_boot_image])
+ logger.info(
+ "Re-generating boot OTA {} using cmd {}".format(filename, args))
ota_from_raw_img.main(args)
@@ -657,6 +701,8 @@
if misc_info.get('avb_enable') == 'true':
RewriteAvbProps(misc_info)
+ RegenerateKernelPartitions(input_tf_zip, output_tf_zip, misc_info)
+
for info in input_tf_zip.infolist():
filename = info.filename
if filename.startswith("IMAGES/"):
@@ -667,10 +713,10 @@
if filename.startswith("OTA/") and filename.endswith(".img"):
continue
- data = input_tf_zip.read(filename)
- out_info = copy.copy(info)
(is_apk, is_compressed, should_be_skipped) = GetApkFileInfo(
filename, compressed_extension, OPTIONS.skip_apks_with_path_prefix)
+ data = input_tf_zip.read(filename)
+ out_info = copy.copy(info)
if is_apk and should_be_skipped:
# Copy skipped APKs verbatim.
@@ -734,8 +780,7 @@
elif filename.endswith(".zip") and IsEntryOtaPackage(input_tf_zip, filename):
logger.info("Re-signing OTA package {}".format(filename))
with tempfile.NamedTemporaryFile() as input_ota, tempfile.NamedTemporaryFile() as output_ota:
- RegenerateBootOTA(input_tf_zip, output_tf_zip,
- misc_info, filename, input_ota)
+ RegenerateBootOTA(input_tf_zip, filename, input_ota)
SignOtaPackage(input_ota.name, output_ota.name)
common.ZipWrite(output_tf_zip, output_ota.name, filename,
@@ -1131,9 +1176,9 @@
common.ZipWriteStr(output_zip, filename, temp_file.getvalue())
-def ReplaceOtaKeys(input_tf_zip, output_tf_zip, misc_info):
+def ReplaceOtaKeys(input_tf_zip: zipfile.ZipFile, output_tf_zip, misc_info):
try:
- keylist = input_tf_zip.read("META/otakeys.txt").split()
+ keylist = input_tf_zip.read("META/otakeys.txt").decode().split()
except KeyError:
raise common.ExternalError("can't read META/otakeys.txt from input")