Merge "Sandbox renderscript genrules" into main
diff --git a/android/config.go b/android/config.go
index 572e006..a69adc3 100644
--- a/android/config.go
+++ b/android/config.go
@@ -570,8 +570,6 @@
config: config,
}
- config.productVariables.Build_from_text_stub = boolPtr(config.BuildFromTextStub())
-
// Soundness check of the build and source directories. This won't catch strange
// configurations with symlinks, but at least checks the obvious case.
absBuildDir, err := filepath.Abs(cmdArgs.SoongOutDir)
@@ -694,6 +692,7 @@
"framework-media": {},
"framework-mediaprovider": {},
"framework-ondevicepersonalization": {},
+ "framework-pdf": {},
"framework-permission": {},
"framework-permission-s": {},
"framework-scheduling": {},
@@ -707,6 +706,8 @@
"i18n.module.public.api": {},
}
+ config.productVariables.Build_from_text_stub = boolPtr(config.BuildFromTextStub())
+
return Config{config}, err
}
@@ -2075,11 +2076,17 @@
return c.IsEnvTrue("EMMA_INSTRUMENT") || c.IsEnvTrue("EMMA_INSTRUMENT_STATIC") || c.IsEnvTrue("EMMA_INSTRUMENT_FRAMEWORK")
}
+func (c *deviceConfig) BuildFromSourceStub() bool {
+ return Bool(c.config.productVariables.BuildFromSourceStub)
+}
+
func (c *config) BuildFromTextStub() bool {
// TODO: b/302320354 - Remove the coverage build specific logic once the
// robust solution for handling native properties in from-text stub build
// is implemented.
- return !c.buildFromSourceStub && !c.JavaCoverageEnabled() && !c.IsEnvTrue("BUILD_FROM_SOURCE_STUB")
+ return !c.buildFromSourceStub &&
+ !c.JavaCoverageEnabled() &&
+ !c.deviceConfig.BuildFromSourceStub()
}
func (c *config) SetBuildFromTextStub(b bool) {
diff --git a/android/variable.go b/android/variable.go
index 648e4cf..fe3a6d7 100644
--- a/android/variable.go
+++ b/android/variable.go
@@ -494,6 +494,8 @@
Release_expose_flagged_api *bool `json:",omitempty"`
BuildFlags map[string]string `json:",omitempty"`
+
+ BuildFromSourceStub *bool `json:",omitempty"`
}
type PartitionQualifiedVariablesType struct {
diff --git a/java/aapt2.go b/java/aapt2.go
index 3bb70b5..17ee6ee 100644
--- a/java/aapt2.go
+++ b/java/aapt2.go
@@ -25,17 +25,23 @@
"android/soong/android"
)
+func isPathValueResource(res android.Path) bool {
+ subDir := filepath.Dir(res.String())
+ subDir, lastDir := filepath.Split(subDir)
+ return strings.HasPrefix(lastDir, "values")
+}
+
// Convert input resource file path to output file path.
// values-[config]/<file>.xml -> values-[config]_<file>.arsc.flat;
// For other resource file, just replace the last "/" with "_" and add .flat extension.
func pathToAapt2Path(ctx android.ModuleContext, res android.Path) android.WritablePath {
name := res.Base()
- subDir := filepath.Dir(res.String())
- subDir, lastDir := filepath.Split(subDir)
- if strings.HasPrefix(lastDir, "values") {
+ if isPathValueResource(res) {
name = strings.TrimSuffix(name, ".xml") + ".arsc"
}
+ subDir := filepath.Dir(res.String())
+ subDir, lastDir := filepath.Split(subDir)
name = lastDir + "_" + name + ".flat"
return android.PathForModuleOut(ctx, "aapt2", subDir, name)
}
@@ -63,7 +69,21 @@
// aapt2Compile compiles resources and puts the results in the requested directory.
func aapt2Compile(ctx android.ModuleContext, dir android.Path, paths android.Paths,
- flags []string) android.WritablePaths {
+ flags []string, productToFilter string) android.WritablePaths {
+ if productToFilter != "" && productToFilter != "default" {
+ // --filter-product leaves only product-specific resources. Product-specific resources only exist
+ // in value resources (values/*.xml), so filter value resource files only. Ignore other types of
+ // resources as they don't need to be in product characteristics RRO (and they will cause aapt2
+ // compile errors)
+ filteredPaths := android.Paths{}
+ for _, path := range paths {
+ if isPathValueResource(path) {
+ filteredPaths = append(filteredPaths, path)
+ }
+ }
+ paths = filteredPaths
+ flags = append([]string{"--filter-product " + productToFilter}, flags...)
+ }
// Shard the input paths so that they can be processed in parallel. If we shard them into too
// small chunks, the additional cost of spinning up aapt2 outweighs the performance gain. The
diff --git a/java/aar.go b/java/aar.go
index 6b89129..e579008 100644
--- a/java/aar.go
+++ b/java/aar.go
@@ -102,6 +102,9 @@
// true if RRO is enforced for any of the dependent modules
RROEnforcedForDependent bool `blueprint:"mutated"`
+
+ // Filter only specified product and ignore other products
+ Filter_product *string `blueprint:"mutated"`
}
type aapt struct {
@@ -162,6 +165,10 @@
return BoolDefault(a.aaptProperties.Use_resource_processor, false)
}
+func (a *aapt) filterProduct() string {
+ return String(a.aaptProperties.Filter_product)
+}
+
func (a *aapt) ExportPackage() android.Path {
return a.exportPackage
}
@@ -432,7 +439,7 @@
var compiledResDirs []android.Paths
for _, dir := range resDirs {
a.resourceFiles = append(a.resourceFiles, dir.files...)
- compiledResDirs = append(compiledResDirs, aapt2Compile(ctx, dir.dir, dir.files, compileFlags).Paths())
+ compiledResDirs = append(compiledResDirs, aapt2Compile(ctx, dir.dir, dir.files, compileFlags, a.filterProduct()).Paths())
}
for i, zip := range resZips {
@@ -491,7 +498,7 @@
}
for _, dir := range overlayDirs {
- compiledOverlay = append(compiledOverlay, aapt2Compile(ctx, dir.dir, dir.files, compileFlags).Paths()...)
+ compiledOverlay = append(compiledOverlay, aapt2Compile(ctx, dir.dir, dir.files, compileFlags, a.filterProduct()).Paths()...)
}
var splitPackages android.WritablePaths
diff --git a/java/androidmk.go b/java/androidmk.go
index 97b303d..84f78c8 100644
--- a/java/androidmk.go
+++ b/java/androidmk.go
@@ -343,10 +343,15 @@
Disabled: true,
}}
}
+ var required []string
+ if proptools.Bool(app.appProperties.Generate_product_characteristics_rro) {
+ required = []string{app.productCharacteristicsRROPackageName()}
+ }
return []android.AndroidMkEntries{android.AndroidMkEntries{
Class: "APPS",
OutputFile: android.OptionalPathForPath(app.outputFile),
Include: "$(BUILD_SYSTEM)/soong_app_prebuilt.mk",
+ Required: required,
ExtraEntries: []android.AndroidMkExtraEntriesFunc{
func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {
// App module names can be overridden.
diff --git a/java/app.go b/java/app.go
index 41848ce..9b7f4c4 100755
--- a/java/app.go
+++ b/java/app.go
@@ -131,6 +131,16 @@
// Specifies the file that contains the allowlist for this app.
Privapp_allowlist *string `android:"path"`
+
+ // If set, create an RRO package which contains only resources having PRODUCT_CHARACTERISTICS
+ // and install the RRO package to /product partition, instead of passing --product argument
+ // to aapt2. Default is false.
+ // Setting this will make this APK identical to all targets, regardless of
+ // PRODUCT_CHARACTERISTICS.
+ Generate_product_characteristics_rro *bool
+
+ ProductCharacteristicsRROPackageName *string `blueprint:"mutated"`
+ ProductCharacteristicsRROManifestModuleName *string `blueprint:"mutated"`
}
// android_app properties that can be overridden by override_android_app
@@ -455,8 +465,9 @@
aaptLinkFlags := []string{}
// Add TARGET_AAPT_CHARACTERISTICS values to AAPT link flags if they exist and --product flags were not provided.
+ autogenerateRRO := proptools.Bool(a.appProperties.Generate_product_characteristics_rro)
hasProduct := android.PrefixInList(a.aaptProperties.Aaptflags, "--product")
- if !hasProduct && len(ctx.Config().ProductAAPTCharacteristics()) > 0 {
+ if !autogenerateRRO && !hasProduct && len(ctx.Config().ProductAAPTCharacteristics()) > 0 {
aaptLinkFlags = append(aaptLinkFlags, "--product", ctx.Config().ProductAAPTCharacteristics())
}
@@ -1057,6 +1068,8 @@
}
case ".export-package.apk":
return []android.Path{a.exportPackage}, nil
+ case ".manifest.xml":
+ return []android.Path{a.aapt.manifestPath}, nil
}
return a.Library.OutputFiles(tag)
}
@@ -1086,6 +1099,14 @@
a.aapt.IDEInfo(dpInfo)
}
+func (a *AndroidApp) productCharacteristicsRROPackageName() string {
+ return proptools.String(a.appProperties.ProductCharacteristicsRROPackageName)
+}
+
+func (a *AndroidApp) productCharacteristicsRROManifestModuleName() string {
+ return proptools.String(a.appProperties.ProductCharacteristicsRROManifestModuleName)
+}
+
// android_app compiles sources and Android resources into an Android application package `.apk` file.
func AndroidAppFactory() android.Module {
module := &AndroidApp{}
@@ -1112,6 +1133,57 @@
android.InitApexModule(module)
android.InitBazelModule(module)
+ android.AddLoadHook(module, func(ctx android.LoadHookContext) {
+ a := ctx.Module().(*AndroidApp)
+
+ characteristics := ctx.Config().ProductAAPTCharacteristics()
+ if characteristics == "default" || characteristics == "" {
+ module.appProperties.Generate_product_characteristics_rro = nil
+ // no need to create RRO
+ return
+ }
+
+ if !proptools.Bool(module.appProperties.Generate_product_characteristics_rro) {
+ return
+ }
+
+ rroPackageName := a.Name() + "__" + strings.ReplaceAll(characteristics, ",", "_") + "__auto_generated_characteristics_rro"
+ rroManifestName := rroPackageName + "_manifest"
+
+ a.appProperties.ProductCharacteristicsRROPackageName = proptools.StringPtr(rroPackageName)
+ a.appProperties.ProductCharacteristicsRROManifestModuleName = proptools.StringPtr(rroManifestName)
+
+ rroManifestProperties := struct {
+ Name *string
+ Tools []string
+ Out []string
+ Srcs []string
+ Cmd *string
+ }{
+ Name: proptools.StringPtr(rroManifestName),
+ Tools: []string{"characteristics_rro_generator"},
+ Out: []string{"AndroidManifest.xml"},
+ Srcs: []string{":" + a.Name() + "{.manifest.xml}"},
+ Cmd: proptools.StringPtr("$(location characteristics_rro_generator) $(in) $(out)"),
+ }
+ ctx.CreateModule(genrule.GenRuleFactory, &rroManifestProperties)
+
+ rroProperties := struct {
+ Name *string
+ Filter_product *string
+ Aaptflags []string
+ Manifest *string
+ Resource_dirs []string
+ }{
+ Name: proptools.StringPtr(rroPackageName),
+ Filter_product: proptools.StringPtr(characteristics),
+ Aaptflags: []string{"--auto-add-overlay"},
+ Manifest: proptools.StringPtr(":" + rroManifestName),
+ Resource_dirs: a.aaptProperties.Resource_dirs,
+ }
+ ctx.CreateModule(RuntimeResourceOverlayFactory, &rroProperties)
+ })
+
return module
}
diff --git a/tests/genrule_sandbox_test.py b/tests/genrule_sandbox_test.py
index 874859a..3799e92 100755
--- a/tests/genrule_sandbox_test.py
+++ b/tests/genrule_sandbox_test.py
@@ -15,12 +15,14 @@
# limitations under the License.
import argparse
+import asyncio
import collections
import json
import os
+import socket
import subprocess
import sys
-import tempfile
+import textwrap
def get_top() -> str:
path = '.'
@@ -30,39 +32,65 @@
path = os.path.join(path, '..')
return os.path.abspath(path)
-def _build_with_soong(targets, target_product, *, keep_going = False, extra_env={}):
- env = {
- **os.environ,
- "TARGET_PRODUCT": target_product,
- "TARGET_BUILD_VARIANT": "userdebug",
- }
- env.update(extra_env)
+async def _build_with_soong(out_dir, targets, *, extra_env={}):
+ env = os.environ | extra_env
+
+ # Use nsjail to remap the out_dir to out/, because some genrules write the path to the out
+ # dir into their artifacts, so if the out directories were different it would cause a diff
+ # that doesn't really matter.
args = [
+ 'prebuilts/build-tools/linux-x86/bin/nsjail',
+ '-q',
+ '--cwd',
+ os.getcwd(),
+ '-e',
+ '-B',
+ '/',
+ '-B',
+ f'{os.path.abspath(out_dir)}:{os.path.abspath("out")}',
+ '--time_limit',
+ '0',
+ '--skip_setsid',
+ '--keep_caps',
+ '--disable_clone_newcgroup',
+ '--disable_clone_newnet',
+ '--rlimit_as',
+ 'soft',
+ '--rlimit_core',
+ 'soft',
+ '--rlimit_cpu',
+ 'soft',
+ '--rlimit_fsize',
+ 'soft',
+ '--rlimit_nofile',
+ 'soft',
+ '--proc_rw',
+ '--hostname',
+ socket.gethostname(),
+ '--',
"build/soong/soong_ui.bash",
"--make-mode",
"--skip-soong-tests",
]
- if keep_going:
- args.append("-k")
args.extend(targets)
- try:
- subprocess.check_output(
- args,
- env=env,
- )
- except subprocess.CalledProcessError as e:
- print(e)
- print(e.stdout)
- print(e.stderr)
- exit(1)
+ process = await asyncio.create_subprocess_exec(
+ *args,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ env=env,
+ )
+ stdout, stderr = await process.communicate()
+ if process.returncode != 0:
+ print(stdout)
+ print(stderr)
+ sys.exit(process.returncode)
-def _find_outputs_for_modules(modules, out_dir, target_product):
- module_path = os.path.join(out_dir, "soong", "module-actions.json")
+async def _find_outputs_for_modules(modules):
+ module_path = "out/soong/module-actions.json"
if not os.path.exists(module_path):
- # Use GENRULE_SANDBOXING=false so that we don't cause re-analysis later when we do the no-sandboxing build
- _build_with_soong(["json-module-graph"], target_product, extra_env={"GENRULE_SANDBOXING": "false"})
+ await _build_with_soong('out', ["json-module-graph"])
with open(module_path) as f:
action_graph = json.load(f)
@@ -71,7 +99,7 @@
for mod in action_graph:
name = mod["Name"]
if name in modules:
- for act in mod["Module"]["Actions"]:
+ for act in (mod["Module"]["Actions"] or []):
if "}generate" in act["Desc"]:
module_to_outs[name].update(act["Outputs"])
return module_to_outs
@@ -89,20 +117,19 @@
return different_modules
-def main():
+async def main():
parser = argparse.ArgumentParser()
parser.add_argument(
- "--target_product",
- "-t",
- default="aosp_cf_arm64_phone",
- help="optional, target product, always runs as eng",
- )
- parser.add_argument(
"modules",
nargs="+",
help="modules to compare builds with genrule sandboxing enabled/not",
)
parser.add_argument(
+ "--check-determinism",
+ action="store_true",
+ help="Don't check for working sandboxing. Instead, run two default builds, and compare their outputs. This is used to check for nondeterminsim, which would also affect the sandboxed test.",
+ )
+ parser.add_argument(
"--show-diff",
"-d",
action="store_true",
@@ -117,10 +144,13 @@
args = parser.parse_args()
os.chdir(get_top())
- out_dir = os.environ.get("OUT_DIR", "out")
+ if "TARGET_PRODUCT" not in os.environ:
+ sys.exit("Please run lunch first")
+ if os.environ.get("OUT_DIR", "out") != "out":
+ sys.exit(f"This script expects OUT_DIR to be 'out', got: '{os.environ.get('OUT_DIR')}'")
print("finding output files for the modules...")
- module_to_outs = _find_outputs_for_modules(set(args.modules), out_dir, args.target_product)
+ module_to_outs = await _find_outputs_for_modules(set(args.modules))
if not module_to_outs:
sys.exit("No outputs found")
@@ -130,33 +160,48 @@
sys.exit(0)
all_outs = list(set.union(*module_to_outs.values()))
+ for i, out in enumerate(all_outs):
+ if not out.startswith("out/"):
+ sys.exit("Expected output file to start with out/, found: " + out)
- print("building without sandboxing...")
- _build_with_soong(all_outs, args.target_product, extra_env={"GENRULE_SANDBOXING": "false"})
- with tempfile.TemporaryDirectory() as tempdir:
- for f in all_outs:
- subprocess.check_call(["cp", "--parents", f, tempdir])
+ other_out_dir = "out_check_determinism" if args.check_determinism else "out_not_sandboxed"
+ other_env = {"GENRULE_SANDBOXING": "false"}
+ if args.check_determinism:
+ other_env = {}
- print("building with sandboxing...")
- _build_with_soong(
- all_outs,
- args.target_product,
- # We've verified these build without sandboxing already, so do the sandboxing build
- # with keep_going = True so that we can find all the genrules that fail to build with
- # sandboxing.
- keep_going = True,
- extra_env={"GENRULE_SANDBOXING": "true"},
- )
+ # nsjail will complain if the out dir doesn't exist
+ os.makedirs("out", exist_ok=True)
+ os.makedirs(other_out_dir, exist_ok=True)
- diffs = _compare_outputs(module_to_outs, tempdir)
- if len(diffs) == 0:
- print("All modules are correct")
- elif args.show_diff:
- for m, d in diffs.items():
- print(f"Module {m} has diffs {d}")
- else:
- print(f"Modules {list(diffs.keys())} have diffs")
+ print("building...")
+ await asyncio.gather(
+ _build_with_soong("out", all_outs),
+ _build_with_soong(other_out_dir, all_outs, extra_env=other_env)
+ )
+
+ diffs = collections.defaultdict(dict)
+ for module, outs in module_to_outs.items():
+ for out in outs:
+ try:
+ subprocess.check_output(["diff", os.path.join(other_out_dir, out.removeprefix("out/")), out])
+ except subprocess.CalledProcessError as e:
+ diffs[module][out] = e.stdout
+
+ if len(diffs) == 0:
+ print("All modules are correct")
+ elif args.show_diff:
+ for m, files in diffs.items():
+ print(f"Module {m} has diffs:")
+ for f, d in files.items():
+ print(" "+f+":")
+ print(textwrap.indent(d, " "))
+ else:
+ print(f"Modules {list(diffs.keys())} have diffs in these files:")
+ all_diff_files = [f for m in diffs.values() for f in m]
+ for f in all_diff_files:
+ print(f)
+
if __name__ == "__main__":
- main()
+ asyncio.run(main())