Merge "Update tests to expect RBE not supported on linux." into main
diff --git a/android/Android.bp b/android/Android.bp
index 774d24a..ce27241 100644
--- a/android/Android.bp
+++ b/android/Android.bp
@@ -93,6 +93,7 @@
"register.go",
"rule_builder.go",
"sandbox.go",
+ "sbom.go",
"sdk.go",
"sdk_version.go",
"shared_properties.go",
diff --git a/android/config.go b/android/config.go
index eff9fdd..cadc929 100644
--- a/android/config.go
+++ b/android/config.go
@@ -1195,6 +1195,10 @@
return Bool(c.productVariables.UseGoma)
}
+func (c *config) UseABFS() bool {
+ return Bool(c.productVariables.UseABFS)
+}
+
func (c *config) UseRBE() bool {
return Bool(c.productVariables.UseRBE)
}
diff --git a/android/sbom.go b/android/sbom.go
new file mode 100644
index 0000000..dd2d2fa
--- /dev/null
+++ b/android/sbom.go
@@ -0,0 +1,100 @@
+// Copyright 2024 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+ "io"
+ "path/filepath"
+ "strings"
+
+ "github.com/google/blueprint"
+)
+
+var (
+ // Command line tool to generate SBOM in Soong
+ genSbom = pctx.HostBinToolVariable("genSbom", "gen_sbom")
+
+ // Command to generate SBOM in Soong.
+ genSbomRule = pctx.AndroidStaticRule("genSbomRule", blueprint.RuleParams{
+ Command: "rm -rf $out && ${genSbom} --output_file ${out} --metadata ${in} --product_out ${productOut} --soong_out ${soongOut} --build_version \"$$(cat ${buildFingerprintFile})\" --product_mfr \"${productManufacturer}\" --json",
+ CommandDeps: []string{"${genSbom}"},
+ }, "productOut", "soongOut", "buildFingerprintFile", "productManufacturer")
+)
+
+func init() {
+ RegisterSbomSingleton(InitRegistrationContext)
+}
+
+func RegisterSbomSingleton(ctx RegistrationContext) {
+ ctx.RegisterParallelSingletonType("sbom_singleton", sbomSingletonFactory)
+}
+
+// sbomSingleton is used to generate build actions of generating SBOM of products.
+type sbomSingleton struct{}
+
+func sbomSingletonFactory() Singleton {
+ return &sbomSingleton{}
+}
+
+// Generates SBOM of products
+func (this *sbomSingleton) GenerateBuildActions(ctx SingletonContext) {
+ if !ctx.Config().HasDeviceProduct() {
+ return
+ }
+ // Get all METADATA files and add them as implicit input
+ metadataFileListFile := PathForArbitraryOutput(ctx, ".module_paths", "METADATA.list")
+ f, err := ctx.Config().fs.Open(metadataFileListFile.String())
+ if err != nil {
+ panic(err)
+ }
+ b, err := io.ReadAll(f)
+ if err != nil {
+ panic(err)
+ }
+ allMetadataFiles := strings.Split(string(b), "\n")
+ implicits := []Path{metadataFileListFile}
+ for _, path := range allMetadataFiles {
+ implicits = append(implicits, PathForSource(ctx, path))
+ }
+ prodVars := ctx.Config().productVariables
+ buildFingerprintFile := PathForArbitraryOutput(ctx, "target", "product", String(prodVars.DeviceName), "build_fingerprint.txt")
+ implicits = append(implicits, buildFingerprintFile)
+
+ // Add installed_files.stamp as implicit input, which depends on all installed files of the product.
+ installedFilesStamp := PathForOutput(ctx, "compliance-metadata", ctx.Config().DeviceProduct(), "installed_files.stamp")
+ implicits = append(implicits, installedFilesStamp)
+
+ metadataDb := PathForOutput(ctx, "compliance-metadata", ctx.Config().DeviceProduct(), "compliance-metadata.db")
+ sbomFile := PathForOutput(ctx, "sbom", ctx.Config().DeviceProduct(), "sbom.spdx.json")
+ ctx.Build(pctx, BuildParams{
+ Rule: genSbomRule,
+ Input: metadataDb,
+ Implicits: implicits,
+ Output: sbomFile,
+ Args: map[string]string{
+ "productOut": filepath.Join(ctx.Config().OutDir(), "target", "product", String(prodVars.DeviceName)),
+ "soongOut": ctx.Config().soongOutDir,
+ "buildFingerprintFile": buildFingerprintFile.String(),
+ "productManufacturer": ctx.Config().ProductVariables().ProductManufacturer,
+ },
+ })
+
+ // Phony rule "soong-sbom". "m soong-sbom" to generate product SBOM in Soong.
+ ctx.Build(pctx, BuildParams{
+ Rule: blueprint.Phony,
+ Inputs: []Path{sbomFile},
+ Output: PathForPhony(ctx, "soong-sbom"),
+ })
+}
diff --git a/android/variable.go b/android/variable.go
index eb0e210..df0e59c 100644
--- a/android/variable.go
+++ b/android/variable.go
@@ -294,6 +294,7 @@
HostStaticBinaries *bool `json:",omitempty"`
Binder32bit *bool `json:",omitempty"`
UseGoma *bool `json:",omitempty"`
+ UseABFS *bool `json:",omitempty"`
UseRBE *bool `json:",omitempty"`
UseRBEJAVAC *bool `json:",omitempty"`
UseRBER8 *bool `json:",omitempty"`
diff --git a/bin/installmod b/bin/installmod
index 1d0d836..1ad5b84 100755
--- a/bin/installmod
+++ b/bin/installmod
@@ -28,7 +28,6 @@
return 1
fi
-local _path
_path=$(outmod ${@:$#:1})
if [ $? -ne 0 ]; then
return 1
@@ -39,7 +38,7 @@
echo "Module '$1' does not produce a file ending with .apk (try 'refreshmod' if there have been build changes?)" >&2
return 1
fi
-local serial_device=""
+serial_device=""
if [[ "$1" == "-s" ]]; then
if [[ $# -le 2 ]]; then
echo "-s requires an argument" >&2
@@ -48,7 +47,7 @@
serial_device="-s $2"
shift 2
fi
-local length=$(( $# - 1 ))
+length=$(( $# - 1 ))
echo adb $serial_device install ${@:1:$length} $_path
adb $serial_device install ${@:1:$length} $_path
diff --git a/cc/proto.go b/cc/proto.go
index 4d72f26..93142b9 100644
--- a/cc/proto.go
+++ b/cc/proto.go
@@ -19,6 +19,8 @@
"github.com/google/blueprint/proptools"
"android/soong/android"
+
+ "strings"
)
const (
@@ -35,13 +37,21 @@
srcSuffix = ".c"
}
+ srcInfix := "pb"
+ for _, value := range flags.proto.Flags {
+ if strings.HasPrefix(value, "--plugin=") && strings.HasSuffix(value, "protoc-gen-grpc-cpp-plugin") {
+ srcInfix = "grpc.pb"
+ break
+ }
+ }
+
if flags.proto.CanonicalPathFromRoot {
- ccFile = android.GenPathWithExt(ctx, "proto", protoFile, "pb"+srcSuffix)
- headerFile = android.GenPathWithExt(ctx, "proto", protoFile, "pb.h")
+ ccFile = android.GenPathWithExt(ctx, "proto", protoFile, srcInfix+srcSuffix)
+ headerFile = android.GenPathWithExt(ctx, "proto", protoFile, srcInfix+".h")
} else {
rel := protoFile.Rel()
- ccFile = android.PathForModuleGen(ctx, "proto", pathtools.ReplaceExtension(rel, "pb"+srcSuffix))
- headerFile = android.PathForModuleGen(ctx, "proto", pathtools.ReplaceExtension(rel, "pb.h"))
+ ccFile = android.PathForModuleGen(ctx, "proto", pathtools.ReplaceExtension(rel, srcInfix+srcSuffix))
+ headerFile = android.PathForModuleGen(ctx, "proto", pathtools.ReplaceExtension(rel, srcInfix+".h"))
}
protoDeps := flags.proto.Deps
diff --git a/cc/proto_test.go b/cc/proto_test.go
index abcb273..a905ea8 100644
--- a/cc/proto_test.go
+++ b/cc/proto_test.go
@@ -68,4 +68,36 @@
}
})
+ t.Run("grpc-cpp-plugin", func(t *testing.T) {
+ ctx := testCc(t, `
+ cc_binary_host {
+ name: "protoc-gen-grpc-cpp-plugin",
+ stl: "none",
+ }
+
+ cc_library_shared {
+ name: "libgrpc",
+ srcs: ["a.proto"],
+ proto: {
+ plugin: "grpc-cpp-plugin",
+ },
+ }`)
+
+ buildOS := ctx.Config().BuildOS.String()
+
+ proto := ctx.ModuleForTests("libgrpc", "android_arm_armv7-a-neon_shared").Output("proto/a.grpc.pb.cc")
+ grpcCppPlugin := ctx.ModuleForTests("protoc-gen-grpc-cpp-plugin", buildOS+"_x86_64")
+
+ cmd := proto.RuleParams.Command
+ if w := "--grpc-cpp-plugin_out="; !strings.Contains(cmd, w) {
+ t.Errorf("expected %q in %q", w, cmd)
+ }
+
+ grpcCppPluginPath := grpcCppPlugin.Module().(android.HostToolProvider).HostToolPath().RelativeToTop().String()
+
+ if w := "--plugin=protoc-gen-grpc-cpp-plugin=" + grpcCppPluginPath; !strings.Contains(cmd, w) {
+ t.Errorf("expected %q in %q", w, cmd)
+ }
+ })
+
}
diff --git a/java/app_import.go b/java/app_import.go
index fa87997..045a89a 100644
--- a/java/app_import.go
+++ b/java/app_import.go
@@ -431,6 +431,9 @@
var extraArgs []string
if a.Privileged() {
extraArgs = append(extraArgs, "--privileged")
+ if ctx.Config().UncompressPrivAppDex() {
+ extraArgs = append(extraArgs, "--uncompress-priv-app-dex")
+ }
}
if proptools.Bool(a.properties.Skip_preprocessed_apk_checks) {
extraArgs = append(extraArgs, "--skip-preprocessed-apk-checks")
diff --git a/java/app_import_test.go b/java/app_import_test.go
index 496fc13..54a5e75 100644
--- a/java/app_import_test.go
+++ b/java/app_import_test.go
@@ -777,30 +777,79 @@
}
func TestAndroidAppImport_Preprocessed(t *testing.T) {
- ctx, _ := testJava(t, `
- android_app_import {
- name: "foo",
- apk: "prebuilts/apk/app.apk",
- presigned: true,
- preprocessed: true,
- }
- `)
+ for _, dontUncompressPrivAppDexs := range []bool{false, true} {
+ name := fmt.Sprintf("dontUncompressPrivAppDexs:%t", dontUncompressPrivAppDexs)
+ t.Run(name, func(t *testing.T) {
+ result := android.GroupFixturePreparers(
+ PrepareForTestWithJavaDefaultModules,
+ android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) {
+ variables.UncompressPrivAppDex = proptools.BoolPtr(!dontUncompressPrivAppDexs)
+ }),
+ ).RunTestWithBp(t, `
+ android_app_import {
+ name: "foo",
+ apk: "prebuilts/apk/app.apk",
+ presigned: true,
+ preprocessed: true,
+ }
- apkName := "foo.apk"
- variant := ctx.ModuleForTests("foo", "android_common")
- outputBuildParams := variant.Output(apkName).BuildParams
- if outputBuildParams.Rule.String() != android.Cp.String() {
- t.Errorf("Unexpected prebuilt android_app_import rule: " + outputBuildParams.Rule.String())
- }
+ android_app_import {
+ name: "bar",
+ apk: "prebuilts/apk/app.apk",
+ presigned: true,
+ privileged: true,
+ preprocessed: true,
+ }
+ `)
- // Make sure compression and aligning were validated.
- if outputBuildParams.Validation == nil {
- t.Errorf("Expected validation rule, but was not found")
- }
+ // non-privileged app
+ apkName := "foo.apk"
+ variant := result.ModuleForTests("foo", "android_common")
+ outputBuildParams := variant.Output(apkName).BuildParams
+ if outputBuildParams.Rule.String() != android.Cp.String() {
+ t.Errorf("Unexpected prebuilt android_app_import rule: " + outputBuildParams.Rule.String())
+ }
- validationBuildParams := variant.Output("validated-prebuilt/check.stamp").BuildParams
- if validationBuildParams.Rule.String() != checkPresignedApkRule.String() {
- t.Errorf("Unexpected validation rule: " + validationBuildParams.Rule.String())
+ // Make sure compression and aligning were validated.
+ if outputBuildParams.Validation == nil {
+ t.Errorf("Expected validation rule, but was not found")
+ }
+
+ validationBuildParams := variant.Output("validated-prebuilt/check.stamp").BuildParams
+ if validationBuildParams.Rule.String() != checkPresignedApkRule.String() {
+ t.Errorf("Unexpected validation rule: " + validationBuildParams.Rule.String())
+ }
+
+ expectedScriptArgs := "--preprocessed"
+ actualScriptArgs := validationBuildParams.Args["extraArgs"]
+ android.AssertStringEquals(t, "check script extraArgs", expectedScriptArgs, actualScriptArgs)
+
+ // privileged app
+ apkName = "bar.apk"
+ variant = result.ModuleForTests("bar", "android_common")
+ outputBuildParams = variant.Output(apkName).BuildParams
+ if outputBuildParams.Rule.String() != android.Cp.String() {
+ t.Errorf("Unexpected prebuilt android_app_import rule: " + outputBuildParams.Rule.String())
+ }
+
+ // Make sure compression and aligning were validated.
+ if outputBuildParams.Validation == nil {
+ t.Errorf("Expected validation rule, but was not found")
+ }
+
+ validationBuildParams = variant.Output("validated-prebuilt/check.stamp").BuildParams
+ if validationBuildParams.Rule.String() != checkPresignedApkRule.String() {
+ t.Errorf("Unexpected validation rule: " + validationBuildParams.Rule.String())
+ }
+
+ expectedScriptArgs = "--privileged"
+ if !dontUncompressPrivAppDexs {
+ expectedScriptArgs += " --uncompress-priv-app-dex"
+ }
+ expectedScriptArgs += " --preprocessed"
+ actualScriptArgs = validationBuildParams.Args["extraArgs"]
+ android.AssertStringEquals(t, "check script extraArgs", expectedScriptArgs, actualScriptArgs)
+ })
}
}
diff --git a/java/ravenwood.go b/java/ravenwood.go
index 908619d..84c285c 100644
--- a/java/ravenwood.go
+++ b/java/ravenwood.go
@@ -33,6 +33,8 @@
var ravenwoodLibContentTag = dependencyTag{name: "ravenwoodlibcontent"}
var ravenwoodUtilsTag = dependencyTag{name: "ravenwoodutils"}
var ravenwoodRuntimeTag = dependencyTag{name: "ravenwoodruntime"}
+var ravenwoodDataTag = dependencyTag{name: "ravenwooddata"}
+var ravenwoodTestResourceApkTag = dependencyTag{name: "ravenwoodtestresapk"}
const ravenwoodUtilsName = "ravenwood-utils"
const ravenwoodRuntimeName = "ravenwood-runtime"
@@ -53,6 +55,13 @@
type ravenwoodTestProperties struct {
Jni_libs []string
+
+ // Specify another android_app module here to copy it to the test directory, so that
+ // the ravenwood test can access it.
+ // TODO: For now, we simply refer to another android_app module and copy it to the
+ // test directory. Eventually, android_ravenwood_test should support all the resource
+ // related properties and build resources from the `res/` directory.
+ Resource_apk *string
}
type ravenwoodTest struct {
@@ -114,6 +123,11 @@
for _, lib := range r.ravenwoodTestProperties.Jni_libs {
ctx.AddVariationDependencies(ctx.Config().BuildOSTarget.Variations(), jniLibTag, lib)
}
+
+ // Resources APK
+ if resourceApk := proptools.String(r.ravenwoodTestProperties.Resource_apk); resourceApk != "" {
+ ctx.AddVariationDependencies(nil, ravenwoodTestResourceApkTag, resourceApk)
+ }
}
func (r *ravenwoodTest) GenerateAndroidBuildActions(ctx android.ModuleContext) {
@@ -175,6 +189,14 @@
installDeps = append(installDeps, installJni)
}
+ resApkInstallPath := installPath.Join(ctx, "ravenwood-res-apks")
+ if resApk := ctx.GetDirectDepsWithTag(ravenwoodTestResourceApkTag); len(resApk) > 0 {
+ for _, installFile := range resApk[0].FilesToInstall() {
+ installResApk := ctx.InstallFile(resApkInstallPath, "ravenwood-res.apk", installFile)
+ installDeps = append(installDeps, installResApk)
+ }
+ }
+
// Install our JAR with all dependencies
ctx.InstallFile(installPath, ctx.ModuleName()+".jar", r.outputFile, installDeps...)
}
@@ -198,6 +220,9 @@
Libs []string
Jni_libs []string
+
+ // We use this to copy framework-res.apk to the ravenwood runtime directory.
+ Data []string
}
type ravenwoodLibgroup struct {
@@ -236,6 +261,9 @@
for _, lib := range r.ravenwoodLibgroupProperties.Jni_libs {
ctx.AddVariationDependencies(ctx.Config().BuildOSTarget.Variations(), jniLibTag, lib)
}
+ for _, data := range r.ravenwoodLibgroupProperties.Data {
+ ctx.AddVariationDependencies(nil, ravenwoodDataTag, data)
+ }
}
func (r *ravenwoodLibgroup) GenerateAndroidBuildActions(ctx android.ModuleContext) {
@@ -266,6 +294,13 @@
ctx.InstallFile(soInstallPath, jniLib.path.Base(), jniLib.path)
}
+ dataInstallPath := installPath.Join(ctx, "ravenwood-data")
+ for _, data := range r.ravenwoodLibgroupProperties.Data {
+ libModule := ctx.GetDirectDepWithTag(data, ravenwoodDataTag)
+ file := android.OutputFileForModule(ctx, libModule, "")
+ ctx.InstallFile(dataInstallPath, file.Base(), file)
+ }
+
// Normal build should perform install steps
ctx.Phony(r.BaseModuleName(), android.PathForPhony(ctx, r.BaseModuleName()+"-install"))
}
diff --git a/java/ravenwood_test.go b/java/ravenwood_test.go
index 5961264..d26db93 100644
--- a/java/ravenwood_test.go
+++ b/java/ravenwood_test.go
@@ -57,6 +57,14 @@
name: "framework-rules.ravenwood",
srcs: ["Rules.java"],
}
+ android_app {
+ name: "app1",
+ sdk_version: "current",
+ }
+ android_app {
+ name: "app2",
+ sdk_version: "current",
+ }
android_ravenwood_libgroup {
name: "ravenwood-runtime",
libs: [
@@ -67,6 +75,9 @@
"ravenwood-runtime-jni1",
"ravenwood-runtime-jni2",
],
+ data: [
+ "app1",
+ ],
}
android_ravenwood_libgroup {
name: "ravenwood-utils",
@@ -102,6 +113,7 @@
runtime.Output(installPathPrefix + "/ravenwood-runtime/lib64/ravenwood-runtime-jni1.so")
runtime.Output(installPathPrefix + "/ravenwood-runtime/lib64/libred.so")
runtime.Output(installPathPrefix + "/ravenwood-runtime/lib64/ravenwood-runtime-jni3.so")
+ runtime.Output(installPathPrefix + "/ravenwood-runtime/ravenwood-data/app1.apk")
utils := ctx.ModuleForTests("ravenwood-utils", "android_common")
utils.Output(installPathPrefix + "/ravenwood-utils/framework-rules.ravenwood.jar")
}
@@ -143,6 +155,7 @@
"jni-lib2",
"ravenwood-runtime-jni2",
],
+ resource_apk: "app2",
sdk_version: "test_current",
}
`)
@@ -169,6 +182,7 @@
module.Output(installPathPrefix + "/ravenwood-test/lib64/jni-lib1.so")
module.Output(installPathPrefix + "/ravenwood-test/lib64/libblue.so")
module.Output(installPathPrefix + "/ravenwood-test/lib64/libpink.so")
+ module.Output(installPathPrefix + "/ravenwood-test/ravenwood-res-apks/ravenwood-res.apk")
// ravenwood-runtime*.so are included in the runtime, so it shouldn't be emitted.
for _, o := range module.AllOutputs() {
diff --git a/phony/phony.go b/phony/phony.go
index b421176..5505a3a 100644
--- a/phony/phony.go
+++ b/phony/phony.go
@@ -20,6 +20,8 @@
"strings"
"android/soong/android"
+
+ "github.com/google/blueprint/proptools"
)
func init() {
@@ -88,14 +90,15 @@
android.ModuleBase
android.DefaultableModuleBase
- properties PhonyProperties
+ phonyDepsModuleNames []string
+ properties PhonyProperties
}
type PhonyProperties struct {
// The Phony_deps is the set of all dependencies for this target,
// and it can function similarly to .PHONY in a makefile.
// Additionally, dependencies within it can even include genrule.
- Phony_deps []string
+ Phony_deps proptools.Configurable[[]string]
}
// The phony_rule provides functionality similar to the .PHONY in a makefile.
@@ -109,13 +112,14 @@
}
func (p *PhonyRule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+ p.phonyDepsModuleNames = p.properties.Phony_deps.GetOrDefault(p.ConfigurableEvaluator(ctx), nil)
}
func (p *PhonyRule) AndroidMk() android.AndroidMkData {
return android.AndroidMkData{
Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
- if len(p.properties.Phony_deps) > 0 {
- depModulesStr := strings.Join(p.properties.Phony_deps, " ")
+ if len(p.phonyDepsModuleNames) > 0 {
+ depModulesStr := strings.Join(p.phonyDepsModuleNames, " ")
fmt.Fprintln(w, ".PHONY:", name)
fmt.Fprintln(w, name, ":", depModulesStr)
}
diff --git a/scripts/check_prebuilt_presigned_apk.py b/scripts/check_prebuilt_presigned_apk.py
index abedfb7..abab2e1 100755
--- a/scripts/check_prebuilt_presigned_apk.py
+++ b/scripts/check_prebuilt_presigned_apk.py
@@ -37,7 +37,7 @@
sys.exit(args.apk + ': Contains compressed JNI libraries')
return True
# It's ok for non-privileged apps to have compressed dex files, see go/gms-uncompressed-jni-slides
- if args.privileged:
+ if args.privileged and args.uncompress_priv_app_dex:
if info.filename.endswith('.dex') and info.compress_type != zipfile.ZIP_STORED:
if fail:
sys.exit(args.apk + ': Contains compressed dex files and is privileged')
@@ -52,6 +52,7 @@
parser.add_argument('--skip-preprocessed-apk-checks', action = 'store_true', help = "the value of the soong property with the same name")
parser.add_argument('--preprocessed', action = 'store_true', help = "the value of the soong property with the same name")
parser.add_argument('--privileged', action = 'store_true', help = "the value of the soong property with the same name")
+ parser.add_argument('--uncompress-priv-app-dex', action = 'store_true', help = "the value of the product variable with the same name")
parser.add_argument('apk', help = "the apk to check")
parser.add_argument('stampfile', help = "a file to touch if successful")
args = parser.parse_args()
diff --git a/tests/sbom_test.sh b/tests/sbom_test.sh
index 8dc1630..794003d 100755
--- a/tests/sbom_test.sh
+++ b/tests/sbom_test.sh
@@ -70,13 +70,14 @@
# m droid, build sbom later in case additional dependencies might be built and included in partition images.
run_soong "${out_dir}" "droid dump.erofs lz4"
+ soong_sbom_out=$out_dir/soong/sbom/$target_product
product_out=$out_dir/target/product/vsoc_x86_64
sbom_test=$product_out/sbom_test
mkdir -p $sbom_test
cp $product_out/*.img $sbom_test
- # m sbom
- run_soong "${out_dir}" sbom
+ # m sbom soong-sbom
+ run_soong "${out_dir}" "sbom soong-sbom"
# Generate installed file list from .img files in PRODUCT_OUT
dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs
@@ -118,6 +119,7 @@
partition_name=$(basename $f | cut -d. -f1)
file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
+ files_in_soong_spdx_file="${sbom_test}/soong-sbom-${partition_name}-files-in-spdx.txt"
rm "$file_list_file" > /dev/null 2>&1 || true
all_dirs="/"
while [ ! -z "$all_dirs" ]; do
@@ -145,6 +147,7 @@
done
sort -n -o "$file_list_file" "$file_list_file"
+ # Diff the file list from image and file list in SBOM created by Make
grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_spdx_file"
if [ "$partition_name" = "system" ]; then
# system partition is mounted to /, so include FileName starts with /root/ too.
@@ -154,6 +157,17 @@
echo ============ Diffing files in $f and SBOM
diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" ""
+
+ # Diff the file list from image and file list in SBOM created by Soong
+ grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_soong_spdx_file"
+ if [ "$partition_name" = "system" ]; then
+ # system partition is mounted to /, so include FileName starts with /root/ too.
+ grep "FileName: /root/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: \/root//' >> "$files_in_soong_spdx_file"
+ fi
+ sort -n -o "$files_in_soong_spdx_file" "$files_in_soong_spdx_file"
+
+ echo ============ Diffing files in $f and SBOM created by Soong
+ diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" ""
done
RAMDISK_IMAGES="$product_out/ramdisk.img"
@@ -161,6 +175,7 @@
partition_name=$(basename $f | cut -d. -f1)
file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
+ files_in_soong_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-soong-spdx.txt"
# lz4 decompress $f to stdout
# cpio list all entries like ls -l
# grep filter normal files and symlinks
@@ -170,11 +185,19 @@
grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_spdx_file"
+ grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_soong_spdx_file"
+
echo ============ Diffing files in $f and SBOM
diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" ""
+
+ echo ============ Diffing files in $f and SBOM created by Soong
+ diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" ""
done
verify_package_verification_code "$product_out/sbom.spdx"
+ verify_package_verification_code "$soong_sbom_out/sbom.spdx"
+
+ verify_packages_licenses "$soong_sbom_out/sbom.spdx"
# Teardown
cleanup "${out_dir}"
@@ -213,6 +236,41 @@
fi
}
+function verify_packages_licenses {
+ local sbom_file="$1"; shift
+
+ num_of_packages=$(grep 'PackageName:' $sbom_file | wc -l)
+ num_of_declared_licenses=$(grep 'PackageLicenseDeclared:' $sbom_file | wc -l)
+ if [ "$num_of_packages" = "$num_of_declared_licenses" ]
+ then
+ echo "Number of packages with declared license is correct."
+ else
+ echo "Number of packages with declared license is WRONG."
+ exit 1
+ fi
+
+ # PRODUCT and 7 prebuilt packages have "PackageLicenseDeclared: NOASSERTION"
+ # All other packages have declared licenses
+ num_of_packages_with_noassertion_license=$(grep 'PackageLicenseDeclared: NOASSERTION' $sbom_file | wc -l)
+ if [ $num_of_packages_with_noassertion_license = 15 ]
+ then
+ echo "Number of packages with NOASSERTION license is correct."
+ else
+ echo "Number of packages with NOASSERTION license is WRONG."
+ exit 1
+ fi
+
+ num_of_files=$(grep 'FileName:' $sbom_file | wc -l)
+ num_of_concluded_licenses=$(grep 'LicenseConcluded:' $sbom_file | wc -l)
+ if [ "$num_of_files" = "$num_of_concluded_licenses" ]
+ then
+ echo "Number of files with concluded license is correct."
+ else
+ echo "Number of files with concluded license is WRONG."
+ exit 1
+ fi
+}
+
function test_sbom_unbundled_apex {
# Setup
out_dir="$(setup)"
@@ -274,7 +332,7 @@
target_product=aosp_cf_x86_64_phone
target_release=trunk_staging
-target_build_variant=userdebug
+target_build_variant=eng
for i in "$@"; do
case $i in
TARGET_PRODUCT=*)
diff --git a/ui/build/build.go b/ui/build/build.go
index c7319ed..49ac791 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -211,9 +211,38 @@
}
}
+func abfsBuildStarted(ctx Context, config Config) {
+ abfsBox := config.PrebuiltBuildTool("abfsbox")
+ cmdArgs := []string{"build-started", "--"}
+ cmdArgs = append(cmdArgs, config.Arguments()...)
+ cmd := Command(ctx, config, "abfsbox", abfsBox, cmdArgs...)
+ cmd.Sandbox = noSandbox
+ cmd.RunAndPrintOrFatal()
+}
+
+func abfsBuildFinished(ctx Context, config Config, finished bool) {
+ var errMsg string
+ if !finished {
+ errMsg = "build was interrupted"
+ }
+ abfsBox := config.PrebuiltBuildTool("abfsbox")
+ cmdArgs := []string{"build-finished", "-e", errMsg, "--"}
+ cmdArgs = append(cmdArgs, config.Arguments()...)
+ cmd := Command(ctx, config, "abfsbox", abfsBox, cmdArgs...)
+ cmd.RunAndPrintOrFatal()
+}
+
// Build the tree. Various flags in `config` govern which components of
// the build to run.
func Build(ctx Context, config Config) {
+ done := false
+ if config.UseABFS() {
+ abfsBuildStarted(ctx, config)
+ defer func() {
+ abfsBuildFinished(ctx, config, done)
+ }()
+ }
+
ctx.Verboseln("Starting build with args:", config.Arguments())
ctx.Verboseln("Environment:", config.Environment().Environ())
@@ -351,6 +380,7 @@
if what&RunDistActions != 0 {
runDistActions(ctx, config)
}
+ done = true
}
func updateBuildIdDir(ctx Context, config Config) {
diff --git a/ui/build/config.go b/ui/build/config.go
index 2470f84..631b76f 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -1270,9 +1270,25 @@
if !c.StubbyExists() && strings.Contains(authType, "use_google_prod_creds") {
return false
}
+ if c.UseABFS() {
+ return false
+ }
return true
}
+func (c *configImpl) UseABFS() bool {
+ if v, ok := c.environ.Get("NO_ABFS"); ok {
+ v = strings.ToLower(strings.TrimSpace(v))
+ if v == "true" || v == "1" {
+ return false
+ }
+ }
+
+ abfsBox := c.PrebuiltBuildTool("abfsbox")
+ err := exec.Command(abfsBox, "hash", srcDirFileCheck).Run()
+ return err == nil
+}
+
func (c *configImpl) UseRBE() bool {
// These alternate modes of running Soong do not use RBE / reclient.
if c.Queryview() || c.JsonModuleGraph() {
@@ -1585,6 +1601,23 @@
}
}
+func (c *configImpl) KatiBin() string {
+ binName := "ckati"
+ if c.UseABFS() {
+ binName = "ckati-wrap"
+ }
+
+ return c.PrebuiltBuildTool(binName)
+}
+
+func (c *configImpl) NinjaBin() string {
+ binName := "ninja"
+ if c.UseABFS() {
+ binName = "ninjago"
+ }
+ return c.PrebuiltBuildTool(binName)
+}
+
func (c *configImpl) PrebuiltBuildTool(name string) string {
if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
if sanitize := strings.Fields(v); inList("address", sanitize) {
diff --git a/ui/build/dumpvars.go b/ui/build/dumpvars.go
index e77df44..5df3a95 100644
--- a/ui/build/dumpvars.go
+++ b/ui/build/dumpvars.go
@@ -93,7 +93,7 @@
defer tool.Finish()
cmd := Command(ctx, config, "dumpvars",
- config.PrebuiltBuildTool("ckati"),
+ config.KatiBin(),
"-f", "build/make/core/config.mk",
"--color_warnings",
"--kati_stats",
diff --git a/ui/build/kati.go b/ui/build/kati.go
index d599c99..a0efd2c 100644
--- a/ui/build/kati.go
+++ b/ui/build/kati.go
@@ -84,7 +84,7 @@
// arguments, and a custom function closure to mutate the environment Kati runs
// in.
func runKati(ctx Context, config Config, extraSuffix string, args []string, envFunc func(*Environment)) {
- executable := config.PrebuiltBuildTool("ckati")
+ executable := config.KatiBin()
// cKati arguments.
args = append([]string{
// Instead of executing commands directly, generate a Ninja file.
diff --git a/ui/build/ninja.go b/ui/build/ninja.go
index 1935e72..b5e74b4 100644
--- a/ui/build/ninja.go
+++ b/ui/build/ninja.go
@@ -49,7 +49,7 @@
nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
defer nr.Close()
- executable := config.PrebuiltBuildTool("ninja")
+ executable := config.NinjaBin()
args := []string{
"-d", "keepdepfile",
"-d", "keeprsp",
diff --git a/ui/build/sandbox_linux.go b/ui/build/sandbox_linux.go
index edb3b66..5c3fec1 100644
--- a/ui/build/sandbox_linux.go
+++ b/ui/build/sandbox_linux.go
@@ -200,6 +200,9 @@
// Only log important warnings / errors
"-q",
}
+ if c.config.UseABFS() {
+ sandboxArgs = append(sandboxArgs, "-B", "{ABFS_DIR}")
+ }
// Mount srcDir RW allowlists as Read-Write
if len(c.config.sandboxConfig.SrcDirRWAllowlist()) > 0 && !c.config.sandboxConfig.SrcDirIsRO() {
diff --git a/ui/build/soong.go b/ui/build/soong.go
index e18cc25..2ccdfec 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -661,7 +661,7 @@
}
ninjaArgs = append(ninjaArgs, targets...)
- ninjaCmd := config.PrebuiltBuildTool("ninja")
+ ninjaCmd := config.NinjaBin()
if config.useN2 {
ninjaCmd = config.PrebuiltBuildTool("n2")
}
diff --git a/ui/build/test_build.go b/ui/build/test_build.go
index 687ad6f..3faa94d 100644
--- a/ui/build/test_build.go
+++ b/ui/build/test_build.go
@@ -15,14 +15,16 @@
package build
import (
- "android/soong/ui/metrics"
- "android/soong/ui/status"
"bufio"
"fmt"
"path/filepath"
+ "regexp"
"runtime"
"sort"
"strings"
+
+ "android/soong/ui/metrics"
+ "android/soong/ui/status"
)
// Checks for files in the out directory that have a rule that depends on them but no rule to
@@ -84,6 +86,10 @@
// before running soong and ninja.
releaseConfigDir := filepath.Join(outDir, "soong", "release-config")
+ // out/target/product/<xxxxx>/build_fingerprint.txt is a source file created in sysprop.mk
+ // ^out/target/product/[^/]+/build_fingerprint.txt$
+ buildFingerPrintFilePattern := regexp.MustCompile("^" + filepath.Join(outDir, "target", "product") + "/[^/]+/build_fingerprint.txt$")
+
danglingRules := make(map[string]bool)
scanner := bufio.NewScanner(stdout)
@@ -100,7 +106,8 @@
line == dexpreoptConfigFilePath ||
line == buildDatetimeFilePath ||
line == bpglob ||
- strings.HasPrefix(line, releaseConfigDir) {
+ strings.HasPrefix(line, releaseConfigDir) ||
+ buildFingerPrintFilePattern.MatchString(line) {
// Leaf node is in one of Soong's bootstrap directories, which do not have
// full build rules in the primary build.ninja file.
continue