Generate SBOM of products in Soong.
Bug: 324465531
Test: CIs
Test: m soong-sbom
Change-Id: If76776851d49282829a79bfb1c33f05b8f57de31
diff --git a/android/Android.bp b/android/Android.bp
index 774d24a..ce27241 100644
--- a/android/Android.bp
+++ b/android/Android.bp
@@ -93,6 +93,7 @@
"register.go",
"rule_builder.go",
"sandbox.go",
+ "sbom.go",
"sdk.go",
"sdk_version.go",
"shared_properties.go",
diff --git a/android/sbom.go b/android/sbom.go
new file mode 100644
index 0000000..dd2d2fa
--- /dev/null
+++ b/android/sbom.go
@@ -0,0 +1,100 @@
+// Copyright 2024 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+ "io"
+ "path/filepath"
+ "strings"
+
+ "github.com/google/blueprint"
+)
+
+var (
+ // Command line tool to generate SBOM in Soong
+ genSbom = pctx.HostBinToolVariable("genSbom", "gen_sbom")
+
+ // Command to generate SBOM in Soong.
+ genSbomRule = pctx.AndroidStaticRule("genSbomRule", blueprint.RuleParams{
+ Command: "rm -rf $out && ${genSbom} --output_file ${out} --metadata ${in} --product_out ${productOut} --soong_out ${soongOut} --build_version \"$$(cat ${buildFingerprintFile})\" --product_mfr \"${productManufacturer}\" --json",
+ CommandDeps: []string{"${genSbom}"},
+ }, "productOut", "soongOut", "buildFingerprintFile", "productManufacturer")
+)
+
+func init() {
+ RegisterSbomSingleton(InitRegistrationContext)
+}
+
+func RegisterSbomSingleton(ctx RegistrationContext) {
+ ctx.RegisterParallelSingletonType("sbom_singleton", sbomSingletonFactory)
+}
+
+// sbomSingleton is used to generate build actions of generating SBOM of products.
+type sbomSingleton struct{}
+
+func sbomSingletonFactory() Singleton {
+ return &sbomSingleton{}
+}
+
+// Generates SBOM of products
+func (this *sbomSingleton) GenerateBuildActions(ctx SingletonContext) {
+ if !ctx.Config().HasDeviceProduct() {
+ return
+ }
+ // Get all METADATA files and add them as implicit input
+ metadataFileListFile := PathForArbitraryOutput(ctx, ".module_paths", "METADATA.list")
+ f, err := ctx.Config().fs.Open(metadataFileListFile.String())
+ if err != nil {
+ panic(err)
+ }
+ b, err := io.ReadAll(f)
+ if err != nil {
+ panic(err)
+ }
+ allMetadataFiles := strings.Split(string(b), "\n")
+ implicits := []Path{metadataFileListFile}
+ for _, path := range allMetadataFiles {
+ implicits = append(implicits, PathForSource(ctx, path))
+ }
+ prodVars := ctx.Config().productVariables
+ buildFingerprintFile := PathForArbitraryOutput(ctx, "target", "product", String(prodVars.DeviceName), "build_fingerprint.txt")
+ implicits = append(implicits, buildFingerprintFile)
+
+ // Add installed_files.stamp as implicit input, which depends on all installed files of the product.
+ installedFilesStamp := PathForOutput(ctx, "compliance-metadata", ctx.Config().DeviceProduct(), "installed_files.stamp")
+ implicits = append(implicits, installedFilesStamp)
+
+ metadataDb := PathForOutput(ctx, "compliance-metadata", ctx.Config().DeviceProduct(), "compliance-metadata.db")
+ sbomFile := PathForOutput(ctx, "sbom", ctx.Config().DeviceProduct(), "sbom.spdx.json")
+ ctx.Build(pctx, BuildParams{
+ Rule: genSbomRule,
+ Input: metadataDb,
+ Implicits: implicits,
+ Output: sbomFile,
+ Args: map[string]string{
+ "productOut": filepath.Join(ctx.Config().OutDir(), "target", "product", String(prodVars.DeviceName)),
+ "soongOut": ctx.Config().soongOutDir,
+ "buildFingerprintFile": buildFingerprintFile.String(),
+ "productManufacturer": ctx.Config().ProductVariables().ProductManufacturer,
+ },
+ })
+
+ // Phony rule "soong-sbom". "m soong-sbom" to generate product SBOM in Soong.
+ ctx.Build(pctx, BuildParams{
+ Rule: blueprint.Phony,
+ Inputs: []Path{sbomFile},
+ Output: PathForPhony(ctx, "soong-sbom"),
+ })
+}
diff --git a/tests/sbom_test.sh b/tests/sbom_test.sh
index 8dc1630..794003d 100755
--- a/tests/sbom_test.sh
+++ b/tests/sbom_test.sh
@@ -70,13 +70,14 @@
# m droid, build sbom later in case additional dependencies might be built and included in partition images.
run_soong "${out_dir}" "droid dump.erofs lz4"
+ soong_sbom_out=$out_dir/soong/sbom/$target_product
product_out=$out_dir/target/product/vsoc_x86_64
sbom_test=$product_out/sbom_test
mkdir -p $sbom_test
cp $product_out/*.img $sbom_test
- # m sbom
- run_soong "${out_dir}" sbom
+ # m sbom soong-sbom
+ run_soong "${out_dir}" "sbom soong-sbom"
# Generate installed file list from .img files in PRODUCT_OUT
dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs
@@ -118,6 +119,7 @@
partition_name=$(basename $f | cut -d. -f1)
file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
+ files_in_soong_spdx_file="${sbom_test}/soong-sbom-${partition_name}-files-in-spdx.txt"
rm "$file_list_file" > /dev/null 2>&1 || true
all_dirs="/"
while [ ! -z "$all_dirs" ]; do
@@ -145,6 +147,7 @@
done
sort -n -o "$file_list_file" "$file_list_file"
+ # Diff the file list from image and file list in SBOM created by Make
grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_spdx_file"
if [ "$partition_name" = "system" ]; then
# system partition is mounted to /, so include FileName starts with /root/ too.
@@ -154,6 +157,17 @@
echo ============ Diffing files in $f and SBOM
diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" ""
+
+ # Diff the file list from image and file list in SBOM created by Soong
+ grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_soong_spdx_file"
+ if [ "$partition_name" = "system" ]; then
+ # system partition is mounted to /, so include FileName starts with /root/ too.
+ grep "FileName: /root/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: \/root//' >> "$files_in_soong_spdx_file"
+ fi
+ sort -n -o "$files_in_soong_spdx_file" "$files_in_soong_spdx_file"
+
+ echo ============ Diffing files in $f and SBOM created by Soong
+ diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" ""
done
RAMDISK_IMAGES="$product_out/ramdisk.img"
@@ -161,6 +175,7 @@
partition_name=$(basename $f | cut -d. -f1)
file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"
files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"
+ files_in_soong_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-soong-spdx.txt"
# lz4 decompress $f to stdout
# cpio list all entries like ls -l
# grep filter normal files and symlinks
@@ -170,11 +185,19 @@
grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_spdx_file"
+ grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_soong_spdx_file"
+
echo ============ Diffing files in $f and SBOM
diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" ""
+
+ echo ============ Diffing files in $f and SBOM created by Soong
+ diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" ""
done
verify_package_verification_code "$product_out/sbom.spdx"
+ verify_package_verification_code "$soong_sbom_out/sbom.spdx"
+
+ verify_packages_licenses "$soong_sbom_out/sbom.spdx"
# Teardown
cleanup "${out_dir}"
@@ -213,6 +236,41 @@
fi
}
+function verify_packages_licenses {
+ local sbom_file="$1"; shift
+
+ num_of_packages=$(grep 'PackageName:' $sbom_file | wc -l)
+ num_of_declared_licenses=$(grep 'PackageLicenseDeclared:' $sbom_file | wc -l)
+ if [ "$num_of_packages" = "$num_of_declared_licenses" ]
+ then
+ echo "Number of packages with declared license is correct."
+ else
+ echo "Number of packages with declared license is WRONG."
+ exit 1
+ fi
+
+ # PRODUCT and 7 prebuilt packages have "PackageLicenseDeclared: NOASSERTION"
+ # All other packages have declared licenses
+ num_of_packages_with_noassertion_license=$(grep 'PackageLicenseDeclared: NOASSERTION' $sbom_file | wc -l)
+ if [ $num_of_packages_with_noassertion_license = 15 ]
+ then
+ echo "Number of packages with NOASSERTION license is correct."
+ else
+ echo "Number of packages with NOASSERTION license is WRONG."
+ exit 1
+ fi
+
+ num_of_files=$(grep 'FileName:' $sbom_file | wc -l)
+ num_of_concluded_licenses=$(grep 'LicenseConcluded:' $sbom_file | wc -l)
+ if [ "$num_of_files" = "$num_of_concluded_licenses" ]
+ then
+ echo "Number of files with concluded license is correct."
+ else
+ echo "Number of files with concluded license is WRONG."
+ exit 1
+ fi
+}
+
function test_sbom_unbundled_apex {
# Setup
out_dir="$(setup)"
@@ -274,7 +332,7 @@
target_product=aosp_cf_x86_64_phone
target_release=trunk_staging
-target_build_variant=userdebug
+target_build_variant=eng
for i in "$@"; do
case $i in
TARGET_PRODUCT=*)
diff --git a/ui/build/test_build.go b/ui/build/test_build.go
index 687ad6f..3faa94d 100644
--- a/ui/build/test_build.go
+++ b/ui/build/test_build.go
@@ -15,14 +15,16 @@
package build
import (
- "android/soong/ui/metrics"
- "android/soong/ui/status"
"bufio"
"fmt"
"path/filepath"
+ "regexp"
"runtime"
"sort"
"strings"
+
+ "android/soong/ui/metrics"
+ "android/soong/ui/status"
)
// Checks for files in the out directory that have a rule that depends on them but no rule to
@@ -84,6 +86,10 @@
// before running soong and ninja.
releaseConfigDir := filepath.Join(outDir, "soong", "release-config")
+ // out/target/product/<xxxxx>/build_fingerprint.txt is a source file created in sysprop.mk
+ // ^out/target/product/[^/]+/build_fingerprint.txt$
+ buildFingerPrintFilePattern := regexp.MustCompile("^" + filepath.Join(outDir, "target", "product") + "/[^/]+/build_fingerprint.txt$")
+
danglingRules := make(map[string]bool)
scanner := bufio.NewScanner(stdout)
@@ -100,7 +106,8 @@
line == dexpreoptConfigFilePath ||
line == buildDatetimeFilePath ||
line == bpglob ||
- strings.HasPrefix(line, releaseConfigDir) {
+ strings.HasPrefix(line, releaseConfigDir) ||
+ buildFingerPrintFilePattern.MatchString(line) {
// Leaf node is in one of Soong's bootstrap directories, which do not have
// full build rules in the primary build.ninja file.
continue