Export Soong modules and build a database from metadata from Make and Soong.
Bug: 324465531
Test: CIs
Test: m compliance-metadata.db
Change-Id: Ia1c9ab0ae874dd47969555ddbfb93405b57a651f
diff --git a/android/Android.bp b/android/Android.bp
index ce8c9b0..985ffa9 100644
--- a/android/Android.bp
+++ b/android/Android.bp
@@ -40,6 +40,7 @@
"base_module_context.go",
"build_prop.go",
"buildinfo_prop.go",
+ "compliance_metadata.go",
"config.go",
"test_config.go",
"configurable_properties.go",
diff --git a/android/compliance_metadata.go b/android/compliance_metadata.go
new file mode 100644
index 0000000..6ea6654
--- /dev/null
+++ b/android/compliance_metadata.go
@@ -0,0 +1,314 @@
+// Copyright 2024 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+ "bytes"
+ "encoding/csv"
+ "fmt"
+ "slices"
+ "strconv"
+ "strings"
+
+ "github.com/google/blueprint"
+)
+
+var (
+ // Constants of property names used in compliance metadata of modules
+ ComplianceMetadataProp = struct {
+ NAME string
+ PACKAGE string
+ MODULE_TYPE string
+ OS string
+ ARCH string
+ IS_PRIMARY_ARCH string
+ VARIANT string
+ IS_STATIC_LIB string
+ INSTALLED_FILES string
+ BUILT_FILES string
+ STATIC_DEPS string
+ STATIC_DEP_FILES string
+ WHOLE_STATIC_DEPS string
+ WHOLE_STATIC_DEP_FILES string
+ LICENSES string
+
+ // module_type=package
+ PKG_DEFAULT_APPLICABLE_LICENSES string
+
+ // module_type=license
+ LIC_LICENSE_KINDS string
+ LIC_LICENSE_TEXT string
+ LIC_PACKAGE_NAME string
+
+ // module_type=license_kind
+ LK_CONDITIONS string
+ LK_URL string
+ }{
+ "name",
+ "package",
+ "module_type",
+ "os",
+ "arch",
+ "is_primary_arch",
+ "variant",
+ "is_static_lib",
+ "installed_files",
+ "built_files",
+ "static_deps",
+ "static_dep_files",
+ "whole_static_deps",
+ "whole_static_dep_files",
+ "licenses",
+
+ "pkg_default_applicable_licenses",
+
+ "lic_license_kinds",
+ "lic_license_text",
+ "lic_package_name",
+
+ "lk_conditions",
+ "lk_url",
+ }
+
+ // A constant list of all property names in compliance metadata
+ // Order of properties here is the order of columns in the exported CSV file.
+ COMPLIANCE_METADATA_PROPS = []string{
+ ComplianceMetadataProp.NAME,
+ ComplianceMetadataProp.PACKAGE,
+ ComplianceMetadataProp.MODULE_TYPE,
+ ComplianceMetadataProp.OS,
+ ComplianceMetadataProp.ARCH,
+ ComplianceMetadataProp.VARIANT,
+ ComplianceMetadataProp.IS_STATIC_LIB,
+ ComplianceMetadataProp.IS_PRIMARY_ARCH,
+ // Space separated installed files
+ ComplianceMetadataProp.INSTALLED_FILES,
+ // Space separated built files
+ ComplianceMetadataProp.BUILT_FILES,
+ // Space separated module names of static dependencies
+ ComplianceMetadataProp.STATIC_DEPS,
+ // Space separated file paths of static dependencies
+ ComplianceMetadataProp.STATIC_DEP_FILES,
+ // Space separated module names of whole static dependencies
+ ComplianceMetadataProp.WHOLE_STATIC_DEPS,
+ // Space separated file paths of whole static dependencies
+ ComplianceMetadataProp.WHOLE_STATIC_DEP_FILES,
+ ComplianceMetadataProp.LICENSES,
+ // module_type=package
+ ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES,
+ // module_type=license
+ ComplianceMetadataProp.LIC_LICENSE_KINDS,
+ ComplianceMetadataProp.LIC_LICENSE_TEXT, // resolve to file paths
+ ComplianceMetadataProp.LIC_PACKAGE_NAME,
+ // module_type=license_kind
+ ComplianceMetadataProp.LK_CONDITIONS,
+ ComplianceMetadataProp.LK_URL,
+ }
+)
+
+// ComplianceMetadataInfo provides all metadata of a module, e.g. name, module type, package, license,
+// dependencies, built/installed files, etc. It is a wrapper on a map[string]string with some utility
+// methods to get/set properties' values.
+type ComplianceMetadataInfo struct {
+ properties map[string]string
+}
+
+func NewComplianceMetadataInfo() *ComplianceMetadataInfo {
+ return &ComplianceMetadataInfo{
+ properties: map[string]string{},
+ }
+}
+
+func (c *ComplianceMetadataInfo) SetStringValue(propertyName string, value string) {
+ if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) {
+ panic(fmt.Errorf("Unknown metadata property: %s.", propertyName))
+ }
+ c.properties[propertyName] = value
+}
+
+func (c *ComplianceMetadataInfo) SetListValue(propertyName string, value []string) {
+ c.SetStringValue(propertyName, strings.TrimSpace(strings.Join(value, " ")))
+}
+
+func (c *ComplianceMetadataInfo) getStringValue(propertyName string) string {
+ if !slices.Contains(COMPLIANCE_METADATA_PROPS, propertyName) {
+ panic(fmt.Errorf("Unknown metadata property: %s.", propertyName))
+ }
+ return c.properties[propertyName]
+}
+
+func (c *ComplianceMetadataInfo) getAllValues() map[string]string {
+ return c.properties
+}
+
+var (
+ ComplianceMetadataProvider = blueprint.NewProvider[*ComplianceMetadataInfo]()
+)
+
+// buildComplianceMetadataProvider starts with the ModuleContext.ComplianceMetadataInfo() and fills in more common metadata
+// for different module types without accessing their private fields but through android.Module interface
+// and public/private fields of package android. The final metadata is stored to a module's ComplianceMetadataProvider.
+func buildComplianceMetadataProvider(ctx ModuleContext, m *ModuleBase) {
+ complianceMetadataInfo := ctx.ComplianceMetadataInfo()
+ complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.NAME, m.Name())
+ complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.PACKAGE, ctx.ModuleDir())
+ complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.MODULE_TYPE, ctx.ModuleType())
+
+ switch ctx.ModuleType() {
+ case "license":
+ licenseModule := m.module.(*licenseModule)
+ complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_KINDS, licenseModule.properties.License_kinds)
+ complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LIC_LICENSE_TEXT, PathsForModuleSrc(ctx, licenseModule.properties.License_text).Strings())
+ complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LIC_PACKAGE_NAME, String(licenseModule.properties.Package_name))
+ case "license_kind":
+ licenseKindModule := m.module.(*licenseKindModule)
+ complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LK_CONDITIONS, licenseKindModule.properties.Conditions)
+ complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.LK_URL, licenseKindModule.properties.Url)
+ default:
+ complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.OS, ctx.Os().String())
+ complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.ARCH, ctx.Arch().String())
+ complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.IS_PRIMARY_ARCH, strconv.FormatBool(ctx.PrimaryArch()))
+ complianceMetadataInfo.SetStringValue(ComplianceMetadataProp.VARIANT, ctx.ModuleSubDir())
+ if m.primaryLicensesProperty != nil && m.primaryLicensesProperty.getName() == "licenses" {
+ complianceMetadataInfo.SetListValue(ComplianceMetadataProp.LICENSES, m.primaryLicensesProperty.getStrings())
+ }
+
+ var installed InstallPaths
+ installed = append(installed, m.module.FilesToInstall()...)
+ installed = append(installed, m.katiInstalls.InstallPaths()...)
+ installed = append(installed, m.katiSymlinks.InstallPaths()...)
+ installed = append(installed, m.katiInitRcInstalls.InstallPaths()...)
+ installed = append(installed, m.katiVintfInstalls.InstallPaths()...)
+ complianceMetadataInfo.SetListValue(ComplianceMetadataProp.INSTALLED_FILES, FirstUniqueStrings(installed.Strings()))
+ }
+ ctx.setProvider(ComplianceMetadataProvider, complianceMetadataInfo)
+}
+
+func init() {
+ RegisterComplianceMetadataSingleton(InitRegistrationContext)
+}
+
+func RegisterComplianceMetadataSingleton(ctx RegistrationContext) {
+ ctx.RegisterParallelSingletonType("compliance_metadata_singleton", complianceMetadataSingletonFactory)
+}
+
+var (
+ // sqlite3 command line tool
+ sqlite3 = pctx.HostBinToolVariable("sqlite3", "sqlite3")
+
+ // Command to import .csv files to sqlite3 database
+ importCsv = pctx.AndroidStaticRule("importCsv",
+ blueprint.RuleParams{
+ Command: `rm -rf $out && ` +
+ `${sqlite3} $out ".import --csv $in modules" && ` +
+ `${sqlite3} $out ".import --csv ${make_metadata} make_metadata" && ` +
+ `${sqlite3} $out ".import --csv ${make_modules} make_modules"`,
+ CommandDeps: []string{"${sqlite3}"},
+ }, "make_metadata", "make_modules")
+)
+
+func complianceMetadataSingletonFactory() Singleton {
+ return &complianceMetadataSingleton{}
+}
+
+type complianceMetadataSingleton struct {
+}
+
+func writerToCsv(csvWriter *csv.Writer, row []string) {
+ err := csvWriter.Write(row)
+ if err != nil {
+ panic(err)
+ }
+}
+
+// Collect compliance metadata from all Soong modules, write to a CSV file and
+// import compliance metadata from Make and Soong to a sqlite3 database.
+func (c *complianceMetadataSingleton) GenerateBuildActions(ctx SingletonContext) {
+ if !ctx.Config().HasDeviceProduct() {
+ return
+ }
+ var buffer bytes.Buffer
+ csvWriter := csv.NewWriter(&buffer)
+
+ // Collect compliance metadata of modules in Soong and write to out/soong/compliance-metadata/<product>/soong-modules.csv file.
+ columnNames := []string{"id"}
+ columnNames = append(columnNames, COMPLIANCE_METADATA_PROPS...)
+ writerToCsv(csvWriter, columnNames)
+
+ rowId := -1
+ ctx.VisitAllModules(func(module Module) {
+ if !module.Enabled(ctx) {
+ return
+ }
+ moduleType := ctx.ModuleType(module)
+ if moduleType == "package" {
+ metadataMap := map[string]string{
+ ComplianceMetadataProp.NAME: ctx.ModuleName(module),
+ ComplianceMetadataProp.MODULE_TYPE: ctx.ModuleType(module),
+ ComplianceMetadataProp.PKG_DEFAULT_APPLICABLE_LICENSES: strings.Join(module.base().primaryLicensesProperty.getStrings(), " "),
+ }
+ rowId = rowId + 1
+ metadata := []string{strconv.Itoa(rowId)}
+ for _, propertyName := range COMPLIANCE_METADATA_PROPS {
+ metadata = append(metadata, metadataMap[propertyName])
+ }
+ writerToCsv(csvWriter, metadata)
+ return
+ }
+ if provider, ok := ctx.moduleProvider(module, ComplianceMetadataProvider); ok {
+ metadataInfo := provider.(*ComplianceMetadataInfo)
+ rowId = rowId + 1
+ metadata := []string{strconv.Itoa(rowId)}
+ for _, propertyName := range COMPLIANCE_METADATA_PROPS {
+ metadata = append(metadata, metadataInfo.getStringValue(propertyName))
+ }
+ writerToCsv(csvWriter, metadata)
+ return
+ }
+ })
+ csvWriter.Flush()
+
+ deviceProduct := ctx.Config().DeviceProduct()
+ modulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "soong-modules.csv")
+ WriteFileRuleVerbatim(ctx, modulesCsv, buffer.String())
+
+ // Metadata generated in Make
+ makeMetadataCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-metadata.csv")
+ makeModulesCsv := PathForOutput(ctx, "compliance-metadata", deviceProduct, "make-modules.csv")
+
+ // Import metadata from Make and Soong to sqlite3 database
+ complianceMetadataDb := PathForOutput(ctx, "compliance-metadata", deviceProduct, "compliance-metadata.db")
+ ctx.Build(pctx, BuildParams{
+ Rule: importCsv,
+ Input: modulesCsv,
+ Implicits: []Path{
+ makeMetadataCsv,
+ makeModulesCsv,
+ },
+ Output: complianceMetadataDb,
+ Args: map[string]string{
+ "make_metadata": makeMetadataCsv.String(),
+ "make_modules": makeModulesCsv.String(),
+ },
+ })
+
+ // Phony rule "compliance-metadata.db". "m compliance-metadata.db" to create the compliance metadata database.
+ ctx.Build(pctx, BuildParams{
+ Rule: blueprint.Phony,
+ Inputs: []Path{complianceMetadataDb},
+ Output: PathForPhony(ctx, "compliance-metadata.db"),
+ })
+
+}
diff --git a/android/module.go b/android/module.go
index a316a1b..eb949e4 100644
--- a/android/module.go
+++ b/android/module.go
@@ -919,6 +919,10 @@
// outputFiles stores the output of a module by tag and is used to set
// the OutputFilesProvider in GenerateBuildActions
outputFiles OutputFilesInfo
+
+ // complianceMetadataInfo is for different module types to dump metadata.
+ // See android.ModuleContext interface.
+ complianceMetadataInfo *ComplianceMetadataInfo
}
func (m *ModuleBase) AddJSONData(d *map[string]interface{}) {
@@ -2049,6 +2053,8 @@
if m.outputFiles.DefaultOutputFiles != nil || m.outputFiles.TaggedOutputFiles != nil {
SetProvider(ctx, OutputFilesProvider, m.outputFiles)
}
+
+ buildComplianceMetadataProvider(ctx, m)
}
func SetJarJarPrefixHandler(handler func(ModuleContext)) {
diff --git a/android/module_context.go b/android/module_context.go
index e2677a4..2e16a24 100644
--- a/android/module_context.go
+++ b/android/module_context.go
@@ -216,6 +216,11 @@
// SetOutputFiles stores the outputFiles to outputFiles property, which is used
// to set the OutputFilesProvider later.
SetOutputFiles(outputFiles Paths, tag string)
+
+ // ComplianceMetadataInfo returns a ComplianceMetadataInfo instance for different module types to dump metadata,
+ // which usually happens in GenerateAndroidBuildActions() of a module type.
+ // See android.ModuleBase.complianceMetadataInfo
+ ComplianceMetadataInfo() *ComplianceMetadataInfo
}
type moduleContext struct {
@@ -729,6 +734,15 @@
}
}
+func (m *moduleContext) ComplianceMetadataInfo() *ComplianceMetadataInfo {
+ if complianceMetadataInfo := m.module.base().complianceMetadataInfo; complianceMetadataInfo != nil {
+ return complianceMetadataInfo
+ }
+ complianceMetadataInfo := NewComplianceMetadataInfo()
+ m.module.base().complianceMetadataInfo = complianceMetadataInfo
+ return complianceMetadataInfo
+}
+
// Returns a list of paths expanded from globs and modules referenced using ":module" syntax. The property must
// be tagged with `android:"path" to support automatic source module dependency resolution.
//