Merge "HWAsan: Increase the malloc fill size."
diff --git a/android/api_domain.go b/android/api_domain.go
index bdd4e6f..587ceae 100644
--- a/android/api_domain.go
+++ b/android/api_domain.go
@@ -32,17 +32,18 @@
// TODO(b/246656800): Reconcile with android.SdkKind
const (
- PublicApi ApiSurface = iota
- SystemApi
- VendorApi
+ // API surface provided by platform and mainline modules to other mainline modules
+ ModuleLibApi ApiSurface = iota
+ PublicApi // Aka NDK
+ VendorApi // Aka LLNDK
)
func (a ApiSurface) String() string {
switch a {
+ case ModuleLibApi:
+ return "module-libapi"
case PublicApi:
return "publicapi"
- case SystemApi:
- return "systemapi"
case VendorApi:
return "vendorapi"
default:
diff --git a/android/bazel_handler.go b/android/bazel_handler.go
index 4a495f0..ac4ced8 100644
--- a/android/bazel_handler.go
+++ b/android/bazel_handler.go
@@ -1135,10 +1135,11 @@
// the 'bazel' package, which cannot depend on 'android' package where ctx is defined,
// because this would cause circular dependency. So, until we move aquery processing
// to the 'android' package, we need to handle special cases here.
- if buildStatement.Mnemonic == "FileWrite" || buildStatement.Mnemonic == "SourceSymlinkManifest" {
+ switch buildStatement.Mnemonic {
+ case "FileWrite", "SourceSymlinkManifest":
out := PathForBazelOut(ctx, buildStatement.OutputPaths[0])
WriteFileRuleVerbatim(ctx, out, buildStatement.FileContents)
- } else if buildStatement.Mnemonic == "SymlinkTree" {
+ case "SymlinkTree":
// build-runfiles arguments are the manifest file and the target directory
// where it creates the symlink tree according to this manifest (and then
// writes the MANIFEST file to it).
@@ -1157,7 +1158,7 @@
"outDir": outDir,
},
})
- } else {
+ default:
panic(fmt.Sprintf("unhandled build statement: %v", buildStatement))
}
}
diff --git a/android/bazel_paths.go b/android/bazel_paths.go
index e151521..bad7baf 100644
--- a/android/bazel_paths.go
+++ b/android/bazel_paths.go
@@ -461,11 +461,6 @@
return fmt.Sprintf("//%s:%s", moduleDir, moduleName)
}
-// ModuleFromBazelLabel reverses the logic in bp2buildModuleLabel
-func ModuleFromBazelLabel(label string) string {
- return strings.Split(label, ":")[1]
-}
-
// BazelOutPath is a Bazel output path compatible to be used for mixed builds within Soong/Ninja.
type BazelOutPath struct {
OutputPath
diff --git a/android/config.go b/android/config.go
index a8b0a40..c305114 100644
--- a/android/config.go
+++ b/android/config.go
@@ -521,27 +521,33 @@
config.AndroidFirstDeviceTarget = FirstTarget(config.Targets[Android], "lib64", "lib32")[0]
}
- if cmdArgs.SymlinkForestMarker != "" {
- config.BuildMode = SymlinkForest
- } else if cmdArgs.Bp2buildMarker != "" {
- config.BuildMode = Bp2build
- } else if cmdArgs.BazelQueryViewDir != "" {
- config.BuildMode = GenerateQueryView
- } else if cmdArgs.BazelApiBp2buildDir != "" {
- config.BuildMode = ApiBp2build
- } else if cmdArgs.ModuleGraphFile != "" {
- config.BuildMode = GenerateModuleGraph
- } else if cmdArgs.DocFile != "" {
- config.BuildMode = GenerateDocFile
- } else if cmdArgs.BazelModeDev {
- config.BuildMode = BazelDevMode
- } else if cmdArgs.BazelMode {
- config.BuildMode = BazelProdMode
- } else if cmdArgs.BazelModeStaging {
- config.BuildMode = BazelStagingMode
- } else {
- config.BuildMode = AnalysisNoBazel
+ setBuildMode := func(arg string, mode SoongBuildMode) {
+ if arg != "" {
+ if config.BuildMode != AnalysisNoBazel {
+ fmt.Fprintf(os.Stderr, "buildMode is already set, illegal argument: %s", arg)
+ os.Exit(1)
+ }
+ config.BuildMode = mode
+ }
}
+ setBazelMode := func(arg bool, argName string, mode SoongBuildMode) {
+ if arg {
+ if config.BuildMode != AnalysisNoBazel {
+ fmt.Fprintf(os.Stderr, "buildMode is already set, illegal argument: %s", argName)
+ os.Exit(1)
+ }
+ config.BuildMode = mode
+ }
+ }
+ setBuildMode(cmdArgs.SymlinkForestMarker, SymlinkForest)
+ setBuildMode(cmdArgs.Bp2buildMarker, Bp2build)
+ setBuildMode(cmdArgs.BazelQueryViewDir, GenerateQueryView)
+ setBuildMode(cmdArgs.BazelApiBp2buildDir, ApiBp2build)
+ setBuildMode(cmdArgs.ModuleGraphFile, GenerateModuleGraph)
+ setBuildMode(cmdArgs.DocFile, GenerateDocFile)
+ setBazelMode(cmdArgs.BazelModeDev, "--bazel-mode-dev", BazelDevMode)
+ setBazelMode(cmdArgs.BazelMode, "--bazel-mode", BazelProdMode)
+ setBazelMode(cmdArgs.BazelModeStaging, "--bazel-mode-staging", BazelStagingMode)
config.BazelContext, err = NewBazelContext(config)
config.Bp2buildPackageConfig = GetBp2BuildAllowList()
@@ -1494,6 +1500,10 @@
return Bool(c.productVariables.CompressedApex) && !c.UnbundledBuildApps()
}
+func (c *config) ApexTrimEnabled() bool {
+ return Bool(c.productVariables.TrimmedApex)
+}
+
func (c *config) EnforceSystemCertificate() bool {
return Bool(c.productVariables.EnforceSystemCertificate)
}
diff --git a/android/testing.go b/android/testing.go
index 29af71f..e4202ae 100644
--- a/android/testing.go
+++ b/android/testing.go
@@ -1145,7 +1145,7 @@
var p AndroidMkDataProvider
var ok bool
if p, ok = mod.(AndroidMkDataProvider); !ok {
- t.Errorf("module does not implement AndroidMkDataProvider: " + mod.Name())
+ t.Fatalf("module does not implement AndroidMkDataProvider: " + mod.Name())
}
data := p.AndroidMk()
data.fillInData(ctx, mod)
diff --git a/android/util.go b/android/util.go
index a0f7160..234bda3 100644
--- a/android/util.go
+++ b/android/util.go
@@ -136,6 +136,32 @@
return IndexList(s, list) != -1
}
+func setFromList[T comparable](l []T) map[T]bool {
+ m := make(map[T]bool, len(l))
+ for _, t := range l {
+ m[t] = true
+ }
+ return m
+}
+
+// ListDifference checks if the two lists contain the same elements
+func ListDifference[T comparable](l1, l2 []T) []T {
+ diff := []T{}
+ m1 := setFromList(l1)
+ m2 := setFromList(l2)
+ for _, t := range l1 {
+ if _, ok := m2[t]; !ok {
+ diff = append(diff, t)
+ }
+ }
+ for _, t := range l2 {
+ if _, ok := m1[t]; !ok {
+ diff = append(diff, t)
+ }
+ }
+ return diff
+}
+
// Returns true if the given string s is prefixed with any string in the given prefix list.
func HasAnyPrefix(s string, prefixList []string) bool {
for _, prefix := range prefixList {
diff --git a/android/variable.go b/android/variable.go
index e838b7c..e714fc4 100644
--- a/android/variable.go
+++ b/android/variable.go
@@ -380,6 +380,7 @@
Ndk_abis *bool `json:",omitempty"`
+ TrimmedApex *bool `json:",omitempty"`
Flatten_apex *bool `json:",omitempty"`
ForceApexSymlinkOptimization *bool `json:",omitempty"`
CompressedApex *bool `json:",omitempty"`
@@ -502,6 +503,7 @@
Malloc_zero_contents: boolPtr(true),
Malloc_pattern_fill_contents: boolPtr(false),
Safestack: boolPtr(false),
+ TrimmedApex: boolPtr(false),
BootJars: ConfiguredJarList{apexes: []string{}, jars: []string{}},
ApexBootJars: ConfiguredJarList{apexes: []string{}, jars: []string{}},
diff --git a/apex/androidmk.go b/apex/androidmk.go
index aadccb7..7babd45 100644
--- a/apex/androidmk.go
+++ b/apex/androidmk.go
@@ -309,7 +309,7 @@
targetRequired = append(targetRequired, fi.targetRequiredModuleNames...)
hostRequired = append(hostRequired, fi.hostRequiredModuleNames...)
}
- android.AndroidMkEmitAssignList(w, "LOCAL_REQUIRED_MODULES", moduleNames, a.requiredDeps, required)
+ android.AndroidMkEmitAssignList(w, "LOCAL_REQUIRED_MODULES", moduleNames, a.makeModulesToInstall, required)
android.AndroidMkEmitAssignList(w, "LOCAL_TARGET_REQUIRED_MODULES", targetRequired)
android.AndroidMkEmitAssignList(w, "LOCAL_HOST_REQUIRED_MODULES", hostRequired)
}
@@ -317,14 +317,14 @@
func (a *apexBundle) androidMkForType() android.AndroidMkData {
return android.AndroidMkData{
Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
+ moduleNames := []string{}
apexType := a.properties.ApexType
+ if a.installable() {
+ apexName := proptools.StringDefault(a.properties.Apex_name, name)
+ moduleNames = a.androidMkForFiles(w, name, apexName, moduleDir, data)
+ }
if apexType == flattenedApex {
- var moduleNames []string = nil
- if a.installable() {
- apexName := proptools.StringDefault(a.properties.Apex_name, name)
- moduleNames = a.androidMkForFiles(w, name, apexName, moduleDir, data)
- }
// Only image APEXes can be flattened.
fmt.Fprintln(w, "\ninclude $(CLEAR_VARS) # apex.apexBundle.flat")
fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir)
@@ -366,7 +366,7 @@
}
android.AndroidMkEmitAssignList(w, "LOCAL_OVERRIDES_MODULES", a.overridableProperties.Overrides)
- a.writeRequiredModules(w, nil)
+ a.writeRequiredModules(w, moduleNames)
fmt.Fprintln(w, "include $(BUILD_PREBUILT)")
diff --git a/apex/apex.go b/apex/apex.go
index ad7da27..ae5dd3b 100644
--- a/apex/apex.go
+++ b/apex/apex.go
@@ -80,6 +80,7 @@
ctx.BottomUp("apex", apexMutator).Parallel()
ctx.BottomUp("apex_directly_in_any", apexDirectlyInAnyMutator).Parallel()
ctx.BottomUp("apex_flattened", apexFlattenedMutator).Parallel()
+ ctx.BottomUp("apex_dcla_deps", apexDCLADepsMutator).Parallel()
// Register after apex_info mutator so that it can use ApexVariationName
ctx.TopDown("apex_strict_updatability_lint", apexStrictUpdatibilityLintMutator).Parallel()
}
@@ -389,6 +390,9 @@
// conditions, e.g., target device needs to support APEX compression, are also fulfilled.
// Default: false.
Compressible *bool
+
+ // Trim against a specific Dynamic Common Lib APEX
+ Trim_against *string
}
type apexBundle struct {
@@ -439,8 +443,8 @@
// GenerateAndroidBuildActions.
filesInfo []apexFile
- // List of other module names that should be installed when this APEX gets installed.
- requiredDeps []string
+ // List of other module names that should be installed when this APEX gets installed (LOCAL_REQUIRED_MODULES).
+ makeModulesToInstall []string
///////////////////////////////////////////////////////////////////////////////////////////
// Outputs (final and intermediates)
@@ -488,8 +492,6 @@
// Optional list of lint report zip files for apexes that contain java or app modules
lintReports android.Paths
- prebuiltFileToDelete string
-
isCompressed bool
// Path of API coverage generate file
@@ -675,6 +677,7 @@
androidAppTag = &dependencyTag{name: "androidApp", payload: true}
bpfTag = &dependencyTag{name: "bpf", payload: true}
certificateTag = &dependencyTag{name: "certificate"}
+ dclaTag = &dependencyTag{name: "dcla"}
executableTag = &dependencyTag{name: "executable", payload: true}
fsTag = &dependencyTag{name: "filesystem", payload: true}
bcpfTag = &dependencyTag{name: "bootclasspathFragment", payload: true, sourceOnly: true, memberType: java.BootclasspathFragmentSdkMemberType}
@@ -908,6 +911,33 @@
}
}
+func apexDCLADepsMutator(mctx android.BottomUpMutatorContext) {
+ if !mctx.Config().ApexTrimEnabled() {
+ return
+ }
+ if a, ok := mctx.Module().(*apexBundle); ok && a.overridableProperties.Trim_against != nil {
+ commonVariation := mctx.Config().AndroidCommonTarget.Variations()
+ mctx.AddFarVariationDependencies(commonVariation, dclaTag, String(a.overridableProperties.Trim_against))
+ } else if o, ok := mctx.Module().(*OverrideApex); ok {
+ for _, p := range o.GetProperties() {
+ properties, ok := p.(*overridableProperties)
+ if !ok {
+ continue
+ }
+ if properties.Trim_against != nil {
+ commonVariation := mctx.Config().AndroidCommonTarget.Variations()
+ mctx.AddFarVariationDependencies(commonVariation, dclaTag, String(properties.Trim_against))
+ }
+ }
+ }
+}
+
+type DCLAInfo struct {
+ ProvidedLibs []string
+}
+
+var DCLAInfoProvider = blueprint.NewMutatorProvider(DCLAInfo{}, "apex_info")
+
type ApexBundleInfo struct {
Contents *android.ApexContents
}
@@ -1035,6 +1065,12 @@
child.(android.ApexModule).BuildForApex(apexInfo) // leave a mark!
return true
})
+
+ if a.dynamic_common_lib_apex() {
+ mctx.SetProvider(DCLAInfoProvider, DCLAInfo{
+ ProvidedLibs: a.properties.Native_shared_libs,
+ })
+ }
}
type ApexInfoMutator interface {
@@ -1531,6 +1567,19 @@
return proptools.BoolDefault(a.properties.Dynamic_common_lib_apex, false)
}
+// See the list of libs to trim
+func (a *apexBundle) libs_to_trim(ctx android.ModuleContext) []string {
+ dclaModules := ctx.GetDirectDepsWithTag(dclaTag)
+ if len(dclaModules) > 1 {
+ panic(fmt.Errorf("expected exactly at most one dcla dependency, got %d", len(dclaModules)))
+ }
+ if len(dclaModules) > 0 {
+ DCLAInfo := ctx.OtherModuleProvider(dclaModules[0], DCLAInfoProvider).(DCLAInfo)
+ return DCLAInfo.ProvidedLibs
+ }
+ return []string{}
+}
+
// These functions are interfacing with cc/sanitizer.go. The entire APEX (along with all of its
// members) can be sanitized, either forcibly, or by the global configuration. For some of the
// sanitizers, extra dependencies can be forcibly added as well.
@@ -1922,11 +1971,9 @@
a.containerCertificateFile = android.PathForBazelOut(ctx, outputs.ContainerKeyInfo[0])
a.containerPrivateKeyFile = android.PathForBazelOut(ctx, outputs.ContainerKeyInfo[1])
- // Ensure ApexInfo.RequiresLibs are installed as part of a bundle build
- for _, bazelLabel := range outputs.RequiresLibs {
- // convert Bazel label back to Soong module name
- a.requiredDeps = append(a.requiredDeps, android.ModuleFromBazelLabel(bazelLabel))
- }
+ // Ensure ApexMkInfo.install_to_system make module names are installed as
+ // part of a bundled build.
+ a.makeModulesToInstall = append(a.makeModulesToInstall, outputs.MakeModulesToInstall...)
apexType := a.properties.ApexType
switch apexType {
@@ -2025,7 +2072,7 @@
a.primaryApexType = true
if ctx.Config().InstallExtraFlattenedApexes() {
- a.requiredDeps = append(a.requiredDeps, a.Name()+flattenedSuffix)
+ a.makeModulesToInstall = append(a.makeModulesToInstall, a.Name()+flattenedSuffix)
}
}
case zipApex:
@@ -2177,7 +2224,7 @@
vctx.filesInfo = append(vctx.filesInfo, apexBootclasspathFragmentFiles(ctx, child)...)
for _, makeModuleName := range bcpfModule.BootImageDeviceInstallMakeModules() {
- a.requiredDeps = append(a.requiredDeps, makeModuleName)
+ a.makeModulesToInstall = append(a.makeModulesToInstall, makeModuleName)
}
return true
case sscpfTag:
@@ -2295,12 +2342,6 @@
} else {
ctx.ModuleErrorf("certificate dependency %q must be an android_app_certificate module", depName)
}
- case android.PrebuiltDepTag:
- // If the prebuilt is force disabled, remember to delete the prebuilt file
- // that might have been installed in the previous builds
- if prebuilt, ok := child.(prebuilt); ok && prebuilt.isForceDisabled() {
- a.prebuiltFileToDelete = prebuilt.InstallFilename()
- }
}
return false
}
@@ -2340,11 +2381,14 @@
//
// Always include if we are a host-apex however since those won't have any
// system libraries.
- if ch.IsStubsImplementationRequired() && !am.DirectlyInAnyApex() {
+ //
+ // Skip the dependency in unbundled builds where the device image is not
+ // being built.
+ if ch.IsStubsImplementationRequired() && !am.DirectlyInAnyApex() && !ctx.Config().UnbundledBuild() {
// we need a module name for Make
name := ch.ImplementationModuleNameForMake(ctx) + ch.Properties.SubName
- if !android.InList(name, a.requiredDeps) {
- a.requiredDeps = append(a.requiredDeps, name)
+ if !android.InList(name, a.makeModulesToInstall) {
+ a.makeModulesToInstall = append(a.makeModulesToInstall, name)
}
}
vctx.requireNativeLibs = append(vctx.requireNativeLibs, af.stem())
@@ -2479,7 +2523,6 @@
}
////////////////////////////////////////////////////////////////////////////////////////////
// 2) traverse the dependency tree to collect apexFile structs from them.
-
// Collect the module directory for IDE info in java/jdeps.go.
a.modulePaths = append(a.modulePaths, ctx.ModuleDir())
diff --git a/apex/apex_test.go b/apex/apex_test.go
index 395da95..ea068ee 100644
--- a/apex/apex_test.go
+++ b/apex/apex_test.go
@@ -526,6 +526,7 @@
data.Custom(&builder, ab.BaseModuleName(), "TARGET_", "", data)
androidMk := builder.String()
+ ensureContains(t, androidMk, "LOCAL_MODULE := mylib.myapex\n")
ensureNotContains(t, androidMk, "LOCAL_MODULE := mylib.com.android.myapex\n")
optFlags := apexRule.Args["opt_flags"]
@@ -2995,7 +2996,7 @@
var builder strings.Builder
data.Custom(&builder, name, prefix, "", data)
androidMk := builder.String()
- ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := libc.vendor libm.vendor libdl.vendor\n")
+ ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := libc++.vendor.myapex:64 mylib.vendor.myapex:64 apex_manifest.pb.myapex apex_pubkey.myapex libc.vendor libm.vendor libdl.vendor\n")
}
func TestAndroidMkWritesCommonProperties(t *testing.T) {
@@ -4147,6 +4148,7 @@
var builder strings.Builder
data.Custom(&builder, name, prefix, "", data)
androidMk := builder.String()
+ ensureContains(t, androidMk, "LOCAL_MODULE := mylib.myapex\n")
ensureNotContains(t, androidMk, "LOCAL_MODULE := mylib.com.android.myapex\n")
}
@@ -5680,6 +5682,12 @@
var builder strings.Builder
data.Custom(&builder, name, prefix, "", data)
androidMk := builder.String()
+ ensureContains(t, androidMk, "LOCAL_MODULE := mytest.myapex\n")
+ ensureContains(t, androidMk, "LOCAL_MODULE := mytest1.myapex\n")
+ ensureContains(t, androidMk, "LOCAL_MODULE := mytest2.myapex\n")
+ ensureContains(t, androidMk, "LOCAL_MODULE := mytest3.myapex\n")
+ ensureContains(t, androidMk, "LOCAL_MODULE := apex_manifest.pb.myapex\n")
+ ensureContains(t, androidMk, "LOCAL_MODULE := apex_pubkey.myapex\n")
ensureContains(t, androidMk, "LOCAL_MODULE := myapex\n")
flatBundle := ctx.ModuleForTests("myapex", "android_common_myapex_flattened").Module().(*apexBundle)
@@ -5708,12 +5716,12 @@
}),
)
ab := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle)
- ensureListContains(t, ab.requiredDeps, "myapex.flattened")
+ ensureListContains(t, ab.makeModulesToInstall, "myapex.flattened")
mk := android.AndroidMkDataForTest(t, ctx, ab)
var builder strings.Builder
mk.Custom(&builder, ab.Name(), "TARGET_", "", mk)
androidMk := builder.String()
- ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := myapex.flattened\n")
+ ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := apex_manifest.pb.myapex apex_pubkey.myapex myapex.flattened\n")
}
func TestErrorsIfDepsAreNotEnabled(t *testing.T) {
@@ -6506,6 +6514,12 @@
var builder strings.Builder
data.Custom(&builder, name, "TARGET_", "", data)
androidMk := builder.String()
+ ensureContains(t, androidMk, "LOCAL_MODULE := override_app.override_myapex")
+ ensureContains(t, androidMk, "LOCAL_MODULE := overrideBpf.o.override_myapex")
+ ensureContains(t, androidMk, "LOCAL_MODULE := apex_manifest.pb.override_myapex")
+ ensureContains(t, androidMk, "LOCAL_MODULE := override_bcplib.override_myapex")
+ ensureContains(t, androidMk, "LOCAL_MODULE := override_systemserverlib.override_myapex")
+ ensureContains(t, androidMk, "LOCAL_MODULE := override_java_library.override_myapex")
ensureContains(t, androidMk, "LOCAL_MODULE_STEM := override_myapex.apex")
ensureContains(t, androidMk, "LOCAL_OVERRIDES_MODULES := unknownapex myapex")
ensureNotContains(t, androidMk, "LOCAL_MODULE := app.myapex")
@@ -7097,7 +7111,7 @@
var builder strings.Builder
data.Custom(&builder, name, prefix, "", data)
androidMk := builder.String()
- ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := a b\n")
+ ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := mylib.myapex:64 apex_manifest.pb.myapex apex_pubkey.myapex a b\n")
ensureContains(t, androidMk, "LOCAL_HOST_REQUIRED_MODULES := c d\n")
ensureContains(t, androidMk, "LOCAL_TARGET_REQUIRED_MODULES := e f\n")
}
@@ -7268,9 +7282,6 @@
"myapex",
"//apex_available:platform",
],
- stubs: {
- versions: ["current"],
- },
}
`)
@@ -7280,10 +7291,11 @@
data.Custom(&builder, apexBundle.BaseModuleName(), "TARGET_", "", data)
androidMk := builder.String()
// `myotherlib` is added to `myapex` as symlink
+ ensureContains(t, androidMk, "LOCAL_MODULE := mylib.myapex\n")
ensureNotContains(t, androidMk, "LOCAL_MODULE := prebuilt_myotherlib.myapex\n")
ensureNotContains(t, androidMk, "LOCAL_MODULE := myotherlib.myapex\n")
// `myapex` should have `myotherlib` in its required line, not `prebuilt_myotherlib`
- ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := myotherlib\n")
+ ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := mylib.myapex:64 myotherlib:64 apex_manifest.pb.myapex apex_pubkey.myapex\n")
}
func TestApexWithJniLibs(t *testing.T) {
@@ -8796,7 +8808,7 @@
// The make level dependency needs to be on otherlib - prebuilt_otherlib isn't
// a thing there.
- ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := otherlib\n")
+ ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := libc++:64 mylib.myapex:64 apex_manifest.pb.myapex apex_pubkey.myapex otherlib\n")
}
func TestExcludeDependency(t *testing.T) {
@@ -9190,7 +9202,7 @@
var builder strings.Builder
data.Custom(&builder, apexBundle.BaseModuleName(), "TARGET_", "", data)
androidMk := builder.String()
- ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := foo-dexpreopt-arm64-apex@myapex@javalib@foo.jar@classes.odex foo-dexpreopt-arm64-apex@myapex@javalib@foo.jar@classes.vdex\n")
+ ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := foo.myapex apex_manifest.pb.myapex apex_pubkey.myapex foo-dexpreopt-arm64-apex@myapex@javalib@foo.jar@classes.odex foo-dexpreopt-arm64-apex@myapex@javalib@foo.jar@classes.vdex\n")
}
func TestAndroidMk_DexpreoptBuiltInstalledForApex_Prebuilt(t *testing.T) {
@@ -9266,7 +9278,7 @@
var builder strings.Builder
data.Custom(&builder, apexBundle.BaseModuleName(), "TARGET_", "", data)
androidMk := builder.String()
- ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := otherapex")
+ ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := foo.myapex apex_manifest.pb.myapex apex_pubkey.myapex otherapex")
}
func TestAndroidMk_RequiredDeps(t *testing.T) {
@@ -9285,15 +9297,15 @@
`)
bundle := ctx.ModuleForTests("myapex", "android_common_myapex_image").Module().(*apexBundle)
- bundle.requiredDeps = append(bundle.requiredDeps, "foo")
+ bundle.makeModulesToInstall = append(bundle.makeModulesToInstall, "foo")
data := android.AndroidMkDataForTest(t, ctx, bundle)
var builder strings.Builder
data.Custom(&builder, bundle.BaseModuleName(), "TARGET_", "", data)
androidMk := builder.String()
- ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := foo\n")
+ ensureContains(t, androidMk, "LOCAL_REQUIRED_MODULES := apex_manifest.pb.myapex apex_pubkey.myapex foo\n")
flattenedBundle := ctx.ModuleForTests("myapex", "android_common_myapex_flattened").Module().(*apexBundle)
- flattenedBundle.requiredDeps = append(flattenedBundle.requiredDeps, "foo")
+ flattenedBundle.makeModulesToInstall = append(flattenedBundle.makeModulesToInstall, "foo")
flattenedData := android.AndroidMkDataForTest(t, ctx, flattenedBundle)
var flattenedBuilder strings.Builder
flattenedData.Custom(&flattenedBuilder, flattenedBundle.BaseModuleName(), "TARGET_", "", flattenedData)
@@ -9537,7 +9549,7 @@
func ensureContainsRequiredDeps(t *testing.T, ctx *android.TestContext, moduleName, variant string, deps []string) {
a := ctx.ModuleForTests(moduleName, variant).Module().(*apexBundle)
for _, dep := range deps {
- android.AssertStringListContains(t, "", a.requiredDeps, dep)
+ android.AssertStringListContains(t, "", a.makeModulesToInstall, dep)
}
}
@@ -9545,7 +9557,7 @@
func ensureDoesNotContainRequiredDeps(t *testing.T, ctx *android.TestContext, moduleName, variant string, deps []string) {
a := ctx.ModuleForTests(moduleName, variant).Module().(*apexBundle)
for _, dep := range deps {
- android.AssertStringListDoesNotContain(t, "", a.requiredDeps, dep)
+ android.AssertStringListDoesNotContain(t, "", a.makeModulesToInstall, dep)
}
}
diff --git a/apex/bp2build_test.go b/apex/bp2build_test.go
index 01afa52..2f2b61e 100644
--- a/apex/bp2build_test.go
+++ b/apex/bp2build_test.go
@@ -42,6 +42,7 @@
OutputBaseDir: outputBaseDir,
LabelToApexInfo: map[string]cquery.ApexInfo{
"//:foo": cquery.ApexInfo{
+ // ApexInfo Starlark provider.
SignedOutput: "signed_out.apex",
SignedCompressedOutput: "signed_out.capex",
UnsignedOutput: "unsigned_out.apex",
@@ -56,6 +57,9 @@
// unused
PackageName: "pkg_name",
ProvidesLibs: []string{"a", "b"},
+
+ // ApexMkInfo Starlark provider
+ MakeModulesToInstall: []string{"c"}, // d deliberately omitted
},
},
}
@@ -111,7 +115,12 @@
if w := "$(call dist-for-goals,checkbuild,out/bazel/execroot/__main__/installed-files.txt:foo-installed-files.txt)"; !strings.Contains(data, w) {
t.Errorf("Expected %q in androidmk data, but did not find %q", w, data)
}
- if w := "LOCAL_REQUIRED_MODULES := c d"; !strings.Contains(data, w) {
+
+ // make modules to be installed to system
+ if len(ab.makeModulesToInstall) != 1 && ab.makeModulesToInstall[0] != "c" {
+ t.Errorf("Expected makeModulesToInstall slice to only contain 'c', got %q", ab.makeModulesToInstall)
+ }
+ if w := "LOCAL_REQUIRED_MODULES := c"; !strings.Contains(data, w) {
t.Errorf("Expected %q in androidmk data, but did not find it in %q", w, data)
}
}
@@ -212,6 +221,7 @@
OutputBaseDir: outputBaseDir,
LabelToApexInfo: map[string]cquery.ApexInfo{
"//:foo": cquery.ApexInfo{
+ // ApexInfo Starlark provider
SignedOutput: "signed_out.apex",
UnsignedOutput: "unsigned_out.apex",
BundleKeyInfo: []string{"public_key", "private_key"},
@@ -225,8 +235,12 @@
// unused
PackageName: "pkg_name",
ProvidesLibs: []string{"a", "b"},
+
+ // ApexMkInfo Starlark provider
+ MakeModulesToInstall: []string{"c"}, // d deliberately omitted
},
"//:override_foo": cquery.ApexInfo{
+ // ApexInfo Starlark provider
SignedOutput: "override_signed_out.apex",
UnsignedOutput: "override_unsigned_out.apex",
BundleKeyInfo: []string{"override_public_key", "override_private_key"},
@@ -240,6 +254,9 @@
// unused
PackageName: "override_pkg_name",
ProvidesLibs: []string{"a", "b"},
+
+ // ApexMkInfo Starlark provider
+ MakeModulesToInstall: []string{"c"}, // d deliberately omitted
},
},
}
@@ -295,7 +312,12 @@
if w := "$(call dist-for-goals,checkbuild,out/bazel/execroot/__main__/override_installed-files.txt:override_foo-installed-files.txt)"; !strings.Contains(data, w) {
t.Errorf("Expected %q in androidmk data, but did not find %q", w, data)
}
- if w := "LOCAL_REQUIRED_MODULES := c d"; !strings.Contains(data, w) {
+
+ // make modules to be installed to system
+ if len(ab.makeModulesToInstall) != 1 && ab.makeModulesToInstall[0] != "c" {
+ t.Errorf("Expected makeModulestoInstall slice to only contain 'c', got %q", ab.makeModulesToInstall)
+ }
+ if w := "LOCAL_REQUIRED_MODULES := c"; !strings.Contains(data, w) {
t.Errorf("Expected %q in androidmk data, but did not find it in %q", w, data)
}
}
diff --git a/apex/builder.go b/apex/builder.go
index 18d0836..4331d3e 100644
--- a/apex/builder.go
+++ b/apex/builder.go
@@ -40,6 +40,8 @@
pctx.Import("android/soong/java")
pctx.HostBinToolVariable("apexer", "apexer")
pctx.HostBinToolVariable("apexer_with_DCLA_preprocessing", "apexer_with_DCLA_preprocessing")
+ pctx.HostBinToolVariable("apexer_with_trim_preprocessing", "apexer_with_trim_preprocessing")
+
// ART minimal builds (using the master-art manifest) do not have the "frameworks/base"
// projects, and hence cannot build 'aapt2'. Use the SDK prebuilt instead.
hostBinToolVariableWithPrebuilt := func(name, prebuiltDir, tool string) {
@@ -146,6 +148,34 @@
}, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key",
"opt_flags", "manifest", "is_DCLA")
+ TrimmedApexRule = pctx.StaticRule("TrimmedApexRule", blueprint.RuleParams{
+ Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` +
+ `(. ${out}.copy_commands) && ` +
+ `APEXER_TOOL_PATH=${tool_path} ` +
+ `${apexer_with_trim_preprocessing} ` +
+ `--apexer ${apexer} ` +
+ `--canned_fs_config ${canned_fs_config} ` +
+ `--manifest ${manifest} ` +
+ `--libs_to_trim ${libs_to_trim} ` +
+ `${image_dir} ` +
+ `${out} ` +
+ `-- ` +
+ `--include_build_info ` +
+ `--force ` +
+ `--payload_type image ` +
+ `--key ${key} ` +
+ `--file_contexts ${file_contexts} ` +
+ `${opt_flags} `,
+ CommandDeps: []string{"${apexer_with_trim_preprocessing}", "${apexer}", "${avbtool}", "${e2fsdroid}",
+ "${merge_zips}", "${mke2fs}", "${resize2fs}", "${sefcontext_compile}", "${make_f2fs}",
+ "${sload_f2fs}", "${make_erofs}", "${soong_zip}", "${zipalign}", "${aapt2}",
+ "prebuilts/sdk/current/public/android.jar"},
+ Rspfile: "${out}.copy_commands",
+ RspfileContent: "${copy_commands}",
+ Description: "APEX ${image_dir} => ${out}",
+ }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key",
+ "opt_flags", "manifest", "libs_to_trim")
+
zipApexRule = pctx.StaticRule("zipApexRule", blueprint.RuleParams{
Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` +
`(. ${out}.copy_commands) && ` +
@@ -706,6 +736,24 @@
"opt_flags": strings.Join(optFlags, " "),
},
})
+ } else if ctx.Config().ApexTrimEnabled() && len(a.libs_to_trim(ctx)) > 0 {
+ ctx.Build(pctx, android.BuildParams{
+ Rule: TrimmedApexRule,
+ Implicits: implicitInputs,
+ Output: unsignedOutputFile,
+ Description: "apex (" + apexType.name() + ")",
+ Args: map[string]string{
+ "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir,
+ "image_dir": imageDir.String(),
+ "copy_commands": strings.Join(copyCommands, " && "),
+ "manifest": a.manifestPbOut.String(),
+ "file_contexts": fileContexts.String(),
+ "canned_fs_config": cannedFsConfig.String(),
+ "key": a.privateKeyFile.String(),
+ "opt_flags": strings.Join(optFlags, " "),
+ "libs_to_trim": strings.Join(a.libs_to_trim(ctx), ","),
+ },
+ })
} else {
ctx.Build(pctx, android.BuildParams{
Rule: apexRule,
diff --git a/apex/vndk.go b/apex/vndk.go
index ef3e5e1..80560cf 100644
--- a/apex/vndk.go
+++ b/apex/vndk.go
@@ -65,8 +65,23 @@
}
vndkVersion := ab.vndkVersion(mctx.DeviceConfig())
+
// Ensure VNDK APEX mount point is formatted as com.android.vndk.v###
ab.properties.Apex_name = proptools.StringPtr(vndkApexNamePrefix + vndkVersion)
+
+ apiLevel, err := android.ApiLevelFromUser(mctx, vndkVersion)
+ if err != nil {
+ mctx.PropertyErrorf("vndk_version", "%s", err.Error())
+ return
+ }
+
+ targets := mctx.MultiTargets()
+ if len(targets) > 0 && apiLevel.LessThan(cc.MinApiForArch(mctx, targets[0].Arch.ArchType)) {
+ // Disable VNDK apexes for VNDK versions less than the minimum supported API level for the primary
+ // architecture.
+ ab.Disable()
+ }
+
}
}
diff --git a/bazel/aquery.go b/bazel/aquery.go
index bc823b3..80cf70a 100644
--- a/bazel/aquery.go
+++ b/bazel/aquery.go
@@ -118,12 +118,11 @@
// A helper type for aquery processing which facilitates retrieval of path IDs from their
// less readable Bazel structures (depset and path fragment).
type aqueryArtifactHandler struct {
- // Switches to true if any depset contains only `bazelToolsDependencySentinel`
- bazelToolsDependencySentinelNeeded bool
// Maps depset id to AqueryDepset, a representation of depset which is
// post-processed for middleman artifact handling, unhandled artifact
// dropping, content hashing, etc.
depsetIdToAqueryDepset map[depsetId]AqueryDepset
+ emptyDepsetIds map[depsetId]struct{}
// Maps content hash to AqueryDepset.
depsetHashToAqueryDepset map[string]AqueryDepset
@@ -145,9 +144,6 @@
// The file name of py3wrapper.sh, which is used by py_binary targets.
const py3wrapperFileName = "/py3wrapper.sh"
-// A file to be put into depsets that are otherwise empty
-const bazelToolsDependencySentinel = "BAZEL_TOOLS_DEPENDENCY_SENTINEL"
-
func indexBy[K comparable, V any](values []V, keyFn func(v V) K) map[K]V {
m := map[K]V{}
for _, v := range values {
@@ -192,6 +188,7 @@
depsetIdToAqueryDepset: map[depsetId]AqueryDepset{},
depsetHashToAqueryDepset: map[string]AqueryDepset{},
depsetHashToArtifactPathsCache: map[string][]string{},
+ emptyDepsetIds: make(map[depsetId]struct{}, 0),
artifactIdToPath: artifactIdToPath,
}
@@ -208,16 +205,16 @@
// Ensures that the handler's depsetIdToAqueryDepset map contains an entry for the given
// depset.
-func (a *aqueryArtifactHandler) populateDepsetMaps(depset depSetOfFiles, middlemanIdToDepsetIds map[artifactId][]depsetId, depsetIdToDepset map[depsetId]depSetOfFiles) (AqueryDepset, error) {
+func (a *aqueryArtifactHandler) populateDepsetMaps(depset depSetOfFiles, middlemanIdToDepsetIds map[artifactId][]depsetId, depsetIdToDepset map[depsetId]depSetOfFiles) (*AqueryDepset, error) {
if aqueryDepset, containsDepset := a.depsetIdToAqueryDepset[depset.Id]; containsDepset {
- return aqueryDepset, nil
+ return &aqueryDepset, nil
}
transitiveDepsetIds := depset.TransitiveDepSetIds
var directArtifactPaths []string
for _, artifactId := range depset.DirectArtifactIds {
path, pathExists := a.artifactIdToPath[artifactId]
if !pathExists {
- return AqueryDepset{}, fmt.Errorf("undefined input artifactId %d", artifactId)
+ return nil, fmt.Errorf("undefined input artifactId %d", artifactId)
}
// Filter out any inputs which are universally dropped, and swap middleman
// artifacts with their corresponding depsets.
@@ -226,6 +223,7 @@
transitiveDepsetIds = append(transitiveDepsetIds, depsetsToUse...)
} else if strings.HasSuffix(path, py3wrapperFileName) ||
strings.HasPrefix(path, "../bazel_tools") {
+ continue
// Drop these artifacts.
// See go/python-binary-host-mixed-build for more details.
// 1) Drop py3wrapper.sh, just use python binary, the launcher script generated by the
@@ -241,20 +239,23 @@
for _, childDepsetId := range transitiveDepsetIds {
childDepset, exists := depsetIdToDepset[childDepsetId]
if !exists {
- return AqueryDepset{}, fmt.Errorf("undefined input depsetId %d (referenced by depsetId %d)", childDepsetId, depset.Id)
+ if _, empty := a.emptyDepsetIds[childDepsetId]; empty {
+ continue
+ } else {
+ return nil, fmt.Errorf("undefined input depsetId %d (referenced by depsetId %d)", childDepsetId, depset.Id)
+ }
}
- childAqueryDepset, err := a.populateDepsetMaps(childDepset, middlemanIdToDepsetIds, depsetIdToDepset)
- if err != nil {
- return AqueryDepset{}, err
+ if childAqueryDepset, err := a.populateDepsetMaps(childDepset, middlemanIdToDepsetIds, depsetIdToDepset); err != nil {
+ return nil, err
+ } else if childAqueryDepset == nil {
+ continue
+ } else {
+ childDepsetHashes = append(childDepsetHashes, childAqueryDepset.ContentHash)
}
- childDepsetHashes = append(childDepsetHashes, childAqueryDepset.ContentHash)
}
if len(directArtifactPaths) == 0 && len(childDepsetHashes) == 0 {
- // We could omit this depset altogether but that requires cleanup on
- // transitive dependents.
- // As a simpler alternative, we use this sentinel file as a dependency.
- directArtifactPaths = append(directArtifactPaths, bazelToolsDependencySentinel)
- a.bazelToolsDependencySentinelNeeded = true
+ a.emptyDepsetIds[depset.Id] = struct{}{}
+ return nil, nil
}
aqueryDepset := AqueryDepset{
ContentHash: depsetContentHash(directArtifactPaths, childDepsetHashes),
@@ -263,7 +264,7 @@
}
a.depsetIdToAqueryDepset[depset.Id] = aqueryDepset
a.depsetHashToAqueryDepset[aqueryDepset.ContentHash] = aqueryDepset
- return aqueryDepset, nil
+ return &aqueryDepset, nil
}
// getInputPaths flattens the depsets of the given IDs and returns all transitive
@@ -392,14 +393,6 @@
}
var buildStatements []BuildStatement
- if aqueryHandler.bazelToolsDependencySentinelNeeded {
- buildStatements = append(buildStatements, BuildStatement{
- Command: fmt.Sprintf("touch '%s'", bazelToolsDependencySentinel),
- OutputPaths: []string{bazelToolsDependencySentinel},
- Mnemonic: bazelToolsDependencySentinel,
- })
- }
-
for _, actionEntry := range aqueryResult.Actions {
if shouldSkipAction(actionEntry) {
continue
@@ -484,7 +477,9 @@
var hashes []string
for _, depsetId := range inputDepsetIds {
if aqueryDepset, exists := a.depsetIdToAqueryDepset[depsetId]; !exists {
- return nil, fmt.Errorf("undefined input depsetId %d", depsetId)
+ if _, empty := a.emptyDepsetIds[depsetId]; !empty {
+ return nil, fmt.Errorf("undefined (not even empty) input depsetId %d", depsetId)
+ }
} else {
hashes = append(hashes, aqueryDepset.ContentHash)
}
diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go
index 2eacafa..4d1503e 100644
--- a/bazel/aquery_test.go
+++ b/bazel/aquery_test.go
@@ -227,7 +227,7 @@
return
}
_, _, err = AqueryBuildStatements(data)
- assertError(t, err, "undefined input depsetId 2")
+ assertError(t, err, "undefined (not even empty) input depsetId 2")
}
func TestInvalidInputDepsetIdFromDepset(t *testing.T) {
@@ -584,13 +584,18 @@
{ "id": 60, "label": ".."}
]
}`
+ /* depsets
+ 1111 2222
+ / \ |
+ ../dep2 ../bazel_tools/dep1
+ */
data, err := JsonToActionGraphContainer(inputString)
if err != nil {
t.Error(err)
return
}
actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data)
- if len(actualDepsets) != 2 {
+ if len(actualDepsets) != 1 {
t.Errorf("expected 1 depset but found %#v", actualDepsets)
return
}
@@ -624,6 +629,82 @@
}
}
+func TestBazelOutRemovalFromTransitiveInputDepsets(t *testing.T) {
+ const inputString = `{
+ "artifacts": [
+ { "id": 1, "path_fragment_id": 10 },
+ { "id": 2, "path_fragment_id": 20 },
+ { "id": 3, "path_fragment_id": 30 }],
+ "dep_set_of_files": [{
+ "id": 1111,
+ "transitive_dep_set_ids": [2222]
+ }, {
+ "id": 2222,
+ "direct_artifact_ids": [3]
+ }, {
+ "id": 3333,
+ "direct_artifact_ids": [3]
+ }, {
+ "id": 4444,
+ "transitive_dep_set_ids": [3333]
+ }],
+ "actions": [{
+ "target_id": 100,
+ "action_key": "x",
+ "input_dep_set_ids": [1111, 4444],
+ "mnemonic": "x",
+ "arguments": ["bogus", "command"],
+ "output_ids": [2],
+ "primary_output_id": 1
+ }],
+ "path_fragments": [
+ { "id": 10, "label": "input" },
+ { "id": 20, "label": "output" },
+ { "id": 30, "label": "dep", "parent_id": 50 },
+ { "id": 50, "label": "bazel_tools", "parent_id": 60 },
+ { "id": 60, "label": ".."}
+ ]
+}`
+ /* depsets
+ 1111 4444
+ || ||
+ 2222 3333
+ | |
+ ../bazel_tools/dep
+ Note: in dep_set_of_files:
+ 1111 appears BEFORE its dependency,2222 while
+ 4444 appears AFTER its dependency 3333
+ and this test shows that that order doesn't affect empty depset pruning
+ */
+ data, err := JsonToActionGraphContainer(inputString)
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ actualBuildStatements, actualDepsets, _ := AqueryBuildStatements(data)
+ if len(actualDepsets) != 0 {
+ t.Errorf("expected 0 depsets but found %#v", actualDepsets)
+ return
+ }
+
+ expectedBuildStatement := BuildStatement{
+ Command: "bogus command",
+ OutputPaths: []string{"output"},
+ Mnemonic: "x",
+ }
+ buildStatementFound := false
+ for _, actualBuildStatement := range actualBuildStatements {
+ if buildStatementEquals(actualBuildStatement, expectedBuildStatement) == "" {
+ buildStatementFound = true
+ break
+ }
+ }
+ if !buildStatementFound {
+ t.Errorf("expected but missing %#v in %#v", expectedBuildStatement, actualBuildStatements)
+ return
+ }
+}
+
func TestMiddlemenAction(t *testing.T) {
const inputString = `
{
diff --git a/bazel/configurability.go b/bazel/configurability.go
index 2b8753b..4680256 100644
--- a/bazel/configurability.go
+++ b/bazel/configurability.go
@@ -343,8 +343,8 @@
}
func (ca *ConfigurationAxis) less(other ConfigurationAxis) bool {
- if ca.configurationType < other.configurationType {
- return true
+ if ca.configurationType == other.configurationType {
+ return ca.subType < other.subType
}
- return ca.subType < other.subType
+ return ca.configurationType < other.configurationType
}
diff --git a/bazel/cquery/request_type.go b/bazel/cquery/request_type.go
index 81c60d9..6654191 100644
--- a/bazel/cquery/request_type.go
+++ b/bazel/cquery/request_type.go
@@ -14,7 +14,14 @@
GetCcUnstrippedInfo = &getCcUnstrippedInfoType{}
)
+type CcAndroidMkInfo struct {
+ LocalStaticLibs []string
+ LocalWholeStaticLibs []string
+ LocalSharedLibs []string
+}
+
type CcInfo struct {
+ CcAndroidMkInfo
OutputFiles []string
CcObjectFiles []string
CcSharedLibraryFiles []string
@@ -180,20 +187,33 @@
if abi_diff_info:
abi_diff_files = [f.path for f in abi_diff_info.diff_files.to_list()]
+local_static_libs = []
+local_whole_static_libs = []
+local_shared_libs = []
+androidmk_tag = "//build/bazel/rules/cc:cc_library_common.bzl%CcAndroidMkInfo"
+if androidmk_tag in p:
+ androidmk_info = p[androidmk_tag]
+ local_static_libs = androidmk_info.local_static_libs
+ local_whole_static_libs = androidmk_info.local_whole_static_libs
+ local_shared_libs = androidmk_info.local_shared_libs
+
return json_encode({
- "OutputFiles": outputFiles,
- "CcObjectFiles": ccObjectFiles,
- "CcSharedLibraryFiles": sharedLibraries,
- "CcStaticLibraryFiles": staticLibraries,
- "Includes": includes,
- "SystemIncludes": system_includes,
- "Headers": headers,
- "RootStaticArchives": rootStaticArchives,
- "RootDynamicLibraries": rootSharedLibraries,
- "TidyFiles": tidy_files,
- "TocFile": toc_file,
- "UnstrippedOutput": unstripped,
- "AbiDiffFiles": abi_diff_files,
+ "OutputFiles": outputFiles,
+ "CcObjectFiles": ccObjectFiles,
+ "CcSharedLibraryFiles": sharedLibraries,
+ "CcStaticLibraryFiles": staticLibraries,
+ "Includes": includes,
+ "SystemIncludes": system_includes,
+ "Headers": headers,
+ "RootStaticArchives": rootStaticArchives,
+ "RootDynamicLibraries": rootSharedLibraries,
+ "TidyFiles": tidy_files,
+ "TocFile": toc_file,
+ "UnstrippedOutput": unstripped,
+ "AbiDiffFiles": abi_diff_files,
+ "LocalStaticLibs": [l for l in local_static_libs],
+ "LocalWholeStaticLibs": [l for l in local_whole_static_libs],
+ "LocalSharedLibs": [l for l in local_shared_libs],
})`
}
@@ -237,6 +257,10 @@
if info.signed_compressed_output:
signed_compressed_output = info.signed_compressed_output.path
+mk_info = providers(target).get("//build/bazel/rules/apex:apex_info.bzl%ApexMkInfo")
+if not mk_info:
+ fail("%s did not provide ApexMkInfo" % id_string)
+
return json_encode({
"signed_output": info.signed_output.path,
"signed_compressed_output": signed_compressed_output,
@@ -251,10 +275,12 @@
"backing_libs": info.backing_libs.path,
"bundle_file": info.base_with_config_zip.path,
"installed_files": info.installed_files.path,
+ "make_modules_to_install": mk_info.make_modules_to_install,
})`
}
type ApexInfo struct {
+ // From the ApexInfo provider
SignedOutput string `json:"signed_output"`
SignedCompressedOutput string `json:"signed_compressed_output"`
UnsignedOutput string `json:"unsigned_output"`
@@ -268,6 +294,9 @@
BackingLibs string `json:"backing_libs"`
BundleFile string `json:"bundle_file"`
InstalledFiles string `json:"installed_files"`
+
+ // From the ApexMkInfo provider
+ MakeModulesToInstall []string `json:"make_modules_to_install"`
}
// ParseResult returns a value obtained by parsing the result of the request's Starlark function.
@@ -289,15 +318,32 @@
}
func (g getCcUnstrippedInfoType) StarlarkFunctionBody() string {
- return `unstripped_tag = "//build/bazel/rules/cc:stripped_cc_common.bzl%CcUnstrippedInfo"
+ return `
p = providers(target)
output_path = target.files.to_list()[0].path
+
unstripped = output_path
+unstripped_tag = "//build/bazel/rules/cc:stripped_cc_common.bzl%CcUnstrippedInfo"
if unstripped_tag in p:
- unstripped = p[unstripped_tag].unstripped.files.to_list()[0].path
+ unstripped_info = p[unstripped_tag]
+ unstripped = unstripped_info.unstripped.files.to_list()[0].path
+
+local_static_libs = []
+local_whole_static_libs = []
+local_shared_libs = []
+androidmk_tag = "//build/bazel/rules/cc:cc_library_common.bzl%CcAndroidMkInfo"
+if androidmk_tag in p:
+ androidmk_info = p[androidmk_tag]
+ local_static_libs = androidmk_info.local_static_libs
+ local_whole_static_libs = androidmk_info.local_whole_static_libs
+ local_shared_libs = androidmk_info.local_shared_libs
+
return json_encode({
"OutputFile": output_path,
"UnstrippedOutput": unstripped,
+ "LocalStaticLibs": [l for l in local_static_libs],
+ "LocalWholeStaticLibs": [l for l in local_whole_static_libs],
+ "LocalSharedLibs": [l for l in local_shared_libs],
})
`
}
@@ -312,6 +358,7 @@
}
type CcUnstrippedInfo struct {
+ CcAndroidMkInfo
OutputFile string
UnstrippedOutput string
}
diff --git a/bazel/cquery/request_type_test.go b/bazel/cquery/request_type_test.go
index 1d30535..7003ce1 100644
--- a/bazel/cquery/request_type_test.go
+++ b/bazel/cquery/request_type_test.go
@@ -177,9 +177,11 @@
"backing_libs":"path/to/backing.txt",
"bundle_file": "dir/bundlefile.zip",
"installed_files":"path/to/installed-files.txt",
- "provides_native_libs":[]
+ "provides_native_libs":[],
+ "make_modules_to_install": ["foo","bar"]
}`,
expectedOutput: ApexInfo{
+ // ApexInfo
SignedOutput: "my.apex",
UnsignedOutput: "my.apex.unsigned",
RequiresLibs: []string{"//bionic/libc:libc", "//bionic/libdl:libdl"},
@@ -191,6 +193,9 @@
BackingLibs: "path/to/backing.txt",
BundleFile: "dir/bundlefile.zip",
InstalledFiles: "path/to/installed-files.txt",
+
+ // ApexMkInfo
+ MakeModulesToInstall: []string{"foo", "bar"},
},
},
}
diff --git a/bazel/properties.go b/bazel/properties.go
index 9be21eb..f4acd26 100644
--- a/bazel/properties.go
+++ b/bazel/properties.go
@@ -73,6 +73,16 @@
}
}
+func SortedConfigurationAxes[T any](m map[ConfigurationAxis]T) []ConfigurationAxis {
+ keys := make([]ConfigurationAxis, 0, len(m))
+ for k := range m {
+ keys = append(keys, k)
+ }
+
+ sort.Slice(keys, func(i, j int) bool { return keys[i].less(keys[j]) })
+ return keys
+}
+
// MakeLabelListFromTargetNames creates a LabelList from unqualified target names
// This is a utiltity function for bp2build converters of Soong modules that have 1:many generated targets
func MakeLabelListFromTargetNames(targetNames []string) LabelList {
@@ -412,13 +422,7 @@
// SortedConfigurationAxes returns all the used ConfigurationAxis in sorted order.
func (la *LabelAttribute) SortedConfigurationAxes() []ConfigurationAxis {
- keys := make([]ConfigurationAxis, 0, len(la.ConfigurableValues))
- for k := range la.ConfigurableValues {
- keys = append(keys, k)
- }
-
- sort.Slice(keys, func(i, j int) bool { return keys[i].less(keys[j]) })
- return keys
+ return SortedConfigurationAxes(la.ConfigurableValues)
}
// MakeLabelAttribute turns a string into a LabelAttribute
@@ -608,13 +612,7 @@
// SortedConfigurationAxes returns all the used ConfigurationAxis in sorted order.
func (ba *BoolAttribute) SortedConfigurationAxes() []ConfigurationAxis {
- keys := make([]ConfigurationAxis, 0, len(ba.ConfigurableValues))
- for k := range ba.ConfigurableValues {
- keys = append(keys, k)
- }
-
- sort.Slice(keys, func(i, j int) bool { return keys[i].less(keys[j]) })
- return keys
+ return SortedConfigurationAxes(ba.ConfigurableValues)
}
// labelListSelectValues supports config-specific label_list typed Bazel attribute values.
@@ -761,13 +759,7 @@
// SortedConfigurationAxes returns all the used ConfigurationAxis in sorted order.
func (lla *LabelListAttribute) SortedConfigurationAxes() []ConfigurationAxis {
- keys := make([]ConfigurationAxis, 0, len(lla.ConfigurableValues))
- for k := range lla.ConfigurableValues {
- keys = append(keys, k)
- }
-
- sort.Slice(keys, func(i, j int) bool { return keys[i].less(keys[j]) })
- return keys
+ return SortedConfigurationAxes(lla.ConfigurableValues)
}
// Append all values, including os and arch specific ones, from another
@@ -1145,13 +1137,7 @@
// SortedConfigurationAxes returns all the used ConfigurationAxis in sorted order.
func (sa *StringAttribute) SortedConfigurationAxes() []ConfigurationAxis {
- keys := make([]ConfigurationAxis, 0, len(sa.ConfigurableValues))
- for k := range sa.ConfigurableValues {
- keys = append(keys, k)
- }
-
- sort.Slice(keys, func(i, j int) bool { return keys[i].less(keys[j]) })
- return keys
+ return SortedConfigurationAxes(sa.ConfigurableValues)
}
// Collapse reduces the configurable axes of the string attribute to a single axis.
@@ -1353,13 +1339,7 @@
// SortedConfigurationAxes returns all the used ConfigurationAxis in sorted order.
func (sla *StringListAttribute) SortedConfigurationAxes() []ConfigurationAxis {
- keys := make([]ConfigurationAxis, 0, len(sla.ConfigurableValues))
- for k := range sla.ConfigurableValues {
- keys = append(keys, k)
- }
-
- sort.Slice(keys, func(i, j int) bool { return keys[i].less(keys[j]) })
- return keys
+ return SortedConfigurationAxes(sla.ConfigurableValues)
}
// DeduplicateAxesFromBase ensures no duplication of items between the no-configuration value and
diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go
index 244ca9c..c11a50d 100644
--- a/bp2build/cc_library_conversion_test.go
+++ b/bp2build/cc_library_conversion_test.go
@@ -2825,7 +2825,7 @@
expectedBazelTargets := []string{
MakeBazelTarget(
"cc_api_library_headers",
- "libfoo.systemapi.headers",
+ "libfoo.module-libapi.headers",
AttrNameToString{
"export_includes": `["dir1"]`,
}),
@@ -2842,18 +2842,18 @@
"api": `"libfoo.map.txt"`,
"library_name": `"libfoo"`,
"api_surfaces": `[
- "systemapi",
+ "module-libapi",
"vendorapi",
]`,
"hdrs": `[
- ":libfoo.systemapi.headers",
+ ":libfoo.module-libapi.headers",
":libfoo.vendorapi.headers",
]`,
}),
}
RunApiBp2BuildTestCase(t, cc.RegisterLibraryBuildComponents, Bp2buildTestCase{
Blueprint: bp,
- Description: "cc API contributions to systemapi and vendorapi",
+ Description: "cc API contributions to module-libapi and vendorapi",
ExpectedBazelTargets: expectedBazelTargets,
})
}
@@ -2872,8 +2872,8 @@
stubs: {symbol_file: "a.map.txt"},
}`,
expectedApi: `"a.map.txt"`,
- expectedApiSurfaces: `["systemapi"]`,
- description: "Library that contributes to systemapi",
+ expectedApiSurfaces: `["module-libapi"]`,
+ description: "Library that contributes to module-libapi",
},
{
bp: `
@@ -2894,10 +2894,10 @@
}`,
expectedApi: `"a.map.txt"`,
expectedApiSurfaces: `[
- "systemapi",
+ "module-libapi",
"vendorapi",
]`,
- description: "Library that contributes to systemapi and vendorapi",
+ description: "Library that contributes to module-libapi and vendorapi",
},
}
for _, testCase := range testCases {
diff --git a/bp2build/cc_object_conversion_test.go b/bp2build/cc_object_conversion_test.go
index 1377c6b..eab84e1 100644
--- a/bp2build/cc_object_conversion_test.go
+++ b/bp2build/cc_object_conversion_test.go
@@ -58,6 +58,7 @@
exclude_srcs: ["a/b/exclude.c"],
sdk_version: "current",
min_sdk_version: "29",
+ crt: true,
}
`,
ExpectedBazelTargets: []string{
@@ -76,6 +77,7 @@
"system_dynamic_deps": `[]`,
"sdk_version": `"current"`,
"min_sdk_version": `"29"`,
+ "crt": "True",
}),
},
})
diff --git a/bp2build/configurability.go b/bp2build/configurability.go
index 4244956..2a0a78e 100644
--- a/bp2build/configurability.go
+++ b/bp2build/configurability.go
@@ -160,6 +160,12 @@
// select statements.
func prettyPrintAttribute(v bazel.Attribute, indent int) (string, error) {
var value reflect.Value
+ // configurableAttrs is the list of individual select statements to be
+ // concatenated together. These select statements should be along different
+ // axes. For example, one element may be
+ // `select({"//color:red": "one", "//color:green": "two"})`, and the second
+ // element may be `select({"//animal:cat": "three", "//animal:dog": "four"}).
+ // These selects should be sorted by axis identifier.
var configurableAttrs []selects
var prepend bool
var defaultSelectValue *string
diff --git a/bp2build/metrics.go b/bp2build/metrics.go
index d6e5cf3..7e29fac 100644
--- a/bp2build/metrics.go
+++ b/bp2build/metrics.go
@@ -188,6 +188,10 @@
)
func (metrics *CodegenMetrics) AddConvertedModule(m blueprint.Module, moduleType string, dir string, conversionType ConversionType) {
+ //a package module has empty name
+ if moduleType == "package" {
+ return
+ }
// Undo prebuilt_ module name prefix modifications
moduleName := android.RemoveOptionalPrebuiltPrefix(m.Name())
metrics.serialized.ConvertedModules = append(metrics.serialized.ConvertedModules, moduleName)
diff --git a/cc/api_level.go b/cc/api_level.go
index fdff5cb..a5571f3 100644
--- a/cc/api_level.go
+++ b/cc/api_level.go
@@ -20,7 +20,9 @@
"android/soong/android"
)
-func minApiForArch(ctx android.EarlyModuleContext,
+// MinApiLevelForArch returns the ApiLevel for the Android version that
+// first supported the architecture.
+func MinApiForArch(ctx android.EarlyModuleContext,
arch android.ArchType) android.ApiLevel {
switch arch {
@@ -38,7 +40,7 @@
func nativeApiLevelFromUser(ctx android.BaseModuleContext,
raw string) (android.ApiLevel, error) {
- min := minApiForArch(ctx, ctx.Arch().ArchType)
+ min := MinApiForArch(ctx, ctx.Arch().ArchType)
if raw == "minimum" {
return min, nil
}
diff --git a/cc/bp2build.go b/cc/bp2build.go
index 35419af..6c5505a 100644
--- a/cc/bp2build.go
+++ b/cc/bp2build.go
@@ -702,7 +702,13 @@
compilerAttrs := compilerAttributes{}
linkerAttrs := linkerAttributes{}
- for axis, configs := range axisToConfigs {
+ // Iterate through these axes in a deterministic order. This is required
+ // because processing certain dependencies may result in concatenating
+ // elements along other axes. (For example, processing NoConfig may result
+ // in elements being added to InApex). This is thus the only way to ensure
+ // that the order of entries in each list is in a predictable order.
+ for _, axis := range bazel.SortedConfigurationAxes(axisToConfigs) {
+ configs := axisToConfigs[axis]
for cfg := range configs {
var allHdrs []string
if baseCompilerProps, ok := archVariantCompilerProps[axis][cfg].(*BaseCompilerProperties); ok {
diff --git a/cc/cc.go b/cc/cc.go
index b194360..753975e 100644
--- a/cc/cc.go
+++ b/cc/cc.go
@@ -1445,6 +1445,8 @@
}
func InstallToBootstrap(name string, config android.Config) bool {
+ // NOTE: also update //build/bazel/rules/apex/cc.bzl#_installed_to_bootstrap
+ // if this list is updated.
if name == "libclang_rt.hwasan" {
return true
}
@@ -1870,7 +1872,8 @@
// in any of the --bazel-mode(s). This filters at the module level and takes
// precedence over the allowlists in allowlists/allowlists.go.
func (c *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool {
- if c.testBinary() && !android.InList(c.Name(), mixedBuildSupportedCcTest) {
+ _, isForTesting := ctx.Config().BazelContext.(android.MockBazelContext)
+ if c.testBinary() && !android.InList(c.Name(), mixedBuildSupportedCcTest) && !isForTesting {
// Per-module rollout of mixed-builds for cc_test modules.
return false
}
@@ -1888,6 +1891,14 @@
bazelCtx := ctx.Config().BazelContext
if ccInfo, err := bazelCtx.GetCcInfo(bazelModuleLabel, android.GetConfigKey(ctx)); err == nil {
c.tidyFiles = android.PathsForBazelOut(ctx, ccInfo.TidyFiles)
+ c.Properties.AndroidMkSharedLibs = ccInfo.LocalSharedLibs
+ c.Properties.AndroidMkStaticLibs = ccInfo.LocalStaticLibs
+ c.Properties.AndroidMkWholeStaticLibs = ccInfo.LocalWholeStaticLibs
+ }
+ if unstrippedInfo, err := bazelCtx.GetCcUnstrippedInfo(bazelModuleLabel, android.GetConfigKey(ctx)); err == nil {
+ c.Properties.AndroidMkSharedLibs = unstrippedInfo.LocalSharedLibs
+ c.Properties.AndroidMkStaticLibs = unstrippedInfo.LocalStaticLibs
+ c.Properties.AndroidMkWholeStaticLibs = unstrippedInfo.LocalWholeStaticLibs
}
c.bazelHandler.ProcessBazelQueryResponse(ctx, bazelModuleLabel)
@@ -2216,6 +2227,13 @@
if err != nil {
ctx.PropertyErrorf("min_sdk_version", err.Error())
}
+
+ // Raise the minSdkVersion to the minimum supported for the architecture.
+ minApiForArch := MinApiForArch(ctx, m.Target().Arch.ArchType)
+ if apiLevel.LessThan(minApiForArch) {
+ apiLevel = minApiForArch
+ }
+
return []blueprint.Variation{
{Mutator: "sdk", Variation: "sdk"},
{Mutator: "version", Variation: apiLevel.String()},
@@ -3697,7 +3715,7 @@
// This allows introducing new architectures in the platform that
// need to be included in apexes that normally require an older
// min_sdk_version.
- minApiForArch := minApiForArch(ctx, c.Target().Arch.ArchType)
+ minApiForArch := MinApiForArch(ctx, c.Target().Arch.ArchType)
if sdkVersion.LessThan(minApiForArch) {
sdkVersion = minApiForArch
}
diff --git a/cc/cc_test.go b/cc/cc_test.go
index 8293f2d..39cc073 100644
--- a/cc/cc_test.go
+++ b/cc/cc_test.go
@@ -25,6 +25,7 @@
"testing"
"android/soong/android"
+ "android/soong/bazel/cquery"
)
func TestMain(m *testing.M) {
@@ -3028,6 +3029,32 @@
}
}
+func checkWholeStaticLibs(t *testing.T, expected []string, module *Module) {
+ t.Helper()
+ actual := module.Properties.AndroidMkWholeStaticLibs
+ if !reflect.DeepEqual(actual, expected) {
+ t.Errorf("incorrect whole_static_libs"+
+ "\nactual: %v"+
+ "\nexpected: %v",
+ actual,
+ expected,
+ )
+ }
+}
+
+func checkSharedLibs(t *testing.T, expected []string, module *Module) {
+ t.Helper()
+ actual := module.Properties.AndroidMkSharedLibs
+ if !reflect.DeepEqual(actual, expected) {
+ t.Errorf("incorrect shared_libs"+
+ "\nactual: %v"+
+ "\nexpected: %v",
+ actual,
+ expected,
+ )
+ }
+}
+
const staticLibAndroidBp = `
cc_library {
name: "lib1",
@@ -3054,6 +3081,156 @@
checkStaticLibs(t, []string{"lib1", "libc++_static", "libc++demangle", "libclang_rt.builtins"}, module)
}
+func TestLibDepAndroidMkExportInMixedBuilds(t *testing.T) {
+ bp := `
+ cc_library {
+ name: "static_dep",
+ }
+ cc_library {
+ name: "whole_static_dep",
+ }
+ cc_library {
+ name: "shared_dep",
+ }
+ cc_library {
+ name: "lib",
+ bazel_module: { label: "//:lib" },
+ static_libs: ["static_dep"],
+ whole_static_libs: ["whole_static_dep"],
+ shared_libs: ["shared_dep"],
+ }
+ cc_test {
+ name: "test",
+ bazel_module: { label: "//:test" },
+ static_libs: ["static_dep"],
+ whole_static_libs: ["whole_static_dep"],
+ shared_libs: ["shared_dep"],
+ gtest: false,
+ }
+ cc_binary {
+ name: "binary",
+ bazel_module: { label: "//:binary" },
+ static_libs: ["static_dep"],
+ whole_static_libs: ["whole_static_dep"],
+ shared_libs: ["shared_dep"],
+ }
+ `
+
+ testCases := []struct {
+ name string
+ moduleName string
+ variant string
+ androidMkInfo cquery.CcAndroidMkInfo
+ }{
+ {
+ name: "shared lib",
+ moduleName: "lib",
+ variant: "android_arm64_armv8-a_shared",
+ androidMkInfo: cquery.CcAndroidMkInfo{
+ LocalStaticLibs: []string{"static_dep"},
+ LocalWholeStaticLibs: []string{"whole_static_dep"},
+ LocalSharedLibs: []string{"shared_dep"},
+ },
+ },
+ {
+ name: "static lib",
+ moduleName: "lib",
+ variant: "android_arm64_armv8-a_static",
+ androidMkInfo: cquery.CcAndroidMkInfo{
+ LocalStaticLibs: []string{"static_dep"},
+ LocalWholeStaticLibs: []string{"whole_static_dep"},
+ LocalSharedLibs: []string{"shared_dep"},
+ },
+ },
+ {
+ name: "cc_test arm64",
+ moduleName: "test",
+ variant: "android_arm64_armv8-a",
+ androidMkInfo: cquery.CcAndroidMkInfo{
+ LocalStaticLibs: []string{"static_dep"},
+ LocalWholeStaticLibs: []string{"whole_static_dep"},
+ LocalSharedLibs: []string{"shared_dep"},
+ },
+ },
+ {
+ name: "cc_test arm",
+ moduleName: "test",
+ variant: "android_arm_armv7-a-neon",
+ androidMkInfo: cquery.CcAndroidMkInfo{
+ LocalStaticLibs: []string{"static_dep"},
+ LocalWholeStaticLibs: []string{"whole_static_dep"},
+ LocalSharedLibs: []string{"shared_dep"},
+ },
+ },
+ {
+ name: "cc_binary",
+ moduleName: "binary",
+ variant: "android_arm64_armv8-a",
+ androidMkInfo: cquery.CcAndroidMkInfo{
+ LocalStaticLibs: []string{"static_dep"},
+ LocalWholeStaticLibs: []string{"whole_static_dep"},
+ LocalSharedLibs: []string{"shared_dep"},
+ },
+ },
+ }
+
+ outputBaseDir := "out/bazel"
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ result := android.GroupFixturePreparers(
+ prepareForCcTest,
+ android.FixtureModifyConfig(func(config android.Config) {
+ config.BazelContext = android.MockBazelContext{
+ OutputBaseDir: outputBaseDir,
+ LabelToCcInfo: map[string]cquery.CcInfo{
+ "//:lib": cquery.CcInfo{
+ CcAndroidMkInfo: tc.androidMkInfo,
+ RootDynamicLibraries: []string{""},
+ },
+ "//:lib_bp2build_cc_library_static": cquery.CcInfo{
+ CcAndroidMkInfo: tc.androidMkInfo,
+ RootStaticArchives: []string{""},
+ },
+ },
+ LabelToCcBinary: map[string]cquery.CcUnstrippedInfo{
+ "//:test": cquery.CcUnstrippedInfo{
+ CcAndroidMkInfo: tc.androidMkInfo,
+ },
+ "//:binary": cquery.CcUnstrippedInfo{
+ CcAndroidMkInfo: tc.androidMkInfo,
+ },
+ },
+ }
+ }),
+ ).RunTestWithBp(t, bp)
+ ctx := result.TestContext
+
+ module := ctx.ModuleForTests(tc.moduleName, tc.variant).Module().(*Module)
+ entries := android.AndroidMkEntriesForTest(t, ctx, module)[0]
+ checkStaticLibs(t, tc.androidMkInfo.LocalStaticLibs, module)
+ missingStaticDeps := android.ListDifference(entries.EntryMap["LOCAL_STATIC_LIBRARIES"], tc.androidMkInfo.LocalStaticLibs)
+ if len(missingStaticDeps) > 0 {
+ t.Errorf("expected LOCAL_STATIC_LIBRARIES to be %q"+
+ " but was %q; difference: %q", tc.androidMkInfo.LocalStaticLibs, entries.EntryMap["LOCAL_STATIC_LIBRARIES"], missingStaticDeps)
+ }
+
+ checkWholeStaticLibs(t, tc.androidMkInfo.LocalWholeStaticLibs, module)
+ missingWholeStaticDeps := android.ListDifference(entries.EntryMap["LOCAL_WHOLE_STATIC_LIBRARIES"], tc.androidMkInfo.LocalWholeStaticLibs)
+ if len(missingWholeStaticDeps) > 0 {
+ t.Errorf("expected LOCAL_WHOLE_STATIC_LIBRARIES to be %q"+
+ " but was %q; difference: %q", tc.androidMkInfo.LocalWholeStaticLibs, entries.EntryMap["LOCAL_WHOLE_STATIC_LIBRARIES"], missingWholeStaticDeps)
+ }
+
+ checkSharedLibs(t, tc.androidMkInfo.LocalSharedLibs, module)
+ missingSharedDeps := android.ListDifference(entries.EntryMap["LOCAL_SHARED_LIBRARIES"], tc.androidMkInfo.LocalSharedLibs)
+ if len(missingSharedDeps) > 0 {
+ t.Errorf("expected LOCAL_SHARED_LIBRARIES to be %q"+
+ " but was %q; difference: %q", tc.androidMkInfo.LocalSharedLibs, entries.EntryMap["LOCAL_SHARED_LIBRARIES"], missingSharedDeps)
+ }
+ })
+ }
+}
+
var compilerFlagsTestCases = []struct {
in string
out bool
diff --git a/cc/config/global.go b/cc/config/global.go
index 2205c9e..d557c0b 100644
--- a/cc/config/global.go
+++ b/cc/config/global.go
@@ -386,30 +386,17 @@
return strings.Join(deviceGlobalCflags, " ")
})
- // Export the static default NoOverrideGlobalCflags and NoOverride64GlobalCflags to Bazel.
+ // Export the static default NoOverrideGlobalCflags to Bazel.
exportedVars.ExportStringList("NoOverrideGlobalCflags", noOverrideGlobalCflags)
- exportedVars.ExportStringList("NoOverride64GlobalCflags", noOverride64GlobalCflags)
pctx.VariableFunc("NoOverrideGlobalCflags", func(ctx android.PackageVarContext) string {
flags := noOverrideGlobalCflags
if ctx.Config().IsEnvTrue("LLVM_NEXT") {
flags = append(noOverrideGlobalCflags, llvmNextExtraCommonGlobalCflags...)
- if ctx.Config().Android64() {
- flags = append(noOverride64GlobalCflags)
- }
}
return strings.Join(flags, " ")
})
- // Export the static default NoOverride64GlobalCflags to Bazel.
- exportedVars.ExportStringList("NoOverride64GlobalCflags", noOverride64GlobalCflags)
- pctx.VariableFunc("NoOverride64GlobalCflags", func(ctx android.PackageVarContext) string {
- flags := noOverride64GlobalCflags
- if ctx.Config().IsEnvTrue("LLVM_NEXT") && ctx.Config().Android64() {
- flags = append(noOverride64GlobalCflags, llvmNextExtraCommonGlobalCflags...)
- }
- return strings.Join(flags, " ")
- })
-
+ exportedVars.ExportStringListStaticVariable("NoOverride64GlobalCflags", noOverride64GlobalCflags)
exportedVars.ExportStringListStaticVariable("HostGlobalCflags", hostGlobalCflags)
exportedVars.ExportStringListStaticVariable("NoOverrideExternalGlobalCflags", noOverrideExternalGlobalCflags)
exportedVars.ExportStringListStaticVariable("CommonGlobalCppflags", commonGlobalCppflags)
diff --git a/cc/library.go b/cc/library.go
index 1291f5c..94984cb 100644
--- a/cc/library.go
+++ b/cc/library.go
@@ -475,10 +475,10 @@
func apiContributionBp2Build(ctx android.TopDownMutatorContext, module *Module) {
apiSurfaces := make([]string, 0)
apiHeaders := make([]string, 0)
- // systemapi (non-null `stubs` property)
+ // module-libapi for apexes (non-null `stubs` property)
if module.HasStubsVariants() {
- apiSurfaces = append(apiSurfaces, android.SystemApi.String())
- apiIncludes := getSystemApiIncludes(ctx, module)
+ apiSurfaces = append(apiSurfaces, android.ModuleLibApi.String())
+ apiIncludes := getModuleLibApiIncludes(ctx, module)
if !apiIncludes.isEmpty() {
createApiHeaderTarget(ctx, apiIncludes)
apiHeaders = append(apiHeaders, apiIncludes.name)
@@ -494,8 +494,8 @@
}
}
// create a target only if this module contributes to an api surface
- // TODO: Currently this does not distinguish systemapi-only headers and vendrorapi-only headers
- // TODO: Update so that systemapi-only headers do not get exported to vendorapi (and vice-versa)
+ // TODO: Currently this does not distinguish modulelibapi-only headers and vendrorapi-only headers
+ // TODO: Update so that modulelibapi-only headers do not get exported to vendorapi (and vice-versa)
if len(apiSurfaces) > 0 {
props := bazel.BazelTargetModuleProperties{
Rule_class: "cc_api_contribution",
@@ -527,8 +527,8 @@
linker := module.linker.(*libraryDecorator)
if llndkApi := linker.Properties.Llndk.Symbol_file; llndkApi != nil {
apiFile = llndkApi
- } else if systemApi := linker.Properties.Stubs.Symbol_file; systemApi != nil {
- apiFile = systemApi
+ } else if moduleLibApi := linker.Properties.Stubs.Symbol_file; moduleLibApi != nil {
+ apiFile = moduleLibApi
} else {
ctx.ModuleErrorf("API surface library does not have any API file")
}
@@ -566,7 +566,8 @@
includes.attrs.Deps.Append(lla)
}
-func getSystemApiIncludes(ctx android.TopDownMutatorContext, c *Module) apiIncludes {
+// includes provided to the module-lib API surface. This API surface is used by apexes.
+func getModuleLibApiIncludes(ctx android.TopDownMutatorContext, c *Module) apiIncludes {
flagProps := c.library.(*libraryDecorator).flagExporter.Properties
linkProps := c.library.(*libraryDecorator).baseLinker.Properties
includes := android.FirstUniqueStrings(flagProps.Export_include_dirs)
@@ -579,7 +580,7 @@
}
return apiIncludes{
- name: c.Name() + ".systemapi.headers",
+ name: c.Name() + ".module-libapi.headers",
attrs: bazelCcApiLibraryHeadersAttributes{
bazelCcLibraryHeadersAttributes: attrs,
},
diff --git a/cc/library_headers.go b/cc/library_headers.go
index 4d38068..c77d253 100644
--- a/cc/library_headers.go
+++ b/cc/library_headers.go
@@ -169,7 +169,7 @@
// For API export, create a top-level arch-agnostic target and list the arch-specific targets as its deps
// arch-agnostic includes
- apiIncludes := getSystemApiIncludes(ctx, module)
+ apiIncludes := getModuleLibApiIncludes(ctx, module)
// arch and os specific includes
archApiIncludes, androidOsIncludes := archOsSpecificApiIncludes(ctx, module)
for _, arch := range allArches { // sorted iteration
@@ -186,7 +186,7 @@
}
if !apiIncludes.isEmpty() {
- // override the name from <mod>.systemapi.headers --> <mod>.contribution
+ // override the name from <mod>.module-libapi.headers --> <mod>.contribution
apiIncludes.name = android.ApiContributionTargetName(module.Name())
createApiHeaderTarget(ctx, apiIncludes)
}
diff --git a/cc/object.go b/cc/object.go
index 6cb1a30..11ce793 100644
--- a/cc/object.go
+++ b/cc/object.go
@@ -142,6 +142,7 @@
Absolute_includes bazel.StringListAttribute
Stl *string
Linker_script bazel.LabelAttribute
+ Crt *bool
sdkAttributes
}
@@ -208,6 +209,7 @@
Absolute_includes: compilerAttrs.absoluteIncludes,
Stl: compilerAttrs.stl,
Linker_script: linkerScript,
+ Crt: m.linker.(*objectLinker).Properties.Crt,
sdkAttributes: bp2BuildParseSdkAttributes(m),
}
diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go
index 712c7fc..e3d1179 100644
--- a/cmd/merge_zips/merge_zips.go
+++ b/cmd/merge_zips/merge_zips.go
@@ -359,7 +359,8 @@
}
for _, file := range inputZip.Entries() {
pyPkg := getPackage(file.Name)
- if filepath.Base(file.Name) == "__init__.py" {
+ baseName := filepath.Base(file.Name)
+ if baseName == "__init__.py" || baseName == "__init__.pyc" {
if _, found := initedPackages[pyPkg]; found {
panic(fmt.Errorf("found __init__.py path duplicates during pars merging: %q", file.Name))
}
diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go
index 8c99988..661bd5d 100644
--- a/cmd/soong_ui/main.go
+++ b/cmd/soong_ui/main.go
@@ -199,11 +199,16 @@
soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
bp2buildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bp2build_metrics.pb")
+ bazelMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bazel_metrics.pb")
+
+ //the profile file generated by Bazel"
+ bazelProfileFile := filepath.Join(logsDir, c.logsPrefix+"analyzed_bazel_profile.txt")
metricsFiles := []string{
buildErrorFile, // build error strings
rbeMetricsFile, // high level metrics related to remote build execution.
bp2buildMetricsFile, // high level metrics related to bp2build.
soongMetricsFile, // high level metrics related to this build system.
+ bazelMetricsFile, // high level metrics related to bazel execution
config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
}
@@ -217,7 +222,7 @@
defer met.Dump(soongMetricsFile)
if !config.SkipMetricsUpload() {
- defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, metricsFiles...)
+ defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, bazelProfileFile, bazelMetricsFile, metricsFiles...)
}
}
diff --git a/java/config/config.go b/java/config/config.go
index 49d88c4..b69a715 100644
--- a/java/config/config.go
+++ b/java/config/config.go
@@ -98,6 +98,8 @@
"-JXmx2048M",
// Disable this optimization as it can impact weak reference semantics. See b/233432839.
"-JDcom.android.tools.r8.disableEnqueuerDeferredTracing=true",
+ // Disable class merging across different files to improve attribution. See b/242881914.
+ "-JDcom.android.tools.r8.enableSameFilePolicy=true",
}, dexerJavaVmFlagsList...))
exportedVars.ExportStringListStaticVariable("CommonJdkFlags", []string{
diff --git a/java/droidstubs.go b/java/droidstubs.go
index 4bbe70a..7ea8d30 100644
--- a/java/droidstubs.go
+++ b/java/droidstubs.go
@@ -148,6 +148,10 @@
// path or filegroup to file defining extension an SDK name <-> numerical ID mapping and
// what APIs exist in which SDKs; passed to metalava via --sdk-extensions-info
Extensions_info_file *string `android:"path"`
+
+ // API surface of this module. If set, the module contributes to an API surface.
+ // For the full list of available API surfaces, refer to soong/android/sdk_version.go
+ Api_surface *string
}
// Used by xsd_config
@@ -178,6 +182,10 @@
&module.Javadoc.properties)
InitDroiddocModule(module, android.HostAndDeviceSupported)
+
+ module.SetDefaultableHook(func(ctx android.DefaultableHookContext) {
+ module.createApiContribution(ctx)
+ })
return module
}
@@ -862,6 +870,23 @@
}, attrs)
}
+func (d *Droidstubs) createApiContribution(ctx android.DefaultableHookContext) {
+ api_file := d.properties.Check_api.Current.Api_file
+ api_surface := d.properties.Api_surface
+
+ props := struct {
+ Name *string
+ Api_surface *string
+ Api_file *string
+ }{}
+
+ props.Name = proptools.StringPtr(d.Name() + ".api.contribution")
+ props.Api_surface = api_surface
+ props.Api_file = api_file
+
+ ctx.CreateModule(ApiContributionFactory, &props)
+}
+
// TODO (b/262014796): Export the API contributions of CorePlatformApi
// A map to populate the api surface of a droidstub from a substring appearing in its name
// This map assumes that droidstubs (either checked-in or created by java_sdk_library)
diff --git a/java/droidstubs_test.go b/java/droidstubs_test.go
index ef2e6dc..6c22937 100644
--- a/java/droidstubs_test.go
+++ b/java/droidstubs_test.go
@@ -346,3 +346,27 @@
android.AssertStringEquals(t, tc.desc, tc.expectedApiSurface, bazelApiSurfaceName(tc.name))
}
}
+
+func TestDroidStubsApiContributionGeneration(t *testing.T) {
+ ctx, _ := testJavaWithFS(t, `
+ droidstubs {
+ name: "foo",
+ srcs: ["A/a.java"],
+ api_surface: "public",
+ check_api: {
+ current: {
+ api_file: "A/current.txt",
+ removed_api_file: "A/removed.txt",
+ }
+ }
+ }
+ `,
+ map[string][]byte{
+ "A/a.java": nil,
+ "A/current.txt": nil,
+ "A/removed.txt": nil,
+ },
+ )
+
+ ctx.ModuleForTests("foo.api.contribution", "")
+}
diff --git a/java/java.go b/java/java.go
index 1907731..659f98a 100644
--- a/java/java.go
+++ b/java/java.go
@@ -1594,7 +1594,11 @@
var JavaApiImportProvider = blueprint.NewProvider(JavaApiImportInfo{})
func (ap *JavaApiContribution) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- apiFile := android.PathForModuleSrc(ctx, String(ap.properties.Api_file))
+ var apiFile android.Path = nil
+ if apiFileString := ap.properties.Api_file; apiFileString != nil {
+ apiFile = android.PathForModuleSrc(ctx, String(apiFileString))
+ }
+
ctx.SetProvider(JavaApiImportProvider, JavaApiImportInfo{
ApiFile: apiFile,
})
@@ -1725,7 +1729,11 @@
switch tag {
case javaApiContributionTag:
provider := ctx.OtherModuleProvider(dep, JavaApiImportProvider).(JavaApiImportInfo)
- srcFiles = append(srcFiles, android.PathForSource(ctx, provider.ApiFile.String()))
+ providerApiFile := provider.ApiFile
+ if providerApiFile == nil {
+ ctx.ModuleErrorf("Error: %s has an empty api file.", dep.Name())
+ }
+ srcFiles = append(srcFiles, android.PathForSource(ctx, providerApiFile.String()))
case libTag:
provider := ctx.OtherModuleProvider(dep, JavaInfoProvider).(JavaInfo)
classPaths = append(classPaths, provider.HeaderJars...)
diff --git a/java/java_test.go b/java/java_test.go
index ae77842..21993ec 100644
--- a/java/java_test.go
+++ b/java/java_test.go
@@ -30,7 +30,6 @@
"android/soong/cc"
"android/soong/dexpreopt"
"android/soong/genrule"
- "android/soong/python"
)
// Legacy preparer used for running tests within the java package.
@@ -49,7 +48,6 @@
// Include all the default java modules.
PrepareForTestWithJavaDefaultModules,
PrepareForTestWithOverlayBuildComponents,
- python.PrepareForTestWithPythonBuildComponents,
android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) {
ctx.RegisterPreSingletonType("sdk_versions", sdkPreSingletonFactory)
}),
@@ -1440,24 +1438,26 @@
}
func TestDataNativeBinaries(t *testing.T) {
- ctx, _ := testJava(t, `
+ ctx := android.GroupFixturePreparers(
+ prepareForJavaTest,
+ android.PrepareForTestWithAllowMissingDependencies).RunTestWithBp(t, `
java_test_host {
name: "foo",
srcs: ["a.java"],
data_native_bins: ["bin"]
}
- python_binary_host {
+ cc_binary_host {
name: "bin",
- srcs: ["bin.py"],
+ srcs: ["bin.cpp"],
}
- `)
+ `).TestContext
buildOS := ctx.Config().BuildOS.String()
test := ctx.ModuleForTests("foo", buildOS+"_common").Module().(*TestHost)
entries := android.AndroidMkEntriesForTest(t, ctx, test)[0]
- expected := []string{"out/soong/.intermediates/bin/" + buildOS + "_x86_64_PY3/bin:bin"}
+ expected := []string{"out/soong/.intermediates/bin/" + buildOS + "_x86_64/bin:bin"}
actual := entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"]
android.AssertStringPathsRelativeToTopEquals(t, "LOCAL_COMPATIBILITY_SUPPORT_FILES", ctx.Config(), expected, actual)
}
@@ -1840,6 +1840,20 @@
}`)
}
+func TestJavaApiContributionEmptyApiFile(t *testing.T) {
+ testJavaError(t,
+ "Error: foo has an empty api file.",
+ `java_api_contribution {
+ name: "foo",
+ }
+ java_api_library {
+ name: "bar",
+ api_surface: "public",
+ api_contributions: ["foo"],
+ }
+ `)
+}
+
func TestJavaApiLibraryAndProviderLink(t *testing.T) {
provider_bp_a := `
java_api_contribution {
diff --git a/java/sdk_library.go b/java/sdk_library.go
index 3b64bf7..b872365 100644
--- a/java/sdk_library.go
+++ b/java/sdk_library.go
@@ -1599,6 +1599,7 @@
Srcs []string
Installable *bool
Sdk_version *string
+ Api_surface *string
System_modules *string
Libs []string
Output_javadoc_comments *bool
@@ -1638,6 +1639,7 @@
props.Srcs = append(props.Srcs, module.properties.Srcs...)
props.Srcs = append(props.Srcs, module.sdkLibraryProperties.Api_srcs...)
props.Sdk_version = module.deviceProperties.Sdk_version
+ props.Api_surface = &apiScope.name
props.System_modules = module.deviceProperties.System_modules
props.Installable = proptools.BoolPtr(false)
// A droiddoc module has only one Libs property and doesn't distinguish between
diff --git a/python/Android.bp b/python/Android.bp
index 4584f1e..7578673 100644
--- a/python/Android.bp
+++ b/python/Android.bp
@@ -9,6 +9,7 @@
"blueprint",
"soong-android",
"soong-tradefed",
+ "soong-cc",
],
srcs: [
"binary.go",
diff --git a/python/binary.go b/python/binary.go
index 95eb2c6..75135f3 100644
--- a/python/binary.go
+++ b/python/binary.go
@@ -21,8 +21,6 @@
"path/filepath"
"strings"
- "github.com/google/blueprint"
-
"android/soong/android"
)
@@ -109,14 +107,14 @@
}
func (p *PythonBinaryModule) buildBinary(ctx android.ModuleContext) {
- depsSrcsZips := p.collectPathsFromTransitiveDeps(ctx)
+ embeddedLauncher := p.isEmbeddedLauncherEnabled()
+ depsSrcsZips := p.collectPathsFromTransitiveDeps(ctx, embeddedLauncher)
main := ""
if p.autorun() {
main = p.getPyMainFile(ctx, p.srcsPathMappings)
}
var launcherPath android.OptionalPath
- embeddedLauncher := p.isEmbeddedLauncherEnabled()
if embeddedLauncher {
ctx.VisitDirectDepsWithTag(launcherTag, func(m android.Module) {
if provider, ok := m.(IntermPathProvider); ok {
@@ -128,9 +126,16 @@
}
})
}
+ srcsZips := make(android.Paths, 0, len(depsSrcsZips)+1)
+ if embeddedLauncher {
+ srcsZips = append(srcsZips, p.precompiledSrcsZip)
+ } else {
+ srcsZips = append(srcsZips, p.srcsZip)
+ }
+ srcsZips = append(srcsZips, depsSrcsZips...)
p.installSource = registerBuildActionForParFile(ctx, embeddedLauncher, launcherPath,
p.getHostInterpreterName(ctx, p.properties.Actual_version),
- main, p.getStem(ctx), append(android.Paths{p.srcsZip}, depsSrcsZips...))
+ main, p.getStem(ctx), srcsZips)
var sharedLibs []string
// if embedded launcher is enabled, we need to collect the shared library dependencies of the
@@ -170,64 +175,8 @@
func (p *PythonBinaryModule) DepsMutator(ctx android.BottomUpMutatorContext) {
p.PythonLibraryModule.DepsMutator(ctx)
- versionVariation := []blueprint.Variation{
- {"python_version", p.properties.Actual_version},
- }
-
- // If this module will be installed and has an embedded launcher, we need to add dependencies for:
- // * standard library
- // * launcher
- // * shared dependencies of the launcher
if p.isEmbeddedLauncherEnabled() {
- var stdLib string
- var launcherModule string
- // Add launcher shared lib dependencies. Ideally, these should be
- // derived from the `shared_libs` property of the launcher. However, we
- // cannot read the property at this stage and it will be too late to add
- // dependencies later.
- launcherSharedLibDeps := []string{
- "libsqlite",
- }
- // Add launcher-specific dependencies for bionic
- if ctx.Target().Os.Bionic() {
- launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm")
- }
- if ctx.Target().Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() {
- launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl")
- }
-
- switch p.properties.Actual_version {
- case pyVersion2:
- stdLib = "py2-stdlib"
-
- launcherModule = "py2-launcher"
- if p.autorun() {
- launcherModule = "py2-launcher-autorun"
- }
-
- launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
-
- case pyVersion3:
- stdLib = "py3-stdlib"
-
- launcherModule = "py3-launcher"
- if p.autorun() {
- launcherModule = "py3-launcher-autorun"
- }
- if ctx.Config().HostStaticBinaries() && ctx.Target().Os == android.LinuxMusl {
- launcherModule += "-static"
- }
-
- if ctx.Device() {
- launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
- }
- default:
- panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
- p.properties.Actual_version, ctx.ModuleName()))
- }
- ctx.AddVariationDependencies(versionVariation, pythonLibTag, stdLib)
- ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherTag, launcherModule)
- ctx.AddFarVariationDependencies(ctx.Target().Variations(), launcherSharedLibTag, launcherSharedLibDeps...)
+ p.AddDepsOnPythonLauncherAndStdlib(ctx, pythonLibTag, launcherTag, launcherSharedLibTag, p.autorun(), ctx.Target())
}
}
diff --git a/python/builder.go b/python/builder.go
index b4ab206..1066493 100644
--- a/python/builder.go
+++ b/python/builder.go
@@ -70,6 +70,17 @@
CommandDeps: []string{"$mergeParCmd"},
},
"srcsZips", "launcher")
+
+ precompile = pctx.AndroidStaticRule("precompilePython", blueprint.RuleParams{
+ Command: `LD_LIBRARY_PATH="$ldLibraryPath" ` +
+ `PYTHONPATH=$stdlibZip/internal/stdlib ` +
+ `$launcher build/soong/python/scripts/precompile_python.py $in $out`,
+ CommandDeps: []string{
+ "$stdlibZip",
+ "$launcher",
+ "build/soong/python/scripts/precompile_python.py",
+ },
+ }, "stdlibZip", "launcher", "ldLibraryPath")
)
func init() {
diff --git a/python/python.go b/python/python.go
index 2b71e83..18e5b68 100644
--- a/python/python.go
+++ b/python/python.go
@@ -146,8 +146,12 @@
// pathMapping: <dest: runfile_path, src: source_path>
dataPathMappings []pathMapping
- // the zip filepath for zipping current module source/data files.
+ // The zip file containing the current module's source/data files.
srcsZip android.Path
+
+ // The zip file containing the current module's source/data files, with the
+ // source files precompiled.
+ precompiledSrcsZip android.Path
}
// newModule generates new Python base module
@@ -164,6 +168,7 @@
getSrcsPathMappings() []pathMapping
getDataPathMappings() []pathMapping
getSrcsZip() android.Path
+ getPrecompiledSrcsZip() android.Path
}
// getSrcsPathMappings gets this module's path mapping of src source path : runfiles destination
@@ -181,6 +186,11 @@
return p.srcsZip
}
+// getSrcsZip returns the filepath where the current module's source/data files are zipped.
+func (p *PythonLibraryModule) getPrecompiledSrcsZip() android.Path {
+ return p.precompiledSrcsZip
+}
+
func (p *PythonLibraryModule) getBaseProperties() *BaseProperties {
return &p.properties
}
@@ -213,16 +223,23 @@
}
var (
- pythonLibTag = dependencyTag{name: "pythonLib"}
- javaDataTag = dependencyTag{name: "javaData"}
+ pythonLibTag = dependencyTag{name: "pythonLib"}
+ javaDataTag = dependencyTag{name: "javaData"}
+ // The python interpreter, with soong module name "py3-launcher" or "py3-launcher-autorun".
launcherTag = dependencyTag{name: "launcher"}
launcherSharedLibTag = installDependencyTag{name: "launcherSharedLib"}
- pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
- pyExt = ".py"
- protoExt = ".proto"
- pyVersion2 = "PY2"
- pyVersion3 = "PY3"
- internalPath = "internal"
+ // The python interpreter built for host so that we can precompile python sources.
+ // This only works because the precompiled sources don't vary by architecture.
+ // The soong module name is "py3-launcher".
+ hostLauncherTag = dependencyTag{name: "hostLauncher"}
+ hostlauncherSharedLibTag = dependencyTag{name: "hostlauncherSharedLib"}
+ hostStdLibTag = dependencyTag{name: "hostStdLib"}
+ pathComponentRegexp = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
+ pyExt = ".py"
+ protoExt = ".proto"
+ pyVersion2 = "PY2"
+ pyVersion3 = "PY3"
+ internalPath = "internal"
)
type basePropertiesProvider interface {
@@ -305,6 +322,76 @@
// so that it can point to java modules.
javaDataVariation := []blueprint.Variation{{"arch", android.Common.String()}}
ctx.AddVariationDependencies(javaDataVariation, javaDataTag, p.properties.Java_data...)
+
+ p.AddDepsOnPythonLauncherAndStdlib(ctx, hostStdLibTag, hostLauncherTag, hostlauncherSharedLibTag, false, ctx.Config().BuildOSTarget)
+}
+
+// AddDepsOnPythonLauncherAndStdlib will make the current module depend on the python stdlib,
+// launcher (interpreter), and the launcher's shared libraries. If autorun is true, it will use
+// the autorun launcher instead of the regular one. This function acceps a targetForDeps argument
+// as the target to use for these dependencies. For embedded launcher python binaries, the launcher
+// that will be embedded will be under the same target as the python module itself. But when
+// precompiling python code, we need to get the python launcher built for host, even if we're
+// compiling the python module for device, so we pass a different target to this function.
+func (p *PythonLibraryModule) AddDepsOnPythonLauncherAndStdlib(ctx android.BottomUpMutatorContext,
+ stdLibTag, launcherTag, launcherSharedLibTag blueprint.DependencyTag,
+ autorun bool, targetForDeps android.Target) {
+ var stdLib string
+ var launcherModule string
+ // Add launcher shared lib dependencies. Ideally, these should be
+ // derived from the `shared_libs` property of the launcher. TODO: read these from
+ // the python launcher itself using ctx.OtherModuleProvider() or similar on the result
+ // of ctx.AddFarVariationDependencies()
+ launcherSharedLibDeps := []string{
+ "libsqlite",
+ }
+ // Add launcher-specific dependencies for bionic
+ if targetForDeps.Os.Bionic() {
+ launcherSharedLibDeps = append(launcherSharedLibDeps, "libc", "libdl", "libm")
+ }
+ if targetForDeps.Os == android.LinuxMusl && !ctx.Config().HostStaticBinaries() {
+ launcherSharedLibDeps = append(launcherSharedLibDeps, "libc_musl")
+ }
+
+ switch p.properties.Actual_version {
+ case pyVersion2:
+ stdLib = "py2-stdlib"
+
+ launcherModule = "py2-launcher"
+ if autorun {
+ launcherModule = "py2-launcher-autorun"
+ }
+
+ launcherSharedLibDeps = append(launcherSharedLibDeps, "libc++")
+ case pyVersion3:
+ stdLib = "py3-stdlib"
+
+ launcherModule = "py3-launcher"
+ if autorun {
+ launcherModule = "py3-launcher-autorun"
+ }
+ if ctx.Config().HostStaticBinaries() && targetForDeps.Os == android.LinuxMusl {
+ launcherModule += "-static"
+ }
+ if ctx.Device() {
+ launcherSharedLibDeps = append(launcherSharedLibDeps, "liblog")
+ }
+ default:
+ panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
+ p.properties.Actual_version, ctx.ModuleName()))
+ }
+ targetVariations := targetForDeps.Variations()
+ if ctx.ModuleName() != stdLib {
+ stdLibVariations := make([]blueprint.Variation, 0, len(targetVariations)+1)
+ stdLibVariations = append(stdLibVariations, blueprint.Variation{Mutator: "python_version", Variation: p.properties.Actual_version})
+ stdLibVariations = append(stdLibVariations, targetVariations...)
+ // Using AddFarVariationDependencies for all of these because they can be for a different
+ // platform, like if the python module itself was being compiled for device, we may want
+ // the python interpreter built for host so that we can precompile python sources.
+ ctx.AddFarVariationDependencies(stdLibVariations, stdLibTag, stdLib)
+ }
+ ctx.AddFarVariationDependencies(targetVariations, launcherTag, launcherModule)
+ ctx.AddFarVariationDependencies(targetVariations, launcherSharedLibTag, launcherSharedLibDeps...)
}
// GenerateAndroidBuildActions performs build actions common to all Python modules
@@ -342,6 +429,7 @@
// generate the zipfile of all source and data files
p.srcsZip = p.createSrcsZip(ctx, pkgPath)
+ p.precompiledSrcsZip = p.precompileSrcs(ctx)
}
func isValidPythonPath(path string) error {
@@ -397,12 +485,8 @@
// createSrcsZip registers build actions to zip current module's sources and data.
func (p *PythonLibraryModule) createSrcsZip(ctx android.ModuleContext, pkgPath string) android.Path {
relativeRootMap := make(map[string]android.Paths)
- pathMappings := append(p.srcsPathMappings, p.dataPathMappings...)
-
var protoSrcs android.Paths
- // "srcs" or "data" properties may contain filegroup so it might happen that
- // the root directory for each source path is different.
- for _, path := range pathMappings {
+ addPathMapping := func(path pathMapping) {
// handle proto sources separately
if path.src.Ext() == protoExt {
protoSrcs = append(protoSrcs, path.src)
@@ -411,6 +495,16 @@
relativeRootMap[relativeRoot] = append(relativeRootMap[relativeRoot], path.src)
}
}
+
+ // "srcs" or "data" properties may contain filegroups so it might happen that
+ // the root directory for each source path is different.
+ for _, path := range p.srcsPathMappings {
+ addPathMapping(path)
+ }
+ for _, path := range p.dataPathMappings {
+ addPathMapping(path)
+ }
+
var zips android.Paths
if len(protoSrcs) > 0 {
protoFlags := android.GetProtoFlags(ctx, &p.protoProperties)
@@ -484,6 +578,65 @@
}
}
+func (p *PythonLibraryModule) precompileSrcs(ctx android.ModuleContext) android.Path {
+ // To precompile the python sources, we need a python interpreter and stdlib built
+ // for host. We then use those to compile the python sources, which may be used on either
+ // host of device. Python bytecode is architecture agnostic, so we're essentially
+ // "cross compiling" for device here purely by virtue of host and device python bytecode
+ // being the same.
+ var stdLib android.Path
+ var launcher android.Path
+ if ctx.ModuleName() == "py3-stdlib" || ctx.ModuleName() == "py2-stdlib" {
+ stdLib = p.srcsZip
+ } else {
+ ctx.VisitDirectDepsWithTag(hostStdLibTag, func(module android.Module) {
+ if dep, ok := module.(pythonDependency); ok {
+ stdLib = dep.getPrecompiledSrcsZip()
+ }
+ })
+ }
+ ctx.VisitDirectDepsWithTag(hostLauncherTag, func(module android.Module) {
+ if dep, ok := module.(IntermPathProvider); ok {
+ optionalLauncher := dep.IntermPathForModuleOut()
+ if optionalLauncher.Valid() {
+ launcher = optionalLauncher.Path()
+ }
+ }
+ })
+ var launcherSharedLibs android.Paths
+ var ldLibraryPath []string
+ ctx.VisitDirectDepsWithTag(hostlauncherSharedLibTag, func(module android.Module) {
+ if dep, ok := module.(IntermPathProvider); ok {
+ optionalPath := dep.IntermPathForModuleOut()
+ if optionalPath.Valid() {
+ launcherSharedLibs = append(launcherSharedLibs, optionalPath.Path())
+ ldLibraryPath = append(ldLibraryPath, filepath.Dir(optionalPath.Path().String()))
+ }
+ }
+ })
+
+ out := android.PathForModuleOut(ctx, ctx.ModuleName()+".srcszipprecompiled")
+ if stdLib == nil || launcher == nil {
+ // This shouldn't happen in a real build because we'll error out when adding dependencies
+ // on the stdlib and launcher if they don't exist. But some tests set
+ // AllowMissingDependencies.
+ return out
+ }
+ ctx.Build(pctx, android.BuildParams{
+ Rule: precompile,
+ Input: p.srcsZip,
+ Output: out,
+ Implicits: launcherSharedLibs,
+ Description: "Precompile the python sources of " + ctx.ModuleName(),
+ Args: map[string]string{
+ "stdlibZip": stdLib.String(),
+ "launcher": launcher.String(),
+ "ldLibraryPath": strings.Join(ldLibraryPath, ":"),
+ },
+ })
+ return out
+}
+
// isPythonLibModule returns whether the given module is a Python library PythonLibraryModule or not
func isPythonLibModule(module blueprint.Module) bool {
if _, ok := module.(*PythonLibraryModule); ok {
@@ -497,7 +650,7 @@
// collectPathsFromTransitiveDeps checks for source/data files for duplicate paths
// for module and its transitive dependencies and collects list of data/source file
// zips for transitive dependencies.
-func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleContext) android.Paths {
+func (p *PythonLibraryModule) collectPathsFromTransitiveDeps(ctx android.ModuleContext, precompiled bool) android.Paths {
// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
// check duplicates.
destToPySrcs := make(map[string]string)
@@ -541,7 +694,11 @@
checkForDuplicateOutputPath(ctx, destToPyData,
path.dest, path.src.String(), ctx.ModuleName(), ctx.OtherModuleName(child))
}
- result = append(result, dep.getSrcsZip())
+ if precompiled {
+ result = append(result, dep.getPrecompiledSrcsZip())
+ } else {
+ result = append(result, dep.getSrcsZip())
+ }
}
return true
})
diff --git a/python/python_test.go b/python/python_test.go
index 6f4223a..75a6a89 100644
--- a/python/python_test.go
+++ b/python/python_test.go
@@ -18,10 +18,10 @@
"fmt"
"os"
"path/filepath"
- "regexp"
"testing"
"android/soong/android"
+ "android/soong/cc"
)
type pyModule struct {
@@ -33,8 +33,10 @@
}
var (
- buildNamePrefix = "soong_python_test"
- moduleVariantErrTemplate = "%s: module %q variant %q: "
+ buildNamePrefix = "soong_python_test"
+ // We allow maching almost anything before the actual variant so that the os/arch variant
+ // is matched.
+ moduleVariantErrTemplate = `%s: module %q variant "[a-zA-Z0-9_]*%s": `
pkgPathErrTemplate = moduleVariantErrTemplate +
"pkg_path: %q must be a relative path contained in par file."
badIdentifierErrTemplate = moduleVariantErrTemplate +
@@ -323,17 +325,26 @@
if d.desc != "module with duplicate runfile path" {
continue
}
- errorPatterns := make([]string, len(d.errors))
- for i, s := range d.errors {
- errorPatterns[i] = regexp.QuoteMeta(s)
- }
+ d.mockFiles[filepath.Join("common", bpFile)] = []byte(`
+python_library {
+ name: "py3-stdlib",
+ host_supported: true,
+}
+cc_binary {
+ name: "py3-launcher",
+ host_supported: true,
+}
+`)
t.Run(d.desc, func(t *testing.T) {
result := android.GroupFixturePreparers(
android.PrepareForTestWithDefaults,
+ android.PrepareForTestWithArchMutator,
+ android.PrepareForTestWithAllowMissingDependencies,
+ cc.PrepareForTestWithCcDefaultModules,
PrepareForTestWithPythonBuildComponents,
d.mockFiles.AddToFixture(),
- ).ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern(errorPatterns)).
+ ).ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern(d.errors)).
RunTest(t)
if len(result.Errs) > 0 {
diff --git a/python/scripts/precompile_python.py b/python/scripts/precompile_python.py
new file mode 100644
index 0000000..e12e7d2
--- /dev/null
+++ b/python/scripts/precompile_python.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# Copyright 2023 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import py_compile
+import os
+import shutil
+import tempfile
+import zipfile
+
+# This file needs to support both python 2 and 3.
+
+
+def process_one_file(name, inf, outzip):
+ if not name.endswith('.py'):
+ outzip.writestr(name, inf.read())
+ return
+
+ # Unfortunately py_compile requires the input/output files to be written
+ # out to disk.
+ with tempfile.NamedTemporaryFile(prefix="Soong_precompile_", delete=False) as tmp:
+ shutil.copyfileobj(inf, tmp)
+ in_name = tmp.name
+ with tempfile.NamedTemporaryFile(prefix="Soong_precompile_", delete=False) as tmp:
+ out_name = tmp.name
+ try:
+ py_compile.compile(in_name, out_name, name, doraise=True)
+ with open(out_name, 'rb') as f:
+ outzip.writestr(name + 'c', f.read())
+ finally:
+ os.remove(in_name)
+ os.remove(out_name)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('src_zip')
+ parser.add_argument('dst_zip')
+ args = parser.parse_args()
+
+ with open(args.dst_zip, 'wb') as outf, open(args.src_zip, 'rb') as inf:
+ with zipfile.ZipFile(outf, mode='w') as outzip, zipfile.ZipFile(inf, mode='r') as inzip:
+ for name in inzip.namelist():
+ with inzip.open(name, mode='r') as inzipf:
+ process_one_file(name, inzipf, outzip)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/tests/par_test.py b/python/tests/par_test.py
index 56a5063..1e03f16 100644
--- a/python/tests/par_test.py
+++ b/python/tests/par_test.py
@@ -27,7 +27,10 @@
failed = True
assert_equal("__name__", __name__, "__main__")
-assert_equal("os.path.basename(__file__)", os.path.basename(__file__), "par_test.py")
+fileName = os.path.basename(__file__)
+if fileName.endswith('.pyc'):
+ fileName = fileName[:-1]
+assert_equal("os.path.basename(__file__)", fileName, "par_test.py")
archive = os.path.dirname(__file__)
diff --git a/python/tests/testpkg/par_test.py b/python/tests/testpkg/par_test.py
index ffad430..b513409 100644
--- a/python/tests/testpkg/par_test.py
+++ b/python/tests/testpkg/par_test.py
@@ -28,7 +28,10 @@
archive = sys.modules["__main__"].__loader__.archive
assert_equal("__name__", __name__, "testpkg.par_test")
-assert_equal("__file__", __file__, os.path.join(archive, "testpkg/par_test.py"))
+fileName = __file__
+if fileName.endswith('.pyc'):
+ fileName = fileName[:-1]
+assert_equal("__file__", fileName, os.path.join(archive, "testpkg/par_test.py"))
# Python3 is returning None here for me, and I haven't found any problems caused by this.
if sys.version_info[0] == 2:
diff --git a/rust/config/global.go b/rust/config/global.go
index 26e2d06..7549969 100644
--- a/rust/config/global.go
+++ b/rust/config/global.go
@@ -24,7 +24,7 @@
var pctx = android.NewPackageContext("android/soong/rust/config")
var (
- RustDefaultVersion = "1.65.0"
+ RustDefaultVersion = "1.65.0.p1"
RustDefaultBase = "prebuilts/rust/"
DefaultEdition = "2021"
Stdlibs = []string{
diff --git a/tests/apex_cc_module_arch_variant_tests.sh b/tests/apex_cc_module_arch_variant_tests.sh
index 97e6576..b0cade1 100755
--- a/tests/apex_cc_module_arch_variant_tests.sh
+++ b/tests/apex_cc_module_arch_variant_tests.sh
@@ -56,7 +56,7 @@
# Number of CppCompile actions with arch variant flag
actions_with_arch_variant_num=$(call_bazel aquery --config=bp2build --config=ci --config=android \
- 'mnemonic("CppCompile", deps(//build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal.apex))' | grep -c "\-march=$ARCH_VARIANT_CFLAG")
+ 'mnemonic("CppCompile", deps(//build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal.apex))' | grep -c \'-march=$ARCH_VARIANT_CFLAG\')
# Number of all CppCompile actions
all_cppcompile_actions_num=0
diff --git a/tests/lib.sh b/tests/lib.sh
index 26bdc97..0973beb 100644
--- a/tests/lib.sh
+++ b/tests/lib.sh
@@ -94,6 +94,8 @@
symlink_directory external/go-cmp
symlink_directory external/golang-protobuf
symlink_directory external/starlark-go
+ symlink_directory external/python
+ symlink_directory external/sqlite
touch "$MOCK_TOP/Android.bp"
}
diff --git a/ui/build/upload.go b/ui/build/upload.go
index 9f14bdd..9959e6f 100644
--- a/ui/build/upload.go
+++ b/ui/build/upload.go
@@ -18,16 +18,21 @@
// another.
import (
+ "bufio"
"fmt"
"io/ioutil"
"os"
"path/filepath"
+ "strconv"
+ "strings"
"time"
+ "android/soong/shared"
"android/soong/ui/metrics"
"google.golang.org/protobuf/proto"
+ bazel_metrics_proto "android/soong/ui/metrics/bazel_metrics_proto"
upload_proto "android/soong/ui/metrics/upload_proto"
)
@@ -73,12 +78,113 @@
return metricsFiles
}
+func parseTimingToNanos(str string) int64 {
+ millisString := removeDecimalPoint(str)
+ timingMillis, _ := strconv.ParseInt(millisString, 10, 64)
+ return timingMillis * 1000000
+}
+
+func parsePercentageToTenThousandths(str string) int32 {
+ percentageString := removeDecimalPoint(str)
+ //remove the % at the end of the string
+ percentage := strings.ReplaceAll(percentageString, "%", "")
+ percentagePortion, _ := strconv.ParseInt(percentage, 10, 32)
+ return int32(percentagePortion)
+}
+
+func removeDecimalPoint(numString string) string {
+ // The format is always 0.425 or 10.425
+ return strings.ReplaceAll(numString, ".", "")
+}
+
+func parseTotal(line string) int64 {
+ words := strings.Fields(line)
+ timing := words[3]
+ return parseTimingToNanos(timing)
+}
+
+func parsePhaseTiming(line string) bazel_metrics_proto.PhaseTiming {
+ words := strings.Fields(line)
+ getPhaseNameAndTimingAndPercentage := func([]string) (string, int64, int32) {
+ // Sample lines include:
+ // Total launch phase time 0.011 s 2.59%
+ // Total target pattern evaluation phase time 0.011 s 2.59%
+ var beginning int
+ var end int
+ for ind, word := range words {
+ if word == "Total" {
+ beginning = ind + 1
+ } else if beginning > 0 && word == "phase" {
+ end = ind
+ break
+ }
+ }
+ phaseName := strings.Join(words[beginning:end], " ")
+
+ // end is now "phase" - advance by 2 for timing and 4 for percentage
+ percentageString := words[end+4]
+ timingString := words[end+2]
+ timing := parseTimingToNanos(timingString)
+ percentagePortion := parsePercentageToTenThousandths(percentageString)
+ return phaseName, timing, percentagePortion
+ }
+
+ phaseName, timing, portion := getPhaseNameAndTimingAndPercentage(words)
+ phaseTiming := bazel_metrics_proto.PhaseTiming{}
+ phaseTiming.DurationNanos = &timing
+ phaseTiming.PortionOfBuildTime = &portion
+
+ phaseTiming.PhaseName = &phaseName
+ return phaseTiming
+}
+
+func processBazelMetrics(bazelProfileFile string, bazelMetricsFile string, ctx Context) {
+ if bazelProfileFile == "" {
+ return
+ }
+
+ readBazelProto := func(filepath string) bazel_metrics_proto.BazelMetrics {
+ //serialize the proto, write it
+ bazelMetrics := bazel_metrics_proto.BazelMetrics{}
+
+ file, err := os.ReadFile(filepath)
+ if err != nil {
+ ctx.Fatalln("Error reading metrics file\n", err)
+ }
+
+ scanner := bufio.NewScanner(strings.NewReader(string(file)))
+ scanner.Split(bufio.ScanLines)
+
+ var phaseTimings []*bazel_metrics_proto.PhaseTiming
+ for scanner.Scan() {
+ line := scanner.Text()
+ if strings.HasPrefix(line, "Total run time") {
+ total := parseTotal(line)
+ bazelMetrics.Total = &total
+ } else if strings.HasPrefix(line, "Total") {
+ phaseTiming := parsePhaseTiming(line)
+ phaseTimings = append(phaseTimings, &phaseTiming)
+ }
+ }
+ bazelMetrics.PhaseTimings = phaseTimings
+
+ return bazelMetrics
+ }
+
+ if _, err := os.Stat(bazelProfileFile); err != nil {
+ // We can assume bazel didn't run if the profile doesn't exist
+ return
+ }
+ bazelProto := readBazelProto(bazelProfileFile)
+ shared.Save(&bazelProto, bazelMetricsFile)
+}
+
// UploadMetrics uploads a set of metrics files to a server for analysis.
// The metrics files are first copied to a temporary directory
// and the uploader is then executed in the background to allow the user/system
// to continue working. Soong communicates to the uploader through the
// upload_proto raw protobuf file.
-func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, paths ...string) {
+func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, bazelProfileFile string, bazelMetricsFile string, paths ...string) {
ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics")
defer ctx.EndTrace()
@@ -88,6 +194,7 @@
return
}
+ processBazelMetrics(bazelProfileFile, bazelMetricsFile, ctx)
// Several of the files might be directories.
metricsFiles := pruneMetricsFiles(paths)
if len(metricsFiles) == 0 {
diff --git a/ui/build/upload_test.go b/ui/build/upload_test.go
index 764a1e1..58d9237 100644
--- a/ui/build/upload_test.go
+++ b/ui/build/upload_test.go
@@ -29,6 +29,30 @@
"android/soong/ui/logger"
)
+func writeBazelProfileFile(dir string) error {
+ contents := `
+
+=== PHASE SUMMARY INFORMATION ===
+
+Total launch phase time 1.193 s 15.77%
+Total init phase time 1.092 s 14.44%
+Total target pattern evaluation phase time 0.580 s 7.67%
+Total interleaved loading-and-analysis phase time 3.646 s 48.21%
+Total preparation phase time 0.022 s 0.30%
+Total execution phase time 0.993 s 13.13%
+Total finish phase time 0.036 s 0.48%
+---------------------------------------------------------------------
+Total run time 7.563 s 100.00%
+
+Critical path (178 ms):
+ Time Percentage Description
+ 178 ms 100.00% action 'BazelWorkspaceStatusAction stable-status.txt'
+
+`
+ file := filepath.Join(dir, "bazel_metrics.txt")
+ return os.WriteFile(file, []byte(contents), 0666)
+}
+
func TestPruneMetricsFiles(t *testing.T) {
rootDir := t.TempDir()
@@ -84,12 +108,12 @@
}, {
description: "non-existent metrics files no upload",
uploader: "echo",
- files: []string{"metrics_file_1", "metrics_file_2", "metrics_file_3"},
+ files: []string{"metrics_file_1", "metrics_file_2", "metrics_file_3, bazel_metrics.pb"},
}, {
description: "trigger upload",
uploader: "echo",
createFiles: true,
- files: []string{"metrics_file_1", "metrics_file_2"},
+ files: []string{"metrics_file_1", "metrics_file_2, bazel_metrics.pb"},
}}
for _, tt := range tests {
@@ -130,6 +154,9 @@
}
}
}
+ if err := writeBazelProfileFile(outDir); err != nil {
+ t.Fatalf("failed to create bazel profile file in dir: %v", outDir)
+ }
config := Config{&configImpl{
environ: &Environment{
@@ -139,7 +166,7 @@
metricsUploader: tt.uploader,
}}
- UploadMetrics(ctx, config, false, time.Now(), metricsFiles...)
+ UploadMetrics(ctx, config, false, time.Now(), "out/bazel_metrics.txt", "out/bazel_metrics.pb", metricsFiles...)
})
}
}
@@ -194,8 +221,79 @@
metricsUploader: "echo",
}}
- UploadMetrics(ctx, config, true, time.Now(), metricsFile)
+ UploadMetrics(ctx, config, true, time.Now(), "", "", metricsFile)
t.Errorf("got nil, expecting %q as a failure", tt.expectedErr)
})
}
}
+
+func TestParsePercentageToTenThousandths(t *testing.T) {
+ // 2.59% should be returned as 259 - representing 259/10000 of the build
+ percentage := parsePercentageToTenThousandths("2.59%")
+ if percentage != 259 {
+ t.Errorf("Expected percentage to be returned as ten-thousandths. Expected 259, have %d\n", percentage)
+ }
+
+ // Test without a leading digit
+ percentage = parsePercentageToTenThousandths(".52%")
+ if percentage != 52 {
+ t.Errorf("Expected percentage to be returned as ten-thousandths. Expected 52, have %d\n", percentage)
+ }
+}
+
+func TestParseTimingToNanos(t *testing.T) {
+ // This parses from seconds (with millis precision) and returns nanos
+ timingNanos := parseTimingToNanos("0.111")
+ if timingNanos != 111000000 {
+ t.Errorf("Error parsing timing. Expected 111000, have %d\n", timingNanos)
+ }
+
+ // Test without a leading digit
+ timingNanos = parseTimingToNanos(".112")
+ if timingNanos != 112000000 {
+ t.Errorf("Error parsing timing. Expected 112000, have %d\n", timingNanos)
+ }
+}
+
+func TestParsePhaseTiming(t *testing.T) {
+ // Sample lines include:
+ // Total launch phase time 0.011 s 2.59%
+ // Total target pattern evaluation phase time 0.012 s 4.59%
+
+ line1 := "Total launch phase time 0.011 s 2.59%"
+ timing := parsePhaseTiming(line1)
+
+ if timing.GetPhaseName() != "launch" {
+ t.Errorf("Failed to parse phase name. Expected launch, have %s\n", timing.GetPhaseName())
+ } else if timing.GetDurationNanos() != 11000000 {
+ t.Errorf("Failed to parse duration nanos. Expected 11000000, have %d\n", timing.GetDurationNanos())
+ } else if timing.GetPortionOfBuildTime() != 259 {
+ t.Errorf("Failed to parse portion of build time. Expected 259, have %d\n", timing.GetPortionOfBuildTime())
+ }
+
+ // Test with a multiword phase name
+ line2 := "Total target pattern evaluation phase time 0.012 s 4.59%"
+
+ timing = parsePhaseTiming(line2)
+ if timing.GetPhaseName() != "target pattern evaluation" {
+ t.Errorf("Failed to parse phase name. Expected target pattern evaluation, have %s\n", timing.GetPhaseName())
+ } else if timing.GetDurationNanos() != 12000000 {
+ t.Errorf("Failed to parse duration nanos. Expected 12000000, have %d\n", timing.GetDurationNanos())
+ } else if timing.GetPortionOfBuildTime() != 459 {
+ t.Errorf("Failed to parse portion of build time. Expected 459, have %d\n", timing.GetPortionOfBuildTime())
+ }
+}
+
+func TestParseTotal(t *testing.T) {
+ // Total line is in the form of:
+ // Total run time 7.563 s 100.00%
+
+ line := "Total run time 7.563 s 100.00%"
+
+ total := parseTotal(line)
+
+ // Only the seconds field is parsed, as nanos
+ if total != 7563000000 {
+ t.Errorf("Failed to parse total build time. Expected 7563000000, have %d\n", total)
+ }
+}
diff --git a/ui/metrics/Android.bp b/ui/metrics/Android.bp
index 2301c56..bd1517c 100644
--- a/ui/metrics/Android.bp
+++ b/ui/metrics/Android.bp
@@ -22,6 +22,7 @@
deps: [
"golang-protobuf-proto",
"soong-ui-bp2build_metrics_proto",
+ "soong-ui-bazel_metrics_proto",
"soong-ui-metrics_upload_proto",
"soong-ui-metrics_proto",
"soong-ui-mk_metrics_proto",
@@ -73,6 +74,18 @@
}
bootstrap_go_package {
+ name: "soong-ui-bazel_metrics_proto",
+ pkgPath: "android/soong/ui/metrics/bazel_metrics_proto",
+ deps: [
+ "golang-protobuf-reflect-protoreflect",
+ "golang-protobuf-runtime-protoimpl",
+ ],
+ srcs: [
+ "bazel_metrics_proto/bazel_metrics.pb.go",
+ ],
+}
+
+bootstrap_go_package {
name: "soong-ui-mk_metrics_proto",
pkgPath: "android/soong/ui/metrics/mk_metrics_proto",
deps: [
diff --git a/zip/cmd/main.go b/zip/cmd/main.go
index cbc73ed..def76aa 100644
--- a/zip/cmd/main.go
+++ b/zip/cmd/main.go
@@ -163,6 +163,7 @@
parallelJobs := flags.Int("parallel", runtime.NumCPU(), "number of parallel threads to use")
cpuProfile := flags.String("cpuprofile", "", "write cpu profile to file")
traceFile := flags.String("trace", "", "write trace to file")
+ sha256Checksum := flags.Bool("sha256", false, "add a zip header to each file containing its SHA256 digest")
flags.Var(&rootPrefix{}, "P", "path prefix within the zip at which to place files")
flags.Var(&listFiles{}, "l", "file containing list of files to zip")
@@ -224,6 +225,7 @@
WriteIfChanged: *writeIfChanged,
StoreSymlinks: *symlinks,
IgnoreMissingFiles: *ignoreMissingFiles,
+ Sha256Checksum: *sha256Checksum,
})
if err != nil {
fmt.Fprintln(os.Stderr, "error:", err.Error())
diff --git a/zip/zip.go b/zip/zip.go
index 955fe68..6f1a8ad 100644
--- a/zip/zip.go
+++ b/zip/zip.go
@@ -17,8 +17,11 @@
import (
"bytes"
"compress/flate"
+ "crypto/sha256"
+ "encoding/binary"
"errors"
"fmt"
+ "hash"
"hash/crc32"
"io"
"io/ioutil"
@@ -38,6 +41,14 @@
"android/soong/third_party/zip"
)
+// Sha256HeaderID is a custom Header ID for the `extra` field in
+// the file header to store the SHA checksum.
+const Sha256HeaderID = 0x4967
+
+// Sha256HeaderSignature is the signature to verify that the extra
+// data block is used to store the SHA checksum.
+const Sha256HeaderSignature = 0x9514
+
// Block size used during parallel compression of a single file.
const parallelBlockSize = 1 * 1024 * 1024 // 1MB
@@ -231,6 +242,8 @@
stderr io.Writer
fs pathtools.FileSystem
+
+ sha256Checksum bool
}
type zipEntry struct {
@@ -257,6 +270,7 @@
WriteIfChanged bool
StoreSymlinks bool
IgnoreMissingFiles bool
+ Sha256Checksum bool
Stderr io.Writer
Filesystem pathtools.FileSystem
@@ -280,6 +294,7 @@
ignoreMissingFiles: args.IgnoreMissingFiles,
stderr: args.Stderr,
fs: args.Filesystem,
+ sha256Checksum: args.Sha256Checksum,
}
if z.fs == nil {
@@ -782,15 +797,17 @@
// this based on actual buffer sizes in RateLimit.
ze.futureReaders = make(chan chan io.Reader, (fileSize/parallelBlockSize)+1)
- // Calculate the CRC in the background, since reading the entire
- // file could take a while.
+ // Calculate the CRC and SHA256 in the background, since reading
+ // the entire file could take a while.
//
// We could split this up into chunks as well, but it's faster
// than the compression. Due to the Go Zip API, we also need to
// know the result before we can begin writing the compressed
// data out to the zipfile.
+ //
+ // We calculate SHA256 only if `-sha256` is set.
wg.Add(1)
- go z.crcFile(r, ze, compressChan, wg)
+ go z.checksumFileAsync(r, ze, compressChan, wg)
for start := int64(0); start < fileSize; start += parallelBlockSize {
sr := io.NewSectionReader(r, start, parallelBlockSize)
@@ -829,20 +846,53 @@
return nil
}
-func (z *ZipWriter) crcFile(r io.Reader, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) {
+func (z *ZipWriter) checksumFileAsync(r io.ReadSeeker, ze *zipEntry, resultChan chan *zipEntry, wg *sync.WaitGroup) {
defer wg.Done()
defer z.cpuRateLimiter.Finish()
+ z.checksumFile(r, ze)
+
+ resultChan <- ze
+ close(resultChan)
+}
+
+func (z *ZipWriter) checksumFile(r io.ReadSeeker, ze *zipEntry) {
crc := crc32.NewIEEE()
- _, err := io.Copy(crc, r)
+ writers := []io.Writer{crc}
+
+ var shaHasher hash.Hash
+ if z.sha256Checksum && !ze.fh.Mode().IsDir() {
+ shaHasher = sha256.New()
+ writers = append(writers, shaHasher)
+ }
+
+ w := io.MultiWriter(writers...)
+
+ _, err := io.Copy(w, r)
if err != nil {
z.errors <- err
return
}
ze.fh.CRC32 = crc.Sum32()
- resultChan <- ze
- close(resultChan)
+ if shaHasher != nil {
+ z.appendSHAToExtra(ze, shaHasher.Sum(nil))
+ }
+}
+
+func (z *ZipWriter) appendSHAToExtra(ze *zipEntry, checksum []byte) {
+ // The block of SHA256 checksum consist of:
+ // - Header ID, equals to Sha256HeaderID (2 bytes)
+ // - Data size (2 bytes)
+ // - Data block:
+ // - Signature, equals to Sha256HeaderSignature (2 bytes)
+ // - Data, SHA checksum value
+ var buf []byte
+ buf = binary.LittleEndian.AppendUint16(buf, Sha256HeaderID)
+ buf = binary.LittleEndian.AppendUint16(buf, uint16(len(checksum)+2))
+ buf = binary.LittleEndian.AppendUint16(buf, Sha256HeaderSignature)
+ buf = append(buf, checksum...)
+ ze.fh.Extra = append(ze.fh.Extra, buf...)
}
func (z *ZipWriter) compressPartialFile(r io.Reader, dict []byte, last bool, resultChan chan io.Reader, wg *sync.WaitGroup) {
@@ -894,17 +944,9 @@
}
func (z *ZipWriter) compressWholeFile(ze *zipEntry, r io.ReadSeeker, compressChan chan *zipEntry) {
+ z.checksumFile(r, ze)
- crc := crc32.NewIEEE()
- _, err := io.Copy(crc, r)
- if err != nil {
- z.errors <- err
- return
- }
-
- ze.fh.CRC32 = crc.Sum32()
-
- _, err = r.Seek(0, 0)
+ _, err := r.Seek(0, 0)
if err != nil {
z.errors <- err
return
diff --git a/zip/zip_test.go b/zip/zip_test.go
index c4832dc..e7fdea8 100644
--- a/zip/zip_test.go
+++ b/zip/zip_test.go
@@ -16,6 +16,7 @@
import (
"bytes"
+ "encoding/hex"
"hash/crc32"
"io"
"os"
@@ -35,6 +36,10 @@
fileEmpty = []byte("")
fileManifest = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\n\n")
+ sha256FileA = "d53eda7a637c99cc7fb566d96e9fa109bf15c478410a3f5eb4d4c4e26cd081f6"
+ sha256FileB = "430c56c5818e62bcb6d478901ef86284e97714c138f3c86aa14fd6a84b7ce5d3"
+ sha256FileC = "31c5ab6111f1d6aa13c2c4e92bb3c0f7c76b61b42d141af1e846eb7f6586a51c"
+
fileCustomManifest = []byte("Custom manifest: true\n")
customManifestAfter = []byte("Manifest-Version: 1.0\nCreated-By: soong_zip\nCustom manifest: true\n\n")
)
@@ -67,6 +72,20 @@
}
}
+func fhWithSHA256(name string, contents []byte, method uint16, sha256 string) zip.FileHeader {
+ h := fh(name, contents, method)
+ // The extra field contains 38 bytes, including 2 bytes of header ID, 2 bytes
+ // of size, 2 bytes of signature, and 32 bytes of checksum data block.
+ var extra [38]byte
+ // The first 6 bytes contains Sha256HeaderID (0x4967), size (unit(34)) and
+ // Sha256HeaderSignature (0x9514)
+ copy(extra[0:], []byte{103, 73, 34, 0, 20, 149})
+ sha256Bytes, _ := hex.DecodeString(sha256)
+ copy(extra[6:], sha256Bytes)
+ h.Extra = append(h.Extra, extra[:]...)
+ return h
+}
+
func fhManifest(contents []byte) zip.FileHeader {
return zip.FileHeader{
Name: "META-INF/MANIFEST.MF",
@@ -87,13 +106,18 @@
}
}
-func fhDir(name string) zip.FileHeader {
+type fhDirOptions struct {
+ extra []byte
+}
+
+func fhDir(name string, opts fhDirOptions) zip.FileHeader {
return zip.FileHeader{
Name: name,
Method: zip.Store,
CRC32: crc32.ChecksumIEEE(nil),
UncompressedSize64: 0,
ExternalAttrs: (syscall.S_IFDIR|0755)<<16 | 0x10,
+ Extra: opts.extra,
}
}
@@ -114,6 +138,7 @@
manifest string
storeSymlinks bool
ignoreMissingFiles bool
+ sha256Checksum bool
files []zip.FileHeader
err error
@@ -320,10 +345,10 @@
emulateJar: true,
files: []zip.FileHeader{
- fhDir("META-INF/"),
+ fhDir("META-INF/", fhDirOptions{extra: []byte{254, 202, 0, 0}}),
fhManifest(fileManifest),
- fhDir("a/"),
- fhDir("a/a/"),
+ fhDir("a/", fhDirOptions{}),
+ fhDir("a/a/", fhDirOptions{}),
fh("a/a/a", fileA, zip.Deflate),
fh("a/a/b", fileB, zip.Deflate),
},
@@ -338,10 +363,10 @@
manifest: "manifest.txt",
files: []zip.FileHeader{
- fhDir("META-INF/"),
+ fhDir("META-INF/", fhDirOptions{extra: []byte{254, 202, 0, 0}}),
fhManifest(customManifestAfter),
- fhDir("a/"),
- fhDir("a/a/"),
+ fhDir("a/", fhDirOptions{}),
+ fhDir("a/a/", fhDirOptions{}),
fh("a/a/a", fileA, zip.Deflate),
fh("a/a/b", fileB, zip.Deflate),
},
@@ -355,8 +380,8 @@
dirEntries: true,
files: []zip.FileHeader{
- fhDir("a/"),
- fhDir("a/a/"),
+ fhDir("a/", fhDirOptions{}),
+ fhDir("a/a/", fhDirOptions{}),
fh("a/a/a", fileA, zip.Deflate),
fh("a/a/b", fileB, zip.Deflate),
},
@@ -412,6 +437,23 @@
fh("a/a/a", fileA, zip.Deflate),
},
},
+ {
+ name: "generate SHA256 checksum",
+ args: fileArgsBuilder().
+ File("a/a/a").
+ File("a/a/b").
+ File("a/a/c").
+ File("c"),
+ compressionLevel: 9,
+ sha256Checksum: true,
+
+ files: []zip.FileHeader{
+ fhWithSHA256("a/a/a", fileA, zip.Deflate, sha256FileA),
+ fhWithSHA256("a/a/b", fileB, zip.Deflate, sha256FileB),
+ fhWithSHA256("a/a/c", fileC, zip.Deflate, sha256FileC),
+ fhWithSHA256("c", fileC, zip.Deflate, sha256FileC),
+ },
+ },
// errors
{
@@ -465,6 +507,7 @@
args.ManifestSourcePath = test.manifest
args.StoreSymlinks = test.storeSymlinks
args.IgnoreMissingFiles = test.ignoreMissingFiles
+ args.Sha256Checksum = test.sha256Checksum
args.Filesystem = mockFs
args.Stderr = &bytes.Buffer{}
@@ -555,6 +598,11 @@
t.Errorf("incorrect file %s method want %v got %v", want.Name,
want.Method, got.Method)
}
+
+ if !bytes.Equal(want.Extra, got.Extra) {
+ t.Errorf("incorrect file %s extra want %v got %v", want.Name,
+ want.Extra, got.Extra)
+ }
}
})
}