Refactor mixed build allowlist handling
This refactoring prepares for introduction of bazel prod mode, an
alternative mechanism for mixed builds allowlist handling.
* Decide bazel-mode as close to soong_build main as possible
* BazelContext itself decides whether a module is allowlisted
* Separate bp2build and mixed build allowlist
Test: m nothing, manually verified all modules are mixed build disabled
(via metrics)
Test: USE_BAZEL_ANALYSIS=1 m nothing, manually verified that mixed build
disabled/enabled modules are identical before and after change.
Change-Id: I0f55d8b85000cb4a871a099edc6d7d868d7df509
diff --git a/android/bazel.go b/android/bazel.go
index aff6116..71eb036 100644
--- a/android/bazel.go
+++ b/android/bazel.go
@@ -231,16 +231,18 @@
// when they have the same type as one listed.
moduleTypeAlwaysConvert map[string]bool
- // Per-module denylist to always opt modules out of both bp2build and mixed builds.
+ // Per-module denylist to always opt modules out of bp2build conversion.
moduleDoNotConvert map[string]bool
// Per-module denylist of cc_library modules to only generate the static
// variant if their shared variant isn't ready or buildable by Bazel.
ccLibraryStaticOnly map[string]bool
+}
- // Per-module denylist to opt modules out of mixed builds. Such modules will
- // still be generated via bp2build.
- mixedBuildsDisabled map[string]bool
+// GenerateCcLibraryStaticOnly returns whether a cc_library module should only
+// generate a static version of itself based on the current global configuration.
+func (a bp2BuildConversionAllowlist) GenerateCcLibraryStaticOnly(moduleName string) bool {
+ return a.ccLibraryStaticOnly[moduleName]
}
// NewBp2BuildAllowlist creates a new, empty bp2BuildConversionAllowlist
@@ -253,7 +255,6 @@
map[string]bool{},
map[string]bool{},
map[string]bool{},
- map[string]bool{},
}
}
@@ -329,43 +330,24 @@
return a
}
-// SetMixedBuildsDisabledList copies the entries from mixedBuildsDisabled into the allowlist
-func (a bp2BuildConversionAllowlist) SetMixedBuildsDisabledList(mixedBuildsDisabled []string) bp2BuildConversionAllowlist {
- if a.mixedBuildsDisabled == nil {
- a.mixedBuildsDisabled = map[string]bool{}
- }
- for _, m := range mixedBuildsDisabled {
- a.mixedBuildsDisabled[m] = true
- }
-
- return a
-}
-
var bp2BuildAllowListKey = NewOnceKey("Bp2BuildAllowlist")
var bp2buildAllowlist OncePer
-func getBp2BuildAllowList() bp2BuildConversionAllowlist {
+func GetBp2BuildAllowList() bp2BuildConversionAllowlist {
return bp2buildAllowlist.Once(bp2BuildAllowListKey, func() interface{} {
return NewBp2BuildAllowlist().SetDefaultConfig(allowlists.Bp2buildDefaultConfig).
SetKeepExistingBuildFile(allowlists.Bp2buildKeepExistingBuildFile).
SetModuleAlwaysConvertList(allowlists.Bp2buildModuleAlwaysConvertList).
SetModuleTypeAlwaysConvertList(allowlists.Bp2buildModuleTypeAlwaysConvertList).
SetModuleDoNotConvertList(allowlists.Bp2buildModuleDoNotConvertList).
- SetCcLibraryStaticOnlyList(allowlists.Bp2buildCcLibraryStaticOnlyList).
- SetMixedBuildsDisabledList(allowlists.MixedBuildsDisabledList)
+ SetCcLibraryStaticOnlyList(allowlists.Bp2buildCcLibraryStaticOnlyList)
}).(bp2BuildConversionAllowlist)
}
-// GenerateCcLibraryStaticOnly returns whether a cc_library module should only
-// generate a static version of itself based on the current global configuration.
-func GenerateCcLibraryStaticOnly(moduleName string) bool {
- return getBp2BuildAllowList().ccLibraryStaticOnly[moduleName]
-}
-
// ShouldKeepExistingBuildFileForDir returns whether an existing BUILD file should be
// added to the build symlink forest based on the current global configuration.
func ShouldKeepExistingBuildFileForDir(dir string) bool {
- return shouldKeepExistingBuildFileForDir(getBp2BuildAllowList(), dir)
+ return shouldKeepExistingBuildFileForDir(GetBp2BuildAllowList(), dir)
}
func shouldKeepExistingBuildFileForDir(allowlist bp2BuildConversionAllowlist, dir string) bool {
@@ -405,20 +387,10 @@
if !ctx.Module().Enabled() {
return false
}
- if !ctx.Config().BazelContext.BazelEnabled() {
- return false
- }
if !convertedToBazel(ctx, ctx.Module()) {
return false
}
-
- if GenerateCcLibraryStaticOnly(ctx.Module().Name()) {
- // Don't use partially-converted cc_library targets in mixed builds,
- // since mixed builds would generally rely on both static and shared
- // variants of a cc_library.
- return false
- }
- return !getBp2BuildAllowList().mixedBuildsDisabled[ctx.Module().Name()]
+ return ctx.Config().BazelContext.BazelAllowlisted(ctx.Module().Name())
}
// ConvertedToBazel returns whether this module has been converted (with bp2build or manually) to Bazel.
diff --git a/android/bazel_handler.go b/android/bazel_handler.go
index a5fa043..d87f988 100644
--- a/android/bazel_handler.go
+++ b/android/bazel_handler.go
@@ -27,6 +27,7 @@
"strings"
"sync"
+ "android/soong/android/allowlists"
"android/soong/bazel/cquery"
"android/soong/shared"
@@ -142,8 +143,11 @@
// queued in the BazelContext.
InvokeBazel(config Config) error
- // Returns true if bazel is enabled for the given configuration.
- BazelEnabled() bool
+ // Returns true if Bazel handling is enabled for the module with the given name.
+ // Note that this only implies "bazel mixed build" allowlisting. The caller
+ // should independently verify the module is eligible for Bazel handling
+ // (for example, that it is MixedBuildBuildable).
+ BazelAllowlisted(moduleName string) bool
// Returns the bazel output base (the root directory for all bazel intermediate outputs).
OutputBase() string
@@ -183,6 +187,17 @@
// Depsets which should be used for Bazel's build statements.
depsets []bazel.AqueryDepset
+
+ // Per-module allowlist/denylist functionality to control whether analysis of
+ // modules are handled by Bazel. For modules which do not have a Bazel definition
+ // (or do not sufficiently support bazel handling via MixedBuildBuildable),
+ // this allowlist will have no effect, even if the module is explicitly allowlisted here.
+ // Per-module denylist to opt modules out of bazel handling.
+ bazelDisabledModules map[string]bool
+ // Per-module allowlist to opt modules in to bazel handling.
+ bazelEnabledModules map[string]bool
+ // If true, modules are bazel-enabled by default, unless present in bazelDisabledModules.
+ modulesDefaultToBazel bool
}
var _ BazelContext = &bazelContext{}
@@ -229,7 +244,7 @@
panic("unimplemented")
}
-func (m MockBazelContext) BazelEnabled() bool {
+func (m MockBazelContext) BazelAllowlisted(moduleName string) bool {
return true
}
@@ -315,7 +330,7 @@
return ""
}
-func (n noopBazelContext) BazelEnabled() bool {
+func (n noopBazelContext) BazelAllowlisted(moduleName string) bool {
return false
}
@@ -328,9 +343,7 @@
}
func NewBazelContext(c *config) (BazelContext, error) {
- // TODO(cparsons): Assess USE_BAZEL=1 instead once "mixed Soong/Bazel builds"
- // are production ready.
- if !c.IsEnvTrue("USE_BAZEL_ANALYSIS") {
+ if !c.IsMixedBuildsEnabled() {
return noopBazelContext{}, nil
}
@@ -338,10 +351,26 @@
if err != nil {
return nil, err
}
+
+ // TODO(cparsons): Use a different allowlist depending on prod vs. dev
+ // bazel mode.
+ disabledModules := map[string]bool{}
+ // Don't use partially-converted cc_library targets in mixed builds,
+ // since mixed builds would generally rely on both static and shared
+ // variants of a cc_library.
+ for staticOnlyModule, _ := range GetBp2BuildAllowList().ccLibraryStaticOnly {
+ disabledModules[staticOnlyModule] = true
+ }
+ for _, disabledDevModule := range allowlists.MixedBuildsDisabledList {
+ disabledModules[disabledDevModule] = true
+ }
+
return &bazelContext{
- bazelRunner: &builtinBazelRunner{},
- paths: p,
- requests: make(map[cqueryKey]bool),
+ bazelRunner: &builtinBazelRunner{},
+ paths: p,
+ requests: make(map[cqueryKey]bool),
+ modulesDefaultToBazel: true,
+ bazelDisabledModules: disabledModules,
}, nil
}
@@ -386,8 +415,14 @@
return p.metricsDir
}
-func (context *bazelContext) BazelEnabled() bool {
- return true
+func (context *bazelContext) BazelAllowlisted(moduleName string) bool {
+ if context.bazelDisabledModules[moduleName] {
+ return false
+ }
+ if context.bazelEnabledModules[moduleName] {
+ return true
+ }
+ return context.modulesDefaultToBazel
}
func pwdPrefix() string {
@@ -851,7 +886,7 @@
func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
// bazelSingleton is a no-op if mixed-soong-bazel-builds are disabled.
- if !ctx.Config().BazelContext.BazelEnabled() {
+ if !ctx.Config().IsMixedBuildsEnabled() {
return
}
diff --git a/android/bazel_test.go b/android/bazel_test.go
index da4a915..98f0a46 100644
--- a/android/bazel_test.go
+++ b/android/bazel_test.go
@@ -389,7 +389,7 @@
}
func TestBp2buildAllowList(t *testing.T) {
- allowlist := getBp2BuildAllowList()
+ allowlist := GetBp2BuildAllowList()
for k, v := range allowlists.Bp2buildDefaultConfig {
if allowlist.defaultConfig[k] != v {
t.Errorf("bp2build default config of %s: expected: %v, got: %v", k, v, allowlist.defaultConfig[k])
@@ -415,9 +415,4 @@
t.Errorf("bp2build cc library static only of %s: expected: true, got: %v", k, allowlist.ccLibraryStaticOnly[k])
}
}
- for _, k := range allowlists.MixedBuildsDisabledList {
- if !allowlist.mixedBuildsDisabled[k] {
- t.Errorf("bp2build mix build disabled of %s: expected: true, got: %v", k, allowlist.mixedBuildsDisabled[k])
- }
- }
}
diff --git a/android/config.go b/android/config.go
index d3f8ab4..a9d465e 100644
--- a/android/config.go
+++ b/android/config.go
@@ -68,6 +68,38 @@
*config
}
+type SoongBuildMode int
+
+// Build modes that soong_build can run as.
+const (
+ // Don't use bazel at all during module analysis.
+ AnalysisNoBazel SoongBuildMode = iota
+
+ // Bp2build mode: Generate BUILD files from blueprint files and exit.
+ Bp2build
+
+ // Generate BUILD files which faithfully represent the dependency graph of
+ // blueprint modules. Individual BUILD targets will not, however, faitfhully
+ // express build semantics.
+ GenerateQueryView
+
+ // Create a JSON representation of the module graph and exit.
+ GenerateModuleGraph
+
+ // Generate a documentation file for module type definitions and exit.
+ GenerateDocFile
+
+ // Use bazel during analysis of many allowlisted build modules. The allowlist
+ // is considered a "developer mode" allowlist, as some modules may be
+ // allowlisted on an experimental basis.
+ BazelDevMode
+
+ // Use bazel during analysis of build modules from an allowlist carefully
+ // curated by the build team to be proven stable.
+ // TODO(cparsons): Implement this mode.
+ BazelProdMode
+)
+
// SoongOutDir returns the build output directory for the configuration.
func (c Config) SoongOutDir() string {
return c.soongOutDir
@@ -157,7 +189,7 @@
fs pathtools.FileSystem
mockBpList string
- runningAsBp2Build bool
+ BuildMode SoongBuildMode
bp2buildPackageConfig bp2BuildConversionAllowlist
Bp2buildSoongConfigDefinitions soongconfig.Bp2BuildSoongConfigDefinitions
@@ -171,6 +203,12 @@
OncePer
+ // These fields are only used for metrics collection. A module should be added
+ // to these maps only if its implementation supports Bazel handling in mixed
+ // builds. A module being in the "enabled" list indicates that there is a
+ // variant of that module for which bazel-handling actually took place.
+ // A module being in the "disabled" list indicates that there is a variant of
+ // that module for which bazel-handling was denied.
mixedBuildsLock sync.Mutex
mixedBuildEnabledModules map[string]struct{}
mixedBuildDisabledModules map[string]struct{}
@@ -346,7 +384,7 @@
// NewConfig creates a new Config object. The srcDir argument specifies the path
// to the root source directory. It also loads the config file, if found.
-func NewConfig(moduleListFile string, runGoTests bool, outDir, soongOutDir string, availableEnv map[string]string) (Config, error) {
+func NewConfig(moduleListFile string, buildMode SoongBuildMode, runGoTests bool, outDir, soongOutDir string, availableEnv map[string]string) (Config, error) {
// Make a config with default options.
config := &config{
ProductVariablesFileName: filepath.Join(soongOutDir, productVariablesFileName),
@@ -443,8 +481,17 @@
config.AndroidFirstDeviceTarget = FirstTarget(config.Targets[Android], "lib64", "lib32")[0]
}
+ // Checking USE_BAZEL_ANALYSIS must be done here instead of in the caller, so
+ // that we can invoke IsEnvTrue (which also registers the env var as a
+ // dependency of the build).
+ // TODO(cparsons): Remove this hack once USE_BAZEL_ANALYSIS is removed.
+ if buildMode == AnalysisNoBazel && config.IsEnvTrue("USE_BAZEL_ANALYSIS") {
+ buildMode = BazelDevMode
+ }
+
+ config.BuildMode = buildMode
config.BazelContext, err = NewBazelContext(config)
- config.bp2buildPackageConfig = getBp2BuildAllowList()
+ config.bp2buildPackageConfig = GetBp2BuildAllowList()
return Config{config}, err
}
@@ -479,6 +526,12 @@
c.mockBpList = blueprint.MockModuleListFile
}
+// Returns true if "Bazel builds" is enabled. In this mode, part of build
+// analysis is handled by Bazel.
+func (c *config) IsMixedBuildsEnabled() bool {
+ return c.BuildMode == BazelProdMode || c.BuildMode == BazelDevMode
+}
+
func (c *config) SetAllowMissingDependencies() {
c.productVariables.Allow_missing_dependencies = proptools.BoolPtr(true)
}
diff --git a/android/defaults.go b/android/defaults.go
index 03b2efb..7906e94 100644
--- a/android/defaults.go
+++ b/android/defaults.go
@@ -448,7 +448,7 @@
}
for _, defaults := range defaultsList {
- if ctx.Config().runningAsBp2Build {
+ if ctx.Config().BuildMode == Bp2build {
applyNamespacedVariableDefaults(defaults, ctx)
}
diff --git a/android/module.go b/android/module.go
index bf62080..eb099f9 100644
--- a/android/module.go
+++ b/android/module.go
@@ -2367,9 +2367,6 @@
}
func (m *ModuleBase) isHandledByBazel(ctx ModuleContext) (MixedBuildBuildable, bool) {
- if !ctx.Config().BazelContext.BazelEnabled() {
- return nil, false
- }
if mixedBuildMod, ok := m.module.(MixedBuildBuildable); ok {
if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) {
return mixedBuildMod, true
diff --git a/android/register.go b/android/register.go
index 5832b1b..d4ce5f1 100644
--- a/android/register.go
+++ b/android/register.go
@@ -164,10 +164,6 @@
return ctx
}
-func (ctx *Context) SetRunningAsBp2build() {
- ctx.config.runningAsBp2Build = true
-}
-
// RegisterForBazelConversion registers an alternate shadow pipeline of
// singletons, module types and mutators to register for converting Blueprint
// files to semantically equivalent BUILD files.
diff --git a/android/soong_config_modules.go b/android/soong_config_modules.go
index b25f248..cd36ae0 100644
--- a/android/soong_config_modules.go
+++ b/android/soong_config_modules.go
@@ -382,7 +382,7 @@
defer r.Close()
mtDef, errs := soongconfig.Parse(r, from)
- if ctx.Config().runningAsBp2Build {
+ if ctx.Config().BuildMode == Bp2build {
ctx.Config().Bp2buildSoongConfigDefinitions.AddVars(*mtDef)
}
@@ -398,7 +398,7 @@
for name, moduleType := range mtDef.ModuleTypes {
factory := globalModuleTypes[moduleType.BaseModuleType]
if factory != nil {
- factories[name] = configModuleFactory(factory, moduleType, ctx.Config().runningAsBp2Build)
+ factories[name] = configModuleFactory(factory, moduleType, ctx.Config().BuildMode == Bp2build)
} else {
reportErrors(ctx, from,
fmt.Errorf("missing global module type factory for %q", moduleType.BaseModuleType))
diff --git a/android/testing.go b/android/testing.go
index b4429ca..ef5e5a9 100644
--- a/android/testing.go
+++ b/android/testing.go
@@ -457,7 +457,7 @@
// RegisterForBazelConversion prepares a test context for bp2build conversion.
func (ctx *TestContext) RegisterForBazelConversion() {
- ctx.SetRunningAsBp2build()
+ ctx.config.BuildMode = Bp2build
RegisterMutatorsForBazelConversion(ctx.Context, ctx.bp2buildPreArch)
}