Merge "Add directories below to Android.mk denylist:" into main
diff --git a/android/androidmk.go b/android/androidmk.go
index 6426835..98f8bb1 100644
--- a/android/androidmk.go
+++ b/android/androidmk.go
@@ -913,6 +913,7 @@
case "*android_sdk.sdkRepoHost": // doesn't go through base_rules
case "*apex.apexBundle": // license properties written
case "*bpf.bpf": // license properties written (both for module and objs)
+ case "*libbpf_prog.libbpfProg": // license properties written (both for module and objs)
case "*genrule.Module": // writes non-custom before adding .phony
case "*java.SystemModules": // doesn't go through base_rules
case "*java.systemModulesImport": // doesn't go through base_rules
@@ -982,11 +983,11 @@
return nil
}
-func ShouldSkipAndroidMkProcessing(ctx ConfigAndErrorContext, module Module) bool {
+func ShouldSkipAndroidMkProcessing(ctx ConfigurableEvaluatorContext, module Module) bool {
return shouldSkipAndroidMkProcessing(ctx, module.base())
}
-func shouldSkipAndroidMkProcessing(ctx ConfigAndErrorContext, module *ModuleBase) bool {
+func shouldSkipAndroidMkProcessing(ctx ConfigurableEvaluatorContext, module *ModuleBase) bool {
if !module.commonProperties.NamespaceExportedToMake {
// TODO(jeffrygaston) do we want to validate that there are no modules being
// exported to Kati that depend on this module?
diff --git a/android/arch_list.go b/android/arch_list.go
index 2937092..9501c87 100644
--- a/android/arch_list.go
+++ b/android/arch_list.go
@@ -26,6 +26,7 @@
"armv8-2a",
"armv8-2a-dotprod",
"armv9-a",
+ "armv9-2a",
},
X86: {
"amberlake",
diff --git a/android/config.go b/android/config.go
index bc53a37..b682c2e 100644
--- a/android/config.go
+++ b/android/config.go
@@ -1828,10 +1828,6 @@
return c.config.productVariables.BuildBrokenTrebleSyspropNeverallow
}
-func (c *deviceConfig) BuildBrokenUsesSoongPython2Modules() bool {
- return c.config.productVariables.BuildBrokenUsesSoongPython2Modules
-}
-
func (c *deviceConfig) BuildDebugfsRestrictionsEnabled() bool {
return c.config.productVariables.BuildDebugfsRestrictionsEnabled
}
diff --git a/android/module.go b/android/module.go
index 009b0df..47fdc23 100644
--- a/android/module.go
+++ b/android/module.go
@@ -58,7 +58,7 @@
base() *ModuleBase
Disable()
- Enabled(ctx ConfigAndErrorContext) bool
+ Enabled(ctx ConfigurableEvaluatorContext) bool
Target() Target
MultiTargets() []Target
@@ -109,12 +109,12 @@
// Get information about the properties that can contain visibility rules.
visibilityProperties() []visibilityProperty
- RequiredModuleNames(ctx ConfigAndErrorContext) []string
+ RequiredModuleNames(ctx ConfigurableEvaluatorContext) []string
HostRequiredModuleNames() []string
TargetRequiredModuleNames() []string
- VintfFragmentModuleNames(ctx ConfigAndErrorContext) []string
+ VintfFragmentModuleNames(ctx ConfigurableEvaluatorContext) []string
- ConfigurableEvaluator(ctx ConfigAndErrorContext) proptools.ConfigurableEvaluator
+ ConfigurableEvaluator(ctx ConfigurableEvaluatorContext) proptools.ConfigurableEvaluator
}
// Qualified id for a module
@@ -1339,7 +1339,7 @@
return partition
}
-func (m *ModuleBase) Enabled(ctx ConfigAndErrorContext) bool {
+func (m *ModuleBase) Enabled(ctx ConfigurableEvaluatorContext) bool {
if m.commonProperties.ForcedDisabled {
return false
}
@@ -1536,7 +1536,7 @@
return m.base().commonProperties.ImageVariation == RecoveryVariation
}
-func (m *ModuleBase) RequiredModuleNames(ctx ConfigAndErrorContext) []string {
+func (m *ModuleBase) RequiredModuleNames(ctx ConfigurableEvaluatorContext) []string {
return m.base().commonProperties.Required.GetOrDefault(m.ConfigurableEvaluator(ctx), nil)
}
@@ -1548,7 +1548,7 @@
return m.base().commonProperties.Target_required
}
-func (m *ModuleBase) VintfFragmentModuleNames(ctx ConfigAndErrorContext) []string {
+func (m *ModuleBase) VintfFragmentModuleNames(ctx ConfigurableEvaluatorContext) []string {
return m.base().commonProperties.Vintf_fragment_modules.GetOrDefault(m.ConfigurableEvaluator(ctx), nil)
}
@@ -2204,17 +2204,17 @@
return proptools.Bool(m.commonProperties.Native_bridge_supported)
}
-type ConfigAndErrorContext interface {
+type ConfigurableEvaluatorContext interface {
Config() Config
OtherModulePropertyErrorf(module Module, property string, fmt string, args ...interface{})
}
type configurationEvalutor struct {
- ctx ConfigAndErrorContext
+ ctx ConfigurableEvaluatorContext
m Module
}
-func (m *ModuleBase) ConfigurableEvaluator(ctx ConfigAndErrorContext) proptools.ConfigurableEvaluator {
+func (m *ModuleBase) ConfigurableEvaluator(ctx ConfigurableEvaluatorContext) proptools.ConfigurableEvaluator {
return configurationEvalutor{
ctx: ctx,
m: m.module,
diff --git a/android/module_context.go b/android/module_context.go
index 3889e40..54fe0be 100644
--- a/android/module_context.go
+++ b/android/module_context.go
@@ -194,7 +194,7 @@
InstallInVendor() bool
InstallForceOS() (*OsType, *ArchType)
- RequiredModuleNames(ctx ConfigAndErrorContext) []string
+ RequiredModuleNames(ctx ConfigurableEvaluatorContext) []string
HostRequiredModuleNames() []string
TargetRequiredModuleNames() []string
@@ -855,7 +855,7 @@
return OptionalPath{}
}
-func (m *moduleContext) RequiredModuleNames(ctx ConfigAndErrorContext) []string {
+func (m *moduleContext) RequiredModuleNames(ctx ConfigurableEvaluatorContext) []string {
return m.module.RequiredModuleNames(ctx)
}
diff --git a/android/sdk.go b/android/sdk.go
index d3f04a4..fb1ac24 100644
--- a/android/sdk.go
+++ b/android/sdk.go
@@ -813,7 +813,7 @@
// SdkMemberContext provides access to information common to a specific member.
type SdkMemberContext interface {
- ConfigAndErrorContext
+ ConfigurableEvaluatorContext
// SdkModuleContext returns the module context of the sdk common os variant which is creating the
// snapshot.
diff --git a/android/singleton_module.go b/android/singleton_module.go
index 2351738..43028e8 100644
--- a/android/singleton_module.go
+++ b/android/singleton_module.go
@@ -68,7 +68,7 @@
func (smb *SingletonModuleBase) GenerateBuildActions(ctx blueprint.ModuleContext) {
smb.lock.Lock()
if smb.variant != "" {
- ctx.ModuleErrorf("GenerateAndroidBuildActions already called for variant %q, SingletonModules can only have one variant", smb.variant)
+ ctx.ModuleErrorf("GenerateAndroidBuildActions already called for variant %q, SingletonModules can only have one variant", smb.variant)
}
smb.variant = ctx.ModuleSubDir()
smb.lock.Unlock()
diff --git a/android/testing.go b/android/testing.go
index 816707d..42cd1eb 100644
--- a/android/testing.go
+++ b/android/testing.go
@@ -1326,7 +1326,7 @@
return ctx.ctx.Config()
}
-func PanickingConfigAndErrorContext(ctx *TestContext) ConfigAndErrorContext {
+func PanickingConfigAndErrorContext(ctx *TestContext) ConfigurableEvaluatorContext {
return &panickingConfigAndErrorContext{
ctx: ctx,
}
diff --git a/android/variable.go b/android/variable.go
index 9026f93..b238c4a 100644
--- a/android/variable.go
+++ b/android/variable.go
@@ -121,6 +121,7 @@
// are used for dogfooding and performance testing, and should be as similar to user builds
// as possible.
Debuggable struct {
+ Apk *string
Cflags []string
Cppflags []string
Init_rc []string
@@ -445,7 +446,6 @@
GenruleSandboxing *bool `json:",omitempty"`
BuildBrokenEnforceSyspropOwner bool `json:",omitempty"`
BuildBrokenTrebleSyspropNeverallow bool `json:",omitempty"`
- BuildBrokenUsesSoongPython2Modules bool `json:",omitempty"`
BuildBrokenVendorPropertyNamespace bool `json:",omitempty"`
BuildBrokenIncorrectPartitionImages bool `json:",omitempty"`
BuildBrokenInputDirModules []string `json:",omitempty"`
diff --git a/bpf/libbpf/Android.bp b/bpf/libbpf/Android.bp
new file mode 100644
index 0000000..f0ba90f
--- /dev/null
+++ b/bpf/libbpf/Android.bp
@@ -0,0 +1,38 @@
+//
+// Copyright (C) 2024 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package {
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+bootstrap_go_package {
+ name: "soong-libbpf",
+ pkgPath: "android/soong/bpf/libbpf",
+ deps: [
+ "blueprint",
+ "blueprint-proptools",
+ "soong-android",
+ "soong-cc",
+ "soong-cc-config",
+ ],
+ srcs: [
+ "libbpf_prog.go",
+ ],
+ testSrcs: [
+ "libbpf_prog_test.go",
+ ],
+ pluginFor: ["soong_build"],
+}
diff --git a/bpf/libbpf/libbpf_prog.go b/bpf/libbpf/libbpf_prog.go
new file mode 100644
index 0000000..1fdb3d6
--- /dev/null
+++ b/bpf/libbpf/libbpf_prog.go
@@ -0,0 +1,278 @@
+// Copyright (C) 2024 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package libbpf_prog
+
+import (
+ "fmt"
+ "io"
+ "runtime"
+ "strings"
+
+ "android/soong/android"
+ "android/soong/cc"
+ "android/soong/genrule"
+
+ "github.com/google/blueprint"
+)
+
+type libbpfProgDepType struct {
+ blueprint.BaseDependencyTag
+}
+
+func init() {
+ registerLibbpfProgBuildComponents(android.InitRegistrationContext)
+ pctx.Import("android/soong/cc/config")
+ pctx.StaticVariable("relPwd", cc.PwdPrefix())
+}
+
+var (
+ pctx = android.NewPackageContext("android/soong/bpf/libbpf_prog")
+
+ libbpfProgCcRule = pctx.AndroidStaticRule("libbpfProgCcRule",
+ blueprint.RuleParams{
+ Depfile: "${out}.d",
+ Deps: blueprint.DepsGCC,
+ Command: "$relPwd $ccCmd --target=bpf -c $cFlags -MD -MF ${out}.d -o $out $in",
+ CommandDeps: []string{"$ccCmd"},
+ },
+ "ccCmd", "cFlags")
+
+ libbpfProgStripRule = pctx.AndroidStaticRule("libbpfProgStripRule",
+ blueprint.RuleParams{
+ Command: `$stripCmd --strip-unneeded --remove-section=.rel.BTF ` +
+ `--remove-section=.rel.BTF.ext --remove-section=.BTF.ext $in -o $out`,
+ CommandDeps: []string{"$stripCmd"},
+ },
+ "stripCmd")
+
+ libbpfProgDepTag = libbpfProgDepType{}
+)
+
+func registerLibbpfProgBuildComponents(ctx android.RegistrationContext) {
+ ctx.RegisterModuleType("libbpf_prog", LibbpfProgFactory)
+}
+
+var PrepareForTestWithLibbpfProg = android.GroupFixturePreparers(
+ android.FixtureRegisterWithContext(registerLibbpfProgBuildComponents),
+ android.FixtureAddFile("libbpf_headers/Foo.h", nil),
+ android.FixtureAddFile("libbpf_headers/Android.bp", []byte(`
+ genrule {
+ name: "libbpf_headers",
+ out: ["foo.h",],
+ }
+ `)),
+ genrule.PrepareForTestWithGenRuleBuildComponents,
+)
+
+type LibbpfProgProperties struct {
+ // source paths to the files.
+ Srcs []string `android:"path"`
+
+ // additional cflags that should be used to build the libbpf variant of
+ // the C/C++ module.
+ Cflags []string `android:"arch_variant"`
+
+ // list of directories relative to the Blueprint file that will
+ // be added to the include path using -I
+ Local_include_dirs []string `android:"arch_variant"`
+
+ // optional subdirectory under which this module is installed into.
+ Relative_install_path string
+}
+
+type libbpfProg struct {
+ android.ModuleBase
+ properties LibbpfProgProperties
+ objs android.Paths
+}
+
+var _ android.ImageInterface = (*libbpfProg)(nil)
+
+func (libbpf *libbpfProg) ImageMutatorBegin(ctx android.BaseModuleContext) {}
+
+func (libbpf *libbpfProg) VendorVariantNeeded(ctx android.BaseModuleContext) bool {
+ return false
+}
+
+func (libbpf *libbpfProg) ProductVariantNeeded(ctx android.BaseModuleContext) bool {
+ return false
+}
+
+func (libbpf *libbpfProg) CoreVariantNeeded(ctx android.BaseModuleContext) bool {
+ return true
+}
+
+func (libbpf *libbpfProg) RamdiskVariantNeeded(ctx android.BaseModuleContext) bool {
+ return false
+}
+
+func (libbpf *libbpfProg) VendorRamdiskVariantNeeded(ctx android.BaseModuleContext) bool {
+ return false
+}
+
+func (libbpf *libbpfProg) DebugRamdiskVariantNeeded(ctx android.BaseModuleContext) bool {
+ return false
+}
+
+func (libbpf *libbpfProg) RecoveryVariantNeeded(ctx android.BaseModuleContext) bool {
+ return false
+}
+
+func (libbpf *libbpfProg) ExtraImageVariations(ctx android.BaseModuleContext) []string {
+ return nil
+}
+
+func (libbpf *libbpfProg) SetImageVariation(ctx android.BaseModuleContext, variation string) {
+}
+
+func (libbpf *libbpfProg) DepsMutator(ctx android.BottomUpMutatorContext) {
+ ctx.AddDependency(ctx.Module(), libbpfProgDepTag, "libbpf_headers")
+}
+
+func (libbpf *libbpfProg) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+ var cFlagsDeps android.Paths
+ cflags := []string{
+ "-nostdlibinc",
+
+ // Make paths in deps files relative
+ "-no-canonical-prefixes",
+
+ "-O2",
+ "-Wall",
+ "-Werror",
+ "-Wextra",
+
+ "-isystem bionic/libc/include",
+ "-isystem bionic/libc/kernel/uapi",
+ // The architecture doesn't matter here, but asm/types.h is included by linux/types.h.
+ "-isystem bionic/libc/kernel/uapi/asm-arm64",
+ "-isystem bionic/libc/kernel/android/uapi",
+ "-I " + ctx.ModuleDir(),
+ "-g", //Libbpf builds require BTF data
+ }
+
+ if runtime.GOOS != "darwin" {
+ cflags = append(cflags, "-fdebug-prefix-map=/proc/self/cwd=")
+ }
+
+ ctx.VisitDirectDeps(func(dep android.Module) {
+ depTag := ctx.OtherModuleDependencyTag(dep)
+ if depTag == libbpfProgDepTag {
+ if genRule, ok := dep.(genrule.SourceFileGenerator); ok {
+ cFlagsDeps = append(cFlagsDeps, genRule.GeneratedDeps()...)
+ dirs := genRule.GeneratedHeaderDirs()
+ for _, dir := range dirs {
+ cflags = append(cflags, "-I "+dir.String())
+ }
+ } else {
+ depName := ctx.OtherModuleName(dep)
+ ctx.ModuleErrorf("module %q is not a genrule", depName)
+ }
+ }
+ })
+
+ for _, dir := range android.PathsForModuleSrc(ctx, libbpf.properties.Local_include_dirs) {
+ cflags = append(cflags, "-I "+dir.String())
+ }
+
+ cflags = append(cflags, libbpf.properties.Cflags...)
+
+ srcs := android.PathsForModuleSrc(ctx, libbpf.properties.Srcs)
+
+ for _, src := range srcs {
+ if strings.ContainsRune(src.Base(), '_') {
+ ctx.ModuleErrorf("invalid character '_' in source name")
+ }
+ obj := android.ObjPathWithExt(ctx, "unstripped", src, "o")
+
+ ctx.Build(pctx, android.BuildParams{
+ Rule: libbpfProgCcRule,
+ Input: src,
+ Implicits: cFlagsDeps,
+ Output: obj,
+ Args: map[string]string{
+ "cFlags": strings.Join(cflags, " "),
+ "ccCmd": "${config.ClangBin}/clang",
+ },
+ })
+
+ objStripped := android.ObjPathWithExt(ctx, "", src, "o")
+ ctx.Build(pctx, android.BuildParams{
+ Rule: libbpfProgStripRule,
+ Input: obj,
+ Output: objStripped,
+ Args: map[string]string{
+ "stripCmd": "${config.ClangBin}/llvm-strip",
+ },
+ })
+ libbpf.objs = append(libbpf.objs, objStripped.WithoutRel())
+ }
+
+ installDir := android.PathForModuleInstall(ctx, "etc", "bpf/libbpf")
+ if len(libbpf.properties.Relative_install_path) > 0 {
+ installDir = installDir.Join(ctx, libbpf.properties.Relative_install_path)
+ }
+ for _, obj := range libbpf.objs {
+ ctx.PackageFile(installDir, obj.Base(), obj)
+ }
+
+ android.SetProvider(ctx, blueprint.SrcsFileProviderKey, blueprint.SrcsFileProviderData{SrcPaths: srcs.Strings()})
+
+ ctx.SetOutputFiles(libbpf.objs, "")
+}
+
+func (libbpf *libbpfProg) AndroidMk() android.AndroidMkData {
+ return android.AndroidMkData{
+ Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
+ var names []string
+ fmt.Fprintln(w)
+ fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir)
+ fmt.Fprintln(w)
+ var localModulePath string
+ localModulePath = "LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/bpf/libbpf"
+ if len(libbpf.properties.Relative_install_path) > 0 {
+ localModulePath += "/" + libbpf.properties.Relative_install_path
+ }
+ for _, obj := range libbpf.objs {
+ objName := name + "_" + obj.Base()
+ names = append(names, objName)
+ fmt.Fprintln(w, "include $(CLEAR_VARS)", " # libbpf.libbpf.obj")
+ fmt.Fprintln(w, "LOCAL_MODULE := ", objName)
+ fmt.Fprintln(w, "LOCAL_PREBUILT_MODULE_FILE :=", obj.String())
+ fmt.Fprintln(w, "LOCAL_MODULE_STEM :=", obj.Base())
+ fmt.Fprintln(w, "LOCAL_MODULE_CLASS := ETC")
+ fmt.Fprintln(w, localModulePath)
+ // AconfigUpdateAndroidMkData may have added elements to Extra. Process them here.
+ for _, extra := range data.Extra {
+ extra(w, nil)
+ }
+ fmt.Fprintln(w, "include $(BUILD_PREBUILT)")
+ fmt.Fprintln(w)
+ }
+ fmt.Fprintln(w, "include $(CLEAR_VARS)", " # libbpf.libbpf")
+ fmt.Fprintln(w, "LOCAL_MODULE := ", name)
+ android.AndroidMkEmitAssignList(w, "LOCAL_REQUIRED_MODULES", names)
+ fmt.Fprintln(w, "include $(BUILD_PHONY_PACKAGE)")
+ },
+ }
+}
+
+func LibbpfProgFactory() android.Module {
+ module := &libbpfProg{}
+
+ module.AddProperties(&module.properties)
+ android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibFirst)
+ return module
+}
diff --git a/bpf/libbpf/libbpf_prog_test.go b/bpf/libbpf/libbpf_prog_test.go
new file mode 100644
index 0000000..f4f5167
--- /dev/null
+++ b/bpf/libbpf/libbpf_prog_test.go
@@ -0,0 +1,69 @@
+// Copyright 2024 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package libbpf_prog
+
+import (
+ "os"
+ "testing"
+
+ "android/soong/android"
+ "android/soong/cc"
+)
+
+func TestMain(m *testing.M) {
+ os.Exit(m.Run())
+}
+
+var prepareForLibbpfProgTest = android.GroupFixturePreparers(
+ cc.PrepareForTestWithCcDefaultModules,
+ android.FixtureMergeMockFs(
+ map[string][]byte{
+ "bpf.c": nil,
+ "bpf_invalid_name.c": nil,
+ "BpfTest.cpp": nil,
+ },
+ ),
+ PrepareForTestWithLibbpfProg,
+)
+
+func TestLibbpfProgDataDependency(t *testing.T) {
+ bp := `
+ libbpf_prog {
+ name: "bpf.o",
+ srcs: ["bpf.c"],
+ }
+
+ cc_test {
+ name: "vts_test_binary_bpf_module",
+ srcs: ["BpfTest.cpp"],
+ data: [":bpf.o"],
+ gtest: false,
+ }
+ `
+
+ prepareForLibbpfProgTest.RunTestWithBp(t, bp)
+}
+
+func TestLibbpfProgSourceName(t *testing.T) {
+ bp := `
+ libbpf_prog {
+ name: "bpf_invalid_name.o",
+ srcs: ["bpf_invalid_name.c"],
+ }
+ `
+ prepareForLibbpfProgTest.ExtendWithErrorHandler(android.FixtureExpectsOneErrorPattern(
+ `invalid character '_' in source name`)).
+ RunTestWithBp(t, bp)
+}
diff --git a/cc/cc.go b/cc/cc.go
index b534737..5b3647e 100644
--- a/cc/cc.go
+++ b/cc/cc.go
@@ -613,7 +613,7 @@
coverageOutputFilePath() android.OptionalPath
// Get the deps that have been explicitly specified in the properties.
- linkerSpecifiedDeps(ctx android.ConfigAndErrorContext, module *Module, specifiedDeps specifiedDeps) specifiedDeps
+ linkerSpecifiedDeps(ctx android.ConfigurableEvaluatorContext, module *Module, specifiedDeps specifiedDeps) specifiedDeps
moduleInfoJSON(ctx ModuleContext, moduleInfoJSON *android.ModuleInfoJSON)
}
@@ -970,7 +970,7 @@
return c.Properties.HideFromMake
}
-func (c *Module) RequiredModuleNames(ctx android.ConfigAndErrorContext) []string {
+func (c *Module) RequiredModuleNames(ctx android.ConfigurableEvaluatorContext) []string {
required := android.CopyOf(c.ModuleBase.RequiredModuleNames(ctx))
if c.ImageVariation().Variation == android.CoreVariation {
required = append(required, c.Properties.Target.Platform.Required...)
diff --git a/cc/check.go b/cc/check.go
index e3af3b2..fa1926d 100644
--- a/cc/check.go
+++ b/cc/check.go
@@ -40,6 +40,8 @@
ctx.PropertyErrorf(prop, "Bad flag: `%s`, use native_coverage instead", flag)
} else if flag == "-fwhole-program-vtables" {
ctx.PropertyErrorf(prop, "Bad flag: `%s`, use whole_program_vtables instead", flag)
+ } else if flag == "-fno-integrated-as" {
+ ctx.PropertyErrorf("Bad flag: `%s` is disallowed as it may invoke the `as` from the build host", flag)
} else if flag == "-Weverything" {
if !ctx.Config().IsEnvTrue("ANDROID_TEMPORARILY_ALLOW_WEVERYTHING") {
ctx.PropertyErrorf(prop, "-Weverything is not allowed in Android.bp files. "+
diff --git a/cc/config/arm64_device.go b/cc/config/arm64_device.go
index 761afcf..0dcf2cf 100644
--- a/cc/config/arm64_device.go
+++ b/cc/config/arm64_device.go
@@ -118,11 +118,9 @@
pctx.StaticVariable("Arm64Cppflags", strings.Join(arm64Cppflags, " "))
- pctx.StaticVariable("Arm64Armv8ACflags", strings.Join(arm64ArchVariantCflags["armv8-a"], " "))
- pctx.StaticVariable("Arm64Armv8ABranchProtCflags", strings.Join(arm64ArchVariantCflags["armv8-a-branchprot"], " "))
- pctx.StaticVariable("Arm64Armv82ACflags", strings.Join(arm64ArchVariantCflags["armv8-2a"], " "))
- pctx.StaticVariable("Arm64Armv82ADotprodCflags", strings.Join(arm64ArchVariantCflags["armv8-2a-dotprod"], " "))
- pctx.StaticVariable("Arm64Armv9ACflags", strings.Join(arm64ArchVariantCflags["armv9-a"], " "))
+ for variant, cflags := range arm64ArchVariantCflags {
+ pctx.StaticVariable("Arm64"+variant+"VariantCflags", strings.Join(cflags, " "))
+ }
pctx.StaticVariable("Arm64CortexA53Cflags", strings.Join(arm64CpuVariantCflags["cortex-a53"], " "))
pctx.StaticVariable("Arm64CortexA55Cflags", strings.Join(arm64CpuVariantCflags["cortex-a55"], " "))
@@ -134,14 +132,6 @@
}
var (
- arm64ArchVariantCflagsVar = map[string]string{
- "armv8-a": "${config.Arm64Armv8ACflags}",
- "armv8-a-branchprot": "${config.Arm64Armv8ABranchProtCflags}",
- "armv8-2a": "${config.Arm64Armv82ACflags}",
- "armv8-2a-dotprod": "${config.Arm64Armv82ADotprodCflags}",
- "armv9-a": "${config.Arm64Armv9ACflags}",
- }
-
arm64CpuVariantCflagsVar = map[string]string{
"cortex-a53": "${config.Arm64CortexA53Cflags}",
"cortex-a55": "${config.Arm64CortexA55Cflags}",
@@ -211,18 +201,12 @@
}
func arm64ToolchainFactory(arch android.Arch) Toolchain {
- switch arch.ArchVariant {
- case "armv8-a":
- case "armv8-a-branchprot":
- case "armv8-2a":
- case "armv8-2a-dotprod":
- case "armv9-a":
- // Nothing extra for armv8-a/armv8-2a
- default:
- panic(fmt.Sprintf("Unknown ARM architecture version: %q", arch.ArchVariant))
+ // Error now rather than having a confusing Ninja error
+ if _, ok := arm64ArchVariantCflags[arch.ArchVariant]; !ok {
+ panic(fmt.Sprintf("Unknown ARM64 architecture version: %q", arch.ArchVariant))
}
- toolchainCflags := []string{arm64ArchVariantCflagsVar[arch.ArchVariant]}
+ toolchainCflags := []string{"${config.Arm64" + arch.ArchVariant + "VariantCflags}"}
toolchainCflags = append(toolchainCflags,
variantOrDefault(arm64CpuVariantCflagsVar, arch.CpuVariant))
diff --git a/cc/config/arm64_linux_host.go b/cc/config/arm64_linux_host.go
index 438e0e6..a19b0ed 100644
--- a/cc/config/arm64_linux_host.go
+++ b/cc/config/arm64_linux_host.go
@@ -87,8 +87,8 @@
}
func linuxBionicArm64ToolchainFactory(arch android.Arch) Toolchain {
- archVariant := "armv8-a" // for host, default to armv8-a
- toolchainCflags := []string{arm64ArchVariantCflagsVar[archVariant]}
+ // for host, default to armv8-a
+ toolchainCflags := []string{"-march=armv8-a"}
// We don't specify CPU architecture for host. Conservatively assume
// the host CPU needs the fix
diff --git a/cc/library.go b/cc/library.go
index 03f7174..0a8fb24 100644
--- a/cc/library.go
+++ b/cc/library.go
@@ -919,7 +919,7 @@
return deps
}
-func (library *libraryDecorator) linkerSpecifiedDeps(ctx android.ConfigAndErrorContext, module *Module, specifiedDeps specifiedDeps) specifiedDeps {
+func (library *libraryDecorator) linkerSpecifiedDeps(ctx android.ConfigurableEvaluatorContext, module *Module, specifiedDeps specifiedDeps) specifiedDeps {
specifiedDeps = library.baseLinker.linkerSpecifiedDeps(ctx, module, specifiedDeps)
var properties StaticOrSharedProperties
if library.static() {
diff --git a/cc/linker.go b/cc/linker.go
index 0056817..1efacad 100644
--- a/cc/linker.go
+++ b/cc/linker.go
@@ -645,7 +645,7 @@
panic(fmt.Errorf("baseLinker doesn't know how to link"))
}
-func (linker *baseLinker) linkerSpecifiedDeps(ctx android.ConfigAndErrorContext, module *Module, specifiedDeps specifiedDeps) specifiedDeps {
+func (linker *baseLinker) linkerSpecifiedDeps(ctx android.ConfigurableEvaluatorContext, module *Module, specifiedDeps specifiedDeps) specifiedDeps {
eval := module.ConfigurableEvaluator(ctx)
specifiedDeps.sharedLibs = append(specifiedDeps.sharedLibs, linker.Properties.Shared_libs.GetOrDefault(eval, nil)...)
diff --git a/cc/object.go b/cc/object.go
index a4f4c84..c89520a 100644
--- a/cc/object.go
+++ b/cc/object.go
@@ -203,7 +203,7 @@
return outputFile
}
-func (object *objectLinker) linkerSpecifiedDeps(ctx android.ConfigAndErrorContext, module *Module, specifiedDeps specifiedDeps) specifiedDeps {
+func (object *objectLinker) linkerSpecifiedDeps(ctx android.ConfigurableEvaluatorContext, module *Module, specifiedDeps specifiedDeps) specifiedDeps {
eval := module.ConfigurableEvaluator(ctx)
specifiedDeps.sharedLibs = append(specifiedDeps.sharedLibs, object.Properties.Shared_libs.GetOrDefault(eval, nil)...)
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index a8be7ec..24a44b4 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -15,7 +15,6 @@
package main
import (
- "bytes"
"encoding/json"
"errors"
"flag"
@@ -29,10 +28,12 @@
"android/soong/android/allowlists"
"android/soong/bp2build"
"android/soong/shared"
+
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/deptools"
"github.com/google/blueprint/metrics"
+ "github.com/google/blueprint/pathtools"
"github.com/google/blueprint/proptools"
androidProtobuf "google.golang.org/protobuf/android"
)
@@ -42,8 +43,6 @@
availableEnvFile string
usedEnvFile string
- globFile string
- globListDir string
delveListen string
delvePath string
@@ -64,8 +63,6 @@
flag.StringVar(&cmdlineArgs.SoongOutDir, "soong_out", "", "Soong output directory (usually $TOP/out/soong)")
flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")
flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
- flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")
- flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
flag.StringVar(&cmdlineArgs.OutDir, "out", "", "the ninja builddir directory")
flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")
@@ -206,20 +203,6 @@
ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile)
}
-func writeBuildGlobsNinjaFile(ctx *android.Context) {
- ctx.EventHandler.Begin("globs_ninja_file")
- defer ctx.EventHandler.End("globs_ninja_file")
-
- globDir := bootstrap.GlobDirectory(ctx.Config().SoongOutDir(), globListDir)
- err := bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{
- GlobLister: ctx.Globs,
- GlobFile: globFile,
- GlobDir: globDir,
- SrcDir: ctx.SrcDir(),
- }, ctx.Config())
- maybeQuit(err, "")
-}
-
func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDeps []string) {
eventHandler.Begin("ninja_deps")
defer eventHandler.End("ninja_deps")
@@ -283,7 +266,9 @@
}
// runSoongOnlyBuild runs the standard Soong build in a number of different modes.
-func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string {
+// It returns the path to the output file (usually the ninja file) and the deps that need
+// to trigger a soong rerun.
+func runSoongOnlyBuild(ctx *android.Context) (string, []string) {
ctx.EventHandler.Begin("soong_build")
defer ctx.EventHandler.End("soong_build")
@@ -299,37 +284,30 @@
ninjaDeps, err := bootstrap.RunBlueprint(cmdlineArgs.Args, stopBefore, ctx.Context, ctx.Config())
maybeQuit(err, "")
- ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
-
- writeBuildGlobsNinjaFile(ctx)
// Convert the Soong module graph into Bazel BUILD files.
switch ctx.Config().BuildMode {
case android.GenerateQueryView:
queryviewMarkerFile := cmdlineArgs.BazelQueryViewDir + ".marker"
runQueryView(cmdlineArgs.BazelQueryViewDir, queryviewMarkerFile, ctx)
- writeDepFile(queryviewMarkerFile, ctx.EventHandler, ninjaDeps)
- return queryviewMarkerFile
+ return queryviewMarkerFile, ninjaDeps
case android.GenerateModuleGraph:
writeJsonModuleGraphAndActions(ctx, cmdlineArgs)
- writeDepFile(cmdlineArgs.ModuleGraphFile, ctx.EventHandler, ninjaDeps)
- return cmdlineArgs.ModuleGraphFile
+ return cmdlineArgs.ModuleGraphFile, ninjaDeps
case android.GenerateDocFile:
// TODO: we could make writeDocs() return the list of documentation files
// written and add them to the .d file. Then soong_docs would be re-run
// whenever one is deleted.
err := writeDocs(ctx, shared.JoinPath(topDir, cmdlineArgs.DocFile))
maybeQuit(err, "error building Soong documentation")
- writeDepFile(cmdlineArgs.DocFile, ctx.EventHandler, ninjaDeps)
- return cmdlineArgs.DocFile
+ return cmdlineArgs.DocFile, ninjaDeps
default:
// The actual output (build.ninja) was written in the RunBlueprint() call
// above
- writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps)
if needToWriteNinjaHint(ctx) {
writeNinjaHint(ctx)
}
- return cmdlineArgs.OutFile
+ return cmdlineArgs.OutFile, ninjaDeps
}
}
@@ -359,6 +337,8 @@
func main() {
flag.Parse()
+ soongStartTime := time.Now()
+
shared.ReexecWithDelveMaybe(delveListen, delvePath)
android.InitSandbox(topDir)
@@ -369,13 +349,6 @@
configuration.SetAllowMissingDependencies()
}
- extraNinjaDeps := []string{configuration.ProductVariablesFileName, usedEnvFile}
- if shared.IsDebugging() {
- // Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is
- // enabled even if it completed successfully.
- extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))
- }
-
// Bypass configuration.Getenv, as LOG_DIR does not need to be dependency tracked. By definition, it will
// change between every CI build, so tracking it would require re-running Soong for every build.
metricsDir := availableEnv["LOG_DIR"]
@@ -393,7 +366,16 @@
ctx.SetIncrementalAnalysis(incremental)
ctx.Register()
- finalOutputFile := runSoongOnlyBuild(ctx, extraNinjaDeps)
+ finalOutputFile, ninjaDeps := runSoongOnlyBuild(ctx)
+
+ ninjaDeps = append(ninjaDeps, usedEnvFile)
+ if shared.IsDebugging() {
+ // Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is
+ // enabled even if it completed successfully.
+ ninjaDeps = append(ninjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))
+ }
+
+ writeDepFile(finalOutputFile, ctx.EventHandler, ninjaDeps)
if ctx.GetIncrementalEnabled() {
data, err := shared.EnvFileContents(configuration.EnvDeps())
@@ -407,6 +389,9 @@
writeUsedEnvironmentFile(configuration)
+ err = writeGlobFile(ctx.EventHandler, finalOutputFile, ctx.Globs(), soongStartTime)
+ maybeQuit(err, "")
+
// Touch the output file so that it's the newest file created by soong_build.
// This is necessary because, if soong_build generated any files which
// are ninja inputs to the main output file, then ninja would superfluously
@@ -423,18 +408,33 @@
data, err := shared.EnvFileContents(configuration.EnvDeps())
maybeQuit(err, "error writing used environment file '%s'\n", usedEnvFile)
- if preexistingData, err := os.ReadFile(path); err != nil {
- if !os.IsNotExist(err) {
- maybeQuit(err, "error reading used environment file '%s'", usedEnvFile)
- }
- } else if bytes.Equal(preexistingData, data) {
- // used environment file is unchanged
- return
- }
- err = os.WriteFile(path, data, 0666)
+ err = pathtools.WriteFileIfChanged(path, data, 0666)
maybeQuit(err, "error writing used environment file '%s'", usedEnvFile)
}
+func writeGlobFile(eventHandler *metrics.EventHandler, finalOutFile string, globs pathtools.MultipleGlobResults, soongStartTime time.Time) error {
+ eventHandler.Begin("writeGlobFile")
+ defer eventHandler.End("writeGlobFile")
+
+ globsFile, err := os.Create(shared.JoinPath(topDir, finalOutFile+".globs"))
+ if err != nil {
+ return err
+ }
+ defer globsFile.Close()
+ globsFileEncoder := json.NewEncoder(globsFile)
+ for _, glob := range globs {
+ if err := globsFileEncoder.Encode(glob); err != nil {
+ return err
+ }
+ }
+
+ return os.WriteFile(
+ shared.JoinPath(topDir, finalOutFile+".globs_time"),
+ []byte(fmt.Sprintf("%d\n", soongStartTime.UnixMicro())),
+ 0666,
+ )
+}
+
func touch(path string) {
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
maybeQuit(err, "Error touching '%s'", path)
diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go
index 5c7ef43..a8f97e3 100644
--- a/filesystem/filesystem.go
+++ b/filesystem/filesystem.go
@@ -136,6 +136,9 @@
// Install aconfig_flags.pb file for the modules installed in this partition.
Gen_aconfig_flags_pb *bool
+ // Update the Base_dir of the $PRODUCT_OUT directory with the packaging files.
+ Update_product_out *bool
+
Fsverity fsverityProperties
}
@@ -331,6 +334,14 @@
return f.CopySpecsToDirs(ctx, builder, dirsToSpecs)
}
+func (f *filesystem) copyFilesToProductOut(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath) {
+ if !proptools.Bool(f.properties.Update_product_out) {
+ return
+ }
+ installPath := android.PathForModuleInPartitionInstall(ctx, f.partitionName())
+ builder.Command().Textf("cp -prf %s/* %s", rebasedDir, installPath)
+}
+
func (f *filesystem) buildImageUsingBuildImage(ctx android.ModuleContext) android.OutputPath {
rootDir := android.PathForModuleOut(ctx, "root").OutputPath
rebasedDir := rootDir
@@ -348,6 +359,7 @@
f.buildFsverityMetadataFiles(ctx, builder, specs, rootDir, rebasedDir)
f.buildEventLogtagsFile(ctx, builder, rebasedDir)
f.buildAconfigFlagsFiles(ctx, builder, specs, rebasedDir)
+ f.copyFilesToProductOut(ctx, builder, rebasedDir)
// run host_init_verifier
// Ideally we should have a concept of pluggable linters that verify the generated image.
@@ -490,6 +502,7 @@
f.buildFsverityMetadataFiles(ctx, builder, specs, rootDir, rebasedDir)
f.buildEventLogtagsFile(ctx, builder, rebasedDir)
f.buildAconfigFlagsFiles(ctx, builder, specs, rebasedDir)
+ f.copyFilesToProductOut(ctx, builder, rebasedDir)
output := android.PathForModuleOut(ctx, f.installFileName()).OutputPath
cmd := builder.Command().
diff --git a/fuzz/fuzz_common.go b/fuzz/fuzz_common.go
index 306d65e..a059837 100644
--- a/fuzz/fuzz_common.go
+++ b/fuzz/fuzz_common.go
@@ -449,7 +449,7 @@
}
}
-func IsValid(ctx android.ConfigAndErrorContext, fuzzModule FuzzModule) bool {
+func IsValid(ctx android.ConfigurableEvaluatorContext, fuzzModule FuzzModule) bool {
// Discard ramdisk + vendor_ramdisk + recovery modules, they're duplicates of
// fuzz targets we're going to package anyway.
if !fuzzModule.Enabled(ctx) || fuzzModule.InRamdisk() || fuzzModule.InVendorRamdisk() || fuzzModule.InRecovery() {
diff --git a/genrule/allowlists.go b/genrule/allowlists.go
index 4f1b320..45a7f72 100644
--- a/genrule/allowlists.go
+++ b/genrule/allowlists.go
@@ -17,7 +17,6 @@
var (
SandboxingDenyModuleList = []string{
// go/keep-sorted start
- "com.google.pixel.camera.hal.manifest",
// go/keep-sorted end
}
)
diff --git a/java/boot_jars.go b/java/boot_jars.go
index 6223ded..3c3bd55 100644
--- a/java/boot_jars.go
+++ b/java/boot_jars.go
@@ -21,7 +21,7 @@
// isActiveModule returns true if the given module should be considered for boot
// jars, i.e. if it's enabled and the preferred one in case of source and
// prebuilt alternatives.
-func isActiveModule(ctx android.ConfigAndErrorContext, module android.Module) bool {
+func isActiveModule(ctx android.ConfigurableEvaluatorContext, module android.Module) bool {
if !module.Enabled(ctx) {
return false
}
diff --git a/python/python.go b/python/python.go
index 8726f02..01ac86c 100644
--- a/python/python.go
+++ b/python/python.go
@@ -264,10 +264,9 @@
variants = append(variants, pyVersion3)
}
if proptools.BoolDefault(props.Version.Py2.Enabled, false) {
- if !ctx.DeviceConfig().BuildBrokenUsesSoongPython2Modules() &&
- ctx.ModuleName() != "py2-cmd" &&
+ if ctx.ModuleName() != "py2-cmd" &&
ctx.ModuleName() != "py2-stdlib" {
- ctx.PropertyErrorf("version.py2.enabled", "Python 2 is no longer supported, please convert to python 3. This error can be temporarily overridden by setting BUILD_BROKEN_USES_SOONG_PYTHON2_MODULES := true in the product configuration")
+ ctx.PropertyErrorf("version.py2.enabled", "Python 2 is no longer supported, please convert to python 3.")
}
variants = append(variants, pyVersion2)
}
diff --git a/sdk/update.go b/sdk/update.go
index 9379f36..93bb861 100644
--- a/sdk/update.go
+++ b/sdk/update.go
@@ -1118,12 +1118,14 @@
// Since module sdks are generated from release branches and dropped to development
// branches, there might be a visibility skew between the sources and prebuilts
// of a specific module.
- // To reconcile this potential skew, change the visibility to public
+ // To reconcile this potential skew, change the visibility to public.
//
- // This is safe for (1) since these are stub libraries.
- // This is ok for (2) since these are host and test exports and are intended for
- // ART development.
- // TODO (b/361303067): This can be removed if ART uses full manifests.
+ // This means dependencies can bypass visibility restrictions when prebuilts are used, so we rely
+ // on source builds in CI to check them.
+ //
+ // TODO (b/361303067): This special case for category (2) can be removed if existing usages
+ // of host/test prebuilts of modules like conscrypt,tzdata,i18n are switched to source builds.
+ // It will also require ART switching to full manifests.
m.AddProperty("visibility", []string{"//visibility:public"})
}
diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh
index 2e40950..715f976 100755
--- a/tests/bootstrap_test.sh
+++ b/tests/bootstrap_test.sh
@@ -145,36 +145,19 @@
run_soong
local -r ninja_mtime1=$(stat -c "%y" out/soong/build."${target_product}".ninja)
- local glob_deps_file=out/soong/globs/"${target_product}"/0.d
-
run_soong
local -r ninja_mtime2=$(stat -c "%y" out/soong/build."${target_product}".ninja)
- # There is an ineffiencency in glob that requires bpglob to rerun once for each glob to update
- # the entry in the .ninja_log. It doesn't update the output file, but we can detect the rerun
- # by checking if the deps file was created.
- if [ ! -e "$glob_deps_file" ]; then
- fail "Glob deps file missing after second build"
- fi
-
- local -r glob_deps_mtime2=$(stat -c "%y" "$glob_deps_file")
-
if [[ "$ninja_mtime1" != "$ninja_mtime2" ]]; then
fail "Ninja file rewritten on null incremental build"
fi
run_soong
local -r ninja_mtime3=$(stat -c "%y" out/soong/build."${target_product}".ninja)
- local -r glob_deps_mtime3=$(stat -c "%y" "$glob_deps_file")
if [[ "$ninja_mtime2" != "$ninja_mtime3" ]]; then
fail "Ninja file rewritten on null incremental build"
fi
-
- # The bpglob commands should not rerun after the first incremental build.
- if [[ "$glob_deps_mtime2" != "$glob_deps_mtime3" ]]; then
- fail "Glob deps file rewritten on second null incremental build"
- fi
}
function test_add_file_to_glob() {
diff --git a/ui/build/build.go b/ui/build/build.go
index 49ac791..28c3284 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -80,7 +80,7 @@
if username, ok = config.environ.Get("BUILD_USERNAME"); !ok {
ctx.Fatalln("Missing BUILD_USERNAME")
}
- buildNumber = fmt.Sprintf("eng.%.6s.00000000.000000", username)
+ buildNumber = fmt.Sprintf("eng.%.6s", username)
writeValueIfChanged(ctx, config, config.OutDir(), "file_name_tag.txt", username)
}
// Write the build number to a file so it can be read back in
diff --git a/ui/build/config.go b/ui/build/config.go
index 08e1957..f02222e 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -1037,10 +1037,6 @@
}
}
-func (c *configImpl) NamedGlobFile(name string) string {
- return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
-}
-
func (c *configImpl) UsedEnvFile(tag string) string {
if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+c.CoverageSuffix()+"."+tag)
diff --git a/ui/build/path.go b/ui/build/path.go
index 075bf2e..cc1d7e9 100644
--- a/ui/build/path.go
+++ b/ui/build/path.go
@@ -125,15 +125,6 @@
prebuiltsPath, _ := filepath.Abs("prebuilts/build-tools/path/" + runtime.GOOS + "-x86")
myPath = prebuiltsPath + string(os.PathListSeparator) + myPath
- if value, _ := config.Environment().Get("BUILD_BROKEN_PYTHON_IS_PYTHON2"); value == "true" {
- py2Path, _ := filepath.Abs("prebuilts/build-tools/path/" + runtime.GOOS + "-x86/py2")
- if info, err := os.Stat(py2Path); err == nil && info.IsDir() {
- myPath = py2Path + string(os.PathListSeparator) + myPath
- }
- } else if value != "" {
- ctx.Fatalf("BUILD_BROKEN_PYTHON_IS_PYTHON2 can only be set to 'true' or an empty string, but got %s\n", value)
- }
-
// Set $PATH to be the directories containing the host tool symlinks, and
// the prebuilts directory for the current host OS.
config.Environment().Set("PATH", myPath)
@@ -270,15 +261,6 @@
prebuiltsPath, _ := filepath.Abs("prebuilts/build-tools/path/" + runtime.GOOS + "-x86")
myPath = prebuiltsPath + string(os.PathListSeparator) + myPath
- if value, _ := config.Environment().Get("BUILD_BROKEN_PYTHON_IS_PYTHON2"); value == "true" {
- py2Path, _ := filepath.Abs("prebuilts/build-tools/path/" + runtime.GOOS + "-x86/py2")
- if info, err := os.Stat(py2Path); err == nil && info.IsDir() {
- myPath = py2Path + string(os.PathListSeparator) + myPath
- }
- } else if value != "" {
- ctx.Fatalf("BUILD_BROKEN_PYTHON_IS_PYTHON2 can only be set to 'true' or an empty string, but got %s\n", value)
- }
-
// Replace the $PATH variable with the path_interposer symlinks, and
// checked-in prebuilts.
config.Environment().Set("PATH", myPath)
diff --git a/ui/build/soong.go b/ui/build/soong.go
index b94ffa5..97bc997 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -15,10 +15,14 @@
package build
import (
+ "encoding/json"
+ "errors"
"fmt"
"io/fs"
"os"
"path/filepath"
+ "runtime"
+ "slices"
"strconv"
"strings"
"sync"
@@ -52,7 +56,7 @@
// bootstrapEpoch is used to determine if an incremental build is incompatible with the current
// version of bootstrap and needs cleaning before continuing the build. Increment this for
- // incompatible changes, for example when moving the location of the bpglob binary that is
+ // incompatible changes, for example when moving the location of a microfactory binary that is
// executed during bootstrap before the primary builder has had a chance to update the path.
bootstrapEpoch = 1
)
@@ -226,10 +230,6 @@
var allArgs []string
allArgs = append(allArgs, pb.specificArgs...)
- globPathName := getGlobPathNameFromPrimaryBuilderFactory(config, pb)
- allArgs = append(allArgs,
- "--globListDir", globPathName,
- "--globFile", pb.config.NamedGlobFile(globPathName))
allArgs = append(allArgs, commonArgs...)
allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...)
@@ -241,10 +241,8 @@
}
allArgs = append(allArgs, "Android.bp")
- globfiles := bootstrap.GlobFileListFiles(bootstrap.GlobDirectory(config.SoongOutDir(), globPathName))
-
return bootstrap.PrimaryBuilderInvocation{
- Implicits: globfiles,
+ Implicits: []string{pb.output + ".glob_results"},
Outputs: []string{pb.output},
Args: allArgs,
Description: pb.description,
@@ -276,24 +274,15 @@
os.Remove(file)
}
}
- for _, globFile := range bootstrapGlobFileList(config) {
- os.Remove(globFile)
- }
+ os.Remove(soongNinjaFile + ".globs")
+ os.Remove(soongNinjaFile + ".globs_time")
+ os.Remove(soongNinjaFile + ".glob_results")
// Mark the tree as up to date with the current epoch by writing the epoch marker file.
writeEmptyFile(ctx, epochPath)
}
}
-func bootstrapGlobFileList(config Config) []string {
- return []string{
- config.NamedGlobFile(getGlobPathName(config)),
- config.NamedGlobFile(jsonModuleGraphTag),
- config.NamedGlobFile(queryviewTag),
- config.NamedGlobFile(soongDocsTag),
- }
-}
-
func bootstrapBlueprint(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
defer ctx.EndTrace()
@@ -411,32 +400,9 @@
runGoTests: !config.skipSoongTests,
// If we want to debug soong_build, we need to compile it for debugging
debugCompilation: delvePort != "",
- subninjas: bootstrapGlobFileList(config),
primaryBuilderInvocations: invocations,
}
- // The glob ninja files are generated during the main build phase. However, the
- // primary buildifer invocation depends on all of its glob files, even before
- // it's been run. Generate a "empty" glob ninja file on the first run,
- // so that the files can be there to satisfy the dependency.
- for _, pb := range pbfs {
- globPathName := getGlobPathNameFromPrimaryBuilderFactory(config, pb)
- globNinjaFile := config.NamedGlobFile(globPathName)
- if _, err := os.Stat(globNinjaFile); os.IsNotExist(err) {
- err := bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{
- GlobLister: func() pathtools.MultipleGlobResults { return nil },
- GlobFile: globNinjaFile,
- GlobDir: bootstrap.GlobDirectory(config.SoongOutDir(), globPathName),
- SrcDir: ".",
- }, blueprintConfig)
- if err != nil {
- ctx.Fatal(err)
- }
- } else if err != nil {
- ctx.Fatal(err)
- }
- }
-
// since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little
// reason to write a `bootstrap.ninja.d` file
_, err := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
@@ -614,9 +580,6 @@
}
}()
- runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob",
- map[string]string{"github.com/google/blueprint": "build/blueprint"})
-
ninja := func(targets ...string) {
ctx.BeginTrace(metrics.RunSoong, "bootstrap")
defer ctx.EndTrace()
@@ -698,6 +661,12 @@
targets = append(targets, config.SoongNinjaFile())
}
+ for _, target := range targets {
+ if err := checkGlobs(ctx, target); err != nil {
+ ctx.Fatalf("Error checking globs: %s", err.Error())
+ }
+ }
+
beforeSoongTimestamp := time.Now()
ninja(targets...)
@@ -724,6 +693,160 @@
}
}
+// checkGlobs manages the globs that cause soong to rerun.
+//
+// When soong_build runs, it will run globs. It will write all the globs
+// it ran into the "{finalOutFile}.globs" file. Then every build,
+// soong_ui will check that file, rerun the globs, and if they changed
+// from the results that soong_build got, update the ".glob_results"
+// file, causing soong_build to rerun. The ".glob_results" file will
+// be empty on the first run of soong_build, because we don't know
+// what the globs are yet, but also remain empty until the globs change
+// so that we don't run soong_build a second time unnecessarily.
+// Both soong_build and soong_ui will also update a ".globs_time" file
+// with the time that they ran at every build. When soong_ui checks
+// globs, it only reruns globs whose dependencies are newer than the
+// time in the ".globs_time" file.
+func checkGlobs(ctx Context, finalOutFile string) error {
+ ctx.BeginTrace(metrics.RunSoong, "check_globs")
+ defer ctx.EndTrace()
+ st := ctx.Status.StartTool()
+ st.Status("Running globs...")
+ defer st.Finish()
+
+ globsFile, err := os.Open(finalOutFile + ".globs")
+ if errors.Is(err, fs.ErrNotExist) {
+ // if the glob file doesn't exist, make sure the glob_results file exists and is empty.
+ if err := os.MkdirAll(filepath.Dir(finalOutFile), 0777); err != nil {
+ return err
+ }
+ f, err := os.Create(finalOutFile + ".glob_results")
+ if err != nil {
+ return err
+ }
+ return f.Close()
+ } else if err != nil {
+ return err
+ }
+ defer globsFile.Close()
+ globsFileDecoder := json.NewDecoder(globsFile)
+
+ globsTimeBytes, err := os.ReadFile(finalOutFile + ".globs_time")
+ if err != nil {
+ return err
+ }
+ globsTimeMicros, err := strconv.ParseInt(strings.TrimSpace(string(globsTimeBytes)), 10, 64)
+ if err != nil {
+ return err
+ }
+ globCheckStartTime := time.Now().UnixMicro()
+
+ globsChan := make(chan pathtools.GlobResult)
+ errorsChan := make(chan error)
+ wg := sync.WaitGroup{}
+ hasChangedGlobs := false
+ for i := 0; i < runtime.NumCPU()*2; i++ {
+ wg.Add(1)
+ go func() {
+ for cachedGlob := range globsChan {
+ // If we've already determined we have changed globs, just finish consuming
+ // the channel without doing any more checks.
+ if hasChangedGlobs {
+ continue
+ }
+ // First, check if any of the deps are newer than the last time globs were checked.
+ // If not, we don't need to rerun the glob.
+ hasNewDep := false
+ for _, dep := range cachedGlob.Deps {
+ info, err := os.Stat(dep)
+ if errors.Is(err, fs.ErrNotExist) {
+ hasNewDep = true
+ break
+ } else if err != nil {
+ errorsChan <- err
+ continue
+ }
+ if info.ModTime().UnixMicro() > globsTimeMicros {
+ hasNewDep = true
+ break
+ }
+ }
+ if !hasNewDep {
+ continue
+ }
+
+ // Then rerun the glob and check if we got the same result as before.
+ result, err := pathtools.Glob(cachedGlob.Pattern, cachedGlob.Excludes, pathtools.FollowSymlinks)
+ if err != nil {
+ errorsChan <- err
+ } else {
+ if !slices.Equal(result.Matches, cachedGlob.Matches) {
+ hasChangedGlobs = true
+ }
+ }
+ }
+ wg.Done()
+ }()
+ }
+ go func() {
+ wg.Wait()
+ close(errorsChan)
+ }()
+
+ errorsWg := sync.WaitGroup{}
+ errorsWg.Add(1)
+ var errFromGoRoutines error
+ go func() {
+ for result := range errorsChan {
+ if errFromGoRoutines == nil {
+ errFromGoRoutines = result
+ }
+ }
+ errorsWg.Done()
+ }()
+
+ var cachedGlob pathtools.GlobResult
+ for globsFileDecoder.More() {
+ if err := globsFileDecoder.Decode(&cachedGlob); err != nil {
+ return err
+ }
+ // Need to clone the GlobResult because the json decoder will
+ // reuse the same slice allocations.
+ globsChan <- cachedGlob.Clone()
+ }
+ close(globsChan)
+ errorsWg.Wait()
+ if errFromGoRoutines != nil {
+ return errFromGoRoutines
+ }
+
+ // Update the globs_time file whether or not we found changed globs,
+ // so that we don't rerun globs in the future that we just saw didn't change.
+ err = os.WriteFile(
+ finalOutFile+".globs_time",
+ []byte(fmt.Sprintf("%d\n", globCheckStartTime)),
+ 0666,
+ )
+ if err != nil {
+ return err
+ }
+
+ if hasChangedGlobs {
+ fmt.Fprintf(os.Stdout, "Globs changed, rerunning soong...\n")
+ // Write the current time to the glob_results file. We just need
+ // some unique value to trigger a rerun, it doesn't matter what it is.
+ err = os.WriteFile(
+ finalOutFile+".glob_results",
+ []byte(fmt.Sprintf("%d\n", globCheckStartTime)),
+ 0666,
+ )
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
// loadSoongBuildMetrics reads out/soong_build_metrics.pb if it was generated by soong_build and copies the
// events stored in it into the soong_ui trace to provide introspection into how long the different phases of
// soong_build are taking.
diff --git a/ui/build/test_build.go b/ui/build/test_build.go
index 3faa94d..ba53119 100644
--- a/ui/build/test_build.go
+++ b/ui/build/test_build.go
@@ -79,9 +79,6 @@
// out/build_date.txt is considered a "source file"
buildDatetimeFilePath := filepath.Join(outDir, "build_date.txt")
- // bpglob is built explicitly using Microfactory
- bpglob := filepath.Join(config.SoongOutDir(), "bpglob")
-
// release-config files are generated from the initial lunch or Kati phase
// before running soong and ninja.
releaseConfigDir := filepath.Join(outDir, "soong", "release-config")
@@ -105,7 +102,6 @@
line == extraVariablesFilePath ||
line == dexpreoptConfigFilePath ||
line == buildDatetimeFilePath ||
- line == bpglob ||
strings.HasPrefix(line, releaseConfigDir) ||
buildFingerPrintFilePattern.MatchString(line) {
// Leaf node is in one of Soong's bootstrap directories, which do not have