Make bp2build be more correct.
It now handles adding .bp files and changing globs.
In order to do this, depfiles are now written separately from RunBlueprint.
This is necessary due to the confluence of a number of seemingly
unrelated factors:
1. The glob filelist dependencies are discovered in globSingleton
2. Singletons need to be registered because otherwise singleton module
types panic
3. Singletons don't work because they require mutators bp2build does not
run
Due to (1), we would need to run the glob singleton. However, due to (2)
and (3), we can't run singletons and have to run Blueprint with
StopBeforeGeneratingBuildActions, which is when the build actions
writing glob files would be generated. So what happens is:
1. When bp2build is run, the glob singleton is disabled
2. At the end of bp2build, the list of glob files is artifically added
to the depfile of the workspace marker file
3. When build.ninja is generated, the Ninja file containing the glob
list file is written by the now-active glob singleton
Test: Presubmits.
Change-Id: I3c5898d8c57c554a93520276c64a952afc912dbe
diff --git a/android/bazel_handler.go b/android/bazel_handler.go
index 2697007..8d561d2 100644
--- a/android/bazel_handler.go
+++ b/android/bazel_handler.go
@@ -737,7 +737,7 @@
// Add ninja file dependencies for files which all bazel invocations require.
bazelBuildList := absolutePath(filepath.Join(
- filepath.Dir(bootstrap.CmdlineModuleListFile()), "bazel.list"))
+ filepath.Dir(bootstrap.CmdlineArgs.ModuleListFile), "bazel.list"))
ctx.AddNinjaFileDeps(bazelBuildList)
data, err := ioutil.ReadFile(bazelBuildList)
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index 7a4cb29..1e796ec 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -26,6 +26,7 @@
"android/soong/bp2build"
"android/soong/shared"
"github.com/google/blueprint/bootstrap"
+ "github.com/google/blueprint/deptools"
"android/soong/android"
)
@@ -89,7 +90,7 @@
}
func newConfig(srcDir, outDir string, availableEnv map[string]string) android.Config {
- configuration, err := android.NewConfig(srcDir, outDir, bootstrap.CmdlineModuleListFile(), availableEnv)
+ configuration, err := android.NewConfig(srcDir, outDir, bootstrap.CmdlineArgs.ModuleListFile, availableEnv)
if err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
@@ -103,21 +104,31 @@
// TODO(cparsons): Don't output any ninja file, as the second pass will overwrite
// the incorrect results from the first pass, and file I/O is expensive.
func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) {
+ var firstArgs, secondArgs bootstrap.Args
+
+ firstArgs = bootstrap.CmdlineArgs
configuration.SetStopBefore(bootstrap.StopBeforeWriteNinja)
- bootstrap.Main(firstCtx.Context, configuration, false, extraNinjaDeps...)
+ bootstrap.RunBlueprint(firstArgs, firstCtx.Context, configuration, extraNinjaDeps...)
+
// Invoke bazel commands and save results for second pass.
if err := configuration.BazelContext.InvokeBazel(); err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
}
// Second pass: Full analysis, using the bazel command results. Output ninja file.
- secondPassConfig, err := android.ConfigForAdditionalRun(configuration)
+ secondConfig, err := android.ConfigForAdditionalRun(configuration)
if err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
}
- secondCtx := newContext(secondPassConfig, true)
- bootstrap.Main(secondCtx.Context, secondPassConfig, false, extraNinjaDeps...)
+ secondCtx := newContext(secondConfig, true)
+ secondArgs = bootstrap.CmdlineArgs
+ ninjaDeps := bootstrap.RunBlueprint(secondArgs, secondCtx.Context, secondConfig, extraNinjaDeps...)
+ err = deptools.WriteDepFile(shared.JoinPath(topDir, secondArgs.DepFile), secondArgs.OutFile, ninjaDeps)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", secondArgs.DepFile, err)
+ os.Exit(1)
+ }
}
// Run the code-generation phase to convert BazelTargetModules to BUILD files.
@@ -132,7 +143,8 @@
func runSoongDocs(configuration android.Config, extraNinjaDeps []string) {
ctx := newContext(configuration, false)
- bootstrap.Main(ctx.Context, configuration, false, extraNinjaDeps...)
+ soongDocsArgs := bootstrap.CmdlineArgs
+ bootstrap.RunBlueprint(soongDocsArgs, ctx.Context, configuration, extraNinjaDeps...)
if err := writeDocs(ctx, configuration, docFile); err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
@@ -166,8 +178,8 @@
generateQueryView := bazelQueryViewDir != ""
jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH")
+ blueprintArgs := bootstrap.CmdlineArgs
prepareBuildActions := !generateQueryView && jsonModuleFile == ""
-
if bazelConversionRequested {
// Run the alternate pipeline of bp2build mutators and singleton to convert
// Blueprint to BUILD files before everything else.
@@ -175,7 +187,7 @@
if bp2buildMarker != "" {
return bp2buildMarker
} else {
- return bootstrap.CmdlineOutFile()
+ return bootstrap.CmdlineArgs.OutFile
}
}
@@ -183,22 +195,27 @@
if mixedModeBuild {
runMixedModeBuild(configuration, ctx, extraNinjaDeps)
} else {
- bootstrap.Main(ctx.Context, configuration, false, extraNinjaDeps...)
+ ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, ctx.Context, configuration, extraNinjaDeps...)
+ err := deptools.WriteDepFile(shared.JoinPath(topDir, blueprintArgs.DepFile), blueprintArgs.OutFile, ninjaDeps)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", blueprintArgs.DepFile, err)
+ os.Exit(1)
+ }
}
// Convert the Soong module graph into Bazel BUILD files.
if generateQueryView {
runQueryView(configuration, ctx)
- return bootstrap.CmdlineOutFile() // TODO: This is a lie
+ return bootstrap.CmdlineArgs.OutFile // TODO: This is a lie
}
if jsonModuleFile != "" {
writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps)
- return bootstrap.CmdlineOutFile() // TODO: This is a lie
+ return bootstrap.CmdlineArgs.OutFile // TODO: This is a lie
}
writeMetrics(configuration)
- return bootstrap.CmdlineOutFile()
+ return bootstrap.CmdlineArgs.OutFile
}
// soong_ui dumps the available environment variables to
@@ -242,7 +259,7 @@
configuration := newConfig(srcDir, outDir, availableEnv)
extraNinjaDeps := []string{
configuration.ProductVariablesFileName,
- shared.JoinPath(outDir, "soong.environment.used"),
+ usedEnvFile,
}
if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
@@ -344,18 +361,18 @@
// Register an alternate set of singletons and mutators for bazel
// conversion for Bazel conversion.
bp2buildCtx := android.NewContext(configuration)
- bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
- bp2buildCtx.RegisterForBazelConversion()
- // No need to generate Ninja build rules/statements from Modules and Singletons.
- configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions)
+ // Propagate "allow misssing dependencies" bit. This is normally set in
+ // newContext(), but we create bp2buildCtx without calling that method.
+ bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
bp2buildCtx.SetNameInterface(newNameResolver(configuration))
+ bp2buildCtx.RegisterForBazelConversion()
// The bp2build process is a purely functional process that only depends on
// Android.bp files. It must not depend on the values of per-build product
// configurations or variables, since those will generate different BUILD
// files based on how the user has configured their tree.
- bp2buildCtx.SetModuleListFile(bootstrap.CmdlineModuleListFile())
+ bp2buildCtx.SetModuleListFile(bootstrap.CmdlineArgs.ModuleListFile)
modulePaths, err := bp2buildCtx.ListModulePaths(configuration.SrcDir())
if err != nil {
panic(err)
@@ -363,10 +380,25 @@
extraNinjaDeps = append(extraNinjaDeps, modulePaths...)
+ // No need to generate Ninja build rules/statements from Modules and Singletons.
+ configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions)
+
// Run the loading and analysis pipeline to prepare the graph of regular
// Modules parsed from Android.bp files, and the BazelTargetModules mapped
// from the regular Modules.
- bootstrap.Main(bp2buildCtx.Context, configuration, false, extraNinjaDeps...)
+ blueprintArgs := bootstrap.CmdlineArgs
+ ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, bp2buildCtx.Context, configuration, extraNinjaDeps...)
+
+ for _, globPath := range bp2buildCtx.Globs() {
+ ninjaDeps = append(ninjaDeps, globPath.FileListFile(configuration.BuildDir()))
+ }
+
+ depFile := bp2buildMarker + ".d"
+ err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err)
+ os.Exit(1)
+ }
// Run the code-generation phase to convert BazelTargetModules to BUILD files
// and print conversion metrics to the user.
diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh
index f85af1a..f1ebca2 100755
--- a/tests/bootstrap_test.sh
+++ b/tests/bootstrap_test.sh
@@ -429,12 +429,47 @@
setup
INTEGRATED_BP2BUILD=1 run_soong
if [[ ! -e out/soong/.bootstrap/bp2build_workspace_marker ]]; then
- fail "b2build marker file not created"
+ fail "bp2build marker file not created"
+ fi
+}
+
+function test_integrated_bp2build_add_android_bp {
+ setup
+
+ mkdir -p a
+ touch a/a.txt
+ cat > a/Android.bp <<'EOF'
+filegroup {
+ name: "a",
+ srcs: ["a.txt"],
+ bazel_module: { bp2build_available: true },
+}
+EOF
+
+ INTEGRATED_BP2BUILD=1 run_soong
+ if [[ ! -e out/soong/bp2build/a/BUILD ]]; then
+ fail "a/BUILD not created";
+ fi
+
+ mkdir -p b
+ touch b/b.txt
+ cat > b/Android.bp <<'EOF'
+filegroup {
+ name: "b",
+ srcs: ["b.txt"],
+ bazel_module: { bp2build_available: true },
+}
+EOF
+
+ INTEGRATED_BP2BUILD=1 run_soong
+ if [[ ! -e out/soong/bp2build/b/BUILD ]]; then
+ fail "b/BUILD not created";
fi
}
function test_integrated_bp2build_null_build {
setup
+
INTEGRATED_BP2BUILD=1 run_soong
local mtime1=$(stat -c "%y" out/soong/build.ninja)
@@ -446,6 +481,27 @@
fi
}
+function test_integrated_bp2build_add_to_glob {
+ setup
+
+ mkdir -p a
+ touch a/a1.txt
+ cat > a/Android.bp <<'EOF'
+filegroup {
+ name: "a",
+ srcs: ["*.txt"],
+ bazel_module: { bp2build_available: true },
+}
+EOF
+
+ INTEGRATED_BP2BUILD=1 run_soong
+ grep -q a1.txt out/soong/bp2build/a/BUILD || fail "a1.txt not in BUILD file"
+
+ touch a/a2.txt
+ INTEGRATED_BP2BUILD=1 run_soong
+ grep -q a2.txt out/soong/bp2build/a/BUILD || fail "a2.txt not in BUILD file"
+}
+
function test_dump_json_module_graph() {
setup
SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong
@@ -468,3 +524,4 @@
test_dump_json_module_graph
test_integrated_bp2build_smoke
test_integrated_bp2build_null_build
+test_integrated_bp2build_add_to_glob
diff --git a/ui/build/soong.go b/ui/build/soong.go
index d77a089..7e94b25 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -21,6 +21,7 @@
"strconv"
"android/soong/shared"
+ "github.com/google/blueprint/deptools"
soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
"github.com/google/blueprint"
@@ -107,6 +108,7 @@
mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")
globFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
+ bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
args.RunGoTests = !config.skipSoongTests
args.UseValidations = true // Use validations to depend on tests
@@ -115,7 +117,6 @@
args.TopFile = "Android.bp"
args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
- args.DepFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
args.GlobFile = globFile
args.GeneratingPrimaryBuilder = true
@@ -171,7 +172,11 @@
debugCompilation: os.Getenv("SOONG_DELVE") != "",
}
- bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
+ bootstrapDeps := bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
+ err := deptools.WriteDepFile(bootstrapDepFile, args.OutFile, bootstrapDeps)
+ if err != nil {
+ ctx.Fatalf("Error writing depfile '%s': %s", bootstrapDepFile, err)
+ }
}
func checkEnvironmentFile(currentEnv *Environment, envFile string) {