Allow running bp2build as part of a regular build.
This is done by setting the INTEGRATED_BP2BUILD environment variable
when invoking the build.
Even though the name of the marker file insinuates that a Bazel
workspace is already created, this is not the case yet.
An issue that remains is that a .d file is not written for the marker
file so it won't be rebuilt if a .bp file changes. Fixing this requires
delicate surgery because writing the .d file is the result of delicate
interplay between Soong and Blueprint.
There are also a number of semi-related fixes:
- The name of soong.environment.{used,available} is now on the command
line of soong_build (soong_docs is still special cased because its
command line in the Ninja file is taken from the os.Args of
soong_build so it's not trivial to remove the --{available,used}_env
from it
- bp2build writes a separate soong.environment.used file
- I had to call SetAllowMissingDependencies() separately when creating
the android.Context for bp2build so that bp2build runs in the
integration tests (it was not obvious how not to do this)
- Fixed a number of integration tests where a command with an expected
exit code of 1 was used as the last one in a test case, thereby
breaking the test suite
Test: Presubmits.
Change-Id: Ibeb61c26022cf801dcb98505b4039151b3409873
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index a4554fc..7a4cb29 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -23,29 +23,42 @@
"strings"
"time"
+ "android/soong/bp2build"
"android/soong/shared"
"github.com/google/blueprint/bootstrap"
"android/soong/android"
- "android/soong/bp2build"
)
var (
- topDir string
- outDir string
+ topDir string
+ outDir string
+ availableEnvFile string
+ usedEnvFile string
+
+ delveListen string
+ delvePath string
+
docFile string
bazelQueryViewDir string
- delveListen string
- delvePath string
+ bp2buildMarker string
)
func init() {
+ // Flags that make sense in every mode
flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree")
flag.StringVar(&outDir, "out", "", "Soong output directory (usually $TOP/out/soong)")
+ flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")
+ flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
+
+ // Debug flags
flag.StringVar(&delveListen, "delve_listen", "", "Delve port to listen on for debugging")
flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set")
+
+ // Flags representing various modes soong_build can run in
flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")
flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
+ flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
}
func newNameResolver(config android.Config) *android.NameResolver {
@@ -147,8 +160,8 @@
writeFakeNinjaFile(extraNinjaDeps, configuration.BuildDir())
}
-func doChosenActivity(configuration android.Config, extraNinjaDeps []string) {
- bazelConversionRequested := configuration.IsEnvTrue("GENERATE_BAZEL_FILES")
+func doChosenActivity(configuration android.Config, extraNinjaDeps []string) string {
+ bazelConversionRequested := configuration.IsEnvTrue("GENERATE_BAZEL_FILES") || bp2buildMarker != ""
mixedModeBuild := configuration.BazelContext.BazelEnabled()
generateQueryView := bazelQueryViewDir != ""
jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH")
@@ -159,7 +172,11 @@
// Run the alternate pipeline of bp2build mutators and singleton to convert
// Blueprint to BUILD files before everything else.
runBp2Build(configuration, extraNinjaDeps)
- return
+ if bp2buildMarker != "" {
+ return bp2buildMarker
+ } else {
+ return bootstrap.CmdlineOutFile()
+ }
}
ctx := newContext(configuration, prepareBuildActions)
@@ -172,15 +189,44 @@
// Convert the Soong module graph into Bazel BUILD files.
if generateQueryView {
runQueryView(configuration, ctx)
- return
+ return bootstrap.CmdlineOutFile() // TODO: This is a lie
}
if jsonModuleFile != "" {
writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps)
- return
+ return bootstrap.CmdlineOutFile() // TODO: This is a lie
}
writeMetrics(configuration)
+ return bootstrap.CmdlineOutFile()
+}
+
+// soong_ui dumps the available environment variables to
+// soong.environment.available . Then soong_build itself is run with an empty
+// environment so that the only way environment variables can be accessed is
+// using Config, which tracks access to them.
+
+// At the end of the build, a file called soong.environment.used is written
+// containing the current value of all used environment variables. The next
+// time soong_ui is run, it checks whether any environment variables that was
+// used had changed and if so, it deletes soong.environment.used to cause a
+// rebuild.
+//
+// The dependency of build.ninja on soong.environment.used is declared in
+// build.ninja.d
+func parseAvailableEnv() map[string]string {
+ if availableEnvFile == "" {
+ fmt.Fprintf(os.Stderr, "--available_env not set\n")
+ os.Exit(1)
+ }
+
+ result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile))
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "error reading available environment file '%s': %s\n", availableEnvFile, err)
+ os.Exit(1)
+ }
+
+ return result
}
func main() {
@@ -189,26 +235,7 @@
shared.ReexecWithDelveMaybe(delveListen, delvePath)
android.InitSandbox(topDir)
- // soong_ui dumps the available environment variables to
- // soong.environment.available . Then soong_build itself is run with an empty
- // environment so that the only way environment variables can be accessed is
- // using Config, which tracks access to them.
-
- // At the end of the build, a file called soong.environment.used is written
- // containing the current value of all used environment variables. The next
- // time soong_ui is run, it checks whether any environment variables that was
- // used had changed and if so, it deletes soong.environment.used to cause a
- // rebuild.
- //
- // The dependency of build.ninja on soong.environment.used is declared in
- // build.ninja.d
- availableEnvFile := shared.JoinPath(topDir, outDir, "soong.environment.available")
- usedEnvFile := shared.JoinPath(topDir, outDir, "soong.environment.used")
- availableEnv, err := shared.EnvFromFile(availableEnvFile)
- if err != nil {
- fmt.Fprintf(os.Stderr, "error reading available environment file %s: %s\n", availableEnvFile, err)
- os.Exit(1)
- }
+ availableEnv := parseAvailableEnv()
// The top-level Blueprints file is passed as the first argument.
srcDir := filepath.Dir(flag.Arg(0))
@@ -233,37 +260,37 @@
// because that is done from within the actual builds as a Ninja action and
// thus it would overwrite the actual used variables file so this is
// special-cased.
+ // TODO: Fix this by not passing --used_env to the soong_docs invocation
runSoongDocs(configuration, extraNinjaDeps)
return
}
- doChosenActivity(configuration, extraNinjaDeps)
- writeUsedEnvironmentFile(usedEnvFile, configuration)
+ finalOutputFile := doChosenActivity(configuration, extraNinjaDeps)
+ writeUsedEnvironmentFile(configuration, finalOutputFile)
}
-func writeUsedEnvironmentFile(path string, configuration android.Config) {
+func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile string) {
+ if usedEnvFile == "" {
+ return
+ }
+
+ path := shared.JoinPath(topDir, usedEnvFile)
data, err := shared.EnvFileContents(configuration.EnvDeps())
if err != nil {
- fmt.Fprintf(os.Stderr, "error writing used environment file %s: %s\n", path, err)
+ fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err)
os.Exit(1)
}
err = ioutil.WriteFile(path, data, 0666)
if err != nil {
- fmt.Fprintf(os.Stderr, "error writing used environment file %s: %s\n", path, err)
+ fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err)
os.Exit(1)
}
- // Touch the output Ninja file so that it's not older than the file we just
+ // Touch the output file so that it's not older than the file we just
// wrote. We can't write the environment file earlier because one an access
// new environment variables while writing it.
- outputNinjaFile := shared.JoinPath(topDir, bootstrap.CmdlineOutFile())
- currentTime := time.Now().Local()
- err = os.Chtimes(outputNinjaFile, currentTime, currentTime)
- if err != nil {
- fmt.Fprintf(os.Stderr, "error touching output file %s: %s\n", outputNinjaFile, err)
- os.Exit(1)
- }
+ touch(shared.JoinPath(topDir, finalOutputFile))
}
// Workarounds to support running bp2build in a clean AOSP checkout with no
@@ -289,6 +316,27 @@
0666)
}
+func touch(path string) {
+ f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err)
+ os.Exit(1)
+ }
+
+ err = f.Close()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err)
+ os.Exit(1)
+ }
+
+ currentTime := time.Now().Local()
+ err = os.Chtimes(path, currentTime, currentTime)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "error touching '%s': %s\n", path, err)
+ os.Exit(1)
+ }
+}
+
// Run Soong in the bp2build mode. This creates a standalone context that registers
// an alternate pipeline of mutators and singletons specifically for generating
// Bazel BUILD files instead of Ninja files.
@@ -296,6 +344,7 @@
// Register an alternate set of singletons and mutators for bazel
// conversion for Bazel conversion.
bp2buildCtx := android.NewContext(configuration)
+ bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
bp2buildCtx.RegisterForBazelConversion()
// No need to generate Ninja build rules/statements from Modules and Singletons.
@@ -330,5 +379,9 @@
metrics.Print()
extraNinjaDeps = append(extraNinjaDeps, codegenContext.AdditionalNinjaDeps()...)
- writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir())
+ if bp2buildMarker != "" {
+ touch(shared.JoinPath(topDir, bp2buildMarker))
+ } else {
+ writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir())
+ }
}
diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh
index 5271f8d..f85af1a 100755
--- a/tests/bootstrap_test.sh
+++ b/tests/bootstrap_test.sh
@@ -114,7 +114,9 @@
rm a/Android.bp
run_soong
- grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja && fail "Old module in output"
+ if grep -q "^# Module:.*my_little_binary_host$" out/soong/build.ninja; then
+ fail "Old module in output"
+ fi
}
function test_add_file_to_glob() {
@@ -404,7 +406,9 @@
grep -q "Engage" out/soong/build.ninja || fail "New action not present"
- grep -q "Make it so" out/soong/build.ninja && fail "Original action still present"
+ if grep -q "Make it so" out/soong/build.ninja; then
+ fail "Original action still present"
+ fi
}
function test_null_build_after_docs {
@@ -421,6 +425,27 @@
fi
}
+function test_integrated_bp2build_smoke {
+ setup
+ INTEGRATED_BP2BUILD=1 run_soong
+ if [[ ! -e out/soong/.bootstrap/bp2build_workspace_marker ]]; then
+ fail "b2build marker file not created"
+ fi
+}
+
+function test_integrated_bp2build_null_build {
+ setup
+ INTEGRATED_BP2BUILD=1 run_soong
+ local mtime1=$(stat -c "%y" out/soong/build.ninja)
+
+ INTEGRATED_BP2BUILD=1 run_soong
+ local mtime2=$(stat -c "%y" out/soong/build.ninja)
+
+ if [[ "$mtime1" != "$mtime2" ]]; then
+ fail "Output Ninja file changed on null build"
+ fi
+}
+
function test_dump_json_module_graph() {
setup
SOONG_DUMP_JSON_MODULE_GRAPH="$MOCK_TOP/modules.json" run_soong
@@ -441,3 +466,5 @@
test_glob_during_bootstrapping
test_soong_build_rerun_iff_environment_changes
test_dump_json_module_graph
+test_integrated_bp2build_smoke
+test_integrated_bp2build_null_build
diff --git a/ui/build/soong.go b/ui/build/soong.go
index 0089075..d77a089 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -33,6 +33,11 @@
"android/soong/ui/status"
)
+const (
+ availableEnvFile = "soong.environment.available"
+ usedEnvFile = "soong.environment.used"
+)
+
func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
data, err := shared.EnvFileContents(envDeps)
if err != nil {
@@ -87,12 +92,22 @@
return c.debugCompilation
}
-func bootstrapBlueprint(ctx Context, config Config) {
+func environmentArgs(config Config, suffix string) []string {
+ return []string{
+ "--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
+ "--used_env", shared.JoinPath(config.SoongOutDir(), usedEnvFile+suffix),
+ }
+}
+func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
defer ctx.EndTrace()
var args bootstrap.Args
+ mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")
+ globFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
+ bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
+
args.RunGoTests = !config.skipSoongTests
args.UseValidations = true // Use validations to depend on tests
args.BuildDir = config.SoongOutDir()
@@ -101,7 +116,7 @@
args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
args.DepFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
- args.GlobFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
+ args.GlobFile = globFile
args.GeneratingPrimaryBuilder = true
args.DelveListen = os.Getenv("SOONG_DELVE")
@@ -109,6 +124,44 @@
args.DelvePath = shared.ResolveDelveBinary()
}
+ commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, bootstrapGlobFile, mainNinjaFile)
+ bp2BuildMarkerFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
+ mainSoongBuildInputs := []string{"Android.bp"}
+
+ if integratedBp2Build {
+ mainSoongBuildInputs = append(mainSoongBuildInputs, bp2BuildMarkerFile)
+ }
+
+ soongBuildArgs := make([]string, 0)
+ soongBuildArgs = append(soongBuildArgs, commonArgs...)
+ soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)
+ soongBuildArgs = append(soongBuildArgs, "Android.bp")
+
+ mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{
+ Inputs: mainSoongBuildInputs,
+ Outputs: []string{mainNinjaFile},
+ Args: soongBuildArgs,
+ }
+
+ if integratedBp2Build {
+ bp2buildArgs := []string{"--bp2build_marker", bp2BuildMarkerFile}
+ bp2buildArgs = append(bp2buildArgs, commonArgs...)
+ bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
+ bp2buildArgs = append(bp2buildArgs, "Android.bp")
+
+ bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
+ Inputs: []string{"Android.bp"},
+ Outputs: []string{bp2BuildMarkerFile},
+ Args: bp2buildArgs,
+ }
+ args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
+ bp2buildInvocation,
+ mainSoongBuildInvocation,
+ }
+ } else {
+ args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{mainSoongBuildInvocation}
+ }
+
blueprintCtx := blueprint.NewContext()
blueprintCtx.SetIgnoreUnknownModuleTypes(true)
blueprintConfig := BlueprintConfig{
@@ -121,6 +174,16 @@
bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
}
+func checkEnvironmentFile(currentEnv *Environment, envFile string) {
+ getenv := func(k string) string {
+ v, _ := currentEnv.Get(k)
+ return v
+ }
+ if stale, _ := shared.StaleEnvFile(envFile, getenv); stale {
+ os.Remove(envFile)
+ }
+}
+
func runSoong(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "soong")
defer ctx.EndTrace()
@@ -129,7 +192,7 @@
// .used with the ones that were actually used. The latter is used to
// determine whether Soong needs to be re-run since why re-run it if only
// unused variables were changed?
- envFile := filepath.Join(config.SoongOutDir(), "soong.environment.available")
+ envFile := filepath.Join(config.SoongOutDir(), availableEnvFile)
for _, n := range []string{".bootstrap", ".minibootstrap"} {
dir := filepath.Join(config.SoongOutDir(), n)
@@ -138,8 +201,10 @@
}
}
+ integratedBp2Build := config.Environment().IsEnvTrue("INTEGRATED_BP2BUILD")
+
// This is done unconditionally, but does not take a measurable amount of time
- bootstrapBlueprint(ctx, config)
+ bootstrapBlueprint(ctx, config, integratedBp2Build)
soongBuildEnv := config.Environment().Copy()
soongBuildEnv.Set("TOP", os.Getenv("TOP"))
@@ -164,13 +229,12 @@
ctx.BeginTrace(metrics.RunSoong, "environment check")
defer ctx.EndTrace()
- envFile := filepath.Join(config.SoongOutDir(), "soong.environment.used")
- getenv := func(k string) string {
- v, _ := soongBuildEnv.Get(k)
- return v
- }
- if stale, _ := shared.StaleEnvFile(envFile, getenv); stale {
- os.Remove(envFile)
+ soongBuildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile)
+ checkEnvironmentFile(soongBuildEnv, soongBuildEnvFile)
+
+ if integratedBp2Build {
+ bp2buildEnvFile := filepath.Join(config.SoongOutDir(), usedEnvFile+".bp2build")
+ checkEnvironmentFile(soongBuildEnv, bp2buildEnvFile)
}
}()