Separate the files of bp2build and soong_build.
The following files are forked:
- build-globs.ninja
- The glob list files
- .d files for the glob list files
- The output file (build.ninja or the bp2build marker file)
This makes bp2build and soong_build not overwrite each other's files
they need for proper incrementality.
Test: Presubmits.
Change-Id: I69d192cbd36ecd9677f46f3fa095dfce6f872227
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index 0099e87..16656e4 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -472,10 +472,10 @@
// Generate out/soong/.bootstrap/build-globs.ninja with the actions to generate flattened globfiles
// containing the globs seen during bp2build conversion
if blueprintArgs.GlobFile != "" {
- bootstrap.WriteBuildGlobsNinjaFile(bootstrap.StageMain, bp2buildCtx.Context, blueprintArgs, configuration)
+ bootstrap.WriteBuildGlobsNinjaFile(blueprintArgs.GlobListDir, bp2buildCtx.Context, blueprintArgs, configuration)
}
// Add the depfile on the expanded globs in out/soong/.primary/globs
- ninjaDeps = append(ninjaDeps, bootstrap.GlobFileListFiles(configuration)...)
+ ninjaDeps = append(ninjaDeps, bootstrap.GlobFileListFiles(configuration, blueprintArgs.GlobListDir)...)
// Run the code-generation phase to convert BazelTargetModules to BUILD files
// and print conversion metrics to the user.
@@ -528,12 +528,4 @@
// Create an empty bp2build marker file.
touch(shared.JoinPath(topDir, bp2buildMarker))
-
- // bp2build *always* writes a fake Ninja file containing just the nothing
- // phony target if it ever re-runs. This allows bp2build to exit early with
- // GENERATE_BAZEL_FILES=1 m nothing.
- //
- // If bp2build is invoked as part of an integrated mixed build, the fake
- // build.ninja file will be rewritten later into the real file anyway.
- writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir())
}
diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh
index 1258684..9342d75 100755
--- a/tests/bootstrap_test.sh
+++ b/tests/bootstrap_test.sh
@@ -144,7 +144,7 @@
run_soong
local ninja_mtime1=$(stat -c "%y" out/soong/build.ninja)
- local glob_deps_file=out/soong/.primary/globs/0.d
+ local glob_deps_file=out/soong/.bootstrap/globs/0.d
if [ -e "$glob_deps_file" ]; then
fail "Glob deps file unexpectedly written on first build"
@@ -526,18 +526,14 @@
[[ -e out/soong/workspace ]] || fail "Bazel workspace not created"
}
-function test_bp2build_generates_fake_ninja_file {
+function test_bp2build_generates_marker_file {
setup
create_mock_bazel
run_bp2build
- if [[ ! -f "./out/soong/build.ninja" ]]; then
- fail "./out/soong/build.ninja was not generated"
- fi
-
- if ! grep "build nothing: phony" "./out/soong/build.ninja"; then
- fail "missing phony nothing target in out/soong/build.ninja"
+ if [[ ! -f "./out/soong/.bootstrap/bp2build_workspace_marker" ]]; then
+ fail "Marker file was not generated"
fi
}
@@ -577,10 +573,10 @@
setup
GENERATE_BAZEL_FILES=1 run_soong
- local mtime1=$(stat -c "%y" out/soong/build.ninja)
+ local mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
GENERATE_BAZEL_FILES=1 run_soong
- local mtime2=$(stat -c "%y" out/soong/build.ninja)
+ local mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
if [[ "$mtime1" != "$mtime2" ]]; then
fail "Output Ninja file changed on null build"
@@ -712,6 +708,41 @@
grep -q "b/${GENERATED_BUILD_FILE_NAME}' exist" "$MOCK_TOP/errors" || fail "Error for b/${GENERATED_BUILD_FILE_NAME} not found"
}
+function test_bp2build_back_and_forth_null_build {
+ setup
+
+ run_soong
+ local output_mtime1=$(stat -c "%y" out/soong/build.ninja)
+
+ GENERATE_BAZEL_FILES=1 run_soong
+ local output_mtime2=$(stat -c "%y" out/soong/build.ninja)
+ if [[ "$output_mtime1" != "$output_mtime2" ]]; then
+ fail "Output Ninja file changed when switching to bp2build"
+ fi
+
+ local marker_mtime1=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
+
+ run_soong
+ local output_mtime3=$(stat -c "%y" out/soong/build.ninja)
+ local marker_mtime2=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
+ if [[ "$output_mtime1" != "$output_mtime3" ]]; then
+ fail "Output Ninja file changed when switching to regular build from bp2build"
+ fi
+ if [[ "$marker_mtime1" != "$marker_mtime2" ]]; then
+ fail "bp2build marker file changed when switching to regular build from bp2build"
+ fi
+
+ GENERATE_BAZEL_FILES=1 run_soong
+ local output_mtime4=$(stat -c "%y" out/soong/build.ninja)
+ local marker_mtime3=$(stat -c "%y" out/soong/.bootstrap/bp2build_workspace_marker)
+ if [[ "$output_mtime1" != "$output_mtime4" ]]; then
+ fail "Output Ninja file changed when switching back to bp2build"
+ fi
+ if [[ "$marker_mtime1" != "$marker_mtime3" ]]; then
+ fail "bp2build marker file changed when switching back to bp2build"
+ fi
+}
+
test_smoke
test_null_build
test_null_build_after_docs
@@ -727,8 +758,9 @@
test_dump_json_module_graph
test_write_to_source_tree
test_bp2build_smoke
-test_bp2build_generates_fake_ninja_file
+test_bp2build_generates_marker_file
test_bp2build_null_build
+test_bp2build_back_and_forth_null_build
test_bp2build_add_android_bp
test_bp2build_add_to_glob
test_bp2build_bazel_workspace_structure
diff --git a/tests/bp2build_bazel_test.sh b/tests/bp2build_bazel_test.sh
index b5c6863..9bd85a4 100755
--- a/tests/bp2build_bazel_test.sh
+++ b/tests/bp2build_bazel_test.sh
@@ -48,60 +48,6 @@
test_bp2build_null_build_with_globs
-function test_soong_after_bp2build_regenerates_build_globs_ninja() {
- setup
-
- mkdir -p foo/bar
- cat > foo/bar/Android.bp <<'EOF'
-filegroup {
- name: "bp2build-files",
- srcs: ["bp2build.*"],
- bazel_module: { bp2build_available: true },
-}
-
-filegroup {
- name: "soong-files",
- srcs: ["soong.*"],
-}
-EOF
- touch foo/bar/bp2build.txt foo/bar/soong.txt
-
- build_globs_file="out/soong/.bootstrap/build-globs.ninja"
-
- # Test: the build-globs file for bp2build should only contain the bp2build-files
- # glob, whereas the build-globs file for soong should contain both bp2build-files
- # and soong-files globs.
-
- run_bp2build
- local output_mtime1=$(stat -c "%y" "${build_globs_file}")
-
- if ! grep "\"foo/bar/bp2build.*\"" "${build_globs_file}"; then
- fail "bp2build filegroup globs not found in bp2build's globs file"
- fi
-
- if grep "\"foo/bar/soong.*\"" "${build_globs_file}"; then
- fail "soong filegroup globs unexpectedly found in bp2build's globs file"
- fi
-
- run_soong
- local output_mtime2=$(stat -c "%y" "${build_globs_file}")
-
- if [[ "$output_mtime1" == "$output_mtime2" ]]; then
- fail "Output build-globs.ninja file did not change"
- fi
-
- if ! grep "\"foo/bar/bp2build.*\"" "${build_globs_file}"; then
- fail "bp2build filegroup globs not found in bp2build's globs file"
- fi
-
- if ! grep "\"foo/bar/soong.*\"" "${build_globs_file}"; then
- fail "soong filegroup globs not found in bp2build's globs file"
- fi
-
-}
-
-test_soong_after_bp2build_regenerates_build_globs_ninja
-
function test_bp2build_generates_all_buildfiles {
setup
create_mock_bazel
diff --git a/ui/build/config.go b/ui/build/config.go
index 9768a44..956406d 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -739,6 +739,14 @@
return filepath.Join(c.OutDir(), "soong")
}
+func (c *configImpl) MainNinjaFile() string {
+ return shared.JoinPath(c.SoongOutDir(), "build.ninja")
+}
+
+func (c *configImpl) Bp2BuildMarkerFile() string {
+ return shared.JoinPath(c.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
+}
+
func (c *configImpl) TempDir() string {
return shared.TempDirForOutDir(c.SoongOutDir())
}
diff --git a/ui/build/soong.go b/ui/build/soong.go
index 190c955..70eabc0 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -114,17 +114,20 @@
}
}
-func bootstrapBlueprint(ctx Context, config Config, integratedBp2Build bool) {
+func bootstrapBlueprint(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
defer ctx.EndTrace()
var args bootstrap.Args
- mainNinjaFile := shared.JoinPath(config.SoongOutDir(), "build.ninja")
bootstrapGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.ninja")
- // .bootstrap/build.ninja "includes" .bootstrap/build-globs.ninja for incremental builds
- // generate an empty glob before running any rule in .bootstrap/build.ninja
+ bp2buildGlobFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build-globs.bp2build.ninja")
+
+ // The glob .ninja files are subninja'd. However, they are generated during
+ // the build itself so we write an empty file so that the subninja doesn't
+ // fail on clean builds
writeEmptyGlobFile(ctx, bootstrapGlobFile)
+ writeEmptyGlobFile(ctx, bp2buildGlobFile)
bootstrapDepFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
args.RunGoTests = !config.skipSoongTests
@@ -137,7 +140,7 @@
// The primary builder (aka soong_build) will use bootstrapGlobFile as the globFile to generate build.ninja(.d)
// Building soong_build does not require a glob file
// Using "" instead of "<soong_build_glob>.ninja" will ensure that an unused glob file is not written to out/soong/.bootstrap during StagePrimary
- args.GlobFile = ""
+ args.Subninjas = []string{bootstrapGlobFile, bp2buildGlobFile}
args.GeneratingPrimaryBuilder = true
args.EmptyNinjaFile = config.EmptyNinjaFile()
@@ -146,42 +149,46 @@
args.DelvePath = shared.ResolveDelveBinary()
}
- commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, bootstrapGlobFile, mainNinjaFile)
- bp2BuildMarkerFile := shared.JoinPath(config.SoongOutDir(), ".bootstrap/bp2build_workspace_marker")
+ commonArgs := bootstrap.PrimaryBuilderExtraFlags(args, config.MainNinjaFile())
mainSoongBuildInputs := []string{"Android.bp"}
- if integratedBp2Build {
- mainSoongBuildInputs = append(mainSoongBuildInputs, bp2BuildMarkerFile)
+ if config.bazelBuildMode() == mixedBuild {
+ mainSoongBuildInputs = append(mainSoongBuildInputs, config.Bp2BuildMarkerFile())
}
- soongBuildArgs := make([]string, 0)
+ soongBuildArgs := []string{
+ "--globListDir", "globs",
+ "--globFile", bootstrapGlobFile,
+ }
+
soongBuildArgs = append(soongBuildArgs, commonArgs...)
soongBuildArgs = append(soongBuildArgs, environmentArgs(config, "")...)
soongBuildArgs = append(soongBuildArgs, "Android.bp")
mainSoongBuildInvocation := bootstrap.PrimaryBuilderInvocation{
Inputs: mainSoongBuildInputs,
- Outputs: []string{mainNinjaFile},
+ Outputs: []string{config.MainNinjaFile()},
Args: soongBuildArgs,
}
- if integratedBp2Build {
- bp2buildArgs := []string{"--bp2build_marker", bp2BuildMarkerFile}
- bp2buildArgs = append(bp2buildArgs, commonArgs...)
- bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
- bp2buildArgs = append(bp2buildArgs, "Android.bp")
+ bp2buildArgs := []string{
+ "--bp2build_marker", config.Bp2BuildMarkerFile(),
+ "--globListDir", "globs.bp2build",
+ "--globFile", bp2buildGlobFile,
+ }
- bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
- Inputs: []string{"Android.bp"},
- Outputs: []string{bp2BuildMarkerFile},
- Args: bp2buildArgs,
- }
- args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{bp2buildInvocation}
- if config.bazelBuildMode() == mixedBuild {
- args.PrimaryBuilderInvocations = append(args.PrimaryBuilderInvocations, mainSoongBuildInvocation)
- }
- } else {
- args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{mainSoongBuildInvocation}
+ bp2buildArgs = append(bp2buildArgs, commonArgs...)
+ bp2buildArgs = append(bp2buildArgs, environmentArgs(config, ".bp2build")...)
+ bp2buildArgs = append(bp2buildArgs, "Android.bp")
+
+ bp2buildInvocation := bootstrap.PrimaryBuilderInvocation{
+ Inputs: []string{"Android.bp"},
+ Outputs: []string{config.Bp2BuildMarkerFile()},
+ Args: bp2buildArgs,
+ }
+ args.PrimaryBuilderInvocations = []bootstrap.PrimaryBuilderInvocation{
+ bp2buildInvocation,
+ mainSoongBuildInvocation,
}
blueprintCtx := blueprint.NewContext()
@@ -231,7 +238,7 @@
integratedBp2Build := (buildMode == mixedBuild) || (buildMode == generateBuildFiles)
// This is done unconditionally, but does not take a measurable amount of time
- bootstrapBlueprint(ctx, config, integratedBp2Build)
+ bootstrapBlueprint(ctx, config)
soongBuildEnv := config.Environment().Copy()
soongBuildEnv.Set("TOP", os.Getenv("TOP"))
@@ -268,7 +275,7 @@
runMicrofactory(ctx, config, ".minibootstrap/bpglob", "github.com/google/blueprint/bootstrap/bpglob",
map[string]string{"github.com/google/blueprint": "build/blueprint"})
- ninja := func(name, file string) {
+ ninja := func(name, ninjaFile string, targets ...string) {
ctx.BeginTrace(metrics.RunSoong, name)
defer ctx.EndTrace()
@@ -276,8 +283,7 @@
nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
defer nr.Close()
- cmd := Command(ctx, config, "soong "+name,
- config.PrebuiltBuildTool("ninja"),
+ ninjaArgs := []string{
"-d", "keepdepfile",
"-d", "stats",
"-o", "usesphonyoutputs=yes",
@@ -287,7 +293,12 @@
"-w", "missingoutfile=err",
"-j", strconv.Itoa(config.Parallel()),
"--frontend_file", fifo,
- "-f", filepath.Join(config.SoongOutDir(), file))
+ "-f", filepath.Join(config.SoongOutDir(), ninjaFile),
+ }
+
+ ninjaArgs = append(ninjaArgs, targets...)
+ cmd := Command(ctx, config, "soong "+name,
+ config.PrebuiltBuildTool("ninja"), ninjaArgs...)
var ninjaEnv Environment
@@ -299,8 +310,17 @@
cmd.Sandbox = soongSandbox
cmd.RunAndStreamOrFatal()
}
- // This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
- ninja("bootstrap", ".bootstrap/build.ninja")
+
+ var target string
+
+ if config.bazelBuildMode() == generateBuildFiles {
+ target = config.Bp2BuildMarkerFile()
+ } else {
+ // This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
+ target = config.MainNinjaFile()
+ }
+
+ ninja("bootstrap", ".bootstrap/build.ninja", target)
var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics
if shouldCollectBuildSoongMetrics(config) {