Move bpglob out of ToolDir
ToolDir is going to become unstable when switching between KatiEnabled
and Soong-only builds while the duplication between out/soong/host and
out/host is resolved. bpglob gets executed very early during bootstrap,
before the primary builder has run to update the paths to match the
current configuration. Move it into SoongOutDir() so that its path
is more stable.
Moving bpglob causes incremental build issues, so add a bootstrapEpoch
constant that can be changed when making incompatible changes and will
wipe bootstrap intermediates.
Bug: 204136549
Test: m nothing
Change-Id: I7b1bd1ebfe1209d11db691b3ee00873ef92658cd
diff --git a/ui/build/soong.go b/ui/build/soong.go
index a0f223b..1c7fbac 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -15,6 +15,7 @@
package build
import (
+ "fmt"
"io/ioutil"
"os"
"path/filepath"
@@ -43,6 +44,12 @@
jsonModuleGraphTag = "modulegraph"
queryviewTag = "queryview"
soongDocsTag = "soong_docs"
+
+ // bootstrapEpoch is used to determine if an incremental build is incompatible with the current
+ // version of bootstrap and needs cleaning before continuing the build. Increment this for
+ // incompatible changes, for example when moving the location of the bpglob binary that is
+ // executed during bootstrap before the primary builder has had a chance to update the path.
+ bootstrapEpoch = 0
)
func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
@@ -121,20 +128,31 @@
}
}
-func writeEmptyGlobFile(ctx Context, path string) {
+func writeEmptyFile(ctx Context, path string) {
err := os.MkdirAll(filepath.Dir(path), 0777)
if err != nil {
- ctx.Fatalf("Failed to create parent directories of empty ninja glob file '%s': %s", path, err)
+ ctx.Fatalf("Failed to create parent directories of empty file '%s': %s", path, err)
}
- if _, err := os.Stat(path); os.IsNotExist(err) {
+ if exists, err := fileExists(path); err != nil {
+ ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err)
+ } else if !exists {
err = ioutil.WriteFile(path, nil, 0666)
if err != nil {
- ctx.Fatalf("Failed to create empty ninja glob file '%s': %s", path, err)
+ ctx.Fatalf("Failed to create empty file '%s': %s", path, err)
}
}
}
+func fileExists(path string) (bool, error) {
+ if _, err := os.Stat(path); os.IsNotExist(err) {
+ return false, nil
+ } else if err != nil {
+ return false, err
+ }
+ return true, nil
+}
+
func primaryBuilderInvocation(config Config, name string, output string, specificArgs []string) bootstrap.PrimaryBuilderInvocation {
commonArgs := make([]string, 0, 0)
@@ -166,10 +184,45 @@
}
}
+// bootstrapEpochCleanup deletes files used by bootstrap during incremental builds across
+// incompatible changes. Incompatible changes are marked by incrementing the bootstrapEpoch
+// constant. A tree is considered out of date for the current epoch of the
+// .soong.bootstrap.epoch.<epoch> file doesn't exist.
+func bootstrapEpochCleanup(ctx Context, config Config) {
+ epochFile := fmt.Sprintf(".soong.bootstrap.epoch.%d", bootstrapEpoch)
+ epochPath := filepath.Join(config.SoongOutDir(), epochFile)
+ if exists, err := fileExists(epochPath); err != nil {
+ ctx.Fatalf("failed to check if bootstrap epoch file %q exists: %q", epochPath, err)
+ } else if !exists {
+ // The tree is out of date for the current epoch, delete files used by bootstrap
+ // and force the primary builder to rerun.
+ os.Remove(filepath.Join(config.SoongOutDir(), "build.ninja"))
+ for _, globFile := range bootstrapGlobFileList(config) {
+ os.Remove(globFile)
+ }
+
+ // Mark the tree as up to date with the current epoch by writing the epoch marker file.
+ writeEmptyFile(ctx, epochPath)
+ }
+}
+
+func bootstrapGlobFileList(config Config) []string {
+ return []string{
+ config.NamedGlobFile(soongBuildTag),
+ config.NamedGlobFile(bp2buildTag),
+ config.NamedGlobFile(jsonModuleGraphTag),
+ config.NamedGlobFile(queryviewTag),
+ config.NamedGlobFile(soongDocsTag),
+ }
+}
+
func bootstrapBlueprint(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
defer ctx.EndTrace()
+ // Clean up some files for incremental builds across incompatible changes.
+ bootstrapEpochCleanup(ctx, config)
+
mainSoongBuildExtraArgs := []string{"-o", config.SoongNinjaFile()}
if config.EmptyNinjaFile() {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
@@ -232,8 +285,8 @@
// The glob .ninja files are subninja'd. However, they are generated during
// the build itself so we write an empty file if the file does not exist yet
// so that the subninja doesn't fail on clean builds
- for _, globFile := range globFiles {
- writeEmptyGlobFile(ctx, globFile)
+ for _, globFile := range bootstrapGlobFileList(config) {
+ writeEmptyFile(ctx, globFile)
}
var blueprintArgs bootstrap.Args
@@ -342,7 +395,7 @@
}
}()
- runMicrofactory(ctx, config, filepath.Join(config.HostToolDir(), "bpglob"), "github.com/google/blueprint/bootstrap/bpglob",
+ runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob",
map[string]string{"github.com/google/blueprint": "build/blueprint"})
ninja := func(name, ninjaFile string, targets ...string) {