Make globs compatible with hash-based ninja semantics
Previously, globs worked by having soong_build rewrite a ninja file
that ran the globs, and then dependended on the results of that ninja
file. soong_build also pre-filled their outputs so that it wouldn't
be immediately rerun on the 2nd build.
However, the pre-filling of outputs worked for ninja, because although
it updated their timestamps, the soong ninja file was then touched
by soong_build after that, so the soong_build ninja file was newer
and ninja wouldn't rerun soong. But N2 reruns actions if their inputs'
mtimes change in any way, not just if they're newer. Similarly,
hashed-based ninja implementations could not enforce an order on
file contents, so they would have the same problem.
To fix this, lift the glob checking out of ninja and into soong_ui.
Soong_build will output a globs report file every time it's run, and
every time soong_ui is run it will check the globs file, and if any
globs change, update an input to soong_build. soong_ui is essentially
doing what was done in ninja with bpglob actions before.
Bug: 364749114
Test: m nothing, m nothing again doesn't reanalyze, create a new file under a glob directory, m nothing again reanalyzes
Change-Id: I0dbc5ec58c89b869b59cd0602b82215c4972d799
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index a8be7ec..24a44b4 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -15,7 +15,6 @@
package main
import (
- "bytes"
"encoding/json"
"errors"
"flag"
@@ -29,10 +28,12 @@
"android/soong/android/allowlists"
"android/soong/bp2build"
"android/soong/shared"
+
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/deptools"
"github.com/google/blueprint/metrics"
+ "github.com/google/blueprint/pathtools"
"github.com/google/blueprint/proptools"
androidProtobuf "google.golang.org/protobuf/android"
)
@@ -42,8 +43,6 @@
availableEnvFile string
usedEnvFile string
- globFile string
- globListDir string
delveListen string
delvePath string
@@ -64,8 +63,6 @@
flag.StringVar(&cmdlineArgs.SoongOutDir, "soong_out", "", "Soong output directory (usually $TOP/out/soong)")
flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")
flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
- flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")
- flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
flag.StringVar(&cmdlineArgs.OutDir, "out", "", "the ninja builddir directory")
flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")
@@ -206,20 +203,6 @@
ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile)
}
-func writeBuildGlobsNinjaFile(ctx *android.Context) {
- ctx.EventHandler.Begin("globs_ninja_file")
- defer ctx.EventHandler.End("globs_ninja_file")
-
- globDir := bootstrap.GlobDirectory(ctx.Config().SoongOutDir(), globListDir)
- err := bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{
- GlobLister: ctx.Globs,
- GlobFile: globFile,
- GlobDir: globDir,
- SrcDir: ctx.SrcDir(),
- }, ctx.Config())
- maybeQuit(err, "")
-}
-
func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDeps []string) {
eventHandler.Begin("ninja_deps")
defer eventHandler.End("ninja_deps")
@@ -283,7 +266,9 @@
}
// runSoongOnlyBuild runs the standard Soong build in a number of different modes.
-func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string {
+// It returns the path to the output file (usually the ninja file) and the deps that need
+// to trigger a soong rerun.
+func runSoongOnlyBuild(ctx *android.Context) (string, []string) {
ctx.EventHandler.Begin("soong_build")
defer ctx.EventHandler.End("soong_build")
@@ -299,37 +284,30 @@
ninjaDeps, err := bootstrap.RunBlueprint(cmdlineArgs.Args, stopBefore, ctx.Context, ctx.Config())
maybeQuit(err, "")
- ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
-
- writeBuildGlobsNinjaFile(ctx)
// Convert the Soong module graph into Bazel BUILD files.
switch ctx.Config().BuildMode {
case android.GenerateQueryView:
queryviewMarkerFile := cmdlineArgs.BazelQueryViewDir + ".marker"
runQueryView(cmdlineArgs.BazelQueryViewDir, queryviewMarkerFile, ctx)
- writeDepFile(queryviewMarkerFile, ctx.EventHandler, ninjaDeps)
- return queryviewMarkerFile
+ return queryviewMarkerFile, ninjaDeps
case android.GenerateModuleGraph:
writeJsonModuleGraphAndActions(ctx, cmdlineArgs)
- writeDepFile(cmdlineArgs.ModuleGraphFile, ctx.EventHandler, ninjaDeps)
- return cmdlineArgs.ModuleGraphFile
+ return cmdlineArgs.ModuleGraphFile, ninjaDeps
case android.GenerateDocFile:
// TODO: we could make writeDocs() return the list of documentation files
// written and add them to the .d file. Then soong_docs would be re-run
// whenever one is deleted.
err := writeDocs(ctx, shared.JoinPath(topDir, cmdlineArgs.DocFile))
maybeQuit(err, "error building Soong documentation")
- writeDepFile(cmdlineArgs.DocFile, ctx.EventHandler, ninjaDeps)
- return cmdlineArgs.DocFile
+ return cmdlineArgs.DocFile, ninjaDeps
default:
// The actual output (build.ninja) was written in the RunBlueprint() call
// above
- writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps)
if needToWriteNinjaHint(ctx) {
writeNinjaHint(ctx)
}
- return cmdlineArgs.OutFile
+ return cmdlineArgs.OutFile, ninjaDeps
}
}
@@ -359,6 +337,8 @@
func main() {
flag.Parse()
+ soongStartTime := time.Now()
+
shared.ReexecWithDelveMaybe(delveListen, delvePath)
android.InitSandbox(topDir)
@@ -369,13 +349,6 @@
configuration.SetAllowMissingDependencies()
}
- extraNinjaDeps := []string{configuration.ProductVariablesFileName, usedEnvFile}
- if shared.IsDebugging() {
- // Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is
- // enabled even if it completed successfully.
- extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))
- }
-
// Bypass configuration.Getenv, as LOG_DIR does not need to be dependency tracked. By definition, it will
// change between every CI build, so tracking it would require re-running Soong for every build.
metricsDir := availableEnv["LOG_DIR"]
@@ -393,7 +366,16 @@
ctx.SetIncrementalAnalysis(incremental)
ctx.Register()
- finalOutputFile := runSoongOnlyBuild(ctx, extraNinjaDeps)
+ finalOutputFile, ninjaDeps := runSoongOnlyBuild(ctx)
+
+ ninjaDeps = append(ninjaDeps, usedEnvFile)
+ if shared.IsDebugging() {
+ // Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is
+ // enabled even if it completed successfully.
+ ninjaDeps = append(ninjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))
+ }
+
+ writeDepFile(finalOutputFile, ctx.EventHandler, ninjaDeps)
if ctx.GetIncrementalEnabled() {
data, err := shared.EnvFileContents(configuration.EnvDeps())
@@ -407,6 +389,9 @@
writeUsedEnvironmentFile(configuration)
+ err = writeGlobFile(ctx.EventHandler, finalOutputFile, ctx.Globs(), soongStartTime)
+ maybeQuit(err, "")
+
// Touch the output file so that it's the newest file created by soong_build.
// This is necessary because, if soong_build generated any files which
// are ninja inputs to the main output file, then ninja would superfluously
@@ -423,18 +408,33 @@
data, err := shared.EnvFileContents(configuration.EnvDeps())
maybeQuit(err, "error writing used environment file '%s'\n", usedEnvFile)
- if preexistingData, err := os.ReadFile(path); err != nil {
- if !os.IsNotExist(err) {
- maybeQuit(err, "error reading used environment file '%s'", usedEnvFile)
- }
- } else if bytes.Equal(preexistingData, data) {
- // used environment file is unchanged
- return
- }
- err = os.WriteFile(path, data, 0666)
+ err = pathtools.WriteFileIfChanged(path, data, 0666)
maybeQuit(err, "error writing used environment file '%s'", usedEnvFile)
}
+func writeGlobFile(eventHandler *metrics.EventHandler, finalOutFile string, globs pathtools.MultipleGlobResults, soongStartTime time.Time) error {
+ eventHandler.Begin("writeGlobFile")
+ defer eventHandler.End("writeGlobFile")
+
+ globsFile, err := os.Create(shared.JoinPath(topDir, finalOutFile+".globs"))
+ if err != nil {
+ return err
+ }
+ defer globsFile.Close()
+ globsFileEncoder := json.NewEncoder(globsFile)
+ for _, glob := range globs {
+ if err := globsFileEncoder.Encode(glob); err != nil {
+ return err
+ }
+ }
+
+ return os.WriteFile(
+ shared.JoinPath(topDir, finalOutFile+".globs_time"),
+ []byte(fmt.Sprintf("%d\n", soongStartTime.UnixMicro())),
+ 0666,
+ )
+}
+
func touch(path string) {
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
maybeQuit(err, "Error touching '%s'", path)