Merge changes I1fb78d7c,Ic836282f
* changes:
Background distGzipFile to speed up CI builds
Fix some problems with soong metrics loading
diff --git a/ui/build/build.go b/ui/build/build.go
index d261f89..aadf4af 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -18,6 +18,7 @@
"io/ioutil"
"os"
"path/filepath"
+ "sync"
"text/template"
"android/soong/ui/metrics"
@@ -205,6 +206,8 @@
return
}
+ defer waitForDist(ctx)
+
// checkProblematicFiles aborts the build if Android.mk or CleanSpec.mk are found at the root of the tree.
checkProblematicFiles(ctx)
@@ -329,8 +332,18 @@
}
}
+var distWaitGroup sync.WaitGroup
+
+// waitForDist waits for all backgrounded distGzipFile and distFile writes to finish
+func waitForDist(ctx Context) {
+ ctx.BeginTrace("soong_ui", "dist")
+ defer ctx.EndTrace()
+
+ distWaitGroup.Wait()
+}
+
// distGzipFile writes a compressed copy of src to the distDir if dist is enabled. Failures
-// are printed but non-fatal.
+// are printed but non-fatal. Uses the distWaitGroup func for backgrounding (optimization).
func distGzipFile(ctx Context, config Config, src string, subDirs ...string) {
if !config.Dist() {
return
@@ -343,13 +356,17 @@
ctx.Printf("failed to mkdir %s: %s", destDir, err.Error())
}
- if err := gzipFileToDir(src, destDir); err != nil {
- ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error())
- }
+ distWaitGroup.Add(1)
+ go func() {
+ defer distWaitGroup.Done()
+ if err := gzipFileToDir(src, destDir); err != nil {
+ ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error())
+ }
+ }()
}
// distFile writes a copy of src to the distDir if dist is enabled. Failures are printed but
-// non-fatal.
+// non-fatal. Uses the distWaitGroup func for backgrounding (optimization).
func distFile(ctx Context, config Config, src string, subDirs ...string) {
if !config.Dist() {
return
@@ -362,7 +379,11 @@
ctx.Printf("failed to mkdir %s: %s", destDir, err.Error())
}
- if _, err := copyFile(src, filepath.Join(destDir, filepath.Base(src))); err != nil {
- ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error())
- }
+ distWaitGroup.Add(1)
+ go func() {
+ defer distWaitGroup.Done()
+ if _, err := copyFile(src, filepath.Join(destDir, filepath.Base(src))); err != nil {
+ ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error())
+ }
+ }()
}
diff --git a/ui/build/soong.go b/ui/build/soong.go
index c7f22f9..8992b4f 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -15,7 +15,9 @@
package build
import (
+ "errors"
"fmt"
+ "io/fs"
"io/ioutil"
"os"
"path/filepath"
@@ -491,10 +493,14 @@
ninja("bootstrap", "bootstrap.ninja", targets...)
- var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics
if shouldCollectBuildSoongMetrics(config) {
soongBuildMetrics := loadSoongBuildMetrics(ctx, config)
- logSoongBuildMetrics(ctx, soongBuildMetrics)
+ if soongBuildMetrics != nil {
+ logSoongBuildMetrics(ctx, soongBuildMetrics)
+ if ctx.Metrics != nil {
+ ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics)
+ }
+ }
}
distGzipFile(ctx, config, config.SoongNinjaFile(), "soong")
@@ -504,9 +510,6 @@
distGzipFile(ctx, config, config.SoongMakeVarsMk(), "soong")
}
- if shouldCollectBuildSoongMetrics(config) && ctx.Metrics != nil {
- ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics)
- }
if config.JsonModuleGraph() {
distGzipFile(ctx, config, config.ModuleGraphFile(), "soong")
}
@@ -538,8 +541,12 @@
func loadSoongBuildMetrics(ctx Context, config Config) *soong_metrics_proto.SoongBuildMetrics {
soongBuildMetricsFile := filepath.Join(config.LogsDir(), "soong_build_metrics.pb")
- buf, err := ioutil.ReadFile(soongBuildMetricsFile)
- if err != nil {
+ buf, err := os.ReadFile(soongBuildMetricsFile)
+ if errors.Is(err, fs.ErrNotExist) {
+ // Soong may not have run during this invocation
+ ctx.Verbosef("Failed to read metrics file, %s: %s", soongBuildMetricsFile, err)
+ return nil
+ } else if err != nil {
ctx.Fatalf("Failed to load %s: %s", soongBuildMetricsFile, err)
}
soongBuildMetrics := &soong_metrics_proto.SoongBuildMetrics{}