Merge "Allow adding extra tradefed options in the Android.bp file"
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index 744a10c..fe928ad 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -277,7 +277,7 @@
switch ctx.Config().BuildMode {
case android.GenerateModuleGraph:
stopBefore = bootstrap.StopBeforeWriteNinja
- case android.GenerateQueryView | android.GenerateDocFile:
+ case android.GenerateQueryView, android.GenerateDocFile:
stopBefore = bootstrap.StopBeforePrepareBuildActions
default:
stopBefore = bootstrap.DoEverything
diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go
index 713ccbe..8c99988 100644
--- a/cmd/soong_ui/main.go
+++ b/cmd/soong_ui/main.go
@@ -58,7 +58,7 @@
stdio func() terminal.StdioInterface
// run the command
- run func(ctx build.Context, config build.Config, args []string, logsDir string)
+ run func(ctx build.Context, config build.Config, args []string)
}
// list of supported commands (flags) supported by soong ui
@@ -91,6 +91,15 @@
config: buildActionConfig,
stdio: stdio,
run: runMake,
+ }, {
+ flag: "--upload-metrics-only",
+ description: "upload metrics without building anything",
+ config: uploadOnlyConfig,
+ stdio: stdio,
+ // Upload-only mode mostly skips to the metrics-uploading phase of soong_ui.
+ // However, this invocation marks the true "end of the build", and thus we
+ // need to update the total runtime of the build to include this upload step.
+ run: updateTotalRealTime,
},
}
@@ -182,18 +191,49 @@
}}
config := c.config(buildCtx, args...)
+ config.SetLogsPrefix(c.logsPrefix)
+ logsDir := config.LogsDir()
+ buildStarted = config.BuildStartedTimeOrDefault(buildStarted)
+ buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
+ soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
+ rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
+ bp2buildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bp2build_metrics.pb")
+ metricsFiles := []string{
+ buildErrorFile, // build error strings
+ rbeMetricsFile, // high level metrics related to remote build execution.
+ bp2buildMetricsFile, // high level metrics related to bp2build.
+ soongMetricsFile, // high level metrics related to this build system.
+ config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
+ }
+
+ os.MkdirAll(logsDir, 0777)
+
+ log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log"))
+
+ trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace"))
+
+ c.run(buildCtx, config, args)
+
+ defer met.Dump(soongMetricsFile)
+ if !config.SkipMetricsUpload() {
+ defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, metricsFiles...)
+ }
+
+}
+
+func logAndSymlinkSetup(buildCtx build.Context, config build.Config) {
+ log := buildCtx.ContextImpl.Logger
+ logsPrefix := config.GetLogsPrefix()
build.SetupOutDir(buildCtx, config)
-
- // Set up files to be outputted in the log directory.
logsDir := config.LogsDir()
// Common list of metric file definition.
- buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
- rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
- soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
- bp2buildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bp2build_metrics.pb")
- soongBuildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_build_metrics.pb")
+ buildErrorFile := filepath.Join(logsDir, logsPrefix+"build_error")
+ rbeMetricsFile := filepath.Join(logsDir, logsPrefix+"rbe_metrics.pb")
+ soongMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_metrics")
+ bp2buildMetricsFile := filepath.Join(logsDir, logsPrefix+"bp2build_metrics.pb")
+ soongBuildMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_build_metrics.pb")
//Delete the stale metrics files
staleFileSlice := []string{buildErrorFile, rbeMetricsFile, soongMetricsFile, bp2buildMetricsFile, soongBuildMetricsFile}
@@ -203,14 +243,12 @@
build.PrintOutDirWarning(buildCtx, config)
- os.MkdirAll(logsDir, 0777)
- log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log"))
- trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace"))
- stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, c.logsPrefix+"verbose.log")))
- stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"error.log")))
+ stat := buildCtx.Status
+ stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, logsPrefix+"verbose.log")))
+ stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, logsPrefix+"error.log")))
stat.AddOutput(status.NewProtoErrorLog(log, buildErrorFile))
stat.AddOutput(status.NewCriticalPath(log))
- stat.AddOutput(status.NewBuildProgressLog(log, filepath.Join(logsDir, c.logsPrefix+"build_progress.pb")))
+ stat.AddOutput(status.NewBuildProgressLog(log, filepath.Join(logsDir, logsPrefix+"build_progress.pb")))
buildCtx.Verbosef("Detected %.3v GB total RAM", float32(config.TotalRAM())/(1024*1024*1024))
buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v",
@@ -218,27 +256,7 @@
setMaxFiles(buildCtx)
- {
- // The order of the function calls is important. The last defer function call
- // is the first one that is executed to save the rbe metrics to a protobuf
- // file. The soong metrics file is then next. Bazel profiles are written
- // before the uploadMetrics is invoked. The written files are then uploaded
- // if the uploading of the metrics is enabled.
- files := []string{
- buildErrorFile, // build error strings
- rbeMetricsFile, // high level metrics related to remote build execution.
- soongBuildMetricsFile, // high level metrics related to soong build(except bp2build).
- bp2buildMetricsFile, // high level metrics related to bp2build.
- soongMetricsFile, // high level metrics related to this build system.
- config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
- }
-
- if !config.SkipMetricsUpload() {
- defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, files...)
- }
- defer met.Dump(soongMetricsFile)
- defer build.CheckProdCreds(buildCtx, config)
- }
+ defer build.CheckProdCreds(buildCtx, config)
// Read the time at the starting point.
if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
@@ -253,7 +271,7 @@
}
if executable, err := os.Executable(); err == nil {
- trace.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace"))
+ buildCtx.ContextImpl.Tracer.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace"))
}
}
@@ -266,8 +284,6 @@
f := build.NewSourceFinder(buildCtx, config)
defer f.Shutdown()
build.FindSources(buildCtx, config, f)
-
- c.run(buildCtx, config, args, logsDir)
}
func fixBadDanglingLink(ctx build.Context, name string) {
@@ -284,7 +300,8 @@
}
}
-func dumpVar(ctx build.Context, config build.Config, args []string, _ string) {
+func dumpVar(ctx build.Context, config build.Config, args []string) {
+ logAndSymlinkSetup(ctx, config)
flags := flag.NewFlagSet("dumpvar", flag.ExitOnError)
flags.SetOutput(ctx.Writer)
@@ -336,7 +353,9 @@
}
}
-func dumpVars(ctx build.Context, config build.Config, args []string, _ string) {
+func dumpVars(ctx build.Context, config build.Config, args []string) {
+ logAndSymlinkSetup(ctx, config)
+
flags := flag.NewFlagSet("dumpvars", flag.ExitOnError)
flags.SetOutput(ctx.Writer)
@@ -421,6 +440,14 @@
return build.NewConfig(ctx)
}
+// uploadOnlyConfig explicitly requires no arguments.
+func uploadOnlyConfig(ctx build.Context, args ...string) build.Config {
+ if len(args) > 0 {
+ fmt.Printf("--upload-only does not require arguments.")
+ }
+ return build.UploadOnlyConfig(ctx)
+}
+
func buildActionConfig(ctx build.Context, args ...string) build.Config {
flags := flag.NewFlagSet("build-mode", flag.ContinueOnError)
flags.SetOutput(ctx.Writer)
@@ -514,7 +541,9 @@
return build.NewBuildActionConfig(buildAction, *dir, ctx, args...)
}
-func runMake(ctx build.Context, config build.Config, _ []string, logsDir string) {
+func runMake(ctx build.Context, config build.Config, _ []string) {
+ logAndSymlinkSetup(ctx, config)
+ logsDir := config.LogsDir()
if config.IsVerbose() {
writer := ctx.Writer
fmt.Fprintln(writer, "! The argument `showcommands` is no longer supported.")
@@ -659,3 +688,19 @@
ctx.Println("Failed to increase file limit:", err)
}
}
+
+func updateTotalRealTime(ctx build.Context, config build.Config, args []string) {
+ soongMetricsFile := filepath.Join(config.LogsDir(), "soong_metrics")
+
+ //read file into proto
+ data, err := os.ReadFile(soongMetricsFile)
+ if err != nil {
+ ctx.Fatal(err)
+ }
+ met := ctx.ContextImpl.Metrics
+
+ err = met.UpdateTotalRealTime(data)
+ if err != nil {
+ ctx.Fatal(err)
+ }
+}
diff --git a/java/base.go b/java/base.go
index 55d77dc..6e42aef 100644
--- a/java/base.go
+++ b/java/base.go
@@ -1488,7 +1488,14 @@
}
// Dex compilation
var dexOutputFile android.OutputPath
- dexOutputFile = j.dexer.compileDex(ctx, flags, j.MinSdkVersion(ctx), implementationAndResourcesJar, jarName)
+ params := &compileDexParams{
+ flags: flags,
+ sdkVersion: j.SdkVersion(ctx),
+ minSdkVersion: j.MinSdkVersion(ctx),
+ classesJar: implementationAndResourcesJar,
+ jarName: jarName,
+ }
+ dexOutputFile = j.dexer.compileDex(ctx, params)
if ctx.Failed() {
return
}
diff --git a/java/dex.go b/java/dex.go
index 40ee99d..ed1f07b 100644
--- a/java/dex.go
+++ b/java/dex.go
@@ -180,7 +180,7 @@
"r8Flags", "zipFlags", "tmpJar", "mergeZipsFlags"}, []string{"implicits"})
func (d *dexer) dexCommonFlags(ctx android.ModuleContext,
- minSdkVersion android.SdkSpec) (flags []string, deps android.Paths) {
+ dexParams *compileDexParams) (flags []string, deps android.Paths) {
flags = d.dexProperties.Dxflags
// Translate all the DX flags to D8 ones until all the build files have been migrated
@@ -209,11 +209,11 @@
// Note: Targets with a min SDK kind of core_platform (e.g., framework.jar) or unspecified (e.g.,
// services.jar), are not classified as stable, which is WAI.
// TODO(b/232073181): Expand to additional min SDK cases after validation.
- if !minSdkVersion.Stable() {
+ if !dexParams.sdkVersion.Stable() {
flags = append(flags, "--android-platform-build")
}
- effectiveVersion, err := minSdkVersion.EffectiveVersion(ctx)
+ effectiveVersion, err := dexParams.minSdkVersion.EffectiveVersion(ctx)
if err != nil {
ctx.PropertyErrorf("min_sdk_version", "%s", err)
}
@@ -317,20 +317,27 @@
return r8Flags, r8Deps
}
-func (d *dexer) compileDex(ctx android.ModuleContext, flags javaBuilderFlags, minSdkVersion android.SdkSpec,
- classesJar android.Path, jarName string) android.OutputPath {
+type compileDexParams struct {
+ flags javaBuilderFlags
+ sdkVersion android.SdkSpec
+ minSdkVersion android.SdkSpec
+ classesJar android.Path
+ jarName string
+}
+
+func (d *dexer) compileDex(ctx android.ModuleContext, dexParams *compileDexParams) android.OutputPath {
// Compile classes.jar into classes.dex and then javalib.jar
- javalibJar := android.PathForModuleOut(ctx, "dex", jarName).OutputPath
+ javalibJar := android.PathForModuleOut(ctx, "dex", dexParams.jarName).OutputPath
outDir := android.PathForModuleOut(ctx, "dex")
- tmpJar := android.PathForModuleOut(ctx, "withres-withoutdex", jarName)
+ tmpJar := android.PathForModuleOut(ctx, "withres-withoutdex", dexParams.jarName)
zipFlags := "--ignore_missing_files"
if proptools.Bool(d.dexProperties.Uncompress_dex) {
zipFlags += " -L 0"
}
- commonFlags, commonDeps := d.dexCommonFlags(ctx, minSdkVersion)
+ commonFlags, commonDeps := d.dexCommonFlags(ctx, dexParams)
// Exclude kotlinc generated files when "exclude_kotlinc_generated_files" is set to true.
mergeZipsFlags := ""
@@ -347,7 +354,7 @@
android.ModuleNameWithPossibleOverride(ctx), "unused.txt")
proguardUsageZip := android.PathForModuleOut(ctx, "proguard_usage.zip")
d.proguardUsageZip = android.OptionalPathForPath(proguardUsageZip)
- r8Flags, r8Deps := d.r8Flags(ctx, flags)
+ r8Flags, r8Deps := d.r8Flags(ctx, dexParams.flags)
r8Deps = append(r8Deps, commonDeps...)
rule := r8
args := map[string]string{
@@ -370,12 +377,12 @@
Description: "r8",
Output: javalibJar,
ImplicitOutputs: android.WritablePaths{proguardDictionary, proguardUsageZip},
- Input: classesJar,
+ Input: dexParams.classesJar,
Implicits: r8Deps,
Args: args,
})
} else {
- d8Flags, d8Deps := d8Flags(flags)
+ d8Flags, d8Deps := d8Flags(dexParams.flags)
d8Deps = append(d8Deps, commonDeps...)
rule := d8
if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_D8") {
@@ -385,7 +392,7 @@
Rule: rule,
Description: "d8",
Output: javalibJar,
- Input: classesJar,
+ Input: dexParams.classesJar,
Implicits: d8Deps,
Args: map[string]string{
"d8Flags": strings.Join(append(commonFlags, d8Flags...), " "),
@@ -397,7 +404,7 @@
})
}
if proptools.Bool(d.dexProperties.Uncompress_dex) {
- alignedJavalibJar := android.PathForModuleOut(ctx, "aligned", jarName).OutputPath
+ alignedJavalibJar := android.PathForModuleOut(ctx, "aligned", dexParams.jarName).OutputPath
TransformZipAlign(ctx, alignedJavalibJar, javalibJar)
javalibJar = alignedJavalibJar
}
diff --git a/java/dex_test.go b/java/dex_test.go
index fc6cd0f..cddd4ad 100644
--- a/java/dex_test.go
+++ b/java/dex_test.go
@@ -41,6 +41,7 @@
name: "core_platform_app",
srcs: ["foo.java"],
sdk_version: "core_platform",
+ min_sdk_version: "31",
}
java_library {
diff --git a/java/java.go b/java/java.go
index 5421a15..e83678e 100644
--- a/java/java.go
+++ b/java/java.go
@@ -2005,7 +2005,15 @@
j.dexpreopter.uncompressedDex = *j.dexProperties.Uncompress_dex
var dexOutputFile android.OutputPath
- dexOutputFile = j.dexer.compileDex(ctx, flags, j.MinSdkVersion(ctx), outputFile, jarName)
+ dexParams := &compileDexParams{
+ flags: flags,
+ sdkVersion: j.SdkVersion(ctx),
+ minSdkVersion: j.MinSdkVersion(ctx),
+ classesJar: outputFile,
+ jarName: jarName,
+ }
+
+ dexOutputFile = j.dexer.compileDex(ctx, dexParams)
if ctx.Failed() {
return
}
diff --git a/ui/build/config.go b/ui/build/config.go
index a6bba15..61f6b1c 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -63,6 +63,7 @@
environ *Environment
distDir string
buildDateTime string
+ logsPrefix string
// From the arguments
parallel int
@@ -84,6 +85,7 @@
skipSoongTests bool
searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces
skipMetricsUpload bool
+ buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
// From the product config
katiArgs []string
@@ -255,6 +257,14 @@
return true
}
+func UploadOnlyConfig(ctx Context, _ ...string) Config {
+ ret := &configImpl{
+ environ: OsEnvironment(),
+ sandboxConfig: &SandboxConfig{},
+ }
+ return Config{ret}
+}
+
func NewConfig(ctx Context, args ...string) Config {
ret := &configImpl{
environ: OsEnvironment(),
@@ -266,9 +276,7 @@
ret.keepGoing = 1
ret.totalRAM = detectTotalRAM(ctx)
-
ret.parseArgs(ctx, args)
-
// Make sure OUT_DIR is set appropriately
if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
@@ -756,6 +764,14 @@
ctx.Metrics.SetBuildCommand([]string{buildCmd})
} else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
+ } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
+ buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
+ val, err := strconv.ParseInt(buildTimeStr, 10, 64)
+ if err == nil {
+ c.buildStartedTime = val
+ } else {
+ ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
+ }
} else if len(arg) > 0 && arg[0] == '-' {
parseArgNum := func(def int) int {
if len(arg) > 2 {
@@ -1092,6 +1108,14 @@
c.includeTags = i
}
+func (c *configImpl) GetLogsPrefix() string {
+ return c.logsPrefix
+}
+
+func (c *configImpl) SetLogsPrefix(prefix string) {
+ c.logsPrefix = prefix
+}
+
func (c *configImpl) HighmemParallel() int {
if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
return i
@@ -1519,6 +1543,15 @@
return c.skipMetricsUpload
}
+// Returns a Time object if one was passed via a command-line flag.
+// Otherwise returns the passed default.
+func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
+ if c.buildStartedTime == 0 {
+ return defaultTime
+ }
+ return time.UnixMilli(c.buildStartedTime)
+}
+
func GetMetricsUploader(topDir string, env *Environment) string {
if p, ok := env.Get("METRICS_UPLOADER"); ok {
metricsUploader := filepath.Join(topDir, p)
diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go
index ce2d946..717530c 100644
--- a/ui/metrics/metrics.go
+++ b/ui/metrics/metrics.go
@@ -32,13 +32,13 @@
// of what an event is and how the metrics system is a stack based system.
import (
+ "fmt"
"os"
"runtime"
"strings"
"time"
"android/soong/shared"
-
"google.golang.org/protobuf/proto"
soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
@@ -223,6 +223,17 @@
m.metrics.BuildDateTimestamp = proto.Int64(buildTimestamp.UnixNano() / int64(time.Second))
}
+func (m *Metrics) UpdateTotalRealTime(data []byte) error {
+ if err := proto.Unmarshal(data, &m.metrics); err != nil {
+ return fmt.Errorf("Failed to unmarshal proto", err)
+ }
+ startTime := *m.metrics.Total.StartTime
+ endTime := uint64(time.Now().UnixNano())
+
+ *m.metrics.Total.RealTime = *proto.Uint64(endTime - startTime)
+ return nil
+}
+
// SetBuildCommand adds the build command specified by the user to the
// list of collected metrics.
func (m *Metrics) SetBuildCommand(cmd []string) {