Add RBE metrics dump in Soong UI.
From aosp/1329396, the RBE metrics protobuf file is part of the
metrics uploading process. The RBE metrics protobuf file is
generated by running the bootstrap shutdown command. A new function
named DumpRBEMetrics was written in order to generate the RBE metrics
protobuf file before sending to the uploading process.
Bug: b/140638454
Test: * Unit test cases
* Ran RBE build on my local host and verified the
metrics protobuf file is created.
* Ran non-RBE build after RBE build and verified that
the previous metrics protobuf file was deleted.
Change-Id: I4b8068905cb67c4b8c2d94793917b98974fed707
diff --git a/ui/build/Android.bp b/ui/build/Android.bp
index 0a0bb16..4ef2721 100644
--- a/ui/build/Android.bp
+++ b/ui/build/Android.bp
@@ -63,6 +63,7 @@
"cleanbuild_test.go",
"config_test.go",
"environment_test.go",
+ "rbe_test.go",
"upload_test.go",
"util_test.go",
"proc_sync_test.go",
diff --git a/ui/build/config.go b/ui/build/config.go
index c4bbad7..e567e40 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -804,6 +804,15 @@
return true
}
+func (c *configImpl) RBEStatsOutputDir() string {
+ for _, f := range []string{"RBE_output_dir", "FLAG_output_dir"} {
+ if v, ok := c.environ.Get(f); ok {
+ return v
+ }
+ }
+ return ""
+}
+
func (c *configImpl) UseRemoteBuild() bool {
return c.UseGoma() || c.UseRBE()
}
diff --git a/ui/build/rbe.go b/ui/build/rbe.go
index ceea4bf..fcdab3b 100644
--- a/ui/build/rbe.go
+++ b/ui/build/rbe.go
@@ -15,14 +15,39 @@
package build
import (
+ "os"
"path/filepath"
"android/soong/ui/metrics"
)
-const bootstrapCmd = "bootstrap"
-const rbeLeastNProcs = 2500
-const rbeLeastNFiles = 16000
+const (
+ rbeLeastNProcs = 2500
+ rbeLeastNFiles = 16000
+
+ // prebuilt RBE binaries
+ bootstrapCmd = "bootstrap"
+
+ // RBE metrics proto buffer file
+ rbeMetricsPBFilename = "rbe_metrics.pb"
+)
+
+func rbeCommand(ctx Context, config Config, rbeCmd string) string {
+ var cmdPath string
+ if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok {
+ cmdPath = filepath.Join(rbeDir, rbeCmd)
+ } else if home, ok := config.Environment().Get("HOME"); ok {
+ cmdPath = filepath.Join(home, "rbe", rbeCmd)
+ } else {
+ ctx.Fatalf("rbe command path not found")
+ }
+
+ if _, err := os.Stat(cmdPath); err != nil && os.IsNotExist(err) {
+ ctx.Fatalf("rbe command %q not found", rbeCmd)
+ }
+
+ return cmdPath
+}
func startRBE(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSetupTool, "rbe_bootstrap")
@@ -35,18 +60,50 @@
ctx.Fatalf("max open files is insufficient: %d; want >= %d.\n", n, rbeLeastNFiles)
}
- var rbeBootstrap string
- if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok {
- rbeBootstrap = filepath.Join(rbeDir, bootstrapCmd)
- } else if home, ok := config.Environment().Get("HOME"); ok {
- rbeBootstrap = filepath.Join(home, "rbe", bootstrapCmd)
- } else {
- ctx.Fatalln("rbe bootstrap not found")
- }
-
- cmd := Command(ctx, config, "boostrap", rbeBootstrap)
+ cmd := Command(ctx, config, "startRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd))
if output, err := cmd.CombinedOutput(); err != nil {
ctx.Fatalf("rbe bootstrap failed with: %v\n%s\n", err, output)
}
}
+
+func stopRBE(ctx Context, config Config) {
+ cmd := Command(ctx, config, "stopRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd), "-shutdown")
+ if output, err := cmd.CombinedOutput(); err != nil {
+ ctx.Fatalf("rbe bootstrap with shutdown failed with: %v\n%s\n", err, output)
+ }
+}
+
+// DumpRBEMetrics creates a metrics protobuf file containing RBE related metrics.
+// The protobuf file is created if RBE is enabled and the proxy service has
+// started. The proxy service is shutdown in order to dump the RBE metrics to the
+// protobuf file.
+func DumpRBEMetrics(ctx Context, config Config, filename string) {
+ ctx.BeginTrace(metrics.RunShutdownTool, "dump_rbe_metrics")
+ defer ctx.EndTrace()
+
+ // Remove the previous metrics file in case there is a failure or RBE has been
+ // disable for this run.
+ os.Remove(filename)
+
+ // If RBE is not enabled then there are no metrics to generate.
+ // If RBE does not require to start, the RBE proxy maybe started
+ // manually for debugging purpose and can generate the metrics
+ // afterwards.
+ if !config.StartRBE() {
+ return
+ }
+
+ outputDir := config.RBEStatsOutputDir()
+ if outputDir == "" {
+ ctx.Fatal("RBE output dir variable not defined. Aborting metrics dumping.")
+ }
+ metricsFile := filepath.Join(outputDir, rbeMetricsPBFilename)
+
+ // Stop the proxy first in order to generate the RBE metrics protobuf file.
+ stopRBE(ctx, config)
+
+ if _, err := copyFile(metricsFile, filename); err != nil {
+ ctx.Fatalf("failed to copy %q to %q: %v\n", metricsFile, filename, err)
+ }
+}
diff --git a/ui/build/rbe_test.go b/ui/build/rbe_test.go
new file mode 100644
index 0000000..2c4995b
--- /dev/null
+++ b/ui/build/rbe_test.go
@@ -0,0 +1,142 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "android/soong/ui/logger"
+)
+
+func TestDumpRBEMetrics(t *testing.T) {
+ ctx := testContext()
+ tests := []struct {
+ description string
+ env []string
+ generated bool
+ }{{
+ description: "RBE disabled",
+ env: []string{
+ "NOSTART_RBE=true",
+ },
+ }, {
+ description: "rbe metrics generated",
+ env: []string{
+ "USE_RBE=true",
+ },
+ generated: true,
+ }}
+
+ for _, tt := range tests {
+ t.Run(tt.description, func(t *testing.T) {
+ tmpDir := t.TempDir()
+
+ rbeBootstrapCmd := filepath.Join(tmpDir, bootstrapCmd)
+ if err := ioutil.WriteFile(rbeBootstrapCmd, []byte(rbeBootstrapProgram), 0755); err != nil {
+ t.Fatalf("failed to create a fake bootstrap command file %s: %v", rbeBootstrapCmd, err)
+ }
+
+ env := Environment(tt.env)
+ env.Set("OUT_DIR", tmpDir)
+ env.Set("RBE_DIR", tmpDir)
+ env.Set("RBE_output_dir", t.TempDir())
+ config := Config{&configImpl{
+ environ: &env,
+ }}
+
+ rbeMetricsFilename := filepath.Join(tmpDir, rbeMetricsPBFilename)
+ DumpRBEMetrics(ctx, config, rbeMetricsFilename)
+
+ // Validate that the rbe metrics file exists if RBE is enabled.
+ if _, err := os.Stat(rbeMetricsFilename); err == nil {
+ if !tt.generated {
+ t.Errorf("got true, want false for rbe metrics file %s to exist.", rbeMetricsFilename)
+ }
+ } else if os.IsNotExist(err) {
+ if tt.generated {
+ t.Errorf("got false, want true for rbe metrics file %s to exist.", rbeMetricsFilename)
+ }
+ } else {
+ t.Errorf("unknown error found on checking %s exists: %v", rbeMetricsFilename, err)
+ }
+ })
+ }
+}
+
+func TestDumpRBEMetricsErrors(t *testing.T) {
+ ctx := testContext()
+ tests := []struct {
+ description string
+ rbeOutputDirDefined bool
+ bootstrapProgram string
+ expectedErr string
+ }{{
+ description: "output_dir not defined",
+ bootstrapProgram: rbeBootstrapProgram,
+ expectedErr: "RBE output dir variable not defined",
+ }, {
+ description: "stopRBE failed",
+ rbeOutputDirDefined: true,
+ bootstrapProgram: "#!/bin/bash\nexit 1",
+ expectedErr: "shutdown failed",
+ }, {
+ description: "failed to copy metrics file",
+ rbeOutputDirDefined: true,
+ bootstrapProgram: "#!/bin/bash",
+ expectedErr: "failed to copy",
+ }}
+
+ for _, tt := range tests {
+ t.Run(tt.description, func(t *testing.T) {
+ defer logger.Recover(func(err error) {
+ got := err.Error()
+ if !strings.Contains(got, tt.expectedErr) {
+ t.Errorf("got %q, want %q to be contained in error", got, tt.expectedErr)
+ }
+ })
+
+ tmpDir := t.TempDir()
+
+ rbeBootstrapCmd := filepath.Join(tmpDir, bootstrapCmd)
+ if err := ioutil.WriteFile(rbeBootstrapCmd, []byte(tt.bootstrapProgram), 0755); err != nil {
+ t.Fatalf("failed to create a fake bootstrap command file %s: %v", rbeBootstrapCmd, err)
+ }
+
+ env := &Environment{}
+ env.Set("USE_RBE", "true")
+ env.Set("OUT_DIR", tmpDir)
+ env.Set("RBE_DIR", tmpDir)
+
+ if tt.rbeOutputDirDefined {
+ env.Set("RBE_output_dir", t.TempDir())
+ }
+
+ config := Config{&configImpl{
+ environ: env,
+ }}
+
+ rbeMetricsFilename := filepath.Join(tmpDir, rbeMetricsPBFilename)
+ DumpRBEMetrics(ctx, config, rbeMetricsFilename)
+ t.Errorf("got nil, expecting %q as a failure", tt.expectedErr)
+ })
+ }
+}
+
+var rbeBootstrapProgram = fmt.Sprintf("#!/bin/bash\necho 1 > $RBE_output_dir/%s", rbeMetricsPBFilename)
diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go
index 3e76d37..e055b76 100644
--- a/ui/metrics/metrics.go
+++ b/ui/metrics/metrics.go
@@ -25,12 +25,13 @@
)
const (
- RunSetupTool = "setup"
- RunKati = "kati"
- RunSoong = "soong"
- PrimaryNinja = "ninja"
- TestRun = "test"
- Total = "total"
+ PrimaryNinja = "ninja"
+ RunKati = "kati"
+ RunSetupTool = "setup"
+ RunShutdownTool = "shutdown"
+ RunSoong = "soong"
+ TestRun = "test"
+ Total = "total"
)
type Metrics struct {