Log bp2build_metrics .pb
Also share `Save(pb proto.Message, filepath string)`
Bug: 201539536
Test: bp2build_metrics.pb has expected content & path
Test: m nothing
Test: {bp2build,mixed_{libc,droid}}.sh
Test: CI
Change-Id: I7d8ad87fca6a4b0355010090a527f5ae67b27c88
diff --git a/bp2build/Android.bp b/bp2build/Android.bp
index 9b66354..337fe86 100644
--- a/bp2build/Android.bp
+++ b/bp2build/Android.bp
@@ -19,6 +19,7 @@
deps: [
"soong-android",
"soong-android-soongconfig",
+ "soong-shared",
"soong-apex",
"soong-bazel",
"soong-cc",
@@ -27,6 +28,7 @@
"soong-genrule",
"soong-python",
"soong-sh",
+ "soong-ui-metrics",
],
testSrcs: [
"android_app_certificate_conversion_test.go",
diff --git a/bp2build/build_conversion.go b/bp2build/build_conversion.go
index eb60cd1..54b59af 100644
--- a/bp2build/build_conversion.go
+++ b/bp2build/build_conversion.go
@@ -259,7 +259,7 @@
// Simple metrics tracking for bp2build
metrics := CodegenMetrics{
- ruleClassCount: make(map[string]int),
+ ruleClassCount: make(map[string]uint64),
}
dirs := make(map[string]bool)
diff --git a/bp2build/metrics.go b/bp2build/metrics.go
index b3d5afb..68ac544 100644
--- a/bp2build/metrics.go
+++ b/bp2build/metrics.go
@@ -2,34 +2,52 @@
import (
"fmt"
+ "os"
+ "path/filepath"
"strings"
"android/soong/android"
+ "android/soong/shared"
+ "android/soong/ui/metrics/bp2build_metrics_proto"
)
// Simple metrics struct to collect information about a Blueprint to BUILD
// conversion process.
type CodegenMetrics struct {
// Total number of Soong modules converted to generated targets
- generatedModuleCount int
+ generatedModuleCount uint64
// Total number of Soong modules converted to handcrafted targets
- handCraftedModuleCount int
+ handCraftedModuleCount uint64
// Total number of unconverted Soong modules
- unconvertedModuleCount int
+ unconvertedModuleCount uint64
// Counts of generated Bazel targets per Bazel rule class
- ruleClassCount map[string]int
+ ruleClassCount map[string]uint64
+ // List of modules with unconverted deps
+ // NOTE: NOT in the .proto
moduleWithUnconvertedDepsMsgs []string
+ // List of converted modules
convertedModules []string
}
+// Serialize returns the protoized version of CodegenMetrics: bp2build_metrics_proto.Bp2BuildMetrics
+func (metrics *CodegenMetrics) Serialize() bp2build_metrics_proto.Bp2BuildMetrics {
+ return bp2build_metrics_proto.Bp2BuildMetrics{
+ GeneratedModuleCount: metrics.generatedModuleCount,
+ HandCraftedModuleCount: metrics.handCraftedModuleCount,
+ UnconvertedModuleCount: metrics.unconvertedModuleCount,
+ RuleClassCount: metrics.ruleClassCount,
+ ConvertedModules: metrics.convertedModules,
+ }
+}
+
// Print the codegen metrics to stdout.
func (metrics *CodegenMetrics) Print() {
- generatedTargetCount := 0
+ generatedTargetCount := uint64(0)
for _, ruleClass := range android.SortedStringKeys(metrics.ruleClassCount) {
count := metrics.ruleClassCount[ruleClass]
fmt.Printf("[bp2build] %s: %d targets\n", ruleClass, count)
@@ -45,6 +63,40 @@
strings.Join(metrics.moduleWithUnconvertedDepsMsgs, "\n\t"))
}
+const bp2buildMetricsFilename = "bp2build_metrics.pb"
+
+// fail prints $PWD to stderr, followed by the given printf string and args (vals),
+// then the given alert, and then exits with 1 for failure
+func fail(err error, alertFmt string, vals ...interface{}) {
+ cwd, wderr := os.Getwd()
+ if wderr != nil {
+ cwd = "FAILED TO GET $PWD: " + wderr.Error()
+ }
+ fmt.Fprintf(os.Stderr, "\nIn "+cwd+":\n"+alertFmt+"\n"+err.Error()+"\n", vals...)
+ os.Exit(1)
+}
+
+// Write the bp2build-protoized codegen metrics into the given directory
+func (metrics *CodegenMetrics) Write(dir string) {
+ if _, err := os.Stat(dir); os.IsNotExist(err) {
+ // The metrics dir doesn't already exist, so create it (and parents)
+ if err := os.MkdirAll(dir, 0755); err != nil { // rx for all; w for user
+ fail(err, "Failed to `mkdir -p` %s", dir)
+ }
+ } else if err != nil {
+ fail(err, "Failed to `stat` %s", dir)
+ }
+ metricsFile := filepath.Join(dir, bp2buildMetricsFilename)
+ if err := metrics.dump(metricsFile); err != nil {
+ fail(err, "Error outputting %s", metricsFile)
+ }
+ if _, err := os.Stat(metricsFile); err != nil {
+ fail(err, "MISSING BP2BUILD METRICS OUTPUT: Failed to `stat` %s", metricsFile)
+ } else {
+ fmt.Printf("\nWrote bp2build metrics to: %s\n", metricsFile)
+ }
+}
+
func (metrics *CodegenMetrics) IncrementRuleClassCount(ruleClass string) {
metrics.ruleClassCount[ruleClass] += 1
}
@@ -53,12 +105,18 @@
metrics.unconvertedModuleCount += 1
}
-func (metrics *CodegenMetrics) TotalModuleCount() int {
+func (metrics *CodegenMetrics) TotalModuleCount() uint64 {
return metrics.handCraftedModuleCount +
metrics.generatedModuleCount +
metrics.unconvertedModuleCount
}
+// Dump serializes the metrics to the given filename
+func (metrics *CodegenMetrics) dump(filename string) (err error) {
+ ser := metrics.Serialize()
+ return shared.Save(&ser, filename)
+}
+
type ConversionType int
const (
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index c81d4bc..f07eafa 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -537,6 +537,7 @@
// for queryview, since that's a total repo-wide conversion and there's a
// 1:1 mapping for each module.
metrics.Print()
+ writeBp2BuildMetrics(&metrics, configuration)
ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
ninjaDeps = append(ninjaDeps, symlinkForestDeps...)
@@ -546,3 +547,13 @@
// Create an empty bp2build marker file.
touch(shared.JoinPath(topDir, bp2buildMarker))
}
+
+// Write Bp2Build metrics into $LOG_DIR
+func writeBp2BuildMetrics(metrics *bp2build.CodegenMetrics, configuration android.Config) {
+ metricsDir := configuration.Getenv("LOG_DIR")
+ if len(metricsDir) < 1 {
+ fmt.Fprintf(os.Stderr, "\nMissing required env var for generating bp2build metrics: LOG_DIR\n")
+ os.Exit(1)
+ }
+ metrics.Write(metricsDir)
+}
diff --git a/shared/Android.bp b/shared/Android.bp
index deb17f8..3c84f55 100644
--- a/shared/Android.bp
+++ b/shared/Android.bp
@@ -9,11 +9,13 @@
"env.go",
"paths.go",
"debug.go",
+ "proto.go",
],
testSrcs: [
"paths_test.go",
],
deps: [
"soong-bazel",
+ "golang-protobuf-proto",
],
}
diff --git a/shared/proto.go b/shared/proto.go
new file mode 100644
index 0000000..232656b
--- /dev/null
+++ b/shared/proto.go
@@ -0,0 +1,41 @@
+// Copyright 2021 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shared
+
+import (
+ "io/ioutil"
+ "os"
+
+ "google.golang.org/protobuf/proto"
+)
+
+// Save takes a protobuf message, marshals to an array of bytes
+// and is then saved to a file.
+func Save(pb proto.Message, filepath string) (err error) {
+ data, err := proto.Marshal(pb)
+ if err != nil {
+ return err
+ }
+ tempFilepath := filepath + ".tmp"
+ if err := ioutil.WriteFile(tempFilepath, []byte(data), 0644 /* rw-r--r-- */); err != nil {
+ return err
+ }
+
+ if err := os.Rename(tempFilepath, filepath); err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/ui/build/config.go b/ui/build/config.go
index c306633..4c26d3e 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -1255,16 +1255,22 @@
return c.metricsUploader
}
-// LogsDir returns the logs directory where build log and metrics
-// files are located. By default, the logs directory is the out
+// LogsDir returns the absolute path to the logs directory where build log and
+// metrics files are located. By default, the logs directory is the out
// directory. If the argument dist is specified, the logs directory
// is <dist_dir>/logs.
func (c *configImpl) LogsDir() string {
+ dir := c.OutDir()
if c.Dist() {
// Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files
- return filepath.Join(c.RealDistDir(), "logs")
+ dir = filepath.Join(c.RealDistDir(), "logs")
}
- return c.OutDir()
+ absDir, err := filepath.Abs(dir)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error())
+ os.Exit(1)
+ }
+ return absDir
}
// BazelMetricsDir returns the <logs dir>/bazel_metrics directory
diff --git a/ui/build/soong.go b/ui/build/soong.go
index ae9a2ce..5360342 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -373,6 +373,7 @@
soongBuildEnv.Set("BAZEL_OUTPUT_BASE", filepath.Join(config.BazelOutDir(), "output"))
soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
+ soongBuildEnv.Set("LOG_DIR", config.LogsDir())
// For Soong bootstrapping tests
if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
diff --git a/ui/metrics/Android.bp b/ui/metrics/Android.bp
index 96f6389..3ba3907 100644
--- a/ui/metrics/Android.bp
+++ b/ui/metrics/Android.bp
@@ -25,6 +25,7 @@
"soong-ui-metrics_proto",
"soong-ui-bp2build_metrics_proto",
"soong-ui-tracer",
+ "soong-shared",
],
srcs: [
"metrics.go",
diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go
index f1bb862..80f8c1a 100644
--- a/ui/metrics/metrics.go
+++ b/ui/metrics/metrics.go
@@ -32,12 +32,12 @@
// of what an event is and how the metrics system is a stack based system.
import (
- "io/ioutil"
"os"
"runtime"
"strings"
"time"
+ "android/soong/shared"
"google.golang.org/protobuf/proto"
soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
@@ -196,7 +196,7 @@
}
m.metrics.HostOs = proto.String(runtime.GOOS)
- return save(&m.metrics, out)
+ return shared.Save(&m.metrics, out)
}
// SetSoongBuildMetrics sets the metrics collected from the soong_build
@@ -228,25 +228,5 @@
// Dump saves the collected CUJs metrics to the raw protobuf file.
func (c *CriticalUserJourneysMetrics) Dump(filename string) (err error) {
- return save(&c.cujs, filename)
-}
-
-// save takes a protobuf message, marshals to an array of bytes
-// and is then saved to a file.
-func save(pb proto.Message, filename string) (err error) {
- data, err := proto.Marshal(pb)
- if err != nil {
- return err
- }
-
- tempFilename := filename + ".tmp"
- if err := ioutil.WriteFile(tempFilename, []byte(data), 0644 /* rw-r--r-- */); err != nil {
- return err
- }
-
- if err := os.Rename(tempFilename, filename); err != nil {
- return err
- }
-
- return nil
+ return shared.Save(&c.cujs, filename)
}