Batch cquery requests for mixed builds
This adds an extra step to mixed builds: creating a master BUILD/bzl
file pair to facilitate running a single cquery command to analyze
all soong->bazel edges in a single request.
Test: Mixed build tested with aosp/1441774, verified ninja outputs
depend on `bazel-out/` intermediates
Test: Manually verified contents of master BUILD and bzl files
Change-Id: I04803bcc91ac4182578f505b3f42893061ddd167
diff --git a/android/bazel_handler.go b/android/bazel_handler.go
index 221aabc..c87a945 100644
--- a/android/bazel_handler.go
+++ b/android/bazel_handler.go
@@ -29,10 +29,16 @@
"github.com/google/blueprint/bootstrap"
)
+type CqueryRequestType int
+
+const (
+ getAllFiles CqueryRequestType = iota
+)
+
// Map key to describe bazel cquery requests.
type cqueryKey struct {
- label string
- starlarkExpr string
+ label string
+ requestType CqueryRequestType
}
type BazelContext interface {
@@ -61,6 +67,7 @@
bazelPath string
outputBase string
workspaceDir string
+ buildDir string
requests map[cqueryKey]bool // cquery requests that have not yet been issued to Bazel
requestMutex sync.Mutex // requests can be written in parallel
@@ -96,8 +103,7 @@
var _ BazelContext = MockBazelContext{}
func (bazelCtx *bazelContext) GetAllFiles(label string) ([]string, bool) {
- starlarkExpr := "', '.join([f.path for f in target.files.to_list()])"
- result, ok := bazelCtx.cquery(label, starlarkExpr)
+ result, ok := bazelCtx.cquery(label, getAllFiles)
if ok {
bazelOutput := strings.TrimSpace(result)
return strings.Split(bazelOutput, ", "), true
@@ -125,7 +131,7 @@
return noopBazelContext{}, nil
}
- bazelCtx := bazelContext{requests: make(map[cqueryKey]bool)}
+ bazelCtx := bazelContext{buildDir: c.buildDir, requests: make(map[cqueryKey]bool)}
missingEnvVars := []string{}
if len(c.Getenv("BAZEL_HOME")) > 1 {
bazelCtx.homeDir = c.Getenv("BAZEL_HOME")
@@ -163,8 +169,8 @@
// If the given request was already made (and the results are available), then
// returns (result, true). If the request is queued but no results are available,
// then returns ("", false).
-func (context *bazelContext) cquery(label string, starlarkExpr string) (string, bool) {
- key := cqueryKey{label, starlarkExpr}
+func (context *bazelContext) cquery(label string, requestType CqueryRequestType) (string, bool) {
+ key := cqueryKey{label, requestType}
if result, ok := context.results[key]; ok {
return result, true
} else {
@@ -186,7 +192,8 @@
func (context *bazelContext) issueBazelCommand(command string, labels []string,
extraFlags ...string) (string, error) {
- cmdFlags := []string{"--output_base=" + context.outputBase, command}
+ cmdFlags := []string{"--bazelrc=build/bazel/common.bazelrc",
+ "--output_base=" + context.outputBase, command}
cmdFlags = append(cmdFlags, labels...)
cmdFlags = append(cmdFlags, extraFlags...)
@@ -204,27 +211,113 @@
}
}
+func (context *bazelContext) mainBzlFileContents() []byte {
+ contents := `
+# This file is generated by soong_build. Do not edit.
+def _mixed_build_root_impl(ctx):
+ return [DefaultInfo(files = depset(ctx.files.deps))]
+
+mixed_build_root = rule(
+ implementation = _mixed_build_root_impl,
+ attrs = {"deps" : attr.label_list()},
+)
+`
+ return []byte(contents)
+}
+
+func (context *bazelContext) mainBuildFileContents() []byte {
+ formatString := `
+# This file is generated by soong_build. Do not edit.
+load(":main.bzl", "mixed_build_root")
+
+mixed_build_root(name = "buildroot",
+ deps = [%s],
+)
+`
+ var buildRootDeps []string = nil
+ for val, _ := range context.requests {
+ buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\"", val.label))
+ }
+ buildRootDepsString := strings.Join(buildRootDeps, ",\n ")
+
+ return []byte(fmt.Sprintf(formatString, buildRootDepsString))
+}
+
+func (context *bazelContext) cqueryStarlarkFileContents() []byte {
+ formatString := `
+# This file is generated by soong_build. Do not edit.
+getAllFilesLabels = {
+ %s
+}
+
+def format(target):
+ if str(target.label) in getAllFilesLabels:
+ return str(target.label) + ">>" + ', '.join([f.path for f in target.files.to_list()])
+ else:
+ # This target was not requested via cquery, and thus must be a dependency
+ # of a requested target.
+ return ""
+`
+ var buildRootDeps []string = nil
+ // TODO(cparsons): Sort by request type instead of assuming all requests
+ // are of GetAllFiles type.
+ for val, _ := range context.requests {
+ buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\" : True", val.label))
+ }
+ buildRootDepsString := strings.Join(buildRootDeps, ",\n ")
+
+ return []byte(fmt.Sprintf(formatString, buildRootDepsString))
+}
+
// Issues commands to Bazel to receive results for all cquery requests
// queued in the BazelContext.
func (context *bazelContext) InvokeBazel() error {
context.results = make(map[cqueryKey]string)
- var labels []string
var cqueryOutput string
var err error
+ err = ioutil.WriteFile(
+ absolutePath(filepath.Join(context.buildDir, "main.bzl")),
+ context.mainBzlFileContents(), 0666)
+ if err != nil {
+ return err
+ }
+ err = ioutil.WriteFile(
+ absolutePath(filepath.Join(context.buildDir, "BUILD.bazel")),
+ context.mainBuildFileContents(), 0666)
+ if err != nil {
+ return err
+ }
+ cquery_file_relpath := filepath.Join(context.buildDir, "buildroot.cquery")
+ err = ioutil.WriteFile(
+ absolutePath(cquery_file_relpath),
+ context.cqueryStarlarkFileContents(), 0666)
+ if err != nil {
+ return err
+ }
+ buildroot_label := fmt.Sprintf("//%s:buildroot", context.buildDir)
+ cqueryOutput, err = context.issueBazelCommand("cquery",
+ []string{fmt.Sprintf("deps(%s)", buildroot_label)},
+ "--output=starlark",
+ "--starlark:file="+cquery_file_relpath)
+
+ if err != nil {
+ return err
+ }
+
+ cqueryResults := map[string]string{}
+ for _, outputLine := range strings.Split(cqueryOutput, "\n") {
+ if strings.Contains(outputLine, ">>") {
+ splitLine := strings.SplitN(outputLine, ">>", 2)
+ cqueryResults[splitLine[0]] = splitLine[1]
+ }
+ }
+
for val, _ := range context.requests {
- labels = append(labels, val.label)
-
- // TODO(cparsons): Combine requests into a batch cquery request.
- // TODO(cparsons): Use --query_file to avoid command line limits.
- cqueryOutput, err = context.issueBazelCommand("cquery", []string{val.label},
- "--output=starlark",
- "--starlark:expr="+val.starlarkExpr)
-
- if err != nil {
- return err
+ if cqueryResult, ok := cqueryResults[val.label]; ok {
+ context.results[val] = string(cqueryResult)
} else {
- context.results[val] = string(cqueryOutput)
+ return fmt.Errorf("missing result for bazel target %s", val.label)
}
}
@@ -233,7 +326,7 @@
// bazel actions should either be added to the Ninja file and executed later,
// or bazel should handle execution.
// TODO(cparsons): Use --target_pattern_file to avoid command line limits.
- _, err = context.issueBazelCommand("build", labels)
+ _, err = context.issueBazelCommand("build", []string{buildroot_label})
if err != nil {
return err