Merge "removing codegen_cli from Bp2buildKeepExistingBuildFile"
diff --git a/android/api_levels.go b/android/api_levels.go
index 6d7552f..9440ee9 100644
--- a/android/api_levels.go
+++ b/android/api_levels.go
@@ -225,6 +225,8 @@
// ApiLevelFromUserWithConfig implements ApiLevelFromUser, see comments for
// ApiLevelFromUser for more details.
func ApiLevelFromUserWithConfig(config Config, raw string) (ApiLevel, error) {
+ // This logic is replicated in starlark, if changing logic here update starlark code too
+ // https://cs.android.com/android/platform/superproject/+/master:build/bazel/rules/common/api.bzl;l=42;drc=231c7e8c8038fd478a79eb68aa5b9f5c64e0e061
if raw == "" {
panic("API level string must be non-empty")
}
@@ -329,6 +331,8 @@
var finalCodenamesMapKey = NewOnceKey("FinalCodenamesMap")
func getFinalCodenamesMap(config Config) map[string]int {
+ // This logic is replicated in starlark, if changing logic here update starlark code too
+ // https://cs.android.com/android/platform/superproject/+/master:build/bazel/rules/common/api.bzl;l=30;drc=231c7e8c8038fd478a79eb68aa5b9f5c64e0e061
return config.Once(finalCodenamesMapKey, func() interface{} {
apiLevelsMap := getApiLevelsMapReleasedVersions()
@@ -355,6 +359,8 @@
// ApiLevelsMap has entries for preview API levels
func GetApiLevelsMap(config Config) map[string]int {
+ // This logic is replicated in starlark, if changing logic here update starlark code too
+ // https://cs.android.com/android/platform/superproject/+/master:build/bazel/rules/common/api.bzl;l=23;drc=231c7e8c8038fd478a79eb68aa5b9f5c64e0e061
return config.Once(apiLevelsMapKey, func() interface{} {
apiLevelsMap := getApiLevelsMapReleasedVersions()
for i, codename := range config.PlatformVersionActiveCodenames() {
diff --git a/android/bazel_handler.go b/android/bazel_handler.go
index e7ff08f..e7b84e3 100644
--- a/android/bazel_handler.go
+++ b/android/bazel_handler.go
@@ -607,7 +607,7 @@
dclaEnabledModules := map[string]bool{}
addToStringSet(dclaEnabledModules, dclaMixedBuildsEnabledList)
return &mixedBuildBazelContext{
- bazelRunner: &builtinBazelRunner{},
+ bazelRunner: &builtinBazelRunner{c.UseBazelProxy, absolutePath(c.outDir)},
paths: &paths,
modulesDefaultToBazel: c.BuildMode == BazelDevMode,
bazelEnabledModules: enabledModules,
@@ -684,23 +684,46 @@
return "", "", nil
}
-type builtinBazelRunner struct{}
+type builtinBazelRunner struct {
+ useBazelProxy bool
+ outDir string
+}
// Issues the given bazel command with given build label and additional flags.
// Returns (stdout, stderr, error). The first and second return values are strings
// containing the stdout and stderr of the run command, and an error is returned if
// the invocation returned an error code.
func (r *builtinBazelRunner) issueBazelCommand(bazelCmd *exec.Cmd, eventHandler *metrics.EventHandler) (string, string, error) {
- eventHandler.Begin("bazel command")
- defer eventHandler.End("bazel command")
- stderr := &bytes.Buffer{}
- bazelCmd.Stderr = stderr
- if output, err := bazelCmd.Output(); err != nil {
- return "", string(stderr.Bytes()),
- fmt.Errorf("bazel command failed: %s\n---command---\n%s\n---env---\n%s\n---stderr---\n%s---",
- err, bazelCmd, strings.Join(bazelCmd.Env, "\n"), stderr)
+ if r.useBazelProxy {
+ eventHandler.Begin("client_proxy")
+ defer eventHandler.End("client_proxy")
+ proxyClient := bazel.NewProxyClient(r.outDir)
+ // Omit the arg containing the Bazel binary, as that is handled by the proxy
+ // server.
+ bazelFlags := bazelCmd.Args[1:]
+ // TODO(b/270989498): Refactor these functions to not take exec.Cmd, as its
+ // not actually executed for client proxying.
+ resp, err := proxyClient.IssueCommand(bazel.CmdRequest{bazelFlags, bazelCmd.Env})
+
+ if err != nil {
+ return "", "", err
+ }
+ if len(resp.ErrorString) > 0 {
+ return "", "", fmt.Errorf(resp.ErrorString)
+ }
+ return resp.Stdout, resp.Stderr, nil
} else {
- return string(output), string(stderr.Bytes()), nil
+ eventHandler.Begin("bazel command")
+ defer eventHandler.End("bazel command")
+ stderr := &bytes.Buffer{}
+ bazelCmd.Stderr = stderr
+ if output, err := bazelCmd.Output(); err != nil {
+ return "", string(stderr.Bytes()),
+ fmt.Errorf("bazel command failed: %s\n---command---\n%s\n---env---\n%s\n---stderr---\n%s---",
+ err, bazelCmd, strings.Join(bazelCmd.Env, "\n"), stderr)
+ } else {
+ return string(output), string(stderr.Bytes()), nil
+ }
}
}
diff --git a/android/config.go b/android/config.go
index 78da320..b37d5c8 100644
--- a/android/config.go
+++ b/android/config.go
@@ -87,6 +87,8 @@
BazelModeDev bool
BazelModeStaging bool
BazelForceEnabledModules string
+
+ UseBazelProxy bool
}
// Build modes that soong_build can run as.
@@ -251,6 +253,10 @@
// specified modules. They are passed via the command-line flag
// "--bazel-force-enabled-modules"
bazelForceEnabledModules map[string]struct{}
+
+ // If true, for any requests to Bazel, communicate with a Bazel proxy using
+ // unix sockets, instead of spawning Bazel as a subprocess.
+ UseBazelProxy bool
}
type deviceConfig struct {
@@ -442,6 +448,8 @@
mixedBuildDisabledModules: make(map[string]struct{}),
mixedBuildEnabledModules: make(map[string]struct{}),
bazelForceEnabledModules: make(map[string]struct{}),
+
+ UseBazelProxy: cmdArgs.UseBazelProxy,
}
config.deviceConfig = &deviceConfig{
@@ -875,6 +883,8 @@
// DefaultAppTargetSdk returns the API level that platform apps are targeting.
// This converts a codename to the exact ApiLevel it represents.
func (c *config) DefaultAppTargetSdk(ctx EarlyModuleContext) ApiLevel {
+ // This logic is replicated in starlark, if changing logic here update starlark code too
+ // https://cs.android.com/android/platform/superproject/+/master:build/bazel/rules/common/api.bzl;l=72;drc=231c7e8c8038fd478a79eb68aa5b9f5c64e0e061
if Bool(c.productVariables.Platform_sdk_final) {
return c.PlatformSdkVersion()
}
@@ -1359,11 +1369,6 @@
}
}
if coverage && len(c.config.productVariables.NativeCoverageExcludePaths) > 0 {
- // Workaround coverage boot failure.
- // http://b/269981180
- if strings.HasPrefix(path, "external/protobuf") {
- coverage = false
- }
if HasAnyPrefix(path, c.config.productVariables.NativeCoverageExcludePaths) {
coverage = false
}
diff --git a/android/sdk_version.go b/android/sdk_version.go
index 8953eae..a7e03dc 100644
--- a/android/sdk_version.go
+++ b/android/sdk_version.go
@@ -211,7 +211,29 @@
if !s.ApiLevel.IsPreview() {
return s.ApiLevel.String(), nil
}
- return ctx.Config().DefaultAppTargetSdk(ctx).String(), nil
+ // Determine the default sdk
+ ret := ctx.Config().DefaultAppTargetSdk(ctx)
+ if !ret.IsPreview() {
+ // If the default sdk has been finalized, return that
+ return ret.String(), nil
+ }
+ // There can be more than one active in-development sdks
+ // If an app is targeting an active sdk, but not the default one, return the requested active sdk.
+ // e.g.
+ // SETUP
+ // In-development: UpsideDownCake, VanillaIceCream
+ // Default: VanillaIceCream
+ // Android.bp
+ // min_sdk_version: `UpsideDownCake`
+ // RETURN
+ // UpsideDownCake and not VanillaIceCream
+ for _, preview := range ctx.Config().PreviewApiLevels() {
+ if s.ApiLevel.String() == preview.String() {
+ return preview.String(), nil
+ }
+ }
+ // Otherwise return the default one
+ return ret.String(), nil
}
var (
diff --git a/android/util.go b/android/util.go
index 6c0ddf4..38e0a4d 100644
--- a/android/util.go
+++ b/android/util.go
@@ -65,21 +65,8 @@
// SortedStringKeys returns the keys of the given map in the ascending order.
//
// Deprecated: Use SortedKeys instead.
-func SortedStringKeys(m interface{}) []string {
- v := reflect.ValueOf(m)
- if v.Kind() != reflect.Map {
- panic(fmt.Sprintf("%#v is not a map", m))
- }
- if v.Len() == 0 {
- return nil
- }
- iter := v.MapRange()
- s := make([]string, 0, v.Len())
- for iter.Next() {
- s = append(s, iter.Key().String())
- }
- sort.Strings(s)
- return s
+func SortedStringKeys[V any](m map[string]V) []string {
+ return SortedKeys(m)
}
type Ordered interface {
diff --git a/android/util_test.go b/android/util_test.go
index 51d8e32..5584b38 100644
--- a/android/util_test.go
+++ b/android/util_test.go
@@ -646,7 +646,7 @@
t.Run(name, func(t *testing.T) {
actual := SortedKeys(input)
if !reflect.DeepEqual(actual, expected) {
- t.Errorf("expected %q, got %q", expected, actual)
+ t.Errorf("expected %v, got %v", expected, actual)
}
})
}
@@ -671,44 +671,6 @@
testSortedKeysHelper(t, "empty", map[string]string{}, nil)
}
-func TestSortedStringKeys(t *testing.T) {
- testCases := []struct {
- name string
- in interface{}
- expected []string
- }{
- {
- name: "nil",
- in: map[string]string(nil),
- expected: nil,
- },
- {
- name: "empty",
- in: map[string]string{},
- expected: nil,
- },
- {
- name: "simple",
- in: map[string]string{"a": "foo", "b": "bar"},
- expected: []string{"a", "b"},
- },
- {
- name: "interface values",
- in: map[string]interface{}{"a": nil, "b": nil},
- expected: []string{"a", "b"},
- },
- }
-
- for _, tt := range testCases {
- t.Run(tt.name, func(t *testing.T) {
- got := SortedStringKeys(tt.in)
- if g, w := got, tt.expected; !reflect.DeepEqual(g, w) {
- t.Errorf("wanted %q, got %q", w, g)
- }
- })
- }
-}
-
func TestSortedStringValues(t *testing.T) {
testCases := []struct {
name string
diff --git a/apex/Android.bp b/apex/Android.bp
index 018d030..7ffca0e 100644
--- a/apex/Android.bp
+++ b/apex/Android.bp
@@ -29,6 +29,7 @@
"bp2build.go",
"deapexer.go",
"key.go",
+ "metadata.go",
"prebuilt.go",
"testing.go",
"vndk.go",
@@ -37,6 +38,7 @@
"apex_test.go",
"bootclasspath_fragment_test.go",
"classpath_element_test.go",
+ "metadata_test.go",
"platform_bootclasspath_test.go",
"systemserver_classpath_fragment_test.go",
"vndk_test.go",
diff --git a/apex/apex.go b/apex/apex.go
index d7d76d1..88eb72f 100644
--- a/apex/apex.go
+++ b/apex/apex.go
@@ -99,6 +99,10 @@
// /system/sepolicy/apex/<module_name>_file_contexts.
File_contexts *string `android:"path"`
+ // By default, file_contexts is amended by force-labelling / and /apex_manifest.pb as system_file
+ // to avoid mistakes. When set as true, no force-labelling.
+ Use_file_contexts_as_is *bool
+
// Path to the canned fs config file for customizing file's uid/gid/mod/capabilities. The
// format is /<path_or_glob> <uid> <gid> <mode> [capabilities=0x<cap>], where path_or_glob is a
// path or glob pattern for a file or set of files, uid/gid are numerial values of user ID
diff --git a/apex/apex_test.go b/apex/apex_test.go
index 53e922c..c94bbbb 100644
--- a/apex/apex_test.go
+++ b/apex/apex_test.go
@@ -784,6 +784,43 @@
}
}
+func TestFileContexts(t *testing.T) {
+ for _, useFileContextsAsIs := range []bool{true, false} {
+ prop := ""
+ if useFileContextsAsIs {
+ prop = "use_file_contexts_as_is: true,\n"
+ }
+ ctx := testApex(t, `
+ apex {
+ name: "myapex",
+ key: "myapex.key",
+ file_contexts: "file_contexts",
+ updatable: false,
+ vendor: true,
+ `+prop+`
+ }
+
+ apex_key {
+ name: "myapex.key",
+ public_key: "testkey.avbpubkey",
+ private_key: "testkey.pem",
+ }
+ `, withFiles(map[string][]byte{
+ "file_contexts": nil,
+ }))
+
+ rule := ctx.ModuleForTests("myapex", "android_common_myapex_image").Output("file_contexts")
+ forceLabellingCommand := "apex_manifest\\\\.pb u:object_r:system_file:s0"
+ if useFileContextsAsIs {
+ android.AssertStringDoesNotContain(t, "should force-label",
+ rule.RuleParams.Command, forceLabellingCommand)
+ } else {
+ android.AssertStringDoesContain(t, "shouldn't force-label",
+ rule.RuleParams.Command, forceLabellingCommand)
+ }
+ }
+}
+
func TestBasicZipApex(t *testing.T) {
ctx := testApex(t, `
apex {
diff --git a/apex/builder.go b/apex/builder.go
index 7248d97..ee6c473 100644
--- a/apex/builder.go
+++ b/apex/builder.go
@@ -333,6 +333,8 @@
ctx.PropertyErrorf("file_contexts", "cannot find file_contexts file: %q", fileContexts.String())
}
+ useFileContextsAsIs := proptools.Bool(a.properties.Use_file_contexts_as_is)
+
output := android.PathForModuleOut(ctx, "file_contexts")
rule := android.NewRuleBuilder(pctx, ctx)
@@ -344,9 +346,11 @@
rule.Command().Text("cat").Input(fileContexts).Text(">>").Output(output)
// new line
rule.Command().Text("echo").Text(">>").Output(output)
- // force-label /apex_manifest.pb and / as system_file so that apexd can read them
- rule.Command().Text("echo").Flag("/apex_manifest\\\\.pb u:object_r:system_file:s0").Text(">>").Output(output)
- rule.Command().Text("echo").Flag("/ u:object_r:system_file:s0").Text(">>").Output(output)
+ if !useFileContextsAsIs {
+ // force-label /apex_manifest.pb and / as system_file so that apexd can read them
+ rule.Command().Text("echo").Flag("/apex_manifest\\\\.pb u:object_r:system_file:s0").Text(">>").Output(output)
+ rule.Command().Text("echo").Flag("/ u:object_r:system_file:s0").Text(">>").Output(output)
+ }
case flattenedApex:
// For flattened apexes, install path should be prepended.
// File_contexts file should be emiited to make via LOCAL_FILE_CONTEXTS
@@ -359,9 +363,11 @@
rule.Command().Text("awk").Text(`'/object_r/{printf("` + apexPath + `%s\n", $0)}'`).Input(fileContexts).Text(">").Output(output)
// new line
rule.Command().Text("echo").Text(">>").Output(output)
- // force-label /apex_manifest.pb and / as system_file so that apexd can read them
- rule.Command().Text("echo").Flag(apexPath + `/apex_manifest\\.pb u:object_r:system_file:s0`).Text(">>").Output(output)
- rule.Command().Text("echo").Flag(apexPath + "/ u:object_r:system_file:s0").Text(">>").Output(output)
+ if !useFileContextsAsIs {
+ // force-label /apex_manifest.pb and / as system_file so that apexd can read them
+ rule.Command().Text("echo").Flag(apexPath + `/apex_manifest\\.pb u:object_r:system_file:s0`).Text(">>").Output(output)
+ rule.Command().Text("echo").Flag(apexPath + "/ u:object_r:system_file:s0").Text(">>").Output(output)
+ }
default:
panic(fmt.Errorf("unsupported type %v", a.properties.ApexType))
}
diff --git a/apex/metadata.go b/apex/metadata.go
new file mode 100644
index 0000000..b1dff3e
--- /dev/null
+++ b/apex/metadata.go
@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package apex
+
+import (
+ "encoding/json"
+
+ "github.com/google/blueprint"
+
+ "android/soong/android"
+)
+
+var (
+ mtctx = android.NewPackageContext("android/soong/multitree_apex")
+)
+
+func init() {
+ RegisterModulesSingleton(android.InitRegistrationContext)
+}
+
+func RegisterModulesSingleton(ctx android.RegistrationContext) {
+ ctx.RegisterSingletonType("apex_multitree_singleton", multitreeAnalysisSingletonFactory)
+}
+
+var PrepareForTestWithApexMultitreeSingleton = android.FixtureRegisterWithContext(RegisterModulesSingleton)
+
+func multitreeAnalysisSingletonFactory() android.Singleton {
+ return &multitreeAnalysisSingleton{}
+}
+
+type multitreeAnalysisSingleton struct {
+ multitreeApexMetadataPath android.OutputPath
+}
+
+type ApexMultitreeMetadataEntry struct {
+ // The name of the apex.
+ Name string
+
+ // TODO: Add other properties as needed.
+}
+
+type ApexMultitreeMetadata struct {
+ // Information about the installable apexes.
+ Apexes map[string]ApexMultitreeMetadataEntry
+}
+
+func (p *multitreeAnalysisSingleton) GenerateBuildActions(context android.SingletonContext) {
+ data := ApexMultitreeMetadata{
+ Apexes: make(map[string]ApexMultitreeMetadataEntry, 0),
+ }
+ context.VisitAllModules(func(module android.Module) {
+ // If this module is not being installed, ignore it.
+ if !module.Enabled() || module.IsSkipInstall() {
+ return
+ }
+ // Actual apexes provide ApexBundleInfoProvider.
+ if _, ok := context.ModuleProvider(module, ApexBundleInfoProvider).(ApexBundleInfo); !ok {
+ return
+ }
+ bundle, ok := module.(*apexBundle)
+ if ok && !bundle.testApex && !bundle.vndkApex && bundle.primaryApexType {
+ name := module.Name()
+ entry := ApexMultitreeMetadataEntry{
+ Name: name,
+ }
+ data.Apexes[name] = entry
+ }
+ })
+ p.multitreeApexMetadataPath = android.PathForOutput(context, "multitree_apex_metadata.json")
+
+ jsonStr, err := json.Marshal(data)
+ if err != nil {
+ context.Errorf(err.Error())
+ }
+ android.WriteFileRule(context, p.multitreeApexMetadataPath, string(jsonStr))
+ // This seems cleaner, but doesn't emit the phony rule in testing.
+ // context.Phony("multitree_apex_metadata", p.multitreeApexMetadataPath)
+
+ context.Build(mtctx, android.BuildParams{
+ Rule: blueprint.Phony,
+ Description: "phony rule for multitree_apex_metadata",
+ Inputs: []android.Path{p.multitreeApexMetadataPath},
+ Output: android.PathForPhony(context, "multitree_apex_metadata"),
+ })
+}
diff --git a/apex/metadata_test.go b/apex/metadata_test.go
new file mode 100644
index 0000000..f6ead42
--- /dev/null
+++ b/apex/metadata_test.go
@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package apex
+
+import (
+ "strings"
+ "testing"
+
+ "android/soong/android"
+ "android/soong/java"
+)
+
+func TestModulesSingleton(t *testing.T) {
+ result := android.GroupFixturePreparers(
+ PrepareForTestWithApexMultitreeSingleton,
+ java.PrepareForTestWithDexpreopt,
+ PrepareForTestWithApexBuildComponents,
+ java.FixtureConfigureApexBootJars("myapex:foo"),
+ java.PrepareForTestWithJavaSdkLibraryFiles,
+ ).RunTestWithBp(t, `
+ prebuilt_apex {
+ name: "myapex",
+ src: "myapex.apex",
+ exported_bootclasspath_fragments: ["mybootclasspath-fragment"],
+ }
+
+ // A prebuilt java_sdk_library_import that is not preferred by default but will be preferred
+ // because AlwaysUsePrebuiltSdks() is true.
+ java_sdk_library_import {
+ name: "foo",
+ prefer: false,
+ shared_library: false,
+ permitted_packages: ["foo"],
+ public: {
+ jars: ["sdk_library/public/foo-stubs.jar"],
+ stub_srcs: ["sdk_library/public/foo_stub_sources"],
+ current_api: "sdk_library/public/foo.txt",
+ removed_api: "sdk_library/public/foo-removed.txt",
+ sdk_version: "current",
+ },
+ apex_available: ["myapex"],
+ }
+
+ prebuilt_bootclasspath_fragment {
+ name: "mybootclasspath-fragment",
+ apex_available: [
+ "myapex",
+ ],
+ contents: [
+ "foo",
+ ],
+ hidden_api: {
+ stub_flags: "prebuilt-stub-flags.csv",
+ annotation_flags: "prebuilt-annotation-flags.csv",
+ metadata: "prebuilt-metadata.csv",
+ index: "prebuilt-index.csv",
+ all_flags: "prebuilt-all-flags.csv",
+ },
+ }
+
+ platform_bootclasspath {
+ name: "myplatform-bootclasspath",
+ fragments: [
+ {
+ apex: "myapex",
+ module:"mybootclasspath-fragment",
+ },
+ ],
+ }
+`,
+ )
+
+ outputs := result.SingletonForTests("apex_multitree_singleton").AllOutputs()
+ for _, output := range outputs {
+ testingBuildParam := result.SingletonForTests("apex_multitree_singleton").Output(output)
+ switch {
+ case strings.Contains(output, "soong/multitree_apex_metadata.json"):
+ android.AssertStringEquals(t, "Invalid build rule", "android/soong/android.writeFile", testingBuildParam.Rule.String())
+ android.AssertIntEquals(t, "Invalid input", len(testingBuildParam.Inputs), 0)
+ android.AssertStringDoesContain(t, "Invalid output path", output, "soong/multitree_apex_metadata.json")
+
+ case strings.HasSuffix(output, "multitree_apex_metadata"):
+ android.AssertStringEquals(t, "Invalid build rule", "<builtin>:phony", testingBuildParam.Rule.String())
+ android.AssertStringEquals(t, "Invalid input", testingBuildParam.Inputs[0].String(), "out/soong/multitree_apex_metadata.json")
+ android.AssertStringEquals(t, "Invalid output path", output, "multitree_apex_metadata")
+ android.AssertIntEquals(t, "Invalid args", len(testingBuildParam.Args), 0)
+ }
+ }
+}
diff --git a/bazel/Android.bp b/bazel/Android.bp
index d11c78b..4709f5c 100644
--- a/bazel/Android.bp
+++ b/bazel/Android.bp
@@ -7,6 +7,7 @@
pkgPath: "android/soong/bazel",
srcs: [
"aquery.go",
+ "bazel_proxy.go",
"configurability.go",
"constants.go",
"properties.go",
diff --git a/bazel/bazel_proxy.go b/bazel/bazel_proxy.go
new file mode 100644
index 0000000..d7f5e64
--- /dev/null
+++ b/bazel/bazel_proxy.go
@@ -0,0 +1,219 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bazel
+
+import (
+ "bytes"
+ "encoding/gob"
+ "fmt"
+ "net"
+ os_lib "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "time"
+)
+
+// Logs fatal events of ProxyServer.
+type ServerLogger interface {
+ Fatal(v ...interface{})
+ Fatalf(format string, v ...interface{})
+}
+
+// CmdRequest is a request to the Bazel Proxy server.
+type CmdRequest struct {
+ // Args to the Bazel command.
+ Argv []string
+ // Environment variables to pass to the Bazel invocation. Strings should be of
+ // the form "KEY=VALUE".
+ Env []string
+}
+
+// CmdResponse is a response from the Bazel Proxy server.
+type CmdResponse struct {
+ Stdout string
+ Stderr string
+ ErrorString string
+}
+
+// ProxyClient is a client which can issue Bazel commands to the Bazel
+// proxy server. Requests are issued (and responses received) via a unix socket.
+// See ProxyServer for more details.
+type ProxyClient struct {
+ outDir string
+}
+
+// ProxyServer is a server which runs as a background goroutine. Each
+// request to the server describes a Bazel command which the server should run.
+// The server then issues the Bazel command, and returns a response describing
+// the stdout/stderr of the command.
+// Client-server communication is done via a unix socket under the output
+// directory.
+// The server is intended to circumvent sandboxing for subprocesses of the
+// build. The build orchestrator (soong_ui) can launch a server to exist outside
+// of sandboxing, and sandboxed processes (such as soong_build) can issue
+// bazel commands through this socket tunnel. This allows a sandboxed process
+// to issue bazel requests to a bazel that resides outside of sandbox. This
+// is particularly useful to maintain a persistent Bazel server which lives
+// past the duration of a single build.
+// The ProxyServer will only live as long as soong_ui does; the
+// underlying Bazel server will live past the duration of the build.
+type ProxyServer struct {
+ logger ServerLogger
+ outDir string
+ workspaceDir string
+ // The server goroutine will listen on this channel and stop handling requests
+ // once it is written to.
+ done chan struct{}
+}
+
+// NewProxyClient is a constructor for a ProxyClient.
+func NewProxyClient(outDir string) *ProxyClient {
+ return &ProxyClient{
+ outDir: outDir,
+ }
+}
+
+func unixSocketPath(outDir string) string {
+ return filepath.Join(outDir, "bazelsocket.sock")
+}
+
+// IssueCommand issues a request to the Bazel Proxy Server to issue a Bazel
+// request. Returns a response describing the output from the Bazel process
+// (if the Bazel process had an error, then the response will include an error).
+// Returns an error if there was an issue with the connection to the Bazel Proxy
+// server.
+func (b *ProxyClient) IssueCommand(req CmdRequest) (CmdResponse, error) {
+ var resp CmdResponse
+ var err error
+ // Check for connections every 1 second. This is chosen to be a relatively
+ // short timeout, because the proxy server should accept requests quite
+ // quickly.
+ d := net.Dialer{Timeout: 1 * time.Second}
+ var conn net.Conn
+ conn, err = d.Dial("unix", unixSocketPath(b.outDir))
+ if err != nil {
+ return resp, err
+ }
+ defer conn.Close()
+
+ enc := gob.NewEncoder(conn)
+ if err = enc.Encode(req); err != nil {
+ return resp, err
+ }
+ dec := gob.NewDecoder(conn)
+ err = dec.Decode(&resp)
+ return resp, err
+}
+
+// NewProxyServer is a constructor for a ProxyServer.
+func NewProxyServer(logger ServerLogger, outDir string, workspaceDir string) *ProxyServer {
+ return &ProxyServer{
+ logger: logger,
+ outDir: outDir,
+ workspaceDir: workspaceDir,
+ done: make(chan struct{}),
+ }
+}
+
+func (b *ProxyServer) handleRequest(conn net.Conn) error {
+ defer conn.Close()
+
+ dec := gob.NewDecoder(conn)
+ var req CmdRequest
+ if err := dec.Decode(&req); err != nil {
+ return fmt.Errorf("Error decoding request: %s", err)
+ }
+
+ bazelCmd := exec.Command("./build/bazel/bin/bazel", req.Argv...)
+ bazelCmd.Dir = b.workspaceDir
+ bazelCmd.Env = req.Env
+
+ stderr := &bytes.Buffer{}
+ bazelCmd.Stderr = stderr
+ var stdout string
+ var bazelErrString string
+
+ if output, err := bazelCmd.Output(); err != nil {
+ bazelErrString = fmt.Sprintf("bazel command failed: %s\n---command---\n%s\n---env---\n%s\n---stderr---\n%s---",
+ err, bazelCmd, strings.Join(bazelCmd.Env, "\n"), stderr)
+ } else {
+ stdout = string(output)
+ }
+
+ resp := CmdResponse{stdout, string(stderr.Bytes()), bazelErrString}
+ enc := gob.NewEncoder(conn)
+ if err := enc.Encode(&resp); err != nil {
+ return fmt.Errorf("Error encoding response: %s", err)
+ }
+ return nil
+}
+
+func (b *ProxyServer) listenUntilClosed(listener net.Listener) error {
+ for {
+ // Check for connections every 1 second. This is a blocking operation, so
+ // if the server is closed, the goroutine will not fully close until this
+ // deadline is reached. Thus, this deadline is short (but not too short
+ // so that the routine churns).
+ listener.(*net.UnixListener).SetDeadline(time.Now().Add(time.Second))
+ conn, err := listener.Accept()
+
+ select {
+ case <-b.done:
+ return nil
+ default:
+ }
+
+ if err != nil {
+ if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() {
+ // Timeout is normal and expected while waiting for client to establish
+ // a connection.
+ continue
+ } else {
+ b.logger.Fatalf("Listener error: %s", err)
+ }
+ }
+
+ err = b.handleRequest(conn)
+ if err != nil {
+ b.logger.Fatal(err)
+ }
+ }
+}
+
+// Start initializes the server unix socket and (in a separate goroutine)
+// handles requests on the socket until the server is closed. Returns an error
+// if a failure occurs during initialization. Will log any post-initialization
+// errors to the server's logger.
+func (b *ProxyServer) Start() error {
+ unixSocketAddr := unixSocketPath(b.outDir)
+ if err := os_lib.RemoveAll(unixSocketAddr); err != nil {
+ return fmt.Errorf("couldn't remove socket '%s': %s", unixSocketAddr, err)
+ }
+ listener, err := net.Listen("unix", unixSocketAddr)
+
+ if err != nil {
+ return fmt.Errorf("error listening on socket '%s': %s", unixSocketAddr, err)
+ }
+
+ go b.listenUntilClosed(listener)
+ return nil
+}
+
+// Close shuts down the server. This will stop the server from listening for
+// additional requests.
+func (b *ProxyServer) Close() {
+ b.done <- struct{}{}
+}
diff --git a/bazel/properties.go b/bazel/properties.go
index f4acd26..40d0ba3 100644
--- a/bazel/properties.go
+++ b/bazel/properties.go
@@ -17,6 +17,7 @@
import (
"fmt"
"path/filepath"
+ "reflect"
"regexp"
"sort"
"strings"
@@ -533,6 +534,37 @@
return result, nil
}
+// ToStringListAttribute creates a StringListAttribute from this BoolAttribute,
+// where each bool corresponds to a string list value generated by the provided
+// function.
+// TODO(b/271425661): Generalize this
+func (ba *BoolAttribute) ToStringListAttribute(valueFunc func(boolPtr *bool, axis ConfigurationAxis, config string) []string) (StringListAttribute, error) {
+ mainVal := valueFunc(ba.Value, NoConfigAxis, "")
+ if !ba.HasConfigurableValues() {
+ return MakeStringListAttribute(mainVal), nil
+ }
+
+ result := StringListAttribute{}
+ if err := ba.Collapse(); err != nil {
+ return result, err
+ }
+
+ for axis, configToBools := range ba.ConfigurableValues {
+ if len(configToBools) < 1 {
+ continue
+ }
+ for config, boolPtr := range configToBools {
+ val := valueFunc(&boolPtr, axis, config)
+ if !reflect.DeepEqual(val, mainVal) {
+ result.SetSelectValue(axis, config, val)
+ }
+ }
+ result.SetSelectValue(axis, ConditionsDefaultConfigKey, mainVal)
+ }
+
+ return result, nil
+}
+
// Collapse reduces the configurable axes of the boolean attribute to a single axis.
// This is necessary for final writing to bp2build, as a configurable boolean
// attribute can only be comprised by a single select.
diff --git a/bp2build/bp2build.go b/bp2build/bp2build.go
index 062eba8..d1dfb9d 100644
--- a/bp2build/bp2build.go
+++ b/bp2build/bp2build.go
@@ -17,21 +17,57 @@
import (
"fmt"
"os"
+ "path/filepath"
"strings"
"android/soong/android"
"android/soong/bazel"
+ "android/soong/shared"
)
+func deleteFilesExcept(ctx *CodegenContext, rootOutputPath android.OutputPath, except []BazelFile) {
+ // Delete files that should no longer be present.
+ bp2buildDirAbs := shared.JoinPath(ctx.topDir, rootOutputPath.String())
+
+ filesToDelete := make(map[string]struct{})
+ err := filepath.Walk(bp2buildDirAbs,
+ func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if !info.IsDir() {
+ relPath, err := filepath.Rel(bp2buildDirAbs, path)
+ if err != nil {
+ return err
+ }
+ filesToDelete[relPath] = struct{}{}
+ }
+ return nil
+ })
+ if err != nil {
+ fmt.Printf("ERROR reading %s: %s", bp2buildDirAbs, err)
+ os.Exit(1)
+ }
+
+ for _, bazelFile := range except {
+ filePath := filepath.Join(bazelFile.Dir, bazelFile.Basename)
+ delete(filesToDelete, filePath)
+ }
+ for f, _ := range filesToDelete {
+ absPath := shared.JoinPath(bp2buildDirAbs, f)
+ if err := os.RemoveAll(absPath); err != nil {
+ fmt.Printf("ERROR deleting %s: %s", absPath, err)
+ os.Exit(1)
+ }
+ }
+}
+
// Codegen is the backend of bp2build. The code generator is responsible for
// writing .bzl files that are equivalent to Android.bp files that are capable
// of being built with Bazel.
func Codegen(ctx *CodegenContext) *CodegenMetrics {
// This directory stores BUILD files that could be eventually checked-in.
bp2buildDir := android.PathForOutput(ctx, "bp2build")
- if err := android.RemoveAllOutputDir(bp2buildDir); err != nil {
- fmt.Printf("ERROR: Encountered error while cleaning %s: %s", bp2buildDir, err.Error())
- }
res, errs := GenerateBazelTargets(ctx, true)
if len(errs) > 0 {
@@ -44,6 +80,12 @@
}
bp2buildFiles := CreateBazelFiles(ctx.Config(), nil, res.buildFileToTargets, ctx.mode)
writeFiles(ctx, bp2buildDir, bp2buildFiles)
+ // Delete files under the bp2build root which weren't just written. An
+ // alternative would have been to delete the whole directory and write these
+ // files. However, this would regenerate files which were otherwise unchanged
+ // since the last bp2build run, which would have negative incremental
+ // performance implications.
+ deleteFilesExcept(ctx, bp2buildDir, bp2buildFiles)
injectionFiles, err := CreateSoongInjectionDirFiles(ctx, res.metrics)
if err != nil {
@@ -51,7 +93,6 @@
os.Exit(1)
}
writeFiles(ctx, android.PathForOutput(ctx, bazel.SoongInjectionDirName), injectionFiles)
-
return &res.metrics
}
diff --git a/bp2build/cc_binary_conversion_test.go b/bp2build/cc_binary_conversion_test.go
index a39ed7d..0315732 100644
--- a/bp2build/cc_binary_conversion_test.go
+++ b/bp2build/cc_binary_conversion_test.go
@@ -868,3 +868,131 @@
},
})
}
+
+func TestCcBinaryWithThinLto(t *testing.T) {
+ runCcBinaryTestCase(t, ccBinaryBp2buildTestCase{
+ description: "cc_binary has correct features when thin LTO is enabled",
+ blueprint: `
+{rule_name} {
+ name: "foo",
+ lto: {
+ thin: true,
+ },
+}`,
+ targets: []testBazelTarget{
+ {"cc_binary", "foo", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `["android_thin_lto"]`,
+ }},
+ },
+ })
+}
+
+func TestCcBinaryWithLtoNever(t *testing.T) {
+ runCcBinaryTestCase(t, ccBinaryBp2buildTestCase{
+ description: "cc_binary has correct features when LTO is explicitly disabled",
+ blueprint: `
+{rule_name} {
+ name: "foo",
+ lto: {
+ never: true,
+ },
+}`,
+ targets: []testBazelTarget{
+ {"cc_binary", "foo", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `["-android_thin_lto"]`,
+ }},
+ },
+ })
+}
+
+func TestCcBinaryWithThinLtoArchSpecific(t *testing.T) {
+ runCcBinaryTestCase(t, ccBinaryBp2buildTestCase{
+ description: "cc_binary has correct features when LTO differs across arch and os variants",
+ blueprint: `
+{rule_name} {
+ name: "foo",
+ target: {
+ android: {
+ lto: {
+ thin: true,
+ },
+ },
+ },
+ arch: {
+ riscv64: {
+ lto: {
+ thin: false,
+ },
+ },
+ },
+}`,
+ targets: []testBazelTarget{
+ {"cc_binary", "foo", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `select({
+ "//build/bazel/platforms/os_arch:android_arm": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_arm64": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_riscv64": ["-android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_x86": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_x86_64": ["android_thin_lto"],
+ "//conditions:default": [],
+ })`,
+ }},
+ },
+ })
+}
+
+func TestCcBinaryWithThinLtoDisabledDefaultEnabledVariant(t *testing.T) {
+ runCcBinaryTestCase(t, ccBinaryBp2buildTestCase{
+ description: "cc_binary has correct features when LTO disabled by default but enabled on a particular variant",
+ blueprint: `
+{rule_name} {
+ name: "foo",
+ lto: {
+ never: true,
+ },
+ target: {
+ android: {
+ lto: {
+ thin: true,
+ never: false,
+ },
+ },
+ },
+}`,
+ targets: []testBazelTarget{
+ {"cc_binary", "foo", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `select({
+ "//build/bazel/platforms/os:android": ["android_thin_lto"],
+ "//conditions:default": ["-android_thin_lto"],
+ })`,
+ }},
+ },
+ })
+}
+
+func TestCcBinaryWithThinLtoAndWholeProgramVtables(t *testing.T) {
+ runCcBinaryTestCase(t, ccBinaryBp2buildTestCase{
+ description: "cc_binary has correct features when thin LTO is enabled with whole_program_vtables",
+ blueprint: `
+{rule_name} {
+ name: "foo",
+ lto: {
+ thin: true,
+ },
+ whole_program_vtables: true,
+}`,
+ targets: []testBazelTarget{
+ {"cc_binary", "foo", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `[
+ "android_thin_lto",
+ "android_thin_lto_whole_program_vtables",
+ ]`,
+ }},
+ },
+ })
+}
diff --git a/bp2build/cc_library_conversion_test.go b/bp2build/cc_library_conversion_test.go
index af14f64..e20cffd 100644
--- a/bp2build/cc_library_conversion_test.go
+++ b/bp2build/cc_library_conversion_test.go
@@ -4137,3 +4137,172 @@
},
})
}
+
+func TestCcLibraryWithThinLto(t *testing.T) {
+ runCcLibraryTestCase(t, Bp2buildTestCase{
+ Description: "cc_library has correct features when thin LTO is enabled",
+ ModuleTypeUnderTest: "cc_library",
+ ModuleTypeUnderTestFactory: cc.LibraryFactory,
+ Blueprint: `
+cc_library {
+ name: "foo",
+ lto: {
+ thin: true,
+ },
+}`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{
+ "features": `["android_thin_lto"]`,
+ "local_includes": `["."]`,
+ }),
+ MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{
+ "features": `["android_thin_lto"]`,
+ "local_includes": `["."]`,
+ }),
+ },
+ })
+}
+
+func TestCcLibraryWithLtoNever(t *testing.T) {
+ runCcLibraryTestCase(t, Bp2buildTestCase{
+ Description: "cc_library has correct features when LTO is explicitly disabled",
+ ModuleTypeUnderTest: "cc_library",
+ ModuleTypeUnderTestFactory: cc.LibraryFactory,
+ Blueprint: `
+cc_library {
+ name: "foo",
+ lto: {
+ never: true,
+ },
+}`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{
+ "features": `["-android_thin_lto"]`,
+ "local_includes": `["."]`,
+ }),
+ MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{
+ "features": `["-android_thin_lto"]`,
+ "local_includes": `["."]`,
+ }),
+ },
+ })
+}
+
+func TestCcLibraryWithThinLtoArchSpecific(t *testing.T) {
+ runCcLibraryTestCase(t, Bp2buildTestCase{
+ Description: "cc_library has correct features when LTO differs across arch and os variants",
+ ModuleTypeUnderTest: "cc_library",
+ ModuleTypeUnderTestFactory: cc.LibraryFactory,
+ Blueprint: `
+cc_library {
+ name: "foo",
+ target: {
+ android: {
+ lto: {
+ thin: true,
+ },
+ },
+ },
+ arch: {
+ riscv64: {
+ lto: {
+ thin: false,
+ },
+ },
+ },
+}`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `select({
+ "//build/bazel/platforms/os_arch:android_arm": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_arm64": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_riscv64": ["-android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_x86": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_x86_64": ["android_thin_lto"],
+ "//conditions:default": [],
+ })`}),
+ MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `select({
+ "//build/bazel/platforms/os_arch:android_arm": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_arm64": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_riscv64": ["-android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_x86": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_x86_64": ["android_thin_lto"],
+ "//conditions:default": [],
+ })`}),
+ },
+ })
+}
+
+func TestCcLibraryWithThinLtoDisabledDefaultEnabledVariant(t *testing.T) {
+ runCcLibraryTestCase(t, Bp2buildTestCase{
+ Description: "cc_library has correct features when LTO disabled by default but enabled on a particular variant",
+ ModuleTypeUnderTest: "cc_library",
+ ModuleTypeUnderTestFactory: cc.LibraryFactory,
+ Blueprint: `
+cc_library {
+ name: "foo",
+ lto: {
+ never: true,
+ },
+ target: {
+ android: {
+ lto: {
+ thin: true,
+ never: false,
+ },
+ },
+ },
+}`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `select({
+ "//build/bazel/platforms/os:android": ["android_thin_lto"],
+ "//conditions:default": ["-android_thin_lto"],
+ })`,
+ }),
+ MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `select({
+ "//build/bazel/platforms/os:android": ["android_thin_lto"],
+ "//conditions:default": ["-android_thin_lto"],
+ })`,
+ }),
+ },
+ })
+}
+
+func TestCcLibraryWithThinLtoWholeProgramVtables(t *testing.T) {
+ runCcLibraryTestCase(t, Bp2buildTestCase{
+ Description: "cc_library has correct features when thin LTO is enabled with whole_program_vtables",
+ ModuleTypeUnderTest: "cc_library",
+ ModuleTypeUnderTestFactory: cc.LibraryFactory,
+ Blueprint: `
+cc_library {
+ name: "foo",
+ lto: {
+ thin: true,
+ },
+ whole_program_vtables: true,
+}`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_static", "foo_bp2build_cc_library_static", AttrNameToString{
+ "features": `[
+ "android_thin_lto",
+ "android_thin_lto_whole_program_vtables",
+ ]`,
+ "local_includes": `["."]`,
+ }),
+ MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{
+ "features": `[
+ "android_thin_lto",
+ "android_thin_lto_whole_program_vtables",
+ ]`,
+ "local_includes": `["."]`,
+ }),
+ },
+ })
+}
diff --git a/bp2build/cc_library_shared_conversion_test.go b/bp2build/cc_library_shared_conversion_test.go
index 6207421..838b297 100644
--- a/bp2build/cc_library_shared_conversion_test.go
+++ b/bp2build/cc_library_shared_conversion_test.go
@@ -974,3 +974,133 @@
},
})
}
+
+func TestCcLibrarySharedWithThinLto(t *testing.T) {
+ runCcLibrarySharedTestCase(t, Bp2buildTestCase{
+ Description: "cc_library_shared has correct features when thin lto is enabled",
+ Blueprint: `
+cc_library_shared {
+ name: "foo",
+ lto: {
+ thin: true,
+ },
+}
+`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{
+ "features": `["android_thin_lto"]`,
+ "local_includes": `["."]`,
+ }),
+ },
+ })
+}
+
+func TestCcLibrarySharedWithLtoNever(t *testing.T) {
+ runCcLibrarySharedTestCase(t, Bp2buildTestCase{
+ Description: "cc_library_shared has correct features when thin lto is enabled",
+ Blueprint: `
+cc_library_shared {
+ name: "foo",
+ lto: {
+ never: true,
+ },
+}
+`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{
+ "features": `["-android_thin_lto"]`,
+ "local_includes": `["."]`,
+ }),
+ },
+ })
+}
+
+func TestCcLibrarySharedWithThinLtoArchSpecific(t *testing.T) {
+ runCcLibrarySharedTestCase(t, Bp2buildTestCase{
+ Description: "cc_library_shared has correct features when LTO differs across arch and os variants",
+ Blueprint: `
+cc_library_shared {
+ name: "foo",
+ target: {
+ android: {
+ lto: {
+ thin: true,
+ },
+ },
+ },
+ arch: {
+ riscv64: {
+ lto: {
+ thin: false,
+ },
+ },
+ },
+}`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `select({
+ "//build/bazel/platforms/os_arch:android_arm": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_arm64": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_riscv64": ["-android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_x86": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_x86_64": ["android_thin_lto"],
+ "//conditions:default": [],
+ })`}),
+ },
+ })
+}
+
+func TestCcLibrarySharedWithThinLtoDisabledDefaultEnabledVariant(t *testing.T) {
+ runCcLibrarySharedTestCase(t, Bp2buildTestCase{
+ Description: "cc_library_shared with thin lto disabled by default but enabled on a particular variant",
+ Blueprint: `
+cc_library_shared {
+ name: "foo",
+ lto: {
+ never: true,
+ },
+ target: {
+ android: {
+ lto: {
+ thin: true,
+ never: false,
+ },
+ },
+ },
+}`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `select({
+ "//build/bazel/platforms/os:android": ["android_thin_lto"],
+ "//conditions:default": ["-android_thin_lto"],
+ })`,
+ }),
+ },
+ })
+}
+
+func TestCcLibrarySharedWithThinLtoAndWholeProgramVtables(t *testing.T) {
+ runCcLibrarySharedTestCase(t, Bp2buildTestCase{
+ Description: "cc_library_shared has correct features when thin LTO is enabled with whole_program_vtables",
+ Blueprint: `
+cc_library_shared {
+ name: "foo",
+ lto: {
+ thin: true,
+ },
+ whole_program_vtables: true,
+}
+`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_shared", "foo", AttrNameToString{
+ "features": `[
+ "android_thin_lto",
+ "android_thin_lto_whole_program_vtables",
+ ]`,
+ "local_includes": `["."]`,
+ }),
+ },
+ })
+}
diff --git a/bp2build/cc_library_static_conversion_test.go b/bp2build/cc_library_static_conversion_test.go
index d5256f6..d16c5cc 100644
--- a/bp2build/cc_library_static_conversion_test.go
+++ b/bp2build/cc_library_static_conversion_test.go
@@ -1889,3 +1889,133 @@
},
})
}
+
+func TestCcLibraryStaticWithThinLto(t *testing.T) {
+ runCcLibraryStaticTestCase(t, Bp2buildTestCase{
+ Description: "cc_library_static has correct features when thin lto is enabled",
+ Blueprint: `
+cc_library_static {
+ name: "foo",
+ lto: {
+ thin: true,
+ },
+}
+`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_static", "foo", AttrNameToString{
+ "features": `["android_thin_lto"]`,
+ "local_includes": `["."]`,
+ }),
+ },
+ })
+}
+
+func TestCcLibraryStaticWithLtoNever(t *testing.T) {
+ runCcLibraryStaticTestCase(t, Bp2buildTestCase{
+ Description: "cc_library_static has correct features when thin lto is enabled",
+ Blueprint: `
+cc_library_static {
+ name: "foo",
+ lto: {
+ never: true,
+ },
+}
+`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_static", "foo", AttrNameToString{
+ "features": `["-android_thin_lto"]`,
+ "local_includes": `["."]`,
+ }),
+ },
+ })
+}
+
+func TestCcLibraryStaticWithThinLtoArchSpecific(t *testing.T) {
+ runCcLibraryStaticTestCase(t, Bp2buildTestCase{
+ Description: "cc_library_static has correct features when LTO differs across arch and os variants",
+ Blueprint: `
+cc_library_static {
+ name: "foo",
+ target: {
+ android: {
+ lto: {
+ thin: true,
+ },
+ },
+ },
+ arch: {
+ riscv64: {
+ lto: {
+ thin: false,
+ },
+ },
+ },
+}`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_static", "foo", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `select({
+ "//build/bazel/platforms/os_arch:android_arm": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_arm64": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_riscv64": ["-android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_x86": ["android_thin_lto"],
+ "//build/bazel/platforms/os_arch:android_x86_64": ["android_thin_lto"],
+ "//conditions:default": [],
+ })`}),
+ },
+ })
+}
+
+func TestCcLibraryStaticWithThinLtoDisabledDefaultEnabledVariant(t *testing.T) {
+ runCcLibraryStaticTestCase(t, Bp2buildTestCase{
+ Description: "cc_library_static has correct features when LTO disabled by default but enabled on a particular variant",
+ Blueprint: `
+cc_library_static {
+ name: "foo",
+ lto: {
+ never: true,
+ },
+ target: {
+ android: {
+ lto: {
+ thin: true,
+ never: false,
+ },
+ },
+ },
+}`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_static", "foo", AttrNameToString{
+ "local_includes": `["."]`,
+ "features": `select({
+ "//build/bazel/platforms/os:android": ["android_thin_lto"],
+ "//conditions:default": ["-android_thin_lto"],
+ })`,
+ }),
+ },
+ })
+}
+
+func TestCcLibraryStaticWithThinLtoAndWholeProgramVtables(t *testing.T) {
+ runCcLibraryStaticTestCase(t, Bp2buildTestCase{
+ Description: "cc_library_static has correct features when thin lto is enabled with whole_program_vtables",
+ Blueprint: `
+cc_library_static {
+ name: "foo",
+ lto: {
+ thin: true,
+ },
+ whole_program_vtables: true,
+}
+`,
+ ExpectedBazelTargets: []string{
+ MakeBazelTarget("cc_library_static", "foo", AttrNameToString{
+ "features": `[
+ "android_thin_lto",
+ "android_thin_lto_whole_program_vtables",
+ ]`,
+ "local_includes": `["."]`,
+ }),
+ },
+ })
+}
diff --git a/cc/bp2build.go b/cc/bp2build.go
index 9751a2e..1ea8bda 100644
--- a/cc/bp2build.go
+++ b/cc/bp2build.go
@@ -817,6 +817,7 @@
compilerAttrs.hdrs.Prepend = true
features := compilerAttrs.features.Clone().Append(linkerAttrs.features).Append(bp2buildSanitizerFeatures(ctx, module))
+ features = features.Append(bp2buildLtoFeatures(ctx, module))
features.DeduplicateAxesFromBase()
addMuslSystemDynamicDeps(ctx, linkerAttrs)
@@ -1459,3 +1460,49 @@
})
return sanitizerFeatures
}
+
+func bp2buildLtoFeatures(ctx android.BazelConversionPathContext, m *Module) bazel.StringListAttribute {
+ lto_feature_name := "android_thin_lto"
+ ltoBoolFeatures := bazel.BoolAttribute{}
+ bp2BuildPropParseHelper(ctx, m, <OProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) {
+ if ltoProps, ok := props.(*LTOProperties); ok {
+ thinProp := ltoProps.Lto.Thin != nil && *ltoProps.Lto.Thin
+ thinPropSetToFalse := ltoProps.Lto.Thin != nil && !*ltoProps.Lto.Thin
+ neverProp := ltoProps.Lto.Never != nil && *ltoProps.Lto.Never
+ if thinProp {
+ ltoBoolFeatures.SetSelectValue(axis, config, BoolPtr(true))
+ return
+ }
+ if neverProp || thinPropSetToFalse {
+ if thinProp {
+ ctx.ModuleErrorf("lto.thin and lto.never are mutually exclusive but were specified together")
+ } else {
+ ltoBoolFeatures.SetSelectValue(axis, config, BoolPtr(false))
+ }
+ return
+ }
+ }
+ ltoBoolFeatures.SetSelectValue(axis, config, nil)
+ })
+
+ props := m.GetArchVariantProperties(ctx, <OProperties{})
+ ltoStringFeatures, err := ltoBoolFeatures.ToStringListAttribute(func(boolPtr *bool, axis bazel.ConfigurationAxis, config string) []string {
+ if boolPtr == nil {
+ return []string{}
+ }
+ if !*boolPtr {
+ return []string{"-" + lto_feature_name}
+ }
+ features := []string{lto_feature_name}
+ if ltoProps, ok := props[axis][config].(*LTOProperties); ok {
+ if ltoProps.Whole_program_vtables != nil && *ltoProps.Whole_program_vtables {
+ features = append(features, "android_thin_lto_whole_program_vtables")
+ }
+ }
+ return features
+ })
+ if err != nil {
+ ctx.ModuleErrorf("Error processing LTO attributes: %s", err)
+ }
+ return ltoStringFeatures
+}
diff --git a/cc/config/global.go b/cc/config/global.go
index d65f883..05dc773 100644
--- a/cc/config/global.go
+++ b/cc/config/global.go
@@ -192,10 +192,6 @@
}
noOverrideGlobalCflags = []string{
- // Workaround for boot loop caused by stack protector.
- // http://b/267839238
- "-mllvm -disable-check-noreturn-call",
-
"-Werror=bool-operation",
"-Werror=implicit-int-float-conversion",
"-Werror=int-in-bool-context",
@@ -257,7 +253,6 @@
"-Wno-bitwise-instead-of-logical",
"-Wno-misleading-indentation",
"-Wno-array-parameter",
- "-Wno-gnu-offsetof-extensions",
}
// Extra cflags for external third-party projects to disable warnings that
@@ -310,8 +305,8 @@
// prebuilts/clang default settings.
ClangDefaultBase = "prebuilts/clang/host"
- ClangDefaultVersion = "clang-r487747"
- ClangDefaultShortVersion = "17"
+ ClangDefaultVersion = "clang-r475365b"
+ ClangDefaultShortVersion = "16.0.2"
// Directories with warnings from Android.bp files.
WarningAllowedProjects = []string{
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index 5f27fa7..5c187f6 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -81,6 +81,7 @@
flag.BoolVar(&cmdlineArgs.BazelMode, "bazel-mode", false, "use bazel for analysis of certain modules")
flag.BoolVar(&cmdlineArgs.BazelModeStaging, "bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules")
flag.BoolVar(&cmdlineArgs.BazelModeDev, "bazel-mode-dev", false, "use bazel for analysis of a large number of modules (less stable)")
+ flag.BoolVar(&cmdlineArgs.UseBazelProxy, "use-bazel-proxy", false, "communicate with bazel using unix socket proxy instead of spawning subprocesses")
// Flags that probably shouldn't be flags of soong_build, but we haven't found
// the time to remove them yet
diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go
index fd718c2..ae026ba 100644
--- a/cmd/soong_ui/main.go
+++ b/cmd/soong_ui/main.go
@@ -200,6 +200,7 @@
rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
bp2buildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bp2build_metrics.pb")
bazelMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bazel_metrics.pb")
+ soongBuildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_build_metrics.pb")
//the profile file generated by Bazel"
bazelProfileFile := filepath.Join(logsDir, c.logsPrefix+"analyzed_bazel_profile.txt")
@@ -209,6 +210,7 @@
bp2buildMetricsFile, // high level metrics related to bp2build.
soongMetricsFile, // high level metrics related to this build system.
bazelMetricsFile, // high level metrics related to bazel execution
+ soongBuildMetricsFile, // high level metrics related to soong build(except bp2build)
config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
}
diff --git a/java/app_test.go b/java/app_test.go
index c77f29d..5b16cea 100644
--- a/java/app_test.go
+++ b/java/app_test.go
@@ -1005,6 +1005,7 @@
platformSdkInt int
platformSdkCodename string
platformSdkFinal bool
+ minSdkVersionBp string
expectedMinSdkVersion string
platformApis bool
activeCodenames []string
@@ -1052,6 +1053,14 @@
platformSdkCodename: "S",
activeCodenames: []string{"S"},
},
+ {
+ name: "two active SDKs",
+ sdkVersion: "module_current",
+ minSdkVersionBp: "UpsideDownCake",
+ expectedMinSdkVersion: "UpsideDownCake", // And not VanillaIceCream
+ platformSdkCodename: "VanillaIceCream",
+ activeCodenames: []string{"UpsideDownCake", "VanillaIceCream"},
+ },
}
for _, moduleType := range []string{"android_app", "android_library"} {
@@ -1061,12 +1070,17 @@
if test.platformApis {
platformApiProp = "platform_apis: true,"
}
+ minSdkVersionProp := ""
+ if test.minSdkVersionBp != "" {
+ minSdkVersionProp = fmt.Sprintf(` min_sdk_version: "%s",`, test.minSdkVersionBp)
+ }
bp := fmt.Sprintf(`%s {
name: "foo",
srcs: ["a.java"],
sdk_version: "%s",
%s
- }`, moduleType, test.sdkVersion, platformApiProp)
+ %s
+ }`, moduleType, test.sdkVersion, platformApiProp, minSdkVersionProp)
result := android.GroupFixturePreparers(
prepareForJavaTest,
diff --git a/java/config/config.go b/java/config/config.go
index 293eb92..838d007 100644
--- a/java/config/config.go
+++ b/java/config/config.go
@@ -93,9 +93,11 @@
"-JXmx4096M",
"-JXX:+TieredCompilation",
"-JXX:TieredStopAtLevel=1",
+ "-JDcom.android.tools.r8.emitRecordAnnotationsInDex",
}, dexerJavaVmFlagsList...))
exportedVars.ExportStringListStaticVariable("R8Flags", append([]string{
"-JXmx2048M",
+ "-JDcom.android.tools.r8.emitRecordAnnotationsInDex",
}, dexerJavaVmFlagsList...))
exportedVars.ExportStringListStaticVariable("CommonJdkFlags", []string{
diff --git a/java/lint.go b/java/lint.go
index a457d44..58b43df 100644
--- a/java/lint.go
+++ b/java/lint.go
@@ -96,9 +96,10 @@
}
type lintOutputs struct {
- html android.Path
- text android.Path
- xml android.Path
+ html android.Path
+ text android.Path
+ xml android.Path
+ referenceBaseline android.Path
depSets LintDepSets
}
@@ -450,7 +451,7 @@
html := android.PathForModuleOut(ctx, "lint", "lint-report.html")
text := android.PathForModuleOut(ctx, "lint", "lint-report.txt")
xml := android.PathForModuleOut(ctx, "lint", "lint-report.xml")
- baseline := android.PathForModuleOut(ctx, "lint", "lint-baseline.xml")
+ referenceBaseline := android.PathForModuleOut(ctx, "lint", "lint-baseline.xml")
depSetsBuilder := NewLintDepSetBuilder().Direct(html, text, xml)
@@ -513,7 +514,7 @@
cmd.FlagWithInput("--baseline ", lintBaseline.Path())
}
- cmd.FlagWithOutput("--write-reference-baseline ", baseline)
+ cmd.FlagWithOutput("--write-reference-baseline ", referenceBaseline)
cmd.Text("; EXITCODE=$?; ")
@@ -535,9 +536,10 @@
rule.Build("lint", "lint")
l.outputs = lintOutputs{
- html: html,
- text: text,
- xml: xml,
+ html: html,
+ text: text,
+ xml: xml,
+ referenceBaseline: referenceBaseline,
depSets: depSetsBuilder.Build(),
}
@@ -569,9 +571,10 @@
}
type lintSingleton struct {
- htmlZip android.WritablePath
- textZip android.WritablePath
- xmlZip android.WritablePath
+ htmlZip android.WritablePath
+ textZip android.WritablePath
+ xmlZip android.WritablePath
+ referenceBaselineZip android.WritablePath
}
func (l *lintSingleton) GenerateBuildActions(ctx android.SingletonContext) {
@@ -684,12 +687,15 @@
l.xmlZip = android.PathForOutput(ctx, "lint-report-xml.zip")
zip(l.xmlZip, func(l *lintOutputs) android.Path { return l.xml })
- ctx.Phony("lint-check", l.htmlZip, l.textZip, l.xmlZip)
+ l.referenceBaselineZip = android.PathForOutput(ctx, "lint-report-reference-baselines.zip")
+ zip(l.referenceBaselineZip, func(l *lintOutputs) android.Path { return l.referenceBaseline })
+
+ ctx.Phony("lint-check", l.htmlZip, l.textZip, l.xmlZip, l.referenceBaselineZip)
}
func (l *lintSingleton) MakeVars(ctx android.MakeVarsContext) {
if !ctx.Config().UnbundledBuild() {
- ctx.DistForGoal("lint-check", l.htmlZip, l.textZip, l.xmlZip)
+ ctx.DistForGoal("lint-check", l.htmlZip, l.textZip, l.xmlZip, l.referenceBaselineZip)
}
}
diff --git a/scripts/check_boot_jars/package_allowed_list.txt b/scripts/check_boot_jars/package_allowed_list.txt
index ce461b1..869fd3f 100644
--- a/scripts/check_boot_jars/package_allowed_list.txt
+++ b/scripts/check_boot_jars/package_allowed_list.txt
@@ -12,6 +12,7 @@
java\.lang\.invoke
java\.lang\.ref
java\.lang\.reflect
+java\.lang\.runtime
java\.math
java\.net
java\.nio
diff --git a/tests/bp2build_bazel_test.sh b/tests/bp2build_bazel_test.sh
index 878b4a1..1ff1b5b 100755
--- a/tests/bp2build_bazel_test.sh
+++ b/tests/bp2build_bazel_test.sh
@@ -21,6 +21,68 @@
fi
}
+# Tests that, if bp2build reruns due to a blueprint file changing, that
+# BUILD files whose contents are unchanged are not regenerated.
+function test_bp2build_unchanged {
+ setup
+
+ mkdir -p pkg
+ touch pkg/x.txt
+ cat > pkg/Android.bp <<'EOF'
+filegroup {
+ name: "x",
+ srcs: ["x.txt"],
+ bazel_module: {bp2build_available: true},
+ }
+EOF
+
+ run_soong bp2build
+ local -r buildfile_mtime1=$(stat -c "%y" out/soong/bp2build/pkg/BUILD.bazel)
+ local -r marker_mtime1=$(stat -c "%y" out/soong/bp2build_workspace_marker)
+
+ # Force bp2build to rerun by updating the timestamp of a blueprint file.
+ touch pkg/Android.bp
+
+ run_soong bp2build
+ local -r buildfile_mtime2=$(stat -c "%y" out/soong/bp2build/pkg/BUILD.bazel)
+ local -r marker_mtime2=$(stat -c "%y" out/soong/bp2build_workspace_marker)
+
+ if [[ "$marker_mtime1" == "$marker_mtime2" ]]; then
+ fail "Expected bp2build marker file to change"
+ fi
+ if [[ "$buildfile_mtime1" != "$buildfile_mtime2" ]]; then
+ fail "BUILD.bazel was updated even though contents are same"
+ fi
+}
+
+# Tests that blueprint files that are deleted are not present when the
+# bp2build tree is regenerated.
+function test_bp2build_deleted_blueprint {
+ setup
+
+ mkdir -p pkg
+ touch pkg/x.txt
+ cat > pkg/Android.bp <<'EOF'
+filegroup {
+ name: "x",
+ srcs: ["x.txt"],
+ bazel_module: {bp2build_available: true},
+ }
+EOF
+
+ run_soong bp2build
+ if [[ ! -e "./out/soong/bp2build/pkg/BUILD.bazel" ]]; then
+ fail "Expected pkg/BUILD.bazel to be generated"
+ fi
+
+ rm pkg/Android.bp
+
+ run_soong bp2build
+ if [[ -e "./out/soong/bp2build/pkg/BUILD.bazel" ]]; then
+ fail "Expected pkg/BUILD.bazel to be deleted"
+ fi
+}
+
function test_bp2build_null_build_with_globs {
setup
diff --git a/tests/persistent_bazel_test.sh b/tests/persistent_bazel_test.sh
new file mode 100755
index 0000000..4e2982a
--- /dev/null
+++ b/tests/persistent_bazel_test.sh
@@ -0,0 +1,83 @@
+#!/bin/bash -eu
+
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o pipefail
+
+source "$(dirname "$0")/lib.sh"
+
+# This test verifies that adding USE_PERSISTENT_BAZEL creates a Bazel process
+# that outlasts the build process.
+# This test should only be run in sandboxed environments (because this test
+# verifies a Bazel process using global process list, and may spawn lingering
+# Bazel processes).
+function test_persistent_bazel {
+ setup
+
+ # Ensure no existing Bazel process.
+ if [[ -e out/bazel/output/server/server.pid.txt ]]; then
+ kill $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null || true
+ if kill -0 $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null ; then
+ fail "Error killing pre-setup bazel"
+ fi
+ fi
+
+ USE_PERSISTENT_BAZEL=1 run_soong nothing
+
+ if ! kill -0 $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null ; then
+ fail "Persistent bazel process expected, but not found after first build"
+ fi
+ BAZEL_PID=$(cat out/bazel/output/server/server.pid.txt)
+
+ USE_PERSISTENT_BAZEL=1 run_soong nothing
+
+ if ! kill -0 $BAZEL_PID 2>/dev/null ; then
+ fail "Bazel pid $BAZEL_PID was killed after second build"
+ fi
+
+ kill $BAZEL_PID 2>/dev/null
+ if ! kill -0 $BAZEL_PID 2>/dev/null ; then
+ fail "Error killing bazel on shutdown"
+ fi
+}
+
+# Verifies that USE_PERSISTENT_BAZEL mode operates as expected in the event
+# that there are Bazel failures.
+function test_bazel_failure {
+ setup
+
+ # Ensure no existing Bazel process.
+ if [[ -e out/bazel/output/server/server.pid.txt ]]; then
+ kill $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null || true
+ if kill -0 $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null ; then
+ fail "Error killing pre-setup bazel"
+ fi
+ fi
+
+ # Introduce a syntax error in a BUILD file which is used in every build
+ # (Note this is a BUILD file which is copied as part of test setup, so this
+ # has no effect on sources outside of this test.
+ rm -rf build/bazel/rules
+
+ USE_PERSISTENT_BAZEL=1 run_soong nothing 1>out/failurelog.txt 2>&1 && fail "Expected build failure" || true
+
+ if ! grep -sq "'build/bazel/rules' is not a package" out/failurelog.txt ; then
+ fail "Expected error to contain 'build/bazel/rules' is not a package, instead got:\n$(cat out/failurelog.txt)"
+ fi
+
+ kill $(cat out/bazel/output/server/server.pid.txt) 2>/dev/null || true
+}
+
+scan_and_run_tests
diff --git a/tests/run_integration_tests.sh b/tests/run_integration_tests.sh
index 8ba2984..a91ccf4 100755
--- a/tests/run_integration_tests.sh
+++ b/tests/run_integration_tests.sh
@@ -7,6 +7,7 @@
"$TOP/build/soong/tests/bootstrap_test.sh"
"$TOP/build/soong/tests/mixed_mode_test.sh"
"$TOP/build/soong/tests/bp2build_bazel_test.sh"
+"$TOP/build/soong/tests/persistent_bazel_test.sh"
"$TOP/build/soong/tests/soong_test.sh"
"$TOP/build/bazel/ci/rbc_regression_test.sh" aosp_arm64-userdebug
diff --git a/ui/build/Android.bp b/ui/build/Android.bp
index 7a8fca9..b79754c 100644
--- a/ui/build/Android.bp
+++ b/ui/build/Android.bp
@@ -50,6 +50,7 @@
"cleanbuild.go",
"config.go",
"context.go",
+ "staging_snapshot.go",
"dumpvars.go",
"environment.go",
"exec.go",
@@ -70,10 +71,11 @@
"cleanbuild_test.go",
"config_test.go",
"environment_test.go",
+ "proc_sync_test.go",
"rbe_test.go",
+ "staging_snapshot_test.go",
"upload_test.go",
"util_test.go",
- "proc_sync_test.go",
],
darwin: {
srcs: [
diff --git a/ui/build/build.go b/ui/build/build.go
index d49a754..edc595d 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -102,9 +102,9 @@
// Whether to include the kati-generated ninja file in the combined ninja.
RunKatiNinja = 1 << iota
// Whether to run ninja on the combined ninja.
- RunNinja = 1 << iota
- RunBuildTests = 1 << iota
- RunAll = RunProductConfig | RunSoong | RunKati | RunKatiNinja | RunNinja
+ RunNinja = 1 << iota
+ RunDistActions = 1 << iota
+ RunBuildTests = 1 << iota
)
// checkBazelMode fails the build if there are conflicting arguments for which bazel
@@ -322,34 +322,42 @@
runNinjaForBuild(ctx, config)
}
+
+ if what&RunDistActions != 0 {
+ runDistActions(ctx, config)
+ }
}
func evaluateWhatToRun(config Config, verboseln func(v ...interface{})) int {
//evaluate what to run
- what := RunAll
+ what := 0
if config.Checkbuild() {
what |= RunBuildTests
}
- if config.SkipConfig() {
+ if !config.SkipConfig() {
+ what |= RunProductConfig
+ } else {
verboseln("Skipping Config as requested")
- what = what &^ RunProductConfig
}
- if config.SkipKati() {
- verboseln("Skipping Kati as requested")
- what = what &^ RunKati
- }
- if config.SkipKatiNinja() {
- verboseln("Skipping use of Kati ninja as requested")
- what = what &^ RunKatiNinja
- }
- if config.SkipSoong() {
+ if !config.SkipSoong() {
+ what |= RunSoong
+ } else {
verboseln("Skipping use of Soong as requested")
- what = what &^ RunSoong
}
-
- if config.SkipNinja() {
+ if !config.SkipKati() {
+ what |= RunKati
+ } else {
+ verboseln("Skipping Kati as requested")
+ }
+ if !config.SkipKatiNinja() {
+ what |= RunKatiNinja
+ } else {
+ verboseln("Skipping use of Kati ninja as requested")
+ }
+ if !config.SkipNinja() {
+ what |= RunNinja
+ } else {
verboseln("Skipping Ninja as requested")
- what = what &^ RunNinja
}
if !config.SoongBuildInvocationNeeded() {
@@ -361,6 +369,11 @@
what = what &^ RunNinja
what = what &^ RunKati
}
+
+ if config.Dist() {
+ what |= RunDistActions
+ }
+
return what
}
@@ -419,3 +432,9 @@
}
}()
}
+
+// Actions to run on every build where 'dist' is in the actions.
+// Be careful, anything added here slows down EVERY CI build
+func runDistActions(ctx Context, config Config) {
+ runStagingSnapshot(ctx, config)
+}
diff --git a/ui/build/config.go b/ui/build/config.go
index 73e2c45..b5ee440 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -1568,6 +1568,10 @@
return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL")
}
+func (c *configImpl) IsPersistentBazelEnabled() bool {
+ return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL")
+}
+
func (c *configImpl) BazelModulesForceEnabledByFlag() string {
return c.bazelForceEnabledModules
}
diff --git a/ui/build/soong.go b/ui/build/soong.go
index e6543ec..a5a3263 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -21,6 +21,7 @@
"strconv"
"strings"
+ "android/soong/bazel"
"android/soong/ui/metrics"
"android/soong/ui/status"
@@ -268,6 +269,9 @@
if config.bazelStagingMode {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging")
}
+ if config.IsPersistentBazelEnabled() {
+ mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--use-bazel-proxy")
+ }
if len(config.bazelForceEnabledModules) > 0 {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-force-enabled-modules="+config.bazelForceEnabledModules)
}
@@ -497,6 +501,12 @@
ctx.BeginTrace(metrics.RunSoong, name)
defer ctx.EndTrace()
+ if config.IsPersistentBazelEnabled() {
+ bazelProxy := bazel.NewProxyServer(ctx.Logger, config.OutDir(), filepath.Join(config.SoongOutDir(), "workspace"))
+ bazelProxy.Start()
+ defer bazelProxy.Close()
+ }
+
fifo := filepath.Join(config.OutDir(), ".ninja_fifo")
nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
defer nr.Close()
diff --git a/ui/build/staging_snapshot.go b/ui/build/staging_snapshot.go
new file mode 100644
index 0000000..377aa64
--- /dev/null
+++ b/ui/build/staging_snapshot.go
@@ -0,0 +1,246 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "crypto/sha1"
+ "encoding/hex"
+ "encoding/json"
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "android/soong/shared"
+ "android/soong/ui/metrics"
+)
+
+// Metadata about a staged file
+type fileEntry struct {
+ Name string `json:"name"`
+ Mode fs.FileMode `json:"mode"`
+ Size int64 `json:"size"`
+ Sha1 string `json:"sha1"`
+}
+
+func fileEntryEqual(a fileEntry, b fileEntry) bool {
+ return a.Name == b.Name && a.Mode == b.Mode && a.Size == b.Size && a.Sha1 == b.Sha1
+}
+
+func sha1_hash(filename string) (string, error) {
+ f, err := os.Open(filename)
+ if err != nil {
+ return "", err
+ }
+ defer f.Close()
+
+ h := sha1.New()
+ if _, err := io.Copy(h, f); err != nil {
+ return "", err
+ }
+
+ return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+// Subdirs of PRODUCT_OUT to scan
+var stagingSubdirs = []string{
+ "apex",
+ "cache",
+ "coverage",
+ "data",
+ "debug_ramdisk",
+ "fake_packages",
+ "installer",
+ "oem",
+ "product",
+ "ramdisk",
+ "recovery",
+ "root",
+ "sysloader",
+ "system",
+ "system_dlkm",
+ "system_ext",
+ "system_other",
+ "testcases",
+ "test_harness_ramdisk",
+ "vendor",
+ "vendor_debug_ramdisk",
+ "vendor_kernel_ramdisk",
+ "vendor_ramdisk",
+}
+
+// Return an array of stagedFileEntrys, one for each file in the staging directories inside
+// productOut
+func takeStagingSnapshot(ctx Context, productOut string, subdirs []string) ([]fileEntry, error) {
+ var outer_err error
+ if !strings.HasSuffix(productOut, "/") {
+ productOut += "/"
+ }
+ result := []fileEntry{}
+ for _, subdir := range subdirs {
+ filepath.WalkDir(productOut+subdir,
+ func(filename string, dirent fs.DirEntry, err error) error {
+ // Ignore errors. The most common one is that one of the subdirectories
+ // hasn't been built, in which case we just report it as empty.
+ if err != nil {
+ ctx.Verbosef("scanModifiedStagingOutputs error: %s", err)
+ return nil
+ }
+ if dirent.Type().IsRegular() {
+ fileInfo, _ := dirent.Info()
+ relative := strings.TrimPrefix(filename, productOut)
+ sha, err := sha1_hash(filename)
+ if err != nil {
+ outer_err = err
+ }
+ result = append(result, fileEntry{
+ Name: relative,
+ Mode: fileInfo.Mode(),
+ Size: fileInfo.Size(),
+ Sha1: sha,
+ })
+ }
+ return nil
+ })
+ }
+
+ sort.Slice(result, func(l, r int) bool { return result[l].Name < result[r].Name })
+
+ return result, outer_err
+}
+
+// Read json into an array of fileEntry. On error return empty array.
+func readJson(filename string) ([]fileEntry, error) {
+ buf, err := os.ReadFile(filename)
+ if err != nil {
+ // Not an error, just missing, which is empty.
+ return []fileEntry{}, nil
+ }
+
+ var result []fileEntry
+ err = json.Unmarshal(buf, &result)
+ if err != nil {
+ // Bad formatting. This is an error
+ return []fileEntry{}, err
+ }
+
+ return result, nil
+}
+
+// Write obj to filename.
+func writeJson(filename string, obj interface{}) error {
+ buf, err := json.MarshalIndent(obj, "", " ")
+ if err != nil {
+ return err
+ }
+
+ return os.WriteFile(filename, buf, 0660)
+}
+
+type snapshotDiff struct {
+ Added []string `json:"added"`
+ Changed []string `json:"changed"`
+ Removed []string `json:"removed"`
+}
+
+// Diff the two snapshots, returning a snapshotDiff.
+func diffSnapshots(previous []fileEntry, current []fileEntry) snapshotDiff {
+ result := snapshotDiff{
+ Added: []string{},
+ Changed: []string{},
+ Removed: []string{},
+ }
+
+ found := make(map[string]bool)
+
+ prev := make(map[string]fileEntry)
+ for _, pre := range previous {
+ prev[pre.Name] = pre
+ }
+
+ for _, cur := range current {
+ pre, ok := prev[cur.Name]
+ found[cur.Name] = true
+ // Added
+ if !ok {
+ result.Added = append(result.Added, cur.Name)
+ continue
+ }
+ // Changed
+ if !fileEntryEqual(pre, cur) {
+ result.Changed = append(result.Changed, cur.Name)
+ }
+ }
+
+ // Removed
+ for _, pre := range previous {
+ if !found[pre.Name] {
+ result.Removed = append(result.Removed, pre.Name)
+ }
+ }
+
+ // Sort the results
+ sort.Strings(result.Added)
+ sort.Strings(result.Changed)
+ sort.Strings(result.Removed)
+
+ return result
+}
+
+// Write a json files to dist:
+// - A list of which files have changed in this build.
+//
+// And record in out/soong:
+// - A list of all files in the staging directories, including their hashes.
+func runStagingSnapshot(ctx Context, config Config) {
+ ctx.BeginTrace(metrics.RunSoong, "runStagingSnapshot")
+ defer ctx.EndTrace()
+
+ snapshotFilename := shared.JoinPath(config.SoongOutDir(), "staged_files.json")
+
+ // Read the existing snapshot file. If it doesn't exist, this is a full
+ // build, so all files will be treated as new.
+ previous, err := readJson(snapshotFilename)
+ if err != nil {
+ ctx.Fatal(err)
+ return
+ }
+
+ // Take a snapshot of the current out directory
+ current, err := takeStagingSnapshot(ctx, config.ProductOut(), stagingSubdirs)
+ if err != nil {
+ ctx.Fatal(err)
+ return
+ }
+
+ // Diff the snapshots
+ diff := diffSnapshots(previous, current)
+
+ // Write the diff (use RealDistDir, not one that might have been faked for bazel)
+ err = writeJson(shared.JoinPath(config.RealDistDir(), "modified_files.json"), diff)
+ if err != nil {
+ ctx.Fatal(err)
+ return
+ }
+
+ // Update the snapshot
+ err = writeJson(snapshotFilename, current)
+ if err != nil {
+ ctx.Fatal(err)
+ return
+ }
+}
diff --git a/ui/build/staging_snapshot_test.go b/ui/build/staging_snapshot_test.go
new file mode 100644
index 0000000..7ac5443
--- /dev/null
+++ b/ui/build/staging_snapshot_test.go
@@ -0,0 +1,188 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+)
+
+func assertDeepEqual(t *testing.T, expected interface{}, actual interface{}) {
+ if !reflect.DeepEqual(actual, expected) {
+ t.Fatalf("expected:\n %#v\n actual:\n %#v", expected, actual)
+ }
+}
+
+// Make a temp directory containing the supplied contents
+func makeTempDir(files []string, directories []string, symlinks []string) string {
+ temp, _ := os.MkdirTemp("", "soon_staging_snapshot_test_")
+
+ for _, file := range files {
+ os.MkdirAll(temp+"/"+filepath.Dir(file), 0700)
+ os.WriteFile(temp+"/"+file, []byte(file), 0600)
+ }
+
+ for _, dir := range directories {
+ os.MkdirAll(temp+"/"+dir, 0770)
+ }
+
+ for _, symlink := range symlinks {
+ os.MkdirAll(temp+"/"+filepath.Dir(symlink), 0770)
+ os.Symlink(temp, temp+"/"+symlink)
+ }
+
+ return temp
+}
+
+// If this is a clean build, we won't have any preexisting files, make sure we get back an empty
+// list and not errors.
+func TestEmptyOut(t *testing.T) {
+ ctx := testContext()
+
+ temp := makeTempDir(nil, nil, nil)
+ defer os.RemoveAll(temp)
+
+ actual, _ := takeStagingSnapshot(ctx, temp, []string{"a", "e", "g"})
+
+ expected := []fileEntry{}
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure only the listed directories are picked up, and only regular files
+func TestNoExtraSubdirs(t *testing.T) {
+ ctx := testContext()
+
+ temp := makeTempDir([]string{"a/b", "a/c", "d", "e/f"}, []string{"g/h"}, []string{"e/symlink"})
+ defer os.RemoveAll(temp)
+
+ actual, _ := takeStagingSnapshot(ctx, temp, []string{"a", "e", "g"})
+
+ expected := []fileEntry{
+ {"a/b", 0600, 3, "3ec69c85a4ff96830024afeef2d4e512181c8f7b"},
+ {"a/c", 0600, 3, "592d70e4e03ee6f6780c71b0bf3b9608dbf1e201"},
+ {"e/f", 0600, 3, "9e164bef74aceede0974b857170100409efe67f1"},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure diff handles empty lists
+func TestDiffEmpty(t *testing.T) {
+ actual := diffSnapshots(nil, []fileEntry{})
+
+ expected := snapshotDiff{
+ Added: []string{},
+ Changed: []string{},
+ Removed: []string{},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure diff handles adding
+func TestDiffAdd(t *testing.T) {
+ actual := diffSnapshots([]fileEntry{
+ {"a", 0600, 1, "1234"},
+ }, []fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 2, "5678"},
+ })
+
+ expected := snapshotDiff{
+ Added: []string{"b"},
+ Changed: []string{},
+ Removed: []string{},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure diff handles changing mode
+func TestDiffChangeMode(t *testing.T) {
+ actual := diffSnapshots([]fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 2, "5678"},
+ }, []fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0600, 2, "5678"},
+ })
+
+ expected := snapshotDiff{
+ Added: []string{},
+ Changed: []string{"b"},
+ Removed: []string{},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure diff handles changing size
+func TestDiffChangeSize(t *testing.T) {
+ actual := diffSnapshots([]fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 2, "5678"},
+ }, []fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 3, "5678"},
+ })
+
+ expected := snapshotDiff{
+ Added: []string{},
+ Changed: []string{"b"},
+ Removed: []string{},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure diff handles changing contents
+func TestDiffChangeContents(t *testing.T) {
+ actual := diffSnapshots([]fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 2, "5678"},
+ }, []fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 2, "aaaa"},
+ })
+
+ expected := snapshotDiff{
+ Added: []string{},
+ Changed: []string{"b"},
+ Removed: []string{},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}
+
+// Make sure diff handles removing
+func TestDiffRemove(t *testing.T) {
+ actual := diffSnapshots([]fileEntry{
+ {"a", 0600, 1, "1234"},
+ {"b", 0700, 2, "5678"},
+ }, []fileEntry{
+ {"a", 0600, 1, "1234"},
+ })
+
+ expected := snapshotDiff{
+ Added: []string{},
+ Changed: []string{},
+ Removed: []string{"b"},
+ }
+
+ assertDeepEqual(t, expected, actual)
+}