Merge "Tell rustc to always use color output"
diff --git a/Android.bp b/Android.bp
index 42a8e5c..63de015 100644
--- a/Android.bp
+++ b/Android.bp
@@ -119,3 +119,14 @@
dexpreopt_systemserver_check {
name: "dexpreopt_systemserver_check",
}
+
+// buildinfo.prop contains common properties for system/build.prop, like ro.build.version.*
+buildinfo_prop {
+ name: "buildinfo.prop",
+
+ // not installable because this will be included to system/build.prop
+ installable: false,
+
+ // Currently, only microdroid can refer to buildinfo.prop
+ visibility: ["//packages/modules/Virtualization/microdroid"],
+}
diff --git a/android/Android.bp b/android/Android.bp
index 87b021f..d583703 100644
--- a/android/Android.bp
+++ b/android/Android.bp
@@ -36,6 +36,7 @@
"bazel.go",
"bazel_handler.go",
"bazel_paths.go",
+ "buildinfo_prop.go",
"config.go",
"config_bp2build.go",
"csuite_config.go",
diff --git a/android/allowlists/allowlists.go b/android/allowlists/allowlists.go
index 4fd4f13..28dc8b4 100644
--- a/android/allowlists/allowlists.go
+++ b/android/allowlists/allowlists.go
@@ -104,12 +104,14 @@
"external/google-benchmark": Bp2BuildDefaultTrueRecursively,
"external/googletest": Bp2BuildDefaultTrueRecursively,
"external/gwp_asan": Bp2BuildDefaultTrueRecursively,
+ "external/hamcrest": Bp2BuildDefaultTrueRecursively,
"external/icu": Bp2BuildDefaultTrueRecursively,
"external/icu/android_icu4j": Bp2BuildDefaultFalse, // java rules incomplete
"external/icu/icu4j": Bp2BuildDefaultFalse, // java rules incomplete
"external/javapoet": Bp2BuildDefaultTrueRecursively,
"external/jemalloc_new": Bp2BuildDefaultTrueRecursively,
"external/jsoncpp": Bp2BuildDefaultTrueRecursively,
+ "external/junit": Bp2BuildDefaultTrueRecursively,
"external/libcap": Bp2BuildDefaultTrueRecursively,
"external/libcxx": Bp2BuildDefaultTrueRecursively,
"external/libcxxabi": Bp2BuildDefaultTrueRecursively,
@@ -122,6 +124,7 @@
"external/pcre": Bp2BuildDefaultTrueRecursively,
"external/protobuf": Bp2BuildDefaultTrueRecursively,
"external/python/six": Bp2BuildDefaultTrueRecursively,
+ "external/rappor": Bp2BuildDefaultTrueRecursively,
"external/scudo": Bp2BuildDefaultTrueRecursively,
"external/selinux/libselinux": Bp2BuildDefaultTrueRecursively,
"external/selinux/libsepol": Bp2BuildDefaultTrueRecursively,
@@ -313,6 +316,7 @@
"host_bionic_linker_asm", // depends on extract_linker, a go binary.
"host_bionic_linker_script", // depends on extract_linker, a go binary.
"libc_musl_sysroot_bionic_arch_headers", // depends on soong_zip
+ "libc_musl_sysroot_zlib_headers", // depends on soong_zip and zip2zip
"libc_musl_sysroot_bionic_headers", // 218405924, depends on soong_zip and generates duplicate srcs
"libc_musl_sysroot_libc++_headers", "libc_musl_sysroot_libc++abi_headers", // depends on soong_zip, zip2zip
"robolectric-sqlite4java-native", // depends on soong_zip, a go binary
diff --git a/android/apex.go b/android/apex.go
index b127f74..030fbdc 100644
--- a/android/apex.go
+++ b/android/apex.go
@@ -870,10 +870,6 @@
"gemmlowp_headers": 30,
"guava-listenablefuture-prebuilt-jar": 30,
"ike-internals": 30,
- "kotlinx-coroutines-android": 28,
- "kotlinx-coroutines-android-nodeps": 30,
- "kotlinx-coroutines-core": 28,
- "kotlinx-coroutines-core-nodeps": 30,
"libbrotli": 30,
"libcrypto_static": 30,
"libeigen": 30,
diff --git a/android/bazel_handler.go b/android/bazel_handler.go
index d851a98..fa26fc8 100644
--- a/android/bazel_handler.go
+++ b/android/bazel_handler.go
@@ -28,6 +28,7 @@
"android/soong/bazel/cquery"
"android/soong/shared"
+ "github.com/google/blueprint"
"android/soong/bazel"
)
@@ -101,6 +102,9 @@
// Returns build statements which should get registered to reflect Bazel's outputs.
BuildStatementsToRegister() []bazel.BuildStatement
+
+ // Returns the depsets defined in Bazel's aquery response.
+ AqueryDepsets() []bazel.AqueryDepset
}
type bazelRunner interface {
@@ -128,6 +132,9 @@
// Build statements which should get registered to reflect Bazel's outputs.
buildStatements []bazel.BuildStatement
+
+ // Depsets which should be used for Bazel's build statements.
+ depsets []bazel.AqueryDepset
}
var _ BazelContext = &bazelContext{}
@@ -175,6 +182,10 @@
return []bazel.BuildStatement{}
}
+func (m MockBazelContext) AqueryDepsets() []bazel.AqueryDepset {
+ return []bazel.AqueryDepset{}
+}
+
var _ BazelContext = MockBazelContext{}
func (bazelCtx *bazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) {
@@ -236,6 +247,10 @@
return []bazel.BuildStatement{}
}
+func (m noopBazelContext) AqueryDepsets() []bazel.AqueryDepset {
+ return []bazel.AqueryDepset{}
+}
+
func NewBazelContext(c *config) (BazelContext, error) {
// TODO(cparsons): Assess USE_BAZEL=1 instead once "mixed Soong/Bazel builds"
// are production ready.
@@ -746,7 +761,7 @@
return err
}
- context.buildStatements, err = bazel.AqueryBuildStatements([]byte(aqueryOutput))
+ context.buildStatements, context.depsets, err = bazel.AqueryBuildStatements([]byte(aqueryOutput))
if err != nil {
return err
}
@@ -772,6 +787,10 @@
return context.buildStatements
}
+func (context *bazelContext) AqueryDepsets() []bazel.AqueryDepset {
+ return context.depsets
+}
+
func (context *bazelContext) OutputBase() string {
return context.paths.outputBase
}
@@ -804,6 +823,23 @@
ctx.AddNinjaFileDeps(file)
}
+ for _, depset := range ctx.Config().BazelContext.AqueryDepsets() {
+ var outputs []Path
+ for _, depsetDepId := range depset.TransitiveDepSetIds {
+ otherDepsetName := bazelDepsetName(depsetDepId)
+ outputs = append(outputs, PathForPhony(ctx, otherDepsetName))
+ }
+ for _, artifactPath := range depset.DirectArtifacts {
+ outputs = append(outputs, PathForBazelOut(ctx, artifactPath))
+ }
+ thisDepsetName := bazelDepsetName(depset.Id)
+ ctx.Build(pctx, BuildParams{
+ Rule: blueprint.Phony,
+ Outputs: []WritablePath{PathForPhony(ctx, thisDepsetName)},
+ Implicits: outputs,
+ })
+ }
+
// Register bazel-owned build statements (obtained from the aquery invocation).
for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() {
if len(buildStatement.Command) < 1 {
@@ -838,6 +874,10 @@
for _, inputPath := range buildStatement.InputPaths {
cmd.Implicit(PathForBazelOut(ctx, inputPath))
}
+ for _, inputDepsetId := range buildStatement.InputDepsetIds {
+ otherDepsetName := bazelDepsetName(inputDepsetId)
+ cmd.Implicit(PathForPhony(ctx, otherDepsetName))
+ }
if depfile := buildStatement.Depfile; depfile != nil {
cmd.ImplicitDepFile(PathForBazelOut(ctx, *depfile))
@@ -853,7 +893,8 @@
// build statement have later timestamps than the outputs.
rule.Restat()
- rule.Build(fmt.Sprintf("bazel %d", index), buildStatement.Mnemonic)
+ desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths)
+ rule.Build(fmt.Sprintf("bazel %d", index), desc)
}
}
@@ -882,3 +923,7 @@
osType: ctx.Os(),
}
}
+
+func bazelDepsetName(depsetId int) string {
+ return fmt.Sprintf("bazel_depset_%d", depsetId)
+}
diff --git a/android/buildinfo_prop.go b/android/buildinfo_prop.go
new file mode 100644
index 0000000..6339a71
--- /dev/null
+++ b/android/buildinfo_prop.go
@@ -0,0 +1,182 @@
+// Copyright 2022 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package android
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/google/blueprint/proptools"
+)
+
+func init() {
+ ctx := InitRegistrationContext
+ ctx.RegisterSingletonModuleType("buildinfo_prop", buildinfoPropFactory)
+}
+
+type buildinfoPropProperties struct {
+ // Whether this module is directly installable to one of the partitions. Default: true.
+ Installable *bool
+}
+
+type buildinfoPropModule struct {
+ SingletonModuleBase
+
+ properties buildinfoPropProperties
+
+ outputFilePath OutputPath
+ installPath InstallPath
+}
+
+var _ OutputFileProducer = (*buildinfoPropModule)(nil)
+
+func (p *buildinfoPropModule) installable() bool {
+ return proptools.BoolDefault(p.properties.Installable, true)
+}
+
+// OutputFileProducer
+func (p *buildinfoPropModule) OutputFiles(tag string) (Paths, error) {
+ if tag != "" {
+ return nil, fmt.Errorf("unsupported tag %q", tag)
+ }
+ return Paths{p.outputFilePath}, nil
+}
+
+func (p *buildinfoPropModule) GenerateAndroidBuildActions(ctx ModuleContext) {
+ p.outputFilePath = PathForModuleOut(ctx, p.Name()).OutputPath
+ if !ctx.Config().KatiEnabled() {
+ WriteFileRule(ctx, p.outputFilePath, "# no buildinfo.prop if kati is disabled")
+ return
+ }
+
+ rule := NewRuleBuilder(pctx, ctx)
+ cmd := rule.Command().Text("(")
+
+ writeString := func(str string) {
+ cmd.Text(`echo "` + str + `" && `)
+ }
+
+ writeString("# begin build properties")
+ writeString("# autogenerated by build/soong/android/buildinfo_prop.go")
+
+ writeProp := func(key, value string) {
+ if strings.Contains(key, "=") {
+ panic(fmt.Errorf("wrong property key %q: key must not contain '='", key))
+ }
+ writeString(key + "=" + value)
+ }
+
+ config := ctx.Config()
+
+ writeProp("ro.build.version.sdk", config.PlatformSdkVersion().String())
+ writeProp("ro.build.version.preview_sdk", config.PlatformPreviewSdkVersion())
+ writeProp("ro.build.version.codename", config.PlatformSdkCodename())
+ writeProp("ro.build.version.all_codenames", strings.Join(config.PlatformVersionActiveCodenames(), ","))
+ writeProp("ro.build.version.release", config.PlatformVersionLastStable())
+ writeProp("ro.build.version.release_or_codename", config.PlatformVersionName())
+ writeProp("ro.build.version.security_patch", config.PlatformSecurityPatch())
+ writeProp("ro.build.version.base_os", config.PlatformBaseOS())
+ writeProp("ro.build.version.min_supported_target_sdk", config.PlatformMinSupportedTargetSdkVersion())
+
+ if config.Eng() {
+ writeProp("ro.build.type", "eng")
+ } else if config.Debuggable() {
+ writeProp("ro.build.type", "userdebug")
+ } else {
+ writeProp("ro.build.type", "user")
+ }
+
+ // Currently, only a few properties are implemented to unblock microdroid use case.
+ // TODO(b/189164487): support below properties as well and replace build/make/tools/buildinfo.sh
+ /*
+ if $BOARD_USE_VBMETA_DIGTEST_IN_FINGERPRINT {
+ writeProp("ro.build.legacy.id", config.BuildID())
+ } else {
+ writeProp("ro.build.id", config.BuildId())
+ }
+ writeProp("ro.build.display.id", $BUILD_DISPLAY_ID)
+ writeProp("ro.build.version.incremental", $BUILD_NUMBER)
+ writeProp("ro.build.version.preview_sdk_fingerprint", $PLATFORM_PREVIEW_SDK_FINGERPRINT)
+ writeProp("ro.build.version.known_codenames", $PLATFORM_VERSION_KNOWN_CODENAMES)
+ writeProp("ro.build.version.release_or_preview_display", $PLATFORM_DISPLAY_VERSION)
+ writeProp("ro.build.date", `$DATE`)
+ writeProp("ro.build.date.utc", `$DATE +%s`)
+ writeProp("ro.build.user", $BUILD_USERNAME)
+ writeProp("ro.build.host", $BUILD_HOSTNAME)
+ writeProp("ro.build.tags", $BUILD_VERSION_TAGS)
+ writeProp("ro.build.flavor", $TARGET_BUILD_FLAVOR)
+ // These values are deprecated, use "ro.product.cpu.abilist"
+ // instead (see below).
+ writeString("# ro.product.cpu.abi and ro.product.cpu.abi2 are obsolete,")
+ writeString("# use ro.product.cpu.abilist instead.")
+ writeProp("ro.product.cpu.abi", $TARGET_CPU_ABI)
+ if [ -n "$TARGET_CPU_ABI2" ] {
+ writeProp("ro.product.cpu.abi2", $TARGET_CPU_ABI2)
+ }
+
+ if [ -n "$PRODUCT_DEFAULT_LOCALE" ] {
+ writeProp("ro.product.locale", $PRODUCT_DEFAULT_LOCALE)
+ }
+ writeProp("ro.wifi.channels", $PRODUCT_DEFAULT_WIFI_CHANNELS)
+ writeString("# ro.build.product is obsolete; use ro.product.device")
+ writeProp("ro.build.product", $TARGET_DEVICE)
+
+ writeString("# Do not try to parse description or thumbprint")
+ writeProp("ro.build.description", $PRIVATE_BUILD_DESC)
+ if [ -n "$BUILD_THUMBPRINT" ] {
+ writeProp("ro.build.thumbprint", $BUILD_THUMBPRINT)
+ }
+ */
+
+ writeString("# end build properties")
+
+ cmd.Text("true) > ").Output(p.outputFilePath)
+ rule.Build("build.prop", "generating build.prop")
+
+ if !p.installable() {
+ p.SkipInstall()
+ }
+
+ p.installPath = PathForModuleInstall(ctx)
+ ctx.InstallFile(p.installPath, p.Name(), p.outputFilePath)
+}
+
+func (f *buildinfoPropModule) GenerateSingletonBuildActions(ctx SingletonContext) {
+ // does nothing; buildinfo_prop is a singeton because two buildinfo modules don't make sense.
+}
+
+func (p *buildinfoPropModule) AndroidMkEntries() []AndroidMkEntries {
+ return []AndroidMkEntries{AndroidMkEntries{
+ Class: "ETC",
+ OutputFile: OptionalPathForPath(p.outputFilePath),
+ ExtraEntries: []AndroidMkExtraEntriesFunc{
+ func(ctx AndroidMkExtraEntriesContext, entries *AndroidMkEntries) {
+ entries.SetString("LOCAL_MODULE_PATH", p.installPath.String())
+ entries.SetString("LOCAL_INSTALLED_MODULE_STEM", p.outputFilePath.Base())
+ entries.SetBoolIfTrue("LOCAL_UNINSTALLABLE_MODULE", !p.installable())
+ },
+ },
+ }}
+}
+
+// buildinfo_prop module generates a build.prop file, which contains a set of common
+// system/build.prop properties, such as ro.build.version.*. Not all properties are implemented;
+// currently this module is only for microdroid.
+func buildinfoPropFactory() SingletonModule {
+ module := &buildinfoPropModule{}
+ module.AddProperties(&module.properties)
+ InitAndroidModule(module)
+ return module
+}
diff --git a/android/config.go b/android/config.go
index 9f1fd6a..ee058e8 100644
--- a/android/config.go
+++ b/android/config.go
@@ -777,6 +777,10 @@
return String(c.productVariables.Platform_base_os)
}
+func (c *config) PlatformVersionLastStable() string {
+ return String(c.productVariables.Platform_version_last_stable)
+}
+
func (c *config) MinSupportedSdkVersion() ApiLevel {
return uncheckedFinalApiLevel(19)
}
@@ -1482,6 +1486,10 @@
return c.productVariables.MissingUsesLibraries
}
+func (c *config) TargetMultitreeUpdateMeta() bool {
+ return c.productVariables.MultitreeUpdateMeta
+}
+
func (c *deviceConfig) DeviceArch() string {
return String(c.config.productVariables.DeviceArch)
}
diff --git a/android/hooks.go b/android/hooks.go
index 5e3a4a7..2ad3b5f 100644
--- a/android/hooks.go
+++ b/android/hooks.go
@@ -89,8 +89,17 @@
l.appendPrependHelper(props, proptools.PrependMatchingProperties)
}
-func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface{}) Module {
- inherited := []interface{}{&l.Module().base().commonProperties}
+func (l *loadHookContext) createModule(factory blueprint.ModuleFactory, name string, props ...interface{}) blueprint.Module {
+ return l.bp.CreateModule(factory, name, props...)
+}
+
+type createModuleContext interface {
+ Module() Module
+ createModule(blueprint.ModuleFactory, string, ...interface{}) blueprint.Module
+}
+
+func createModule(ctx createModuleContext, factory ModuleFactory, ext string, props ...interface{}) Module {
+ inherited := []interface{}{&ctx.Module().base().commonProperties}
var typeName string
if typeNameLookup, ok := ModuleTypeByFactory()[reflect.ValueOf(factory)]; ok {
@@ -101,12 +110,12 @@
filePath, _ := factoryFunc.FileLine(factoryPtr)
typeName = fmt.Sprintf("%s_%s", path.Base(filePath), factoryFunc.Name())
}
- typeName = typeName + "_loadHookModule"
+ typeName = typeName + "_" + ext
- module := l.bp.CreateModule(ModuleFactoryAdaptor(factory), typeName, append(inherited, props...)...).(Module)
+ module := ctx.createModule(ModuleFactoryAdaptor(factory), typeName, append(inherited, props...)...).(Module)
- if l.Module().base().variableProperties != nil && module.base().variableProperties != nil {
- src := l.Module().base().variableProperties
+ if ctx.Module().base().variableProperties != nil && module.base().variableProperties != nil {
+ src := ctx.Module().base().variableProperties
dst := []interface{}{
module.base().variableProperties,
// Put an empty copy of the src properties into dst so that properties in src that are not in dst
@@ -122,6 +131,10 @@
return module
}
+func (l *loadHookContext) CreateModule(factory ModuleFactory, props ...interface{}) Module {
+ return createModule(l, factory, "_loadHookModule", props...)
+}
+
func (l *loadHookContext) registerScopedModuleType(name string, factory blueprint.ModuleFactory) {
l.bp.RegisterScopedModuleType(name, factory)
}
diff --git a/android/mutator.go b/android/mutator.go
index 739e4ee..02a6143 100644
--- a/android/mutator.go
+++ b/android/mutator.go
@@ -15,12 +15,9 @@
package android
import (
- "reflect"
-
"android/soong/bazel"
"github.com/google/blueprint"
- "github.com/google/blueprint/proptools"
)
// Phases:
@@ -553,29 +550,16 @@
t.Module().base().commonProperties.DebugName = name
}
+func (t *topDownMutatorContext) createModule(factory blueprint.ModuleFactory, name string, props ...interface{}) blueprint.Module {
+ return t.bp.CreateModule(factory, name, props...)
+}
+
func (t *topDownMutatorContext) CreateModule(factory ModuleFactory, props ...interface{}) Module {
- inherited := []interface{}{&t.Module().base().commonProperties}
- module := t.bp.CreateModule(ModuleFactoryAdaptor(factory), append(inherited, props...)...).(Module)
-
- if t.Module().base().variableProperties != nil && module.base().variableProperties != nil {
- src := t.Module().base().variableProperties
- dst := []interface{}{
- module.base().variableProperties,
- // Put an empty copy of the src properties into dst so that properties in src that are not in dst
- // don't cause a "failed to find property to extend" error.
- proptools.CloneEmptyProperties(reflect.ValueOf(src)).Interface(),
- }
- err := proptools.AppendMatchingProperties(dst, src, nil)
- if err != nil {
- panic(err)
- }
- }
-
- return module
+ return createModule(t, factory, "_topDownMutatorModule", props...)
}
func (t *topDownMutatorContext) createModuleWithoutInheritance(factory ModuleFactory, props ...interface{}) Module {
- module := t.bp.CreateModule(ModuleFactoryAdaptor(factory), props...).(Module)
+ module := t.bp.CreateModule(ModuleFactoryAdaptor(factory), "", props...).(Module)
return module
}
diff --git a/android/paths.go b/android/paths.go
index e7829b9..e0e5ae5 100644
--- a/android/paths.go
+++ b/android/paths.go
@@ -1057,7 +1057,8 @@
}
// absolute path already checked by validateSafePath
- if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) {
+ // special-case api surface gen files for now
+ if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) && !strings.Contains(ret.String(), ctx.Config().soongOutDir+"/.export") {
return ret, fmt.Errorf("source path %q is in output", ret.String())
}
@@ -1073,7 +1074,8 @@
}
// absolute path already checked by validatePath
- if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) {
+ // special-case for now
+ if strings.HasPrefix(ret.String(), ctx.Config().soongOutDir) && !strings.Contains(ret.String(), ctx.Config().soongOutDir+"/.export") {
return ret, fmt.Errorf("source path %q is in output", ret.String())
}
diff --git a/android/variable.go b/android/variable.go
index 077b810..90cb6ff 100644
--- a/android/variable.go
+++ b/android/variable.go
@@ -199,6 +199,7 @@
Platform_preview_sdk_version *string `json:",omitempty"`
Platform_min_supported_target_sdk_version *string `json:",omitempty"`
Platform_base_os *string `json:",omitempty"`
+ Platform_version_last_stable *string `json:",omitempty"`
DeviceName *string `json:",omitempty"`
DeviceProduct *string `json:",omitempty"`
@@ -351,6 +352,8 @@
RecoverySnapshotDirsIncluded []string `json:",omitempty"`
HostFakeSnapshotEnabled bool `json:",omitempty"`
+ MultitreeUpdateMeta bool `json:",omitempty"`
+
BoardVendorSepolicyDirs []string `json:",omitempty"`
BoardOdmSepolicyDirs []string `json:",omitempty"`
BoardReqdMaskPolicy []string `json:",omitempty"`
diff --git a/androidmk/parser/make_strings.go b/androidmk/parser/make_strings.go
index 8030326..416e430 100644
--- a/androidmk/parser/make_strings.go
+++ b/androidmk/parser/make_strings.go
@@ -279,7 +279,7 @@
func (ms *MakeString) EndsWith(ch rune) bool {
s := ms.Strings[len(ms.Strings)-1]
- return s[len(s)-1] == uint8(ch)
+ return len(s) > 0 && s[len(s)-1] == uint8(ch)
}
func (ms *MakeString) ReplaceLiteral(input string, output string) {
diff --git a/androidmk/parser/make_strings_test.go b/androidmk/parser/make_strings_test.go
index fbb289b..e243ece 100644
--- a/androidmk/parser/make_strings_test.go
+++ b/androidmk/parser/make_strings_test.go
@@ -217,6 +217,36 @@
}
}
+var endsWithTestCases = []struct {
+ in *MakeString
+ endsWith rune
+ expected bool
+}{
+ {
+ in: genMakeString("foo", "X", "bar ="),
+ endsWith: '=',
+ expected: true,
+ },
+ {
+ in: genMakeString("foo", "X", "bar ="),
+ endsWith: ':',
+ expected: false,
+ },
+ {
+ in: genMakeString("foo", "X", ""),
+ endsWith: '=',
+ expected: false,
+ },
+}
+
+func TestMakeStringEndsWith(t *testing.T) {
+ for _, test := range endsWithTestCases {
+ if test.in.EndsWith(test.endsWith) != test.expected {
+ t.Errorf("with:\n%q\nexpected:\n%t\ngot:\n%t", test.in.Dump(), test.expected, !test.expected)
+ }
+ }
+}
+
func dumpArray(a []*MakeString) string {
ret := make([]string, len(a))
diff --git a/apex/Android.bp b/apex/Android.bp
index 41224ec..d3417c2 100644
--- a/apex/Android.bp
+++ b/apex/Android.bp
@@ -14,6 +14,7 @@
"soong-cc",
"soong-filesystem",
"soong-java",
+ "soong-multitree",
"soong-provenance",
"soong-python",
"soong-rust",
diff --git a/apex/apex.go b/apex/apex.go
index 62013cf..c779868 100644
--- a/apex/apex.go
+++ b/apex/apex.go
@@ -19,7 +19,6 @@
import (
"fmt"
"path/filepath"
- "regexp"
"sort"
"strings"
@@ -34,6 +33,7 @@
prebuilt_etc "android/soong/etc"
"android/soong/filesystem"
"android/soong/java"
+ "android/soong/multitree"
"android/soong/python"
"android/soong/rust"
"android/soong/sh"
@@ -359,6 +359,7 @@
android.OverridableModuleBase
android.SdkBase
android.BazelModuleBase
+ multitree.ExportableModuleBase
// Properties
properties apexBundleProperties
@@ -1360,6 +1361,21 @@
}
}
+var _ multitree.Exportable = (*apexBundle)(nil)
+
+func (a *apexBundle) Exportable() bool {
+ if a.properties.ApexType == flattenedApex {
+ return false
+ }
+ return true
+}
+
+func (a *apexBundle) TaggedOutputs() map[string]android.Paths {
+ ret := make(map[string]android.Paths)
+ ret["apex"] = android.Paths{a.outputFile}
+ return ret
+}
+
var _ cc.Coverage = (*apexBundle)(nil)
// Implements cc.Coverage
@@ -1657,20 +1673,7 @@
var _ androidApp = (*java.AndroidApp)(nil)
var _ androidApp = (*java.AndroidAppImport)(nil)
-func sanitizedBuildIdForPath(ctx android.BaseModuleContext) string {
- buildId := ctx.Config().BuildId()
-
- // The build ID is used as a suffix for a filename, so ensure that
- // the set of characters being used are sanitized.
- // - any word character: [a-zA-Z0-9_]
- // - dots: .
- // - dashes: -
- validRegex := regexp.MustCompile(`^[\w\.\-\_]+$`)
- if !validRegex.MatchString(buildId) {
- ctx.ModuleErrorf("Unable to use build id %s as filename suffix, valid characters are [a-z A-Z 0-9 _ . -].", buildId)
- }
- return buildId
-}
+const APEX_VERSION_PLACEHOLDER = "__APEX_VERSION_PLACEHOLDER__"
func apexFileForAndroidApp(ctx android.BaseModuleContext, aapp androidApp) apexFile {
appDir := "app"
@@ -1681,7 +1684,7 @@
// TODO(b/224589412, b/226559955): Ensure that the subdirname is suffixed
// so that PackageManager correctly invalidates the existing installed apk
// in favour of the new APK-in-APEX. See bugs for more information.
- dirInApex := filepath.Join(appDir, aapp.InstallApkName()+"@"+sanitizedBuildIdForPath(ctx))
+ dirInApex := filepath.Join(appDir, aapp.InstallApkName()+"@"+APEX_VERSION_PLACEHOLDER)
fileToCopy := aapp.OutputFile()
af := newApexFile(ctx, fileToCopy, aapp.BaseModuleName(), dirInApex, app, aapp)
@@ -1920,7 +1923,7 @@
// suffixed so that PackageManager correctly invalidates the
// existing installed apk in favour of the new APK-in-APEX.
// See bugs for more information.
- appDirName := filepath.Join(appDir, ap.BaseModuleName()+"@"+sanitizedBuildIdForPath(ctx))
+ appDirName := filepath.Join(appDir, ap.BaseModuleName()+"@"+APEX_VERSION_PLACEHOLDER)
af := newApexFile(ctx, ap.OutputFile(), ap.BaseModuleName(), appDirName, appSet, ap)
af.certificate = java.PresignedCertificate
filesInfo = append(filesInfo, af)
@@ -2386,6 +2389,7 @@
android.InitSdkAwareModule(module)
android.InitOverridableModule(module, &module.overridableProperties.Overrides)
android.InitBazelModule(module)
+ multitree.InitExportableModule(module)
return module
}
@@ -3519,7 +3523,7 @@
props := bazel.BazelTargetModuleProperties{
Rule_class: "apex",
- Bzl_load_location: "//build/bazel/rules:apex.bzl",
+ Bzl_load_location: "//build/bazel/rules/apex:apex.bzl",
}
ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: a.Name()}, attrs)
diff --git a/apex/apex_test.go b/apex/apex_test.go
index 7b29058..4a52115 100644
--- a/apex/apex_test.go
+++ b/apex/apex_test.go
@@ -223,7 +223,6 @@
// not because of these tests specifically (it's not used by the tests)
variables.Platform_version_active_codenames = []string{"Q", "Tiramisu"}
variables.Platform_vndk_version = proptools.StringPtr("29")
- variables.BuildId = proptools.StringPtr("TEST.BUILD_ID")
}),
)
@@ -683,7 +682,7 @@
"etc/myetc",
"javalib/myjar.jar",
"lib64/mylib.so",
- "app/AppFoo@TEST.BUILD_ID/AppFoo.apk",
+ "app/AppFoo@__APEX_VERSION_PLACEHOLDER__/AppFoo.apk",
"overlay/blue/rro.apk",
"etc/bpf/bpf.o",
"etc/bpf/bpf2.o",
@@ -5683,8 +5682,8 @@
apexRule := module.Rule("apexRule")
copyCmds := apexRule.Args["copy_commands"]
- ensureContains(t, copyCmds, "image.apex/app/AppFoo@TEST.BUILD_ID/AppFoo.apk")
- ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPriv@TEST.BUILD_ID/AppFooPriv.apk")
+ ensureContains(t, copyCmds, "image.apex/app/AppFoo@__APEX_VERSION_PLACEHOLDER__/AppFoo.apk")
+ ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPriv@__APEX_VERSION_PLACEHOLDER__/AppFooPriv.apk")
appZipRule := ctx.ModuleForTests("AppFoo", "android_common_apex10000").Description("zip jni libs")
// JNI libraries are uncompressed
@@ -5701,36 +5700,6 @@
}
}
-func TestApexWithAppImportBuildId(t *testing.T) {
- invalidBuildIds := []string{"../", "a b", "a/b", "a/b/../c", "/a"}
- for _, id := range invalidBuildIds {
- message := fmt.Sprintf("Unable to use build id %s as filename suffix", id)
- fixture := android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) {
- variables.BuildId = proptools.StringPtr(id)
- })
- testApexError(t, message, `apex {
- name: "myapex",
- key: "myapex.key",
- apps: ["AppFooPrebuilt"],
- updatable: false,
- }
-
- apex_key {
- name: "myapex.key",
- public_key: "testkey.avbpubkey",
- private_key: "testkey.pem",
- }
-
- android_app_import {
- name: "AppFooPrebuilt",
- apk: "PrebuiltAppFoo.apk",
- presigned: true,
- apex_available: ["myapex"],
- }
- `, fixture)
- }
-}
-
func TestApexWithAppImports(t *testing.T) {
ctx := testApex(t, `
apex {
@@ -5776,8 +5745,8 @@
apexRule := module.Rule("apexRule")
copyCmds := apexRule.Args["copy_commands"]
- ensureContains(t, copyCmds, "image.apex/app/AppFooPrebuilt@TEST.BUILD_ID/AppFooPrebuilt.apk")
- ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPrivPrebuilt@TEST.BUILD_ID/AwesomePrebuiltAppFooPriv.apk")
+ ensureContains(t, copyCmds, "image.apex/app/AppFooPrebuilt@__APEX_VERSION_PLACEHOLDER__/AppFooPrebuilt.apk")
+ ensureContains(t, copyCmds, "image.apex/priv-app/AppFooPrivPrebuilt@__APEX_VERSION_PLACEHOLDER__/AwesomePrebuiltAppFooPriv.apk")
}
func TestApexWithAppImportsPrefer(t *testing.T) {
@@ -5818,7 +5787,7 @@
}))
ensureExactContents(t, ctx, "myapex", "android_common_myapex_image", []string{
- "app/AppFoo@TEST.BUILD_ID/AppFooPrebuilt.apk",
+ "app/AppFoo@__APEX_VERSION_PLACEHOLDER__/AppFooPrebuilt.apk",
})
}
@@ -5851,7 +5820,7 @@
apexRule := module.Rule("apexRule")
copyCmds := apexRule.Args["copy_commands"]
- ensureContains(t, copyCmds, "image.apex/app/TesterHelpAppFoo@TEST.BUILD_ID/TesterHelpAppFoo.apk")
+ ensureContains(t, copyCmds, "image.apex/app/TesterHelpAppFoo@__APEX_VERSION_PLACEHOLDER__/TesterHelpAppFoo.apk")
}
func TestApexPropertiesShouldBeDefaultable(t *testing.T) {
@@ -6294,8 +6263,8 @@
apexRule := module.Rule("apexRule")
copyCmds := apexRule.Args["copy_commands"]
- ensureNotContains(t, copyCmds, "image.apex/app/app@TEST.BUILD_ID/app.apk")
- ensureContains(t, copyCmds, "image.apex/app/override_app@TEST.BUILD_ID/override_app.apk")
+ ensureNotContains(t, copyCmds, "image.apex/app/app@__APEX_VERSION_PLACEHOLDER__/app.apk")
+ ensureContains(t, copyCmds, "image.apex/app/override_app@__APEX_VERSION_PLACEHOLDER__/override_app.apk")
ensureNotContains(t, copyCmds, "image.apex/etc/bpf/bpf.o")
ensureContains(t, copyCmds, "image.apex/etc/bpf/override_bpf.o")
@@ -7199,7 +7168,7 @@
content := bundleConfigRule.Args["content"]
ensureContains(t, content, `"compression":{"uncompressed_glob":["apex_payload.img","apex_manifest.*"]}`)
- ensureContains(t, content, `"apex_config":{"apex_embedded_apk_config":[{"package_name":"com.android.foo","path":"app/AppFoo@TEST.BUILD_ID/AppFoo.apk"}]}`)
+ ensureContains(t, content, `"apex_config":{"apex_embedded_apk_config":[{"package_name":"com.android.foo","path":"app/AppFoo@__APEX_VERSION_PLACEHOLDER__/AppFoo.apk"}]}`)
}
func TestAppSetBundle(t *testing.T) {
@@ -7230,9 +7199,9 @@
if len(copyCmds) != 3 {
t.Fatalf("Expected 3 commands, got %d in:\n%s", len(copyCmds), s)
}
- ensureMatches(t, copyCmds[0], "^rm -rf .*/app/AppSet@TEST.BUILD_ID$")
- ensureMatches(t, copyCmds[1], "^mkdir -p .*/app/AppSet@TEST.BUILD_ID$")
- ensureMatches(t, copyCmds[2], "^unzip .*-d .*/app/AppSet@TEST.BUILD_ID .*/AppSet.zip$")
+ ensureMatches(t, copyCmds[0], "^rm -rf .*/app/AppSet@__APEX_VERSION_PLACEHOLDER__$")
+ ensureMatches(t, copyCmds[1], "^mkdir -p .*/app/AppSet@__APEX_VERSION_PLACEHOLDER__$")
+ ensureMatches(t, copyCmds[2], "^unzip .*-d .*/app/AppSet@__APEX_VERSION_PLACEHOLDER__ .*/AppSet.zip$")
}
func TestAppSetBundlePrebuilt(t *testing.T) {
diff --git a/apex/builder.go b/apex/builder.go
index abbf8ad..d4765d0 100644
--- a/apex/builder.go
+++ b/apex/builder.go
@@ -107,14 +107,16 @@
`--canned_fs_config ${canned_fs_config} ` +
`--include_build_info ` +
`--payload_type image ` +
- `--key ${key} ${opt_flags} ${image_dir} ${out} `,
+ `--key ${key} ` +
+ `--apex_version_placeholder ${apex_version_placeholder} ` +
+ `${opt_flags} ${image_dir} ${out} `,
CommandDeps: []string{"${apexer}", "${avbtool}", "${e2fsdroid}", "${merge_zips}",
"${mke2fs}", "${resize2fs}", "${sefcontext_compile}", "${make_f2fs}", "${sload_f2fs}", "${make_erofs}",
"${soong_zip}", "${zipalign}", "${aapt2}", "prebuilts/sdk/current/public/android.jar"},
Rspfile: "${out}.copy_commands",
RspfileContent: "${copy_commands}",
Description: "APEX ${image_dir} => ${out}",
- }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", "opt_flags", "manifest", "payload_fs_type")
+ }, "tool_path", "image_dir", "copy_commands", "file_contexts", "canned_fs_config", "key", "opt_flags", "manifest", "payload_fs_type", "apex_version_placeholder")
zipApexRule = pctx.StaticRule("zipApexRule", blueprint.RuleParams{
Command: `rm -rf ${image_dir} && mkdir -p ${image_dir} && ` +
@@ -122,12 +124,13 @@
`APEXER_TOOL_PATH=${tool_path} ` +
`${apexer} --force --manifest ${manifest} ` +
`--payload_type zip ` +
+ `--apex_version_placeholder ${apex_version_placeholder} ` +
`${image_dir} ${out} `,
CommandDeps: []string{"${apexer}", "${merge_zips}", "${soong_zip}", "${zipalign}", "${aapt2}"},
Rspfile: "${out}.copy_commands",
RspfileContent: "${copy_commands}",
Description: "ZipAPEX ${image_dir} => ${out}",
- }, "tool_path", "image_dir", "copy_commands", "manifest")
+ }, "tool_path", "image_dir", "copy_commands", "manifest", "apex_version_placeholder")
apexProtoConvertRule = pctx.AndroidStaticRule("apexProtoConvertRule",
blueprint.RuleParams{
@@ -658,14 +661,15 @@
Output: unsignedOutputFile,
Description: "apex (" + apexType.name() + ")",
Args: map[string]string{
- "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir,
- "image_dir": imageDir.String(),
- "copy_commands": strings.Join(copyCommands, " && "),
- "manifest": a.manifestPbOut.String(),
- "file_contexts": fileContexts.String(),
- "canned_fs_config": cannedFsConfig.String(),
- "key": a.privateKeyFile.String(),
- "opt_flags": strings.Join(optFlags, " "),
+ "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir,
+ "image_dir": imageDir.String(),
+ "copy_commands": strings.Join(copyCommands, " && "),
+ "manifest": a.manifestPbOut.String(),
+ "file_contexts": fileContexts.String(),
+ "canned_fs_config": cannedFsConfig.String(),
+ "key": a.privateKeyFile.String(),
+ "opt_flags": strings.Join(optFlags, " "),
+ "apex_version_placeholder": APEX_VERSION_PLACEHOLDER,
},
})
@@ -757,10 +761,11 @@
Output: unsignedOutputFile,
Description: "apex (" + apexType.name() + ")",
Args: map[string]string{
- "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir,
- "image_dir": imageDir.String(),
- "copy_commands": strings.Join(copyCommands, " && "),
- "manifest": a.manifestPbOut.String(),
+ "tool_path": outHostBinDir + ":" + prebuiltSdkToolsBinDir,
+ "image_dir": imageDir.String(),
+ "copy_commands": strings.Join(copyCommands, " && "),
+ "manifest": a.manifestPbOut.String(),
+ "apex_version_placeholder": APEX_VERSION_PLACEHOLDER,
},
})
}
diff --git a/apex/key.go b/apex/key.go
index 829410e..9c5bb05 100644
--- a/apex/key.go
+++ b/apex/key.go
@@ -230,7 +230,7 @@
props := bazel.BazelTargetModuleProperties{
Rule_class: "apex_key",
- Bzl_load_location: "//build/bazel/rules:apex_key.bzl",
+ Bzl_load_location: "//build/bazel/rules/apex:apex_key.bzl",
}
ctx.CreateBazelTargetModule(props, android.CommonAttributes{Name: module.Name()}, attrs)
diff --git a/bazel/aquery.go b/bazel/aquery.go
index fd8cf67..e05cbd6 100644
--- a/bazel/aquery.go
+++ b/bazel/aquery.go
@@ -43,6 +43,18 @@
Value string
}
+// AqueryDepset is a depset definition from Bazel's aquery response. This is
+// akin to the `depSetOfFiles` in the response proto, except that direct
+// artifacts are enumerated by full path instead of by ID.
+// A depset is a data structure for efficient transitive handling of artifact
+// paths. A single depset consists of one or more artifact paths and one or
+// more "child" depsets.
+type AqueryDepset struct {
+ Id int
+ DirectArtifacts []string
+ TransitiveDepSetIds []int
+}
+
// depSetOfFiles contains relevant portions of Bazel's aquery proto, DepSetOfFiles.
// Represents a data structure containing one or more files. Depsets in Bazel are an efficient
// data structure for storing large numbers of file paths.
@@ -79,21 +91,21 @@
Command string
Depfile *string
OutputPaths []string
- InputPaths []string
SymlinkPaths []string
Env []KeyValuePair
Mnemonic string
+
+ // Inputs of this build statement, either as unexpanded depsets or expanded
+ // input paths. There should be no overlap between these fields; an input
+ // path should either be included as part of an unexpanded depset or a raw
+ // input path string, but not both.
+ InputDepsetIds []int
+ InputPaths []string
}
// A helper type for aquery processing which facilitates retrieval of path IDs from their
// less readable Bazel structures (depset and path fragment).
type aqueryArtifactHandler struct {
- // Maps middleman artifact Id to input artifact depset ID.
- // Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example,
- // if we find a middleman action which has outputs [foo, bar], and output [baz_middleman], then,
- // for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for
- // that action instead.
- middlemanIdToDepsetIds map[int][]int
// Maps depset Id to depset struct.
depsetIdToDepset map[int]depSetOfFiles
// depsetIdToArtifactIdsCache is a memoization of depset flattening, because flattening
@@ -132,12 +144,11 @@
artifactIdToPath[artifact.Id] = artifactPath
}
- depsetIdToDepset := map[int]depSetOfFiles{}
- for _, depset := range aqueryResult.DepSetOfFiles {
- depsetIdToDepset[depset.Id] = depset
- }
-
- // Do a pass through all actions to identify which artifacts are middleman artifacts.
+ // Map middleman artifact Id to input artifact depset ID.
+ // Middleman artifacts are treated as "substitute" artifacts for mixed builds. For example,
+ // if we find a middleman action which has outputs [foo, bar], and output [baz_middleman], then,
+ // for each other action which has input [baz_middleman], we add [foo, bar] to the inputs for
+ // that action instead.
middlemanIdToDepsetIds := map[int][]int{}
for _, actionEntry := range aqueryResult.Actions {
if actionEntry.Mnemonic == "Middleman" {
@@ -146,14 +157,64 @@
}
}
}
+
+ // Store all depset IDs to validate all depset links are resolvable.
+ depsetIds := map[int]bool{}
+ for _, depset := range aqueryResult.DepSetOfFiles {
+ depsetIds[depset.Id] = true
+ }
+
+ depsetIdToDepset := map[int]depSetOfFiles{}
+ // Validate and adjust aqueryResult.DepSetOfFiles values.
+ for _, depset := range aqueryResult.DepSetOfFiles {
+ filteredArtifactIds := []int{}
+ for _, artifactId := range depset.DirectArtifactIds {
+ path, pathExists := artifactIdToPath[artifactId]
+ if !pathExists {
+ return nil, fmt.Errorf("undefined input artifactId %d", artifactId)
+ }
+ // Filter out any inputs which are universally dropped, and swap middleman
+ // artifacts with their corresponding depsets.
+ if depsetsToUse, isMiddleman := middlemanIdToDepsetIds[artifactId]; isMiddleman {
+ // Swap middleman artifacts with their corresponding depsets and drop the middleman artifacts.
+ depset.TransitiveDepSetIds = append(depset.TransitiveDepSetIds, depsetsToUse...)
+ } else if strings.HasSuffix(path, py3wrapperFileName) || manifestFilePattern.MatchString(path) {
+ // Drop these artifacts.
+ // See go/python-binary-host-mixed-build for more details.
+ // 1) For py3wrapper.sh, there is no action for creating py3wrapper.sh in the aquery output of
+ // Bazel py_binary targets, so there is no Ninja build statements generated for creating it.
+ // 2) For MANIFEST file, SourceSymlinkManifest action is in aquery output of Bazel py_binary targets,
+ // but it doesn't contain sufficient information so no Ninja build statements are generated
+ // for creating it.
+ // So in mixed build mode, when these two are used as input of some Ninja build statement,
+ // since there is no build statement to create them, they should be removed from input paths.
+ // TODO(b/197135294): Clean up this custom runfiles handling logic when
+ // SourceSymlinkManifest and SymlinkTree actions are supported.
+ } else {
+ // TODO(b/216194240): Filter out bazel tools.
+ filteredArtifactIds = append(filteredArtifactIds, artifactId)
+ }
+ }
+ depset.DirectArtifactIds = filteredArtifactIds
+ for _, childDepsetId := range depset.TransitiveDepSetIds {
+ if _, exists := depsetIds[childDepsetId]; !exists {
+ return nil, fmt.Errorf("undefined input depsetId %d (referenced by depsetId %d)", childDepsetId, depset.Id)
+ }
+ }
+ depsetIdToDepset[depset.Id] = depset
+ }
+
return &aqueryArtifactHandler{
- middlemanIdToDepsetIds: middlemanIdToDepsetIds,
depsetIdToDepset: depsetIdToDepset,
depsetIdToArtifactIdsCache: map[int][]int{},
artifactIdToPath: artifactIdToPath,
}, nil
}
+// getInputPaths flattens the depsets of the given IDs and returns all transitive
+// input paths contained in these depsets.
+// This is a potentially expensive operation, and should not be invoked except
+// for actions which need specialized input handling.
func (a *aqueryArtifactHandler) getInputPaths(depsetIds []int) ([]string, error) {
inputPaths := []string{}
@@ -163,48 +224,15 @@
return nil, err
}
for _, inputId := range inputArtifacts {
- if middlemanInputDepsetIds, isMiddlemanArtifact := a.middlemanIdToDepsetIds[inputId]; isMiddlemanArtifact {
- // Add all inputs from middleman actions which created middleman artifacts which are
- // in the inputs for this action.
- swappedInputPaths, err := a.getInputPaths(middlemanInputDepsetIds)
- if err != nil {
- return nil, err
- }
- inputPaths = append(inputPaths, swappedInputPaths...)
- } else {
- inputPath, exists := a.artifactIdToPath[inputId]
- if !exists {
- return nil, fmt.Errorf("undefined input artifactId %d", inputId)
- }
- inputPaths = append(inputPaths, inputPath)
+ inputPath, exists := a.artifactIdToPath[inputId]
+ if !exists {
+ return nil, fmt.Errorf("undefined input artifactId %d", inputId)
}
+ inputPaths = append(inputPaths, inputPath)
}
}
- // TODO(b/197135294): Clean up this custom runfiles handling logic when
- // SourceSymlinkManifest and SymlinkTree actions are supported.
- filteredInputPaths := filterOutPy3wrapperAndManifestFileFromInputPaths(inputPaths)
-
- return filteredInputPaths, nil
-}
-
-// See go/python-binary-host-mixed-build for more details.
-// 1) For py3wrapper.sh, there is no action for creating py3wrapper.sh in the aquery output of
-// Bazel py_binary targets, so there is no Ninja build statements generated for creating it.
-// 2) For MANIFEST file, SourceSymlinkManifest action is in aquery output of Bazel py_binary targets,
-// but it doesn't contain sufficient information so no Ninja build statements are generated
-// for creating it.
-// So in mixed build mode, when these two are used as input of some Ninja build statement,
-// since there is no build statement to create them, they should be removed from input paths.
-func filterOutPy3wrapperAndManifestFileFromInputPaths(inputPaths []string) []string {
- filteredInputPaths := []string{}
- for _, path := range inputPaths {
- if strings.HasSuffix(path, py3wrapperFileName) || manifestFilePattern.MatchString(path) {
- continue
- }
- filteredInputPaths = append(filteredInputPaths, path)
- }
- return filteredInputPaths
+ return inputPaths, nil
}
func (a *aqueryArtifactHandler) artifactIdsFromDepsetId(depsetId int) ([]int, error) {
@@ -227,115 +255,233 @@
}
}
-// AqueryBuildStatements returns an array of BuildStatements which should be registered (and output
-// to a ninja file) to correspond one-to-one with the given action graph json proto (from a bazel
-// aquery invocation).
-func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) {
+// AqueryBuildStatements returns a slice of BuildStatements and a slice of AqueryDepset
+// which should be registered (and output to a ninja file) to correspond with Bazel's
+// action graph, as described by the given action graph json proto.
+// BuildStatements are one-to-one with actions in the given action graph, and AqueryDepsets
+// are one-to-one with Bazel's depSetOfFiles objects.
+func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, []AqueryDepset, error) {
buildStatements := []BuildStatement{}
+ depsets := []AqueryDepset{}
var aqueryResult actionGraphContainer
err := json.Unmarshal(aqueryJsonProto, &aqueryResult)
if err != nil {
- return nil, err
+ return nil, nil, err
}
aqueryHandler, err := newAqueryHandler(aqueryResult)
if err != nil {
- return nil, err
+ return nil, nil, err
}
for _, actionEntry := range aqueryResult.Actions {
if shouldSkipAction(actionEntry) {
continue
}
- outputPaths := []string{}
- var depfile *string
- for _, outputId := range actionEntry.OutputIds {
- outputPath, exists := aqueryHandler.artifactIdToPath[outputId]
- if !exists {
- return nil, fmt.Errorf("undefined outputId %d", outputId)
- }
- ext := filepath.Ext(outputPath)
- if ext == ".d" {
- if depfile != nil {
- return nil, fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath)
- } else {
- depfile = &outputPath
- }
- } else {
- outputPaths = append(outputPaths, outputPath)
- }
- }
- inputPaths, err := aqueryHandler.getInputPaths(actionEntry.InputDepSetIds)
- if err != nil {
- return nil, err
- }
- buildStatement := BuildStatement{
- Command: strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " "),
- Depfile: depfile,
- OutputPaths: outputPaths,
- InputPaths: inputPaths,
- Env: actionEntry.EnvironmentVariables,
- Mnemonic: actionEntry.Mnemonic,
- }
-
+ var buildStatement BuildStatement
if isSymlinkAction(actionEntry) {
- if len(inputPaths) != 1 || len(outputPaths) != 1 {
- return nil, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths)
- }
- out := outputPaths[0]
- outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out))
- out = proptools.ShellEscapeIncludingSpaces(out)
- in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0]))
- // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`).
- buildStatement.Command = fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in)
- buildStatement.SymlinkPaths = outputPaths[:]
+ buildStatement, err = aqueryHandler.symlinkActionBuildStatement(actionEntry)
} else if isTemplateExpandAction(actionEntry) && len(actionEntry.Arguments) < 1 {
- if len(outputPaths) != 1 {
- return nil, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths)
- }
- expandedTemplateContent := expandTemplateContent(actionEntry)
- // The expandedTemplateContent is escaped for being used in double quotes and shell unescape,
- // and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might
- // change \n to space and mess up the format of Python programs.
- // sed is used to convert \\n back to \n before saving to output file.
- // See go/python-binary-host-mixed-build for more details.
- command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`,
- escapeCommandlineArgument(expandedTemplateContent), outputPaths[0])
- buildStatement.Command = command
+ buildStatement, err = aqueryHandler.templateExpandActionBuildStatement(actionEntry)
} else if isPythonZipperAction(actionEntry) {
- if len(inputPaths) < 1 || len(outputPaths) != 1 {
- return nil, fmt.Errorf("Expect 1+ input and 1 output to python zipper action, got: input %q, output %q", inputPaths, outputPaths)
- }
- buildStatement.InputPaths, buildStatement.Command = removePy3wrapperScript(buildStatement)
- buildStatement.Command = addCommandForPyBinaryRunfilesDir(buildStatement, inputPaths[0], outputPaths[0])
- // Add the python zip file as input of the corresponding python binary stub script in Ninja build statements.
- // In Ninja build statements, the outputs of dependents of a python binary have python binary stub script as input,
- // which is not sufficient without the python zip file from which runfiles directory is created for py_binary.
- //
- // The following logic relies on that Bazel aquery output returns actions in the order that
- // PythonZipper is after TemplateAction of creating Python binary stub script. If later Bazel doesn't return actions
- // in that order, the following logic might not find the build statement generated for Python binary
- // stub script and the build might fail. So the check of pyBinaryFound is added to help debug in case later Bazel might change aquery output.
- // See go/python-binary-host-mixed-build for more details.
- pythonZipFilePath := outputPaths[0]
- pyBinaryFound := false
- for i, _ := range buildStatements {
- if len(buildStatements[i].OutputPaths) == 1 && buildStatements[i].OutputPaths[0]+".zip" == pythonZipFilePath {
- buildStatements[i].InputPaths = append(buildStatements[i].InputPaths, pythonZipFilePath)
- pyBinaryFound = true
- }
- }
- if !pyBinaryFound {
- return nil, fmt.Errorf("Could not find the correspondinging Python binary stub script of PythonZipper: %q", outputPaths)
- }
+ buildStatement, err = aqueryHandler.pythonZipperActionBuildStatement(actionEntry, buildStatements)
} else if len(actionEntry.Arguments) < 1 {
- return nil, fmt.Errorf("received action with no command: [%v]", buildStatement)
+ return nil, nil, fmt.Errorf("received action with no command: [%s]", actionEntry.Mnemonic)
+ } else {
+ buildStatement, err = aqueryHandler.normalActionBuildStatement(actionEntry)
+ }
+
+ if err != nil {
+ return nil, nil, err
}
buildStatements = append(buildStatements, buildStatement)
}
- return buildStatements, nil
+ // Iterate over depset IDs in the initial aquery order to preserve determinism.
+ for _, depset := range aqueryResult.DepSetOfFiles {
+ // Use the depset in the aqueryHandler, as this contains the augmented depsets.
+ depset = aqueryHandler.depsetIdToDepset[depset.Id]
+ directPaths := []string{}
+ for _, artifactId := range depset.DirectArtifactIds {
+ pathString := aqueryHandler.artifactIdToPath[artifactId]
+ directPaths = append(directPaths, pathString)
+ }
+ aqueryDepset := AqueryDepset{
+ Id: depset.Id,
+ DirectArtifacts: directPaths,
+ TransitiveDepSetIds: depset.TransitiveDepSetIds,
+ }
+ depsets = append(depsets, aqueryDepset)
+ }
+ return buildStatements, depsets, nil
+}
+
+func (aqueryHandler *aqueryArtifactHandler) validateInputDepsets(inputDepsetIds []int) ([]int, error) {
+ // Validate input depsets correspond to real depsets.
+ for _, depsetId := range inputDepsetIds {
+ if _, exists := aqueryHandler.depsetIdToDepset[depsetId]; !exists {
+ return nil, fmt.Errorf("undefined input depsetId %d", depsetId)
+ }
+ }
+ return inputDepsetIds, nil
+}
+
+func (aqueryHandler *aqueryArtifactHandler) normalActionBuildStatement(actionEntry action) (BuildStatement, error) {
+ command := strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " ")
+ inputDepsetIds, err := aqueryHandler.validateInputDepsets(actionEntry.InputDepSetIds)
+ if err != nil {
+ return BuildStatement{}, err
+ }
+ outputPaths, depfile, err := aqueryHandler.getOutputPaths(actionEntry)
+ if err != nil {
+ return BuildStatement{}, err
+ }
+
+ buildStatement := BuildStatement{
+ Command: command,
+ Depfile: depfile,
+ OutputPaths: outputPaths,
+ InputDepsetIds: inputDepsetIds,
+ Env: actionEntry.EnvironmentVariables,
+ Mnemonic: actionEntry.Mnemonic,
+ }
+ return buildStatement, nil
+}
+
+func (aqueryHandler *aqueryArtifactHandler) pythonZipperActionBuildStatement(actionEntry action, prevBuildStatements []BuildStatement) (BuildStatement, error) {
+ inputPaths, err := aqueryHandler.getInputPaths(actionEntry.InputDepSetIds)
+ if err != nil {
+ return BuildStatement{}, err
+ }
+ outputPaths, depfile, err := aqueryHandler.getOutputPaths(actionEntry)
+ if err != nil {
+ return BuildStatement{}, err
+ }
+
+ if len(inputPaths) < 1 || len(outputPaths) != 1 {
+ return BuildStatement{}, fmt.Errorf("Expect 1+ input and 1 output to python zipper action, got: input %q, output %q", inputPaths, outputPaths)
+ }
+ command := strings.Join(proptools.ShellEscapeListIncludingSpaces(actionEntry.Arguments), " ")
+ inputPaths, command = removePy3wrapperScript(inputPaths, command)
+ command = addCommandForPyBinaryRunfilesDir(command, inputPaths[0], outputPaths[0])
+ // Add the python zip file as input of the corresponding python binary stub script in Ninja build statements.
+ // In Ninja build statements, the outputs of dependents of a python binary have python binary stub script as input,
+ // which is not sufficient without the python zip file from which runfiles directory is created for py_binary.
+ //
+ // The following logic relies on that Bazel aquery output returns actions in the order that
+ // PythonZipper is after TemplateAction of creating Python binary stub script. If later Bazel doesn't return actions
+ // in that order, the following logic might not find the build statement generated for Python binary
+ // stub script and the build might fail. So the check of pyBinaryFound is added to help debug in case later Bazel might change aquery output.
+ // See go/python-binary-host-mixed-build for more details.
+ pythonZipFilePath := outputPaths[0]
+ pyBinaryFound := false
+ for i, _ := range prevBuildStatements {
+ if len(prevBuildStatements[i].OutputPaths) == 1 && prevBuildStatements[i].OutputPaths[0]+".zip" == pythonZipFilePath {
+ prevBuildStatements[i].InputPaths = append(prevBuildStatements[i].InputPaths, pythonZipFilePath)
+ pyBinaryFound = true
+ }
+ }
+ if !pyBinaryFound {
+ return BuildStatement{}, fmt.Errorf("Could not find the correspondinging Python binary stub script of PythonZipper: %q", outputPaths)
+ }
+
+ buildStatement := BuildStatement{
+ Command: command,
+ Depfile: depfile,
+ OutputPaths: outputPaths,
+ InputPaths: inputPaths,
+ Env: actionEntry.EnvironmentVariables,
+ Mnemonic: actionEntry.Mnemonic,
+ }
+ return buildStatement, nil
+}
+
+func (aqueryHandler *aqueryArtifactHandler) templateExpandActionBuildStatement(actionEntry action) (BuildStatement, error) {
+ outputPaths, depfile, err := aqueryHandler.getOutputPaths(actionEntry)
+ if err != nil {
+ return BuildStatement{}, err
+ }
+ if len(outputPaths) != 1 {
+ return BuildStatement{}, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths)
+ }
+ expandedTemplateContent := expandTemplateContent(actionEntry)
+ // The expandedTemplateContent is escaped for being used in double quotes and shell unescape,
+ // and the new line characters (\n) are also changed to \\n which avoids some Ninja escape on \n, which might
+ // change \n to space and mess up the format of Python programs.
+ // sed is used to convert \\n back to \n before saving to output file.
+ // See go/python-binary-host-mixed-build for more details.
+ command := fmt.Sprintf(`/bin/bash -c 'echo "%[1]s" | sed "s/\\\\n/\\n/g" > %[2]s && chmod a+x %[2]s'`,
+ escapeCommandlineArgument(expandedTemplateContent), outputPaths[0])
+ inputDepsetIds, err := aqueryHandler.validateInputDepsets(actionEntry.InputDepSetIds)
+ if err != nil {
+ return BuildStatement{}, err
+ }
+
+ buildStatement := BuildStatement{
+ Command: command,
+ Depfile: depfile,
+ OutputPaths: outputPaths,
+ InputDepsetIds: inputDepsetIds,
+ Env: actionEntry.EnvironmentVariables,
+ Mnemonic: actionEntry.Mnemonic,
+ }
+ return buildStatement, nil
+}
+
+func (aqueryHandler *aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry action) (BuildStatement, error) {
+ outputPaths, depfile, err := aqueryHandler.getOutputPaths(actionEntry)
+ if err != nil {
+ return BuildStatement{}, err
+ }
+
+ inputPaths, err := aqueryHandler.getInputPaths(actionEntry.InputDepSetIds)
+ if err != nil {
+ return BuildStatement{}, err
+ }
+ if len(inputPaths) != 1 || len(outputPaths) != 1 {
+ return BuildStatement{}, fmt.Errorf("Expect 1 input and 1 output to symlink action, got: input %q, output %q", inputPaths, outputPaths)
+ }
+ out := outputPaths[0]
+ outDir := proptools.ShellEscapeIncludingSpaces(filepath.Dir(out))
+ out = proptools.ShellEscapeIncludingSpaces(out)
+ in := filepath.Join("$PWD", proptools.ShellEscapeIncludingSpaces(inputPaths[0]))
+ // Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`).
+ command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, in)
+ symlinkPaths := outputPaths[:]
+
+ buildStatement := BuildStatement{
+ Command: command,
+ Depfile: depfile,
+ OutputPaths: outputPaths,
+ InputPaths: inputPaths,
+ Env: actionEntry.EnvironmentVariables,
+ Mnemonic: actionEntry.Mnemonic,
+ SymlinkPaths: symlinkPaths,
+ }
+ return buildStatement, nil
+}
+
+func (aqueryHandler *aqueryArtifactHandler) getOutputPaths(actionEntry action) (outputPaths []string, depfile *string, err error) {
+ for _, outputId := range actionEntry.OutputIds {
+ outputPath, exists := aqueryHandler.artifactIdToPath[outputId]
+ if !exists {
+ err = fmt.Errorf("undefined outputId %d", outputId)
+ return
+ }
+ ext := filepath.Ext(outputPath)
+ if ext == ".d" {
+ if depfile != nil {
+ err = fmt.Errorf("found multiple potential depfiles %q, %q", *depfile, outputPath)
+ return
+ } else {
+ depfile = &outputPath
+ }
+ } else {
+ outputPaths = append(outputPaths, outputPath)
+ }
+ }
+ return
}
// expandTemplateContent substitutes the tokens in a template.
@@ -372,10 +518,10 @@
// removed from input paths and command of creating python zip file.
// See go/python-binary-host-mixed-build for more details.
// TODO(b/205879240) remove this after py3wrapper.sh could be created in the mixed build mode.
-func removePy3wrapperScript(bs BuildStatement) (newInputPaths []string, newCommand string) {
+func removePy3wrapperScript(inputPaths []string, command string) (newInputPaths []string, newCommand string) {
// Remove from inputs
filteredInputPaths := []string{}
- for _, path := range bs.InputPaths {
+ for _, path := range inputPaths {
if !strings.HasSuffix(path, py3wrapperFileName) {
filteredInputPaths = append(filteredInputPaths, path)
}
@@ -384,7 +530,7 @@
// Remove from command line
var re = regexp.MustCompile(`\S*` + py3wrapperFileName)
- newCommand = re.ReplaceAllString(bs.Command, "")
+ newCommand = re.ReplaceAllString(command, "")
return
}
@@ -395,14 +541,14 @@
// so MANIFEST file could not be created, which also blocks the creation of runfiles directory.
// See go/python-binary-host-mixed-build for more details.
// TODO(b/197135294) create runfiles directory from MANIFEST file once it can be created from SourceSymlinkManifest action.
-func addCommandForPyBinaryRunfilesDir(bs BuildStatement, zipperCommandPath, zipFilePath string) string {
+func addCommandForPyBinaryRunfilesDir(oldCommand string, zipperCommandPath, zipFilePath string) string {
// Unzip the zip file, zipFilePath looks like <python_binary>.zip
runfilesDirName := zipFilePath[0:len(zipFilePath)-4] + ".runfiles"
command := fmt.Sprintf("%s x %s -d %s", zipperCommandPath, zipFilePath, runfilesDirName)
// Create a symbolic link in <python_binary>.runfiles/, which is the expected structure
// when running the python binary stub script.
command += fmt.Sprintf(" && ln -sf runfiles/__main__ %s", runfilesDirName)
- return bs.Command + " && " + command
+ return oldCommand + " && " + command
}
func isSymlinkAction(a action) bool {
diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go
index 68e50c2..2328411 100644
--- a/bazel/aquery_test.go
+++ b/bazel/aquery_test.go
@@ -223,7 +223,7 @@
"parentId": 13
}]
}`
- actualbuildStatements, _ := AqueryBuildStatements([]byte(inputString))
+ actualbuildStatements, actualDepsets, _ := AqueryBuildStatements([]byte(inputString))
expectedBuildStatements := []BuildStatement{}
for _, arch := range []string{"arm", "arm64", "x86", "x86_64"} {
expectedBuildStatements = append(expectedBuildStatements,
@@ -234,11 +234,7 @@
OutputPaths: []string{
fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/bionic/libc/syscalls-%s.S", arch),
},
- InputPaths: []string{
- "../sourceroot/bionic/libc/SYSCALLS.TXT",
- "../sourceroot/bionic/libc/tools/gensyscalls.py",
- "../bazel_tools/tools/genrule/genrule-setup.sh",
- },
+ InputDepsetIds: []int{1},
Env: []KeyValuePair{
KeyValuePair{Key: "PATH", Value: "/bin:/usr/bin:/usr/local/bin"},
},
@@ -246,6 +242,16 @@
})
}
assertBuildStatements(t, expectedBuildStatements, actualbuildStatements)
+
+ expectedFlattenedInputs := []string{
+ "../sourceroot/bionic/libc/SYSCALLS.TXT",
+ "../sourceroot/bionic/libc/tools/gensyscalls.py",
+ "../bazel_tools/tools/genrule/genrule-setup.sh",
+ }
+ actualFlattenedInputs := flattenDepsets([]int{1}, actualDepsets)
+ if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) {
+ t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs)
+ }
}
func TestInvalidOutputId(t *testing.T) {
@@ -280,11 +286,11 @@
}]
}`
- _, err := AqueryBuildStatements([]byte(inputString))
+ _, _, err := AqueryBuildStatements([]byte(inputString))
assertError(t, err, "undefined outputId 3")
}
-func TestInvalidInputDepsetId(t *testing.T) {
+func TestInvalidInputDepsetIdFromAction(t *testing.T) {
const inputString = `
{
"artifacts": [{
@@ -316,10 +322,47 @@
}]
}`
- _, err := AqueryBuildStatements([]byte(inputString))
+ _, _, err := AqueryBuildStatements([]byte(inputString))
assertError(t, err, "undefined input depsetId 2")
}
+func TestInvalidInputDepsetIdFromDepset(t *testing.T) {
+ const inputString = `
+{
+ "artifacts": [{
+ "id": 1,
+ "pathFragmentId": 1
+ }, {
+ "id": 2,
+ "pathFragmentId": 2
+ }],
+ "actions": [{
+ "targetId": 1,
+ "actionKey": "x",
+ "mnemonic": "x",
+ "arguments": ["touch", "foo"],
+ "inputDepSetIds": [1],
+ "outputIds": [1],
+ "primaryOutputId": 1
+ }],
+ "depSetOfFiles": [{
+ "id": 1,
+ "directArtifactIds": [1, 2],
+ "transitiveDepSetIds": [42]
+ }],
+ "pathFragments": [{
+ "id": 1,
+ "label": "one"
+ }, {
+ "id": 2,
+ "label": "two"
+ }]
+}`
+
+ _, _, err := AqueryBuildStatements([]byte(inputString))
+ assertError(t, err, "undefined input depsetId 42 (referenced by depsetId 1)")
+}
+
func TestInvalidInputArtifactId(t *testing.T) {
const inputString = `
{
@@ -352,7 +395,7 @@
}]
}`
- _, err := AqueryBuildStatements([]byte(inputString))
+ _, _, err := AqueryBuildStatements([]byte(inputString))
assertError(t, err, "undefined input artifactId 3")
}
@@ -389,7 +432,7 @@
}]
}`
- _, err := AqueryBuildStatements([]byte(inputString))
+ _, _, err := AqueryBuildStatements([]byte(inputString))
assertError(t, err, "undefined path fragment id 3")
}
@@ -431,7 +474,7 @@
}]
}`
- actual, err := AqueryBuildStatements([]byte(inputString))
+ actual, _, err := AqueryBuildStatements([]byte(inputString))
if err != nil {
t.Errorf("Unexpected error %q", err)
}
@@ -492,7 +535,7 @@
}]
}`
- _, err := AqueryBuildStatements([]byte(inputString))
+ _, _, err := AqueryBuildStatements([]byte(inputString))
assertError(t, err, `found multiple potential depfiles "two.d", "other.d"`)
}
@@ -699,23 +742,31 @@
}]
}`
- actualbuildStatements, _ := AqueryBuildStatements([]byte(inputString))
- // Inputs for the action are test_{i} from 1 to 20, and test_root. These inputs
- // are given via a deep depset, but the depset is flattened when returned as a
- // BuildStatement slice.
- inputPaths := []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_root"}
- for i := 1; i < 20; i++ {
- inputPaths = append(inputPaths, fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_%d", i))
- }
+ actualbuildStatements, actualDepsets, _ := AqueryBuildStatements([]byte(inputString))
+
expectedBuildStatements := []BuildStatement{
BuildStatement{
- Command: "/bin/bash -c 'touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out'",
- OutputPaths: []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"},
- InputPaths: inputPaths,
- Mnemonic: "Action",
+ Command: "/bin/bash -c 'touch bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out'",
+ OutputPaths: []string{"bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_out"},
+ InputDepsetIds: []int{1},
+ Mnemonic: "Action",
},
}
assertBuildStatements(t, expectedBuildStatements, actualbuildStatements)
+
+ // Inputs for the action are test_{i} from 1 to 20, and test_root. These inputs
+ // are given via a deep depset, but the depset is flattened when returned as a
+ // BuildStatement slice.
+ expectedFlattenedInputs := []string{}
+ for i := 1; i < 20; i++ {
+ expectedFlattenedInputs = append(expectedFlattenedInputs, fmt.Sprintf("bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_%d", i))
+ }
+ expectedFlattenedInputs = append(expectedFlattenedInputs, "bazel-out/sourceroot/k8-fastbuild/bin/testpkg/test_root")
+
+ actualFlattenedInputs := flattenDepsets([]int{1}, actualDepsets)
+ if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) {
+ t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs)
+ }
}
func TestMiddlemenAction(t *testing.T) {
@@ -785,24 +836,74 @@
}]
}`
- actual, err := AqueryBuildStatements([]byte(inputString))
+ actualBuildStatements, actualDepsets, err := AqueryBuildStatements([]byte(inputString))
if err != nil {
t.Errorf("Unexpected error %q", err)
}
- if expected := 1; len(actual) != expected {
- t.Fatalf("Expected %d build statements, got %d", expected, len(actual))
+ if expected := 1; len(actualBuildStatements) != expected {
+ t.Fatalf("Expected %d build statements, got %d", expected, len(actualBuildStatements))
}
- bs := actual[0]
- expectedInputs := []string{"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"}
- if !reflect.DeepEqual(bs.InputPaths, expectedInputs) {
- t.Errorf("Expected main action inputs %q, but got %q", expectedInputs, bs.InputPaths)
+ bs := actualBuildStatements[0]
+ if len(bs.InputPaths) > 0 {
+ t.Errorf("Expected main action raw inputs to be empty, but got %q", bs.InputPaths)
+ }
+
+ expectedInputDepsets := []int{2}
+ if !reflect.DeepEqual(bs.InputDepsetIds, expectedInputDepsets) {
+ t.Errorf("Expected main action depset IDs %v, but got %v", expectedInputDepsets, bs.InputDepsetIds)
}
expectedOutputs := []string{"output"}
if !reflect.DeepEqual(bs.OutputPaths, expectedOutputs) {
t.Errorf("Expected main action outputs %q, but got %q", expectedOutputs, bs.OutputPaths)
}
+
+ expectedAllDepsets := []AqueryDepset{
+ {
+ Id: 1,
+ DirectArtifacts: []string{"middleinput_one", "middleinput_two"},
+ },
+ {
+ Id: 2,
+ DirectArtifacts: []string{"maininput_one", "maininput_two"},
+ TransitiveDepSetIds: []int{1},
+ },
+ }
+ if !reflect.DeepEqual(actualDepsets, expectedAllDepsets) {
+ t.Errorf("Expected depsets %v, but got %v", expectedAllDepsets, actualDepsets)
+ }
+
+ expectedFlattenedInputs := []string{"middleinput_one", "middleinput_two", "maininput_one", "maininput_two"}
+ actualFlattenedInputs := flattenDepsets(bs.InputDepsetIds, actualDepsets)
+
+ if !reflect.DeepEqual(actualFlattenedInputs, expectedFlattenedInputs) {
+ t.Errorf("Expected flattened inputs %v, but got %v", expectedFlattenedInputs, actualFlattenedInputs)
+ }
+}
+
+// Returns the contents of given depsets in concatenated post order.
+func flattenDepsets(depsetIdsToFlatten []int, allDepsets []AqueryDepset) []string {
+ depsetsById := map[int]AqueryDepset{}
+ for _, depset := range allDepsets {
+ depsetsById[depset.Id] = depset
+ }
+ result := []string{}
+ for _, depsetId := range depsetIdsToFlatten {
+ result = append(result, flattenDepset(depsetId, depsetsById)...)
+ }
+ return result
+}
+
+// Returns the contents of a given depset in post order.
+func flattenDepset(depsetIdToFlatten int, allDepsets map[int]AqueryDepset) []string {
+ depset := allDepsets[depsetIdToFlatten]
+ result := []string{}
+ for _, depsetId := range depset.TransitiveDepSetIds {
+ result = append(result, flattenDepset(depsetId, allDepsets)...)
+ }
+ result = append(result, depset.DirectArtifacts...)
+ return result
}
func TestSimpleSymlink(t *testing.T) {
@@ -849,7 +950,7 @@
}]
}`
- actual, err := AqueryBuildStatements([]byte(inputString))
+ actual, _, err := AqueryBuildStatements([]byte(inputString))
if err != nil {
t.Errorf("Unexpected error %q", err)
@@ -913,7 +1014,7 @@
}]
}`
- actual, err := AqueryBuildStatements([]byte(inputString))
+ actual, _, err := AqueryBuildStatements([]byte(inputString))
if err != nil {
t.Errorf("Unexpected error %q", err)
@@ -970,7 +1071,7 @@
}]
}`
- _, err := AqueryBuildStatements([]byte(inputString))
+ _, _, err := AqueryBuildStatements([]byte(inputString))
assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file" "other_file"], output ["symlink"]`)
}
@@ -1011,7 +1112,7 @@
}]
}`
- _, err := AqueryBuildStatements([]byte(inputString))
+ _, _, err := AqueryBuildStatements([]byte(inputString))
assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file"], output ["symlink" "other_symlink"]`)
}
@@ -1045,7 +1146,7 @@
}]
}`
- actual, err := AqueryBuildStatements([]byte(inputString))
+ actual, _, err := AqueryBuildStatements([]byte(inputString))
if err != nil {
t.Errorf("Unexpected error %q", err)
@@ -1091,7 +1192,7 @@
}]
}`
- _, err := AqueryBuildStatements([]byte(inputString))
+ _, _, err := AqueryBuildStatements([]byte(inputString))
assertError(t, err, `Expect 1 output to template expand action, got: output []`)
}
@@ -1211,7 +1312,7 @@
"label": "python_binary"
}]
}`
- actual, err := AqueryBuildStatements([]byte(inputString))
+ actual, _, err := AqueryBuildStatements([]byte(inputString))
if err != nil {
t.Errorf("Unexpected error %q", err)
@@ -1264,7 +1365,7 @@
"label": "python_binary.zip"
}]
}`
- _, err := AqueryBuildStatements([]byte(inputString))
+ _, _, err := AqueryBuildStatements([]byte(inputString))
assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input [], output ["python_binary.zip"]`)
}
@@ -1360,7 +1461,7 @@
"parentId": 11
}]
}`
- _, err := AqueryBuildStatements([]byte(inputString))
+ _, _, err := AqueryBuildStatements([]byte(inputString))
assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input ["../bazel_tools/tools/zip/zipper/zipper" "python_binary.py"], output []`)
}
diff --git a/bp2build/android_app_conversion_test.go b/bp2build/android_app_conversion_test.go
index 3824586..a216c9d 100644
--- a/bp2build/android_app_conversion_test.go
+++ b/bp2build/android_app_conversion_test.go
@@ -74,7 +74,8 @@
package_name: "com.google",
resource_dirs: ["resa", "resb"],
manifest: "manifest/AndroidManifest.xml",
- static_libs: ["static_lib_dep"]
+ static_libs: ["static_lib_dep"],
+ java_version: "7",
}
`,
expectedBazelTargets: []string{
@@ -87,6 +88,7 @@
]`,
"custom_package": `"com.google"`,
"deps": `[":static_lib_dep"]`,
+ "javacopts": `["-source 1.7 -target 1.7"]`,
}),
}})
}
diff --git a/bp2build/cc_library_shared_conversion_test.go b/bp2build/cc_library_shared_conversion_test.go
index 22c9dfe..7c2c100 100644
--- a/bp2build/cc_library_shared_conversion_test.go
+++ b/bp2build/cc_library_shared_conversion_test.go
@@ -176,8 +176,8 @@
":whole_static_lib_1",
":whole_static_lib_2",
]`,
- "sdk_version": `"current"`,
- "min_sdk_version": `"29"`,
+ "sdk_version": `"current"`,
+ "min_sdk_version": `"29"`,
}),
},
})
@@ -496,3 +496,27 @@
},
)
}
+
+func TestCcLibrarySharedSystemSharedLibsSharedEmpty(t *testing.T) {
+ runCcLibrarySharedTestCase(t, bp2buildTestCase{
+ description: "cc_library_shared system_shared_libs empty shared default",
+ moduleTypeUnderTest: "cc_library_shared",
+ moduleTypeUnderTestFactory: cc.LibrarySharedFactory,
+ blueprint: soongCcLibrarySharedPreamble + `
+cc_defaults {
+ name: "empty_defaults",
+ shared: {
+ system_shared_libs: [],
+ },
+ include_build_directory: false,
+}
+cc_library_shared {
+ name: "empty",
+ defaults: ["empty_defaults"],
+}
+`,
+ expectedBazelTargets: []string{makeBazelTarget("cc_library_shared", "empty", attrNameToString{
+ "system_dynamic_deps": "[]",
+ })},
+ })
+}
diff --git a/bp2build/java_binary_host_conversion_test.go b/bp2build/java_binary_host_conversion_test.go
index 4fc07e0..d7a76a8 100644
--- a/bp2build/java_binary_host_conversion_test.go
+++ b/bp2build/java_binary_host_conversion_test.go
@@ -52,6 +52,7 @@
jni_libs: ["jni-lib-1"],
javacflags: ["-Xdoclint:all/protected"],
bazel_module: { bp2build_available: true },
+ java_version: "8",
}`,
expectedBazelTargets: []string{
makeBazelTarget("java_binary", "java-binary-host-1", attrNameToString{
@@ -59,7 +60,10 @@
"main_class": `"com.android.test.MainClass"`,
"deps": `["//other:jni-lib-1"]`,
"jvm_flags": `["-Djava.library.path=$${RUNPATH}other"]`,
- "javacopts": `["-Xdoclint:all/protected"]`,
+ "javacopts": `[
+ "-Xdoclint:all/protected",
+ "-source 1.8 -target 1.8",
+ ]`,
"target_compatible_with": `select({
"//build/bazel/platforms/os:android": ["@platforms//:incompatible"],
"//conditions:default": [],
diff --git a/bp2build/java_library_conversion_test.go b/bp2build/java_library_conversion_test.go
index ccc52ef..3b66369 100644
--- a/bp2build/java_library_conversion_test.go
+++ b/bp2build/java_library_conversion_test.go
@@ -158,6 +158,22 @@
})
}
+func TestJavaLibraryJavaVersion(t *testing.T) {
+ runJavaLibraryTestCase(t, bp2buildTestCase{
+ blueprint: `java_library {
+ name: "java-lib-1",
+ srcs: ["a.java"],
+ java_version: "11",
+}`,
+ expectedBazelTargets: []string{
+ makeBazelTarget("java_library", "java-lib-1", attrNameToString{
+ "srcs": `["a.java"]`,
+ "javacopts": `["-source 11 -target 11"]`,
+ }),
+ },
+ })
+}
+
func TestJavaLibraryErrorproneJavacflagsEnabledManually(t *testing.T) {
runJavaLibraryTestCase(t, bp2buildTestCase{
blueprint: `java_library {
diff --git a/bp2build/java_library_host_conversion_test.go b/bp2build/java_library_host_conversion_test.go
index 73abdd2..83cc551 100644
--- a/bp2build/java_library_host_conversion_test.go
+++ b/bp2build/java_library_host_conversion_test.go
@@ -43,6 +43,7 @@
name: "java-lib-host-2",
srcs: ["c.java"],
bazel_module: { bp2build_available: true },
+ java_version: "9",
}`,
expectedBazelTargets: []string{
makeBazelTarget("java_library", "java-lib-host-1", attrNameToString{
@@ -54,7 +55,8 @@
})`,
}),
makeBazelTarget("java_library", "java-lib-host-2", attrNameToString{
- "srcs": `["c.java"]`,
+ "javacopts": `["-source 1.9 -target 1.9"]`,
+ "srcs": `["c.java"]`,
"target_compatible_with": `select({
"//build/bazel/platforms/os:android": ["@platforms//:incompatible"],
"//conditions:default": [],
diff --git a/bp2build/java_plugin_conversion_test.go b/bp2build/java_plugin_conversion_test.go
index c2a2182..dc763e7 100644
--- a/bp2build/java_plugin_conversion_test.go
+++ b/bp2build/java_plugin_conversion_test.go
@@ -39,6 +39,7 @@
libs: ["java-lib-1"],
static_libs: ["java-lib-2"],
bazel_module: { bp2build_available: true },
+ java_version: "7",
}
java_library {
@@ -66,6 +67,7 @@
"a.java",
"b.java",
]`,
+ "javacopts": `["-source 1.7 -target 1.7"]`,
}),
},
})
diff --git a/bp2build/java_proto_conversion_test.go b/bp2build/java_proto_conversion_test.go
index 67f8044..c6feeb8 100644
--- a/bp2build/java_proto_conversion_test.go
+++ b/bp2build/java_proto_conversion_test.go
@@ -102,6 +102,7 @@
blueprint: `java_library_static {
name: "java-protos",
srcs: ["a.proto"],
+ java_version: "7",
}
`,
expectedBazelTargets: []string{
@@ -115,7 +116,8 @@
"deps": `[":java-protos_proto"]`,
}),
makeBazelTarget("java_library", "java-protos", attrNameToString{
- "exports": `[":java-protos_java_proto_lite"]`,
+ "exports": `[":java-protos_java_proto_lite"]`,
+ "javacopts": `["-source 1.7 -target 1.7"]`,
}),
},
})
diff --git a/build_test.bash b/build_test.bash
index 1dc6660..8b91e2c 100755
--- a/build_test.bash
+++ b/build_test.bash
@@ -25,7 +25,8 @@
# Products that are broken or otherwise don't work with multiproduct_kati
SKIPPED_PRODUCTS=(
- # Both of these products are for soong-only builds, and will fail the kati stage.
+ # These products are for soong-only builds, and will fail the kati stage.
+ linux_bionic
mainline_sdk
ndk
)
diff --git a/cc/Android.bp b/cc/Android.bp
index 9103a48..60d329e 100644
--- a/cc/Android.bp
+++ b/cc/Android.bp
@@ -15,6 +15,7 @@
"soong-etc",
"soong-fuzz",
"soong-genrule",
+ "soong-multitree",
"soong-snapshot",
"soong-tradefed",
],
@@ -65,6 +66,7 @@
"library.go",
"library_headers.go",
"library_sdk_member.go",
+ "library_stub.go",
"native_bridge_sdk_trait.go",
"object.go",
"test.go",
@@ -94,12 +96,14 @@
"gen_test.go",
"genrule_test.go",
"library_headers_test.go",
+ "library_stub_test.go",
"library_test.go",
"object_test.go",
"prebuilt_test.go",
"proto_test.go",
"sanitize_test.go",
"test_data_test.go",
+ "tidy_test.go",
"vendor_public_library_test.go",
"vendor_snapshot_test.go",
],
diff --git a/cc/OWNERS b/cc/OWNERS
index a438b15..ffbf14a 100644
--- a/cc/OWNERS
+++ b/cc/OWNERS
@@ -1,4 +1,4 @@
per-file ndk_*.go = danalbert@google.com
-per-file tidy.go = srhines@google.com, chh@google.com
+per-file tidy*.go = srhines@google.com, chh@google.com
per-file afdo.go,afdo_test.go,lto.go,pgo.go = srhines@google.com, pirama@google.com, yikong@google.com
per-file coverage.go = pirama@google.com, srhines@google.com, allenhair@google.com
diff --git a/cc/bp2build.go b/cc/bp2build.go
index cc378b3..19855fa 100644
--- a/cc/bp2build.go
+++ b/cc/bp2build.go
@@ -167,21 +167,17 @@
attrs.System_dynamic_deps.ForceSpecifyEmptyList = true
if isStatic {
- for axis, configToProps := range module.GetArchVariantProperties(ctx, &StaticProperties{}) {
- for config, props := range configToProps {
- if staticOrSharedProps, ok := props.(*StaticProperties); ok {
- setAttrs(axis, config, staticOrSharedProps.Static)
- }
+ bp2BuildPropParseHelper(ctx, module, &StaticProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) {
+ if staticOrSharedProps, ok := props.(*StaticProperties); ok {
+ setAttrs(axis, config, staticOrSharedProps.Static)
}
- }
+ })
} else {
- for axis, configToProps := range module.GetArchVariantProperties(ctx, &SharedProperties{}) {
- for config, props := range configToProps {
- if staticOrSharedProps, ok := props.(*SharedProperties); ok {
- setAttrs(axis, config, staticOrSharedProps.Shared)
- }
+ bp2BuildPropParseHelper(ctx, module, &SharedProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) {
+ if staticOrSharedProps, ok := props.(*SharedProperties); ok {
+ setAttrs(axis, config, staticOrSharedProps.Shared)
}
- }
+ })
}
partitionedSrcs := groupSrcsByExtension(ctx, attrs.Srcs)
@@ -359,21 +355,18 @@
}
func (ca *compilerAttributes) convertStlProps(ctx android.ArchVariantContext, module *Module) {
- stlPropsByArch := module.GetArchVariantProperties(ctx, &StlProperties{})
- for _, configToProps := range stlPropsByArch {
- for _, props := range configToProps {
- if stlProps, ok := props.(*StlProperties); ok {
- if stlProps.Stl == nil {
- continue
- }
- if ca.stl == nil {
- ca.stl = stlProps.Stl
- } else if ca.stl != stlProps.Stl {
- ctx.ModuleErrorf("Unsupported conversion: module with different stl for different variants: %s and %s", *ca.stl, stlProps.Stl)
- }
+ bp2BuildPropParseHelper(ctx, module, &StlProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) {
+ if stlProps, ok := props.(*StlProperties); ok {
+ if stlProps.Stl == nil {
+ return
+ }
+ if ca.stl == nil {
+ ca.stl = stlProps.Stl
+ } else if ca.stl != stlProps.Stl {
+ ctx.ModuleErrorf("Unsupported conversion: module with different stl for different variants: %s and %s", *ca.stl, stlProps.Stl)
}
}
- }
+ })
}
func (ca *compilerAttributes) convertProductVariables(ctx android.BazelConversionPathContext, productVariableProps android.ProductConfigProperties) {
@@ -713,17 +706,15 @@
}
func (la *linkerAttributes) convertStripProps(ctx android.BazelConversionPathContext, module *Module) {
- for axis, configToProps := range module.GetArchVariantProperties(ctx, &StripProperties{}) {
- for config, props := range configToProps {
- if stripProperties, ok := props.(*StripProperties); ok {
- la.stripKeepSymbols.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols)
- la.stripKeepSymbolsList.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_list)
- la.stripKeepSymbolsAndDebugFrame.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_and_debug_frame)
- la.stripAll.SetSelectValue(axis, config, stripProperties.Strip.All)
- la.stripNone.SetSelectValue(axis, config, stripProperties.Strip.None)
- }
+ bp2BuildPropParseHelper(ctx, module, &StripProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) {
+ if stripProperties, ok := props.(*StripProperties); ok {
+ la.stripKeepSymbols.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols)
+ la.stripKeepSymbolsList.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_list)
+ la.stripKeepSymbolsAndDebugFrame.SetSelectValue(axis, config, stripProperties.Strip.Keep_symbols_and_debug_frame)
+ la.stripAll.SetSelectValue(axis, config, stripProperties.Strip.All)
+ la.stripNone.SetSelectValue(axis, config, stripProperties.Strip.None)
}
- }
+ })
}
func (la *linkerAttributes) convertProductVariables(ctx android.BazelConversionPathContext, productVariableProps android.ProductConfigProperties) {
@@ -859,18 +850,16 @@
} else {
exported = BazelIncludes{}
}
- for axis, configToProps := range module.GetArchVariantProperties(ctx, &FlagExporterProperties{}) {
- for config, props := range configToProps {
- if flagExporterProperties, ok := props.(*FlagExporterProperties); ok {
- if len(flagExporterProperties.Export_include_dirs) > 0 {
- exported.Includes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.Includes.SelectValue(axis, config), flagExporterProperties.Export_include_dirs...)))
- }
- if len(flagExporterProperties.Export_system_include_dirs) > 0 {
- exported.SystemIncludes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.SystemIncludes.SelectValue(axis, config), flagExporterProperties.Export_system_include_dirs...)))
- }
+ bp2BuildPropParseHelper(ctx, module, &FlagExporterProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) {
+ if flagExporterProperties, ok := props.(*FlagExporterProperties); ok {
+ if len(flagExporterProperties.Export_include_dirs) > 0 {
+ exported.Includes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.Includes.SelectValue(axis, config), flagExporterProperties.Export_include_dirs...)))
+ }
+ if len(flagExporterProperties.Export_system_include_dirs) > 0 {
+ exported.SystemIncludes.SetSelectValue(axis, config, android.FirstUniqueStrings(append(exported.SystemIncludes.SelectValue(axis, config), flagExporterProperties.Export_system_include_dirs...)))
}
}
- }
+ })
exported.AbsoluteIncludes.DeduplicateAxesFromBase()
exported.Includes.DeduplicateAxesFromBase()
exported.SystemIncludes.DeduplicateAxesFromBase()
@@ -938,22 +927,19 @@
func bp2buildBinaryLinkerProps(ctx android.BazelConversionPathContext, m *Module) binaryLinkerAttrs {
attrs := binaryLinkerAttrs{}
- archVariantProps := m.GetArchVariantProperties(ctx, &BinaryLinkerProperties{})
- for axis, configToProps := range archVariantProps {
- for _, p := range configToProps {
- props := p.(*BinaryLinkerProperties)
- staticExecutable := props.Static_executable
- if axis == bazel.NoConfigAxis {
- if linkBinaryShared := !proptools.Bool(staticExecutable); !linkBinaryShared {
- attrs.Linkshared = &linkBinaryShared
- }
- } else if staticExecutable != nil {
- // TODO(b/202876379): Static_executable is arch-variant; however, linkshared is a
- // nonconfigurable attribute. Only 4 AOSP modules use this feature, defer handling
- ctx.ModuleErrorf("bp2build cannot migrate a module with arch/target-specific static_executable values")
+ bp2BuildPropParseHelper(ctx, m, &BinaryLinkerProperties{}, func(axis bazel.ConfigurationAxis, config string, props interface{}) {
+ linkerProps := props.(*BinaryLinkerProperties)
+ staticExecutable := linkerProps.Static_executable
+ if axis == bazel.NoConfigAxis {
+ if linkBinaryShared := !proptools.Bool(staticExecutable); !linkBinaryShared {
+ attrs.Linkshared = &linkBinaryShared
}
+ } else if staticExecutable != nil {
+ // TODO(b/202876379): Static_executable is arch-variant; however, linkshared is a
+ // nonconfigurable attribute. Only 4 AOSP modules use this feature, defer handling
+ ctx.ModuleErrorf("bp2build cannot migrate a module with arch/target-specific static_executable values")
}
- }
+ })
return attrs
}
diff --git a/cc/builder.go b/cc/builder.go
index 525b1a1..107cd58 100644
--- a/cc/builder.go
+++ b/cc/builder.go
@@ -855,13 +855,24 @@
deps = append(deps, crtBegin...)
deps = append(deps, crtEnd...)
+ var depFile android.WritablePath
+ var depFileLdFlags string
+ depsType := blueprint.DepsNone
+ if !ctx.Windows() && !ctx.Darwin() {
+ // lld only supports --dependency-file for elf files
+ depFile = outputFile.ReplaceExtension(ctx, "d")
+ depFileLdFlags = " -Wl,--dependency-file=" + depFile.String()
+ depsType = blueprint.DepsGCC
+ implicitOutputs = append(implicitOutputs, depFile)
+ }
+
rule := ld
args := map[string]string{
"ldCmd": ldCmd,
"crtBegin": strings.Join(crtBegin.Strings(), " "),
"libFlags": strings.Join(libFlagsList, " "),
"extraLibFlags": flags.extraLibFlags,
- "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags,
+ "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags + depFileLdFlags,
"crtEnd": strings.Join(crtEnd.Strings(), " "),
}
if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_CXX_LINKS") {
@@ -872,6 +883,8 @@
ctx.Build(pctx, android.BuildParams{
Rule: rule,
+ Deps: depsType,
+ Depfile: depFile,
Description: "link " + outputFile.Base(),
Output: outputFile,
ImplicitOutputs: implicitOutputs,
@@ -1025,18 +1038,33 @@
ldCmd := "${config.ClangBin}/clang++"
+ var implicitOutputs android.WritablePaths
+ var depFile android.WritablePath
+ var depFileLdFlags string
+ depsType := blueprint.DepsNone
+ if !ctx.Windows() && !ctx.Darwin() {
+ // lld only supports --dependency-file for elf files
+ depFile = outputFile.ReplaceExtension(ctx, "d")
+ depFileLdFlags = " -Wl,--dependency-file=" + depFile.String()
+ depsType = blueprint.DepsGCC
+ implicitOutputs = append(implicitOutputs, depFile)
+ }
+
rule := partialLd
args := map[string]string{
"ldCmd": ldCmd,
- "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags,
+ "ldFlags": flags.globalLdFlags + " " + flags.localLdFlags + depFileLdFlags,
}
if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_CXX_LINKS") {
rule = partialLdRE
args["inCommaList"] = strings.Join(objFiles.Strings(), ",")
args["implicitInputs"] = strings.Join(deps.Strings(), ",")
+ args["implicitOutputs"] = strings.Join(implicitOutputs.Strings(), ",")
}
ctx.Build(pctx, android.BuildParams{
Rule: rule,
+ Deps: depsType,
+ Depfile: depFile,
Description: "link " + outputFile.Base(),
Output: outputFile,
Inputs: objFiles,
diff --git a/cc/cc.go b/cc/cc.go
index 456b736..c2d0f6e 100644
--- a/cc/cc.go
+++ b/cc/cc.go
@@ -746,6 +746,7 @@
runtimeDepTag = installDependencyTag{name: "runtime lib"}
testPerSrcDepTag = dependencyTag{name: "test_per_src"}
stubImplDepTag = dependencyTag{name: "stub_impl"}
+ JniFuzzLibTag = dependencyTag{name: "jni_fuzz_lib_tag"}
)
func IsSharedDepTag(depTag blueprint.DependencyTag) bool {
diff --git a/cc/config/global.go b/cc/config/global.go
index 65bfbf0..dc6310c 100644
--- a/cc/config/global.go
+++ b/cc/config/global.go
@@ -286,8 +286,8 @@
// prebuilts/clang default settings.
ClangDefaultBase = "prebuilts/clang/host"
- ClangDefaultVersion = "clang-r450784c"
- ClangDefaultShortVersion = "14.0.5"
+ ClangDefaultVersion = "clang-r450784e"
+ ClangDefaultShortVersion = "14.0.7"
// Directories with warnings from Android.bp files.
WarningAllowedProjects = []string{
diff --git a/cc/libbuildversion/tests/Android.bp b/cc/libbuildversion/tests/Android.bp
index 0e97fed..c616a33 100644
--- a/cc/libbuildversion/tests/Android.bp
+++ b/cc/libbuildversion/tests/Android.bp
@@ -35,6 +35,16 @@
dir: "host/",
},
},
+ linux_musl_x86: {
+ dist: {
+ dir: "host32/",
+ },
+ },
+ linux_musl_x86_64: {
+ dist: {
+ dir: "host/",
+ },
+ },
linux_glibc_x86: {
dist: {
dir: "host32/",
diff --git a/cc/library_stub.go b/cc/library_stub.go
new file mode 100644
index 0000000..4d0148d
--- /dev/null
+++ b/cc/library_stub.go
@@ -0,0 +1,163 @@
+// Copyright 2021 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cc
+
+import (
+ "android/soong/android"
+ "android/soong/multitree"
+)
+
+func init() {
+ RegisterLibraryStubBuildComponents(android.InitRegistrationContext)
+}
+
+func RegisterLibraryStubBuildComponents(ctx android.RegistrationContext) {
+ // cc_api_stub_library shares a lot of ndk_library, and this will be refactored later
+ ctx.RegisterModuleType("cc_api_stub_library", CcApiStubLibraryFactory)
+ ctx.RegisterModuleType("cc_api_contribution", CcApiContributionFactory)
+}
+
+func CcApiStubLibraryFactory() android.Module {
+ module, decorator := NewLibrary(android.DeviceSupported)
+ apiStubDecorator := &apiStubDecorator{
+ libraryDecorator: decorator,
+ }
+ apiStubDecorator.BuildOnlyShared()
+
+ module.compiler = apiStubDecorator
+ module.linker = apiStubDecorator
+ module.installer = nil
+ module.library = apiStubDecorator
+ module.Properties.HideFromMake = true // TODO: remove
+
+ android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibBoth)
+ module.AddProperties(&module.Properties,
+ &apiStubDecorator.properties,
+ &apiStubDecorator.MutatedProperties,
+ &apiStubDecorator.apiStubLibraryProperties)
+ return module
+}
+
+type apiStubLiraryProperties struct {
+ Imported_includes []string `android:"path"`
+}
+
+type apiStubDecorator struct {
+ *libraryDecorator
+ properties libraryProperties
+ apiStubLibraryProperties apiStubLiraryProperties
+}
+
+func (compiler *apiStubDecorator) stubsVersions(ctx android.BaseMutatorContext) []string {
+ firstVersion := String(compiler.properties.First_version)
+ return ndkLibraryVersions(ctx, android.ApiLevelOrPanic(ctx, firstVersion))
+}
+
+func (decorator *apiStubDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) Objects {
+ if decorator.stubsVersion() == "" {
+ decorator.setStubsVersion("current")
+ } // TODO: fix
+ symbolFile := String(decorator.properties.Symbol_file)
+ nativeAbiResult := parseNativeAbiDefinition(ctx, symbolFile,
+ android.ApiLevelOrPanic(ctx, decorator.stubsVersion()),
+ "")
+ return compileStubLibrary(ctx, flags, nativeAbiResult.stubSrc)
+}
+
+func (decorator *apiStubDecorator) link(ctx ModuleContext, flags Flags, deps PathDeps, objects Objects) android.Path {
+ decorator.reexportDirs(android.PathsForModuleSrc(ctx, decorator.apiStubLibraryProperties.Imported_includes)...)
+ return decorator.libraryDecorator.link(ctx, flags, deps, objects)
+}
+
+func init() {
+ pctx.HostBinToolVariable("gen_api_surface_build_files", "gen_api_surface_build_files")
+}
+
+type CcApiContribution struct {
+ android.ModuleBase
+ properties ccApiContributionProperties
+}
+
+type ccApiContributionProperties struct {
+ Symbol_file *string `android:"path"`
+ First_version *string
+ Export_include_dir *string
+}
+
+func CcApiContributionFactory() android.Module {
+ module := &CcApiContribution{}
+ module.AddProperties(&module.properties)
+ android.InitAndroidModule(module)
+ return module
+}
+
+// Do some simple validations
+// Majority of the build rules will be created in the ctx of the api surface this module contributes to
+func (contrib *CcApiContribution) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+ if contrib.properties.Symbol_file == nil {
+ ctx.PropertyErrorf("symbol_file", "%v does not have symbol file", ctx.ModuleName())
+ }
+ if contrib.properties.First_version == nil {
+ ctx.PropertyErrorf("first_version", "%v does not have first_version for stub variants", ctx.ModuleName())
+ }
+}
+
+// Path is out/soong/.export/ but will be different in final multi-tree layout
+func outPathApiSurface(ctx android.ModuleContext, myModuleName string, pathComponent string) android.OutputPath {
+ return android.PathForOutput(ctx, ".export", ctx.ModuleName(), myModuleName, pathComponent)
+}
+
+func (contrib *CcApiContribution) CopyFilesWithTag(apiSurfaceContext android.ModuleContext) map[string]android.Paths {
+ // copy map.txt for now
+ // hardlinks cannot be created since nsjail creates a different mountpoint for out/
+ myDir := apiSurfaceContext.OtherModuleDir(contrib)
+ genMapTxt := outPathApiSurface(apiSurfaceContext, contrib.Name(), String(contrib.properties.Symbol_file))
+ apiSurfaceContext.Build(pctx, android.BuildParams{
+ Rule: android.Cp,
+ Description: "import map.txt file",
+ Input: android.PathForSource(apiSurfaceContext, myDir, String(contrib.properties.Symbol_file)),
+ Output: genMapTxt,
+ })
+
+ outputs := make(map[string]android.Paths)
+ outputs["map"] = []android.Path{genMapTxt}
+
+ if contrib.properties.Export_include_dir != nil {
+ includeDir := android.PathForSource(apiSurfaceContext, myDir, String(contrib.properties.Export_include_dir))
+ outputs["export_include_dir"] = []android.Path{includeDir}
+ }
+ return outputs
+}
+
+var _ multitree.ApiContribution = (*CcApiContribution)(nil)
+
+/*
+func (contrib *CcApiContribution) GenerateBuildFiles(apiSurfaceContext android.ModuleContext) android.Paths {
+ genAndroidBp := outPathApiSurface(apiSurfaceContext, contrib.Name(), "Android.bp")
+
+ // generate Android.bp
+ apiSurfaceContext.Build(pctx, android.BuildParams{
+ Rule: genApiSurfaceBuildFiles,
+ Description: "generate API surface build files",
+ Outputs: []android.WritablePath{genAndroidBp},
+ Args: map[string]string{
+ "name": contrib.Name() + "." + apiSurfaceContext.ModuleName(), //e.g. liblog.ndk
+ "symbol_file": String(contrib.properties.Symbol_file),
+ "first_version": String(contrib.properties.First_version),
+ },
+ })
+ return []android.Path{genAndroidBp}
+}
+*/
diff --git a/cc/library_stub_test.go b/cc/library_stub_test.go
new file mode 100644
index 0000000..15b56d2
--- /dev/null
+++ b/cc/library_stub_test.go
@@ -0,0 +1,108 @@
+// Copyright 2021 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cc
+
+import (
+ _ "fmt"
+ _ "sort"
+
+ "testing"
+
+ "android/soong/android"
+ "android/soong/multitree"
+)
+
+func TestCcApiStubLibraryOutputFiles(t *testing.T) {
+ bp := `
+ cc_api_stub_library {
+ name: "foo",
+ symbol_file: "foo.map.txt",
+ first_version: "29",
+ }
+ `
+ result := prepareForCcTest.RunTestWithBp(t, bp)
+ outputs := result.ModuleForTests("foo", "android_arm64_armv8-a_shared").AllOutputs()
+ expected_file_suffixes := []string{".c", "stub.map", ".o", ".so"}
+ for _, expected_file_suffix := range expected_file_suffixes {
+ android.AssertBoolEquals(t, expected_file_suffix+" file not found in output", true, android.SuffixInList(outputs, expected_file_suffix))
+ }
+}
+
+func TestCcApiStubLibraryVariants(t *testing.T) {
+ bp := `
+ cc_api_stub_library {
+ name: "foo",
+ symbol_file: "foo.map.txt",
+ first_version: "29",
+ }
+ `
+ result := prepareForCcTest.RunTestWithBp(t, bp)
+ variants := result.ModuleVariantsForTests("foo")
+ expected_variants := []string{"29", "30", "S", "Tiramisu"} //TODO: make this test deterministic by using fixtures
+ for _, expected_variant := range expected_variants {
+ android.AssertBoolEquals(t, expected_variant+" variant not found in foo", true, android.SubstringInList(variants, expected_variant))
+ }
+}
+
+func TestCcLibraryUsesCcApiStubLibrary(t *testing.T) {
+ bp := `
+ cc_api_stub_library {
+ name: "foo",
+ symbol_file: "foo.map.txt",
+ first_version: "29",
+ }
+ cc_library {
+ name: "foo_user",
+ shared_libs: [
+ "foo#29",
+ ],
+ }
+
+ `
+ prepareForCcTest.RunTestWithBp(t, bp)
+}
+
+func TestApiSurfaceOutputs(t *testing.T) {
+ bp := `
+ api_surface {
+ name: "mysdk",
+ contributions: [
+ "foo",
+ ],
+ }
+
+ cc_api_contribution {
+ name: "foo",
+ symbol_file: "foo.map.txt",
+ first_version: "29",
+ }
+ `
+ result := android.GroupFixturePreparers(
+ prepareForCcTest,
+ multitree.PrepareForTestWithApiSurface,
+ ).RunTestWithBp(t, bp)
+ mysdk := result.ModuleForTests("mysdk", "")
+
+ actual_surface_inputs := mysdk.Rule("phony").BuildParams.Inputs.Strings()
+ expected_file_suffixes := []string{"mysdk/foo/foo.map.txt"}
+ for _, expected_file_suffix := range expected_file_suffixes {
+ android.AssertBoolEquals(t, expected_file_suffix+" file not found in input", true, android.SuffixInList(actual_surface_inputs, expected_file_suffix))
+ }
+
+ // check args/inputs to rule
+ /*api_surface_gen_rule_args := result.ModuleForTests("mysdk", "").Rule("genApiSurfaceBuildFiles").Args
+ android.AssertStringEquals(t, "name", "foo.mysdk", api_surface_gen_rule_args["name"])
+ android.AssertStringEquals(t, "symbol_file", "foo.map.txt", api_surface_gen_rule_args["symbol_file"])*/
+}
diff --git a/cc/ndk_library.go b/cc/ndk_library.go
index 5ef41ea..0879257 100644
--- a/cc/ndk_library.go
+++ b/cc/ndk_library.go
@@ -93,7 +93,7 @@
type libraryProperties struct {
// Relative path to the symbol map.
// An example file can be seen here: TODO(danalbert): Make an example.
- Symbol_file *string
+ Symbol_file *string `android:"path"`
// The first API level a library was available. A library will be generated
// for every API level beginning with this one.
@@ -284,6 +284,10 @@
}
func compileStubLibrary(ctx ModuleContext, flags Flags, src android.Path) Objects {
+ // libc/libm stubs libraries end up mismatching with clang's internal definition of these
+ // functions (which have noreturn attributes and other things). Because we just want to create a
+ // stub with symbol definitions, and types aren't important in C, ignore the mismatch.
+ flags.Local.ConlyFlags = append(flags.Local.ConlyFlags, "-fno-builtin")
return compileObjs(ctx, flagsToBuilderFlags(flags), "",
android.Paths{src}, nil, nil, nil, nil)
}
diff --git a/cc/sanitize.go b/cc/sanitize.go
index 814fef6..53169de 100644
--- a/cc/sanitize.go
+++ b/cc/sanitize.go
@@ -969,6 +969,22 @@
})
}
} else if sanitizeable, ok := mctx.Module().(Sanitizeable); ok {
+ // If it's a Java module with native dependencies through jni,
+ // set the sanitizer for them
+ if jniSanitizeable, ok := mctx.Module().(JniSanitizeable); ok {
+ if jniSanitizeable.IsSanitizerEnabledForJni(mctx, t.name()) {
+ mctx.VisitDirectDeps(func(child android.Module) {
+ if c, ok := child.(PlatformSanitizeable); ok &&
+ mctx.OtherModuleDependencyTag(child) == JniFuzzLibTag &&
+ c.SanitizePropDefined() &&
+ !c.SanitizeNever() &&
+ !c.IsSanitizerExplicitlyDisabled(t) {
+ c.SetSanitizeDep(true)
+ }
+ })
+ }
+ }
+
// If an APEX module includes a lib which is enabled for a sanitizer T, then
// the APEX module is also enabled for the same sanitizer type.
mctx.VisitDirectDeps(func(child android.Module) {
@@ -1280,6 +1296,11 @@
AddSanitizerDependencies(ctx android.BottomUpMutatorContext, sanitizerName string)
}
+type JniSanitizeable interface {
+ android.Module
+ IsSanitizerEnabledForJni(ctx android.BaseModuleContext, sanitizerName string) bool
+}
+
func (c *Module) MinimalRuntimeDep() bool {
return c.sanitize.Properties.MinimalRuntimeDep
}
@@ -1407,7 +1428,7 @@
}
c.SetSanitizeDep(false)
} else if sanitizeable, ok := mctx.Module().(Sanitizeable); ok && sanitizeable.IsSanitizerEnabled(mctx, t.name()) {
- // APEX modules fall here
+ // APEX and Java fuzz modules fall here
sanitizeable.AddSanitizerDependencies(mctx, t.name())
mctx.CreateVariations(t.variationName())
} else if c, ok := mctx.Module().(*Module); ok {
diff --git a/cc/testing.go b/cc/testing.go
index 32f7c60..ecdae8b 100644
--- a/cc/testing.go
+++ b/cc/testing.go
@@ -29,6 +29,7 @@
RegisterBinaryBuildComponents(ctx)
RegisterLibraryBuildComponents(ctx)
RegisterLibraryHeadersBuildComponents(ctx)
+ RegisterLibraryStubBuildComponents(ctx)
ctx.RegisterModuleType("cc_benchmark", BenchmarkFactory)
ctx.RegisterModuleType("cc_object", ObjectFactory)
diff --git a/cc/tidy.go b/cc/tidy.go
index 750e9de..03e967d 100644
--- a/cc/tidy.go
+++ b/cc/tidy.go
@@ -76,9 +76,10 @@
// the global WITH_TIDY or module 'tidy' property is true.
flags.Tidy = true
- // If explicitly enabled, by global default or local tidy property,
+ // If explicitly enabled, by global WITH_TIDY or local tidy:true property,
// set flags.NeedTidyFiles to make this module depend on .tidy files.
- if ctx.Config().ClangTidy() || Bool(tidy.Properties.Tidy) {
+ // Note that locally set tidy:true is ignored if ALLOW_LOCAL_TIDY_TRUE is not set to true.
+ if ctx.Config().IsEnvTrue("WITH_TIDY") || (ctx.Config().IsEnvTrue("ALLOW_LOCAL_TIDY_TRUE") && Bool(tidy.Properties.Tidy)) {
flags.NeedTidyFiles = true
}
diff --git a/cc/tidy_test.go b/cc/tidy_test.go
new file mode 100644
index 0000000..339b302
--- /dev/null
+++ b/cc/tidy_test.go
@@ -0,0 +1,98 @@
+// Copyright 2022 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cc
+
+import (
+ "fmt"
+ "testing"
+
+ "android/soong/android"
+)
+
+func TestWithTidy(t *testing.T) {
+ // When WITH_TIDY=1 or (ALLOW_LOCAL_TIDY_TRUE=1 and local tidy:true)
+ // a C++ library should depend on .tidy files.
+ testCases := []struct {
+ withTidy, allowLocalTidyTrue string // "_" means undefined
+ needTidyFile []bool // for {libfoo_0, libfoo_1} and {libbar_0, libbar_1}
+ }{
+ {"_", "_", []bool{false, false, false}},
+ {"_", "0", []bool{false, false, false}},
+ {"_", "1", []bool{false, true, false}},
+ {"_", "true", []bool{false, true, false}},
+ {"0", "_", []bool{false, false, false}},
+ {"0", "1", []bool{false, true, false}},
+ {"1", "_", []bool{true, true, false}},
+ {"1", "false", []bool{true, true, false}},
+ {"1", "1", []bool{true, true, false}},
+ {"true", "_", []bool{true, true, false}},
+ }
+ bp := `
+ cc_library_shared {
+ name: "libfoo_0", // depends on .tidy if WITH_TIDY=1
+ srcs: ["foo.c"],
+ }
+ cc_library_shared { // depends on .tidy if WITH_TIDY=1 or ALLOW_LOCAL_TIDY_TRUE=1
+ name: "libfoo_1",
+ srcs: ["foo.c"],
+ tidy: true,
+ }
+ cc_library_shared { // no .tidy
+ name: "libfoo_2",
+ srcs: ["foo.c"],
+ tidy: false,
+ }
+ cc_library_static {
+ name: "libbar_0", // depends on .tidy if WITH_TIDY=1
+ srcs: ["bar.c"],
+ }
+ cc_library_static { // depends on .tidy if WITH_TIDY=1 or ALLOW_LOCAL_TIDY_TRUE=1
+ name: "libbar_1",
+ srcs: ["bar.c"],
+ tidy: true,
+ }
+ cc_library_static { // no .tidy
+ name: "libbar_2",
+ srcs: ["bar.c"],
+ tidy: false,
+ }`
+ for index, test := range testCases {
+ testName := fmt.Sprintf("case%d,%v,%v", index, test.withTidy, test.allowLocalTidyTrue)
+ t.Run(testName, func(t *testing.T) {
+ testEnv := map[string]string{}
+ if test.withTidy != "_" {
+ testEnv["WITH_TIDY"] = test.withTidy
+ }
+ if test.allowLocalTidyTrue != "_" {
+ testEnv["ALLOW_LOCAL_TIDY_TRUE"] = test.allowLocalTidyTrue
+ }
+ ctx := android.GroupFixturePreparers(prepareForCcTest, android.FixtureMergeEnv(testEnv)).RunTestWithBp(t, bp)
+ for n := 0; n < 3; n++ {
+ checkLibraryRule := func(foo, variant, ruleName string) {
+ libName := fmt.Sprintf("lib%s_%d", foo, n)
+ tidyFile := "out/soong/.intermediates/" + libName + "/" + variant + "/obj/" + foo + ".tidy"
+ depFiles := ctx.ModuleForTests(libName, variant).Rule(ruleName).Validations.Strings()
+ if test.needTidyFile[n] {
+ android.AssertStringListContains(t, libName+" needs .tidy file", depFiles, tidyFile)
+ } else {
+ android.AssertStringListDoesNotContain(t, libName+" does not need .tidy file", depFiles, tidyFile)
+ }
+ }
+ checkLibraryRule("foo", "android_arm64_armv8-a_shared", "ld")
+ checkLibraryRule("bar", "android_arm64_armv8-a_static", "ar")
+ }
+ })
+ }
+}
diff --git a/cc/util.go b/cc/util.go
index b256b9a..4e10037 100644
--- a/cc/util.go
+++ b/cc/util.go
@@ -15,9 +15,7 @@
package cc
import (
- "fmt"
"path/filepath"
- "regexp"
"strings"
"android/soong/android"
@@ -30,30 +28,12 @@
return android.JoinWithPrefix(dirs.Strings(), "-I")
}
-func ldDirsToFlags(dirs []string) string {
- return android.JoinWithPrefix(dirs, "-L")
-}
-
-func libNamesToFlags(names []string) string {
- return android.JoinWithPrefix(names, "-l")
-}
-
var indexList = android.IndexList
var inList = android.InList
var filterList = android.FilterList
var removeListFromList = android.RemoveListFromList
var removeFromList = android.RemoveFromList
-var libNameRegexp = regexp.MustCompile(`^lib(.*)$`)
-
-func moduleToLibName(module string) (string, error) {
- matches := libNameRegexp.FindStringSubmatch(module)
- if matches == nil {
- return "", fmt.Errorf("Library module name %s does not start with lib", module)
- }
- return matches[1], nil
-}
-
func flagsToBuilderFlags(in Flags) builderFlags {
return builderFlags{
globalCommonFlags: strings.Join(in.Global.CommonFlags, " "),
@@ -113,13 +93,6 @@
return list
}
-func addSuffix(list []string, suffix string) []string {
- for i := range list {
- list[i] = list[i] + suffix
- }
- return list
-}
-
// linkDirOnDevice/linkName -> target
func makeSymlinkCmd(linkDirOnDevice string, linkName string, target string) string {
dir := filepath.Join("$(PRODUCT_OUT)", linkDirOnDevice)
diff --git a/dexpreopt/class_loader_context.go b/dexpreopt/class_loader_context.go
index 36513b6..d0a6a39 100644
--- a/dexpreopt/class_loader_context.go
+++ b/dexpreopt/class_loader_context.go
@@ -678,6 +678,9 @@
jClcMap := make(jsonClassLoaderContextMap)
for sdkVer, clcs := range clcMap {
sdkVerStr := fmt.Sprintf("%d", sdkVer)
+ if sdkVer == AnySdkVersion {
+ sdkVerStr = "any"
+ }
jClcMap[sdkVerStr] = toJsonClassLoaderContextRec(clcs)
}
return jClcMap
diff --git a/dexpreopt/class_loader_context_test.go b/dexpreopt/class_loader_context_test.go
index 5d3a9d9..614681f 100644
--- a/dexpreopt/class_loader_context_test.go
+++ b/dexpreopt/class_loader_context_test.go
@@ -389,6 +389,38 @@
})
}
+// Test that CLC is correctly serialized to JSON.
+func TestCLCtoJSON(t *testing.T) {
+ ctx := testContext()
+ optional := false
+ implicit := true
+ m := make(ClassLoaderContextMap)
+ m.AddContext(ctx, 28, "a", optional, implicit, buildPath(ctx, "a"), installPath(ctx, "a"), nil)
+ m.AddContext(ctx, AnySdkVersion, "b", optional, implicit, buildPath(ctx, "b"), installPath(ctx, "b"), nil)
+ android.AssertStringEquals(t, "output CLCM ", `{
+ "28": [
+ {
+ "Name": "a",
+ "Optional": false,
+ "Implicit": true,
+ "Host": "out/soong/a.jar",
+ "Device": "/system/a.jar",
+ "Subcontexts": []
+ }
+ ],
+ "any": [
+ {
+ "Name": "b",
+ "Optional": false,
+ "Implicit": true,
+ "Host": "out/soong/b.jar",
+ "Device": "/system/b.jar",
+ "Subcontexts": []
+ }
+ ]
+}`, m.Dump())
+}
+
func checkError(t *testing.T, have error, want string) {
if have == nil {
t.Errorf("\nwant error: '%s'\nhave: none", want)
diff --git a/dexpreopt/config.go b/dexpreopt/config.go
index 153b025..ab9e418 100644
--- a/dexpreopt/config.go
+++ b/dexpreopt/config.go
@@ -36,8 +36,6 @@
PreoptWithUpdatableBcp bool // If updatable boot jars are included in dexpreopt or not.
- UseArtImage bool // use the art image (use other boot class path dex files without image)
-
HasSystemOther bool // store odex files that match PatternsOnSystemOther on the system_other partition
PatternsOnSystemOther []string // patterns (using '%' to denote a prefix match) to put odex on the system_other partition
diff --git a/fuzz/fuzz_common.go b/fuzz/fuzz_common.go
index 89f8187..700cdf0 100644
--- a/fuzz/fuzz_common.go
+++ b/fuzz/fuzz_common.go
@@ -82,6 +82,9 @@
Hwasan_options []string `json:"hwasan_options,omitempty"`
// Additional options to be passed to HWASAN when running on host in Haiku.
Asan_options []string `json:"asan_options,omitempty"`
+ // If there's a Java fuzzer with JNI, a different version of Jazzer would
+ // need to be added to the fuzzer package than one without JNI
+ IsJni *bool `json:"is_jni,omitempty"`
}
type FuzzProperties struct {
diff --git a/java/android_manifest.go b/java/android_manifest.go
index 7772b70..3fa3520 100644
--- a/java/android_manifest.go
+++ b/java/android_manifest.go
@@ -82,8 +82,8 @@
if minSdkVersion.FinalOrFutureInt() >= 23 {
args = append(args, fmt.Sprintf("--extract-native-libs=%v", !params.UseEmbeddedNativeLibs))
} else if params.UseEmbeddedNativeLibs {
- ctx.ModuleErrorf("module attempted to store uncompressed native libraries, but minSdkVersion=%d doesn't support it",
- minSdkVersion)
+ ctx.ModuleErrorf("module attempted to store uncompressed native libraries, but minSdkVersion=%s doesn't support it",
+ minSdkVersion.String())
}
}
diff --git a/java/androidmk.go b/java/androidmk.go
index 80b828d..f6ea6a9 100644
--- a/java/androidmk.go
+++ b/java/androidmk.go
@@ -132,6 +132,16 @@
return entriesList
}
+func (j *JavaFuzzLibrary) AndroidMkEntries() []android.AndroidMkEntries {
+ entriesList := j.Library.AndroidMkEntries()
+ entries := &entriesList[0]
+ entries.ExtraEntries = append(entries.ExtraEntries, func(ctx android.AndroidMkExtraEntriesContext, entries *android.AndroidMkEntries) {
+ entries.AddStrings("LOCAL_COMPATIBILITY_SUITE", "null-suite")
+ androidMkWriteTestData(j.jniFilePaths, entries)
+ })
+ return entriesList
+}
+
// Called for modules that are a component of a test suite.
func testSuiteComponent(entries *android.AndroidMkEntries, test_suites []string, perTestcaseDirectory bool) {
entries.SetString("LOCAL_MODULE_TAGS", "tests")
diff --git a/java/base.go b/java/base.go
index b925350..245ffbb 100644
--- a/java/base.go
+++ b/java/base.go
@@ -442,9 +442,6 @@
// manifest file to use instead of properties.Manifest
overrideManifest android.OptionalPath
- // map of SDK version to class loader context
- classLoaderContexts dexpreopt.ClassLoaderContextMap
-
// list of plugins that this java module is exporting
exportedPluginJars android.Paths
@@ -1481,11 +1478,11 @@
}
if ctx.Device() {
- lintSDKVersionString := func(sdkSpec android.SdkSpec) string {
+ lintSDKVersion := func(sdkSpec android.SdkSpec) android.ApiLevel {
if v := sdkSpec.ApiLevel; !v.IsPreview() {
- return v.String()
+ return v
} else {
- return ctx.Config().DefaultAppTargetSdk(ctx).String()
+ return ctx.Config().DefaultAppTargetSdk(ctx)
}
}
@@ -1494,9 +1491,9 @@
j.linter.srcJars = srcJars
j.linter.classpath = append(append(android.Paths(nil), flags.bootClasspath...), flags.classpath...)
j.linter.classes = j.implementationJarFile
- j.linter.minSdkVersion = lintSDKVersionString(j.MinSdkVersion(ctx))
- j.linter.targetSdkVersion = lintSDKVersionString(j.TargetSdkVersion(ctx))
- j.linter.compileSdkVersion = lintSDKVersionString(j.SdkVersion(ctx))
+ j.linter.minSdkVersion = lintSDKVersion(j.MinSdkVersion(ctx))
+ j.linter.targetSdkVersion = lintSDKVersion(j.TargetSdkVersion(ctx))
+ j.linter.compileSdkVersion = lintSDKVersion(j.SdkVersion(ctx))
j.linter.compileSdkKind = j.SdkVersion(ctx).Kind
j.linter.javaLanguageLevel = flags.javaVersion.String()
j.linter.kotlinLanguageLevel = "1.3"
diff --git a/java/config/config.go b/java/config/config.go
index 46c91a2..d744002 100644
--- a/java/config/config.go
+++ b/java/config/config.go
@@ -68,6 +68,11 @@
"-J-XX:+TieredCompilation",
"-J-XX:TieredStopAtLevel=1",
}
+ dexerJavaVmFlagsList = []string{
+ `-JXX:OnError="cat hs_err_pid%p.log"`,
+ "-JXX:CICompilerCount=6",
+ "-JXX:+UseDynamicNumberOfGCThreads",
+ }
)
func init() {
@@ -83,19 +88,14 @@
// D8 invocations are shorter lived, so we restrict their JIT tiering relative to R8.
// Note that the `-JXX` prefix syntax is specific to the R8/D8 invocation wrappers.
- exportedVars.ExportStringListStaticVariable("D8Flags", []string{
- `-JXX:OnError="cat hs_err_pid%p.log"`,
- "-JXX:CICompilerCount=6",
- "-JXX:+UseDynamicNumberOfGCThreads",
+ exportedVars.ExportStringListStaticVariable("D8Flags", append([]string{
+ "-JXmx2048M",
"-JXX:+TieredCompilation",
"-JXX:TieredStopAtLevel=1",
- })
-
- exportedVars.ExportStringListStaticVariable("R8Flags", []string{
- `-JXX:OnError="cat hs_err_pid%p.log"`,
- "-JXX:CICompilerCount=6",
- "-JXX:+UseDynamicNumberOfGCThreads",
- })
+ }, dexerJavaVmFlagsList...))
+ exportedVars.ExportStringListStaticVariable("R8Flags", append([]string{
+ "-JXmx2048M",
+ }, dexerJavaVmFlagsList...))
exportedVars.ExportStringListStaticVariable("CommonJdkFlags", []string{
`-Xmaxerrs 9999999`,
diff --git a/java/config/makevars.go b/java/config/makevars.go
index bc6848f..273aca0 100644
--- a/java/config/makevars.go
+++ b/java/config/makevars.go
@@ -43,9 +43,10 @@
ctx.Strict("JAVADOC", "${JavadocCmd}")
ctx.Strict("COMMON_JDK_FLAGS", "${CommonJdkFlags}")
- ctx.Strict("DX", "${D8Cmd}")
- ctx.Strict("DX_COMMAND", "${D8Cmd} -JXms16M -JXmx2048M")
- ctx.Strict("R8_COMPAT_PROGUARD", "${R8Cmd}")
+ ctx.Strict("D8", "${D8Cmd}")
+ ctx.Strict("R8", "${R8Cmd}")
+ ctx.Strict("D8_COMMAND", "${D8Cmd} ${D8Flags}")
+ ctx.Strict("R8_COMMAND", "${R8Cmd} ${R8Flags}")
ctx.Strict("TURBINE", "${TurbineJar}")
@@ -78,9 +79,6 @@
ctx.Strict("CLASS2NONSDKLIST", "${Class2NonSdkList}")
ctx.Strict("HIDDENAPI", "${HiddenAPI}")
- ctx.Strict("D8_FLAGS", "${D8Flags}")
- ctx.Strict("R8_FLAGS", "${R8Flags}")
-
ctx.Strict("AIDL", "${AidlCmd}")
ctx.Strict("AAPT2", "${Aapt2Cmd}")
ctx.Strict("ZIPALIGN", "${ZipAlign}")
diff --git a/java/dexpreopt.go b/java/dexpreopt.go
index 7c5f055..0adaf99 100644
--- a/java/dexpreopt.go
+++ b/java/dexpreopt.go
@@ -259,10 +259,6 @@
isSystemServerJar := global.AllSystemServerJars(ctx).ContainsJar(moduleName(ctx))
bootImage := defaultBootImageConfig(ctx)
- if global.UseArtImage {
- bootImage = artBootImageConfig(ctx)
- }
-
dexFiles, dexLocations := bcpForDexpreopt(ctx, global.PreoptWithUpdatableBcp)
targets := ctx.MultiTargets()
diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go
index 3d91aec..7c4da3e 100644
--- a/java/dexpreopt_bootjars.go
+++ b/java/dexpreopt_bootjars.go
@@ -213,12 +213,6 @@
// writes out a few DEXPREOPT_IMAGE_* variables for Make; these variables contain boot image names,
// paths and so on.
//
-// 2.5. JIT-Zygote configuration
-// -----------------------------
-//
-// One special configuration is JIT-Zygote build, when the primary ART image is used for compiling
-// apps instead of the Framework boot image extension (see DEXPREOPT_USE_ART_IMAGE and UseArtImage).
-//
var artApexNames = []string{
"com.android.art",
@@ -938,11 +932,8 @@
ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS", strings.Join(dexLocations, " "))
var imageNames []string
- // TODO: the primary ART boot image should not be exposed to Make, as it is installed in a
- // different way as a part of the ART APEX. However, there is a special JIT-Zygote build
- // configuration which uses the primary ART image instead of the Framework boot image
- // extension, and it relies on the ART image being exposed to Make. To fix this, it is
- // necessary to rework the logic in makefiles.
+ // The primary ART boot image is exposed to Make for testing (gtests) and benchmarking
+ // (golem) purposes.
for _, current := range append(d.otherImages, image) {
imageNames = append(imageNames, current.name)
for _, variant := range current.variants {
diff --git a/java/fuzz.go b/java/fuzz.go
index 257f343..584c80b 100644
--- a/java/fuzz.go
+++ b/java/fuzz.go
@@ -15,14 +15,25 @@
package java
import (
- "github.com/google/blueprint/proptools"
"sort"
"strings"
+ "github.com/google/blueprint"
+ "github.com/google/blueprint/proptools"
+
"android/soong/android"
+ "android/soong/cc"
"android/soong/fuzz"
)
+type jniProperties struct {
+ // list of jni libs
+ Jni_libs []string
+
+ // sanitization
+ Sanitizers []string
+}
+
func init() {
RegisterJavaFuzzBuildComponents(android.InitRegistrationContext)
}
@@ -35,11 +46,60 @@
type JavaFuzzLibrary struct {
Library
fuzzPackagedModule fuzz.FuzzPackagedModule
+ jniProperties jniProperties
+ jniFilePaths android.Paths
+}
+
+// IsSanitizerEnabled implemented to make JavaFuzzLibrary implement
+// cc.Sanitizeable
+func (j *JavaFuzzLibrary) IsSanitizerEnabled(ctx android.BaseModuleContext, sanitizerName string) bool {
+ for _, s := range j.jniProperties.Sanitizers {
+ if sanitizerName == s {
+ return true
+ }
+ }
+ return false
+}
+
+// IsSanitizerEnabledForJni implemented to make JavaFuzzLibrary implement
+// cc.JniSanitizeable. It returns a bool for whether a cc dependency should be
+// sanitized for the given sanitizer or not.
+func (j *JavaFuzzLibrary) IsSanitizerEnabledForJni(ctx android.BaseModuleContext, sanitizerName string) bool {
+ return j.IsSanitizerEnabled(ctx, sanitizerName)
+}
+
+// EnableSanitizer implemented to make JavaFuzzLibrary implement
+// cc.Sanitizeable
+func (j *JavaFuzzLibrary) EnableSanitizer(sanitizerName string) {
+}
+
+// AddSanitizerDependencies implemented to make JavaFuzzLibrary implement
+// cc.Sanitizeable
+func (j *JavaFuzzLibrary) AddSanitizerDependencies(mctx android.BottomUpMutatorContext, sanitizerName string) {
+}
+
+// To verify that JavaFuzzLibrary implements cc.Sanitizeable
+var _ cc.Sanitizeable = (*JavaFuzzLibrary)(nil)
+
+func (j *JavaFuzzLibrary) DepsMutator(mctx android.BottomUpMutatorContext) {
+ if len(j.jniProperties.Jni_libs) > 0 {
+ if j.fuzzPackagedModule.FuzzProperties.Fuzz_config == nil {
+ config := &fuzz.FuzzConfig{}
+ j.fuzzPackagedModule.FuzzProperties.Fuzz_config = config
+ }
+ // this will be used by the ingestion pipeline to determine the version
+ // of jazzer to add to the fuzzer package
+ j.fuzzPackagedModule.FuzzProperties.Fuzz_config.IsJni = proptools.BoolPtr(true)
+
+ for _, target := range mctx.MultiTargets() {
+ sharedLibVariations := append(target.Variations(), blueprint.Variation{Mutator: "link", Variation: "shared"})
+ mctx.AddFarVariationDependencies(sharedLibVariations, cc.JniFuzzLibTag, j.jniProperties.Jni_libs...)
+ }
+ }
+ j.Library.DepsMutator(mctx)
}
func (j *JavaFuzzLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- j.Library.GenerateAndroidBuildActions(ctx)
-
if j.fuzzPackagedModule.FuzzProperties.Corpus != nil {
j.fuzzPackagedModule.Corpus = android.PathsForModuleSrc(ctx, j.fuzzPackagedModule.FuzzProperties.Corpus)
}
@@ -55,6 +115,23 @@
android.WriteFileRule(ctx, configPath, j.fuzzPackagedModule.FuzzProperties.Fuzz_config.String())
j.fuzzPackagedModule.Config = configPath
}
+
+ ctx.VisitDirectDepsWithTag(cc.JniFuzzLibTag, func(dep android.Module) {
+ sharedLibInfo := ctx.OtherModuleProvider(dep, cc.SharedLibraryInfoProvider).(cc.SharedLibraryInfo)
+ if sharedLibInfo.SharedLibrary != nil {
+ libPath := android.PathForModuleOut(ctx, sharedLibInfo.SharedLibrary.Base())
+ ctx.Build(pctx, android.BuildParams{
+ Rule: android.Cp,
+ Input: sharedLibInfo.SharedLibrary,
+ Output: libPath,
+ })
+ j.jniFilePaths = append(j.jniFilePaths, libPath)
+ } else {
+ ctx.PropertyErrorf("jni_libs", "%q of type %q is not supported", dep.Name(), ctx.OtherModuleType(dep))
+ }
+ })
+
+ j.Library.GenerateAndroidBuildActions(ctx)
}
// java_fuzz builds and links sources into a `.jar` file for the host.
@@ -65,7 +142,8 @@
module := &JavaFuzzLibrary{}
module.addHostProperties()
- module.Module.properties.Installable = proptools.BoolPtr(false)
+ module.AddProperties(&module.jniProperties)
+ module.Module.properties.Installable = proptools.BoolPtr(true)
module.AddProperties(&module.fuzzPackagedModule.FuzzProperties)
// java_fuzz packaging rules collide when both linux_glibc and linux_bionic are enabled, disable the linux_bionic variants.
@@ -83,7 +161,7 @@
module.initModuleAndImport(module)
android.InitSdkAwareModule(module)
- InitJavaModule(module, android.HostSupported)
+ InitJavaModuleMultiTargets(module, android.HostSupported)
return module
}
@@ -106,26 +184,26 @@
ctx.VisitAllModules(func(module android.Module) {
// Discard non-fuzz targets.
- javaModule, ok := module.(*JavaFuzzLibrary)
+ javaFuzzModule, ok := module.(*JavaFuzzLibrary)
if !ok {
return
}
fuzzModuleValidator := fuzz.FuzzModule{
- javaModule.ModuleBase,
- javaModule.DefaultableModuleBase,
- javaModule.ApexModuleBase,
+ javaFuzzModule.ModuleBase,
+ javaFuzzModule.DefaultableModuleBase,
+ javaFuzzModule.ApexModuleBase,
}
- if ok := fuzz.IsValid(fuzzModuleValidator); !ok || *javaModule.Module.properties.Installable {
+ if ok := fuzz.IsValid(fuzzModuleValidator); !ok {
return
}
hostOrTargetString := "target"
- if javaModule.Host() {
+ if javaFuzzModule.Host() {
hostOrTargetString = "host"
}
- archString := javaModule.Arch().ArchType.String()
+ archString := javaFuzzModule.Arch().ArchType.String()
archDir := android.PathForIntermediates(ctx, "fuzz", hostOrTargetString, archString)
archOs := fuzz.ArchOs{HostOrTarget: hostOrTargetString, Arch: archString, Dir: archDir.String()}
@@ -134,12 +212,17 @@
builder := android.NewRuleBuilder(pctx, ctx)
// Package the artifacts (data, corpus, config and dictionary into a zipfile.
- files = s.PackageArtifacts(ctx, module, javaModule.fuzzPackagedModule, archDir, builder)
+ files = s.PackageArtifacts(ctx, module, javaFuzzModule.fuzzPackagedModule, archDir, builder)
// Add .jar
- files = append(files, fuzz.FileToZip{javaModule.outputFile, ""})
+ files = append(files, fuzz.FileToZip{javaFuzzModule.outputFile, ""})
- archDirs[archOs], ok = s.BuildZipFile(ctx, module, javaModule.fuzzPackagedModule, files, builder, archDir, archString, "host", archOs, archDirs)
+ // Add jni .so files
+ for _, fPath := range javaFuzzModule.jniFilePaths {
+ files = append(files, fuzz.FileToZip{fPath, ""})
+ }
+
+ archDirs[archOs], ok = s.BuildZipFile(ctx, module, javaFuzzModule.fuzzPackagedModule, files, builder, archDir, archString, hostOrTargetString, archOs, archDirs)
if !ok {
return
}
diff --git a/java/fuzz_test.go b/java/fuzz_test.go
index cf063eb..0a2c945 100644
--- a/java/fuzz_test.go
+++ b/java/fuzz_test.go
@@ -15,13 +15,17 @@
package java
import (
- "android/soong/android"
"path/filepath"
+ "runtime"
"testing"
+
+ "android/soong/android"
+ "android/soong/cc"
)
var prepForJavaFuzzTest = android.GroupFixturePreparers(
PrepareForTestWithJavaDefaultModules,
+ cc.PrepareForTestWithCcBuildComponents,
android.FixtureRegisterWithContext(RegisterJavaFuzzBuildComponents),
)
@@ -32,6 +36,13 @@
srcs: ["a.java"],
libs: ["bar"],
static_libs: ["baz"],
+ jni_libs: [
+ "libjni",
+ ],
+ sanitizers: [
+ "address",
+ "fuzzer",
+ ],
}
java_library_host {
@@ -42,11 +53,21 @@
java_library_host {
name: "baz",
srcs: ["c.java"],
- }`)
+ }
+
+ cc_library_shared {
+ name: "libjni",
+ host_supported: true,
+ device_supported: false,
+ stl: "none",
+ }
+ `)
osCommonTarget := result.Config.BuildOSCommonTarget.String()
- javac := result.ModuleForTests("foo", osCommonTarget).Rule("javac")
- combineJar := result.ModuleForTests("foo", osCommonTarget).Description("for javac")
+
+ osCommonTargetWithSan := osCommonTarget + "_asan" + "_fuzzer"
+ javac := result.ModuleForTests("foo", osCommonTargetWithSan).Rule("javac")
+ combineJar := result.ModuleForTests("foo", osCommonTargetWithSan).Description("for javac")
if len(javac.Inputs) != 1 || javac.Inputs[0].String() != "a.java" {
t.Errorf(`foo inputs %v != ["a.java"]`, javac.Inputs)
@@ -62,4 +83,18 @@
if len(combineJar.Inputs) != 2 || combineJar.Inputs[1].String() != baz {
t.Errorf("foo combineJar inputs %v does not contain %q", combineJar.Inputs, baz)
}
+
+ ctx := result.TestContext
+ foo := ctx.ModuleForTests("foo", osCommonTargetWithSan).Module().(*JavaFuzzLibrary)
+
+ expected := "libjni.so"
+ if runtime.GOOS == "darwin" {
+ expected = "libjni.dylib"
+ }
+
+ fooJniFilePaths := foo.jniFilePaths
+ if len(fooJniFilePaths) != 1 || fooJniFilePaths[0].Rel() != expected {
+ t.Errorf(`expected foo test data relative path [%q], got %q`,
+ expected, fooJniFilePaths.Strings())
+ }
}
diff --git a/java/hiddenapi.go b/java/hiddenapi.go
index 3af5f1c..cf9c7ad 100644
--- a/java/hiddenapi.go
+++ b/java/hiddenapi.go
@@ -65,6 +65,8 @@
type hiddenAPIModule interface {
android.Module
hiddenAPIIntf
+
+ MinSdkVersion(ctx android.EarlyModuleContext) android.SdkSpec
}
type hiddenAPIIntf interface {
@@ -148,7 +150,7 @@
// Create a copy of the dex jar which has been encoded with hiddenapi flags.
flagsCSV := hiddenAPISingletonPaths(ctx).flags
outputDir := android.PathForModuleOut(ctx, "hiddenapi").OutputPath
- encodedDex := hiddenAPIEncodeDex(ctx, dexJar, flagsCSV, uncompressDex, outputDir)
+ encodedDex := hiddenAPIEncodeDex(ctx, dexJar, flagsCSV, uncompressDex, android.SdkSpecNone, outputDir)
// Use the encoded dex jar from here onwards.
return encodedDex
@@ -246,7 +248,7 @@
// The encode dex rule requires unzipping, encoding and rezipping the classes.dex files along with
// all the resources from the input jar. It also ensures that if it was uncompressed in the input
// it stays uncompressed in the output.
-func hiddenAPIEncodeDex(ctx android.ModuleContext, dexInput, flagsCSV android.Path, uncompressDex bool, outputDir android.OutputPath) android.OutputPath {
+func hiddenAPIEncodeDex(ctx android.ModuleContext, dexInput, flagsCSV android.Path, uncompressDex bool, minSdkVersion android.SdkSpec, outputDir android.OutputPath) android.OutputPath {
// The output file has the same name as the input file and is in the output directory.
output := outputDir.Join(ctx, dexInput.Base())
@@ -274,6 +276,15 @@
hiddenapiFlags = "--no-force-assign-all"
}
+ // If the library is targeted for Q and/or R then make sure that they do not
+ // have any S+ flags encoded as that will break the runtime.
+ minApiLevel := minSdkVersion.ApiLevel
+ if !minApiLevel.IsNone() {
+ if minApiLevel.LessThanOrEqualTo(android.ApiLevelOrPanic(ctx, "R")) {
+ hiddenapiFlags = hiddenapiFlags + " --max-hiddenapi-level=max-target-r"
+ }
+ }
+
ctx.Build(pctx, android.BuildParams{
Rule: hiddenAPIEncodeDexRule,
Description: "hiddenapi encode dex",
diff --git a/java/hiddenapi_modular.go b/java/hiddenapi_modular.go
index 534a814..c90b2ff 100644
--- a/java/hiddenapi_modular.go
+++ b/java/hiddenapi_modular.go
@@ -1104,7 +1104,7 @@
for _, name := range android.SortedStringKeys(bootDexInfoByModule) {
bootDexInfo := bootDexInfoByModule[name]
unencodedDex := bootDexInfo.path
- encodedDex := hiddenAPIEncodeDex(ctx, unencodedDex, allFlagsCSV, bootDexInfo.uncompressDex, outputDir)
+ encodedDex := hiddenAPIEncodeDex(ctx, unencodedDex, allFlagsCSV, bootDexInfo.uncompressDex, bootDexInfo.minSdkVersion, outputDir)
encodedBootDexJarsByModule[name] = encodedDex
}
@@ -1188,6 +1188,9 @@
// Indicates whether the dex jar needs uncompressing before encoding.
uncompressDex bool
+
+ // The minimum sdk version that the dex jar will be used on.
+ minSdkVersion android.SdkSpec
}
// bootDexInfoByModule is a map from module name (as returned by module.Name()) to the boot dex
@@ -1213,6 +1216,7 @@
bootDexJarsByModule[module.Name()] = bootDexInfo{
path: bootDexJar,
uncompressDex: *hiddenAPIModule.uncompressDex(),
+ minSdkVersion: hiddenAPIModule.MinSdkVersion(ctx),
}
}
diff --git a/java/java.go b/java/java.go
index b34d6de..4e7e14c 100644
--- a/java/java.go
+++ b/java/java.go
@@ -2089,6 +2089,11 @@
if m.properties.Javacflags != nil {
javacopts = append(javacopts, m.properties.Javacflags...)
}
+ if m.properties.Java_version != nil {
+ javaVersion := normalizeJavaVersion(ctx, *m.properties.Java_version).String()
+ javacopts = append(javacopts, fmt.Sprintf("-source %s -target %s", javaVersion, javaVersion))
+ }
+
epEnabled := m.properties.Errorprone.Enabled
//TODO(b/227504307) add configuration that depends on RUN_ERROR_PRONE environment variable
if Bool(epEnabled) {
diff --git a/java/lint.go b/java/lint.go
index e97c9c2..426a2af 100644
--- a/java/lint.go
+++ b/java/lint.go
@@ -75,9 +75,9 @@
extraLintCheckJars android.Paths
test bool
library bool
- minSdkVersion string
- targetSdkVersion string
- compileSdkVersion string
+ minSdkVersion android.ApiLevel
+ targetSdkVersion android.ApiLevel
+ compileSdkVersion android.ApiLevel
compileSdkKind android.SdkKind
javaLanguageLevel string
kotlinLanguageLevel string
@@ -300,7 +300,7 @@
Text(`echo "<manifest xmlns:android='http://schemas.android.com/apk/res/android'" &&`).
Text(`echo " android:versionCode='1' android:versionName='1' >" &&`).
Textf(`echo " <uses-sdk android:minSdkVersion='%s' android:targetSdkVersion='%s'/>" &&`,
- l.minSdkVersion, l.targetSdkVersion).
+ l.minSdkVersion.String(), l.targetSdkVersion.String()).
Text(`echo "</manifest>"`).
Text(") >").Output(manifestPath)
@@ -325,7 +325,7 @@
return
}
- if l.minSdkVersion != l.compileSdkVersion {
+ if l.minSdkVersion.CompareTo(l.compileSdkVersion) == -1 {
l.extraMainlineLintErrors = append(l.extraMainlineLintErrors, updatabilityChecks...)
_, filtered := android.FilterList(l.properties.Lint.Warning_checks, updatabilityChecks)
if len(filtered) != 0 {
@@ -427,7 +427,7 @@
FlagWithOutput("--html ", html).
FlagWithOutput("--text ", text).
FlagWithOutput("--xml ", xml).
- FlagWithArg("--compile-sdk-version ", l.compileSdkVersion).
+ FlagWithArg("--compile-sdk-version ", l.compileSdkVersion.String()).
FlagWithArg("--java-language-level ", l.javaLanguageLevel).
FlagWithArg("--kotlin-language-level ", l.kotlinLanguageLevel).
FlagWithArg("--url ", fmt.Sprintf(".=.,%s=out", android.PathForOutput(ctx).String())).
diff --git a/java/lint_defaults.txt b/java/lint_defaults.txt
index 4bc0c5f..1eee354 100644
--- a/java/lint_defaults.txt
+++ b/java/lint_defaults.txt
@@ -28,6 +28,11 @@
--disable_check SuspiciousImport
--disable_check UnusedResources
--disable_check ViewConstructor
+# Disable NewApi checks for the platform since platform is the one that implements
+# the API. This prevents noisy lint warnings like b/228956345#1
+# NewApi checks will continue to be enforced for apex deps since
+# lint.strict_updatability_linting will be true for those Soong modules
+--disable_check NewApi
# Downgrade existing errors to warnings
--warning_check AppCompatResource # 55 occurences in 10 modules
@@ -66,7 +71,6 @@
--warning_check MissingTvBanner # 3 occurences in 3 modules
--warning_check NamespaceTypo # 3 occurences in 3 modules
--warning_check NetworkSecurityConfig # 46 occurences in 12 modules
---warning_check NewApi # 1996 occurences in 122 modules
--warning_check NotSibling # 15 occurences in 10 modules
--warning_check ObjectAnimatorBinding # 14 occurences in 5 modules
--warning_check OnClick # 49 occurences in 21 modules
diff --git a/java/sdk_library.go b/java/sdk_library.go
index c37ed1a..cd8e875 100644
--- a/java/sdk_library.go
+++ b/java/sdk_library.go
@@ -377,6 +377,9 @@
// List of Java libraries that will be in the classpath when building the implementation lib
Impl_only_libs []string `android:"arch_variant"`
+ // List of Java libraries that will included in the implementation lib.
+ Impl_only_static_libs []string `android:"arch_variant"`
+
// List of Java libraries that will be in the classpath when building stubs
Stub_only_libs []string `android:"arch_variant"`
@@ -1346,10 +1349,12 @@
visibility := childModuleVisibility(module.sdkLibraryProperties.Impl_library_visibility)
props := struct {
- Name *string
- Visibility []string
- Instrument bool
- Libs []string
+ Name *string
+ Visibility []string
+ Instrument bool
+ Libs []string
+ Static_libs []string
+ Apex_available []string
}{
Name: proptools.StringPtr(module.implLibraryModuleName()),
Visibility: visibility,
@@ -1358,6 +1363,12 @@
// Set the impl_only libs. Note that the module's "Libs" get appended as well, via the
// addition of &module.properties below.
Libs: module.sdkLibraryProperties.Impl_only_libs,
+ // Set the impl_only static libs. Note that the module's "static_libs" get appended as well, via the
+ // addition of &module.properties below.
+ Static_libs: module.sdkLibraryProperties.Impl_only_static_libs,
+ // Pass the apex_available settings down so that the impl library can be statically
+ // embedded within a library that is added to an APEX. Needed for updatable-media.
+ Apex_available: module.ApexAvailable(),
}
properties := []interface{}{
@@ -1814,8 +1825,9 @@
*javaSdkLibraries = append(*javaSdkLibraries, module.BaseModuleName())
}
- // Add the impl_only_libs *after* we're done using the Libs prop in submodules.
+ // Add the impl_only_libs and impl_only_static_libs *after* we're done using them in submodules.
module.properties.Libs = append(module.properties.Libs, module.sdkLibraryProperties.Impl_only_libs...)
+ module.properties.Static_libs = append(module.properties.Static_libs, module.sdkLibraryProperties.Impl_only_static_libs...)
}
func (module *SdkLibrary) InitSdkLibraryProperties() {
@@ -2211,8 +2223,23 @@
return module.uniqueApexVariations()
}
+// MinSdkVersion - Implements hiddenAPIModule
+func (module *SdkLibraryImport) MinSdkVersion(ctx android.EarlyModuleContext) android.SdkSpec {
+ return android.SdkSpecNone
+}
+
+var _ hiddenAPIModule = (*SdkLibraryImport)(nil)
+
func (module *SdkLibraryImport) OutputFiles(tag string) (android.Paths, error) {
- return module.commonOutputFiles(tag)
+ paths, err := module.commonOutputFiles(tag)
+ if paths != nil || err != nil {
+ return paths, err
+ }
+ if module.implLibraryModule != nil {
+ return module.implLibraryModule.OutputFiles(tag)
+ } else {
+ return nil, nil
+ }
}
func (module *SdkLibraryImport) GenerateAndroidBuildActions(ctx android.ModuleContext) {
diff --git a/mk2rbc/cmd/mk2rbc.go b/mk2rbc/cmd/mk2rbc.go
index e84eacd..cc83430 100644
--- a/mk2rbc/cmd/mk2rbc.go
+++ b/mk2rbc/cmd/mk2rbc.go
@@ -173,7 +173,7 @@
}
ok := true
for _, mkFile := range files {
- ok = convertOne(mkFile) && ok
+ ok = convertOne(mkFile, []string{}) && ok
}
if *launcher != "" {
@@ -183,7 +183,7 @@
if *inputVariables == "" {
quit(fmt.Errorf("the product launcher requires an input variables file"))
}
- if !convertOne(*inputVariables) {
+ if !convertOne(*inputVariables, []string{}) {
quit(fmt.Errorf("the product launcher input variables file failed to convert"))
}
@@ -201,7 +201,7 @@
if *inputVariables == "" {
quit(fmt.Errorf("the board launcher requires an input variables file"))
}
- if !convertOne(*inputVariables) {
+ if !convertOne(*inputVariables, []string{}) {
quit(fmt.Errorf("the board launcher input variables file failed to convert"))
}
err := writeGenerated(*boardlauncher, mk2rbc.BoardLauncher(
@@ -310,9 +310,13 @@
// the output hierarchy, or to the stdout.
// Optionally, recursively convert the files this one includes by
// $(call inherit-product) or an include statement.
-func convertOne(mkFile string) (ok bool) {
+func convertOne(mkFile string, loadStack []string) (ok bool) {
if v, ok := converted[mkFile]; ok {
- return v != nil
+ if v == nil {
+ fmt.Fprintf(os.Stderr, "Cycle in load graph:\n%s\n%s\n\n", strings.Join(loadStack, "\n"), mkFile)
+ return false
+ }
+ return true
}
converted[mkFile] = nil
defer func() {
@@ -356,6 +360,7 @@
return false
}
}
+ loadStack = append(loadStack, mkFile)
ok = true
if *recurse {
for _, sub := range ss.SubConfigFiles() {
@@ -363,7 +368,7 @@
if _, err := os.Stat(sub); os.IsNotExist(err) {
continue
}
- ok = convertOne(sub) && ok
+ ok = convertOne(sub, loadStack) && ok
}
}
converted[mkFile] = ss
diff --git a/mk2rbc/expr.go b/mk2rbc/expr.go
index 9266520..6a6eb46 100644
--- a/mk2rbc/expr.go
+++ b/mk2rbc/expr.go
@@ -741,8 +741,8 @@
return starlarkTypeUnknown
}
-func (_ *badExpr) emitListVarCopy(_ *generationContext) {
- panic("implement me")
+func (b *badExpr) emitListVarCopy(gctx *generationContext) {
+ b.emit(gctx)
}
func (b *badExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go
index 8f4fea4..1eae63f 100644
--- a/mk2rbc/mk2rbc.go
+++ b/mk2rbc/mk2rbc.go
@@ -86,7 +86,7 @@
"find-copy-subdir-files": &simpleCallParser{name: baseName + ".find_and_copy", returnType: starlarkTypeList},
"filter": &simpleCallParser{name: baseName + ".filter", returnType: starlarkTypeList},
"filter-out": &simpleCallParser{name: baseName + ".filter_out", returnType: starlarkTypeList},
- "firstword": &firstOrLastwordCallParser{isLastWord: false},
+ "firstword": &simpleCallParser{name: baseName + ".first_word", returnType: starlarkTypeString},
"foreach": &foreachCallParser{},
"if": &ifCallParser{},
"info": &makeControlFuncParser{name: baseName + ".mkinfo"},
@@ -97,7 +97,7 @@
"is-product-in-list": &isProductInListCallParser{},
"is-vendor-board-platform": &isVendorBoardPlatformCallParser{},
"is-vendor-board-qcom": &isVendorBoardQcomCallParser{},
- "lastword": &firstOrLastwordCallParser{isLastWord: true},
+ "lastword": &simpleCallParser{name: baseName + ".last_word", returnType: starlarkTypeString},
"notdir": &simpleCallParser{name: baseName + ".notdir", returnType: starlarkTypeString},
"math_max": &mathMaxOrMinCallParser{function: "max"},
"math_min": &mathMaxOrMinCallParser{function: "min"},
@@ -116,6 +116,7 @@
"subst": &substCallParser{fname: "subst"},
"warning": &makeControlFuncParser{name: baseName + ".mkwarning"},
"word": &wordCallParser{},
+ "words": &wordsCallParser{},
"wildcard": &simpleCallParser{name: baseName + ".expand_wildcard", returnType: starlarkTypeList},
}
@@ -130,6 +131,14 @@
"foreach": &foreachCallNodeParser{},
}
+// These look like variables, but are actually functions, and would give
+// undefined variable errors if we converted them as variables. Instead,
+// emit an error instead of converting them.
+var unsupportedFunctions = map[string]bool{
+ "local-generated-sources-dir": true,
+ "local-intermediates-dir": true,
+}
+
// These are functions that we don't implement conversions for, but
// we allow seeing their definitions in the product config files.
var ignoredDefines = map[string]bool{
@@ -459,6 +468,7 @@
predefined := []struct{ name, value string }{
{"SRC_TARGET_DIR", filepath.Join("build", "make", "target")},
{"LOCAL_PATH", filepath.Dir(ss.mkFile)},
+ {"MAKEFILE_LIST", ss.mkFile},
{"TOPDIR", ""}, // TOPDIR is just set to an empty string in cleanbuild.mk and core.mk
// TODO(asmundak): maybe read it from build/make/core/envsetup.mk?
{"TARGET_COPY_OUT_SYSTEM", "system"},
@@ -531,7 +541,7 @@
func (ctx *parseContext) handleAssignment(a *mkparser.Assignment) []starlarkNode {
// Handle only simple variables
- if !a.Name.Const() {
+ if !a.Name.Const() || a.Target != nil {
return []starlarkNode{ctx.newBadNode(a, "Only simple variables are handled")}
}
name := a.Name.Strings[0]
@@ -542,6 +552,12 @@
if strings.HasPrefix(name, "override ") {
return []starlarkNode{ctx.newBadNode(a, "cannot handle override directive")}
}
+ if name == ".KATI_READONLY" {
+ // Skip assignments to .KATI_READONLY. If it was in the output file, it
+ // would be an error because it would be sorted before the definition of
+ // the variable it's trying to make readonly.
+ return []starlarkNode{}
+ }
// Soong configuration
if strings.HasPrefix(name, soongNsPrefix) {
@@ -556,9 +572,6 @@
if lhs.valueType() == starlarkTypeUnknown {
// Try to divine variable type from the RHS
asgn.value = ctx.parseMakeString(a, a.Value)
- if xBad, ok := asgn.value.(*badExpr); ok {
- return []starlarkNode{&exprNode{xBad}}
- }
inferred_type := asgn.value.typ()
if inferred_type != starlarkTypeUnknown {
lhs.setValueType(inferred_type)
@@ -567,21 +580,19 @@
if lhs.valueType() == starlarkTypeList {
xConcat, xBad := ctx.buildConcatExpr(a)
if xBad != nil {
- return []starlarkNode{&exprNode{expr: xBad}}
- }
- switch len(xConcat.items) {
- case 0:
- asgn.value = &listExpr{}
- case 1:
- asgn.value = xConcat.items[0]
- default:
- asgn.value = xConcat
+ asgn.value = xBad
+ } else {
+ switch len(xConcat.items) {
+ case 0:
+ asgn.value = &listExpr{}
+ case 1:
+ asgn.value = xConcat.items[0]
+ default:
+ asgn.value = xConcat
+ }
}
} else {
asgn.value = ctx.parseMakeString(a, a.Value)
- if xBad, ok := asgn.value.(*badExpr); ok {
- return []starlarkNode{&exprNode{expr: xBad}}
- }
}
if asgn.lhs.valueType() == starlarkTypeString &&
@@ -811,35 +822,40 @@
// rblf.inherit(handle, _e[0], _e[1])
//
var matchingPaths []string
- varPath, ok := pathExpr.(*interpolateExpr)
- if !ok {
+ var needsWarning = false
+ if interpolate, ok := pathExpr.(*interpolateExpr); ok {
+ pathPattern := []string{interpolate.chunks[0]}
+ for _, chunk := range interpolate.chunks[1:] {
+ if chunk != "" {
+ pathPattern = append(pathPattern, chunk)
+ }
+ }
+ if pathPattern[0] == "" && len(ctx.includeTops) > 0 {
+ // If pattern starts from the top. restrict it to the directories where
+ // we know inherit-product uses dynamically calculated path.
+ for _, p := range ctx.includeTops {
+ pathPattern[0] = p
+ matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...)
+ }
+ } else {
+ matchingPaths = ctx.findMatchingPaths(pathPattern)
+ }
+ needsWarning = pathPattern[0] == "" && len(ctx.includeTops) == 0
+ } else if len(ctx.includeTops) > 0 {
+ for _, p := range ctx.includeTops {
+ matchingPaths = append(matchingPaths, ctx.findMatchingPaths([]string{p, ""})...)
+ }
+ } else {
return []starlarkNode{ctx.newBadNode(v, "inherit-product/include argument is too complex")}
}
- pathPattern := []string{varPath.chunks[0]}
- for _, chunk := range varPath.chunks[1:] {
- if chunk != "" {
- pathPattern = append(pathPattern, chunk)
- }
- }
- if pathPattern[0] == "" && len(ctx.includeTops) > 0 {
- // If pattern starts from the top. restrict it to the directories where
- // we know inherit-product uses dynamically calculated path.
- for _, p := range ctx.includeTops {
- pathPattern[0] = p
- matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...)
- }
- } else {
- matchingPaths = ctx.findMatchingPaths(pathPattern)
- }
// Safeguard against $(call inherit-product,$(PRODUCT_PATH))
const maxMatchingFiles = 150
if len(matchingPaths) > maxMatchingFiles {
return []starlarkNode{ctx.newBadNode(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)}
}
- needsWarning := pathPattern[0] == "" && len(ctx.includeTops) == 0
- res := inheritedDynamicModule{*varPath, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning}
+ res := inheritedDynamicModule{pathExpr, []*moduleInfo{}, loadAlways, ctx.errorLocation(v), needsWarning}
for _, p := range matchingPaths {
// A product configuration files discovered dynamically may attempt to inherit
// from another one which does not exist in this source tree. Prevent load errors
@@ -889,8 +905,9 @@
})
}
-func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) []starlarkNode {
- return ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) starlarkNode {
+func (ctx *parseContext) handleInclude(v *mkparser.Directive) []starlarkNode {
+ loadAlways := v.Name[0] != '-'
+ return ctx.handleSubConfig(v, ctx.parseMakeString(v, v.Args), loadAlways, func(im inheritedModule) starlarkNode {
return &includeNode{im, loadAlways}
})
}
@@ -1068,6 +1085,18 @@
return otherOperand
}
}
+ if otherOperand.typ() == starlarkTypeList {
+ fields := strings.Fields(stringOperand)
+ elements := make([]starlarkExpr, len(fields))
+ for i, s := range fields {
+ elements[i] = &stringLiteralExpr{literal: s}
+ }
+ return &eqExpr{
+ left: otherOperand,
+ right: &listExpr{elements},
+ isEq: isEq,
+ }
+ }
if intOperand, err := strconv.Atoi(strings.TrimSpace(stringOperand)); err == nil && otherOperand.typ() == starlarkTypeInt {
return &eqExpr{
left: otherOperand,
@@ -1113,8 +1142,6 @@
switch call.name {
case baseName + ".filter":
return ctx.parseCompareFilterFuncResult(directive, call, value, isEq)
- case baseName + ".expand_wildcard":
- return ctx.parseCompareWildcardFuncResult(directive, call, value, !isEq), true
case baseName + ".findstring":
return ctx.parseCheckFindstringFuncResult(directive, call, value, !isEq), true
case baseName + ".strip":
@@ -1159,22 +1186,6 @@
}
}
-func (ctx *parseContext) parseCompareWildcardFuncResult(directive *mkparser.Directive,
- xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
- if !isEmptyString(xValue) {
- return ctx.newBadExpr(directive, "wildcard result can be compared only to empty: %s", xValue)
- }
- callFunc := baseName + ".file_wildcard_exists"
- if s, ok := xCall.args[0].(*stringLiteralExpr); ok && !strings.ContainsAny(s.literal, "*?{[") {
- callFunc = baseName + ".file_exists"
- }
- var cc starlarkExpr = &callExpr{name: callFunc, args: xCall.args, returnType: starlarkTypeBool}
- if !negate {
- cc = ¬Expr{cc}
- }
- return cc
-}
-
func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive,
xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
if isEmptyString(xValue) {
@@ -1262,6 +1273,12 @@
return ctx.newBadExpr(node, "reference is too complex: %s", refDump)
}
+ if name, _, ok := ctx.maybeParseFunctionCall(node, ref); ok {
+ if _, unsupported := unsupportedFunctions[name]; unsupported {
+ return ctx.newBadExpr(node, "%s is not supported", refDump)
+ }
+ }
+
// If it is a single word, it can be a simple variable
// reference or a function call
if len(words) == 1 && !isMakeControlFunc(refDump) && refDump != "shell" && refDump != "eval" {
@@ -1309,9 +1326,8 @@
} else {
return ctx.newBadExpr(node, "cannot handle invoking %s", name)
}
- } else {
- return ctx.newBadExpr(node, "cannot handle %s", refDump)
}
+ return ctx.newBadExpr(node, "cannot handle %s", refDump)
}
type simpleCallParser struct {
@@ -1587,6 +1603,16 @@
for _, n := range a.actions {
transformNode(n, transformer)
}
+ case *inheritNode:
+ if b, ok := a.module.(inheritedDynamicModule); ok {
+ b.path = b.path.transform(transformer)
+ a.module = b
+ }
+ case *includeNode:
+ if b, ok := a.module.(inheritedDynamicModule); ok {
+ b.path = b.path.transform(transformer)
+ a.module = b
+ }
}
}
@@ -1637,9 +1663,11 @@
if len(words) != 2 {
return ctx.newBadExpr(node, "word function should have 2 arguments")
}
- var index uint64 = 0
+ var index = 0
if words[0].Const() {
- index, _ = strconv.ParseUint(strings.TrimSpace(words[0].Strings[0]), 10, 64)
+ if i, err := strconv.Atoi(strings.TrimSpace(words[0].Strings[0])); err == nil {
+ index = i
+ }
}
if index < 1 {
return ctx.newBadExpr(node, "word index should be constant positive integer")
@@ -1647,35 +1675,40 @@
words[1].TrimLeftSpaces()
words[1].TrimRightSpaces()
array := ctx.parseMakeString(node, words[1])
- if xBad, ok := array.(*badExpr); ok {
- return xBad
- }
- if array.typ() != starlarkTypeList {
- array = &callExpr{object: array, name: "split", returnType: starlarkTypeList}
- }
- return &indexExpr{array, &intLiteralExpr{int(index - 1)}}
-}
-
-type firstOrLastwordCallParser struct {
- isLastWord bool
-}
-
-func (p *firstOrLastwordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
- arg := ctx.parseMakeString(node, args)
- if bad, ok := arg.(*badExpr); ok {
+ if bad, ok := array.(*badExpr); ok {
return bad
}
- index := &intLiteralExpr{0}
- if p.isLastWord {
- if v, ok := arg.(*variableRefExpr); ok && v.ref.name() == "MAKEFILE_LIST" {
- return &stringLiteralExpr{ctx.script.mkFile}
+ if array.typ() != starlarkTypeList {
+ array = &callExpr{
+ name: baseName + ".words",
+ args: []starlarkExpr{array},
+ returnType: starlarkTypeList,
}
- index.literal = -1
}
- if arg.typ() == starlarkTypeList {
- return &indexExpr{arg, index}
+ return &indexExpr{array, &intLiteralExpr{index - 1}}
+}
+
+type wordsCallParser struct{}
+
+func (p *wordsCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
+ args.TrimLeftSpaces()
+ args.TrimRightSpaces()
+ array := ctx.parseMakeString(node, args)
+ if bad, ok := array.(*badExpr); ok {
+ return bad
}
- return &indexExpr{&callExpr{object: arg, name: "split", returnType: starlarkTypeList}, index}
+ if array.typ() != starlarkTypeList {
+ array = &callExpr{
+ name: baseName + ".words",
+ args: []starlarkExpr{array},
+ returnType: starlarkTypeList,
+ }
+ }
+ return &callExpr{
+ name: "len",
+ args: []starlarkExpr{array},
+ returnType: starlarkTypeInt,
+ }
}
func parseIntegerArguments(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString, expectedArgs int) ([]starlarkExpr, error) {
@@ -1759,10 +1792,23 @@
}
case *mkparser.Comment:
return []starlarkNode{&commentNode{strings.TrimSpace("#" + n.Comment)}}
+ case *mkparser.Directive:
+ if n.Name == "include" || n.Name == "-include" {
+ return ctx.handleInclude(n)
+ }
+ case *mkparser.Variable:
+ // Technically inherit-product(-if-exists) don't need to be put inside
+ // an eval, but some makefiles do it, presumably because they copy+pasted
+ // from a $(eval include ...)
+ if name, _, ok := ctx.maybeParseFunctionCall(n, n.Name); ok {
+ if name == "inherit-product" || name == "inherit-product-if-exists" {
+ return ctx.handleVariable(n)
+ }
+ }
}
}
- return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments and comments are supported")}
+ return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments, comments, includes, and inherit-products are supported")}
}
func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr {
@@ -1822,7 +1868,7 @@
result = []starlarkNode{res}
}
case "include", "-include":
- result = ctx.handleInclude(node, ctx.parseMakeString(node, x.Args), x.Name[0] != '-')
+ result = ctx.handleInclude(x)
case "ifeq", "ifneq", "ifdef", "ifndef":
result = []starlarkNode{ctx.handleIfBlock(x)}
default:
diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go
index de75129..7f236bb 100644
--- a/mk2rbc/mk2rbc_test.go
+++ b/mk2rbc/mk2rbc_test.go
@@ -117,8 +117,8 @@
def init(g, handle):
cfg = rblf.cfg(handle)
- rblf.mk2rbc_error("product.mk:2", "cannot handle invoking foo1")
- rblf.mk2rbc_error("product.mk:3", "cannot handle invoking foo0")
+ cfg["PRODUCT_NAME"] = rblf.mk2rbc_error("product.mk:2", "cannot handle invoking foo1")
+ cfg["PRODUCT_NAME"] = rblf.mk2rbc_error("product.mk:3", "cannot handle invoking foo0")
`,
},
{
@@ -568,14 +568,18 @@
endif
ifneq (,$(wildcard foo*.mk))
endif
+ifeq (foo1.mk foo2.mk barxyz.mk,$(wildcard foo*.mk bar*.mk))
+endif
`,
expected: `load("//build/make/core:product_config.rbc", "rblf")
def init(g, handle):
cfg = rblf.cfg(handle)
- if not rblf.file_exists("foo.mk"):
+ if not rblf.expand_wildcard("foo.mk"):
pass
- if rblf.file_wildcard_exists("foo*.mk"):
+ if rblf.expand_wildcard("foo*.mk"):
+ pass
+ if rblf.expand_wildcard("foo*.mk bar*.mk") == ["foo1.mk", "foo2.mk", "barxyz.mk"]:
pass
`,
},
@@ -808,6 +812,10 @@
PRODUCT_COPY_FILES := $(addprefix pfx-,a b c)
PRODUCT_COPY_FILES := $(addsuffix .sff, a b c)
PRODUCT_NAME := $(word 1, $(subst ., ,$(TARGET_BOARD_PLATFORM)))
+ifeq (1,$(words $(SOME_UNKNOWN_VARIABLE)))
+endif
+ifeq ($(words $(SOME_OTHER_VARIABLE)),$(SOME_INT_VARIABLE))
+endif
$(info $(patsubst %.pub,$(PRODUCT_NAME)%,$(PRODUCT_ADB_KEYS)))
$(info $$(dir foo/bar): $(dir foo/bar))
$(info $(firstword $(PRODUCT_COPY_FILES)))
@@ -830,14 +838,18 @@
cfg = rblf.cfg(handle)
cfg["PRODUCT_COPY_FILES"] = rblf.addprefix("pfx-", "a b c")
cfg["PRODUCT_COPY_FILES"] = rblf.addsuffix(".sff", "a b c")
- cfg["PRODUCT_NAME"] = ((g.get("TARGET_BOARD_PLATFORM", "")).replace(".", " ")).split()[0]
+ cfg["PRODUCT_NAME"] = rblf.words((g.get("TARGET_BOARD_PLATFORM", "")).replace(".", " "))[0]
+ if len(rblf.words(g.get("SOME_UNKNOWN_VARIABLE", ""))) == 1:
+ pass
+ if ("%d" % (len(rblf.words(g.get("SOME_OTHER_VARIABLE", ""))))) == g.get("SOME_INT_VARIABLE", ""):
+ pass
rblf.mkinfo("product.mk", rblf.mkpatsubst("%.pub", "%s%%" % cfg["PRODUCT_NAME"], g.get("PRODUCT_ADB_KEYS", "")))
rblf.mkinfo("product.mk", "$(dir foo/bar): %s" % rblf.dir("foo/bar"))
- rblf.mkinfo("product.mk", cfg["PRODUCT_COPY_FILES"][0])
- rblf.mkinfo("product.mk", cfg["PRODUCT_COPY_FILES"][-1])
- rblf.mkinfo("product.mk", rblf.dir("product.mk"))
- rblf.mkinfo("product.mk", rblf.dir(cfg["PRODUCT_COPY_FILES"][-1]))
- rblf.mkinfo("product.mk", rblf.dir((_foobar).split()[-1]))
+ rblf.mkinfo("product.mk", rblf.first_word(cfg["PRODUCT_COPY_FILES"]))
+ rblf.mkinfo("product.mk", rblf.last_word(cfg["PRODUCT_COPY_FILES"]))
+ rblf.mkinfo("product.mk", rblf.dir(rblf.last_word("product.mk")))
+ rblf.mkinfo("product.mk", rblf.dir(rblf.last_word(cfg["PRODUCT_COPY_FILES"])))
+ rblf.mkinfo("product.mk", rblf.dir(rblf.last_word(_foobar)))
rblf.mkinfo("product.mk", rblf.abspath("foo/bar"))
rblf.mkinfo("product.mk", rblf.notdir("foo/bar"))
rblf.soong_config_namespace(g, "snsconfig")
@@ -975,7 +987,7 @@
rblf.soong_config_namespace(g, "cvd")
rblf.soong_config_set(g, "cvd", "launch_configs", "cvd_config_auto.json")
rblf.soong_config_append(g, "cvd", "grub_config", "grub.cfg")
- rblf.mk2rbc_error("product.mk:7", "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: SOONG_CONFIG_cvd_grub_config")
+ _x = rblf.mk2rbc_error("product.mk:7", "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: SOONG_CONFIG_cvd_grub_config")
`,
}, {
desc: "soong namespace accesses",
@@ -1142,6 +1154,11 @@
MY_PATH:=foo
#RBC# include_top vendor/foo1
$(call inherit-product,$(MY_PATH)/cfg.mk)
+#RBC# include_top vendor/foo1
+$(call inherit-product,$(MY_OTHER_PATH))
+#RBC# include_top vendor/foo1
+$(foreach f,$(MY_MAKEFILES), \
+ $(call inherit-product,$(f)))
`,
expected: `load("//build/make/core:product_config.rbc", "rblf")
load("//vendor/foo1:cfg.star|init", _cfg_init = "init")
@@ -1156,6 +1173,21 @@
if not _varmod_init:
rblf.mkerror("product.mk", "Cannot find %s" % ("%s/cfg.mk" % g["MY_PATH"]))
rblf.inherit(handle, _varmod, _varmod_init)
+ _entry = {
+ "vendor/foo1/cfg.mk": ("vendor/foo1/cfg", _cfg_init),
+ }.get(g.get("MY_OTHER_PATH", ""))
+ (_varmod, _varmod_init) = _entry if _entry else (None, None)
+ if not _varmod_init:
+ rblf.mkerror("product.mk", "Cannot find %s" % (g.get("MY_OTHER_PATH", "")))
+ rblf.inherit(handle, _varmod, _varmod_init)
+ for f in rblf.words(g.get("MY_MAKEFILES", "")):
+ _entry = {
+ "vendor/foo1/cfg.mk": ("vendor/foo1/cfg", _cfg_init),
+ }.get(f)
+ (_varmod, _varmod_init) = _entry if _entry else (None, None)
+ if not _varmod_init:
+ rblf.mkerror("product.mk", "Cannot find %s" % (f))
+ rblf.inherit(handle, _varmod, _varmod_init)
`,
},
{
@@ -1242,13 +1274,15 @@
desc: "Ignore make rules",
mkname: "product.mk",
in: `
+foo: PRIVATE_VARIABLE = some_tool $< $@
foo: foo.c
gcc -o $@ $*`,
expected: `load("//build/make/core:product_config.rbc", "rblf")
def init(g, handle):
cfg = rblf.cfg(handle)
- rblf.mk2rbc_error("product.mk:2", "unsupported line rule: foo: foo.c\n#gcc -o $@ $*")
+ rblf.mk2rbc_error("product.mk:2", "Only simple variables are handled")
+ rblf.mk2rbc_error("product.mk:3", "unsupported line rule: foo: foo.c\n#gcc -o $@ $*")
`,
},
{
@@ -1269,6 +1303,7 @@
in: `
ifeq (,$(call foobar))
endif
+my_sources := $(local-generated-sources-dir)
`,
expected: `load("//build/make/core:product_config.rbc", "rblf")
@@ -1276,6 +1311,7 @@
cfg = rblf.cfg(handle)
if rblf.mk2rbc_error("build/product.mk:2", "cannot handle invoking foobar"):
pass
+ _my_sources = rblf.mk2rbc_error("build/product.mk:4", "local-generated-sources-dir is not supported")
`,
},
{
@@ -1509,24 +1545,54 @@
$(eval MY_VAR := foo)
$(eval # This is a test of eval functions)
$(eval $(TOO_COMPLICATED) := bar)
+$(eval include foo/font.mk)
+$(eval $(call inherit-product,vendor/foo1/cfg.mk))
+
$(foreach x,$(MY_LIST_VAR), \
$(eval PRODUCT_COPY_FILES += foo/bar/$(x):$(TARGET_COPY_OUT_VENDOR)/etc/$(x)) \
- $(if $(MY_OTHER_VAR),$(eval PRODUCT_COPY_FILES += $(MY_OTHER_VAR):foo/bar/$(x))) \
-)
+ $(if $(MY_OTHER_VAR),$(eval PRODUCT_COPY_FILES += $(MY_OTHER_VAR):foo/bar/$(x))))
+$(foreach x,$(MY_LIST_VAR), \
+ $(eval include foo/$(x).mk))
+`,
+ expected: `load("//build/make/core:product_config.rbc", "rblf")
+load("//foo:font.star", _font_init = "init")
+load("//vendor/foo1:cfg.star", _cfg_init = "init")
+
+def init(g, handle):
+ cfg = rblf.cfg(handle)
+ g["MY_VAR"] = "foo"
+ # This is a test of eval functions
+ rblf.mk2rbc_error("product.mk:5", "Eval expression too complex; only assignments, comments, includes, and inherit-products are supported")
+ _font_init(g, handle)
+ rblf.inherit(handle, "vendor/foo1/cfg", _cfg_init)
+ for x in rblf.words(g.get("MY_LIST_VAR", "")):
+ rblf.setdefault(handle, "PRODUCT_COPY_FILES")
+ cfg["PRODUCT_COPY_FILES"] += ("foo/bar/%s:%s/etc/%s" % (x, g.get("TARGET_COPY_OUT_VENDOR", ""), x)).split()
+ if g.get("MY_OTHER_VAR", ""):
+ cfg["PRODUCT_COPY_FILES"] += ("%s:foo/bar/%s" % (g.get("MY_OTHER_VAR", ""), x)).split()
+ for x in rblf.words(g.get("MY_LIST_VAR", "")):
+ _entry = {
+ "foo/font.mk": ("foo/font", _font_init),
+ }.get("foo/%s.mk" % _x)
+ (_varmod, _varmod_init) = _entry if _entry else (None, None)
+ if not _varmod_init:
+ rblf.mkerror("product.mk", "Cannot find %s" % ("foo/%s.mk" % _x))
+ _varmod_init(g, handle)
+`,
+ },
+ {
+ desc: ".KATI_READONLY",
+ mkname: "product.mk",
+ in: `
+MY_VAR := foo
+.KATI_READONLY := MY_VAR
`,
expected: `load("//build/make/core:product_config.rbc", "rblf")
def init(g, handle):
cfg = rblf.cfg(handle)
g["MY_VAR"] = "foo"
- # This is a test of eval functions
- rblf.mk2rbc_error("product.mk:5", "Eval expression too complex; only assignments and comments are supported")
- for x in rblf.words(g.get("MY_LIST_VAR", "")):
- rblf.setdefault(handle, "PRODUCT_COPY_FILES")
- cfg["PRODUCT_COPY_FILES"] += ("foo/bar/%s:%s/etc/%s" % (x, g.get("TARGET_COPY_OUT_VENDOR", ""), x)).split()
- if g.get("MY_OTHER_VAR", ""):
- cfg["PRODUCT_COPY_FILES"] += ("%s:foo/bar/%s" % (g.get("MY_OTHER_VAR", ""), x)).split()
`,
},
}
diff --git a/mk2rbc/node.go b/mk2rbc/node.go
index 7c39b9e..a01abd8 100644
--- a/mk2rbc/node.go
+++ b/mk2rbc/node.go
@@ -83,7 +83,7 @@
}
type inheritedDynamicModule struct {
- path interpolateExpr
+ path starlarkExpr
candidateModules []*moduleInfo
loadAlways bool
location ErrorLocation
@@ -120,7 +120,7 @@
}
func (i inheritedDynamicModule) pathExpr() starlarkExpr {
- return &i.path
+ return i.path
}
func (i inheritedDynamicModule) needsLoadCheck() bool {
diff --git a/multitree/Android.bp b/multitree/Android.bp
new file mode 100644
index 0000000..9b16d20
--- /dev/null
+++ b/multitree/Android.bp
@@ -0,0 +1,19 @@
+package {
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+bootstrap_go_package {
+ name: "soong-multitree",
+ pkgPath: "android/soong/multitree",
+ deps: [
+ "blueprint",
+ "soong-android",
+ ],
+ srcs: [
+ "api_surface.go",
+ "export.go",
+ "metadata.go",
+ "import.go",
+ ],
+ pluginFor: ["soong_build"],
+}
diff --git a/multitree/api_surface.go b/multitree/api_surface.go
new file mode 100644
index 0000000..f739a24
--- /dev/null
+++ b/multitree/api_surface.go
@@ -0,0 +1,119 @@
+// Copyright 2021 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package multitree
+
+import (
+ "android/soong/android"
+ "fmt"
+
+ "github.com/google/blueprint"
+)
+
+var (
+ pctx = android.NewPackageContext("android/soong/multitree")
+)
+
+func init() {
+ RegisterApiSurfaceBuildComponents(android.InitRegistrationContext)
+}
+
+var PrepareForTestWithApiSurface = android.FixtureRegisterWithContext(RegisterApiSurfaceBuildComponents)
+
+func RegisterApiSurfaceBuildComponents(ctx android.RegistrationContext) {
+ ctx.RegisterModuleType("api_surface", ApiSurfaceFactory)
+}
+
+type ApiSurface struct {
+ android.ModuleBase
+ ExportableModuleBase
+ properties apiSurfaceProperties
+
+ allOutputs android.Paths
+ taggedOutputs map[string]android.Paths
+}
+
+type apiSurfaceProperties struct {
+ Contributions []string
+}
+
+func ApiSurfaceFactory() android.Module {
+ module := &ApiSurface{}
+ module.AddProperties(&module.properties)
+ android.InitAndroidModule(module)
+ InitExportableModule(module)
+ return module
+}
+
+func (surface *ApiSurface) DepsMutator(ctx android.BottomUpMutatorContext) {
+ if surface.properties.Contributions != nil {
+ ctx.AddVariationDependencies(nil, nil, surface.properties.Contributions...)
+ }
+
+}
+func (surface *ApiSurface) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+ contributionFiles := make(map[string]android.Paths)
+ var allOutputs android.Paths
+ ctx.WalkDeps(func(child, parent android.Module) bool {
+ if contribution, ok := child.(ApiContribution); ok {
+ copied := contribution.CopyFilesWithTag(ctx)
+ for tag, files := range copied {
+ contributionFiles[child.Name()+"#"+tag] = files
+ }
+ for _, paths := range copied {
+ allOutputs = append(allOutputs, paths...)
+ }
+ return false // no transitive dependencies
+ }
+ return false
+ })
+
+ // phony target
+ ctx.Build(pctx, android.BuildParams{
+ Rule: blueprint.Phony,
+ Output: android.PathForPhony(ctx, ctx.ModuleName()),
+ Inputs: allOutputs,
+ })
+
+ surface.allOutputs = allOutputs
+ surface.taggedOutputs = contributionFiles
+}
+
+func (surface *ApiSurface) OutputFiles(tag string) (android.Paths, error) {
+ if tag != "" {
+ return nil, fmt.Errorf("unknown tag: %q", tag)
+ }
+ return surface.allOutputs, nil
+}
+
+func (surface *ApiSurface) TaggedOutputs() map[string]android.Paths {
+ return surface.taggedOutputs
+}
+
+func (surface *ApiSurface) Exportable() bool {
+ return true
+}
+
+var _ android.OutputFileProducer = (*ApiSurface)(nil)
+var _ Exportable = (*ApiSurface)(nil)
+
+type ApiContribution interface {
+ // copy files necessaryt to construct an API surface
+ // For C, it will be map.txt and .h files
+ // For Java, it will be api.txt
+ CopyFilesWithTag(ctx android.ModuleContext) map[string]android.Paths // output paths
+
+ // Generate Android.bp in out/ to use the exported .txt files
+ // GenerateBuildFiles(ctx ModuleContext) Paths //output paths
+}
diff --git a/multitree/export.go b/multitree/export.go
new file mode 100644
index 0000000..aecade5
--- /dev/null
+++ b/multitree/export.go
@@ -0,0 +1,67 @@
+// Copyright 2022 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package multitree
+
+import (
+ "android/soong/android"
+
+ "github.com/google/blueprint/proptools"
+)
+
+type moduleExportProperty struct {
+ // True if the module is exported to the other components in a multi-tree.
+ // Any components in the multi-tree can import this module to use.
+ Export *bool
+}
+
+type ExportableModuleBase struct {
+ properties moduleExportProperty
+}
+
+type Exportable interface {
+ // Properties for the exporable module.
+ exportableModuleProps() *moduleExportProperty
+
+ // Check if this module can be exported.
+ // If this returns false, the module will not be exported regardless of the 'export' value.
+ Exportable() bool
+
+ // Returns 'true' if this module has 'export: true'
+ // This module will not be exported if it returns 'false' to 'Exportable()' interface even if
+ // it has 'export: true'.
+ IsExported() bool
+
+ // Map from tags to outputs.
+ // Each module can tag their outputs for convenience.
+ TaggedOutputs() map[string]android.Paths
+}
+
+type ExportableModule interface {
+ android.Module
+ android.OutputFileProducer
+ Exportable
+}
+
+func InitExportableModule(module ExportableModule) {
+ module.AddProperties(module.exportableModuleProps())
+}
+
+func (m *ExportableModuleBase) exportableModuleProps() *moduleExportProperty {
+ return &m.properties
+}
+
+func (m *ExportableModuleBase) IsExported() bool {
+ return proptools.Bool(m.properties.Export)
+}
diff --git a/multitree/import.go b/multitree/import.go
new file mode 100644
index 0000000..1e5c421
--- /dev/null
+++ b/multitree/import.go
@@ -0,0 +1,96 @@
+// Copyright 2022 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package multitree
+
+import (
+ "android/soong/android"
+)
+
+var (
+ nameSuffix = ".imported"
+)
+
+type MultitreeImportedModuleInterface interface {
+ GetMultitreeImportedModuleName() string
+}
+
+func init() {
+ android.RegisterModuleType("imported_filegroup", importedFileGroupFactory)
+
+ android.PreArchMutators(RegisterMultitreePreArchMutators)
+}
+
+type importedFileGroupProperties struct {
+ // Imported modules from the other components in a multi-tree
+ Imported []string
+}
+
+type importedFileGroup struct {
+ android.ModuleBase
+
+ properties importedFileGroupProperties
+ srcs android.Paths
+}
+
+func (ifg *importedFileGroup) Name() string {
+ return ifg.BaseModuleName() + nameSuffix
+}
+
+func importedFileGroupFactory() android.Module {
+ module := &importedFileGroup{}
+ module.AddProperties(&module.properties)
+
+ android.InitAndroidModule(module)
+ return module
+}
+
+var _ MultitreeImportedModuleInterface = (*importedFileGroup)(nil)
+
+func (ifg *importedFileGroup) GetMultitreeImportedModuleName() string {
+ // The base module name of the imported filegroup is used as the imported module name
+ return ifg.BaseModuleName()
+}
+
+var _ android.SourceFileProducer = (*importedFileGroup)(nil)
+
+func (ifg *importedFileGroup) Srcs() android.Paths {
+ return ifg.srcs
+}
+
+func (ifg *importedFileGroup) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+ // srcs from this module must not be used. Adding a dot path to avoid the empty
+ // source failure. Still soong returns error when a module wants to build against
+ // this source, which is intended.
+ ifg.srcs = android.PathsForModuleSrc(ctx, []string{"."})
+}
+
+func RegisterMultitreePreArchMutators(ctx android.RegisterMutatorsContext) {
+ ctx.BottomUp("multitree_imported_rename", MultitreeImportedRenameMutator).Parallel()
+}
+
+func MultitreeImportedRenameMutator(ctx android.BottomUpMutatorContext) {
+ if m, ok := ctx.Module().(MultitreeImportedModuleInterface); ok {
+ name := m.GetMultitreeImportedModuleName()
+ if !ctx.OtherModuleExists(name) {
+ // Provide an empty filegroup not to break the build while updating the metadata.
+ // In other cases, soong will report an error to guide users to run 'm update-meta'
+ // first.
+ if !ctx.Config().TargetMultitreeUpdateMeta() {
+ ctx.ModuleErrorf("\"%s\" filegroup must be imported.\nRun 'm update-meta' first to import the filegroup.", name)
+ }
+ ctx.Rename(name)
+ }
+ }
+}
diff --git a/multitree/metadata.go b/multitree/metadata.go
new file mode 100644
index 0000000..3fd7215
--- /dev/null
+++ b/multitree/metadata.go
@@ -0,0 +1,74 @@
+// Copyright 2022 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package multitree
+
+import (
+ "android/soong/android"
+ "encoding/json"
+)
+
+func init() {
+ android.RegisterSingletonType("update-meta", UpdateMetaSingleton)
+}
+
+func UpdateMetaSingleton() android.Singleton {
+ return &updateMetaSingleton{}
+}
+
+type jsonImported struct {
+ FileGroups map[string][]string `json:",omitempty"`
+}
+
+type metadataJsonFlags struct {
+ Imported jsonImported `json:",omitempty"`
+ Exported map[string][]string `json:",omitempty"`
+}
+
+type updateMetaSingleton struct {
+ importedModules []string
+ generatedMetadataFile android.OutputPath
+}
+
+func (s *updateMetaSingleton) GenerateBuildActions(ctx android.SingletonContext) {
+ metadata := metadataJsonFlags{
+ Imported: jsonImported{
+ FileGroups: make(map[string][]string),
+ },
+ Exported: make(map[string][]string),
+ }
+ ctx.VisitAllModules(func(module android.Module) {
+ if ifg, ok := module.(*importedFileGroup); ok {
+ metadata.Imported.FileGroups[ifg.BaseModuleName()] = ifg.properties.Imported
+ }
+ if e, ok := module.(ExportableModule); ok {
+ if e.IsExported() && e.Exportable() {
+ for tag, files := range e.TaggedOutputs() {
+ // TODO(b/219846705): refactor this to a dictionary
+ metadata.Exported[e.Name()+":"+tag] = append(metadata.Exported[e.Name()+":"+tag], files.Strings()...)
+ }
+ }
+ }
+ })
+ jsonStr, err := json.Marshal(metadata)
+ if err != nil {
+ ctx.Errorf(err.Error())
+ }
+ s.generatedMetadataFile = android.PathForOutput(ctx, "multitree", "metadata.json")
+ android.WriteFileRule(ctx, s.generatedMetadataFile, string(jsonStr))
+}
+
+func (s *updateMetaSingleton) MakeVars(ctx android.MakeVarsContext) {
+ ctx.Strict("MULTITREE_METADATA", s.generatedMetadataFile.String())
+}
diff --git a/rust/config/allowed_list.go b/rust/config/allowed_list.go
index 802e1da..7468579 100644
--- a/rust/config/allowed_list.go
+++ b/rust/config/allowed_list.go
@@ -8,6 +8,7 @@
RustAllowedPaths = []string{
"device/google/cuttlefish",
"external/adhd",
+ "external/boringssl",
"external/crosvm",
"external/libchromeos-rs",
"external/minijail",
diff --git a/rust/config/global.go b/rust/config/global.go
index bcbfe9f..d11665c 100644
--- a/rust/config/global.go
+++ b/rust/config/global.go
@@ -24,7 +24,7 @@
var pctx = android.NewPackageContext("android/soong/rust/config")
var (
- RustDefaultVersion = "1.59.0"
+ RustDefaultVersion = "1.60.0"
RustDefaultBase = "prebuilts/rust/"
DefaultEdition = "2021"
Stdlibs = []string{
diff --git a/rust/test.go b/rust/test.go
index 250b765..6e53935 100644
--- a/rust/test.go
+++ b/rust/test.go
@@ -15,6 +15,8 @@
package rust
import (
+ "path/filepath"
+
"github.com/google/blueprint/proptools"
"android/soong/android"
@@ -151,9 +153,15 @@
ctx.ModuleErrorf("data_lib %q is not a linkable module", depName)
}
if linkableDep.OutputFile().Valid() {
+ // Copy the output in "lib[64]" so that it's compatible with
+ // the default rpath values.
+ libDir := "lib"
+ if linkableDep.Target().Arch.ArchType.Multilib == "lib64" {
+ libDir = "lib64"
+ }
test.data = append(test.data,
android.DataPath{SrcPath: linkableDep.OutputFile().Path(),
- RelativeInstallPath: linkableDep.RelativeInstallPath()})
+ RelativeInstallPath: filepath.Join(libDir, linkableDep.RelativeInstallPath())})
}
})
diff --git a/rust/test_test.go b/rust/test_test.go
index 1124176..8906f1c 100644
--- a/rust/test_test.go
+++ b/rust/test_test.go
@@ -187,12 +187,12 @@
t.Errorf("expected test output file to be 'main_test', but was '%s'", outputPath)
}
entries := android.AndroidMkEntriesForTest(t, ctx, module)[0]
- if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][0], ":test_lib.so:foo/bar/baz") {
- t.Errorf("expected LOCAL_TEST_DATA to end with `:test_lib.so:foo/bar/baz`,"+
+ if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][0], ":test_lib.so:lib64/foo/bar/baz") {
+ t.Errorf("expected LOCAL_TEST_DATA to end with `:test_lib.so:lib64/foo/bar/baz`,"+
" but was '%s'", entries.EntryMap["LOCAL_TEST_DATA"][0])
}
- if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][1], ":librust_test_lib.so:foo/bar/baz") {
- t.Errorf("expected LOCAL_TEST_DATA to end with `:librust_test_lib.so:foo/bar/baz`,"+
+ if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][1], ":librust_test_lib.so:lib64/foo/bar/baz") {
+ t.Errorf("expected LOCAL_TEST_DATA to end with `:librust_test_lib.so:lib64/foo/bar/baz`,"+
" but was '%s'", entries.EntryMap["LOCAL_TEST_DATA"][1])
}
if !strings.HasSuffix(entries.EntryMap["LOCAL_TEST_DATA"][2], ":rusty:foo/bar/baz") {
diff --git a/ui/build/build.go b/ui/build/build.go
index d261f89..aadf4af 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -18,6 +18,7 @@
"io/ioutil"
"os"
"path/filepath"
+ "sync"
"text/template"
"android/soong/ui/metrics"
@@ -205,6 +206,8 @@
return
}
+ defer waitForDist(ctx)
+
// checkProblematicFiles aborts the build if Android.mk or CleanSpec.mk are found at the root of the tree.
checkProblematicFiles(ctx)
@@ -329,8 +332,18 @@
}
}
+var distWaitGroup sync.WaitGroup
+
+// waitForDist waits for all backgrounded distGzipFile and distFile writes to finish
+func waitForDist(ctx Context) {
+ ctx.BeginTrace("soong_ui", "dist")
+ defer ctx.EndTrace()
+
+ distWaitGroup.Wait()
+}
+
// distGzipFile writes a compressed copy of src to the distDir if dist is enabled. Failures
-// are printed but non-fatal.
+// are printed but non-fatal. Uses the distWaitGroup func for backgrounding (optimization).
func distGzipFile(ctx Context, config Config, src string, subDirs ...string) {
if !config.Dist() {
return
@@ -343,13 +356,17 @@
ctx.Printf("failed to mkdir %s: %s", destDir, err.Error())
}
- if err := gzipFileToDir(src, destDir); err != nil {
- ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error())
- }
+ distWaitGroup.Add(1)
+ go func() {
+ defer distWaitGroup.Done()
+ if err := gzipFileToDir(src, destDir); err != nil {
+ ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error())
+ }
+ }()
}
// distFile writes a copy of src to the distDir if dist is enabled. Failures are printed but
-// non-fatal.
+// non-fatal. Uses the distWaitGroup func for backgrounding (optimization).
func distFile(ctx Context, config Config, src string, subDirs ...string) {
if !config.Dist() {
return
@@ -362,7 +379,11 @@
ctx.Printf("failed to mkdir %s: %s", destDir, err.Error())
}
- if _, err := copyFile(src, filepath.Join(destDir, filepath.Base(src))); err != nil {
- ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error())
- }
+ distWaitGroup.Add(1)
+ go func() {
+ defer distWaitGroup.Done()
+ if _, err := copyFile(src, filepath.Join(destDir, filepath.Base(src))); err != nil {
+ ctx.Printf("failed to dist %s: %s", filepath.Base(src), err.Error())
+ }
+ }()
}
diff --git a/ui/build/config.go b/ui/build/config.go
index e271bfc..0092ff1 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -1223,6 +1223,21 @@
return "RBE_use_application_default_credentials", "true"
}
+func (c *configImpl) IsGooglerEnvironment() bool {
+ cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG"
+ if v, ok := c.environ.Get(cf); ok {
+ return v == "googler"
+ }
+ return false
+}
+
+func (c *configImpl) GoogleProdCredsExist() bool {
+ if _, err := exec.Command("/usr/bin/prodcertstatus", "--simple_output", "--nocheck_loas").Output(); err != nil {
+ return false
+ }
+ return true
+}
+
func (c *configImpl) UseRemoteBuild() bool {
return c.UseGoma() || c.UseRBE()
}
diff --git a/ui/build/rbe.go b/ui/build/rbe.go
index 8f9a699..78d37b4 100644
--- a/ui/build/rbe.go
+++ b/ui/build/rbe.go
@@ -119,6 +119,7 @@
}
func stopRBE(ctx Context, config Config) {
+ defer checkProdCreds(ctx, config)
cmd := Command(ctx, config, "stopRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd), "-shutdown")
output, err := cmd.CombinedOutput()
if err != nil {
@@ -131,6 +132,15 @@
}
}
+func checkProdCreds(ctx Context, config Config) {
+ if !config.IsGooglerEnvironment() || config.GoogleProdCredsExist() {
+ return
+ }
+ fmt.Fprintln(ctx.Writer, "")
+ fmt.Fprintln(ctx.Writer, "\033[33mWARNING: Missing LOAS credentials, please run `gcert`. This will result in failing RBE builds in the future, see go/build-fast#authentication.\033[0m")
+ fmt.Fprintln(ctx.Writer, "")
+}
+
// DumpRBEMetrics creates a metrics protobuf file containing RBE related metrics.
// The protobuf file is created if RBE is enabled and the proxy service has
// started. The proxy service is shutdown in order to dump the RBE metrics to the
diff --git a/ui/build/soong.go b/ui/build/soong.go
index c7f22f9..8992b4f 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -15,7 +15,9 @@
package build
import (
+ "errors"
"fmt"
+ "io/fs"
"io/ioutil"
"os"
"path/filepath"
@@ -491,10 +493,14 @@
ninja("bootstrap", "bootstrap.ninja", targets...)
- var soongBuildMetrics *soong_metrics_proto.SoongBuildMetrics
if shouldCollectBuildSoongMetrics(config) {
soongBuildMetrics := loadSoongBuildMetrics(ctx, config)
- logSoongBuildMetrics(ctx, soongBuildMetrics)
+ if soongBuildMetrics != nil {
+ logSoongBuildMetrics(ctx, soongBuildMetrics)
+ if ctx.Metrics != nil {
+ ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics)
+ }
+ }
}
distGzipFile(ctx, config, config.SoongNinjaFile(), "soong")
@@ -504,9 +510,6 @@
distGzipFile(ctx, config, config.SoongMakeVarsMk(), "soong")
}
- if shouldCollectBuildSoongMetrics(config) && ctx.Metrics != nil {
- ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics)
- }
if config.JsonModuleGraph() {
distGzipFile(ctx, config, config.ModuleGraphFile(), "soong")
}
@@ -538,8 +541,12 @@
func loadSoongBuildMetrics(ctx Context, config Config) *soong_metrics_proto.SoongBuildMetrics {
soongBuildMetricsFile := filepath.Join(config.LogsDir(), "soong_build_metrics.pb")
- buf, err := ioutil.ReadFile(soongBuildMetricsFile)
- if err != nil {
+ buf, err := os.ReadFile(soongBuildMetricsFile)
+ if errors.Is(err, fs.ErrNotExist) {
+ // Soong may not have run during this invocation
+ ctx.Verbosef("Failed to read metrics file, %s: %s", soongBuildMetricsFile, err)
+ return nil
+ } else if err != nil {
ctx.Fatalf("Failed to load %s: %s", soongBuildMetricsFile, err)
}
soongBuildMetrics := &soong_metrics_proto.SoongBuildMetrics{}