Merge "Nullify stl and sanitize of llndk_headers"
diff --git a/android/config.go b/android/config.go
index 6f73a12..d680b65 100644
--- a/android/config.go
+++ b/android/config.go
@@ -93,8 +93,9 @@
deviceConfig *deviceConfig
- srcDir string // the path of the root source directory
- buildDir string // the path of the build output directory
+ srcDir string // the path of the root source directory
+ buildDir string // the path of the build output directory
+ moduleListFile string // the path to the file which lists blueprint files to parse.
env map[string]string
envLock sync.Mutex
@@ -316,7 +317,7 @@
// New creates a new Config object. The srcDir argument specifies the path to
// the root source directory. It also loads the config file, if found.
-func NewConfig(srcDir, buildDir string) (Config, error) {
+func NewConfig(srcDir, buildDir string, moduleListFile string) (Config, error) {
// Make a config with default options
config := &config{
ConfigFileName: filepath.Join(buildDir, configFileName),
@@ -328,7 +329,8 @@
buildDir: buildDir,
multilibConflicts: make(map[ArchType]bool),
- fs: pathtools.NewOsFs(absSrcDir),
+ moduleListFile: moduleListFile,
+ fs: pathtools.NewOsFs(absSrcDir),
}
config.deviceConfig = &deviceConfig{
@@ -873,12 +875,12 @@
enforceList := c.productVariables.EnforceRROTargets
// TODO(b/150820813) Some modules depend on static overlay, remove this after eliminating the dependency.
exemptedList := c.productVariables.EnforceRROExemptedTargets
- if exemptedList != nil {
+ if len(exemptedList) > 0 {
if InList(name, exemptedList) {
return false
}
}
- if enforceList != nil {
+ if len(enforceList) > 0 {
if InList("*", enforceList) {
return true
}
@@ -889,7 +891,7 @@
func (c *config) EnforceRROExcludedOverlay(path string) bool {
excluded := c.productVariables.EnforceRROExcludedOverlays
- if excluded != nil {
+ if len(excluded) > 0 {
return HasAnyPrefix(path, excluded)
}
return false
@@ -1057,7 +1059,7 @@
HasAnyPrefix(path, c.config.productVariables.JavaCoveragePaths) {
coverage = true
}
- if coverage && c.config.productVariables.JavaCoverageExcludePaths != nil {
+ if coverage && len(c.config.productVariables.JavaCoverageExcludePaths) > 0 {
if HasAnyPrefix(path, c.config.productVariables.JavaCoverageExcludePaths) {
coverage = false
}
@@ -1086,12 +1088,12 @@
// NativeCoveragePaths represents any path.
func (c *deviceConfig) NativeCoverageEnabledForPath(path string) bool {
coverage := false
- if c.config.productVariables.NativeCoveragePaths != nil {
+ if len(c.config.productVariables.NativeCoveragePaths) > 0 {
if InList("*", c.config.productVariables.NativeCoveragePaths) || HasAnyPrefix(path, c.config.productVariables.NativeCoveragePaths) {
coverage = true
}
}
- if coverage && c.config.productVariables.NativeCoverageExcludePaths != nil {
+ if coverage && len(c.config.productVariables.NativeCoverageExcludePaths) > 0 {
if HasAnyPrefix(path, c.config.productVariables.NativeCoverageExcludePaths) {
coverage = false
}
@@ -1162,21 +1164,21 @@
}
func (c *config) IntegerOverflowDisabledForPath(path string) bool {
- if c.productVariables.IntegerOverflowExcludePaths == nil {
+ if len(c.productVariables.IntegerOverflowExcludePaths) == 0 {
return false
}
return HasAnyPrefix(path, c.productVariables.IntegerOverflowExcludePaths)
}
func (c *config) CFIDisabledForPath(path string) bool {
- if c.productVariables.CFIExcludePaths == nil {
+ if len(c.productVariables.CFIExcludePaths) == 0 {
return false
}
return HasAnyPrefix(path, c.productVariables.CFIExcludePaths)
}
func (c *config) CFIEnabledForPath(path string) bool {
- if c.productVariables.CFIIncludePaths == nil {
+ if len(c.productVariables.CFIIncludePaths) == 0 {
return false
}
return HasAnyPrefix(path, c.productVariables.CFIIncludePaths)
diff --git a/android/module.go b/android/module.go
index 06079ca..f80f37e 100644
--- a/android/module.go
+++ b/android/module.go
@@ -228,6 +228,16 @@
// For more information, see Module.GenerateBuildActions within Blueprint's module_ctx.go
GenerateAndroidBuildActions(ModuleContext)
+ // Add dependencies to the components of a module, i.e. modules that are created
+ // by the module and which are considered to be part of the creating module.
+ //
+ // This is called before prebuilts are renamed so as to allow a dependency to be
+ // added directly to a prebuilt child module instead of depending on a source module
+ // and relying on prebuilt processing to switch to the prebuilt module if preferred.
+ //
+ // A dependency on a prebuilt must include the "prebuilt_" prefix.
+ ComponentDepsMutator(ctx BottomUpMutatorContext)
+
DepsMutator(BottomUpMutatorContext)
base() *ModuleBase
@@ -769,6 +779,8 @@
prefer32 func(ctx BaseModuleContext, base *ModuleBase, class OsClass) bool
}
+func (m *ModuleBase) ComponentDepsMutator(BottomUpMutatorContext) {}
+
func (m *ModuleBase) DepsMutator(BottomUpMutatorContext) {}
func (m *ModuleBase) AddProperties(props ...interface{}) {
@@ -2099,15 +2111,6 @@
m.checkbuildFiles = append(m.checkbuildFiles, srcPath)
}
-func findStringInSlice(str string, slice []string) int {
- for i, s := range slice {
- if s == str {
- return i
- }
- }
- return -1
-}
-
// SrcIsModule decodes module references in the format ":name" into the module name, or empty string if the input
// was not a module reference.
func SrcIsModule(s string) (module string) {
diff --git a/android/mutator.go b/android/mutator.go
index 77d5f43..b70c4ff 100644
--- a/android/mutator.go
+++ b/android/mutator.go
@@ -115,6 +115,18 @@
// a DefaultableHook.
RegisterDefaultsPreArchMutators,
+ // Add dependencies on any components so that any component references can be
+ // resolved within the deps mutator.
+ //
+ // Must be run after defaults so it can be used to create dependencies on the
+ // component modules that are creating in a DefaultableHook.
+ //
+ // Must be run before RegisterPrebuiltsPreArchMutators, i.e. before prebuilts are
+ // renamed. That is so that if a module creates components using a prebuilt module
+ // type that any dependencies (which must use prebuilt_ prefixes) are resolved to
+ // the prebuilt module and not the source module.
+ RegisterComponentsMutator,
+
// Create an association between prebuilt modules and their corresponding source
// modules (if any).
//
@@ -202,6 +214,7 @@
AddFarVariationDependencies([]blueprint.Variation, blueprint.DependencyTag, ...string)
AddInterVariantDependency(tag blueprint.DependencyTag, from, to blueprint.Module)
ReplaceDependencies(string)
+ ReplaceDependenciesIf(string, blueprint.ReplaceDependencyPredicate)
AliasVariation(variationName string)
}
@@ -252,8 +265,21 @@
return mutator
}
+func RegisterComponentsMutator(ctx RegisterMutatorsContext) {
+ ctx.BottomUp("component-deps", componentDepsMutator).Parallel()
+}
+
+// A special mutator that runs just prior to the deps mutator to allow the dependencies
+// on component modules to be added so that they can depend directly on a prebuilt
+// module.
+func componentDepsMutator(ctx BottomUpMutatorContext) {
+ if m := ctx.Module(); m.Enabled() {
+ m.ComponentDepsMutator(ctx)
+ }
+}
+
func depsMutator(ctx BottomUpMutatorContext) {
- if m, ok := ctx.Module().(Module); ok && m.Enabled() {
+ if m := ctx.Module(); m.Enabled() {
m.DepsMutator(ctx)
}
}
@@ -403,6 +429,10 @@
b.bp.ReplaceDependencies(name)
}
+func (b *bottomUpMutatorContext) ReplaceDependenciesIf(name string, predicate blueprint.ReplaceDependencyPredicate) {
+ b.bp.ReplaceDependenciesIf(name, predicate)
+}
+
func (b *bottomUpMutatorContext) AliasVariation(variationName string) {
b.bp.AliasVariation(variationName)
}
diff --git a/android/paths.go b/android/paths.go
index d8d51a7..d13b6d8 100644
--- a/android/paths.go
+++ b/android/paths.go
@@ -380,6 +380,18 @@
}
func expandOneSrcPath(ctx ModuleContext, s string, expandedExcludes []string) (Paths, error) {
+ excludePaths := func(paths Paths) Paths {
+ if len(expandedExcludes) == 0 {
+ return paths
+ }
+ remainder := make(Paths, 0, len(paths))
+ for _, p := range paths {
+ if !InList(p.String(), expandedExcludes) {
+ remainder = append(remainder, p)
+ }
+ }
+ return remainder
+ }
if m, t := SrcIsModuleWithTag(s); m != "" {
module := ctx.GetDirectDepWithTag(m, sourceOrOutputDepTag(t))
if module == nil {
@@ -390,20 +402,11 @@
if err != nil {
return nil, fmt.Errorf("path dependency %q: %s", s, err)
}
- return outputFiles, nil
+ return excludePaths(outputFiles), nil
} else if t != "" {
return nil, fmt.Errorf("path dependency %q is not an output file producing module", s)
} else if srcProducer, ok := module.(SourceFileProducer); ok {
- moduleSrcs := srcProducer.Srcs()
- for _, e := range expandedExcludes {
- for j := 0; j < len(moduleSrcs); j++ {
- if moduleSrcs[j].String() == e {
- moduleSrcs = append(moduleSrcs[:j], moduleSrcs[j+1:]...)
- j--
- }
- }
- }
- return moduleSrcs, nil
+ return excludePaths(srcProducer.Srcs()), nil
} else {
return nil, fmt.Errorf("path dependency %q is not a source file producing module", s)
}
@@ -418,8 +421,7 @@
reportPathErrorf(ctx, "module source path %q does not exist", p)
}
- j := findStringInSlice(p.String(), expandedExcludes)
- if j >= 0 {
+ if InList(p.String(), expandedExcludes) {
return nil, nil
}
return Paths{p}, nil
@@ -913,6 +915,22 @@
var _ Path = OutputPath{}
var _ WritablePath = OutputPath{}
+// toolDepPath is a Path representing a dependency of the build tool.
+type toolDepPath struct {
+ basePath
+}
+
+var _ Path = toolDepPath{}
+
+// pathForBuildToolDep returns a toolDepPath representing the given path string.
+// There is no validation for the path, as it is "trusted": It may fail
+// normal validation checks. For example, it may be an absolute path.
+// Only use this function to construct paths for dependencies of the build
+// tool invocation.
+func pathForBuildToolDep(ctx PathContext, path string) toolDepPath {
+ return toolDepPath{basePath{path, ctx.Config(), ""}}
+}
+
// PathForOutput joins the provided paths and returns an OutputPath that is
// validated to not escape the build dir.
// On error, it will return a usable, but invalid OutputPath, and report a ModuleError.
diff --git a/android/paths_test.go b/android/paths_test.go
index 9b45d3f..a9cd22b 100644
--- a/android/paths_test.go
+++ b/android/paths_test.go
@@ -1091,6 +1091,21 @@
rels: []string{"gen/c"},
},
{
+ name: "output file provider with exclude",
+ bp: `
+ test {
+ name: "foo",
+ srcs: [":b", ":c"],
+ exclude_srcs: [":c"]
+ }
+ output_file_provider {
+ name: "c",
+ outs: ["gen/c"],
+ }`,
+ srcs: []string{buildDir + "/.intermediates/ofp/b/gen/b"},
+ rels: []string{"gen/b"},
+ },
+ {
name: "special characters glob",
bp: `
test {
diff --git a/android/prebuilt.go b/android/prebuilt.go
index a29ec91..9f4df28 100644
--- a/android/prebuilt.go
+++ b/android/prebuilt.go
@@ -30,6 +30,16 @@
ctx.PostDepsMutators(RegisterPrebuiltsPostDepsMutators)
}
+// Marks a dependency tag as possibly preventing a reference to a source from being
+// replaced with the prebuilt.
+type ReplaceSourceWithPrebuilt interface {
+ blueprint.DependencyTag
+
+ // Return true if the dependency defined by this tag should be replaced with the
+ // prebuilt.
+ ReplaceSourceWithPrebuilt() bool
+}
+
type prebuiltDependencyTag struct {
blueprint.BaseDependencyTag
}
@@ -260,7 +270,13 @@
name := m.base().BaseModuleName()
if p.properties.UsePrebuilt {
if p.properties.SourceExists {
- ctx.ReplaceDependencies(name)
+ ctx.ReplaceDependenciesIf(name, func(from blueprint.Module, tag blueprint.DependencyTag, to blueprint.Module) bool {
+ if t, ok := tag.(ReplaceSourceWithPrebuilt); ok {
+ return t.ReplaceSourceWithPrebuilt()
+ }
+
+ return true
+ })
}
} else {
m.SkipInstall()
diff --git a/android/sdk.go b/android/sdk.go
index e823106..8115b69 100644
--- a/android/sdk.go
+++ b/android/sdk.go
@@ -266,6 +266,9 @@
SdkMemberType() SdkMemberType
}
+var _ SdkMemberTypeDependencyTag = (*sdkMemberDependencyTag)(nil)
+var _ ReplaceSourceWithPrebuilt = (*sdkMemberDependencyTag)(nil)
+
type sdkMemberDependencyTag struct {
blueprint.BaseDependencyTag
memberType SdkMemberType
@@ -275,6 +278,12 @@
return t.memberType
}
+// Prevent dependencies from the sdk/module_exports onto their members from being
+// replaced with a preferred prebuilt.
+func (t *sdkMemberDependencyTag) ReplaceSourceWithPrebuilt() bool {
+ return false
+}
+
func DependencyTagForSdkMemberType(memberType SdkMemberType) SdkMemberTypeDependencyTag {
return &sdkMemberDependencyTag{memberType: memberType}
}
diff --git a/android/writedocs.go b/android/writedocs.go
index 9e43e80..4eb15e6 100644
--- a/android/writedocs.go
+++ b/android/writedocs.go
@@ -44,6 +44,10 @@
}
func (c *docsSingleton) GenerateBuildActions(ctx SingletonContext) {
+ var deps Paths
+ deps = append(deps, pathForBuildToolDep(ctx, ctx.Config().moduleListFile))
+ deps = append(deps, pathForBuildToolDep(ctx, ctx.Config().ProductVariablesFileName))
+
// Generate build system docs for the primary builder. Generating docs reads the source
// files used to build the primary builder, but that dependency will be picked up through
// the dependency on the primary builder itself. There are no dependencies on the
@@ -63,6 +67,7 @@
ctx.Build(pctx, BuildParams{
Rule: soongDocs,
Output: docsFile,
+ Inputs: deps,
Args: map[string]string{
"outDir": PathForOutput(ctx, "docs").String(),
},
diff --git a/apex/apex.go b/apex/apex.go
index d0c1a09..b29017d 100644
--- a/apex/apex.go
+++ b/apex/apex.go
@@ -1750,9 +1750,15 @@
return false
}
+ dt := ctx.OtherModuleDependencyTag(child)
+
+ if _, ok := dt.(android.ExcludeFromApexContentsTag); ok {
+ return false
+ }
+
// Check for the direct dependencies that contribute to the payload
- if dt, ok := ctx.OtherModuleDependencyTag(child).(dependencyTag); ok {
- if dt.payload {
+ if adt, ok := dt.(dependencyTag); ok {
+ if adt.payload {
return do(ctx, parent, am, false /* externalDep */)
}
// As soon as the dependency graph crosses the APEX boundary, don't go further.
diff --git a/apex/apex_test.go b/apex/apex_test.go
index befb814..f064338 100644
--- a/apex/apex_test.go
+++ b/apex/apex_test.go
@@ -233,6 +233,7 @@
ctx.RegisterModuleType("apex_set", apexSetFactory)
ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators)
+ ctx.PreArchMutators(android.RegisterComponentsMutator)
ctx.PostDepsMutators(android.RegisterOverridePostDepsMutators)
cc.RegisterRequiredBuildComponentsForTest(ctx)
@@ -5790,6 +5791,41 @@
}
}
+func TestNonPreferredPrebuiltDependency(t *testing.T) {
+ _, _ = testApex(t, `
+ apex {
+ name: "myapex",
+ key: "myapex.key",
+ native_shared_libs: ["mylib"],
+ }
+
+ apex_key {
+ name: "myapex.key",
+ public_key: "testkey.avbpubkey",
+ private_key: "testkey.pem",
+ }
+
+ cc_library {
+ name: "mylib",
+ srcs: ["mylib.cpp"],
+ stubs: {
+ versions: ["10000"],
+ },
+ apex_available: ["myapex"],
+ }
+
+ cc_prebuilt_library_shared {
+ name: "mylib",
+ prefer: false,
+ srcs: ["prebuilt.so"],
+ stubs: {
+ versions: ["10000"],
+ },
+ apex_available: ["myapex"],
+ }
+ `)
+}
+
func TestMain(m *testing.M) {
run := func() int {
setUp()
diff --git a/cc/cc.go b/cc/cc.go
index 5f774c9..0d5b6bd 100644
--- a/cc/cc.go
+++ b/cc/cc.go
@@ -96,6 +96,9 @@
HeaderLibs []string
RuntimeLibs []string
+ // Used for data dependencies adjacent to tests
+ DataLibs []string
+
StaticUnwinderIfLegacy bool
ReexportSharedLibHeaders, ReexportStaticLibHeaders, ReexportHeaderLibHeaders []string
@@ -423,6 +426,7 @@
}
var (
+ dataLibDepTag = DependencyTag{Name: "data_lib", Library: true, Shared: true}
sharedExportDepTag = DependencyTag{Name: "shared", Library: true, Shared: true, ReexportFlags: true}
earlySharedDepTag = DependencyTag{Name: "early_shared", Library: true, Shared: true}
lateSharedDepTag = DependencyTag{Name: "late shared", Library: true, Shared: true}
@@ -1982,6 +1986,10 @@
actx.AddVariationDependencies([]blueprint.Variation{
{Mutator: "link", Variation: "shared"},
+ }, dataLibDepTag, deps.DataLibs...)
+
+ actx.AddVariationDependencies([]blueprint.Variation{
+ {Mutator: "link", Variation: "shared"},
}, runtimeDepTag, deps.RuntimeLibs...)
actx.AddDependency(c, genSourceDepTag, deps.GeneratedSources...)
@@ -3004,6 +3012,9 @@
lib.baseCompiler.Properties.Exclude_srcs = append(lib.baseCompiler.Properties.Exclude_srcs,
lib.baseCompiler.Properties.Target.Vendor.Exclude_srcs...)
+
+ lib.baseCompiler.Properties.Exclude_generated_sources = append(lib.baseCompiler.Properties.Exclude_generated_sources,
+ lib.baseCompiler.Properties.Target.Vendor.Exclude_generated_sources...)
}
}
@@ -3014,6 +3025,9 @@
lib.baseCompiler.Properties.Exclude_srcs = append(lib.baseCompiler.Properties.Exclude_srcs,
lib.baseCompiler.Properties.Target.Recovery.Exclude_srcs...)
+
+ lib.baseCompiler.Properties.Exclude_generated_sources = append(lib.baseCompiler.Properties.Exclude_generated_sources,
+ lib.baseCompiler.Properties.Target.Recovery.Exclude_generated_sources...)
}
}
diff --git a/cc/cc_test.go b/cc/cc_test.go
index f73e021..8a1c8ed 100644
--- a/cc/cc_test.go
+++ b/cc/cc_test.go
@@ -526,6 +526,56 @@
checkVndkLibrariesOutput(t, ctx, "vndkcorevariant.libraries.txt", []string{"libc++.so", "libvndk2.so", "libvndk_sp.so"})
}
+func TestDataLibs(t *testing.T) {
+ bp := `
+ cc_test_library {
+ name: "test_lib",
+ srcs: ["test_lib.cpp"],
+ gtest: false,
+ }
+
+ cc_test {
+ name: "main_test",
+ data_libs: ["test_lib"],
+ gtest: false,
+ }
+ `
+
+ config := TestConfig(buildDir, android.Android, nil, bp, nil)
+ config.TestProductVariables.DeviceVndkVersion = StringPtr("current")
+ config.TestProductVariables.Platform_vndk_version = StringPtr("VER")
+ config.TestProductVariables.VndkUseCoreVariant = BoolPtr(true)
+
+ ctx := testCcWithConfig(t, config)
+ module := ctx.ModuleForTests("main_test", "android_arm_armv7-a-neon").Module()
+ testBinary := module.(*Module).linker.(*testBinary)
+ outputFiles, err := module.(android.OutputFileProducer).OutputFiles("")
+ if err != nil {
+ t.Errorf("Expected cc_test to produce output files, error: %s", err)
+ return
+ }
+ if len(outputFiles) != 1 {
+ t.Errorf("expected exactly one output file. output files: [%s]", outputFiles)
+ return
+ }
+ if len(testBinary.dataPaths()) != 1 {
+ t.Errorf("expected exactly one test data file. test data files: [%s]", testBinary.dataPaths())
+ return
+ }
+
+ outputPath := outputFiles[0].String()
+ testBinaryPath := testBinary.dataPaths()[0].String()
+
+ if !strings.HasSuffix(outputPath, "/main_test") {
+ t.Errorf("expected test output file to be 'main_test', but was '%s'", outputPath)
+ return
+ }
+ if !strings.HasSuffix(testBinaryPath, "/test_lib.so") {
+ t.Errorf("expected test data file to be 'test_lib.so', but was '%s'", testBinaryPath)
+ return
+ }
+}
+
func TestVndkWhenVndkVersionIsNotSet(t *testing.T) {
ctx := testCcNoVndk(t, `
cc_library {
diff --git a/cc/ccdeps.go b/cc/ccdeps.go
index 225405c..b96d8b0 100644
--- a/cc/ccdeps.go
+++ b/cc/ccdeps.go
@@ -38,8 +38,11 @@
}
type ccdepsGeneratorSingleton struct {
+ outputPath android.Path
}
+var _ android.SingletonMakeVarsProvider = (*ccdepsGeneratorSingleton)(nil)
+
const (
// Environment variables used to control the behavior of this singleton.
envVariableCollectCCDeps = "SOONG_COLLECT_CC_DEPS"
@@ -110,6 +113,21 @@
if err != nil {
ctx.Errorf(err.Error())
}
+ c.outputPath = ccfpath
+
+ // This is necessary to satisfy the dangling rules check as this file is written by Soong rather than a rule.
+ ctx.Build(pctx, android.BuildParams{
+ Rule: android.Touch,
+ Output: ccfpath,
+ })
+}
+
+func (c *ccdepsGeneratorSingleton) MakeVars(ctx android.MakeVarsContext) {
+ if c.outputPath == nil {
+ return
+ }
+
+ ctx.DistForGoal("general-tests", c.outputPath)
}
func parseCompilerCCParameters(ctx android.SingletonContext, params []string) ccParameters {
diff --git a/cc/compiler.go b/cc/compiler.go
index b5f297c..d5ea2c3 100644
--- a/cc/compiler.go
+++ b/cc/compiler.go
@@ -86,6 +86,10 @@
// genrule modules.
Generated_sources []string `android:"arch_variant"`
+ // list of generated sources that should not be used to build the C/C++ module.
+ // This is most useful in the arch/multilib variants to remove non-common files
+ Exclude_generated_sources []string `android:"arch_variant"`
+
// list of generated headers to add to the include path. These are the names
// of genrule modules.
Generated_headers []string `android:"arch_variant"`
@@ -150,6 +154,10 @@
// List of additional cflags that should be used to build the vendor
// variant of the C/C++ module.
Cflags []string
+
+ // list of generated sources that should not be used to
+ // build the vendor variant of the C/C++ module.
+ Exclude_generated_sources []string
}
Recovery struct {
// list of source files that should only be used in the
@@ -163,6 +171,10 @@
// List of additional cflags that should be used to build the recovery
// variant of the C/C++ module.
Cflags []string
+
+ // list of generated sources that should not be used to
+ // build the recovery variant of the C/C++ module.
+ Exclude_generated_sources []string
}
}
@@ -227,6 +239,7 @@
func (compiler *baseCompiler) compilerDeps(ctx DepsContext, deps Deps) Deps {
deps.GeneratedSources = append(deps.GeneratedSources, compiler.Properties.Generated_sources...)
+ deps.GeneratedSources = removeListFromList(deps.GeneratedSources, compiler.Properties.Exclude_generated_sources)
deps.GeneratedHeaders = append(deps.GeneratedHeaders, compiler.Properties.Generated_headers...)
android.ProtoDeps(ctx, &compiler.Proto)
diff --git a/cc/config/global.go b/cc/config/global.go
index 7b651bc..373fc77 100644
--- a/cc/config/global.go
+++ b/cc/config/global.go
@@ -32,6 +32,7 @@
"-Wno-unused",
"-Winit-self",
"-Wpointer-arith",
+ "-Wunreachable-code-loop-increment",
// Make paths in deps files relative
"-no-canonical-prefixes",
@@ -51,6 +52,7 @@
"-Werror=date-time",
"-Werror=pragma-pack",
"-Werror=pragma-pack-suspicious-include",
+ "-Werror=unreachable-code-loop-increment",
}
commonGlobalConlyflags = []string{}
diff --git a/cc/library_sdk_member.go b/cc/library_sdk_member.go
index 9c54399..4b9eb30 100644
--- a/cc/library_sdk_member.go
+++ b/cc/library_sdk_member.go
@@ -234,7 +234,7 @@
for _, propertyInfo := range includeDirProperties {
// Calculate the base directory in the snapshot into which the files will be copied.
// lib.ArchType is "" for common properties.
- targetDir := filepath.Join(libInfo.archType, propertyInfo.snapshotDir)
+ targetDir := filepath.Join(libInfo.OsPrefix(), libInfo.archType, propertyInfo.snapshotDir)
propertyName := propertyInfo.propertyName
diff --git a/cc/test.go b/cc/test.go
index 09da976..37afb0c 100644
--- a/cc/test.go
+++ b/cc/test.go
@@ -58,6 +58,9 @@
// the test
Data []string `android:"path,arch_variant"`
+ // list of shared library modules that should be installed alongside the test
+ Data_libs []string `android:"arch_variant"`
+
// list of compatibility suites (for example "cts", "vts") that the module should be
// installed into.
Test_suites []string `android:"arch_variant"`
@@ -325,6 +328,7 @@
func (test *testBinary) linkerDeps(ctx DepsContext, deps Deps) Deps {
deps = test.testDecorator.linkerDeps(ctx, deps)
deps = test.binaryDecorator.linkerDeps(ctx, deps)
+ deps.DataLibs = append(deps.DataLibs, test.Properties.Data_libs...)
return deps
}
@@ -336,6 +340,21 @@
func (test *testBinary) install(ctx ModuleContext, file android.Path) {
test.data = android.PathsForModuleSrc(ctx, test.Properties.Data)
+
+ ctx.VisitDirectDepsWithTag(dataLibDepTag, func(dep android.Module) {
+ depName := ctx.OtherModuleName(dep)
+ ccDep, ok := dep.(LinkableInterface)
+
+ if !ok {
+ ctx.ModuleErrorf("data_lib %q is not a linkable cc module", depName)
+ }
+ if ccDep.OutputFile().Valid() {
+ test.data = append(test.data, ccDep.OutputFile().Path())
+ } else {
+ ctx.ModuleErrorf("data_lib %q has no output file", depName)
+ }
+ })
+
var api_level_prop string
var configs []tradefed.Config
var min_level string
@@ -507,6 +526,7 @@
func (benchmark *benchmarkDecorator) install(ctx ModuleContext, file android.Path) {
benchmark.data = android.PathsForModuleSrc(ctx, benchmark.Properties.Data)
+
var configs []tradefed.Config
if Bool(benchmark.Properties.Require_root) {
configs = append(configs, tradefed.Object{"target_preparer", "com.android.tradefed.targetprep.RootTargetPreparer", nil})
diff --git a/cc/testing.go b/cc/testing.go
index b0c3c162..a106d46 100644
--- a/cc/testing.go
+++ b/cc/testing.go
@@ -500,6 +500,7 @@
ctx := android.NewTestArchContext()
ctx.RegisterModuleType("cc_fuzz", FuzzFactory)
ctx.RegisterModuleType("cc_test", TestFactory)
+ ctx.RegisterModuleType("cc_test_library", TestLibraryFactory)
ctx.RegisterModuleType("llndk_headers", llndkHeadersFactory)
ctx.RegisterModuleType("vendor_public_library", vendorPublicLibraryFactory)
ctx.RegisterModuleType("filegroup", android.FileGroupFactory)
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index 905f206..532d9e4 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -59,7 +59,7 @@
ctx := android.NewContext()
ctx.Register()
- configuration, err := android.NewConfig(srcDir, bootstrap.BuildDir)
+ configuration, err := android.NewConfig(srcDir, bootstrap.BuildDir, bootstrap.ModuleListFile)
if err != nil {
fmt.Fprintf(os.Stderr, "%s", err)
os.Exit(1)
diff --git a/cmd/soong_build/writedocs.go b/cmd/soong_build/writedocs.go
index 9424b6c..c136846 100644
--- a/cmd/soong_build/writedocs.go
+++ b/cmd/soong_build/writedocs.go
@@ -44,7 +44,7 @@
"name": 0,
"src": 1,
"srcs": 2,
- "defautls": 3,
+ "defaults": 3,
"host_supported": 4,
"device_supported": 5,
}
diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go
index c965107..d3802f9 100644
--- a/cmd/soong_ui/main.go
+++ b/cmd/soong_ui/main.go
@@ -187,6 +187,7 @@
config.Parallel(), config.RemoteParallel(), config.HighmemParallel())
defer met.Dump(soongMetricsFile)
+ defer build.DumpRBEMetrics(buildCtx, config, rbeMetricsFile)
if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
if !strings.HasSuffix(start, "N") {
diff --git a/docs/perf.md b/docs/perf.md
index 538adff..86a27b4 100644
--- a/docs/perf.md
+++ b/docs/perf.md
@@ -12,6 +12,41 @@
+### Critical path
+
+soong_ui logs the wall time of the longest dependency chain compared to the
+elapsed wall time in `$OUT_DIR/soong.log`. For example:
+```
+critical path took 3m10s
+elapsed time 5m16s
+perfect parallelism ratio 60%
+critical path:
+ 0:00 build out/target/product/generic_arm64/obj/FAKE/sepolicy_neverallows_intermediates/policy_2.conf
+ 0:04 build out/target/product/generic_arm64/obj/FAKE/sepolicy_neverallows_intermediates/sepolicy_neverallows
+ 0:13 build out/target/product/generic_arm64/obj/ETC/plat_sepolicy.cil_intermediates/plat_sepolicy.cil
+ 0:01 build out/target/product/generic_arm64/obj/ETC/plat_pub_versioned.cil_intermediates/plat_pub_versioned.cil
+ 0:02 build out/target/product/generic_arm64/obj/ETC/vendor_sepolicy.cil_intermediates/vendor_sepolicy.cil
+ 0:16 build out/target/product/generic_arm64/obj/ETC/sepolicy_intermediates/sepolicy
+ 0:00 build out/target/product/generic_arm64/obj/ETC/plat_seapp_contexts_intermediates/plat_seapp_contexts
+ 0:00 Install: out/target/product/generic_arm64/system/etc/selinux/plat_seapp_contexts
+ 0:02 build out/target/product/generic_arm64/obj/NOTICE.txt
+ 0:00 build out/target/product/generic_arm64/obj/NOTICE.xml.gz
+ 0:00 build out/target/product/generic_arm64/system/etc/NOTICE.xml.gz
+ 0:01 Installed file list: out/target/product/generic_arm64/installed-files.txt
+ 1:00 Target system fs image: out/target/product/generic_arm64/obj/PACKAGING/systemimage_intermediates/system.img
+ 0:01 Install system fs image: out/target/product/generic_arm64/system.img
+ 0:01 Target vbmeta image: out/target/product/generic_arm64/vbmeta.img
+ 1:26 Package target files: out/target/product/generic_arm64/obj/PACKAGING/target_files_intermediates/aosp_arm64-target_files-6663974.zip
+ 0:01 Package: out/target/product/generic_arm64/aosp_arm64-img-6663974.zip
+ 0:01 Dist: /buildbot/dist_dirs/aosp-master-linux-aosp_arm64-userdebug/6663974/aosp_arm64-img-6663974.zip
+```
+
+If the elapsed time is much longer than the critical path then additional
+parallelism on the build machine will improve total build times. If there are
+long individual times listed in the critical path then improving build times
+for those steps or adjusting dependencies so that those steps can run earlier
+in the build graph will improve total build times.
+
### Soong
Soong can be traced and profiled using the standard Go tools. It understands
diff --git a/finder/finder.go b/finder/finder.go
index f20bed2..6513fa3 100644
--- a/finder/finder.go
+++ b/finder/finder.go
@@ -325,7 +325,12 @@
// Shutdown declares that the finder is no longer needed and waits for its cleanup to complete
// Currently, that only entails waiting for the database dump to complete.
func (f *Finder) Shutdown() {
- f.waitForDbDump()
+ f.WaitForDbDump()
+}
+
+// WaitForDbDump returns once the database has been written to f.DbPath.
+func (f *Finder) WaitForDbDump() {
+ f.shutdownWaitgroup.Wait()
}
// End of public api
@@ -345,10 +350,6 @@
}
}
-func (f *Finder) waitForDbDump() {
- f.shutdownWaitgroup.Wait()
-}
-
// joinCleanPaths is like filepath.Join but is faster because
// joinCleanPaths doesn't have to support paths ending in "/" or containing ".."
func joinCleanPaths(base string, leaf string) string {
diff --git a/java/java.go b/java/java.go
index 0d46672..46ef98b 100644
--- a/java/java.go
+++ b/java/java.go
@@ -140,10 +140,15 @@
// Findbugs
type CompilerProperties struct {
- // list of source files used to compile the Java module. May be .java, .logtags, .proto,
+ // list of source files used to compile the Java module. May be .java, .kt, .logtags, .proto,
// or .aidl files.
Srcs []string `android:"path,arch_variant"`
+ // list Kotlin of source files containing Kotlin code that should be treated as common code in
+ // a codebase that supports Kotlin multiplatform. See
+ // https://kotlinlang.org/docs/reference/multiplatform.html. May be only be .kt files.
+ Common_srcs []string `android:"path,arch_variant"`
+
// list of source files that should not be used to build the Java module.
// This is most useful in the arch/multilib variants to remove non-common files
Exclude_srcs []string `android:"path,arch_variant"`
@@ -1300,6 +1305,11 @@
flags = protoFlags(ctx, &j.properties, &j.protoProperties, flags)
}
+ kotlinCommonSrcFiles := android.PathsForModuleSrcExcludes(ctx, j.properties.Common_srcs, nil)
+ if len(kotlinCommonSrcFiles.FilterOutByExt(".kt")) > 0 {
+ ctx.PropertyErrorf("common_srcs", "common_srcs must be .kt files")
+ }
+
srcFiles = j.genSources(ctx, srcFiles, flags)
srcJars := srcFiles.FilterByExt(".srcjar")
@@ -1353,6 +1363,7 @@
// Collect .kt files for AIDEGen
j.expandIDEInfoCompiledSrcs = append(j.expandIDEInfoCompiledSrcs, srcFiles.FilterByExt(".kt").Strings()...)
+ j.expandIDEInfoCompiledSrcs = append(j.expandIDEInfoCompiledSrcs, kotlinCommonSrcFiles.Strings()...)
flags.classpath = append(flags.classpath, deps.kotlinStdlib...)
flags.classpath = append(flags.classpath, deps.kotlinAnnotations...)
@@ -1364,7 +1375,7 @@
// Use kapt for annotation processing
kaptSrcJar := android.PathForModuleOut(ctx, "kapt", "kapt-sources.jar")
kaptResJar := android.PathForModuleOut(ctx, "kapt", "kapt-res.jar")
- kotlinKapt(ctx, kaptSrcJar, kaptResJar, kotlinSrcFiles, srcJars, flags)
+ kotlinKapt(ctx, kaptSrcJar, kaptResJar, kotlinSrcFiles, kotlinCommonSrcFiles, srcJars, flags)
srcJars = append(srcJars, kaptSrcJar)
kotlinJars = append(kotlinJars, kaptResJar)
// Disable annotation processing in javac, it's already been handled by kapt
@@ -1373,7 +1384,7 @@
}
kotlinJar := android.PathForModuleOut(ctx, "kotlin", jarName)
- kotlinCompile(ctx, kotlinJar, kotlinSrcFiles, srcJars, flags)
+ kotlinCompile(ctx, kotlinJar, kotlinSrcFiles, kotlinCommonSrcFiles, srcJars, flags)
if ctx.Failed() {
return
}
diff --git a/java/java_test.go b/java/java_test.go
index def42db..fb00361 100644
--- a/java/java_test.go
+++ b/java/java_test.go
@@ -25,7 +25,6 @@
"strings"
"testing"
- "github.com/google/blueprint"
"github.com/google/blueprint/proptools"
"android/soong/android"
@@ -86,6 +85,7 @@
RegisterStubsBuildComponents(ctx)
RegisterSdkLibraryBuildComponents(ctx)
ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators)
+ ctx.PreArchMutators(android.RegisterComponentsMutator)
RegisterPrebuiltApisBuildComponents(ctx)
@@ -172,20 +172,6 @@
}
}
-func checkModuleDependencies(t *testing.T, ctx *android.TestContext, name, variant string, expected []string) {
- t.Helper()
- module := ctx.ModuleForTests(name, variant).Module()
- deps := []string{}
- ctx.VisitDirectDeps(module, func(m blueprint.Module) {
- deps = append(deps, m.Name())
- })
- sort.Strings(deps)
-
- if actual := deps; !reflect.DeepEqual(expected, actual) {
- t.Errorf("expected %#q, found %#q", expected, actual)
- }
-}
-
func TestJavaLinkType(t *testing.T) {
testJava(t, `
java_library {
@@ -646,7 +632,7 @@
}
}
- checkModuleDependencies(t, ctx, "sdklib", "android_common", []string{
+ CheckModuleDependencies(t, ctx, "sdklib", "android_common", []string{
`prebuilt_sdklib.stubs`,
`prebuilt_sdklib.stubs.source.test`,
`prebuilt_sdklib.stubs.system`,
@@ -674,7 +660,7 @@
}
`)
- checkModuleDependencies(t, ctx, "sdklib", "android_common", []string{
+ CheckModuleDependencies(t, ctx, "sdklib", "android_common", []string{
`dex2oatd`,
`prebuilt_sdklib`,
`sdklib.impl`,
@@ -683,12 +669,12 @@
`sdklib.xml`,
})
- checkModuleDependencies(t, ctx, "prebuilt_sdklib", "android_common", []string{
+ CheckModuleDependencies(t, ctx, "prebuilt_sdklib", "android_common", []string{
+ `prebuilt_sdklib.stubs`,
`sdklib.impl`,
// This should be prebuilt_sdklib.stubs but is set to sdklib.stubs because the
// dependency is added after prebuilts may have been renamed and so has to use
// the renamed name.
- `sdklib.stubs`,
`sdklib.xml`,
})
}
@@ -714,17 +700,16 @@
}
`)
- checkModuleDependencies(t, ctx, "sdklib", "android_common", []string{
+ CheckModuleDependencies(t, ctx, "sdklib", "android_common", []string{
`dex2oatd`,
`prebuilt_sdklib`,
- // This should be sdklib.stubs but is switched to the prebuilt because it is preferred.
- `prebuilt_sdklib.stubs`,
`sdklib.impl`,
+ `sdklib.stubs`,
`sdklib.stubs.source`,
`sdklib.xml`,
})
- checkModuleDependencies(t, ctx, "prebuilt_sdklib", "android_common", []string{
+ CheckModuleDependencies(t, ctx, "prebuilt_sdklib", "android_common", []string{
`prebuilt_sdklib.stubs`,
`sdklib.impl`,
`sdklib.xml`,
@@ -1491,7 +1476,7 @@
}
`)
- checkModuleDependencies(t, ctx, "sdklib", "android_common", []string{
+ CheckModuleDependencies(t, ctx, "sdklib", "android_common", []string{
`dex2oatd`,
`sdklib.impl`,
`sdklib.stubs`,
diff --git a/java/jdeps.go b/java/jdeps.go
index 9f43887..2b5ee74 100644
--- a/java/jdeps.go
+++ b/java/jdeps.go
@@ -34,8 +34,11 @@
}
type jdepsGeneratorSingleton struct {
+ outputPath android.Path
}
+var _ android.SingletonMakeVarsProvider = (*jdepsGeneratorSingleton)(nil)
+
const (
// Environment variables used to modify behavior of this singleton.
envVariableCollectJavaDeps = "SOONG_COLLECT_JAVA_DEPS"
@@ -97,6 +100,21 @@
if err != nil {
ctx.Errorf(err.Error())
}
+ j.outputPath = jfpath
+
+ // This is necessary to satisfy the dangling rules check as this file is written by Soong rather than a rule.
+ ctx.Build(pctx, android.BuildParams{
+ Rule: android.Touch,
+ Output: jfpath,
+ })
+}
+
+func (j *jdepsGeneratorSingleton) MakeVars(ctx android.MakeVarsContext) {
+ if j.outputPath == nil {
+ return
+ }
+
+ ctx.DistForGoal("general-tests", j.outputPath)
}
func createJsonFile(moduleInfos map[string]android.IdeInfo, jfpath android.WritablePath) error {
diff --git a/java/kotlin.go b/java/kotlin.go
index e3356be..e8c030a 100644
--- a/java/kotlin.go
+++ b/java/kotlin.go
@@ -33,7 +33,7 @@
`${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` +
`${config.GenKotlinBuildFileCmd} --classpath "$classpath" --name "$name"` +
` --out_dir "$classesDir" --srcs "$out.rsp" --srcs "$srcJarDir/list"` +
- ` --out "$kotlinBuildFile" && ` +
+ ` $commonSrcFilesArg --out "$kotlinBuildFile" && ` +
`${config.KotlincCmd} ${config.JavacHeapFlags} $kotlincFlags ` +
`-jvm-target $kotlinJvmTarget -Xbuild-file=$kotlinBuildFile -kotlin-home $emptyDir && ` +
`${config.SoongZipCmd} -jar -o $out -C $classesDir -D $classesDir && ` +
@@ -54,21 +54,43 @@
Rspfile: "$out.rsp",
RspfileContent: `$in`,
},
- "kotlincFlags", "classpath", "srcJars", "srcJarDir", "classesDir", "kotlinJvmTarget", "kotlinBuildFile",
- "emptyDir", "name")
+ "kotlincFlags", "classpath", "srcJars", "commonSrcFilesArg", "srcJarDir", "classesDir",
+ "kotlinJvmTarget", "kotlinBuildFile", "emptyDir", "name")
+
+func kotlinCommonSrcsList(ctx android.ModuleContext, commonSrcFiles android.Paths) android.OptionalPath {
+ if len(commonSrcFiles) > 0 {
+ // The list of common_srcs may be too long to put on the command line, but
+ // we can't use the rsp file because it is already being used for srcs.
+ // Insert a second rule to write out the list of resources to a file.
+ commonSrcsList := android.PathForModuleOut(ctx, "kotlinc_common_srcs.list")
+ rule := android.NewRuleBuilder()
+ rule.Command().Text("cp").FlagWithRspFileInputList("", commonSrcFiles).Output(commonSrcsList)
+ rule.Build(pctx, ctx, "kotlin_common_srcs_list", "kotlin common_srcs list")
+ return android.OptionalPathForPath(commonSrcsList)
+ }
+ return android.OptionalPath{}
+}
// kotlinCompile takes .java and .kt sources and srcJars, and compiles the .kt sources into a classes jar in outputFile.
func kotlinCompile(ctx android.ModuleContext, outputFile android.WritablePath,
- srcFiles, srcJars android.Paths,
+ srcFiles, commonSrcFiles, srcJars android.Paths,
flags javaBuilderFlags) {
var deps android.Paths
deps = append(deps, flags.kotlincClasspath...)
deps = append(deps, srcJars...)
+ deps = append(deps, commonSrcFiles...)
kotlinName := filepath.Join(ctx.ModuleDir(), ctx.ModuleSubDir(), ctx.ModuleName())
kotlinName = strings.ReplaceAll(kotlinName, "/", "__")
+ commonSrcsList := kotlinCommonSrcsList(ctx, commonSrcFiles)
+ commonSrcFilesArg := ""
+ if commonSrcsList.Valid() {
+ deps = append(deps, commonSrcsList.Path())
+ commonSrcFilesArg = "--common_srcs " + commonSrcsList.String()
+ }
+
ctx.Build(pctx, android.BuildParams{
Rule: kotlinc,
Description: "kotlinc",
@@ -76,13 +98,14 @@
Inputs: srcFiles,
Implicits: deps,
Args: map[string]string{
- "classpath": flags.kotlincClasspath.FormJavaClassPath(""),
- "kotlincFlags": flags.kotlincFlags,
- "srcJars": strings.Join(srcJars.Strings(), " "),
- "classesDir": android.PathForModuleOut(ctx, "kotlinc", "classes").String(),
- "srcJarDir": android.PathForModuleOut(ctx, "kotlinc", "srcJars").String(),
- "kotlinBuildFile": android.PathForModuleOut(ctx, "kotlinc-build.xml").String(),
- "emptyDir": android.PathForModuleOut(ctx, "kotlinc", "empty").String(),
+ "classpath": flags.kotlincClasspath.FormJavaClassPath(""),
+ "kotlincFlags": flags.kotlincFlags,
+ "commonSrcFilesArg": commonSrcFilesArg,
+ "srcJars": strings.Join(srcJars.Strings(), " "),
+ "classesDir": android.PathForModuleOut(ctx, "kotlinc", "classes").String(),
+ "srcJarDir": android.PathForModuleOut(ctx, "kotlinc", "srcJars").String(),
+ "kotlinBuildFile": android.PathForModuleOut(ctx, "kotlinc-build.xml").String(),
+ "emptyDir": android.PathForModuleOut(ctx, "kotlinc", "empty").String(),
// http://b/69160377 kotlinc only supports -jvm-target 1.6 and 1.8
"kotlinJvmTarget": "1.8",
"name": kotlinName,
@@ -97,7 +120,7 @@
`${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` +
`${config.GenKotlinBuildFileCmd} --classpath "$classpath" --name "$name"` +
` --srcs "$out.rsp" --srcs "$srcJarDir/list"` +
- ` --out "$kotlinBuildFile" && ` +
+ ` $commonSrcFilesArg --out "$kotlinBuildFile" && ` +
`${config.KotlincCmd} ${config.KotlincSuppressJDK9Warnings} ${config.JavacHeapFlags} $kotlincFlags ` +
`-Xplugin=${config.KotlinKaptJar} ` +
`-P plugin:org.jetbrains.kotlin.kapt3:sources=$kaptDir/sources ` +
@@ -124,21 +147,31 @@
RspfileContent: `$in`,
},
"kotlincFlags", "encodedJavacFlags", "kaptProcessorPath", "kaptProcessor",
- "classpath", "srcJars", "srcJarDir", "kaptDir", "kotlinJvmTarget", "kotlinBuildFile", "name",
- "classesJarOut")
+ "classpath", "srcJars", "commonSrcFilesArg", "srcJarDir", "kaptDir", "kotlinJvmTarget",
+ "kotlinBuildFile", "name", "classesJarOut")
// kotlinKapt performs Kotlin-compatible annotation processing. It takes .kt and .java sources and srcjars, and runs
// annotation processors over all of them, producing a srcjar of generated code in outputFile. The srcjar should be
// added as an additional input to kotlinc and javac rules, and the javac rule should have annotation processing
// disabled.
func kotlinKapt(ctx android.ModuleContext, srcJarOutputFile, resJarOutputFile android.WritablePath,
- srcFiles, srcJars android.Paths,
+ srcFiles, commonSrcFiles, srcJars android.Paths,
flags javaBuilderFlags) {
+ srcFiles = append(android.Paths(nil), srcFiles...)
+
var deps android.Paths
deps = append(deps, flags.kotlincClasspath...)
deps = append(deps, srcJars...)
deps = append(deps, flags.processorPath...)
+ deps = append(deps, commonSrcFiles...)
+
+ commonSrcsList := kotlinCommonSrcsList(ctx, commonSrcFiles)
+ commonSrcFilesArg := ""
+ if commonSrcsList.Valid() {
+ deps = append(deps, commonSrcsList.Path())
+ commonSrcFilesArg = "--common_srcs " + commonSrcsList.String()
+ }
kaptProcessorPath := flags.processorPath.FormRepeatedClassPath("-P plugin:org.jetbrains.kotlin.kapt3:apclasspath=")
@@ -168,6 +201,7 @@
Args: map[string]string{
"classpath": flags.kotlincClasspath.FormJavaClassPath(""),
"kotlincFlags": flags.kotlincFlags,
+ "commonSrcFilesArg": commonSrcFilesArg,
"srcJars": strings.Join(srcJars.Strings(), " "),
"srcJarDir": android.PathForModuleOut(ctx, "kapt", "srcJars").String(),
"kotlinBuildFile": android.PathForModuleOut(ctx, "kapt", "build.xml").String(),
diff --git a/java/sdk.go b/java/sdk.go
index c4861e3..6564f6d 100644
--- a/java/sdk.go
+++ b/java/sdk.go
@@ -449,7 +449,12 @@
case sdkTest:
return toModule([]string{"android_test_stubs_current"}, "framework-res", sdkFrameworkAidlPath(ctx))
case sdkCore:
- return toModule([]string{"core.current.stubs"}, "", nil)
+ return sdkDep{
+ useModule: true,
+ bootclasspath: []string{"core.current.stubs", config.DefaultLambdaStubsLibrary},
+ systemModules: "core-current-stubs-system-modules",
+ noFrameworksLibs: true,
+ }
case sdkModule:
// TODO(146757305): provide .apk and .aidl that have more APIs for modules
return toModule([]string{"android_module_lib_stubs_current"}, "framework-res", nonUpdatableFrameworkAidlPath(ctx))
diff --git a/java/sdk_library.go b/java/sdk_library.go
index 8f8f8ce..a5aa328 100644
--- a/java/sdk_library.go
+++ b/java/sdk_library.go
@@ -70,6 +70,12 @@
}
}
+var _ android.ReplaceSourceWithPrebuilt = (*scopeDependencyTag)(nil)
+
+func (tag scopeDependencyTag) ReplaceSourceWithPrebuilt() bool {
+ return false
+}
+
// Provides information about an api scope, e.g. public, system, test.
type apiScope struct {
// The name of the api scope, e.g. public, system, test
@@ -973,7 +979,8 @@
var implLibraryTag = sdkLibraryComponentTag{name: "impl-library"}
-func (module *SdkLibrary) DepsMutator(ctx android.BottomUpMutatorContext) {
+// Add the dependencies on the child modules in the component deps mutator.
+func (module *SdkLibrary) ComponentDepsMutator(ctx android.BottomUpMutatorContext) {
for _, apiScope := range module.getGeneratedApiScopes(ctx) {
// Add dependencies to the stubs library
ctx.AddVariationDependencies(nil, apiScope.stubsTag, module.stubsLibraryModuleName(apiScope))
@@ -998,7 +1005,12 @@
// Add dependency to the rule for generating the xml permissions file
ctx.AddDependency(module, xmlPermissionsFileTag, module.xmlPermissionsModuleName())
}
+ }
+}
+// Add other dependencies as normal.
+func (module *SdkLibrary) DepsMutator(ctx android.BottomUpMutatorContext) {
+ if module.requiresRuntimeImplementationLibrary() {
// Only add the deps for the library if it is actually going to be built.
module.Library.deps(ctx)
}
@@ -1874,20 +1886,26 @@
props.Prefer = proptools.BoolPtr(module.prebuilt.Prefer())
}
-func (module *SdkLibraryImport) DepsMutator(ctx android.BottomUpMutatorContext) {
+// Add the dependencies on the child module in the component deps mutator so that it
+// creates references to the prebuilt and not the source modules.
+func (module *SdkLibraryImport) ComponentDepsMutator(ctx android.BottomUpMutatorContext) {
for apiScope, scopeProperties := range module.scopeProperties {
if len(scopeProperties.Jars) == 0 {
continue
}
// Add dependencies to the prebuilt stubs library
- ctx.AddVariationDependencies(nil, apiScope.stubsTag, module.stubsLibraryModuleName(apiScope))
+ ctx.AddVariationDependencies(nil, apiScope.stubsTag, "prebuilt_"+module.stubsLibraryModuleName(apiScope))
if len(scopeProperties.Stub_srcs) > 0 {
// Add dependencies to the prebuilt stubs source library
- ctx.AddVariationDependencies(nil, apiScope.stubsSourceTag, module.stubsSourceModuleName(apiScope))
+ ctx.AddVariationDependencies(nil, apiScope.stubsSourceTag, "prebuilt_"+module.stubsSourceModuleName(apiScope))
}
}
+}
+
+// Add other dependencies as normal.
+func (module *SdkLibraryImport) DepsMutator(ctx android.BottomUpMutatorContext) {
implName := module.implLibraryModuleName()
if ctx.OtherModuleExists(implName) {
diff --git a/java/sdk_test.go b/java/sdk_test.go
index 773a70c..1f23b14 100644
--- a/java/sdk_test.go
+++ b/java/sdk_test.go
@@ -139,11 +139,10 @@
},
{
- name: "core_current",
- properties: `sdk_version: "core_current",`,
- bootclasspath: []string{"core.current.stubs", "core-lambda-stubs"},
- system: "core-current-stubs-system-modules",
- java9classpath: []string{"core.current.stubs"},
+ name: "core_current",
+ properties: `sdk_version: "core_current",`,
+ bootclasspath: []string{"core.current.stubs", "core-lambda-stubs"},
+ system: "core-current-stubs-system-modules",
},
{
diff --git a/java/testing.go b/java/testing.go
index f5688e6..94f054e 100644
--- a/java/testing.go
+++ b/java/testing.go
@@ -16,9 +16,13 @@
import (
"fmt"
+ "reflect"
+ "sort"
+ "testing"
"android/soong/android"
"android/soong/cc"
+ "github.com/google/blueprint"
)
func TestConfig(buildDir string, env map[string]string, bp string, fs map[string][]byte) android.Config {
@@ -216,3 +220,17 @@
return bp
}
+
+func CheckModuleDependencies(t *testing.T, ctx *android.TestContext, name, variant string, expected []string) {
+ t.Helper()
+ module := ctx.ModuleForTests(name, variant).Module()
+ deps := []string{}
+ ctx.VisitDirectDeps(module, func(m blueprint.Module) {
+ deps = append(deps, m.Name())
+ })
+ sort.Strings(deps)
+
+ if actual := deps; !reflect.DeepEqual(expected, actual) {
+ t.Errorf("expected %#q, found %#q", expected, actual)
+ }
+}
diff --git a/rust/binary.go b/rust/binary.go
index a1cd410..9fc52cd 100644
--- a/rust/binary.go
+++ b/rust/binary.go
@@ -106,7 +106,8 @@
func (binary *binaryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path {
fileName := binary.getStem(ctx) + ctx.toolchain().ExecutableSuffix()
- srcPath := srcPathFromModuleSrcs(ctx, binary.baseCompiler.Properties.Srcs)
+ srcPath, paths := srcPathFromModuleSrcs(ctx, binary.baseCompiler.Properties.Srcs)
+ deps.SrcDeps = paths
outputFile := android.PathForModuleOut(ctx, fileName)
binary.unstrippedOutputFile = outputFile
diff --git a/rust/builder.go b/rust/builder.go
index 16d7306..7f94bb5 100644
--- a/rust/builder.go
+++ b/rust/builder.go
@@ -166,6 +166,7 @@
implicits = append(implicits, rustLibsToPaths(deps.ProcMacros)...)
implicits = append(implicits, deps.StaticLibs...)
implicits = append(implicits, deps.SharedLibs...)
+ implicits = append(implicits, deps.SrcDeps...)
if deps.CrtBegin.Valid() {
implicits = append(implicits, deps.CrtBegin.Path(), deps.CrtEnd.Path())
}
diff --git a/rust/compiler.go b/rust/compiler.go
index 92a3b07..c20179b 100644
--- a/rust/compiler.go
+++ b/rust/compiler.go
@@ -253,10 +253,24 @@
return String(compiler.Properties.Relative_install_path)
}
-func srcPathFromModuleSrcs(ctx ModuleContext, srcs []string) android.Path {
- srcPaths := android.PathsForModuleSrc(ctx, srcs)
- if len(srcPaths) != 1 {
- ctx.PropertyErrorf("srcs", "srcs can only contain one path for rust modules")
+func srcPathFromModuleSrcs(ctx ModuleContext, srcs []string) (android.Path, android.Paths) {
+ // The srcs can contain strings with prefix ":".
+ // They are dependent modules of this module, with android.SourceDepTag.
+ // They are not the main source file compiled by rustc.
+ numSrcs := 0
+ srcIndex := 0
+ for i, s := range srcs {
+ if android.SrcIsModule(s) == "" {
+ numSrcs++
+ srcIndex = i
+ }
}
- return srcPaths[0]
+ if numSrcs != 1 {
+ ctx.PropertyErrorf("srcs", "srcs can only contain one path for a rust file")
+ }
+ if srcIndex != 0 {
+ ctx.PropertyErrorf("srcs", "main source file must be the first in srcs")
+ }
+ paths := android.PathsForModuleSrc(ctx, srcs)
+ return paths[srcIndex], paths
}
diff --git a/rust/compiler_test.go b/rust/compiler_test.go
index bcde757..58ca52a 100644
--- a/rust/compiler_test.go
+++ b/rust/compiler_test.go
@@ -43,7 +43,7 @@
// Test that we reject multiple source files.
func TestEnforceSingleSourceFile(t *testing.T) {
- singleSrcError := "srcs can only contain one path for rust modules"
+ singleSrcError := "srcs can only contain one path for a rust file"
// Test libraries
testRustError(t, singleSrcError, `
diff --git a/rust/library.go b/rust/library.go
index 8b8e797..d718eb8 100644
--- a/rust/library.go
+++ b/rust/library.go
@@ -368,7 +368,8 @@
func (library *libraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path {
var outputFile android.WritablePath
- srcPath := srcPathFromModuleSrcs(ctx, library.baseCompiler.Properties.Srcs)
+ srcPath, paths := srcPathFromModuleSrcs(ctx, library.baseCompiler.Properties.Srcs)
+ deps.SrcDeps = paths
flags.RustFlags = append(flags.RustFlags, deps.depFlags...)
diff --git a/rust/prebuilt.go b/rust/prebuilt.go
index 67d649d..3b4f40a 100644
--- a/rust/prebuilt.go
+++ b/rust/prebuilt.go
@@ -95,7 +95,8 @@
func (prebuilt *prebuiltLibraryDecorator) compile(ctx ModuleContext, flags Flags, deps PathDeps) android.Path {
prebuilt.exportLinkDirs(android.PathsForModuleSrc(ctx, prebuilt.Properties.Link_dirs).Strings()...)
- srcPath := srcPathFromModuleSrcs(ctx, prebuilt.prebuiltSrcs())
+ srcPath, paths := srcPathFromModuleSrcs(ctx, prebuilt.prebuiltSrcs())
+ deps.SrcDeps = paths
prebuilt.unstrippedOutputFile = srcPath
diff --git a/rust/proc_macro.go b/rust/proc_macro.go
index 2719161..49dbd8d 100644
--- a/rust/proc_macro.go
+++ b/rust/proc_macro.go
@@ -65,7 +65,8 @@
fileName := procMacro.getStem(ctx) + ctx.toolchain().ProcMacroSuffix()
outputFile := android.PathForModuleOut(ctx, fileName)
- srcPath := srcPathFromModuleSrcs(ctx, procMacro.baseCompiler.Properties.Srcs)
+ srcPath, paths := srcPathFromModuleSrcs(ctx, procMacro.baseCompiler.Properties.Srcs)
+ deps.SrcDeps = paths
procMacro.unstrippedOutputFile = outputFile
diff --git a/rust/rust.go b/rust/rust.go
index 72301a7..7a98c64 100644
--- a/rust/rust.go
+++ b/rust/rust.go
@@ -239,6 +239,9 @@
CrtBegin android.OptionalPath
CrtEnd android.OptionalPath
+
+ // Paths to generated source files
+ SrcDeps android.Paths
}
type RustLibraries []RustLibrary
@@ -843,6 +846,7 @@
// Dedup exported flags from dependencies
depPaths.linkDirs = android.FirstUniqueStrings(depPaths.linkDirs)
depPaths.depFlags = android.FirstUniqueStrings(depPaths.depFlags)
+ depPaths.SrcDeps = android.FirstUniquePaths(depPaths.SrcDeps)
return depPaths
}
diff --git a/rust/rust_test.go b/rust/rust_test.go
index 08bc8ca..e803925 100644
--- a/rust/rust_test.go
+++ b/rust/rust_test.go
@@ -61,6 +61,7 @@
"foo.rs": nil,
"foo.c": nil,
"src/bar.rs": nil,
+ "src/any.h": nil,
"liby.so": nil,
"libz.so": nil,
}
@@ -181,7 +182,7 @@
}
rust_library_host_rlib {
name: "librlib",
- srcs: ["foo.rs"],
+ srcs: ["foo.rs", ":my_generator"],
crate_name: "rlib",
}
rust_proc_macro {
@@ -189,17 +190,38 @@
srcs: ["foo.rs"],
crate_name: "pm",
}
+ genrule {
+ name: "my_generator",
+ tools: ["any_rust_binary"],
+ cmd: "$(location) -o $(out) $(in)",
+ srcs: ["src/any.h"],
+ out: ["src/any.rs"],
+ }
rust_binary_host {
- name: "fizz-buzz",
+ name: "fizz-buzz-dep",
dylibs: ["libdylib"],
rlibs: ["librlib"],
proc_macros: ["libpm"],
static_libs: ["libstatic"],
shared_libs: ["libshared"],
- srcs: ["foo.rs"],
+ srcs: [
+ "foo.rs",
+ ":my_generator",
+ ],
}
`)
- module := ctx.ModuleForTests("fizz-buzz", "linux_glibc_x86_64").Module().(*Module)
+ module := ctx.ModuleForTests("fizz-buzz-dep", "linux_glibc_x86_64").Module().(*Module)
+ rlibmodule := ctx.ModuleForTests("librlib", "linux_glibc_x86_64_rlib").Module().(*Module)
+
+ srcs := module.compiler.(*binaryDecorator).baseCompiler.Properties.Srcs
+ if len(srcs) != 2 || !android.InList(":my_generator", srcs) {
+ t.Errorf("missing module dependency in fizz-buzz)")
+ }
+
+ srcs = rlibmodule.compiler.(*libraryDecorator).baseCompiler.Properties.Srcs
+ if len(srcs) != 2 || !android.InList(":my_generator", srcs) {
+ t.Errorf("missing module dependency in rlib")
+ }
// Since dependencies are added to AndroidMk* properties, we can check these to see if they've been picked up.
if !android.InList("libdylib", module.Properties.AndroidMkDylibs) {
diff --git a/rust/testing.go b/rust/testing.go
index 3d583e1..430b40b 100644
--- a/rust/testing.go
+++ b/rust/testing.go
@@ -17,6 +17,7 @@
import (
"android/soong/android"
"android/soong/cc"
+ "android/soong/genrule"
)
func GatherRequiredDepsForTest() string {
@@ -77,6 +78,7 @@
func CreateTestContext() *android.TestContext {
ctx := android.NewTestArchContext()
cc.RegisterRequiredBuildComponentsForTest(ctx)
+ ctx.RegisterModuleType("genrule", genrule.GenRuleFactory)
ctx.RegisterModuleType("rust_binary", RustBinaryFactory)
ctx.RegisterModuleType("rust_binary_host", RustBinaryHostFactory)
ctx.RegisterModuleType("rust_test", RustTestFactory)
diff --git a/sdk/cc_sdk_test.go b/sdk/cc_sdk_test.go
index 123fe70..b168cd0 100644
--- a/sdk/cc_sdk_test.go
+++ b/sdk/cc_sdk_test.go
@@ -69,6 +69,28 @@
ensureListContains(t, inputs, arm64Output.String())
}
+func TestSdkCompileMultilibOverride(t *testing.T) {
+ result := testSdkWithCc(t, `
+ sdk {
+ name: "mysdk",
+ native_shared_libs: ["sdkmember"],
+ compile_multilib: "64",
+ }
+
+ cc_library_shared {
+ name: "sdkmember",
+ srcs: ["Test.cpp"],
+ stl: "none",
+ compile_multilib: "64",
+ }
+ `)
+
+ result.CheckSnapshot("mysdk", "",
+ checkAllCopyRules(`
+.intermediates/sdkmember/android_arm64_armv8-a_shared/sdkmember.so -> arm64/lib/sdkmember.so
+`))
+}
+
func TestBasicSdkWithCc(t *testing.T) {
result := testSdkWithCc(t, `
sdk {
@@ -79,6 +101,8 @@
cc_library_shared {
name: "sdkmember",
system_shared_libs: [],
+ stl: "none",
+ apex_available: ["mysdkapex"],
}
sdk_snapshot {
@@ -152,6 +176,13 @@
key: "myapex.key",
certificate: ":myapex.cert",
}
+
+ apex {
+ name: "mysdkapex",
+ native_shared_libs: ["sdkmember"],
+ key: "myapex.key",
+ certificate: ":myapex.cert",
+ }
`)
sdkMemberV1 := result.ModuleForTests("sdkmember_mysdk_1", "android_arm64_armv8-a_shared_myapex").Rule("toc").Output
@@ -1583,13 +1614,13 @@
sdk_member_name: "mynativeheaders",
host_supported: true,
stl: "none",
- export_system_include_dirs: ["include/include"],
+ export_system_include_dirs: ["common_os/include/include"],
target: {
android: {
- export_include_dirs: ["include/include-android"],
+ export_include_dirs: ["android/include/include-android"],
},
linux_glibc: {
- export_include_dirs: ["include/include-host"],
+ export_include_dirs: ["linux_glibc/include/include-host"],
},
},
}
@@ -1599,13 +1630,13 @@
prefer: false,
host_supported: true,
stl: "none",
- export_system_include_dirs: ["include/include"],
+ export_system_include_dirs: ["common_os/include/include"],
target: {
android: {
- export_include_dirs: ["include/include-android"],
+ export_include_dirs: ["android/include/include-android"],
},
linux_glibc: {
- export_include_dirs: ["include/include-host"],
+ export_include_dirs: ["linux_glibc/include/include-host"],
},
},
}
@@ -1617,9 +1648,9 @@
}
`),
checkAllCopyRules(`
-include/Test.h -> include/include/Test.h
-include-android/AndroidTest.h -> include/include-android/AndroidTest.h
-include-host/HostTest.h -> include/include-host/HostTest.h
+include/Test.h -> common_os/include/include/Test.h
+include-android/AndroidTest.h -> android/include/include-android/AndroidTest.h
+include-host/HostTest.h -> linux_glibc/include/include-host/HostTest.h
`),
)
}
diff --git a/sdk/java_sdk_test.go b/sdk/java_sdk_test.go
index 7496b20..79da3f0 100644
--- a/sdk/java_sdk_test.go
+++ b/sdk/java_sdk_test.go
@@ -16,6 +16,8 @@
import (
"testing"
+
+ "android/soong/java"
)
func testSdkWithJava(t *testing.T, bp string) *testSdkResult {
@@ -26,6 +28,9 @@
"resource.test": nil,
"aidl/foo/bar/Test.aidl": nil,
+ // For java_import
+ "prebuilt.jar": nil,
+
// For java_sdk_library
"api/current.txt": nil,
"api/removed.txt": nil,
@@ -85,6 +90,52 @@
// Contains tests for SDK members provided by the java package.
+func TestSdkDependsOnSourceEvenWhenPrebuiltPreferred(t *testing.T) {
+ result := testSdkWithJava(t, `
+ sdk {
+ name: "mysdk",
+ java_header_libs: ["sdkmember"],
+ }
+
+ java_library {
+ name: "sdkmember",
+ srcs: ["Test.java"],
+ system_modules: "none",
+ sdk_version: "none",
+ }
+
+ java_import {
+ name: "sdkmember",
+ prefer: true,
+ jars: ["prebuilt.jar"],
+ }
+ `)
+
+ // Make sure that the mysdk module depends on "sdkmember" and not "prebuilt_sdkmember".
+ java.CheckModuleDependencies(t, result.ctx, "mysdk", "android_common", []string{"sdkmember"})
+
+ result.CheckSnapshot("mysdk", "",
+ checkAndroidBpContents(`// This is auto-generated. DO NOT EDIT.
+
+java_import {
+ name: "mysdk_sdkmember@current",
+ sdk_member_name: "sdkmember",
+ jars: ["java/sdkmember.jar"],
+}
+
+java_import {
+ name: "sdkmember",
+ prefer: false,
+ jars: ["java/sdkmember.jar"],
+}
+
+sdk_snapshot {
+ name: "mysdk@current",
+ java_header_libs: ["mysdk_sdkmember@current"],
+}
+`))
+}
+
func TestBasicSdkWithJavaLibrary(t *testing.T) {
result := testSdkWithJava(t, `
sdk {
diff --git a/sdk/sdk.go b/sdk/sdk.go
index b9b8199..3e76008 100644
--- a/sdk/sdk.go
+++ b/sdk/sdk.go
@@ -218,7 +218,7 @@
Compile_multilib *string
}
p := &props{Compile_multilib: proptools.StringPtr("both")}
- ctx.AppendProperties(p)
+ ctx.PrependProperties(p)
})
return s
}
@@ -330,6 +330,11 @@
blueprint.BaseDependencyTag
}
+// Mark this tag so dependencies that use it are excluded from APEX contents.
+func (t dependencyTag) ExcludeFromApexContents() {}
+
+var _ android.ExcludeFromApexContentsTag = dependencyTag{}
+
// For dependencies from an in-development version of an SDK member to frozen versions of the same member
// e.g. libfoo -> libfoo.mysdk.11 and libfoo.mysdk.12
type sdkMemberVersionedDepTag struct {
diff --git a/sdk/testing.go b/sdk/testing.go
index 4361754..4d029e4 100644
--- a/sdk/testing.go
+++ b/sdk/testing.go
@@ -44,14 +44,15 @@
` + cc.GatherRequiredDepsForTest(android.Android, android.Windows)
mockFS := map[string][]byte{
- "build/make/target/product/security": nil,
- "apex_manifest.json": nil,
- "system/sepolicy/apex/myapex-file_contexts": nil,
- "system/sepolicy/apex/myapex2-file_contexts": nil,
- "myapex.avbpubkey": nil,
- "myapex.pem": nil,
- "myapex.x509.pem": nil,
- "myapex.pk8": nil,
+ "build/make/target/product/security": nil,
+ "apex_manifest.json": nil,
+ "system/sepolicy/apex/myapex-file_contexts": nil,
+ "system/sepolicy/apex/myapex2-file_contexts": nil,
+ "system/sepolicy/apex/mysdkapex-file_contexts": nil,
+ "myapex.avbpubkey": nil,
+ "myapex.pem": nil,
+ "myapex.x509.pem": nil,
+ "myapex.pk8": nil,
}
cc.GatherRequiredFilesForTest(mockFS)
@@ -84,6 +85,7 @@
android.RegisterPackageBuildComponents(ctx)
ctx.PreArchMutators(android.RegisterVisibilityRuleChecker)
ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators)
+ ctx.PreArchMutators(android.RegisterComponentsMutator)
ctx.PreArchMutators(android.RegisterVisibilityRuleGatherer)
ctx.PostDepsMutators(android.RegisterVisibilityRuleEnforcer)
diff --git a/ui/build/Android.bp b/ui/build/Android.bp
index 0a0bb16..4ef2721 100644
--- a/ui/build/Android.bp
+++ b/ui/build/Android.bp
@@ -63,6 +63,7 @@
"cleanbuild_test.go",
"config_test.go",
"environment_test.go",
+ "rbe_test.go",
"upload_test.go",
"util_test.go",
"proc_sync_test.go",
diff --git a/ui/build/config.go b/ui/build/config.go
index c4bbad7..e567e40 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -804,6 +804,15 @@
return true
}
+func (c *configImpl) RBEStatsOutputDir() string {
+ for _, f := range []string{"RBE_output_dir", "FLAG_output_dir"} {
+ if v, ok := c.environ.Get(f); ok {
+ return v
+ }
+ }
+ return ""
+}
+
func (c *configImpl) UseRemoteBuild() bool {
return c.UseGoma() || c.UseRBE()
}
diff --git a/ui/build/finder.go b/ui/build/finder.go
index 6786337..c019ea2 100644
--- a/ui/build/finder.go
+++ b/ui/build/finder.go
@@ -126,6 +126,11 @@
if err != nil {
ctx.Fatalf("Could not find modules: %v", err)
}
+
+ if config.Dist() {
+ f.WaitForDbDump()
+ distFile(ctx, config, f.DbPath, "module_paths")
+ }
}
func dumpListToFile(ctx Context, config Config, list []string, filePath string) (err error) {
diff --git a/ui/build/rbe.go b/ui/build/rbe.go
index ceea4bf..fcdab3b 100644
--- a/ui/build/rbe.go
+++ b/ui/build/rbe.go
@@ -15,14 +15,39 @@
package build
import (
+ "os"
"path/filepath"
"android/soong/ui/metrics"
)
-const bootstrapCmd = "bootstrap"
-const rbeLeastNProcs = 2500
-const rbeLeastNFiles = 16000
+const (
+ rbeLeastNProcs = 2500
+ rbeLeastNFiles = 16000
+
+ // prebuilt RBE binaries
+ bootstrapCmd = "bootstrap"
+
+ // RBE metrics proto buffer file
+ rbeMetricsPBFilename = "rbe_metrics.pb"
+)
+
+func rbeCommand(ctx Context, config Config, rbeCmd string) string {
+ var cmdPath string
+ if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok {
+ cmdPath = filepath.Join(rbeDir, rbeCmd)
+ } else if home, ok := config.Environment().Get("HOME"); ok {
+ cmdPath = filepath.Join(home, "rbe", rbeCmd)
+ } else {
+ ctx.Fatalf("rbe command path not found")
+ }
+
+ if _, err := os.Stat(cmdPath); err != nil && os.IsNotExist(err) {
+ ctx.Fatalf("rbe command %q not found", rbeCmd)
+ }
+
+ return cmdPath
+}
func startRBE(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSetupTool, "rbe_bootstrap")
@@ -35,18 +60,50 @@
ctx.Fatalf("max open files is insufficient: %d; want >= %d.\n", n, rbeLeastNFiles)
}
- var rbeBootstrap string
- if rbeDir, ok := config.Environment().Get("RBE_DIR"); ok {
- rbeBootstrap = filepath.Join(rbeDir, bootstrapCmd)
- } else if home, ok := config.Environment().Get("HOME"); ok {
- rbeBootstrap = filepath.Join(home, "rbe", bootstrapCmd)
- } else {
- ctx.Fatalln("rbe bootstrap not found")
- }
-
- cmd := Command(ctx, config, "boostrap", rbeBootstrap)
+ cmd := Command(ctx, config, "startRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd))
if output, err := cmd.CombinedOutput(); err != nil {
ctx.Fatalf("rbe bootstrap failed with: %v\n%s\n", err, output)
}
}
+
+func stopRBE(ctx Context, config Config) {
+ cmd := Command(ctx, config, "stopRBE bootstrap", rbeCommand(ctx, config, bootstrapCmd), "-shutdown")
+ if output, err := cmd.CombinedOutput(); err != nil {
+ ctx.Fatalf("rbe bootstrap with shutdown failed with: %v\n%s\n", err, output)
+ }
+}
+
+// DumpRBEMetrics creates a metrics protobuf file containing RBE related metrics.
+// The protobuf file is created if RBE is enabled and the proxy service has
+// started. The proxy service is shutdown in order to dump the RBE metrics to the
+// protobuf file.
+func DumpRBEMetrics(ctx Context, config Config, filename string) {
+ ctx.BeginTrace(metrics.RunShutdownTool, "dump_rbe_metrics")
+ defer ctx.EndTrace()
+
+ // Remove the previous metrics file in case there is a failure or RBE has been
+ // disable for this run.
+ os.Remove(filename)
+
+ // If RBE is not enabled then there are no metrics to generate.
+ // If RBE does not require to start, the RBE proxy maybe started
+ // manually for debugging purpose and can generate the metrics
+ // afterwards.
+ if !config.StartRBE() {
+ return
+ }
+
+ outputDir := config.RBEStatsOutputDir()
+ if outputDir == "" {
+ ctx.Fatal("RBE output dir variable not defined. Aborting metrics dumping.")
+ }
+ metricsFile := filepath.Join(outputDir, rbeMetricsPBFilename)
+
+ // Stop the proxy first in order to generate the RBE metrics protobuf file.
+ stopRBE(ctx, config)
+
+ if _, err := copyFile(metricsFile, filename); err != nil {
+ ctx.Fatalf("failed to copy %q to %q: %v\n", metricsFile, filename, err)
+ }
+}
diff --git a/ui/build/rbe_test.go b/ui/build/rbe_test.go
new file mode 100644
index 0000000..2c4995b
--- /dev/null
+++ b/ui/build/rbe_test.go
@@ -0,0 +1,142 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "android/soong/ui/logger"
+)
+
+func TestDumpRBEMetrics(t *testing.T) {
+ ctx := testContext()
+ tests := []struct {
+ description string
+ env []string
+ generated bool
+ }{{
+ description: "RBE disabled",
+ env: []string{
+ "NOSTART_RBE=true",
+ },
+ }, {
+ description: "rbe metrics generated",
+ env: []string{
+ "USE_RBE=true",
+ },
+ generated: true,
+ }}
+
+ for _, tt := range tests {
+ t.Run(tt.description, func(t *testing.T) {
+ tmpDir := t.TempDir()
+
+ rbeBootstrapCmd := filepath.Join(tmpDir, bootstrapCmd)
+ if err := ioutil.WriteFile(rbeBootstrapCmd, []byte(rbeBootstrapProgram), 0755); err != nil {
+ t.Fatalf("failed to create a fake bootstrap command file %s: %v", rbeBootstrapCmd, err)
+ }
+
+ env := Environment(tt.env)
+ env.Set("OUT_DIR", tmpDir)
+ env.Set("RBE_DIR", tmpDir)
+ env.Set("RBE_output_dir", t.TempDir())
+ config := Config{&configImpl{
+ environ: &env,
+ }}
+
+ rbeMetricsFilename := filepath.Join(tmpDir, rbeMetricsPBFilename)
+ DumpRBEMetrics(ctx, config, rbeMetricsFilename)
+
+ // Validate that the rbe metrics file exists if RBE is enabled.
+ if _, err := os.Stat(rbeMetricsFilename); err == nil {
+ if !tt.generated {
+ t.Errorf("got true, want false for rbe metrics file %s to exist.", rbeMetricsFilename)
+ }
+ } else if os.IsNotExist(err) {
+ if tt.generated {
+ t.Errorf("got false, want true for rbe metrics file %s to exist.", rbeMetricsFilename)
+ }
+ } else {
+ t.Errorf("unknown error found on checking %s exists: %v", rbeMetricsFilename, err)
+ }
+ })
+ }
+}
+
+func TestDumpRBEMetricsErrors(t *testing.T) {
+ ctx := testContext()
+ tests := []struct {
+ description string
+ rbeOutputDirDefined bool
+ bootstrapProgram string
+ expectedErr string
+ }{{
+ description: "output_dir not defined",
+ bootstrapProgram: rbeBootstrapProgram,
+ expectedErr: "RBE output dir variable not defined",
+ }, {
+ description: "stopRBE failed",
+ rbeOutputDirDefined: true,
+ bootstrapProgram: "#!/bin/bash\nexit 1",
+ expectedErr: "shutdown failed",
+ }, {
+ description: "failed to copy metrics file",
+ rbeOutputDirDefined: true,
+ bootstrapProgram: "#!/bin/bash",
+ expectedErr: "failed to copy",
+ }}
+
+ for _, tt := range tests {
+ t.Run(tt.description, func(t *testing.T) {
+ defer logger.Recover(func(err error) {
+ got := err.Error()
+ if !strings.Contains(got, tt.expectedErr) {
+ t.Errorf("got %q, want %q to be contained in error", got, tt.expectedErr)
+ }
+ })
+
+ tmpDir := t.TempDir()
+
+ rbeBootstrapCmd := filepath.Join(tmpDir, bootstrapCmd)
+ if err := ioutil.WriteFile(rbeBootstrapCmd, []byte(tt.bootstrapProgram), 0755); err != nil {
+ t.Fatalf("failed to create a fake bootstrap command file %s: %v", rbeBootstrapCmd, err)
+ }
+
+ env := &Environment{}
+ env.Set("USE_RBE", "true")
+ env.Set("OUT_DIR", tmpDir)
+ env.Set("RBE_DIR", tmpDir)
+
+ if tt.rbeOutputDirDefined {
+ env.Set("RBE_output_dir", t.TempDir())
+ }
+
+ config := Config{&configImpl{
+ environ: env,
+ }}
+
+ rbeMetricsFilename := filepath.Join(tmpDir, rbeMetricsPBFilename)
+ DumpRBEMetrics(ctx, config, rbeMetricsFilename)
+ t.Errorf("got nil, expecting %q as a failure", tt.expectedErr)
+ })
+ }
+}
+
+var rbeBootstrapProgram = fmt.Sprintf("#!/bin/bash\necho 1 > $RBE_output_dir/%s", rbeMetricsPBFilename)
diff --git a/ui/build/test_build.go b/ui/build/test_build.go
index 4ff9483..83b3807 100644
--- a/ui/build/test_build.go
+++ b/ui/build/test_build.go
@@ -66,6 +66,8 @@
outDir := config.OutDir()
bootstrapDir := filepath.Join(outDir, "soong", ".bootstrap")
miniBootstrapDir := filepath.Join(outDir, "soong", ".minibootstrap")
+ modulePathsDir := filepath.Join(outDir, ".module_paths")
+ variablesFilePath := filepath.Join(outDir, "soong", "soong.variables")
danglingRules := make(map[string]bool)
@@ -76,7 +78,10 @@
// Leaf node is not in the out directory.
continue
}
- if strings.HasPrefix(line, bootstrapDir) || strings.HasPrefix(line, miniBootstrapDir) {
+ if strings.HasPrefix(line, bootstrapDir) ||
+ strings.HasPrefix(line, miniBootstrapDir) ||
+ strings.HasPrefix(line, modulePathsDir) ||
+ line == variablesFilePath {
// Leaf node is in one of Soong's bootstrap directories, which do not have
// full build rules in the primary build.ninja file.
continue
diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go
index 3e76d37..e055b76 100644
--- a/ui/metrics/metrics.go
+++ b/ui/metrics/metrics.go
@@ -25,12 +25,13 @@
)
const (
- RunSetupTool = "setup"
- RunKati = "kati"
- RunSoong = "soong"
- PrimaryNinja = "ninja"
- TestRun = "test"
- Total = "total"
+ PrimaryNinja = "ninja"
+ RunKati = "kati"
+ RunSetupTool = "setup"
+ RunShutdownTool = "shutdown"
+ RunSoong = "soong"
+ TestRun = "test"
+ Total = "total"
)
type Metrics struct {