Fix staging dir creation for adevice
Adevice does a `ninja -t inputs -i droid` and filters the results
for out/target/product/ to find files in the staging directories.
We were just copying the staging directories without creating ninja
rules before, so that failed. Actually create the ninja rules.
The rules are created in the android_device module instead of the
filesystem so that if you have multiple filesystems, they don't create
conflicting installation rules.
We should provide a way for tools like adevice to query the build system
for installed files, no matter where they are in the out/ dir, and
use that to replace this logic.
This cl produces the following diff in out/target/product compared
to soong-only builds before this change:
https://paste.googleplex.com/4624734193713152
It's mostly empty directories, which I don't think matter for adevice.
Bug: 393464638
Bug: 392957226
Test: `m installclean && m && launch_cvd && adevice update` says the device is up-to-date.
Change-Id: Ifb52aaf5bc3edd2e3900546bf9df73a52d1022c0
diff --git a/filesystem/aconfig_files.go b/filesystem/aconfig_files.go
index 9a3ca54..6d03402 100644
--- a/filesystem/aconfig_files.go
+++ b/filesystem/aconfig_files.go
@@ -34,7 +34,13 @@
var importAconfigDependencyTag = interPartitionDepTag{}
-func (f *filesystem) buildAconfigFlagsFiles(ctx android.ModuleContext, builder *android.RuleBuilder, specs map[string]android.PackagingSpec, dir android.OutputPath) {
+func (f *filesystem) buildAconfigFlagsFiles(
+ ctx android.ModuleContext,
+ builder *android.RuleBuilder,
+ specs map[string]android.PackagingSpec,
+ dir android.OutputPath,
+ fullInstallPaths *[]FullInstallPathInfo,
+) {
var caches []android.Path
for _, ps := range specs {
caches = append(caches, ps.GetAconfigPaths()...)
@@ -70,6 +76,10 @@
for _, cache := range caches {
cmd.FlagWithInput("--cache ", cache)
}
+ *fullInstallPaths = append(*fullInstallPaths, FullInstallPathInfo{
+ FullInstallPath: android.PathForModuleInPartitionInstall(ctx, f.PartitionType(), "etc/aconfig_flags.pb"),
+ SourcePath: installAconfigFlagsPath,
+ })
f.appendToEntry(ctx, installAconfigFlagsPath)
installAconfigStorageDir := dir.Join(ctx, "etc", "aconfig")
@@ -90,6 +100,10 @@
FlagWithOutput("--out ", outputPath).
FlagWithArg("--cache ", installAconfigFlagsPath.String()).
FlagWithArg("--version ", strconv.Itoa(storageFilesVersion))
+ *fullInstallPaths = append(*fullInstallPaths, FullInstallPathInfo{
+ FullInstallPath: android.PathForModuleInPartitionInstall(ctx, f.PartitionType(), "etc/aconfig", fileName),
+ SourcePath: outputPath,
+ })
f.appendToEntry(ctx, outputPath)
}
diff --git a/filesystem/android_device.go b/filesystem/android_device.go
index 3e2f61f..6c04828 100644
--- a/filesystem/android_device.go
+++ b/filesystem/android_device.go
@@ -210,7 +210,7 @@
// https://cs.android.com/android/platform/superproject/main/+/main:build/make/core/main.mk;l=1396;drc=6595459cdd8164a6008335f6372c9f97b9094060
ctx.Phony("droidcore-unbundled", allImagesStamp)
- validations = append(validations, a.copyFilesToProductOutForSoongOnly(ctx))
+ deps = append(deps, a.copyFilesToProductOutForSoongOnly(ctx))
}
ctx.Build(pctx, android.BuildParams{
diff --git a/filesystem/android_device_product_out.go b/filesystem/android_device_product_out.go
index a6177e3..405d710 100644
--- a/filesystem/android_device_product_out.go
+++ b/filesystem/android_device_product_out.go
@@ -16,7 +16,6 @@
import (
"android/soong/android"
- "fmt"
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
@@ -36,18 +35,6 @@
func (a *androidDevice) copyFilesToProductOutForSoongOnly(ctx android.ModuleContext) android.Path {
filesystemInfos := a.getFsInfos(ctx)
- // The current logic to copy the staging directories to PRODUCT_OUT isn't very sound.
- // We only track dependencies on the image file, so if the image file wasn't changed, the
- // staging directory won't be re-copied. If you do an installclean, it would remove the copied
- // staging directories but not affect the intermediates path image file, so the next build
- // wouldn't re-copy them. As a hack, create a presence detector that would be deleted on
- // an installclean to use as a dep for the staging dir copies.
- productOutPresenceDetector := android.PathForModuleInPartitionInstall(ctx, "", "product_out_presence_detector.txt")
- ctx.Build(pctx, android.BuildParams{
- Rule: android.Touch,
- Output: productOutPresenceDetector,
- })
-
var deps android.Paths
for _, partition := range android.SortedKeys(filesystemInfos) {
@@ -58,30 +45,53 @@
Input: info.Output,
Output: imgInstallPath,
})
- dirStamp := android.PathForModuleOut(ctx, partition+"_staging_dir_copy_stamp.txt")
- dirInstallPath := android.PathForModuleInPartitionInstall(ctx, "", partition)
- ctx.Build(pctx, android.BuildParams{
- Rule: copyStagingDirRule,
- Output: dirStamp,
- Implicits: []android.Path{
- info.Output,
- productOutPresenceDetector,
- },
- Args: map[string]string{
- "dir": info.RebasedDir.String(),
- "dest": dirInstallPath.String(),
- },
- })
// Make it so doing `m <moduleName>` or `m <partitionType>image` will copy the files to
// PRODUCT_OUT
- ctx.Phony(info.ModuleName, dirStamp, imgInstallPath)
if partition == "system_ext" {
partition = "systemext"
}
- ctx.Phony(fmt.Sprintf("%simage", partition), dirStamp, imgInstallPath)
+ partition = partition + "imgage"
+ ctx.Phony(info.ModuleName, imgInstallPath)
+ ctx.Phony(partition, imgInstallPath)
+ for _, fip := range info.FullInstallPaths {
+ // TODO: Directories. But maybe they're not necessary? Adevice doesn't care
+ // about empty directories, still need to check if adb sync does.
+ if !fip.IsDir {
+ if !fip.RequiresFullInstall {
+ // Some modules set requires_full_install: false, which causes their staging
+ // directory file to not be installed. This is usually because the file appears
+ // in both PRODUCT_COPY_FILES and a soong module for the handwritten soong system
+ // image. In this case, that module's installed files would conflict with the
+ // PRODUCT_COPY_FILES. However, in soong-only builds, we don't automatically
+ // create rules for PRODUCT_COPY_FILES unless they're needed in the partition.
+ // So in that case, nothing is creating the installed path. Create them now
+ // if that's the case.
+ if fip.SymlinkTarget == "" {
+ ctx.Build(pctx, android.BuildParams{
+ Rule: android.Cp,
+ Input: fip.SourcePath,
+ Output: fip.FullInstallPath,
+ })
+ } else {
+ ctx.Build(pctx, android.BuildParams{
+ Rule: android.SymlinkWithBash,
+ Output: fip.FullInstallPath,
+ Args: map[string]string{
+ "fromPath": fip.SymlinkTarget,
+ },
+ })
+ }
+ }
+ ctx.Phony(info.ModuleName, fip.FullInstallPath)
+ ctx.Phony(partition, fip.FullInstallPath)
+ deps = append(deps, fip.FullInstallPath)
+ ctx.Phony("sync_"+partition, fip.FullInstallPath)
+ ctx.Phony("sync", fip.FullInstallPath)
+ }
+ }
- deps = append(deps, imgInstallPath, dirStamp)
+ deps = append(deps, imgInstallPath)
}
// List all individual files to be copied to PRODUCT_OUT here
diff --git a/filesystem/filesystem.go b/filesystem/filesystem.go
index 815113e..e3f3ce8 100644
--- a/filesystem/filesystem.go
+++ b/filesystem/filesystem.go
@@ -79,7 +79,7 @@
}
type filesystemBuilder interface {
- BuildLinkerConfigFile(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath)
+ BuildLinkerConfigFile(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath, fullInstallPaths *[]FullInstallPathInfo)
// Function that filters PackagingSpec in PackagingBase.GatherPackagingSpecs()
FilterPackagingSpec(spec android.PackagingSpec) bool
// Function that modifies PackagingSpec in PackagingBase.GatherPackagingSpecs() to customize.
@@ -389,6 +389,34 @@
BuildImagePropFileDeps android.Paths
// Packaging specs to be installed on the system_other image, for the initial boot's dexpreopt.
SpecsForSystemOther map[string]android.PackagingSpec
+
+ FullInstallPaths []FullInstallPathInfo
+}
+
+// FullInstallPathInfo contains information about the "full install" paths of all the files
+// inside this partition. The full install paths are the files installed in
+// out/target/product/<device>/<partition>. This is essentially legacy behavior, maintained for
+// tools like adb sync and adevice, but we should update them to query the build system for the
+// installed files no matter where they are.
+type FullInstallPathInfo struct {
+ // RequiresFullInstall tells us if the origional module did the install to FullInstallPath
+ // already. If it's false, the android_device module needs to emit the install rule.
+ RequiresFullInstall bool
+ // The "full install" paths for the files in this filesystem. This is the paths in the
+ // out/target/product/<device>/<partition> folder. They're not used by this filesystem,
+ // but can be depended on by the top-level android_device module to cause the staging
+ // directories to be built.
+ FullInstallPath android.InstallPath
+
+ // The file that's copied to FullInstallPath. May be nil if SymlinkTarget is set or IsDir is
+ // true.
+ SourcePath android.Path
+
+ // The target of the symlink, if this file is a symlink.
+ SymlinkTarget string
+
+ // If this file is a directory. Only used for empty directories, which are mostly mount points.
+ IsDir bool
}
var FilesystemProvider = blueprint.NewProvider[FilesystemInfo]()
@@ -485,13 +513,23 @@
// Wipe the root dir to get rid of leftover files from prior builds
builder.Command().Textf("rm -rf %s && mkdir -p %s", rootDir, rootDir)
specs := f.gatherFilteredPackagingSpecs(ctx)
- f.entries = f.copyPackagingSpecs(ctx, builder, specs, rootDir, rebasedDir)
- f.buildNonDepsFiles(ctx, builder, rootDir)
- f.buildFsverityMetadataFiles(ctx, builder, specs, rootDir, rebasedDir)
- f.buildEventLogtagsFile(ctx, builder, rebasedDir)
- f.buildAconfigFlagsFiles(ctx, builder, specs, rebasedDir)
- f.filesystemBuilder.BuildLinkerConfigFile(ctx, builder, rebasedDir)
+ var fullInstallPaths []FullInstallPathInfo
+ for _, spec := range specs {
+ fullInstallPaths = append(fullInstallPaths, FullInstallPathInfo{
+ FullInstallPath: spec.FullInstallPath(),
+ RequiresFullInstall: spec.RequiresFullInstall(),
+ SourcePath: spec.SrcPath(),
+ SymlinkTarget: spec.ToGob().SymlinkTarget,
+ })
+ }
+
+ f.entries = f.copyPackagingSpecs(ctx, builder, specs, rootDir, rebasedDir)
+ f.buildNonDepsFiles(ctx, builder, rootDir, rebasedDir, &fullInstallPaths)
+ f.buildFsverityMetadataFiles(ctx, builder, specs, rootDir, rebasedDir, &fullInstallPaths)
+ f.buildEventLogtagsFile(ctx, builder, rebasedDir, &fullInstallPaths)
+ f.buildAconfigFlagsFiles(ctx, builder, specs, rebasedDir, &fullInstallPaths)
+ f.filesystemBuilder.BuildLinkerConfigFile(ctx, builder, rebasedDir, &fullInstallPaths)
var mapFile android.Path
var outputHermetic android.Path
@@ -531,6 +569,7 @@
BuildImagePropFile: buildImagePropFile,
BuildImagePropFileDeps: buildImagePropFileDeps,
SpecsForSystemOther: f.systemOtherFiles(ctx),
+ FullInstallPaths: fullInstallPaths,
}
android.SetProvider(ctx, FilesystemProvider, fsInfo)
@@ -645,11 +684,36 @@
// Copy extra files/dirs that are not from the `deps` property to `rootDir`, checking for conflicts with files
// already in `rootDir`.
-func (f *filesystem) buildNonDepsFiles(ctx android.ModuleContext, builder *android.RuleBuilder, rootDir android.OutputPath) {
+func (f *filesystem) buildNonDepsFiles(
+ ctx android.ModuleContext,
+ builder *android.RuleBuilder,
+ rootDir android.OutputPath,
+ rebasedDir android.OutputPath,
+ fullInstallPaths *[]FullInstallPathInfo,
+) {
+ rebasedPrefix, err := filepath.Rel(rootDir.String(), rebasedDir.String())
+ if err != nil || strings.HasPrefix(rebasedPrefix, "../") {
+ panic("rebasedDir could not be made relative to rootDir")
+ }
+ if !strings.HasSuffix(rebasedPrefix, "/") {
+ rebasedPrefix += "/"
+ }
+ if rebasedPrefix == "./" {
+ rebasedPrefix = ""
+ }
+
// create dirs and symlinks
for _, dir := range f.properties.Dirs.GetOrDefault(ctx, nil) {
// OutputPath.Join verifies dir
builder.Command().Text("mkdir -p").Text(rootDir.Join(ctx, dir).String())
+ // Only add the fullInstallPath logic for files in the rebased dir. The root dir
+ // is harder to install to.
+ if strings.HasPrefix(dir, rebasedPrefix) {
+ *fullInstallPaths = append(*fullInstallPaths, FullInstallPathInfo{
+ FullInstallPath: android.PathForModuleInPartitionInstall(ctx, f.PartitionType(), strings.TrimPrefix(dir, rebasedPrefix)),
+ IsDir: true,
+ })
+ }
}
for _, symlink := range f.properties.Symlinks {
@@ -672,6 +736,14 @@
builder.Command().Text("mkdir -p").Text(filepath.Dir(dst.String()))
builder.Command().Text("ln -sf").Text(proptools.ShellEscape(target)).Text(dst.String())
f.appendToEntry(ctx, dst)
+ // Only add the fullInstallPath logic for files in the rebased dir. The root dir
+ // is harder to install to.
+ if strings.HasPrefix(name, rebasedPrefix) {
+ *fullInstallPaths = append(*fullInstallPaths, FullInstallPathInfo{
+ FullInstallPath: android.PathForModuleInPartitionInstall(ctx, f.PartitionType(), strings.TrimPrefix(name, rebasedPrefix)),
+ SymlinkTarget: target,
+ })
+ }
}
// https://cs.android.com/android/platform/superproject/main/+/main:build/make/core/Makefile;l=2835;drc=b186569ef00ff2f2a1fab28aedc75ebc32bcd67b
@@ -1019,7 +1091,12 @@
"recovery",
}
-func (f *filesystem) buildEventLogtagsFile(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath) {
+func (f *filesystem) buildEventLogtagsFile(
+ ctx android.ModuleContext,
+ builder *android.RuleBuilder,
+ rebasedDir android.OutputPath,
+ fullInstallPaths *[]FullInstallPathInfo,
+) {
if !proptools.Bool(f.properties.Build_logtags) {
return
}
@@ -1029,10 +1106,20 @@
builder.Command().Text("mkdir").Flag("-p").Text(etcPath.String())
builder.Command().Text("cp").Input(android.MergedLogtagsPath(ctx)).Text(eventLogtagsPath.String())
+ *fullInstallPaths = append(*fullInstallPaths, FullInstallPathInfo{
+ FullInstallPath: android.PathForModuleInPartitionInstall(ctx, f.PartitionType(), "etc", "event-log-tags"),
+ SourcePath: android.MergedLogtagsPath(ctx),
+ })
+
f.appendToEntry(ctx, eventLogtagsPath)
}
-func (f *filesystem) BuildLinkerConfigFile(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath) {
+func (f *filesystem) BuildLinkerConfigFile(
+ ctx android.ModuleContext,
+ builder *android.RuleBuilder,
+ rebasedDir android.OutputPath,
+ fullInstallPaths *[]FullInstallPathInfo,
+) {
if !proptools.Bool(f.properties.Linker_config.Gen_linker_config) {
return
}
@@ -1043,6 +1130,11 @@
output := rebasedDir.Join(ctx, "etc", "linker.config.pb")
builder.Command().Text("cp").Input(intermediateOutput).Output(output)
+ *fullInstallPaths = append(*fullInstallPaths, FullInstallPathInfo{
+ FullInstallPath: android.PathForModuleInPartitionInstall(ctx, f.PartitionType(), "etc", "linker.config.pb"),
+ SourcePath: intermediateOutput,
+ })
+
f.appendToEntry(ctx, output)
}
diff --git a/filesystem/fsverity_metadata.go b/filesystem/fsverity_metadata.go
index c3f1936..a3a2086 100644
--- a/filesystem/fsverity_metadata.go
+++ b/filesystem/fsverity_metadata.go
@@ -44,7 +44,14 @@
android.WriteFileRuleVerbatim(ctx, outputPath, buf.String())
}
-func (f *filesystem) buildFsverityMetadataFiles(ctx android.ModuleContext, builder *android.RuleBuilder, specs map[string]android.PackagingSpec, rootDir android.OutputPath, rebasedDir android.OutputPath) {
+func (f *filesystem) buildFsverityMetadataFiles(
+ ctx android.ModuleContext,
+ builder *android.RuleBuilder,
+ specs map[string]android.PackagingSpec,
+ rootDir android.OutputPath,
+ rebasedDir android.OutputPath,
+ fullInstallPaths *[]FullInstallPathInfo,
+) {
match := func(path string) bool {
for _, pattern := range f.properties.Fsverity.Inputs.GetOrDefault(ctx, nil) {
if matched, err := filepath.Match(pattern, path); matched {
@@ -82,9 +89,13 @@
FlagWithInput("--fsverity-path ", fsverityPath).
FlagWithArg("--signature ", "none").
FlagWithArg("--hash-alg ", "sha256").
- FlagWithArg("--output ", destPath.String()).
+ FlagWithOutput("--output ", destPath).
Text(srcPath.String())
f.appendToEntry(ctx, destPath)
+ *fullInstallPaths = append(*fullInstallPaths, FullInstallPathInfo{
+ SourcePath: destPath,
+ FullInstallPath: android.PathForModuleInPartitionInstall(ctx, f.PartitionType(), spec.RelPathInPackage()+".fsv_meta"),
+ })
}
fsVerityBaseDir := rootDir.String()
@@ -148,6 +159,10 @@
FlagWithArg("--version-name ", ctx.Config().AppsDefaultVersionName()).
FlagWithInput("--manifest ", manifestTemplatePath).
Text(" --rename-manifest-package com.android.security.fsverity_metadata." + f.partitionName())
+ *fullInstallPaths = append(*fullInstallPaths, FullInstallPathInfo{
+ SourcePath: apkPath,
+ FullInstallPath: android.PathForModuleInPartitionInstall(ctx, f.PartitionType(), fmt.Sprintf("etc/security/fsverity/BuildManifest%s.apk", apkNameSuffix)),
+ })
f.appendToEntry(ctx, apkPath)
@@ -160,6 +175,10 @@
FlagWithInput("--cert ", pemPath).
FlagWithInput("--key ", keyPath).
ImplicitOutput(idsigPath)
+ *fullInstallPaths = append(*fullInstallPaths, FullInstallPathInfo{
+ SourcePath: idsigPath,
+ FullInstallPath: android.PathForModuleInPartitionInstall(ctx, f.PartitionType(), fmt.Sprintf("etc/security/fsverity/BuildManifest%s.apk.idsig", apkNameSuffix)),
+ })
f.appendToEntry(ctx, idsigPath)
}
diff --git a/filesystem/system_image.go b/filesystem/system_image.go
index 874d20d..cc9093f 100644
--- a/filesystem/system_image.go
+++ b/filesystem/system_image.go
@@ -44,7 +44,12 @@
return s.filesystem.properties
}
-func (s *systemImage) BuildLinkerConfigFile(ctx android.ModuleContext, builder *android.RuleBuilder, rebasedDir android.OutputPath) {
+func (s *systemImage) BuildLinkerConfigFile(
+ ctx android.ModuleContext,
+ builder *android.RuleBuilder,
+ rebasedDir android.OutputPath,
+ fullInstallPaths *[]FullInstallPathInfo,
+) {
if !proptools.Bool(s.filesystem.properties.Linker_config.Gen_linker_config) {
return
}
@@ -55,6 +60,11 @@
intermediateOutput := android.PathForModuleOut(ctx, "linker.config.pb")
linkerconfig.BuildLinkerConfig(ctx, android.PathsForModuleSrc(ctx, s.filesystem.properties.Linker_config.Linker_config_srcs), provideModules, requireModules, intermediateOutput)
builder.Command().Text("cp").Input(intermediateOutput).Output(output)
+
+ *fullInstallPaths = append(*fullInstallPaths, FullInstallPathInfo{
+ FullInstallPath: android.PathForModuleInPartitionInstall(ctx, s.PartitionType(), "etc", "linker.config.pb"),
+ SourcePath: intermediateOutput,
+ })
} else {
// TODO: This branch is the logic that make uses for the linker config file, which is
// different than linkerconfig.BuildLinkerConfig used above. Keeping both branches for now
@@ -87,6 +97,11 @@
Implicit(llndkMovedToApexLibraries)
// TODO: Make also supports adding an extra append command with PRODUCT_EXTRA_STUB_LIBRARIES,
// but that variable appears to have no usages.
+
+ *fullInstallPaths = append(*fullInstallPaths, FullInstallPathInfo{
+ FullInstallPath: android.PathForModuleInPartitionInstall(ctx, s.PartitionType(), "etc", "linker.config.pb"),
+ SourcePath: output,
+ })
}
s.appendToEntry(ctx, output)