Support disting in soong-only builds
Disting was normally handled by the packaging kati step, which was
not run in soong-only builds. Disting needs to be done in a separate
ninja file, so that if you toggle dist on/off you only need to
regenerate a small ninja file instead of the main one.
This change makes it so that the kati packaging step is also run in
soong-only builds. (which makes them not technically soong-only, but
the packaging step is very fast so it's fine)
Phonies in soong were normally exported to the main make invocation
and then make would emit the ninja rules for them. In soong-only builds
soong would emit the ninja phone rules directly. This is problematic
when supporting disting in soong, because some of the phony rules have
the same name as dist targets, causing ninja to complain about duplicate
rules. To resolve this, make soong export its phonies to the packaging
step, which will dedup them with the dist phonies.
Bug: 388312357
Test: m --soong-only dist
Change-Id: If6de8c28274fe96dac674f449b9bc0b488540655
diff --git a/android/androidmk.go b/android/androidmk.go
index 590cce3..9e721e1 100644
--- a/android/androidmk.go
+++ b/android/androidmk.go
@@ -701,11 +701,6 @@
type androidMkSingleton struct{}
func (c *androidMkSingleton) GenerateBuildActions(ctx SingletonContext) {
- // Skip if Soong wasn't invoked from Make.
- if !ctx.Config().KatiEnabled() {
- return
- }
-
var androidMkModulesList []blueprint.Module
ctx.VisitAllModulesBlueprint(func(module blueprint.Module) {
@@ -718,6 +713,12 @@
return ctx.ModuleName(androidMkModulesList[i]) < ctx.ModuleName(androidMkModulesList[j])
})
+ // If running in soong-only mode, do a different, more limited version of this singleton
+ if !ctx.Config().KatiEnabled() {
+ c.soongOnlyBuildActions(ctx, androidMkModulesList)
+ return
+ }
+
transMk := PathForOutput(ctx, "Android"+String(ctx.Config().productVariables.Make_suffix)+".mk")
if ctx.Failed() {
return
@@ -736,6 +737,97 @@
})
}
+// In soong-only mode, we don't do most of the androidmk stuff. But disted files are still largely
+// defined through the androidmk mechanisms, so this function is an alternate implementation of
+// the androidmk singleton that just focuses on getting the dist contributions
+func (c *androidMkSingleton) soongOnlyBuildActions(ctx SingletonContext, mods []blueprint.Module) {
+ allDistContributions := getDistContributionsFromMods(ctx, mods)
+
+ distMkFile := absolutePath(filepath.Join(ctx.Config().katiPackageMkDir(), "dist.mk"))
+
+ var goalOutputPairs []string
+ var buf strings.Builder
+ buf.WriteString("DIST_SRC_DST_PAIRS := $(sort")
+ for _, contributions := range allDistContributions {
+ for _, copiesForGoal := range contributions.copiesForGoals {
+ goals := strings.Fields(copiesForGoal.goals)
+ for _, copy := range copiesForGoal.copies {
+ for _, goal := range goals {
+ goalOutputPairs = append(goalOutputPairs, fmt.Sprintf("%s:%s", goal, copy.dest))
+ }
+ buf.WriteString(fmt.Sprintf(" %s:%s", copy.from.String(), copy.dest))
+ }
+ }
+ }
+ buf.WriteString(")\nDIST_GOAL_OUTPUT_PAIRS := $(sort ")
+ buf.WriteString(strings.Join(goalOutputPairs, " "))
+ buf.WriteString(")\n")
+
+ writeValueIfChanged(ctx, distMkFile, buf.String())
+}
+
+func writeValueIfChanged(ctx SingletonContext, path string, value string) {
+ if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
+ ctx.Errorf("%s\n", err)
+ return
+ }
+ previousValue := ""
+ rawPreviousValue, err := os.ReadFile(path)
+ if err == nil {
+ previousValue = string(rawPreviousValue)
+ }
+
+ if previousValue != value {
+ if err = os.WriteFile(path, []byte(value), 0666); err != nil {
+ ctx.Errorf("Failed to write: %v", err)
+ }
+ }
+}
+
+func getDistContributionsFromMods(ctx fillInEntriesContext, mods []blueprint.Module) []distContributions {
+ var allDistContributions []distContributions
+ for _, mod := range mods {
+ if amod, ok := mod.(Module); ok && shouldSkipAndroidMkProcessing(ctx, amod.base()) {
+ continue
+ }
+ if info, ok := OtherModuleProvider(ctx, mod, AndroidMkInfoProvider); ok {
+ if contribution := info.PrimaryInfo.getDistContributions(ctx, mod); contribution != nil {
+ allDistContributions = append(allDistContributions, *contribution)
+ }
+ for _, ei := range info.ExtraInfo {
+ if contribution := ei.getDistContributions(ctx, mod); contribution != nil {
+ allDistContributions = append(allDistContributions, *contribution)
+ }
+ }
+ } else {
+ switch x := mod.(type) {
+ case AndroidMkDataProvider:
+ data := x.AndroidMk()
+
+ if data.Include == "" {
+ data.Include = "$(BUILD_PREBUILT)"
+ }
+
+ data.fillInData(ctx, mod)
+ if contribution := data.Entries.getDistContributions(mod); contribution != nil {
+ allDistContributions = append(allDistContributions, *contribution)
+ }
+ case AndroidMkEntriesProvider:
+ entriesList := x.AndroidMkEntries()
+ for _, entries := range entriesList {
+ entries.fillInEntries(ctx, mod)
+ if contribution := entries.getDistContributions(mod); contribution != nil {
+ allDistContributions = append(allDistContributions, *contribution)
+ }
+ }
+ default:
+ // Not exported to make so no make variables to set.
+ }
+ }
+ }
+ return allDistContributions
+}
+
func translateAndroidMk(ctx SingletonContext, absMkFile string, moduleInfoJSONPath WritablePath, mods []blueprint.Module) error {
buf := &bytes.Buffer{}
diff --git a/android/config.go b/android/config.go
index e9cb2cd..70c5e4f 100644
--- a/android/config.go
+++ b/android/config.go
@@ -83,6 +83,7 @@
OutDir string
SoongOutDir string
SoongVariables string
+ KatiSuffix string
ModuleGraphFile string
ModuleActionsFile string
@@ -349,6 +350,7 @@
// Changes behavior based on whether Kati runs after soong_build, or if soong_build
// runs standalone.
katiEnabled bool
+ katiSuffix string
captureBuild bool // true for tests, saves build parameters for each module
ignoreEnvironment bool // true for tests, returns empty from all Getenv calls
@@ -620,6 +622,7 @@
outDir: cmdArgs.OutDir,
soongOutDir: cmdArgs.SoongOutDir,
runGoTests: cmdArgs.RunGoTests,
+ katiSuffix: cmdArgs.KatiSuffix,
multilibConflicts: make(map[ArchType]bool),
moduleListFile: cmdArgs.ModuleListFile,
@@ -1512,6 +1515,10 @@
return c.productVariables.GetBuildFlagBool("RELEASE_BOARD_API_LEVEL_FROZEN")
}
+func (c *config) katiPackageMkDir() string {
+ return filepath.Join(c.soongOutDir, "kati_packaging"+c.katiSuffix)
+}
+
func (c *deviceConfig) Arches() []Arch {
var arches []Arch
for _, target := range c.config.Targets[Android] {
diff --git a/android/module.go b/android/module.go
index b8f2cae..2f505aa 100644
--- a/android/module.go
+++ b/android/module.go
@@ -1679,14 +1679,13 @@
}
})
- var deps Paths
-
var namespacePrefix string
nameSpace := ctx.Namespace().Path
if nameSpace != "." {
namespacePrefix = strings.ReplaceAll(nameSpace, "/", ".") + "-"
}
+ var deps Paths
var info FinalModuleBuildTargetsInfo
if len(allInstalledFiles) > 0 {
@@ -1853,9 +1852,9 @@
var SourceFilesInfoKey = blueprint.NewProvider[SourceFilesInfo]()
+// FinalModuleBuildTargetsInfo is used by buildTargetSingleton to create checkbuild and
+// per-directory build targets. Only set on the final variant of each module
type FinalModuleBuildTargetsInfo struct {
- // Used by buildTargetSingleton to create checkbuild and per-directory build targets
- // Only set on the final variant of each module
InstallTarget WritablePath
CheckbuildTarget WritablePath
BlueprintDir string
diff --git a/android/phony.go b/android/phony.go
index f8db88d..7bdd9d3 100644
--- a/android/phony.go
+++ b/android/phony.go
@@ -15,6 +15,7 @@
package android
import (
+ "strings"
"sync"
"github.com/google/blueprint"
@@ -68,13 +69,25 @@
}
if !ctx.Config().KatiEnabled() {
+ // In soong-only builds, the phonies can conflict with dist targets that will
+ // be generated in the packaging step. Instead of emitting a blueprint/ninja phony directly,
+ // create a makefile that defines the phonies that will be included in the packaging step.
+ // Make will dedup the phonies there.
+ var buildPhonyFileContents strings.Builder
for _, phony := range p.phonyList {
- ctx.Build(pctx, BuildParams{
- Rule: blueprint.Phony,
- Outputs: []WritablePath{PathForPhony(ctx, phony)},
- Implicits: p.phonyMap[phony],
- })
+ buildPhonyFileContents.WriteString(".PHONY: ")
+ buildPhonyFileContents.WriteString(phony)
+ buildPhonyFileContents.WriteString("\n")
+ buildPhonyFileContents.WriteString(phony)
+ buildPhonyFileContents.WriteString(":")
+ for _, dep := range p.phonyMap[phony] {
+ buildPhonyFileContents.WriteString(" ")
+ buildPhonyFileContents.WriteString(dep.String())
+ }
+ buildPhonyFileContents.WriteString("\n")
}
+ buildPhonyFile := PathForOutput(ctx, "soong_phony_targets.mk")
+ writeValueIfChanged(ctx, absolutePath(buildPhonyFile.String()), buildPhonyFileContents.String())
}
}
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index 6642023..cd4e9bd 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -64,6 +64,7 @@
flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
flag.StringVar(&cmdlineArgs.OutDir, "out", "", "the ninja builddir directory")
flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")
+ flag.StringVar(&cmdlineArgs.KatiSuffix, "kati_suffix", "", "the suffix for kati and ninja files, so that different configurations don't clobber each other")
// Debug flags
flag.StringVar(&delveListen, "delve_listen", "", "Delve port to listen on for debugging")
diff --git a/ui/build/build.go b/ui/build/build.go
index 1dc6dbd..26f5969 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -107,8 +107,11 @@
{{end -}}
pool highmem_pool
depth = {{.HighmemParallel}}
-{{if and (not .SkipKatiNinja) .HasKatiSuffix}}subninja {{.KatiBuildNinjaFile}}
+{{if and (not .SkipKatiNinja) .HasKatiSuffix}}
+subninja {{.KatiBuildNinjaFile}}
subninja {{.KatiPackageNinjaFile}}
+{{else}}
+subninja {{.KatiSoongOnlyPackageNinjaFile}}
{{end -}}
subninja {{.SoongNinjaFile}}
`))
@@ -346,25 +349,31 @@
return
}
+ // Still generate the kati suffix in soong-only builds because soong-only still uses kati for
+ // the packaging step. Also, the kati suffix is used for the combined ninja file.
+ genKatiSuffix(ctx, config)
+
if what&RunSoong != 0 {
runSoong(ctx, config)
}
if what&RunKati != 0 {
- genKatiSuffix(ctx, config)
runKatiCleanSpec(ctx, config)
runKatiBuild(ctx, config)
- runKatiPackage(ctx, config)
+ runKatiPackage(ctx, config, false)
- ioutil.WriteFile(config.LastKatiSuffixFile(), []byte(config.KatiSuffix()), 0666) // a+rw
} else if what&RunKatiNinja != 0 {
// Load last Kati Suffix if it exists
- if katiSuffix, err := ioutil.ReadFile(config.LastKatiSuffixFile()); err == nil {
+ if katiSuffix, err := os.ReadFile(config.LastKatiSuffixFile()); err == nil {
ctx.Verboseln("Loaded previous kati config:", string(katiSuffix))
config.SetKatiSuffix(string(katiSuffix))
}
+ } else if what&RunSoong != 0 {
+ runKatiPackage(ctx, config, true)
}
+ os.WriteFile(config.LastKatiSuffixFile(), []byte(config.KatiSuffix()), 0666) // a+rw
+
// Write combined ninja file
createCombinedBuildNinjaFile(ctx, config)
diff --git a/ui/build/config.go b/ui/build/config.go
index dc468c2..4f2d213 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -1592,6 +1592,10 @@
return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja")
}
+func (c *configImpl) KatiSoongOnlyPackageNinjaFile() string {
+ return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiSoongOnlyPackageSuffix+".ninja")
+}
+
func (c *configImpl) SoongVarsFile() string {
targetProduct, err := c.TargetProductOrErr()
if err != nil {
@@ -1647,7 +1651,7 @@
}
func (c *configImpl) KatiPackageMkDir() string {
- return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging")
+ return filepath.Join(c.SoongOutDir(), "kati_packaging"+c.KatiSuffix())
}
func (c *configImpl) hostOutRoot() string {
diff --git a/ui/build/kati.go b/ui/build/kati.go
index acd5254..6519573 100644
--- a/ui/build/kati.go
+++ b/ui/build/kati.go
@@ -31,6 +31,7 @@
const katiBuildSuffix = ""
const katiCleanspecSuffix = "-cleanspec"
const katiPackageSuffix = "-package"
+const katiSoongOnlyPackageSuffix = "-soong-only-package"
// genKatiSuffix creates a filename suffix for kati-generated files so that we
// can cache them based on their inputs. Such files include the generated Ninja
@@ -40,8 +41,12 @@
// Currently that includes the TARGET_PRODUCT and kati-processed command line
// arguments.
func genKatiSuffix(ctx Context, config Config) {
+ targetProduct := "unknown"
+ if p, err := config.TargetProductOrErr(); err == nil {
+ targetProduct = p
+ }
// Construct the base suffix.
- katiSuffix := "-" + config.TargetProduct() + config.CoverageSuffix()
+ katiSuffix := "-" + targetProduct + config.CoverageSuffix()
// Append kati arguments to the suffix.
if args := config.KatiArgs(); len(args) > 0 {
@@ -68,13 +73,13 @@
func writeValueIfChanged(ctx Context, config Config, dir string, filename string, value string) {
filePath := filepath.Join(dir, filename)
previousValue := ""
- rawPreviousValue, err := ioutil.ReadFile(filePath)
+ rawPreviousValue, err := os.ReadFile(filePath)
if err == nil {
previousValue = string(rawPreviousValue)
}
if previousValue != value {
- if err = ioutil.WriteFile(filePath, []byte(value), 0666); err != nil {
+ if err = os.WriteFile(filePath, []byte(value), 0666); err != nil {
ctx.Fatalf("Failed to write: %v", err)
}
}
@@ -334,10 +339,19 @@
// Generate the Ninja file containing the packaging command lines for the dist
// dir.
-func runKatiPackage(ctx Context, config Config) {
+func runKatiPackage(ctx Context, config Config, soongOnly bool) {
ctx.BeginTrace(metrics.RunKati, "kati package")
defer ctx.EndTrace()
+ entryPoint := "build/make/packaging/main.mk"
+ suffix := katiPackageSuffix
+ ninjaFile := config.KatiPackageNinjaFile()
+ if soongOnly {
+ entryPoint = "build/make/packaging/main_soong_only.mk"
+ suffix = katiSoongOnlyPackageSuffix
+ ninjaFile = config.KatiSoongOnlyPackageNinjaFile()
+ }
+
args := []string{
// Mark the dist dir as writable.
"--writable", config.DistDir() + "/",
@@ -346,14 +360,14 @@
// Fail when redefining / duplicating a target.
"--werror_overriding_commands",
// Entry point.
- "-f", "build/make/packaging/main.mk",
+ "-f", entryPoint,
// Directory containing .mk files for packaging purposes, such as
// the dist.mk file, containing dist-for-goals data.
"KATI_PACKAGE_MK_DIR=" + config.KatiPackageMkDir(),
}
// Run Kati against a restricted set of environment variables.
- runKati(ctx, config, katiPackageSuffix, args, func(env *Environment) {
+ runKati(ctx, config, suffix, args, func(env *Environment) {
env.Allow([]string{
// Some generic basics
"LANG",
@@ -381,7 +395,7 @@
})
// Compress and dist the packaging Ninja file.
- distGzipFile(ctx, config, config.KatiPackageNinjaFile())
+ distGzipFile(ctx, config, ninjaFile)
}
// Run Kati on the cleanspec files to clean the build.
diff --git a/ui/build/soong.go b/ui/build/soong.go
index 82e5c96..58334a9 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -197,6 +197,8 @@
func (pb PrimaryBuilderFactory) primaryBuilderInvocation(config Config) bootstrap.PrimaryBuilderInvocation {
commonArgs := make([]string, 0, 0)
+ commonArgs = append(commonArgs, "--kati_suffix", config.KatiSuffix())
+
if !pb.config.skipSoongTests {
commonArgs = append(commonArgs, "-t")
}