blob: b8543d929a27af7ecd4fc42ba06c261c6883f569 [file] [log] [blame]
Dan Willemsen1e704462016-08-21 15:17:17 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
Colin Cross9191df22021-10-29 13:11:32 -070018 "fmt"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070019 "os"
Dan Willemsen1e704462016-08-21 15:17:17 -070020 "path/filepath"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070021 "strconv"
Usta Shrestha8dc8b0a2022-08-10 17:39:37 -040022 "strings"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070023
Chris Parsons9402ca82023-02-23 17:28:06 -050024 "android/soong/bazel"
Dan Willemsen4591b642021-05-24 14:24:12 -070025 "android/soong/ui/metrics"
Dan Willemsen4591b642021-05-24 14:24:12 -070026 "android/soong/ui/status"
27
28 "android/soong/shared"
29
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010030 "github.com/google/blueprint"
31 "github.com/google/blueprint/bootstrap"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070032 "github.com/google/blueprint/microfactory"
Dan Willemsen1e704462016-08-21 15:17:17 -070033)
34
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020035const (
36 availableEnvFile = "soong.environment.available"
37 usedEnvFile = "soong.environment.used"
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +020038
Lukacs T. Berkic541cd22022-10-26 07:26:50 +000039 soongBuildTag = "build"
40 bp2buildFilesTag = "bp2build_files"
41 bp2buildWorkspaceTag = "bp2build_workspace"
42 jsonModuleGraphTag = "modulegraph"
43 queryviewTag = "queryview"
44 apiBp2buildTag = "api_bp2build"
45 soongDocsTag = "soong_docs"
Colin Cross9191df22021-10-29 13:11:32 -070046
47 // bootstrapEpoch is used to determine if an incremental build is incompatible with the current
48 // version of bootstrap and needs cleaning before continuing the build. Increment this for
49 // incompatible changes, for example when moving the location of the bpglob binary that is
50 // executed during bootstrap before the primary builder has had a chance to update the path.
Lukacs T. Berki9985d9a2021-11-04 11:47:42 +010051 bootstrapEpoch = 1
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020052)
53
Sasha Smundak4cbe83a2022-11-28 17:02:40 -080054func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error {
Lukacs T. Berki7690c092021-02-26 14:27:36 +010055 data, err := shared.EnvFileContents(envDeps)
56 if err != nil {
57 return err
58 }
59
Sasha Smundak4cbe83a2022-11-28 17:02:40 -080060 return os.WriteFile(envFile, data, 0644)
Lukacs T. Berki7690c092021-02-26 14:27:36 +010061}
62
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000063// This uses Android.bp files and various tools to generate <builddir>/build.ninja.
64//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010065// However, the execution of <builddir>/build.ninja happens later in
66// build/soong/ui/build/build.go#Build()
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000067//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010068// We want to rely on as few prebuilts as possible, so we need to bootstrap
69// Soong. The process is as follows:
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000070//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010071// 1. We use "Microfactory", a simple tool to compile Go code, to build
72// first itself, then soong_ui from soong_ui.bash. This binary contains
73// parts of soong_build that are needed to build itself.
74// 2. This simplified version of soong_build then reads the Blueprint files
75// that describe itself and emits .bootstrap/build.ninja that describes
76// how to build its full version and use that to produce the final Ninja
77// file Soong emits.
78// 3. soong_ui executes .bootstrap/build.ninja
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000079//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010080// (After this, Kati is executed to parse the Makefiles, but that's not part of
81// bootstrapping Soong)
82
83// A tiny struct used to tell Blueprint that it's in bootstrap mode. It would
84// probably be nicer to use a flag in bootstrap.Args instead.
85type BlueprintConfig struct {
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +020086 toolDir string
87 soongOutDir string
88 outDir string
89 runGoTests bool
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +020090 debugCompilation bool
91 subninjas []string
92 primaryBuilderInvocations []bootstrap.PrimaryBuilderInvocation
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010093}
94
Lukacs T. Berkia806e412021-09-01 08:57:48 +020095func (c BlueprintConfig) HostToolDir() string {
96 return c.toolDir
97}
98
Lukacs T. Berki9f6c24a2021-08-26 15:07:24 +020099func (c BlueprintConfig) SoongOutDir() string {
100 return c.soongOutDir
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100101}
102
Lukacs T. Berki9f6c24a2021-08-26 15:07:24 +0200103func (c BlueprintConfig) OutDir() string {
104 return c.outDir
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100105}
106
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +0200107func (c BlueprintConfig) RunGoTests() bool {
108 return c.runGoTests
109}
110
Lukacs T. Berki5f6cb1d2021-03-17 15:03:14 +0100111func (c BlueprintConfig) DebugCompilation() bool {
112 return c.debugCompilation
113}
114
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +0200115func (c BlueprintConfig) Subninjas() []string {
116 return c.subninjas
117}
118
119func (c BlueprintConfig) PrimaryBuilderInvocations() []bootstrap.PrimaryBuilderInvocation {
120 return c.primaryBuilderInvocations
121}
122
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200123func environmentArgs(config Config, tag string) []string {
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200124 return []string{
125 "--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200126 "--used_env", config.UsedEnvFile(tag),
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200127 }
128}
Spandan Das8f99ae62021-06-11 16:48:06 +0000129
Colin Cross9191df22021-10-29 13:11:32 -0700130func writeEmptyFile(ctx Context, path string) {
Spandan Das8f99ae62021-06-11 16:48:06 +0000131 err := os.MkdirAll(filepath.Dir(path), 0777)
132 if err != nil {
Colin Cross9191df22021-10-29 13:11:32 -0700133 ctx.Fatalf("Failed to create parent directories of empty file '%s': %s", path, err)
Spandan Das8f99ae62021-06-11 16:48:06 +0000134 }
135
Colin Cross9191df22021-10-29 13:11:32 -0700136 if exists, err := fileExists(path); err != nil {
137 ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err)
138 } else if !exists {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800139 err = os.WriteFile(path, nil, 0666)
Spandan Das8f99ae62021-06-11 16:48:06 +0000140 if err != nil {
Colin Cross9191df22021-10-29 13:11:32 -0700141 ctx.Fatalf("Failed to create empty file '%s': %s", path, err)
Spandan Das8f99ae62021-06-11 16:48:06 +0000142 }
143 }
144}
145
Colin Cross9191df22021-10-29 13:11:32 -0700146func fileExists(path string) (bool, error) {
147 if _, err := os.Stat(path); os.IsNotExist(err) {
148 return false, nil
149 } else if err != nil {
150 return false, err
151 }
152 return true, nil
153}
154
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800155type PrimaryBuilderFactory struct {
156 name string
157 description string
158 config Config
159 output string
160 specificArgs []string
161 debugPort string
162}
163
Jeongik Chaccf37002023-08-04 01:46:32 +0900164func getGlobPathName(config Config) string {
165 globPathName, ok := config.TargetProductOrErr()
166 if ok != nil {
167 globPathName = soongBuildTag
168 }
169 return globPathName
170}
171
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800172func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuilderInvocation {
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200173 commonArgs := make([]string, 0, 0)
174
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800175 if !pb.config.skipSoongTests {
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200176 commonArgs = append(commonArgs, "-t")
177 }
178
LaMont Jones80f70352023-03-20 19:58:25 +0000179 if pb.config.multitreeBuild {
LaMont Jones52a72432023-03-09 18:19:35 +0000180 commonArgs = append(commonArgs, "--multitree-build")
181 }
Jihoon Kang1bff0342023-01-17 20:40:22 +0000182 if pb.config.buildFromTextStub {
183 commonArgs = append(commonArgs, "--build-from-text-stub")
184 }
LaMont Jones52a72432023-03-09 18:19:35 +0000185
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800186 commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list"))
Lukacs T. Berki13644272022-01-05 10:29:56 +0100187 invocationEnv := make(map[string]string)
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800188 if pb.debugPort != "" {
ustafb67fd12022-08-19 19:26:00 -0400189 //debug mode
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800190 commonArgs = append(commonArgs, "--delve_listen", pb.debugPort,
191 "--delve_path", shared.ResolveDelveBinary())
Lukacs T. Berki13644272022-01-05 10:29:56 +0100192 // GODEBUG=asyncpreemptoff=1 disables the preemption of goroutines. This
193 // is useful because the preemption happens by sending SIGURG to the OS
194 // thread hosting the goroutine in question and each signal results in
195 // work that needs to be done by Delve; it uses ptrace to debug the Go
196 // process and the tracer process must deal with every signal (it is not
197 // possible to selectively ignore SIGURG). This makes debugging slower,
198 // sometimes by an order of magnitude depending on luck.
199 // The original reason for adding async preemption to Go is here:
200 // https://github.com/golang/proposal/blob/master/design/24543-non-cooperative-preemption.md
201 invocationEnv["GODEBUG"] = "asyncpreemptoff=1"
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200202 }
203
ustafb67fd12022-08-19 19:26:00 -0400204 var allArgs []string
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800205 allArgs = append(allArgs, pb.specificArgs...)
Jeongik Chaccf37002023-08-04 01:46:32 +0900206 globPathName := pb.name
207 // Glob path for soong build would be separated per product target
208 if pb.name == soongBuildTag {
209 globPathName = getGlobPathName(pb.config)
210 }
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200211 allArgs = append(allArgs,
Jeongik Chaccf37002023-08-04 01:46:32 +0900212 "--globListDir", globPathName,
213 "--globFile", pb.config.NamedGlobFile(globPathName))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200214
215 allArgs = append(allArgs, commonArgs...)
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800216 allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800217 if profileCpu := os.Getenv("SOONG_PROFILE_CPU"); profileCpu != "" {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800218 allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+pb.name)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800219 }
220 if profileMem := os.Getenv("SOONG_PROFILE_MEM"); profileMem != "" {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800221 allArgs = append(allArgs, "--memprofile", profileMem+"."+pb.name)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800222 }
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200223 allArgs = append(allArgs, "Android.bp")
224
225 return bootstrap.PrimaryBuilderInvocation{
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000226 Inputs: []string{"Android.bp"},
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800227 Outputs: []string{pb.output},
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000228 Args: allArgs,
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800229 Description: pb.description,
Lukacs T. Berki2fad3412022-01-04 14:40:13 +0100230 // NB: Changing the value of this environment variable will not result in a
231 // rebuild. The bootstrap Ninja file will change, but apparently Ninja does
232 // not consider changing the pool specified in a statement a change that's
233 // worth rebuilding for.
234 Console: os.Getenv("SOONG_UNBUFFERED_OUTPUT") == "1",
Lukacs T. Berki13644272022-01-05 10:29:56 +0100235 Env: invocationEnv,
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200236 }
237}
238
Colin Cross9191df22021-10-29 13:11:32 -0700239// bootstrapEpochCleanup deletes files used by bootstrap during incremental builds across
240// incompatible changes. Incompatible changes are marked by incrementing the bootstrapEpoch
241// constant. A tree is considered out of date for the current epoch of the
242// .soong.bootstrap.epoch.<epoch> file doesn't exist.
243func bootstrapEpochCleanup(ctx Context, config Config) {
244 epochFile := fmt.Sprintf(".soong.bootstrap.epoch.%d", bootstrapEpoch)
245 epochPath := filepath.Join(config.SoongOutDir(), epochFile)
246 if exists, err := fileExists(epochPath); err != nil {
247 ctx.Fatalf("failed to check if bootstrap epoch file %q exists: %q", epochPath, err)
248 } else if !exists {
249 // The tree is out of date for the current epoch, delete files used by bootstrap
250 // and force the primary builder to rerun.
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900251 os.Remove(config.SoongNinjaFile())
Colin Cross9191df22021-10-29 13:11:32 -0700252 for _, globFile := range bootstrapGlobFileList(config) {
253 os.Remove(globFile)
254 }
255
256 // Mark the tree as up to date with the current epoch by writing the epoch marker file.
257 writeEmptyFile(ctx, epochPath)
258 }
259}
260
261func bootstrapGlobFileList(config Config) []string {
262 return []string{
Jeongik Chaccf37002023-08-04 01:46:32 +0900263 config.NamedGlobFile(getGlobPathName(config)),
Lukacs T. Berkic541cd22022-10-26 07:26:50 +0000264 config.NamedGlobFile(bp2buildFilesTag),
Colin Cross9191df22021-10-29 13:11:32 -0700265 config.NamedGlobFile(jsonModuleGraphTag),
266 config.NamedGlobFile(queryviewTag),
Spandan Das5af0bd32022-09-28 20:43:08 +0000267 config.NamedGlobFile(apiBp2buildTag),
Colin Cross9191df22021-10-29 13:11:32 -0700268 config.NamedGlobFile(soongDocsTag),
269 }
270}
271
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200272func bootstrapBlueprint(ctx Context, config Config) {
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100273 ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
274 defer ctx.EndTrace()
275
Colin Cross9191df22021-10-29 13:11:32 -0700276 // Clean up some files for incremental builds across incompatible changes.
277 bootstrapEpochCleanup(ctx, config)
278
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900279 baseArgs := []string{"--soong_variables", config.SoongVarsFile()}
280
281 mainSoongBuildExtraArgs := append(baseArgs, "-o", config.SoongNinjaFile())
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200282 if config.EmptyNinjaFile() {
283 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
Lukacs T. Berki745380c2021-04-12 12:07:44 +0200284 }
Chris Parsonsef615e52022-08-18 22:04:11 -0400285 if config.bazelProdMode {
286 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode")
287 }
MarkDacekb78465d2022-10-18 20:10:16 +0000288 if config.bazelStagingMode {
289 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging")
290 }
Chris Parsons9402ca82023-02-23 17:28:06 -0500291 if config.IsPersistentBazelEnabled() {
292 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--use-bazel-proxy")
293 }
MarkDacekd06db5d2022-11-29 00:47:59 +0000294 if len(config.bazelForceEnabledModules) > 0 {
295 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-force-enabled-modules="+config.bazelForceEnabledModules)
296 }
LaMont Jones52a72432023-03-09 18:19:35 +0000297 if config.MultitreeBuild() {
298 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--multitree-build")
299 }
Jihoon Kang1bff0342023-01-17 20:40:22 +0000300 if config.buildFromTextStub {
301 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--build-from-text-stub")
302 }
MarkDacekf47e1422023-04-19 16:47:36 +0000303 if config.ensureAllowlistIntegrity {
304 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--ensure-allowlist-integrity")
305 }
MarkDacekd06db5d2022-11-29 00:47:59 +0000306
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000307 queryviewDir := filepath.Join(config.SoongOutDir(), "queryview")
Spandan Das5af0bd32022-09-28 20:43:08 +0000308 // The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files)
309 // The final workspace will be generated in out/soong/api_bp2build
310 apiBp2buildDir := filepath.Join(config.SoongOutDir(), ".api_bp2build")
Spandan Das5af0bd32022-09-28 20:43:08 +0000311
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800312 pbfs := []PrimaryBuilderFactory{
313 {
314 name: soongBuildTag,
315 description: fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()),
316 config: config,
317 output: config.SoongNinjaFile(),
318 specificArgs: mainSoongBuildExtraArgs,
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000319 },
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800320 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900321 name: bp2buildFilesTag,
322 description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
323 config: config,
324 output: config.Bp2BuildFilesMarkerFile(),
325 specificArgs: append(baseArgs,
326 "--bp2build_marker", config.Bp2BuildFilesMarkerFile(),
327 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800328 },
329 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900330 name: bp2buildWorkspaceTag,
331 description: "Creating Bazel symlink forest",
332 config: config,
333 output: config.Bp2BuildWorkspaceMarkerFile(),
334 specificArgs: append(baseArgs,
335 "--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile(),
336 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800337 },
338 {
339 name: jsonModuleGraphTag,
340 description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
341 config: config,
342 output: config.ModuleGraphFile(),
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900343 specificArgs: append(baseArgs,
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800344 "--module_graph_file", config.ModuleGraphFile(),
345 "--module_actions_file", config.ModuleActionsFile(),
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900346 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800347 },
348 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900349 name: queryviewTag,
350 description: fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir),
351 config: config,
352 output: config.QueryviewMarkerFile(),
353 specificArgs: append(baseArgs,
354 "--bazel_queryview_dir", queryviewDir,
355 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800356 },
357 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900358 name: apiBp2buildTag,
359 description: fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir),
360 config: config,
361 output: config.ApiBp2buildMarkerFile(),
362 specificArgs: append(baseArgs,
363 "--bazel_api_bp2build_dir", apiBp2buildDir,
364 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800365 },
366 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900367 name: soongDocsTag,
368 description: fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()),
369 config: config,
370 output: config.SoongDocsHtml(),
371 specificArgs: append(baseArgs,
372 "--soong_docs", config.SoongDocsHtml(),
373 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800374 },
375 }
376
377 // Figure out which invocations will be run under the debugger:
378 // * SOONG_DELVE if set specifies listening port
379 // * SOONG_DELVE_STEPS if set specifies specific invocations to be debugged, otherwise all are
380 debuggedInvocations := make(map[string]bool)
381 delvePort := os.Getenv("SOONG_DELVE")
382 if delvePort != "" {
383 if steps := os.Getenv("SOONG_DELVE_STEPS"); steps != "" {
384 var validSteps []string
385 for _, pbf := range pbfs {
386 debuggedInvocations[pbf.name] = false
387 validSteps = append(validSteps, pbf.name)
388
389 }
390 for _, step := range strings.Split(steps, ",") {
391 if _, ok := debuggedInvocations[step]; ok {
392 debuggedInvocations[step] = true
393 } else {
394 ctx.Fatalf("SOONG_DELVE_STEPS contains unknown soong_build step %s\n"+
395 "Valid steps are %v", step, validSteps)
396 }
397 }
398 } else {
399 // SOONG_DELVE_STEPS is not set, run all steps in the debugger
400 for _, pbf := range pbfs {
401 debuggedInvocations[pbf.name] = true
402 }
403 }
404 }
405
406 var invocations []bootstrap.PrimaryBuilderInvocation
407 for _, pbf := range pbfs {
408 if debuggedInvocations[pbf.name] {
409 pbf.debugPort = delvePort
410 }
411 pbi := pbf.primaryBuilderInvocation()
412 // Some invocations require adjustment:
413 switch pbf.name {
414 case soongBuildTag:
415 if config.BazelBuildEnabled() {
416 // Mixed builds call Bazel from soong_build and they therefore need the
417 // Bazel workspace to be available. Make that so by adding a dependency on
418 // the bp2build marker file to the action that invokes soong_build .
419 pbi.OrderOnlyInputs = append(pbi.OrderOnlyInputs, config.Bp2BuildWorkspaceMarkerFile())
420 }
421 case bp2buildWorkspaceTag:
422 pbi.Inputs = append(pbi.Inputs,
423 config.Bp2BuildFilesMarkerFile(),
424 filepath.Join(config.FileListDir(), "bazel.list"))
Wei Li2c9e8d62023-05-05 01:07:15 -0700425 case bp2buildFilesTag:
426 pbi.Inputs = append(pbi.Inputs, filepath.Join(config.FileListDir(), "METADATA.list"))
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800427 }
428 invocations = append(invocations, pbi)
429 }
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200430
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200431 // The glob .ninja files are subninja'd. However, they are generated during
432 // the build itself so we write an empty file if the file does not exist yet
433 // so that the subninja doesn't fail on clean builds
Colin Cross9191df22021-10-29 13:11:32 -0700434 for _, globFile := range bootstrapGlobFileList(config) {
435 writeEmptyFile(ctx, globFile)
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200436 }
437
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800438 blueprintArgs := bootstrap.Args{
439 ModuleListFile: filepath.Join(config.FileListDir(), "Android.bp.list"),
440 OutFile: shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja"),
441 EmptyNinjaFile: false,
442 }
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200443
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100444 blueprintCtx := blueprint.NewContext()
Spandan Dasc5763832022-11-08 18:42:16 +0000445 blueprintCtx.AddIncludeTags(config.GetIncludeTags()...)
Sam Delmerico98a73292023-02-21 11:50:29 -0500446 blueprintCtx.AddSourceRootDirs(config.GetSourceRootDirs()...)
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100447 blueprintCtx.SetIgnoreUnknownModuleTypes(true)
448 blueprintConfig := BlueprintConfig{
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200449 soongOutDir: config.SoongOutDir(),
450 toolDir: config.HostToolDir(),
451 outDir: config.OutDir(),
452 runGoTests: !config.skipSoongTests,
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200453 // If we want to debug soong_build, we need to compile it for debugging
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800454 debugCompilation: delvePort != "",
455 subninjas: bootstrapGlobFileList(config),
456 primaryBuilderInvocations: invocations,
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100457 }
458
usta5bb4a5d2022-08-24 12:53:46 -0400459 // since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little
460 // reason to write a `bootstrap.ninja.d` file
Lukacs T. Berkic357c812023-06-20 09:30:06 +0000461 _, err := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
462 if err != nil {
463 ctx.Fatal(err)
464 }
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100465}
466
Jason Wu2520f5e2023-05-30 19:45:36 -0400467func checkEnvironmentFile(ctx Context, currentEnv *Environment, envFile string) {
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200468 getenv := func(k string) string {
469 v, _ := currentEnv.Get(k)
470 return v
471 }
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200472
Jason Wu2520f5e2023-05-30 19:45:36 -0400473 // Log the changed environment variables to ChangedEnvironmentVariable field
474 if stale, changedEnvironmentVariableList, _ := shared.StaleEnvFile(envFile, getenv); stale {
475 for _, changedEnvironmentVariable := range changedEnvironmentVariableList {
476 ctx.Metrics.AddChangedEnvironmentVariable(changedEnvironmentVariable)
477 }
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200478 os.Remove(envFile)
479 }
480}
481
Dan Willemsen1e704462016-08-21 15:17:17 -0700482func runSoong(ctx Context, config Config) {
Nan Zhang17f27672018-12-12 16:01:49 -0800483 ctx.BeginTrace(metrics.RunSoong, "soong")
Dan Willemsend9f6fa22016-08-21 15:17:17 -0700484 defer ctx.EndTrace()
485
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100486 // We have two environment files: .available is the one with every variable,
487 // .used with the ones that were actually used. The latter is used to
488 // determine whether Soong needs to be re-run since why re-run it if only
489 // unused variables were changed?
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200490 envFile := filepath.Join(config.SoongOutDir(), availableEnvFile)
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100491
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100492 // This is done unconditionally, but does not take a measurable amount of time
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200493 bootstrapBlueprint(ctx, config)
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700494
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100495 soongBuildEnv := config.Environment().Copy()
496 soongBuildEnv.Set("TOP", os.Getenv("TOP"))
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100497 // For Bazel mixed builds.
Joe Onoratoba29f382022-10-24 06:38:11 -0700498 soongBuildEnv.Set("BAZEL_PATH", "./build/bazel/bin/bazel")
Chris Parsonsa9bef142022-09-28 15:07:46 -0400499 // Bazel's HOME var is set to an output subdirectory which doesn't exist. This
500 // prevents Bazel from file I/O in the actual user HOME directory.
501 soongBuildEnv.Set("BAZEL_HOME", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazelhome")))
Liz Kammer2af5ea82022-11-11 14:21:03 -0500502 soongBuildEnv.Set("BAZEL_OUTPUT_BASE", config.bazelOutputBase())
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100503 soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
504 soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -0500505 soongBuildEnv.Set("LOG_DIR", config.LogsDir())
Jingwen Chen3b13b612022-10-17 12:14:26 +0000506 soongBuildEnv.Set("BAZEL_DEPS_FILE", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazel.list")))
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100507
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100508 // For Soong bootstrapping tests
509 if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
510 soongBuildEnv.Set("ALLOW_MISSING_DEPENDENCIES", "true")
511 }
512
Paul Duffin5e85c662021-03-05 12:26:14 +0000513 err := writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap())
514 if err != nil {
515 ctx.Fatalf("failed to write environment file %s: %s", envFile, err)
516 }
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100517
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700518 func() {
Nan Zhang17f27672018-12-12 16:01:49 -0800519 ctx.BeginTrace(metrics.RunSoong, "environment check")
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700520 defer ctx.EndTrace()
521
Jason Wu2520f5e2023-05-30 19:45:36 -0400522 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(soongBuildTag))
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200523
Chris Parsonsef615e52022-08-18 22:04:11 -0400524 if config.BazelBuildEnabled() || config.Bp2Build() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400525 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(bp2buildFilesTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200526 }
527
528 if config.JsonModuleGraph() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400529 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(jsonModuleGraphTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200530 }
531
532 if config.Queryview() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400533 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(queryviewTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200534 }
535
Spandan Das5af0bd32022-09-28 20:43:08 +0000536 if config.ApiBp2build() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400537 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(apiBp2buildTag))
Spandan Das5af0bd32022-09-28 20:43:08 +0000538 }
539
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200540 if config.SoongDocs() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400541 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(soongDocsTag))
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700542 }
543 }()
544
Colin Cross9191df22021-10-29 13:11:32 -0700545 runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob",
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700546 map[string]string{"github.com/google/blueprint": "build/blueprint"})
Dan Willemsen5af1cbe2018-07-05 21:46:51 -0700547
usta49012ee2023-05-22 16:33:27 -0400548 ninja := func(targets ...string) {
549 ctx.BeginTrace(metrics.RunSoong, "bootstrap")
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700550 defer ctx.EndTrace()
551
Chris Parsons9402ca82023-02-23 17:28:06 -0500552 if config.IsPersistentBazelEnabled() {
Chris Parsonsc83398f2023-05-31 18:41:41 +0000553 bazelProxy := bazel.NewProxyServer(ctx.Logger, config.OutDir(), filepath.Join(config.SoongOutDir(), "workspace"), config.GetBazeliskBazelVersion())
usta49012ee2023-05-22 16:33:27 -0400554 if err := bazelProxy.Start(); err != nil {
555 ctx.Fatalf("Failed to create bazel proxy")
556 }
Chris Parsons9402ca82023-02-23 17:28:06 -0500557 defer bazelProxy.Close()
558 }
559
Dan Willemsenb82471a2018-05-17 16:37:09 -0700560 fifo := filepath.Join(config.OutDir(), ".ninja_fifo")
Colin Crossb98d3bc2019-03-21 16:02:58 -0700561 nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
562 defer nr.Close()
Dan Willemsenb82471a2018-05-17 16:37:09 -0700563
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200564 ninjaArgs := []string{
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700565 "-d", "keepdepfile",
Dan Willemsen08218222020-05-18 14:02:02 -0700566 "-d", "stats",
Dan Willemsen6587bed2020-04-18 20:25:59 -0700567 "-o", "usesphonyoutputs=yes",
568 "-o", "preremoveoutputs=yes",
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700569 "-w", "dupbuild=err",
Dan Willemsen6587bed2020-04-18 20:25:59 -0700570 "-w", "outputdir=err",
571 "-w", "missingoutfile=err",
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700572 "-j", strconv.Itoa(config.Parallel()),
Dan Willemsen02736672018-07-17 17:54:31 -0700573 "--frontend_file", fifo,
usta49012ee2023-05-22 16:33:27 -0400574 "-f", filepath.Join(config.SoongOutDir(), "bootstrap.ninja"),
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200575 }
576
Usta Shrestha8dc8b0a2022-08-10 17:39:37 -0400577 if extra, ok := config.Environment().Get("SOONG_UI_NINJA_ARGS"); ok {
578 ctx.Printf(`CAUTION: arguments in $SOONG_UI_NINJA_ARGS=%q, e.g. "-n", can make soong_build FAIL or INCORRECT`, extra)
579 ninjaArgs = append(ninjaArgs, strings.Fields(extra)...)
580 }
581
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200582 ninjaArgs = append(ninjaArgs, targets...)
usta49012ee2023-05-22 16:33:27 -0400583 cmd := Command(ctx, config, "soong bootstrap",
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200584 config.PrebuiltBuildTool("ninja"), ninjaArgs...)
Jingwen Chen7c6089a2020-11-02 02:56:20 -0500585
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100586 var ninjaEnv Environment
Lukacs T. Berki73ab9282021-03-10 10:48:39 +0100587
588 // This is currently how the command line to invoke soong_build finds the
589 // root of the source tree and the output root
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100590 ninjaEnv.Set("TOP", os.Getenv("TOP"))
Lukacs T. Berki7d613bf2021-03-02 10:09:41 +0100591
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100592 cmd.Environment = &ninjaEnv
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700593 cmd.Sandbox = soongSandbox
Colin Cross7b97ecd2019-06-19 13:17:59 -0700594 cmd.RunAndStreamOrFatal()
Dan Willemsen1e704462016-08-21 15:17:17 -0700595 }
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200596
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200597 targets := make([]string, 0, 0)
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200598
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200599 if config.JsonModuleGraph() {
600 targets = append(targets, config.ModuleGraphFile())
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200601 }
602
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200603 if config.Bp2Build() {
Lukacs T. Berkic541cd22022-10-26 07:26:50 +0000604 targets = append(targets, config.Bp2BuildWorkspaceMarkerFile())
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200605 }
606
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200607 if config.Queryview() {
608 targets = append(targets, config.QueryviewMarkerFile())
609 }
610
Spandan Das5af0bd32022-09-28 20:43:08 +0000611 if config.ApiBp2build() {
612 targets = append(targets, config.ApiBp2buildMarkerFile())
613 }
614
Lukacs T. Berkic6012f32021-09-06 18:31:46 +0200615 if config.SoongDocs() {
616 targets = append(targets, config.SoongDocsHtml())
617 }
618
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200619 if config.SoongBuildInvocationNeeded() {
620 // This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
Cole Faust65f298c2021-10-28 16:05:13 -0700621 targets = append(targets, config.SoongNinjaFile())
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200622 }
623
usta49012ee2023-05-22 16:33:27 -0400624 ninja(targets...)
Colin Crossb72c9092020-02-10 11:23:49 -0800625
Colin Cross8ba7d472020-06-25 11:27:52 -0700626 distGzipFile(ctx, config, config.SoongNinjaFile(), "soong")
Jihoon Kang9f4f8a32022-08-16 00:57:30 +0000627 distFile(ctx, config, config.SoongVarsFile(), "soong")
Colin Cross8ba7d472020-06-25 11:27:52 -0700628
Anton Hansson5e5c48b2020-11-27 12:35:20 +0000629 if !config.SkipKati() {
Colin Cross8ba7d472020-06-25 11:27:52 -0700630 distGzipFile(ctx, config, config.SoongAndroidMk(), "soong")
631 distGzipFile(ctx, config, config.SoongMakeVarsMk(), "soong")
632 }
633
Rob Seymour33cd10d2022-03-02 23:10:25 +0000634 if config.JsonModuleGraph() {
635 distGzipFile(ctx, config, config.ModuleGraphFile(), "soong")
636 }
Colin Crossb72c9092020-02-10 11:23:49 -0800637}
638
Lukacs T. Berki90b43342021-11-02 14:42:04 +0100639func runMicrofactory(ctx Context, config Config, name string, pkg string, mapping map[string]string) {
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700640 ctx.BeginTrace(metrics.RunSoong, name)
641 defer ctx.EndTrace()
642 cfg := microfactory.Config{TrimPath: absPath(ctx, ".")}
643 for pkgPrefix, pathPrefix := range mapping {
644 cfg.Map(pkgPrefix, pathPrefix)
645 }
646
Lukacs T. Berki90b43342021-11-02 14:42:04 +0100647 exePath := filepath.Join(config.SoongOutDir(), name)
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700648 dir := filepath.Dir(exePath)
649 if err := os.MkdirAll(dir, 0777); err != nil {
650 ctx.Fatalf("cannot create %s: %s", dir, err)
651 }
652 if _, err := microfactory.Build(&cfg, exePath, pkg); err != nil {
653 ctx.Fatalf("failed to build %s: %s", name, err)
654 }
655}