blob: 59d4fa87501d2868cd0a0228c737429bbfd4cab0 [file] [log] [blame]
Dan Willemsen1e704462016-08-21 15:17:17 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
Colin Cross9191df22021-10-29 13:11:32 -070018 "fmt"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070019 "os"
Dan Willemsen1e704462016-08-21 15:17:17 -070020 "path/filepath"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070021 "strconv"
Usta Shrestha8dc8b0a2022-08-10 17:39:37 -040022 "strings"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070023
Chris Parsons9402ca82023-02-23 17:28:06 -050024 "android/soong/bazel"
Dan Willemsen4591b642021-05-24 14:24:12 -070025 "android/soong/ui/metrics"
Dan Willemsen4591b642021-05-24 14:24:12 -070026 "android/soong/ui/status"
27
28 "android/soong/shared"
29
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010030 "github.com/google/blueprint"
31 "github.com/google/blueprint/bootstrap"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070032 "github.com/google/blueprint/microfactory"
Dan Willemsen1e704462016-08-21 15:17:17 -070033)
34
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020035const (
36 availableEnvFile = "soong.environment.available"
37 usedEnvFile = "soong.environment.used"
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +020038
Lukacs T. Berkic541cd22022-10-26 07:26:50 +000039 soongBuildTag = "build"
40 bp2buildFilesTag = "bp2build_files"
41 bp2buildWorkspaceTag = "bp2build_workspace"
42 jsonModuleGraphTag = "modulegraph"
43 queryviewTag = "queryview"
44 apiBp2buildTag = "api_bp2build"
45 soongDocsTag = "soong_docs"
Colin Cross9191df22021-10-29 13:11:32 -070046
47 // bootstrapEpoch is used to determine if an incremental build is incompatible with the current
48 // version of bootstrap and needs cleaning before continuing the build. Increment this for
49 // incompatible changes, for example when moving the location of the bpglob binary that is
50 // executed during bootstrap before the primary builder has had a chance to update the path.
Lukacs T. Berki9985d9a2021-11-04 11:47:42 +010051 bootstrapEpoch = 1
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020052)
53
Sasha Smundak4cbe83a2022-11-28 17:02:40 -080054func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error {
Lukacs T. Berki7690c092021-02-26 14:27:36 +010055 data, err := shared.EnvFileContents(envDeps)
56 if err != nil {
57 return err
58 }
59
Sasha Smundak4cbe83a2022-11-28 17:02:40 -080060 return os.WriteFile(envFile, data, 0644)
Lukacs T. Berki7690c092021-02-26 14:27:36 +010061}
62
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000063// This uses Android.bp files and various tools to generate <builddir>/build.ninja.
64//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010065// However, the execution of <builddir>/build.ninja happens later in
66// build/soong/ui/build/build.go#Build()
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000067//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010068// We want to rely on as few prebuilts as possible, so we need to bootstrap
69// Soong. The process is as follows:
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000070//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010071// 1. We use "Microfactory", a simple tool to compile Go code, to build
72// first itself, then soong_ui from soong_ui.bash. This binary contains
73// parts of soong_build that are needed to build itself.
74// 2. This simplified version of soong_build then reads the Blueprint files
75// that describe itself and emits .bootstrap/build.ninja that describes
76// how to build its full version and use that to produce the final Ninja
77// file Soong emits.
78// 3. soong_ui executes .bootstrap/build.ninja
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000079//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010080// (After this, Kati is executed to parse the Makefiles, but that's not part of
81// bootstrapping Soong)
82
83// A tiny struct used to tell Blueprint that it's in bootstrap mode. It would
84// probably be nicer to use a flag in bootstrap.Args instead.
85type BlueprintConfig struct {
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +020086 toolDir string
87 soongOutDir string
88 outDir string
89 runGoTests bool
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +020090 debugCompilation bool
91 subninjas []string
92 primaryBuilderInvocations []bootstrap.PrimaryBuilderInvocation
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010093}
94
Lukacs T. Berkia806e412021-09-01 08:57:48 +020095func (c BlueprintConfig) HostToolDir() string {
96 return c.toolDir
97}
98
Lukacs T. Berki9f6c24a2021-08-26 15:07:24 +020099func (c BlueprintConfig) SoongOutDir() string {
100 return c.soongOutDir
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100101}
102
Lukacs T. Berki9f6c24a2021-08-26 15:07:24 +0200103func (c BlueprintConfig) OutDir() string {
104 return c.outDir
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100105}
106
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +0200107func (c BlueprintConfig) RunGoTests() bool {
108 return c.runGoTests
109}
110
Lukacs T. Berki5f6cb1d2021-03-17 15:03:14 +0100111func (c BlueprintConfig) DebugCompilation() bool {
112 return c.debugCompilation
113}
114
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +0200115func (c BlueprintConfig) Subninjas() []string {
116 return c.subninjas
117}
118
119func (c BlueprintConfig) PrimaryBuilderInvocations() []bootstrap.PrimaryBuilderInvocation {
120 return c.primaryBuilderInvocations
121}
122
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200123func environmentArgs(config Config, tag string) []string {
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200124 return []string{
125 "--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200126 "--used_env", config.UsedEnvFile(tag),
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200127 }
128}
Spandan Das8f99ae62021-06-11 16:48:06 +0000129
Colin Cross9191df22021-10-29 13:11:32 -0700130func writeEmptyFile(ctx Context, path string) {
Spandan Das8f99ae62021-06-11 16:48:06 +0000131 err := os.MkdirAll(filepath.Dir(path), 0777)
132 if err != nil {
Colin Cross9191df22021-10-29 13:11:32 -0700133 ctx.Fatalf("Failed to create parent directories of empty file '%s': %s", path, err)
Spandan Das8f99ae62021-06-11 16:48:06 +0000134 }
135
Colin Cross9191df22021-10-29 13:11:32 -0700136 if exists, err := fileExists(path); err != nil {
137 ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err)
138 } else if !exists {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800139 err = os.WriteFile(path, nil, 0666)
Spandan Das8f99ae62021-06-11 16:48:06 +0000140 if err != nil {
Colin Cross9191df22021-10-29 13:11:32 -0700141 ctx.Fatalf("Failed to create empty file '%s': %s", path, err)
Spandan Das8f99ae62021-06-11 16:48:06 +0000142 }
143 }
144}
145
Colin Cross9191df22021-10-29 13:11:32 -0700146func fileExists(path string) (bool, error) {
147 if _, err := os.Stat(path); os.IsNotExist(err) {
148 return false, nil
149 } else if err != nil {
150 return false, err
151 }
152 return true, nil
153}
154
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800155type PrimaryBuilderFactory struct {
156 name string
157 description string
158 config Config
159 output string
160 specificArgs []string
161 debugPort string
162}
163
164func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuilderInvocation {
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200165 commonArgs := make([]string, 0, 0)
166
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800167 if !pb.config.skipSoongTests {
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200168 commonArgs = append(commonArgs, "-t")
169 }
170
LaMont Jones80f70352023-03-20 19:58:25 +0000171 if pb.config.multitreeBuild {
LaMont Jones52a72432023-03-09 18:19:35 +0000172 commonArgs = append(commonArgs, "--multitree-build")
173 }
Jihoon Kang1bff0342023-01-17 20:40:22 +0000174 if pb.config.buildFromTextStub {
175 commonArgs = append(commonArgs, "--build-from-text-stub")
176 }
LaMont Jones52a72432023-03-09 18:19:35 +0000177
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800178 commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list"))
Lukacs T. Berki13644272022-01-05 10:29:56 +0100179 invocationEnv := make(map[string]string)
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800180 if pb.debugPort != "" {
ustafb67fd12022-08-19 19:26:00 -0400181 //debug mode
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800182 commonArgs = append(commonArgs, "--delve_listen", pb.debugPort,
183 "--delve_path", shared.ResolveDelveBinary())
Lukacs T. Berki13644272022-01-05 10:29:56 +0100184 // GODEBUG=asyncpreemptoff=1 disables the preemption of goroutines. This
185 // is useful because the preemption happens by sending SIGURG to the OS
186 // thread hosting the goroutine in question and each signal results in
187 // work that needs to be done by Delve; it uses ptrace to debug the Go
188 // process and the tracer process must deal with every signal (it is not
189 // possible to selectively ignore SIGURG). This makes debugging slower,
190 // sometimes by an order of magnitude depending on luck.
191 // The original reason for adding async preemption to Go is here:
192 // https://github.com/golang/proposal/blob/master/design/24543-non-cooperative-preemption.md
193 invocationEnv["GODEBUG"] = "asyncpreemptoff=1"
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200194 }
195
ustafb67fd12022-08-19 19:26:00 -0400196 var allArgs []string
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800197 allArgs = append(allArgs, pb.specificArgs...)
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200198 allArgs = append(allArgs,
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800199 "--globListDir", pb.name,
200 "--globFile", pb.config.NamedGlobFile(pb.name))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200201
202 allArgs = append(allArgs, commonArgs...)
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800203 allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800204 if profileCpu := os.Getenv("SOONG_PROFILE_CPU"); profileCpu != "" {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800205 allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+pb.name)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800206 }
207 if profileMem := os.Getenv("SOONG_PROFILE_MEM"); profileMem != "" {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800208 allArgs = append(allArgs, "--memprofile", profileMem+"."+pb.name)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800209 }
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200210 allArgs = append(allArgs, "Android.bp")
211
212 return bootstrap.PrimaryBuilderInvocation{
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000213 Inputs: []string{"Android.bp"},
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800214 Outputs: []string{pb.output},
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000215 Args: allArgs,
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800216 Description: pb.description,
Lukacs T. Berki2fad3412022-01-04 14:40:13 +0100217 // NB: Changing the value of this environment variable will not result in a
218 // rebuild. The bootstrap Ninja file will change, but apparently Ninja does
219 // not consider changing the pool specified in a statement a change that's
220 // worth rebuilding for.
221 Console: os.Getenv("SOONG_UNBUFFERED_OUTPUT") == "1",
Lukacs T. Berki13644272022-01-05 10:29:56 +0100222 Env: invocationEnv,
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200223 }
224}
225
Colin Cross9191df22021-10-29 13:11:32 -0700226// bootstrapEpochCleanup deletes files used by bootstrap during incremental builds across
227// incompatible changes. Incompatible changes are marked by incrementing the bootstrapEpoch
228// constant. A tree is considered out of date for the current epoch of the
229// .soong.bootstrap.epoch.<epoch> file doesn't exist.
230func bootstrapEpochCleanup(ctx Context, config Config) {
231 epochFile := fmt.Sprintf(".soong.bootstrap.epoch.%d", bootstrapEpoch)
232 epochPath := filepath.Join(config.SoongOutDir(), epochFile)
233 if exists, err := fileExists(epochPath); err != nil {
234 ctx.Fatalf("failed to check if bootstrap epoch file %q exists: %q", epochPath, err)
235 } else if !exists {
236 // The tree is out of date for the current epoch, delete files used by bootstrap
237 // and force the primary builder to rerun.
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900238 os.Remove(config.SoongNinjaFile())
Colin Cross9191df22021-10-29 13:11:32 -0700239 for _, globFile := range bootstrapGlobFileList(config) {
240 os.Remove(globFile)
241 }
242
243 // Mark the tree as up to date with the current epoch by writing the epoch marker file.
244 writeEmptyFile(ctx, epochPath)
245 }
246}
247
248func bootstrapGlobFileList(config Config) []string {
249 return []string{
250 config.NamedGlobFile(soongBuildTag),
Lukacs T. Berkic541cd22022-10-26 07:26:50 +0000251 config.NamedGlobFile(bp2buildFilesTag),
Colin Cross9191df22021-10-29 13:11:32 -0700252 config.NamedGlobFile(jsonModuleGraphTag),
253 config.NamedGlobFile(queryviewTag),
Spandan Das5af0bd32022-09-28 20:43:08 +0000254 config.NamedGlobFile(apiBp2buildTag),
Colin Cross9191df22021-10-29 13:11:32 -0700255 config.NamedGlobFile(soongDocsTag),
256 }
257}
258
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200259func bootstrapBlueprint(ctx Context, config Config) {
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100260 ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
261 defer ctx.EndTrace()
262
Colin Cross9191df22021-10-29 13:11:32 -0700263 // Clean up some files for incremental builds across incompatible changes.
264 bootstrapEpochCleanup(ctx, config)
265
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900266 baseArgs := []string{"--soong_variables", config.SoongVarsFile()}
267
268 mainSoongBuildExtraArgs := append(baseArgs, "-o", config.SoongNinjaFile())
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200269 if config.EmptyNinjaFile() {
270 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
Lukacs T. Berki745380c2021-04-12 12:07:44 +0200271 }
Chris Parsonsef615e52022-08-18 22:04:11 -0400272 if config.bazelProdMode {
273 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode")
274 }
MarkDacekb78465d2022-10-18 20:10:16 +0000275 if config.bazelStagingMode {
276 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging")
277 }
Chris Parsons9402ca82023-02-23 17:28:06 -0500278 if config.IsPersistentBazelEnabled() {
279 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--use-bazel-proxy")
280 }
MarkDacekd06db5d2022-11-29 00:47:59 +0000281 if len(config.bazelForceEnabledModules) > 0 {
282 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-force-enabled-modules="+config.bazelForceEnabledModules)
283 }
LaMont Jones52a72432023-03-09 18:19:35 +0000284 if config.MultitreeBuild() {
285 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--multitree-build")
286 }
Jihoon Kang1bff0342023-01-17 20:40:22 +0000287 if config.buildFromTextStub {
288 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--build-from-text-stub")
289 }
MarkDacekf47e1422023-04-19 16:47:36 +0000290 if config.ensureAllowlistIntegrity {
291 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--ensure-allowlist-integrity")
292 }
MarkDacekd06db5d2022-11-29 00:47:59 +0000293
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000294 queryviewDir := filepath.Join(config.SoongOutDir(), "queryview")
Spandan Das5af0bd32022-09-28 20:43:08 +0000295 // The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files)
296 // The final workspace will be generated in out/soong/api_bp2build
297 apiBp2buildDir := filepath.Join(config.SoongOutDir(), ".api_bp2build")
Spandan Das5af0bd32022-09-28 20:43:08 +0000298
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800299 pbfs := []PrimaryBuilderFactory{
300 {
301 name: soongBuildTag,
302 description: fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()),
303 config: config,
304 output: config.SoongNinjaFile(),
305 specificArgs: mainSoongBuildExtraArgs,
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000306 },
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800307 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900308 name: bp2buildFilesTag,
309 description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
310 config: config,
311 output: config.Bp2BuildFilesMarkerFile(),
312 specificArgs: append(baseArgs,
313 "--bp2build_marker", config.Bp2BuildFilesMarkerFile(),
314 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800315 },
316 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900317 name: bp2buildWorkspaceTag,
318 description: "Creating Bazel symlink forest",
319 config: config,
320 output: config.Bp2BuildWorkspaceMarkerFile(),
321 specificArgs: append(baseArgs,
322 "--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile(),
323 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800324 },
325 {
326 name: jsonModuleGraphTag,
327 description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
328 config: config,
329 output: config.ModuleGraphFile(),
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900330 specificArgs: append(baseArgs,
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800331 "--module_graph_file", config.ModuleGraphFile(),
332 "--module_actions_file", config.ModuleActionsFile(),
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900333 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800334 },
335 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900336 name: queryviewTag,
337 description: fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir),
338 config: config,
339 output: config.QueryviewMarkerFile(),
340 specificArgs: append(baseArgs,
341 "--bazel_queryview_dir", queryviewDir,
342 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800343 },
344 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900345 name: apiBp2buildTag,
346 description: fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir),
347 config: config,
348 output: config.ApiBp2buildMarkerFile(),
349 specificArgs: append(baseArgs,
350 "--bazel_api_bp2build_dir", apiBp2buildDir,
351 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800352 },
353 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900354 name: soongDocsTag,
355 description: fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()),
356 config: config,
357 output: config.SoongDocsHtml(),
358 specificArgs: append(baseArgs,
359 "--soong_docs", config.SoongDocsHtml(),
360 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800361 },
362 }
363
364 // Figure out which invocations will be run under the debugger:
365 // * SOONG_DELVE if set specifies listening port
366 // * SOONG_DELVE_STEPS if set specifies specific invocations to be debugged, otherwise all are
367 debuggedInvocations := make(map[string]bool)
368 delvePort := os.Getenv("SOONG_DELVE")
369 if delvePort != "" {
370 if steps := os.Getenv("SOONG_DELVE_STEPS"); steps != "" {
371 var validSteps []string
372 for _, pbf := range pbfs {
373 debuggedInvocations[pbf.name] = false
374 validSteps = append(validSteps, pbf.name)
375
376 }
377 for _, step := range strings.Split(steps, ",") {
378 if _, ok := debuggedInvocations[step]; ok {
379 debuggedInvocations[step] = true
380 } else {
381 ctx.Fatalf("SOONG_DELVE_STEPS contains unknown soong_build step %s\n"+
382 "Valid steps are %v", step, validSteps)
383 }
384 }
385 } else {
386 // SOONG_DELVE_STEPS is not set, run all steps in the debugger
387 for _, pbf := range pbfs {
388 debuggedInvocations[pbf.name] = true
389 }
390 }
391 }
392
393 var invocations []bootstrap.PrimaryBuilderInvocation
394 for _, pbf := range pbfs {
395 if debuggedInvocations[pbf.name] {
396 pbf.debugPort = delvePort
397 }
398 pbi := pbf.primaryBuilderInvocation()
399 // Some invocations require adjustment:
400 switch pbf.name {
401 case soongBuildTag:
402 if config.BazelBuildEnabled() {
403 // Mixed builds call Bazel from soong_build and they therefore need the
404 // Bazel workspace to be available. Make that so by adding a dependency on
405 // the bp2build marker file to the action that invokes soong_build .
406 pbi.OrderOnlyInputs = append(pbi.OrderOnlyInputs, config.Bp2BuildWorkspaceMarkerFile())
407 }
408 case bp2buildWorkspaceTag:
409 pbi.Inputs = append(pbi.Inputs,
410 config.Bp2BuildFilesMarkerFile(),
411 filepath.Join(config.FileListDir(), "bazel.list"))
Wei Li2c9e8d62023-05-05 01:07:15 -0700412 case bp2buildFilesTag:
413 pbi.Inputs = append(pbi.Inputs, filepath.Join(config.FileListDir(), "METADATA.list"))
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800414 }
415 invocations = append(invocations, pbi)
416 }
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200417
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200418 // The glob .ninja files are subninja'd. However, they are generated during
419 // the build itself so we write an empty file if the file does not exist yet
420 // so that the subninja doesn't fail on clean builds
Colin Cross9191df22021-10-29 13:11:32 -0700421 for _, globFile := range bootstrapGlobFileList(config) {
422 writeEmptyFile(ctx, globFile)
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200423 }
424
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800425 blueprintArgs := bootstrap.Args{
426 ModuleListFile: filepath.Join(config.FileListDir(), "Android.bp.list"),
427 OutFile: shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja"),
428 EmptyNinjaFile: false,
429 }
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200430
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100431 blueprintCtx := blueprint.NewContext()
Spandan Dasc5763832022-11-08 18:42:16 +0000432 blueprintCtx.AddIncludeTags(config.GetIncludeTags()...)
Sam Delmerico98a73292023-02-21 11:50:29 -0500433 blueprintCtx.AddSourceRootDirs(config.GetSourceRootDirs()...)
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100434 blueprintCtx.SetIgnoreUnknownModuleTypes(true)
435 blueprintConfig := BlueprintConfig{
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200436 soongOutDir: config.SoongOutDir(),
437 toolDir: config.HostToolDir(),
438 outDir: config.OutDir(),
439 runGoTests: !config.skipSoongTests,
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200440 // If we want to debug soong_build, we need to compile it for debugging
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800441 debugCompilation: delvePort != "",
442 subninjas: bootstrapGlobFileList(config),
443 primaryBuilderInvocations: invocations,
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100444 }
445
usta5bb4a5d2022-08-24 12:53:46 -0400446 // since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little
447 // reason to write a `bootstrap.ninja.d` file
448 _ = bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100449}
450
Jason Wu2520f5e2023-05-30 19:45:36 -0400451func checkEnvironmentFile(ctx Context, currentEnv *Environment, envFile string) {
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200452 getenv := func(k string) string {
453 v, _ := currentEnv.Get(k)
454 return v
455 }
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200456
Jason Wu2520f5e2023-05-30 19:45:36 -0400457 // Log the changed environment variables to ChangedEnvironmentVariable field
458 if stale, changedEnvironmentVariableList, _ := shared.StaleEnvFile(envFile, getenv); stale {
459 for _, changedEnvironmentVariable := range changedEnvironmentVariableList {
460 ctx.Metrics.AddChangedEnvironmentVariable(changedEnvironmentVariable)
461 }
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200462 os.Remove(envFile)
463 }
464}
465
Dan Willemsen1e704462016-08-21 15:17:17 -0700466func runSoong(ctx Context, config Config) {
Nan Zhang17f27672018-12-12 16:01:49 -0800467 ctx.BeginTrace(metrics.RunSoong, "soong")
Dan Willemsend9f6fa22016-08-21 15:17:17 -0700468 defer ctx.EndTrace()
469
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100470 // We have two environment files: .available is the one with every variable,
471 // .used with the ones that were actually used. The latter is used to
472 // determine whether Soong needs to be re-run since why re-run it if only
473 // unused variables were changed?
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200474 envFile := filepath.Join(config.SoongOutDir(), availableEnvFile)
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100475
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100476 // This is done unconditionally, but does not take a measurable amount of time
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200477 bootstrapBlueprint(ctx, config)
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700478
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100479 soongBuildEnv := config.Environment().Copy()
480 soongBuildEnv.Set("TOP", os.Getenv("TOP"))
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100481 // For Bazel mixed builds.
Joe Onoratoba29f382022-10-24 06:38:11 -0700482 soongBuildEnv.Set("BAZEL_PATH", "./build/bazel/bin/bazel")
Chris Parsonsa9bef142022-09-28 15:07:46 -0400483 // Bazel's HOME var is set to an output subdirectory which doesn't exist. This
484 // prevents Bazel from file I/O in the actual user HOME directory.
485 soongBuildEnv.Set("BAZEL_HOME", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazelhome")))
Liz Kammer2af5ea82022-11-11 14:21:03 -0500486 soongBuildEnv.Set("BAZEL_OUTPUT_BASE", config.bazelOutputBase())
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100487 soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
488 soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -0500489 soongBuildEnv.Set("LOG_DIR", config.LogsDir())
Jingwen Chen3b13b612022-10-17 12:14:26 +0000490 soongBuildEnv.Set("BAZEL_DEPS_FILE", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazel.list")))
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100491
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100492 // For Soong bootstrapping tests
493 if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
494 soongBuildEnv.Set("ALLOW_MISSING_DEPENDENCIES", "true")
495 }
496
Paul Duffin5e85c662021-03-05 12:26:14 +0000497 err := writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap())
498 if err != nil {
499 ctx.Fatalf("failed to write environment file %s: %s", envFile, err)
500 }
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100501
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700502 func() {
Nan Zhang17f27672018-12-12 16:01:49 -0800503 ctx.BeginTrace(metrics.RunSoong, "environment check")
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700504 defer ctx.EndTrace()
505
Jason Wu2520f5e2023-05-30 19:45:36 -0400506 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(soongBuildTag))
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200507
Chris Parsonsef615e52022-08-18 22:04:11 -0400508 if config.BazelBuildEnabled() || config.Bp2Build() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400509 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(bp2buildFilesTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200510 }
511
512 if config.JsonModuleGraph() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400513 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(jsonModuleGraphTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200514 }
515
516 if config.Queryview() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400517 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(queryviewTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200518 }
519
Spandan Das5af0bd32022-09-28 20:43:08 +0000520 if config.ApiBp2build() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400521 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(apiBp2buildTag))
Spandan Das5af0bd32022-09-28 20:43:08 +0000522 }
523
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200524 if config.SoongDocs() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400525 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(soongDocsTag))
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700526 }
527 }()
528
Colin Cross9191df22021-10-29 13:11:32 -0700529 runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob",
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700530 map[string]string{"github.com/google/blueprint": "build/blueprint"})
Dan Willemsen5af1cbe2018-07-05 21:46:51 -0700531
usta49012ee2023-05-22 16:33:27 -0400532 ninja := func(targets ...string) {
533 ctx.BeginTrace(metrics.RunSoong, "bootstrap")
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700534 defer ctx.EndTrace()
535
Chris Parsons9402ca82023-02-23 17:28:06 -0500536 if config.IsPersistentBazelEnabled() {
Chris Parsonsc83398f2023-05-31 18:41:41 +0000537 bazelProxy := bazel.NewProxyServer(ctx.Logger, config.OutDir(), filepath.Join(config.SoongOutDir(), "workspace"), config.GetBazeliskBazelVersion())
usta49012ee2023-05-22 16:33:27 -0400538 if err := bazelProxy.Start(); err != nil {
539 ctx.Fatalf("Failed to create bazel proxy")
540 }
Chris Parsons9402ca82023-02-23 17:28:06 -0500541 defer bazelProxy.Close()
542 }
543
Dan Willemsenb82471a2018-05-17 16:37:09 -0700544 fifo := filepath.Join(config.OutDir(), ".ninja_fifo")
Colin Crossb98d3bc2019-03-21 16:02:58 -0700545 nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
546 defer nr.Close()
Dan Willemsenb82471a2018-05-17 16:37:09 -0700547
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200548 ninjaArgs := []string{
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700549 "-d", "keepdepfile",
Dan Willemsen08218222020-05-18 14:02:02 -0700550 "-d", "stats",
Dan Willemsen6587bed2020-04-18 20:25:59 -0700551 "-o", "usesphonyoutputs=yes",
552 "-o", "preremoveoutputs=yes",
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700553 "-w", "dupbuild=err",
Dan Willemsen6587bed2020-04-18 20:25:59 -0700554 "-w", "outputdir=err",
555 "-w", "missingoutfile=err",
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700556 "-j", strconv.Itoa(config.Parallel()),
Dan Willemsen02736672018-07-17 17:54:31 -0700557 "--frontend_file", fifo,
usta49012ee2023-05-22 16:33:27 -0400558 "-f", filepath.Join(config.SoongOutDir(), "bootstrap.ninja"),
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200559 }
560
Usta Shrestha8dc8b0a2022-08-10 17:39:37 -0400561 if extra, ok := config.Environment().Get("SOONG_UI_NINJA_ARGS"); ok {
562 ctx.Printf(`CAUTION: arguments in $SOONG_UI_NINJA_ARGS=%q, e.g. "-n", can make soong_build FAIL or INCORRECT`, extra)
563 ninjaArgs = append(ninjaArgs, strings.Fields(extra)...)
564 }
565
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200566 ninjaArgs = append(ninjaArgs, targets...)
usta49012ee2023-05-22 16:33:27 -0400567 cmd := Command(ctx, config, "soong bootstrap",
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200568 config.PrebuiltBuildTool("ninja"), ninjaArgs...)
Jingwen Chen7c6089a2020-11-02 02:56:20 -0500569
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100570 var ninjaEnv Environment
Lukacs T. Berki73ab9282021-03-10 10:48:39 +0100571
572 // This is currently how the command line to invoke soong_build finds the
573 // root of the source tree and the output root
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100574 ninjaEnv.Set("TOP", os.Getenv("TOP"))
Lukacs T. Berki7d613bf2021-03-02 10:09:41 +0100575
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100576 cmd.Environment = &ninjaEnv
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700577 cmd.Sandbox = soongSandbox
Colin Cross7b97ecd2019-06-19 13:17:59 -0700578 cmd.RunAndStreamOrFatal()
Dan Willemsen1e704462016-08-21 15:17:17 -0700579 }
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200580
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200581 targets := make([]string, 0, 0)
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200582
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200583 if config.JsonModuleGraph() {
584 targets = append(targets, config.ModuleGraphFile())
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200585 }
586
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200587 if config.Bp2Build() {
Lukacs T. Berkic541cd22022-10-26 07:26:50 +0000588 targets = append(targets, config.Bp2BuildWorkspaceMarkerFile())
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200589 }
590
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200591 if config.Queryview() {
592 targets = append(targets, config.QueryviewMarkerFile())
593 }
594
Spandan Das5af0bd32022-09-28 20:43:08 +0000595 if config.ApiBp2build() {
596 targets = append(targets, config.ApiBp2buildMarkerFile())
597 }
598
Lukacs T. Berkic6012f32021-09-06 18:31:46 +0200599 if config.SoongDocs() {
600 targets = append(targets, config.SoongDocsHtml())
601 }
602
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200603 if config.SoongBuildInvocationNeeded() {
604 // This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
Cole Faust65f298c2021-10-28 16:05:13 -0700605 targets = append(targets, config.SoongNinjaFile())
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200606 }
607
usta49012ee2023-05-22 16:33:27 -0400608 ninja(targets...)
Colin Crossb72c9092020-02-10 11:23:49 -0800609
Colin Cross8ba7d472020-06-25 11:27:52 -0700610 distGzipFile(ctx, config, config.SoongNinjaFile(), "soong")
Jihoon Kang9f4f8a32022-08-16 00:57:30 +0000611 distFile(ctx, config, config.SoongVarsFile(), "soong")
Colin Cross8ba7d472020-06-25 11:27:52 -0700612
Anton Hansson5e5c48b2020-11-27 12:35:20 +0000613 if !config.SkipKati() {
Colin Cross8ba7d472020-06-25 11:27:52 -0700614 distGzipFile(ctx, config, config.SoongAndroidMk(), "soong")
615 distGzipFile(ctx, config, config.SoongMakeVarsMk(), "soong")
616 }
617
Rob Seymour33cd10d2022-03-02 23:10:25 +0000618 if config.JsonModuleGraph() {
619 distGzipFile(ctx, config, config.ModuleGraphFile(), "soong")
620 }
Colin Crossb72c9092020-02-10 11:23:49 -0800621}
622
Lukacs T. Berki90b43342021-11-02 14:42:04 +0100623func runMicrofactory(ctx Context, config Config, name string, pkg string, mapping map[string]string) {
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700624 ctx.BeginTrace(metrics.RunSoong, name)
625 defer ctx.EndTrace()
626 cfg := microfactory.Config{TrimPath: absPath(ctx, ".")}
627 for pkgPrefix, pathPrefix := range mapping {
628 cfg.Map(pkgPrefix, pathPrefix)
629 }
630
Lukacs T. Berki90b43342021-11-02 14:42:04 +0100631 exePath := filepath.Join(config.SoongOutDir(), name)
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700632 dir := filepath.Dir(exePath)
633 if err := os.MkdirAll(dir, 0777); err != nil {
634 ctx.Fatalf("cannot create %s: %s", dir, err)
635 }
636 if _, err := microfactory.Build(&cfg, exePath, pkg); err != nil {
637 ctx.Fatalf("failed to build %s: %s", name, err)
638 }
639}