blob: 44ce3ad087d854eee7943dbf5939acc0f1da1dd3 [file] [log] [blame]
Dan Willemsen1e704462016-08-21 15:17:17 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
Colin Cross9191df22021-10-29 13:11:32 -070018 "fmt"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070019 "os"
Dan Willemsen1e704462016-08-21 15:17:17 -070020 "path/filepath"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070021 "strconv"
Usta Shrestha8dc8b0a2022-08-10 17:39:37 -040022 "strings"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070023
Chris Parsons9402ca82023-02-23 17:28:06 -050024 "android/soong/bazel"
Dan Willemsen4591b642021-05-24 14:24:12 -070025 "android/soong/ui/metrics"
Dan Willemsen4591b642021-05-24 14:24:12 -070026 "android/soong/ui/status"
27
28 "android/soong/shared"
29
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010030 "github.com/google/blueprint"
31 "github.com/google/blueprint/bootstrap"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070032 "github.com/google/blueprint/microfactory"
Dan Willemsen1e704462016-08-21 15:17:17 -070033)
34
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020035const (
36 availableEnvFile = "soong.environment.available"
37 usedEnvFile = "soong.environment.used"
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +020038
Lukacs T. Berkic541cd22022-10-26 07:26:50 +000039 soongBuildTag = "build"
40 bp2buildFilesTag = "bp2build_files"
41 bp2buildWorkspaceTag = "bp2build_workspace"
42 jsonModuleGraphTag = "modulegraph"
43 queryviewTag = "queryview"
44 apiBp2buildTag = "api_bp2build"
45 soongDocsTag = "soong_docs"
Colin Cross9191df22021-10-29 13:11:32 -070046
47 // bootstrapEpoch is used to determine if an incremental build is incompatible with the current
48 // version of bootstrap and needs cleaning before continuing the build. Increment this for
49 // incompatible changes, for example when moving the location of the bpglob binary that is
50 // executed during bootstrap before the primary builder has had a chance to update the path.
Lukacs T. Berki9985d9a2021-11-04 11:47:42 +010051 bootstrapEpoch = 1
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020052)
53
Sasha Smundak4cbe83a2022-11-28 17:02:40 -080054func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error {
Lukacs T. Berki7690c092021-02-26 14:27:36 +010055 data, err := shared.EnvFileContents(envDeps)
56 if err != nil {
57 return err
58 }
59
Sasha Smundak4cbe83a2022-11-28 17:02:40 -080060 return os.WriteFile(envFile, data, 0644)
Lukacs T. Berki7690c092021-02-26 14:27:36 +010061}
62
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000063// This uses Android.bp files and various tools to generate <builddir>/build.ninja.
64//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010065// However, the execution of <builddir>/build.ninja happens later in
66// build/soong/ui/build/build.go#Build()
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000067//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010068// We want to rely on as few prebuilts as possible, so we need to bootstrap
69// Soong. The process is as follows:
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000070//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010071// 1. We use "Microfactory", a simple tool to compile Go code, to build
72// first itself, then soong_ui from soong_ui.bash. This binary contains
73// parts of soong_build that are needed to build itself.
74// 2. This simplified version of soong_build then reads the Blueprint files
75// that describe itself and emits .bootstrap/build.ninja that describes
76// how to build its full version and use that to produce the final Ninja
77// file Soong emits.
78// 3. soong_ui executes .bootstrap/build.ninja
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000079//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010080// (After this, Kati is executed to parse the Makefiles, but that's not part of
81// bootstrapping Soong)
82
83// A tiny struct used to tell Blueprint that it's in bootstrap mode. It would
84// probably be nicer to use a flag in bootstrap.Args instead.
85type BlueprintConfig struct {
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +020086 toolDir string
87 soongOutDir string
88 outDir string
89 runGoTests bool
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +020090 debugCompilation bool
91 subninjas []string
92 primaryBuilderInvocations []bootstrap.PrimaryBuilderInvocation
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010093}
94
Lukacs T. Berkia806e412021-09-01 08:57:48 +020095func (c BlueprintConfig) HostToolDir() string {
96 return c.toolDir
97}
98
Lukacs T. Berki9f6c24a2021-08-26 15:07:24 +020099func (c BlueprintConfig) SoongOutDir() string {
100 return c.soongOutDir
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100101}
102
Lukacs T. Berki9f6c24a2021-08-26 15:07:24 +0200103func (c BlueprintConfig) OutDir() string {
104 return c.outDir
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100105}
106
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +0200107func (c BlueprintConfig) RunGoTests() bool {
108 return c.runGoTests
109}
110
Lukacs T. Berki5f6cb1d2021-03-17 15:03:14 +0100111func (c BlueprintConfig) DebugCompilation() bool {
112 return c.debugCompilation
113}
114
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +0200115func (c BlueprintConfig) Subninjas() []string {
116 return c.subninjas
117}
118
119func (c BlueprintConfig) PrimaryBuilderInvocations() []bootstrap.PrimaryBuilderInvocation {
120 return c.primaryBuilderInvocations
121}
122
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200123func environmentArgs(config Config, tag string) []string {
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200124 return []string{
125 "--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200126 "--used_env", config.UsedEnvFile(tag),
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200127 }
128}
Spandan Das8f99ae62021-06-11 16:48:06 +0000129
Colin Cross9191df22021-10-29 13:11:32 -0700130func writeEmptyFile(ctx Context, path string) {
Spandan Das8f99ae62021-06-11 16:48:06 +0000131 err := os.MkdirAll(filepath.Dir(path), 0777)
132 if err != nil {
Colin Cross9191df22021-10-29 13:11:32 -0700133 ctx.Fatalf("Failed to create parent directories of empty file '%s': %s", path, err)
Spandan Das8f99ae62021-06-11 16:48:06 +0000134 }
135
Colin Cross9191df22021-10-29 13:11:32 -0700136 if exists, err := fileExists(path); err != nil {
137 ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err)
138 } else if !exists {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800139 err = os.WriteFile(path, nil, 0666)
Spandan Das8f99ae62021-06-11 16:48:06 +0000140 if err != nil {
Colin Cross9191df22021-10-29 13:11:32 -0700141 ctx.Fatalf("Failed to create empty file '%s': %s", path, err)
Spandan Das8f99ae62021-06-11 16:48:06 +0000142 }
143 }
144}
145
Colin Cross9191df22021-10-29 13:11:32 -0700146func fileExists(path string) (bool, error) {
147 if _, err := os.Stat(path); os.IsNotExist(err) {
148 return false, nil
149 } else if err != nil {
150 return false, err
151 }
152 return true, nil
153}
154
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800155type PrimaryBuilderFactory struct {
156 name string
157 description string
158 config Config
159 output string
160 specificArgs []string
161 debugPort string
162}
163
164func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuilderInvocation {
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200165 commonArgs := make([]string, 0, 0)
166
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800167 if !pb.config.skipSoongTests {
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200168 commonArgs = append(commonArgs, "-t")
169 }
170
LaMont Jones80f70352023-03-20 19:58:25 +0000171 if pb.config.multitreeBuild {
LaMont Jones52a72432023-03-09 18:19:35 +0000172 commonArgs = append(commonArgs, "--multitree-build")
173 }
Jihoon Kang1bff0342023-01-17 20:40:22 +0000174 if pb.config.buildFromTextStub {
175 commonArgs = append(commonArgs, "--build-from-text-stub")
176 }
LaMont Jones52a72432023-03-09 18:19:35 +0000177
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800178 commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list"))
Lukacs T. Berki13644272022-01-05 10:29:56 +0100179 invocationEnv := make(map[string]string)
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800180 if pb.debugPort != "" {
ustafb67fd12022-08-19 19:26:00 -0400181 //debug mode
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800182 commonArgs = append(commonArgs, "--delve_listen", pb.debugPort,
183 "--delve_path", shared.ResolveDelveBinary())
Lukacs T. Berki13644272022-01-05 10:29:56 +0100184 // GODEBUG=asyncpreemptoff=1 disables the preemption of goroutines. This
185 // is useful because the preemption happens by sending SIGURG to the OS
186 // thread hosting the goroutine in question and each signal results in
187 // work that needs to be done by Delve; it uses ptrace to debug the Go
188 // process and the tracer process must deal with every signal (it is not
189 // possible to selectively ignore SIGURG). This makes debugging slower,
190 // sometimes by an order of magnitude depending on luck.
191 // The original reason for adding async preemption to Go is here:
192 // https://github.com/golang/proposal/blob/master/design/24543-non-cooperative-preemption.md
193 invocationEnv["GODEBUG"] = "asyncpreemptoff=1"
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200194 }
195
ustafb67fd12022-08-19 19:26:00 -0400196 var allArgs []string
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800197 allArgs = append(allArgs, pb.specificArgs...)
Jeongik Cha03e50cb2023-08-02 00:57:32 +0900198 globPathName, ok := pb.config.TargetProductOrErr()
199 if ok != nil {
200 globPathName = pb.name
201 }
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200202 allArgs = append(allArgs,
Jeongik Cha03e50cb2023-08-02 00:57:32 +0900203 "--globListDir", globPathName,
204 "--globFile", pb.config.NamedGlobFile(globPathName))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200205
206 allArgs = append(allArgs, commonArgs...)
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800207 allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800208 if profileCpu := os.Getenv("SOONG_PROFILE_CPU"); profileCpu != "" {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800209 allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+pb.name)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800210 }
211 if profileMem := os.Getenv("SOONG_PROFILE_MEM"); profileMem != "" {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800212 allArgs = append(allArgs, "--memprofile", profileMem+"."+pb.name)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800213 }
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200214 allArgs = append(allArgs, "Android.bp")
215
216 return bootstrap.PrimaryBuilderInvocation{
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000217 Inputs: []string{"Android.bp"},
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800218 Outputs: []string{pb.output},
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000219 Args: allArgs,
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800220 Description: pb.description,
Lukacs T. Berki2fad3412022-01-04 14:40:13 +0100221 // NB: Changing the value of this environment variable will not result in a
222 // rebuild. The bootstrap Ninja file will change, but apparently Ninja does
223 // not consider changing the pool specified in a statement a change that's
224 // worth rebuilding for.
225 Console: os.Getenv("SOONG_UNBUFFERED_OUTPUT") == "1",
Lukacs T. Berki13644272022-01-05 10:29:56 +0100226 Env: invocationEnv,
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200227 }
228}
229
Colin Cross9191df22021-10-29 13:11:32 -0700230// bootstrapEpochCleanup deletes files used by bootstrap during incremental builds across
231// incompatible changes. Incompatible changes are marked by incrementing the bootstrapEpoch
232// constant. A tree is considered out of date for the current epoch of the
233// .soong.bootstrap.epoch.<epoch> file doesn't exist.
234func bootstrapEpochCleanup(ctx Context, config Config) {
235 epochFile := fmt.Sprintf(".soong.bootstrap.epoch.%d", bootstrapEpoch)
236 epochPath := filepath.Join(config.SoongOutDir(), epochFile)
237 if exists, err := fileExists(epochPath); err != nil {
238 ctx.Fatalf("failed to check if bootstrap epoch file %q exists: %q", epochPath, err)
239 } else if !exists {
240 // The tree is out of date for the current epoch, delete files used by bootstrap
241 // and force the primary builder to rerun.
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900242 os.Remove(config.SoongNinjaFile())
Colin Cross9191df22021-10-29 13:11:32 -0700243 for _, globFile := range bootstrapGlobFileList(config) {
244 os.Remove(globFile)
245 }
246
247 // Mark the tree as up to date with the current epoch by writing the epoch marker file.
248 writeEmptyFile(ctx, epochPath)
249 }
250}
251
252func bootstrapGlobFileList(config Config) []string {
253 return []string{
254 config.NamedGlobFile(soongBuildTag),
Lukacs T. Berkic541cd22022-10-26 07:26:50 +0000255 config.NamedGlobFile(bp2buildFilesTag),
Colin Cross9191df22021-10-29 13:11:32 -0700256 config.NamedGlobFile(jsonModuleGraphTag),
257 config.NamedGlobFile(queryviewTag),
Spandan Das5af0bd32022-09-28 20:43:08 +0000258 config.NamedGlobFile(apiBp2buildTag),
Colin Cross9191df22021-10-29 13:11:32 -0700259 config.NamedGlobFile(soongDocsTag),
260 }
261}
262
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200263func bootstrapBlueprint(ctx Context, config Config) {
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100264 ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
265 defer ctx.EndTrace()
266
Colin Cross9191df22021-10-29 13:11:32 -0700267 // Clean up some files for incremental builds across incompatible changes.
268 bootstrapEpochCleanup(ctx, config)
269
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900270 baseArgs := []string{"--soong_variables", config.SoongVarsFile()}
271
272 mainSoongBuildExtraArgs := append(baseArgs, "-o", config.SoongNinjaFile())
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200273 if config.EmptyNinjaFile() {
274 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
Lukacs T. Berki745380c2021-04-12 12:07:44 +0200275 }
Chris Parsonsef615e52022-08-18 22:04:11 -0400276 if config.bazelProdMode {
277 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode")
278 }
MarkDacekb78465d2022-10-18 20:10:16 +0000279 if config.bazelStagingMode {
280 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging")
281 }
Chris Parsons9402ca82023-02-23 17:28:06 -0500282 if config.IsPersistentBazelEnabled() {
283 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--use-bazel-proxy")
284 }
MarkDacekd06db5d2022-11-29 00:47:59 +0000285 if len(config.bazelForceEnabledModules) > 0 {
286 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-force-enabled-modules="+config.bazelForceEnabledModules)
287 }
LaMont Jones52a72432023-03-09 18:19:35 +0000288 if config.MultitreeBuild() {
289 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--multitree-build")
290 }
Jihoon Kang1bff0342023-01-17 20:40:22 +0000291 if config.buildFromTextStub {
292 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--build-from-text-stub")
293 }
MarkDacekf47e1422023-04-19 16:47:36 +0000294 if config.ensureAllowlistIntegrity {
295 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--ensure-allowlist-integrity")
296 }
MarkDacekd06db5d2022-11-29 00:47:59 +0000297
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000298 queryviewDir := filepath.Join(config.SoongOutDir(), "queryview")
Spandan Das5af0bd32022-09-28 20:43:08 +0000299 // The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files)
300 // The final workspace will be generated in out/soong/api_bp2build
301 apiBp2buildDir := filepath.Join(config.SoongOutDir(), ".api_bp2build")
Spandan Das5af0bd32022-09-28 20:43:08 +0000302
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800303 pbfs := []PrimaryBuilderFactory{
304 {
305 name: soongBuildTag,
306 description: fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()),
307 config: config,
308 output: config.SoongNinjaFile(),
309 specificArgs: mainSoongBuildExtraArgs,
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000310 },
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800311 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900312 name: bp2buildFilesTag,
313 description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
314 config: config,
315 output: config.Bp2BuildFilesMarkerFile(),
316 specificArgs: append(baseArgs,
317 "--bp2build_marker", config.Bp2BuildFilesMarkerFile(),
318 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800319 },
320 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900321 name: bp2buildWorkspaceTag,
322 description: "Creating Bazel symlink forest",
323 config: config,
324 output: config.Bp2BuildWorkspaceMarkerFile(),
325 specificArgs: append(baseArgs,
326 "--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile(),
327 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800328 },
329 {
330 name: jsonModuleGraphTag,
331 description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
332 config: config,
333 output: config.ModuleGraphFile(),
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900334 specificArgs: append(baseArgs,
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800335 "--module_graph_file", config.ModuleGraphFile(),
336 "--module_actions_file", config.ModuleActionsFile(),
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900337 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800338 },
339 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900340 name: queryviewTag,
341 description: fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir),
342 config: config,
343 output: config.QueryviewMarkerFile(),
344 specificArgs: append(baseArgs,
345 "--bazel_queryview_dir", queryviewDir,
346 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800347 },
348 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900349 name: apiBp2buildTag,
350 description: fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir),
351 config: config,
352 output: config.ApiBp2buildMarkerFile(),
353 specificArgs: append(baseArgs,
354 "--bazel_api_bp2build_dir", apiBp2buildDir,
355 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800356 },
357 {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +0900358 name: soongDocsTag,
359 description: fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()),
360 config: config,
361 output: config.SoongDocsHtml(),
362 specificArgs: append(baseArgs,
363 "--soong_docs", config.SoongDocsHtml(),
364 ),
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800365 },
366 }
367
368 // Figure out which invocations will be run under the debugger:
369 // * SOONG_DELVE if set specifies listening port
370 // * SOONG_DELVE_STEPS if set specifies specific invocations to be debugged, otherwise all are
371 debuggedInvocations := make(map[string]bool)
372 delvePort := os.Getenv("SOONG_DELVE")
373 if delvePort != "" {
374 if steps := os.Getenv("SOONG_DELVE_STEPS"); steps != "" {
375 var validSteps []string
376 for _, pbf := range pbfs {
377 debuggedInvocations[pbf.name] = false
378 validSteps = append(validSteps, pbf.name)
379
380 }
381 for _, step := range strings.Split(steps, ",") {
382 if _, ok := debuggedInvocations[step]; ok {
383 debuggedInvocations[step] = true
384 } else {
385 ctx.Fatalf("SOONG_DELVE_STEPS contains unknown soong_build step %s\n"+
386 "Valid steps are %v", step, validSteps)
387 }
388 }
389 } else {
390 // SOONG_DELVE_STEPS is not set, run all steps in the debugger
391 for _, pbf := range pbfs {
392 debuggedInvocations[pbf.name] = true
393 }
394 }
395 }
396
397 var invocations []bootstrap.PrimaryBuilderInvocation
398 for _, pbf := range pbfs {
399 if debuggedInvocations[pbf.name] {
400 pbf.debugPort = delvePort
401 }
402 pbi := pbf.primaryBuilderInvocation()
403 // Some invocations require adjustment:
404 switch pbf.name {
405 case soongBuildTag:
406 if config.BazelBuildEnabled() {
407 // Mixed builds call Bazel from soong_build and they therefore need the
408 // Bazel workspace to be available. Make that so by adding a dependency on
409 // the bp2build marker file to the action that invokes soong_build .
410 pbi.OrderOnlyInputs = append(pbi.OrderOnlyInputs, config.Bp2BuildWorkspaceMarkerFile())
411 }
412 case bp2buildWorkspaceTag:
413 pbi.Inputs = append(pbi.Inputs,
414 config.Bp2BuildFilesMarkerFile(),
415 filepath.Join(config.FileListDir(), "bazel.list"))
Wei Li2c9e8d62023-05-05 01:07:15 -0700416 case bp2buildFilesTag:
417 pbi.Inputs = append(pbi.Inputs, filepath.Join(config.FileListDir(), "METADATA.list"))
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800418 }
419 invocations = append(invocations, pbi)
420 }
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200421
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200422 // The glob .ninja files are subninja'd. However, they are generated during
423 // the build itself so we write an empty file if the file does not exist yet
424 // so that the subninja doesn't fail on clean builds
Colin Cross9191df22021-10-29 13:11:32 -0700425 for _, globFile := range bootstrapGlobFileList(config) {
426 writeEmptyFile(ctx, globFile)
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200427 }
428
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800429 blueprintArgs := bootstrap.Args{
430 ModuleListFile: filepath.Join(config.FileListDir(), "Android.bp.list"),
431 OutFile: shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja"),
432 EmptyNinjaFile: false,
433 }
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200434
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100435 blueprintCtx := blueprint.NewContext()
Spandan Dasc5763832022-11-08 18:42:16 +0000436 blueprintCtx.AddIncludeTags(config.GetIncludeTags()...)
Sam Delmerico98a73292023-02-21 11:50:29 -0500437 blueprintCtx.AddSourceRootDirs(config.GetSourceRootDirs()...)
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100438 blueprintCtx.SetIgnoreUnknownModuleTypes(true)
439 blueprintConfig := BlueprintConfig{
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200440 soongOutDir: config.SoongOutDir(),
441 toolDir: config.HostToolDir(),
442 outDir: config.OutDir(),
443 runGoTests: !config.skipSoongTests,
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200444 // If we want to debug soong_build, we need to compile it for debugging
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800445 debugCompilation: delvePort != "",
446 subninjas: bootstrapGlobFileList(config),
447 primaryBuilderInvocations: invocations,
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100448 }
449
usta5bb4a5d2022-08-24 12:53:46 -0400450 // since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little
451 // reason to write a `bootstrap.ninja.d` file
Lukacs T. Berkic357c812023-06-20 09:30:06 +0000452 _, err := bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
453 if err != nil {
454 ctx.Fatal(err)
455 }
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100456}
457
Jason Wu2520f5e2023-05-30 19:45:36 -0400458func checkEnvironmentFile(ctx Context, currentEnv *Environment, envFile string) {
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200459 getenv := func(k string) string {
460 v, _ := currentEnv.Get(k)
461 return v
462 }
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200463
Jason Wu2520f5e2023-05-30 19:45:36 -0400464 // Log the changed environment variables to ChangedEnvironmentVariable field
465 if stale, changedEnvironmentVariableList, _ := shared.StaleEnvFile(envFile, getenv); stale {
466 for _, changedEnvironmentVariable := range changedEnvironmentVariableList {
467 ctx.Metrics.AddChangedEnvironmentVariable(changedEnvironmentVariable)
468 }
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200469 os.Remove(envFile)
470 }
471}
472
Dan Willemsen1e704462016-08-21 15:17:17 -0700473func runSoong(ctx Context, config Config) {
Nan Zhang17f27672018-12-12 16:01:49 -0800474 ctx.BeginTrace(metrics.RunSoong, "soong")
Dan Willemsend9f6fa22016-08-21 15:17:17 -0700475 defer ctx.EndTrace()
476
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100477 // We have two environment files: .available is the one with every variable,
478 // .used with the ones that were actually used. The latter is used to
479 // determine whether Soong needs to be re-run since why re-run it if only
480 // unused variables were changed?
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200481 envFile := filepath.Join(config.SoongOutDir(), availableEnvFile)
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100482
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100483 // This is done unconditionally, but does not take a measurable amount of time
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200484 bootstrapBlueprint(ctx, config)
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700485
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100486 soongBuildEnv := config.Environment().Copy()
487 soongBuildEnv.Set("TOP", os.Getenv("TOP"))
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100488 // For Bazel mixed builds.
Joe Onoratoba29f382022-10-24 06:38:11 -0700489 soongBuildEnv.Set("BAZEL_PATH", "./build/bazel/bin/bazel")
Chris Parsonsa9bef142022-09-28 15:07:46 -0400490 // Bazel's HOME var is set to an output subdirectory which doesn't exist. This
491 // prevents Bazel from file I/O in the actual user HOME directory.
492 soongBuildEnv.Set("BAZEL_HOME", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazelhome")))
Liz Kammer2af5ea82022-11-11 14:21:03 -0500493 soongBuildEnv.Set("BAZEL_OUTPUT_BASE", config.bazelOutputBase())
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100494 soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
495 soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -0500496 soongBuildEnv.Set("LOG_DIR", config.LogsDir())
Jingwen Chen3b13b612022-10-17 12:14:26 +0000497 soongBuildEnv.Set("BAZEL_DEPS_FILE", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazel.list")))
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100498
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100499 // For Soong bootstrapping tests
500 if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
501 soongBuildEnv.Set("ALLOW_MISSING_DEPENDENCIES", "true")
502 }
503
Paul Duffin5e85c662021-03-05 12:26:14 +0000504 err := writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap())
505 if err != nil {
506 ctx.Fatalf("failed to write environment file %s: %s", envFile, err)
507 }
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100508
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700509 func() {
Nan Zhang17f27672018-12-12 16:01:49 -0800510 ctx.BeginTrace(metrics.RunSoong, "environment check")
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700511 defer ctx.EndTrace()
512
Jason Wu2520f5e2023-05-30 19:45:36 -0400513 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(soongBuildTag))
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200514
Chris Parsonsef615e52022-08-18 22:04:11 -0400515 if config.BazelBuildEnabled() || config.Bp2Build() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400516 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(bp2buildFilesTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200517 }
518
519 if config.JsonModuleGraph() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400520 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(jsonModuleGraphTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200521 }
522
523 if config.Queryview() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400524 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(queryviewTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200525 }
526
Spandan Das5af0bd32022-09-28 20:43:08 +0000527 if config.ApiBp2build() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400528 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(apiBp2buildTag))
Spandan Das5af0bd32022-09-28 20:43:08 +0000529 }
530
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200531 if config.SoongDocs() {
Jason Wu2520f5e2023-05-30 19:45:36 -0400532 checkEnvironmentFile(ctx, soongBuildEnv, config.UsedEnvFile(soongDocsTag))
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700533 }
534 }()
535
Colin Cross9191df22021-10-29 13:11:32 -0700536 runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob",
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700537 map[string]string{"github.com/google/blueprint": "build/blueprint"})
Dan Willemsen5af1cbe2018-07-05 21:46:51 -0700538
usta49012ee2023-05-22 16:33:27 -0400539 ninja := func(targets ...string) {
540 ctx.BeginTrace(metrics.RunSoong, "bootstrap")
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700541 defer ctx.EndTrace()
542
Chris Parsons9402ca82023-02-23 17:28:06 -0500543 if config.IsPersistentBazelEnabled() {
Chris Parsonsc83398f2023-05-31 18:41:41 +0000544 bazelProxy := bazel.NewProxyServer(ctx.Logger, config.OutDir(), filepath.Join(config.SoongOutDir(), "workspace"), config.GetBazeliskBazelVersion())
usta49012ee2023-05-22 16:33:27 -0400545 if err := bazelProxy.Start(); err != nil {
546 ctx.Fatalf("Failed to create bazel proxy")
547 }
Chris Parsons9402ca82023-02-23 17:28:06 -0500548 defer bazelProxy.Close()
549 }
550
Dan Willemsenb82471a2018-05-17 16:37:09 -0700551 fifo := filepath.Join(config.OutDir(), ".ninja_fifo")
Colin Crossb98d3bc2019-03-21 16:02:58 -0700552 nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
553 defer nr.Close()
Dan Willemsenb82471a2018-05-17 16:37:09 -0700554
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200555 ninjaArgs := []string{
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700556 "-d", "keepdepfile",
Dan Willemsen08218222020-05-18 14:02:02 -0700557 "-d", "stats",
Dan Willemsen6587bed2020-04-18 20:25:59 -0700558 "-o", "usesphonyoutputs=yes",
559 "-o", "preremoveoutputs=yes",
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700560 "-w", "dupbuild=err",
Dan Willemsen6587bed2020-04-18 20:25:59 -0700561 "-w", "outputdir=err",
562 "-w", "missingoutfile=err",
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700563 "-j", strconv.Itoa(config.Parallel()),
Dan Willemsen02736672018-07-17 17:54:31 -0700564 "--frontend_file", fifo,
usta49012ee2023-05-22 16:33:27 -0400565 "-f", filepath.Join(config.SoongOutDir(), "bootstrap.ninja"),
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200566 }
567
Usta Shrestha8dc8b0a2022-08-10 17:39:37 -0400568 if extra, ok := config.Environment().Get("SOONG_UI_NINJA_ARGS"); ok {
569 ctx.Printf(`CAUTION: arguments in $SOONG_UI_NINJA_ARGS=%q, e.g. "-n", can make soong_build FAIL or INCORRECT`, extra)
570 ninjaArgs = append(ninjaArgs, strings.Fields(extra)...)
571 }
572
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200573 ninjaArgs = append(ninjaArgs, targets...)
usta49012ee2023-05-22 16:33:27 -0400574 cmd := Command(ctx, config, "soong bootstrap",
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200575 config.PrebuiltBuildTool("ninja"), ninjaArgs...)
Jingwen Chen7c6089a2020-11-02 02:56:20 -0500576
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100577 var ninjaEnv Environment
Lukacs T. Berki73ab9282021-03-10 10:48:39 +0100578
579 // This is currently how the command line to invoke soong_build finds the
580 // root of the source tree and the output root
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100581 ninjaEnv.Set("TOP", os.Getenv("TOP"))
Lukacs T. Berki7d613bf2021-03-02 10:09:41 +0100582
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100583 cmd.Environment = &ninjaEnv
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700584 cmd.Sandbox = soongSandbox
Colin Cross7b97ecd2019-06-19 13:17:59 -0700585 cmd.RunAndStreamOrFatal()
Dan Willemsen1e704462016-08-21 15:17:17 -0700586 }
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200587
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200588 targets := make([]string, 0, 0)
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200589
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200590 if config.JsonModuleGraph() {
591 targets = append(targets, config.ModuleGraphFile())
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200592 }
593
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200594 if config.Bp2Build() {
Lukacs T. Berkic541cd22022-10-26 07:26:50 +0000595 targets = append(targets, config.Bp2BuildWorkspaceMarkerFile())
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200596 }
597
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200598 if config.Queryview() {
599 targets = append(targets, config.QueryviewMarkerFile())
600 }
601
Spandan Das5af0bd32022-09-28 20:43:08 +0000602 if config.ApiBp2build() {
603 targets = append(targets, config.ApiBp2buildMarkerFile())
604 }
605
Lukacs T. Berkic6012f32021-09-06 18:31:46 +0200606 if config.SoongDocs() {
607 targets = append(targets, config.SoongDocsHtml())
608 }
609
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200610 if config.SoongBuildInvocationNeeded() {
611 // This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
Cole Faust65f298c2021-10-28 16:05:13 -0700612 targets = append(targets, config.SoongNinjaFile())
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200613 }
614
usta49012ee2023-05-22 16:33:27 -0400615 ninja(targets...)
Colin Crossb72c9092020-02-10 11:23:49 -0800616
Colin Cross8ba7d472020-06-25 11:27:52 -0700617 distGzipFile(ctx, config, config.SoongNinjaFile(), "soong")
Jihoon Kang9f4f8a32022-08-16 00:57:30 +0000618 distFile(ctx, config, config.SoongVarsFile(), "soong")
Colin Cross8ba7d472020-06-25 11:27:52 -0700619
Anton Hansson5e5c48b2020-11-27 12:35:20 +0000620 if !config.SkipKati() {
Colin Cross8ba7d472020-06-25 11:27:52 -0700621 distGzipFile(ctx, config, config.SoongAndroidMk(), "soong")
622 distGzipFile(ctx, config, config.SoongMakeVarsMk(), "soong")
623 }
624
Rob Seymour33cd10d2022-03-02 23:10:25 +0000625 if config.JsonModuleGraph() {
626 distGzipFile(ctx, config, config.ModuleGraphFile(), "soong")
627 }
Colin Crossb72c9092020-02-10 11:23:49 -0800628}
629
Lukacs T. Berki90b43342021-11-02 14:42:04 +0100630func runMicrofactory(ctx Context, config Config, name string, pkg string, mapping map[string]string) {
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700631 ctx.BeginTrace(metrics.RunSoong, name)
632 defer ctx.EndTrace()
633 cfg := microfactory.Config{TrimPath: absPath(ctx, ".")}
634 for pkgPrefix, pathPrefix := range mapping {
635 cfg.Map(pkgPrefix, pathPrefix)
636 }
637
Lukacs T. Berki90b43342021-11-02 14:42:04 +0100638 exePath := filepath.Join(config.SoongOutDir(), name)
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700639 dir := filepath.Dir(exePath)
640 if err := os.MkdirAll(dir, 0777); err != nil {
641 ctx.Fatalf("cannot create %s: %s", dir, err)
642 }
643 if _, err := microfactory.Build(&cfg, exePath, pkg); err != nil {
644 ctx.Fatalf("failed to build %s: %s", name, err)
645 }
646}