blob: de3179a3d3ccaec1a0f19c9d16ae496cb78a89b9 [file] [log] [blame]
Dan Willemsen1e704462016-08-21 15:17:17 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
Dan Willemsende360442022-04-20 21:21:55 -070018 "errors"
Colin Cross9191df22021-10-29 13:11:32 -070019 "fmt"
Dan Willemsende360442022-04-20 21:21:55 -070020 "io/fs"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070021 "os"
Dan Willemsen1e704462016-08-21 15:17:17 -070022 "path/filepath"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070023 "strconv"
Usta Shrestha8dc8b0a2022-08-10 17:39:37 -040024 "strings"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070025
Dan Willemsen4591b642021-05-24 14:24:12 -070026 "android/soong/ui/metrics"
Colin Crossb72c9092020-02-10 11:23:49 -080027 soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
Dan Willemsen4591b642021-05-24 14:24:12 -070028 "android/soong/ui/status"
29
30 "android/soong/shared"
31
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010032 "github.com/google/blueprint"
33 "github.com/google/blueprint/bootstrap"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070034 "github.com/google/blueprint/microfactory"
Dan Willemsenb82471a2018-05-17 16:37:09 -070035
Dan Willemsen4591b642021-05-24 14:24:12 -070036 "google.golang.org/protobuf/proto"
Dan Willemsen1e704462016-08-21 15:17:17 -070037)
38
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020039const (
40 availableEnvFile = "soong.environment.available"
41 usedEnvFile = "soong.environment.used"
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +020042
Lukacs T. Berkic541cd22022-10-26 07:26:50 +000043 soongBuildTag = "build"
44 bp2buildFilesTag = "bp2build_files"
45 bp2buildWorkspaceTag = "bp2build_workspace"
46 jsonModuleGraphTag = "modulegraph"
47 queryviewTag = "queryview"
48 apiBp2buildTag = "api_bp2build"
49 soongDocsTag = "soong_docs"
Colin Cross9191df22021-10-29 13:11:32 -070050
51 // bootstrapEpoch is used to determine if an incremental build is incompatible with the current
52 // version of bootstrap and needs cleaning before continuing the build. Increment this for
53 // incompatible changes, for example when moving the location of the bpglob binary that is
54 // executed during bootstrap before the primary builder has had a chance to update the path.
Lukacs T. Berki9985d9a2021-11-04 11:47:42 +010055 bootstrapEpoch = 1
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020056)
57
Sasha Smundak4cbe83a2022-11-28 17:02:40 -080058func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error {
Lukacs T. Berki7690c092021-02-26 14:27:36 +010059 data, err := shared.EnvFileContents(envDeps)
60 if err != nil {
61 return err
62 }
63
Sasha Smundak4cbe83a2022-11-28 17:02:40 -080064 return os.WriteFile(envFile, data, 0644)
Lukacs T. Berki7690c092021-02-26 14:27:36 +010065}
66
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000067// This uses Android.bp files and various tools to generate <builddir>/build.ninja.
68//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010069// However, the execution of <builddir>/build.ninja happens later in
70// build/soong/ui/build/build.go#Build()
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000071//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010072// We want to rely on as few prebuilts as possible, so we need to bootstrap
73// Soong. The process is as follows:
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000074//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010075// 1. We use "Microfactory", a simple tool to compile Go code, to build
76// first itself, then soong_ui from soong_ui.bash. This binary contains
77// parts of soong_build that are needed to build itself.
78// 2. This simplified version of soong_build then reads the Blueprint files
79// that describe itself and emits .bootstrap/build.ninja that describes
80// how to build its full version and use that to produce the final Ninja
81// file Soong emits.
82// 3. soong_ui executes .bootstrap/build.ninja
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000083//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010084// (After this, Kati is executed to parse the Makefiles, but that's not part of
85// bootstrapping Soong)
86
87// A tiny struct used to tell Blueprint that it's in bootstrap mode. It would
88// probably be nicer to use a flag in bootstrap.Args instead.
89type BlueprintConfig struct {
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +020090 toolDir string
91 soongOutDir string
92 outDir string
93 runGoTests bool
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +020094 debugCompilation bool
95 subninjas []string
96 primaryBuilderInvocations []bootstrap.PrimaryBuilderInvocation
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010097}
98
Lukacs T. Berkia806e412021-09-01 08:57:48 +020099func (c BlueprintConfig) HostToolDir() string {
100 return c.toolDir
101}
102
Lukacs T. Berki9f6c24a2021-08-26 15:07:24 +0200103func (c BlueprintConfig) SoongOutDir() string {
104 return c.soongOutDir
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100105}
106
Lukacs T. Berki9f6c24a2021-08-26 15:07:24 +0200107func (c BlueprintConfig) OutDir() string {
108 return c.outDir
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100109}
110
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +0200111func (c BlueprintConfig) RunGoTests() bool {
112 return c.runGoTests
113}
114
Lukacs T. Berki5f6cb1d2021-03-17 15:03:14 +0100115func (c BlueprintConfig) DebugCompilation() bool {
116 return c.debugCompilation
117}
118
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +0200119func (c BlueprintConfig) Subninjas() []string {
120 return c.subninjas
121}
122
123func (c BlueprintConfig) PrimaryBuilderInvocations() []bootstrap.PrimaryBuilderInvocation {
124 return c.primaryBuilderInvocations
125}
126
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200127func environmentArgs(config Config, tag string) []string {
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200128 return []string{
129 "--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200130 "--used_env", config.UsedEnvFile(tag),
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200131 }
132}
Spandan Das8f99ae62021-06-11 16:48:06 +0000133
Colin Cross9191df22021-10-29 13:11:32 -0700134func writeEmptyFile(ctx Context, path string) {
Spandan Das8f99ae62021-06-11 16:48:06 +0000135 err := os.MkdirAll(filepath.Dir(path), 0777)
136 if err != nil {
Colin Cross9191df22021-10-29 13:11:32 -0700137 ctx.Fatalf("Failed to create parent directories of empty file '%s': %s", path, err)
Spandan Das8f99ae62021-06-11 16:48:06 +0000138 }
139
Colin Cross9191df22021-10-29 13:11:32 -0700140 if exists, err := fileExists(path); err != nil {
141 ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err)
142 } else if !exists {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800143 err = os.WriteFile(path, nil, 0666)
Spandan Das8f99ae62021-06-11 16:48:06 +0000144 if err != nil {
Colin Cross9191df22021-10-29 13:11:32 -0700145 ctx.Fatalf("Failed to create empty file '%s': %s", path, err)
Spandan Das8f99ae62021-06-11 16:48:06 +0000146 }
147 }
148}
149
Colin Cross9191df22021-10-29 13:11:32 -0700150func fileExists(path string) (bool, error) {
151 if _, err := os.Stat(path); os.IsNotExist(err) {
152 return false, nil
153 } else if err != nil {
154 return false, err
155 }
156 return true, nil
157}
158
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800159type PrimaryBuilderFactory struct {
160 name string
161 description string
162 config Config
163 output string
164 specificArgs []string
165 debugPort string
166}
167
168func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuilderInvocation {
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200169 commonArgs := make([]string, 0, 0)
170
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800171 if !pb.config.skipSoongTests {
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200172 commonArgs = append(commonArgs, "-t")
173 }
174
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800175 commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list"))
Lukacs T. Berki13644272022-01-05 10:29:56 +0100176 invocationEnv := make(map[string]string)
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800177 if pb.debugPort != "" {
ustafb67fd12022-08-19 19:26:00 -0400178 //debug mode
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800179 commonArgs = append(commonArgs, "--delve_listen", pb.debugPort,
180 "--delve_path", shared.ResolveDelveBinary())
Lukacs T. Berki13644272022-01-05 10:29:56 +0100181 // GODEBUG=asyncpreemptoff=1 disables the preemption of goroutines. This
182 // is useful because the preemption happens by sending SIGURG to the OS
183 // thread hosting the goroutine in question and each signal results in
184 // work that needs to be done by Delve; it uses ptrace to debug the Go
185 // process and the tracer process must deal with every signal (it is not
186 // possible to selectively ignore SIGURG). This makes debugging slower,
187 // sometimes by an order of magnitude depending on luck.
188 // The original reason for adding async preemption to Go is here:
189 // https://github.com/golang/proposal/blob/master/design/24543-non-cooperative-preemption.md
190 invocationEnv["GODEBUG"] = "asyncpreemptoff=1"
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200191 }
192
ustafb67fd12022-08-19 19:26:00 -0400193 var allArgs []string
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800194 allArgs = append(allArgs, pb.specificArgs...)
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200195 allArgs = append(allArgs,
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800196 "--globListDir", pb.name,
197 "--globFile", pb.config.NamedGlobFile(pb.name))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200198
199 allArgs = append(allArgs, commonArgs...)
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800200 allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800201 if profileCpu := os.Getenv("SOONG_PROFILE_CPU"); profileCpu != "" {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800202 allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+pb.name)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800203 }
204 if profileMem := os.Getenv("SOONG_PROFILE_MEM"); profileMem != "" {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800205 allArgs = append(allArgs, "--memprofile", profileMem+"."+pb.name)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800206 }
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200207 allArgs = append(allArgs, "Android.bp")
208
209 return bootstrap.PrimaryBuilderInvocation{
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000210 Inputs: []string{"Android.bp"},
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800211 Outputs: []string{pb.output},
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000212 Args: allArgs,
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800213 Description: pb.description,
Lukacs T. Berki2fad3412022-01-04 14:40:13 +0100214 // NB: Changing the value of this environment variable will not result in a
215 // rebuild. The bootstrap Ninja file will change, but apparently Ninja does
216 // not consider changing the pool specified in a statement a change that's
217 // worth rebuilding for.
218 Console: os.Getenv("SOONG_UNBUFFERED_OUTPUT") == "1",
Lukacs T. Berki13644272022-01-05 10:29:56 +0100219 Env: invocationEnv,
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200220 }
221}
222
Colin Cross9191df22021-10-29 13:11:32 -0700223// bootstrapEpochCleanup deletes files used by bootstrap during incremental builds across
224// incompatible changes. Incompatible changes are marked by incrementing the bootstrapEpoch
225// constant. A tree is considered out of date for the current epoch of the
226// .soong.bootstrap.epoch.<epoch> file doesn't exist.
227func bootstrapEpochCleanup(ctx Context, config Config) {
228 epochFile := fmt.Sprintf(".soong.bootstrap.epoch.%d", bootstrapEpoch)
229 epochPath := filepath.Join(config.SoongOutDir(), epochFile)
230 if exists, err := fileExists(epochPath); err != nil {
231 ctx.Fatalf("failed to check if bootstrap epoch file %q exists: %q", epochPath, err)
232 } else if !exists {
233 // The tree is out of date for the current epoch, delete files used by bootstrap
234 // and force the primary builder to rerun.
235 os.Remove(filepath.Join(config.SoongOutDir(), "build.ninja"))
236 for _, globFile := range bootstrapGlobFileList(config) {
237 os.Remove(globFile)
238 }
239
240 // Mark the tree as up to date with the current epoch by writing the epoch marker file.
241 writeEmptyFile(ctx, epochPath)
242 }
243}
244
245func bootstrapGlobFileList(config Config) []string {
246 return []string{
247 config.NamedGlobFile(soongBuildTag),
Lukacs T. Berkic541cd22022-10-26 07:26:50 +0000248 config.NamedGlobFile(bp2buildFilesTag),
Colin Cross9191df22021-10-29 13:11:32 -0700249 config.NamedGlobFile(jsonModuleGraphTag),
250 config.NamedGlobFile(queryviewTag),
Spandan Das5af0bd32022-09-28 20:43:08 +0000251 config.NamedGlobFile(apiBp2buildTag),
Colin Cross9191df22021-10-29 13:11:32 -0700252 config.NamedGlobFile(soongDocsTag),
253 }
254}
255
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200256func bootstrapBlueprint(ctx Context, config Config) {
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100257 ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
258 defer ctx.EndTrace()
259
Colin Cross9191df22021-10-29 13:11:32 -0700260 // Clean up some files for incremental builds across incompatible changes.
261 bootstrapEpochCleanup(ctx, config)
262
Cole Faust65f298c2021-10-28 16:05:13 -0700263 mainSoongBuildExtraArgs := []string{"-o", config.SoongNinjaFile()}
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200264 if config.EmptyNinjaFile() {
265 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
Lukacs T. Berki745380c2021-04-12 12:07:44 +0200266 }
Chris Parsonsef615e52022-08-18 22:04:11 -0400267 if config.bazelProdMode {
268 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode")
269 }
270 if config.bazelDevMode {
271 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-dev")
272 }
MarkDacekb78465d2022-10-18 20:10:16 +0000273 if config.bazelStagingMode {
274 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging")
275 }
MarkDacekd06db5d2022-11-29 00:47:59 +0000276 if len(config.bazelForceEnabledModules) > 0 {
277 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-force-enabled-modules="+config.bazelForceEnabledModules)
278 }
279
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000280 queryviewDir := filepath.Join(config.SoongOutDir(), "queryview")
Spandan Das5af0bd32022-09-28 20:43:08 +0000281 // The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files)
282 // The final workspace will be generated in out/soong/api_bp2build
283 apiBp2buildDir := filepath.Join(config.SoongOutDir(), ".api_bp2build")
Spandan Das5af0bd32022-09-28 20:43:08 +0000284
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800285 pbfs := []PrimaryBuilderFactory{
286 {
287 name: soongBuildTag,
288 description: fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()),
289 config: config,
290 output: config.SoongNinjaFile(),
291 specificArgs: mainSoongBuildExtraArgs,
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000292 },
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800293 {
294 name: bp2buildFilesTag,
295 description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
296 config: config,
297 output: config.Bp2BuildFilesMarkerFile(),
298 specificArgs: []string{"--bp2build_marker", config.Bp2BuildFilesMarkerFile()},
299 },
300 {
301 name: bp2buildWorkspaceTag,
302 description: "Creating Bazel symlink forest",
303 config: config,
304 output: config.Bp2BuildWorkspaceMarkerFile(),
305 specificArgs: []string{"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile()},
306 },
307 {
308 name: jsonModuleGraphTag,
309 description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
310 config: config,
311 output: config.ModuleGraphFile(),
312 specificArgs: []string{
313 "--module_graph_file", config.ModuleGraphFile(),
314 "--module_actions_file", config.ModuleActionsFile(),
315 },
316 },
317 {
318 name: queryviewTag,
319 description: fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir),
320 config: config,
321 output: config.QueryviewMarkerFile(),
322 specificArgs: []string{"--bazel_queryview_dir", queryviewDir},
323 },
324 {
325 name: apiBp2buildTag,
326 description: fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir),
327 config: config,
328 output: config.ApiBp2buildMarkerFile(),
329 specificArgs: []string{"--bazel_api_bp2build_dir", apiBp2buildDir},
330 },
331 {
332 name: soongDocsTag,
333 description: fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()),
334 config: config,
335 output: config.SoongDocsHtml(),
336 specificArgs: []string{"--soong_docs", config.SoongDocsHtml()},
337 },
338 }
339
340 // Figure out which invocations will be run under the debugger:
341 // * SOONG_DELVE if set specifies listening port
342 // * SOONG_DELVE_STEPS if set specifies specific invocations to be debugged, otherwise all are
343 debuggedInvocations := make(map[string]bool)
344 delvePort := os.Getenv("SOONG_DELVE")
345 if delvePort != "" {
346 if steps := os.Getenv("SOONG_DELVE_STEPS"); steps != "" {
347 var validSteps []string
348 for _, pbf := range pbfs {
349 debuggedInvocations[pbf.name] = false
350 validSteps = append(validSteps, pbf.name)
351
352 }
353 for _, step := range strings.Split(steps, ",") {
354 if _, ok := debuggedInvocations[step]; ok {
355 debuggedInvocations[step] = true
356 } else {
357 ctx.Fatalf("SOONG_DELVE_STEPS contains unknown soong_build step %s\n"+
358 "Valid steps are %v", step, validSteps)
359 }
360 }
361 } else {
362 // SOONG_DELVE_STEPS is not set, run all steps in the debugger
363 for _, pbf := range pbfs {
364 debuggedInvocations[pbf.name] = true
365 }
366 }
367 }
368
369 var invocations []bootstrap.PrimaryBuilderInvocation
370 for _, pbf := range pbfs {
371 if debuggedInvocations[pbf.name] {
372 pbf.debugPort = delvePort
373 }
374 pbi := pbf.primaryBuilderInvocation()
375 // Some invocations require adjustment:
376 switch pbf.name {
377 case soongBuildTag:
378 if config.BazelBuildEnabled() {
379 // Mixed builds call Bazel from soong_build and they therefore need the
380 // Bazel workspace to be available. Make that so by adding a dependency on
381 // the bp2build marker file to the action that invokes soong_build .
382 pbi.OrderOnlyInputs = append(pbi.OrderOnlyInputs, config.Bp2BuildWorkspaceMarkerFile())
383 }
384 case bp2buildWorkspaceTag:
385 pbi.Inputs = append(pbi.Inputs,
386 config.Bp2BuildFilesMarkerFile(),
387 filepath.Join(config.FileListDir(), "bazel.list"))
388 }
389 invocations = append(invocations, pbi)
390 }
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200391
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200392 // The glob .ninja files are subninja'd. However, they are generated during
393 // the build itself so we write an empty file if the file does not exist yet
394 // so that the subninja doesn't fail on clean builds
Colin Cross9191df22021-10-29 13:11:32 -0700395 for _, globFile := range bootstrapGlobFileList(config) {
396 writeEmptyFile(ctx, globFile)
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200397 }
398
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800399 blueprintArgs := bootstrap.Args{
400 ModuleListFile: filepath.Join(config.FileListDir(), "Android.bp.list"),
401 OutFile: shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja"),
402 EmptyNinjaFile: false,
403 }
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200404
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100405 blueprintCtx := blueprint.NewContext()
406 blueprintCtx.SetIgnoreUnknownModuleTypes(true)
407 blueprintConfig := BlueprintConfig{
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200408 soongOutDir: config.SoongOutDir(),
409 toolDir: config.HostToolDir(),
410 outDir: config.OutDir(),
411 runGoTests: !config.skipSoongTests,
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200412 // If we want to debug soong_build, we need to compile it for debugging
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800413 debugCompilation: delvePort != "",
414 subninjas: bootstrapGlobFileList(config),
415 primaryBuilderInvocations: invocations,
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100416 }
417
usta5bb4a5d2022-08-24 12:53:46 -0400418 // since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little
419 // reason to write a `bootstrap.ninja.d` file
420 _ = bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100421}
422
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200423func checkEnvironmentFile(currentEnv *Environment, envFile string) {
424 getenv := func(k string) string {
425 v, _ := currentEnv.Get(k)
426 return v
427 }
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200428
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200429 if stale, _ := shared.StaleEnvFile(envFile, getenv); stale {
430 os.Remove(envFile)
431 }
432}
433
Dan Willemsen1e704462016-08-21 15:17:17 -0700434func runSoong(ctx Context, config Config) {
Nan Zhang17f27672018-12-12 16:01:49 -0800435 ctx.BeginTrace(metrics.RunSoong, "soong")
Dan Willemsend9f6fa22016-08-21 15:17:17 -0700436 defer ctx.EndTrace()
437
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100438 // We have two environment files: .available is the one with every variable,
439 // .used with the ones that were actually used. The latter is used to
440 // determine whether Soong needs to be re-run since why re-run it if only
441 // unused variables were changed?
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200442 envFile := filepath.Join(config.SoongOutDir(), availableEnvFile)
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100443
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100444 // This is done unconditionally, but does not take a measurable amount of time
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200445 bootstrapBlueprint(ctx, config)
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700446
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100447 soongBuildEnv := config.Environment().Copy()
448 soongBuildEnv.Set("TOP", os.Getenv("TOP"))
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100449 // For Bazel mixed builds.
Joe Onoratoba29f382022-10-24 06:38:11 -0700450 soongBuildEnv.Set("BAZEL_PATH", "./build/bazel/bin/bazel")
Chris Parsonsa9bef142022-09-28 15:07:46 -0400451 // Bazel's HOME var is set to an output subdirectory which doesn't exist. This
452 // prevents Bazel from file I/O in the actual user HOME directory.
453 soongBuildEnv.Set("BAZEL_HOME", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazelhome")))
Liz Kammer2af5ea82022-11-11 14:21:03 -0500454 soongBuildEnv.Set("BAZEL_OUTPUT_BASE", config.bazelOutputBase())
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100455 soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
456 soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -0500457 soongBuildEnv.Set("LOG_DIR", config.LogsDir())
Jingwen Chen3b13b612022-10-17 12:14:26 +0000458 soongBuildEnv.Set("BAZEL_DEPS_FILE", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazel.list")))
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100459
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100460 // For Soong bootstrapping tests
461 if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
462 soongBuildEnv.Set("ALLOW_MISSING_DEPENDENCIES", "true")
463 }
464
Paul Duffin5e85c662021-03-05 12:26:14 +0000465 err := writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap())
466 if err != nil {
467 ctx.Fatalf("failed to write environment file %s: %s", envFile, err)
468 }
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100469
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700470 func() {
Nan Zhang17f27672018-12-12 16:01:49 -0800471 ctx.BeginTrace(metrics.RunSoong, "environment check")
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700472 defer ctx.EndTrace()
473
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200474 checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(soongBuildTag))
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200475
Chris Parsonsef615e52022-08-18 22:04:11 -0400476 if config.BazelBuildEnabled() || config.Bp2Build() {
Lukacs T. Berkic541cd22022-10-26 07:26:50 +0000477 checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(bp2buildFilesTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200478 }
479
480 if config.JsonModuleGraph() {
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200481 checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(jsonModuleGraphTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200482 }
483
484 if config.Queryview() {
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200485 checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(queryviewTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200486 }
487
Spandan Das5af0bd32022-09-28 20:43:08 +0000488 if config.ApiBp2build() {
489 checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(apiBp2buildTag))
490 }
491
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200492 if config.SoongDocs() {
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200493 checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(soongDocsTag))
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700494 }
495 }()
496
Colin Cross9191df22021-10-29 13:11:32 -0700497 runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob",
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700498 map[string]string{"github.com/google/blueprint": "build/blueprint"})
Dan Willemsen5af1cbe2018-07-05 21:46:51 -0700499
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200500 ninja := func(name, ninjaFile string, targets ...string) {
Nan Zhang17f27672018-12-12 16:01:49 -0800501 ctx.BeginTrace(metrics.RunSoong, name)
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700502 defer ctx.EndTrace()
503
Dan Willemsenb82471a2018-05-17 16:37:09 -0700504 fifo := filepath.Join(config.OutDir(), ".ninja_fifo")
Colin Crossb98d3bc2019-03-21 16:02:58 -0700505 nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
506 defer nr.Close()
Dan Willemsenb82471a2018-05-17 16:37:09 -0700507
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200508 ninjaArgs := []string{
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700509 "-d", "keepdepfile",
Dan Willemsen08218222020-05-18 14:02:02 -0700510 "-d", "stats",
Dan Willemsen6587bed2020-04-18 20:25:59 -0700511 "-o", "usesphonyoutputs=yes",
512 "-o", "preremoveoutputs=yes",
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700513 "-w", "dupbuild=err",
Dan Willemsen6587bed2020-04-18 20:25:59 -0700514 "-w", "outputdir=err",
515 "-w", "missingoutfile=err",
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700516 "-j", strconv.Itoa(config.Parallel()),
Dan Willemsen02736672018-07-17 17:54:31 -0700517 "--frontend_file", fifo,
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200518 "-f", filepath.Join(config.SoongOutDir(), ninjaFile),
519 }
520
Usta Shrestha8dc8b0a2022-08-10 17:39:37 -0400521 if extra, ok := config.Environment().Get("SOONG_UI_NINJA_ARGS"); ok {
522 ctx.Printf(`CAUTION: arguments in $SOONG_UI_NINJA_ARGS=%q, e.g. "-n", can make soong_build FAIL or INCORRECT`, extra)
523 ninjaArgs = append(ninjaArgs, strings.Fields(extra)...)
524 }
525
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200526 ninjaArgs = append(ninjaArgs, targets...)
527 cmd := Command(ctx, config, "soong "+name,
528 config.PrebuiltBuildTool("ninja"), ninjaArgs...)
Jingwen Chen7c6089a2020-11-02 02:56:20 -0500529
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100530 var ninjaEnv Environment
Lukacs T. Berki73ab9282021-03-10 10:48:39 +0100531
532 // This is currently how the command line to invoke soong_build finds the
533 // root of the source tree and the output root
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100534 ninjaEnv.Set("TOP", os.Getenv("TOP"))
Lukacs T. Berki7d613bf2021-03-02 10:09:41 +0100535
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100536 cmd.Environment = &ninjaEnv
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700537 cmd.Sandbox = soongSandbox
Colin Cross7b97ecd2019-06-19 13:17:59 -0700538 cmd.RunAndStreamOrFatal()
Dan Willemsen1e704462016-08-21 15:17:17 -0700539 }
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200540
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200541 targets := make([]string, 0, 0)
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200542
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200543 if config.JsonModuleGraph() {
544 targets = append(targets, config.ModuleGraphFile())
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200545 }
546
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200547 if config.Bp2Build() {
Lukacs T. Berkic541cd22022-10-26 07:26:50 +0000548 targets = append(targets, config.Bp2BuildWorkspaceMarkerFile())
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200549 }
550
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200551 if config.Queryview() {
552 targets = append(targets, config.QueryviewMarkerFile())
553 }
554
Spandan Das5af0bd32022-09-28 20:43:08 +0000555 if config.ApiBp2build() {
556 targets = append(targets, config.ApiBp2buildMarkerFile())
557 }
558
Lukacs T. Berkic6012f32021-09-06 18:31:46 +0200559 if config.SoongDocs() {
560 targets = append(targets, config.SoongDocsHtml())
561 }
562
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200563 if config.SoongBuildInvocationNeeded() {
564 // This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
Cole Faust65f298c2021-10-28 16:05:13 -0700565 targets = append(targets, config.SoongNinjaFile())
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200566 }
567
Jingwen Cheneb76c432021-01-28 08:22:12 -0500568 if shouldCollectBuildSoongMetrics(config) {
usta7f56eaa2022-10-27 23:01:02 -0400569 soongBuildMetricsFile := filepath.Join(config.LogsDir(), "soong_build_metrics.pb")
570 if err := os.Remove(soongBuildMetricsFile); err != nil && !os.IsNotExist(err) {
571 ctx.Verbosef("Failed to remove %s", soongBuildMetricsFile)
Dan Willemsende360442022-04-20 21:21:55 -0700572 }
usta7f56eaa2022-10-27 23:01:02 -0400573 defer func() {
574 soongBuildMetrics := loadSoongBuildMetrics(ctx, soongBuildMetricsFile)
575 if soongBuildMetrics != nil {
576 logSoongBuildMetrics(ctx, soongBuildMetrics)
577 if ctx.Metrics != nil {
578 ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics)
579 }
580 }
581 }()
Jingwen Cheneb76c432021-01-28 08:22:12 -0500582 }
usta7f56eaa2022-10-27 23:01:02 -0400583 ninja("bootstrap", "bootstrap.ninja", targets...)
Colin Crossb72c9092020-02-10 11:23:49 -0800584
Colin Cross8ba7d472020-06-25 11:27:52 -0700585 distGzipFile(ctx, config, config.SoongNinjaFile(), "soong")
Jihoon Kang9f4f8a32022-08-16 00:57:30 +0000586 distFile(ctx, config, config.SoongVarsFile(), "soong")
Colin Cross8ba7d472020-06-25 11:27:52 -0700587
Anton Hansson5e5c48b2020-11-27 12:35:20 +0000588 if !config.SkipKati() {
Colin Cross8ba7d472020-06-25 11:27:52 -0700589 distGzipFile(ctx, config, config.SoongAndroidMk(), "soong")
590 distGzipFile(ctx, config, config.SoongMakeVarsMk(), "soong")
591 }
592
Rob Seymour33cd10d2022-03-02 23:10:25 +0000593 if config.JsonModuleGraph() {
594 distGzipFile(ctx, config, config.ModuleGraphFile(), "soong")
595 }
Colin Crossb72c9092020-02-10 11:23:49 -0800596}
597
Lukacs T. Berki90b43342021-11-02 14:42:04 +0100598func runMicrofactory(ctx Context, config Config, name string, pkg string, mapping map[string]string) {
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700599 ctx.BeginTrace(metrics.RunSoong, name)
600 defer ctx.EndTrace()
601 cfg := microfactory.Config{TrimPath: absPath(ctx, ".")}
602 for pkgPrefix, pathPrefix := range mapping {
603 cfg.Map(pkgPrefix, pathPrefix)
604 }
605
Lukacs T. Berki90b43342021-11-02 14:42:04 +0100606 exePath := filepath.Join(config.SoongOutDir(), name)
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700607 dir := filepath.Dir(exePath)
608 if err := os.MkdirAll(dir, 0777); err != nil {
609 ctx.Fatalf("cannot create %s: %s", dir, err)
610 }
611 if _, err := microfactory.Build(&cfg, exePath, pkg); err != nil {
612 ctx.Fatalf("failed to build %s: %s", name, err)
613 }
614}
615
Jingwen Cheneb76c432021-01-28 08:22:12 -0500616func shouldCollectBuildSoongMetrics(config Config) bool {
Jingwen Chendd9725c2021-06-24 08:41:16 +0000617 // Do not collect metrics protobuf if the soong_build binary ran as the
618 // bp2build converter or the JSON graph dump.
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200619 return config.SoongBuildInvocationNeeded()
Jingwen Cheneb76c432021-01-28 08:22:12 -0500620}
621
usta7f56eaa2022-10-27 23:01:02 -0400622func loadSoongBuildMetrics(ctx Context, soongBuildMetricsFile string) *soong_metrics_proto.SoongBuildMetrics {
Dan Willemsende360442022-04-20 21:21:55 -0700623 buf, err := os.ReadFile(soongBuildMetricsFile)
624 if errors.Is(err, fs.ErrNotExist) {
625 // Soong may not have run during this invocation
Usta Shresthadb46a9b2022-07-11 11:29:56 -0400626 ctx.Verbosef("Failed to read metrics file, %s: %s", soongBuildMetricsFile, err)
Dan Willemsende360442022-04-20 21:21:55 -0700627 return nil
628 } else if err != nil {
Colin Crossb72c9092020-02-10 11:23:49 -0800629 ctx.Fatalf("Failed to load %s: %s", soongBuildMetricsFile, err)
630 }
631 soongBuildMetrics := &soong_metrics_proto.SoongBuildMetrics{}
632 err = proto.Unmarshal(buf, soongBuildMetrics)
633 if err != nil {
634 ctx.Fatalf("Failed to unmarshal %s: %s", soongBuildMetricsFile, err)
635 }
636 return soongBuildMetrics
637}
638
639func logSoongBuildMetrics(ctx Context, metrics *soong_metrics_proto.SoongBuildMetrics) {
640 ctx.Verbosef("soong_build metrics:")
641 ctx.Verbosef(" modules: %v", metrics.GetModules())
642 ctx.Verbosef(" variants: %v", metrics.GetVariants())
643 ctx.Verbosef(" max heap size: %v MB", metrics.GetMaxHeapSize()/1e6)
644 ctx.Verbosef(" total allocation count: %v", metrics.GetTotalAllocCount())
645 ctx.Verbosef(" total allocation size: %v MB", metrics.GetTotalAllocSize()/1e6)
646
Dan Willemsen1e704462016-08-21 15:17:17 -0700647}