blob: 837f0a4f44d7599013fa54acaf1d5611479f7171 [file] [log] [blame]
Dan Willemsen1e704462016-08-21 15:17:17 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
Dan Willemsende360442022-04-20 21:21:55 -070018 "errors"
Colin Cross9191df22021-10-29 13:11:32 -070019 "fmt"
Dan Willemsende360442022-04-20 21:21:55 -070020 "io/fs"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070021 "os"
Dan Willemsen1e704462016-08-21 15:17:17 -070022 "path/filepath"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070023 "strconv"
Usta Shrestha8dc8b0a2022-08-10 17:39:37 -040024 "strings"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070025
Dan Willemsen4591b642021-05-24 14:24:12 -070026 "android/soong/ui/metrics"
Colin Crossb72c9092020-02-10 11:23:49 -080027 soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
Dan Willemsen4591b642021-05-24 14:24:12 -070028 "android/soong/ui/status"
29
30 "android/soong/shared"
31
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010032 "github.com/google/blueprint"
33 "github.com/google/blueprint/bootstrap"
Dan Willemsen99a75cd2017-08-04 16:04:04 -070034 "github.com/google/blueprint/microfactory"
Dan Willemsenb82471a2018-05-17 16:37:09 -070035
Dan Willemsen4591b642021-05-24 14:24:12 -070036 "google.golang.org/protobuf/proto"
Dan Willemsen1e704462016-08-21 15:17:17 -070037)
38
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020039const (
40 availableEnvFile = "soong.environment.available"
41 usedEnvFile = "soong.environment.used"
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +020042
Lukacs T. Berkic541cd22022-10-26 07:26:50 +000043 soongBuildTag = "build"
44 bp2buildFilesTag = "bp2build_files"
45 bp2buildWorkspaceTag = "bp2build_workspace"
46 jsonModuleGraphTag = "modulegraph"
47 queryviewTag = "queryview"
48 apiBp2buildTag = "api_bp2build"
49 soongDocsTag = "soong_docs"
Colin Cross9191df22021-10-29 13:11:32 -070050
51 // bootstrapEpoch is used to determine if an incremental build is incompatible with the current
52 // version of bootstrap and needs cleaning before continuing the build. Increment this for
53 // incompatible changes, for example when moving the location of the bpglob binary that is
54 // executed during bootstrap before the primary builder has had a chance to update the path.
Lukacs T. Berki9985d9a2021-11-04 11:47:42 +010055 bootstrapEpoch = 1
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020056)
57
Sasha Smundak4cbe83a2022-11-28 17:02:40 -080058func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error {
Lukacs T. Berki7690c092021-02-26 14:27:36 +010059 data, err := shared.EnvFileContents(envDeps)
60 if err != nil {
61 return err
62 }
63
Sasha Smundak4cbe83a2022-11-28 17:02:40 -080064 return os.WriteFile(envFile, data, 0644)
Lukacs T. Berki7690c092021-02-26 14:27:36 +010065}
66
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000067// This uses Android.bp files and various tools to generate <builddir>/build.ninja.
68//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010069// However, the execution of <builddir>/build.ninja happens later in
70// build/soong/ui/build/build.go#Build()
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000071//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010072// We want to rely on as few prebuilts as possible, so we need to bootstrap
73// Soong. The process is as follows:
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000074//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010075// 1. We use "Microfactory", a simple tool to compile Go code, to build
76// first itself, then soong_ui from soong_ui.bash. This binary contains
77// parts of soong_build that are needed to build itself.
78// 2. This simplified version of soong_build then reads the Blueprint files
79// that describe itself and emits .bootstrap/build.ninja that describes
80// how to build its full version and use that to produce the final Ninja
81// file Soong emits.
82// 3. soong_ui executes .bootstrap/build.ninja
Rupert Shuttleworthb7d97102020-11-25 10:19:29 +000083//
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010084// (After this, Kati is executed to parse the Makefiles, but that's not part of
85// bootstrapping Soong)
86
87// A tiny struct used to tell Blueprint that it's in bootstrap mode. It would
88// probably be nicer to use a flag in bootstrap.Args instead.
89type BlueprintConfig struct {
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +020090 toolDir string
91 soongOutDir string
92 outDir string
93 runGoTests bool
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +020094 debugCompilation bool
95 subninjas []string
96 primaryBuilderInvocations []bootstrap.PrimaryBuilderInvocation
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010097}
98
Lukacs T. Berkia806e412021-09-01 08:57:48 +020099func (c BlueprintConfig) HostToolDir() string {
100 return c.toolDir
101}
102
Lukacs T. Berki9f6c24a2021-08-26 15:07:24 +0200103func (c BlueprintConfig) SoongOutDir() string {
104 return c.soongOutDir
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100105}
106
Lukacs T. Berki9f6c24a2021-08-26 15:07:24 +0200107func (c BlueprintConfig) OutDir() string {
108 return c.outDir
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100109}
110
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +0200111func (c BlueprintConfig) RunGoTests() bool {
112 return c.runGoTests
113}
114
Lukacs T. Berki5f6cb1d2021-03-17 15:03:14 +0100115func (c BlueprintConfig) DebugCompilation() bool {
116 return c.debugCompilation
117}
118
Lukacs T. Berkiea1a31c2021-09-02 09:58:09 +0200119func (c BlueprintConfig) Subninjas() []string {
120 return c.subninjas
121}
122
123func (c BlueprintConfig) PrimaryBuilderInvocations() []bootstrap.PrimaryBuilderInvocation {
124 return c.primaryBuilderInvocations
125}
126
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200127func environmentArgs(config Config, tag string) []string {
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200128 return []string{
129 "--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile),
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200130 "--used_env", config.UsedEnvFile(tag),
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200131 }
132}
Spandan Das8f99ae62021-06-11 16:48:06 +0000133
Colin Cross9191df22021-10-29 13:11:32 -0700134func writeEmptyFile(ctx Context, path string) {
Spandan Das8f99ae62021-06-11 16:48:06 +0000135 err := os.MkdirAll(filepath.Dir(path), 0777)
136 if err != nil {
Colin Cross9191df22021-10-29 13:11:32 -0700137 ctx.Fatalf("Failed to create parent directories of empty file '%s': %s", path, err)
Spandan Das8f99ae62021-06-11 16:48:06 +0000138 }
139
Colin Cross9191df22021-10-29 13:11:32 -0700140 if exists, err := fileExists(path); err != nil {
141 ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err)
142 } else if !exists {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800143 err = os.WriteFile(path, nil, 0666)
Spandan Das8f99ae62021-06-11 16:48:06 +0000144 if err != nil {
Colin Cross9191df22021-10-29 13:11:32 -0700145 ctx.Fatalf("Failed to create empty file '%s': %s", path, err)
Spandan Das8f99ae62021-06-11 16:48:06 +0000146 }
147 }
148}
149
Colin Cross9191df22021-10-29 13:11:32 -0700150func fileExists(path string) (bool, error) {
151 if _, err := os.Stat(path); os.IsNotExist(err) {
152 return false, nil
153 } else if err != nil {
154 return false, err
155 }
156 return true, nil
157}
158
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800159type PrimaryBuilderFactory struct {
160 name string
161 description string
162 config Config
163 output string
164 specificArgs []string
165 debugPort string
166}
167
168func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuilderInvocation {
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200169 commonArgs := make([]string, 0, 0)
170
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800171 if !pb.config.skipSoongTests {
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200172 commonArgs = append(commonArgs, "-t")
173 }
174
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800175 commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list"))
Lukacs T. Berki13644272022-01-05 10:29:56 +0100176 invocationEnv := make(map[string]string)
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800177 if pb.debugPort != "" {
ustafb67fd12022-08-19 19:26:00 -0400178 //debug mode
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800179 commonArgs = append(commonArgs, "--delve_listen", pb.debugPort,
180 "--delve_path", shared.ResolveDelveBinary())
Lukacs T. Berki13644272022-01-05 10:29:56 +0100181 // GODEBUG=asyncpreemptoff=1 disables the preemption of goroutines. This
182 // is useful because the preemption happens by sending SIGURG to the OS
183 // thread hosting the goroutine in question and each signal results in
184 // work that needs to be done by Delve; it uses ptrace to debug the Go
185 // process and the tracer process must deal with every signal (it is not
186 // possible to selectively ignore SIGURG). This makes debugging slower,
187 // sometimes by an order of magnitude depending on luck.
188 // The original reason for adding async preemption to Go is here:
189 // https://github.com/golang/proposal/blob/master/design/24543-non-cooperative-preemption.md
190 invocationEnv["GODEBUG"] = "asyncpreemptoff=1"
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200191 }
192
ustafb67fd12022-08-19 19:26:00 -0400193 var allArgs []string
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800194 allArgs = append(allArgs, pb.specificArgs...)
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200195 allArgs = append(allArgs,
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800196 "--globListDir", pb.name,
197 "--globFile", pb.config.NamedGlobFile(pb.name))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200198
199 allArgs = append(allArgs, commonArgs...)
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800200 allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800201 if profileCpu := os.Getenv("SOONG_PROFILE_CPU"); profileCpu != "" {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800202 allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+pb.name)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800203 }
204 if profileMem := os.Getenv("SOONG_PROFILE_MEM"); profileMem != "" {
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800205 allArgs = append(allArgs, "--memprofile", profileMem+"."+pb.name)
Sasha Smundakfaa97b72022-11-18 15:32:49 -0800206 }
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200207 allArgs = append(allArgs, "Android.bp")
208
209 return bootstrap.PrimaryBuilderInvocation{
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000210 Inputs: []string{"Android.bp"},
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800211 Outputs: []string{pb.output},
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000212 Args: allArgs,
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800213 Description: pb.description,
Lukacs T. Berki2fad3412022-01-04 14:40:13 +0100214 // NB: Changing the value of this environment variable will not result in a
215 // rebuild. The bootstrap Ninja file will change, but apparently Ninja does
216 // not consider changing the pool specified in a statement a change that's
217 // worth rebuilding for.
218 Console: os.Getenv("SOONG_UNBUFFERED_OUTPUT") == "1",
Lukacs T. Berki13644272022-01-05 10:29:56 +0100219 Env: invocationEnv,
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200220 }
221}
222
Colin Cross9191df22021-10-29 13:11:32 -0700223// bootstrapEpochCleanup deletes files used by bootstrap during incremental builds across
224// incompatible changes. Incompatible changes are marked by incrementing the bootstrapEpoch
225// constant. A tree is considered out of date for the current epoch of the
226// .soong.bootstrap.epoch.<epoch> file doesn't exist.
227func bootstrapEpochCleanup(ctx Context, config Config) {
228 epochFile := fmt.Sprintf(".soong.bootstrap.epoch.%d", bootstrapEpoch)
229 epochPath := filepath.Join(config.SoongOutDir(), epochFile)
230 if exists, err := fileExists(epochPath); err != nil {
231 ctx.Fatalf("failed to check if bootstrap epoch file %q exists: %q", epochPath, err)
232 } else if !exists {
233 // The tree is out of date for the current epoch, delete files used by bootstrap
234 // and force the primary builder to rerun.
235 os.Remove(filepath.Join(config.SoongOutDir(), "build.ninja"))
236 for _, globFile := range bootstrapGlobFileList(config) {
237 os.Remove(globFile)
238 }
239
240 // Mark the tree as up to date with the current epoch by writing the epoch marker file.
241 writeEmptyFile(ctx, epochPath)
242 }
243}
244
245func bootstrapGlobFileList(config Config) []string {
246 return []string{
247 config.NamedGlobFile(soongBuildTag),
Lukacs T. Berkic541cd22022-10-26 07:26:50 +0000248 config.NamedGlobFile(bp2buildFilesTag),
Colin Cross9191df22021-10-29 13:11:32 -0700249 config.NamedGlobFile(jsonModuleGraphTag),
250 config.NamedGlobFile(queryviewTag),
Spandan Das5af0bd32022-09-28 20:43:08 +0000251 config.NamedGlobFile(apiBp2buildTag),
Colin Cross9191df22021-10-29 13:11:32 -0700252 config.NamedGlobFile(soongDocsTag),
253 }
254}
255
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200256func bootstrapBlueprint(ctx Context, config Config) {
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100257 ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
258 defer ctx.EndTrace()
259
Colin Cross9191df22021-10-29 13:11:32 -0700260 // Clean up some files for incremental builds across incompatible changes.
261 bootstrapEpochCleanup(ctx, config)
262
Cole Faust65f298c2021-10-28 16:05:13 -0700263 mainSoongBuildExtraArgs := []string{"-o", config.SoongNinjaFile()}
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200264 if config.EmptyNinjaFile() {
265 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
Lukacs T. Berki745380c2021-04-12 12:07:44 +0200266 }
Chris Parsonsef615e52022-08-18 22:04:11 -0400267 if config.bazelProdMode {
268 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode")
269 }
270 if config.bazelDevMode {
271 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-dev")
272 }
MarkDacekb78465d2022-10-18 20:10:16 +0000273 if config.bazelStagingMode {
274 mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging")
275 }
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000276 queryviewDir := filepath.Join(config.SoongOutDir(), "queryview")
Spandan Das5af0bd32022-09-28 20:43:08 +0000277 // The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files)
278 // The final workspace will be generated in out/soong/api_bp2build
279 apiBp2buildDir := filepath.Join(config.SoongOutDir(), ".api_bp2build")
Spandan Das5af0bd32022-09-28 20:43:08 +0000280
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800281 pbfs := []PrimaryBuilderFactory{
282 {
283 name: soongBuildTag,
284 description: fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()),
285 config: config,
286 output: config.SoongNinjaFile(),
287 specificArgs: mainSoongBuildExtraArgs,
Jingwen Chen78fd87f2021-12-06 13:27:43 +0000288 },
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800289 {
290 name: bp2buildFilesTag,
291 description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()),
292 config: config,
293 output: config.Bp2BuildFilesMarkerFile(),
294 specificArgs: []string{"--bp2build_marker", config.Bp2BuildFilesMarkerFile()},
295 },
296 {
297 name: bp2buildWorkspaceTag,
298 description: "Creating Bazel symlink forest",
299 config: config,
300 output: config.Bp2BuildWorkspaceMarkerFile(),
301 specificArgs: []string{"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile()},
302 },
303 {
304 name: jsonModuleGraphTag,
305 description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()),
306 config: config,
307 output: config.ModuleGraphFile(),
308 specificArgs: []string{
309 "--module_graph_file", config.ModuleGraphFile(),
310 "--module_actions_file", config.ModuleActionsFile(),
311 },
312 },
313 {
314 name: queryviewTag,
315 description: fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir),
316 config: config,
317 output: config.QueryviewMarkerFile(),
318 specificArgs: []string{"--bazel_queryview_dir", queryviewDir},
319 },
320 {
321 name: apiBp2buildTag,
322 description: fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir),
323 config: config,
324 output: config.ApiBp2buildMarkerFile(),
325 specificArgs: []string{"--bazel_api_bp2build_dir", apiBp2buildDir},
326 },
327 {
328 name: soongDocsTag,
329 description: fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()),
330 config: config,
331 output: config.SoongDocsHtml(),
332 specificArgs: []string{"--soong_docs", config.SoongDocsHtml()},
333 },
334 }
335
336 // Figure out which invocations will be run under the debugger:
337 // * SOONG_DELVE if set specifies listening port
338 // * SOONG_DELVE_STEPS if set specifies specific invocations to be debugged, otherwise all are
339 debuggedInvocations := make(map[string]bool)
340 delvePort := os.Getenv("SOONG_DELVE")
341 if delvePort != "" {
342 if steps := os.Getenv("SOONG_DELVE_STEPS"); steps != "" {
343 var validSteps []string
344 for _, pbf := range pbfs {
345 debuggedInvocations[pbf.name] = false
346 validSteps = append(validSteps, pbf.name)
347
348 }
349 for _, step := range strings.Split(steps, ",") {
350 if _, ok := debuggedInvocations[step]; ok {
351 debuggedInvocations[step] = true
352 } else {
353 ctx.Fatalf("SOONG_DELVE_STEPS contains unknown soong_build step %s\n"+
354 "Valid steps are %v", step, validSteps)
355 }
356 }
357 } else {
358 // SOONG_DELVE_STEPS is not set, run all steps in the debugger
359 for _, pbf := range pbfs {
360 debuggedInvocations[pbf.name] = true
361 }
362 }
363 }
364
365 var invocations []bootstrap.PrimaryBuilderInvocation
366 for _, pbf := range pbfs {
367 if debuggedInvocations[pbf.name] {
368 pbf.debugPort = delvePort
369 }
370 pbi := pbf.primaryBuilderInvocation()
371 // Some invocations require adjustment:
372 switch pbf.name {
373 case soongBuildTag:
374 if config.BazelBuildEnabled() {
375 // Mixed builds call Bazel from soong_build and they therefore need the
376 // Bazel workspace to be available. Make that so by adding a dependency on
377 // the bp2build marker file to the action that invokes soong_build .
378 pbi.OrderOnlyInputs = append(pbi.OrderOnlyInputs, config.Bp2BuildWorkspaceMarkerFile())
379 }
380 case bp2buildWorkspaceTag:
381 pbi.Inputs = append(pbi.Inputs,
382 config.Bp2BuildFilesMarkerFile(),
383 filepath.Join(config.FileListDir(), "bazel.list"))
384 }
385 invocations = append(invocations, pbi)
386 }
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200387
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200388 // The glob .ninja files are subninja'd. However, they are generated during
389 // the build itself so we write an empty file if the file does not exist yet
390 // so that the subninja doesn't fail on clean builds
Colin Cross9191df22021-10-29 13:11:32 -0700391 for _, globFile := range bootstrapGlobFileList(config) {
392 writeEmptyFile(ctx, globFile)
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200393 }
394
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800395 blueprintArgs := bootstrap.Args{
396 ModuleListFile: filepath.Join(config.FileListDir(), "Android.bp.list"),
397 OutFile: shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja"),
398 EmptyNinjaFile: false,
399 }
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200400
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100401 blueprintCtx := blueprint.NewContext()
402 blueprintCtx.SetIgnoreUnknownModuleTypes(true)
403 blueprintConfig := BlueprintConfig{
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200404 soongOutDir: config.SoongOutDir(),
405 toolDir: config.HostToolDir(),
406 outDir: config.OutDir(),
407 runGoTests: !config.skipSoongTests,
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200408 // If we want to debug soong_build, we need to compile it for debugging
Sasha Smundak4cbe83a2022-11-28 17:02:40 -0800409 debugCompilation: delvePort != "",
410 subninjas: bootstrapGlobFileList(config),
411 primaryBuilderInvocations: invocations,
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100412 }
413
usta5bb4a5d2022-08-24 12:53:46 -0400414 // since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little
415 // reason to write a `bootstrap.ninja.d` file
416 _ = bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig)
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100417}
418
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200419func checkEnvironmentFile(currentEnv *Environment, envFile string) {
420 getenv := func(k string) string {
421 v, _ := currentEnv.Get(k)
422 return v
423 }
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200424
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200425 if stale, _ := shared.StaleEnvFile(envFile, getenv); stale {
426 os.Remove(envFile)
427 }
428}
429
Dan Willemsen1e704462016-08-21 15:17:17 -0700430func runSoong(ctx Context, config Config) {
Nan Zhang17f27672018-12-12 16:01:49 -0800431 ctx.BeginTrace(metrics.RunSoong, "soong")
Dan Willemsend9f6fa22016-08-21 15:17:17 -0700432 defer ctx.EndTrace()
433
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100434 // We have two environment files: .available is the one with every variable,
435 // .used with the ones that were actually used. The latter is used to
436 // determine whether Soong needs to be re-run since why re-run it if only
437 // unused variables were changed?
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200438 envFile := filepath.Join(config.SoongOutDir(), availableEnvFile)
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100439
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100440 // This is done unconditionally, but does not take a measurable amount of time
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200441 bootstrapBlueprint(ctx, config)
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700442
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100443 soongBuildEnv := config.Environment().Copy()
444 soongBuildEnv.Set("TOP", os.Getenv("TOP"))
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100445 // For Bazel mixed builds.
Joe Onoratoba29f382022-10-24 06:38:11 -0700446 soongBuildEnv.Set("BAZEL_PATH", "./build/bazel/bin/bazel")
Chris Parsonsa9bef142022-09-28 15:07:46 -0400447 // Bazel's HOME var is set to an output subdirectory which doesn't exist. This
448 // prevents Bazel from file I/O in the actual user HOME directory.
449 soongBuildEnv.Set("BAZEL_HOME", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazelhome")))
Liz Kammer2af5ea82022-11-11 14:21:03 -0500450 soongBuildEnv.Set("BAZEL_OUTPUT_BASE", config.bazelOutputBase())
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100451 soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
452 soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -0500453 soongBuildEnv.Set("LOG_DIR", config.LogsDir())
Jingwen Chen3b13b612022-10-17 12:14:26 +0000454 soongBuildEnv.Set("BAZEL_DEPS_FILE", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazel.list")))
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100455
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100456 // For Soong bootstrapping tests
457 if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
458 soongBuildEnv.Set("ALLOW_MISSING_DEPENDENCIES", "true")
459 }
460
Paul Duffin5e85c662021-03-05 12:26:14 +0000461 err := writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap())
462 if err != nil {
463 ctx.Fatalf("failed to write environment file %s: %s", envFile, err)
464 }
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100465
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700466 func() {
Nan Zhang17f27672018-12-12 16:01:49 -0800467 ctx.BeginTrace(metrics.RunSoong, "environment check")
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700468 defer ctx.EndTrace()
469
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200470 checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(soongBuildTag))
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200471
Chris Parsonsef615e52022-08-18 22:04:11 -0400472 if config.BazelBuildEnabled() || config.Bp2Build() {
Lukacs T. Berkic541cd22022-10-26 07:26:50 +0000473 checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(bp2buildFilesTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200474 }
475
476 if config.JsonModuleGraph() {
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200477 checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(jsonModuleGraphTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200478 }
479
480 if config.Queryview() {
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200481 checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(queryviewTag))
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200482 }
483
Spandan Das5af0bd32022-09-28 20:43:08 +0000484 if config.ApiBp2build() {
485 checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(apiBp2buildTag))
486 }
487
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200488 if config.SoongDocs() {
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +0200489 checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(soongDocsTag))
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700490 }
491 }()
492
Colin Cross9191df22021-10-29 13:11:32 -0700493 runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob",
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700494 map[string]string{"github.com/google/blueprint": "build/blueprint"})
Dan Willemsen5af1cbe2018-07-05 21:46:51 -0700495
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200496 ninja := func(name, ninjaFile string, targets ...string) {
Nan Zhang17f27672018-12-12 16:01:49 -0800497 ctx.BeginTrace(metrics.RunSoong, name)
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700498 defer ctx.EndTrace()
499
Dan Willemsenb82471a2018-05-17 16:37:09 -0700500 fifo := filepath.Join(config.OutDir(), ".ninja_fifo")
Colin Crossb98d3bc2019-03-21 16:02:58 -0700501 nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo)
502 defer nr.Close()
Dan Willemsenb82471a2018-05-17 16:37:09 -0700503
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200504 ninjaArgs := []string{
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700505 "-d", "keepdepfile",
Dan Willemsen08218222020-05-18 14:02:02 -0700506 "-d", "stats",
Dan Willemsen6587bed2020-04-18 20:25:59 -0700507 "-o", "usesphonyoutputs=yes",
508 "-o", "preremoveoutputs=yes",
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700509 "-w", "dupbuild=err",
Dan Willemsen6587bed2020-04-18 20:25:59 -0700510 "-w", "outputdir=err",
511 "-w", "missingoutfile=err",
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700512 "-j", strconv.Itoa(config.Parallel()),
Dan Willemsen02736672018-07-17 17:54:31 -0700513 "--frontend_file", fifo,
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200514 "-f", filepath.Join(config.SoongOutDir(), ninjaFile),
515 }
516
Usta Shrestha8dc8b0a2022-08-10 17:39:37 -0400517 if extra, ok := config.Environment().Get("SOONG_UI_NINJA_ARGS"); ok {
518 ctx.Printf(`CAUTION: arguments in $SOONG_UI_NINJA_ARGS=%q, e.g. "-n", can make soong_build FAIL or INCORRECT`, extra)
519 ninjaArgs = append(ninjaArgs, strings.Fields(extra)...)
520 }
521
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200522 ninjaArgs = append(ninjaArgs, targets...)
523 cmd := Command(ctx, config, "soong "+name,
524 config.PrebuiltBuildTool("ninja"), ninjaArgs...)
Jingwen Chen7c6089a2020-11-02 02:56:20 -0500525
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100526 var ninjaEnv Environment
Lukacs T. Berki73ab9282021-03-10 10:48:39 +0100527
528 // This is currently how the command line to invoke soong_build finds the
529 // root of the source tree and the output root
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100530 ninjaEnv.Set("TOP", os.Getenv("TOP"))
Lukacs T. Berki7d613bf2021-03-02 10:09:41 +0100531
Lukacs T. Berkib14ad7b2021-03-09 10:43:57 +0100532 cmd.Environment = &ninjaEnv
Dan Willemsen99a75cd2017-08-04 16:04:04 -0700533 cmd.Sandbox = soongSandbox
Colin Cross7b97ecd2019-06-19 13:17:59 -0700534 cmd.RunAndStreamOrFatal()
Dan Willemsen1e704462016-08-21 15:17:17 -0700535 }
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200536
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200537 targets := make([]string, 0, 0)
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200538
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200539 if config.JsonModuleGraph() {
540 targets = append(targets, config.ModuleGraphFile())
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +0200541 }
542
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200543 if config.Bp2Build() {
Lukacs T. Berkic541cd22022-10-26 07:26:50 +0000544 targets = append(targets, config.Bp2BuildWorkspaceMarkerFile())
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200545 }
546
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200547 if config.Queryview() {
548 targets = append(targets, config.QueryviewMarkerFile())
549 }
550
Spandan Das5af0bd32022-09-28 20:43:08 +0000551 if config.ApiBp2build() {
552 targets = append(targets, config.ApiBp2buildMarkerFile())
553 }
554
Lukacs T. Berkic6012f32021-09-06 18:31:46 +0200555 if config.SoongDocs() {
556 targets = append(targets, config.SoongDocsHtml())
557 }
558
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200559 if config.SoongBuildInvocationNeeded() {
560 // This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
Cole Faust65f298c2021-10-28 16:05:13 -0700561 targets = append(targets, config.SoongNinjaFile())
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200562 }
563
Jingwen Cheneb76c432021-01-28 08:22:12 -0500564 if shouldCollectBuildSoongMetrics(config) {
usta7f56eaa2022-10-27 23:01:02 -0400565 soongBuildMetricsFile := filepath.Join(config.LogsDir(), "soong_build_metrics.pb")
566 if err := os.Remove(soongBuildMetricsFile); err != nil && !os.IsNotExist(err) {
567 ctx.Verbosef("Failed to remove %s", soongBuildMetricsFile)
Dan Willemsende360442022-04-20 21:21:55 -0700568 }
usta7f56eaa2022-10-27 23:01:02 -0400569 defer func() {
570 soongBuildMetrics := loadSoongBuildMetrics(ctx, soongBuildMetricsFile)
571 if soongBuildMetrics != nil {
572 logSoongBuildMetrics(ctx, soongBuildMetrics)
573 if ctx.Metrics != nil {
574 ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics)
575 }
576 }
577 }()
Jingwen Cheneb76c432021-01-28 08:22:12 -0500578 }
usta7f56eaa2022-10-27 23:01:02 -0400579 ninja("bootstrap", "bootstrap.ninja", targets...)
Colin Crossb72c9092020-02-10 11:23:49 -0800580
Colin Cross8ba7d472020-06-25 11:27:52 -0700581 distGzipFile(ctx, config, config.SoongNinjaFile(), "soong")
Jihoon Kang9f4f8a32022-08-16 00:57:30 +0000582 distFile(ctx, config, config.SoongVarsFile(), "soong")
Colin Cross8ba7d472020-06-25 11:27:52 -0700583
Anton Hansson5e5c48b2020-11-27 12:35:20 +0000584 if !config.SkipKati() {
Colin Cross8ba7d472020-06-25 11:27:52 -0700585 distGzipFile(ctx, config, config.SoongAndroidMk(), "soong")
586 distGzipFile(ctx, config, config.SoongMakeVarsMk(), "soong")
587 }
588
Rob Seymour33cd10d2022-03-02 23:10:25 +0000589 if config.JsonModuleGraph() {
590 distGzipFile(ctx, config, config.ModuleGraphFile(), "soong")
591 }
Colin Crossb72c9092020-02-10 11:23:49 -0800592}
593
Lukacs T. Berki90b43342021-11-02 14:42:04 +0100594func runMicrofactory(ctx Context, config Config, name string, pkg string, mapping map[string]string) {
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700595 ctx.BeginTrace(metrics.RunSoong, name)
596 defer ctx.EndTrace()
597 cfg := microfactory.Config{TrimPath: absPath(ctx, ".")}
598 for pkgPrefix, pathPrefix := range mapping {
599 cfg.Map(pkgPrefix, pathPrefix)
600 }
601
Lukacs T. Berki90b43342021-11-02 14:42:04 +0100602 exePath := filepath.Join(config.SoongOutDir(), name)
Sasha Smundak7ae80a72021-04-09 12:03:51 -0700603 dir := filepath.Dir(exePath)
604 if err := os.MkdirAll(dir, 0777); err != nil {
605 ctx.Fatalf("cannot create %s: %s", dir, err)
606 }
607 if _, err := microfactory.Build(&cfg, exePath, pkg); err != nil {
608 ctx.Fatalf("failed to build %s: %s", name, err)
609 }
610}
611
Jingwen Cheneb76c432021-01-28 08:22:12 -0500612func shouldCollectBuildSoongMetrics(config Config) bool {
Jingwen Chendd9725c2021-06-24 08:41:16 +0000613 // Do not collect metrics protobuf if the soong_build binary ran as the
614 // bp2build converter or the JSON graph dump.
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200615 return config.SoongBuildInvocationNeeded()
Jingwen Cheneb76c432021-01-28 08:22:12 -0500616}
617
usta7f56eaa2022-10-27 23:01:02 -0400618func loadSoongBuildMetrics(ctx Context, soongBuildMetricsFile string) *soong_metrics_proto.SoongBuildMetrics {
Dan Willemsende360442022-04-20 21:21:55 -0700619 buf, err := os.ReadFile(soongBuildMetricsFile)
620 if errors.Is(err, fs.ErrNotExist) {
621 // Soong may not have run during this invocation
Usta Shresthadb46a9b2022-07-11 11:29:56 -0400622 ctx.Verbosef("Failed to read metrics file, %s: %s", soongBuildMetricsFile, err)
Dan Willemsende360442022-04-20 21:21:55 -0700623 return nil
624 } else if err != nil {
Colin Crossb72c9092020-02-10 11:23:49 -0800625 ctx.Fatalf("Failed to load %s: %s", soongBuildMetricsFile, err)
626 }
627 soongBuildMetrics := &soong_metrics_proto.SoongBuildMetrics{}
628 err = proto.Unmarshal(buf, soongBuildMetrics)
629 if err != nil {
630 ctx.Fatalf("Failed to unmarshal %s: %s", soongBuildMetricsFile, err)
631 }
632 return soongBuildMetrics
633}
634
635func logSoongBuildMetrics(ctx Context, metrics *soong_metrics_proto.SoongBuildMetrics) {
636 ctx.Verbosef("soong_build metrics:")
637 ctx.Verbosef(" modules: %v", metrics.GetModules())
638 ctx.Verbosef(" variants: %v", metrics.GetVariants())
639 ctx.Verbosef(" max heap size: %v MB", metrics.GetMaxHeapSize()/1e6)
640 ctx.Verbosef(" total allocation count: %v", metrics.GetTotalAllocCount())
641 ctx.Verbosef(" total allocation size: %v MB", metrics.GetTotalAllocSize()/1e6)
642
Dan Willemsen1e704462016-08-21 15:17:17 -0700643}