Dan Willemsen | 1e70446 | 2016-08-21 15:17:17 -0700 | [diff] [blame] | 1 | // Copyright 2017 Google Inc. All rights reserved. |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | package build |
| 16 | |
| 17 | import ( |
Dan Willemsen | de36044 | 2022-04-20 21:21:55 -0700 | [diff] [blame] | 18 | "errors" |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 19 | "fmt" |
Dan Willemsen | de36044 | 2022-04-20 21:21:55 -0700 | [diff] [blame] | 20 | "io/fs" |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 21 | "os" |
Dan Willemsen | 1e70446 | 2016-08-21 15:17:17 -0700 | [diff] [blame] | 22 | "path/filepath" |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 23 | "strconv" |
Usta Shrestha | 8dc8b0a | 2022-08-10 17:39:37 -0400 | [diff] [blame] | 24 | "strings" |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 25 | |
Dan Willemsen | 4591b64 | 2021-05-24 14:24:12 -0700 | [diff] [blame] | 26 | "android/soong/ui/metrics" |
Colin Cross | b72c909 | 2020-02-10 11:23:49 -0800 | [diff] [blame] | 27 | soong_metrics_proto "android/soong/ui/metrics/metrics_proto" |
Dan Willemsen | 4591b64 | 2021-05-24 14:24:12 -0700 | [diff] [blame] | 28 | "android/soong/ui/status" |
| 29 | |
| 30 | "android/soong/shared" |
| 31 | |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 32 | "github.com/google/blueprint" |
| 33 | "github.com/google/blueprint/bootstrap" |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 34 | "github.com/google/blueprint/microfactory" |
Dan Willemsen | b82471a | 2018-05-17 16:37:09 -0700 | [diff] [blame] | 35 | |
Dan Willemsen | 4591b64 | 2021-05-24 14:24:12 -0700 | [diff] [blame] | 36 | "google.golang.org/protobuf/proto" |
Dan Willemsen | 1e70446 | 2016-08-21 15:17:17 -0700 | [diff] [blame] | 37 | ) |
| 38 | |
Lukacs T. Berki | f8e2428 | 2021-04-14 10:31:00 +0200 | [diff] [blame] | 39 | const ( |
| 40 | availableEnvFile = "soong.environment.available" |
| 41 | usedEnvFile = "soong.environment.used" |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 42 | |
Lukacs T. Berki | c541cd2 | 2022-10-26 07:26:50 +0000 | [diff] [blame] | 43 | soongBuildTag = "build" |
| 44 | bp2buildFilesTag = "bp2build_files" |
| 45 | bp2buildWorkspaceTag = "bp2build_workspace" |
| 46 | jsonModuleGraphTag = "modulegraph" |
| 47 | queryviewTag = "queryview" |
| 48 | apiBp2buildTag = "api_bp2build" |
| 49 | soongDocsTag = "soong_docs" |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 50 | |
| 51 | // bootstrapEpoch is used to determine if an incremental build is incompatible with the current |
| 52 | // version of bootstrap and needs cleaning before continuing the build. Increment this for |
| 53 | // incompatible changes, for example when moving the location of the bpglob binary that is |
| 54 | // executed during bootstrap before the primary builder has had a chance to update the path. |
Lukacs T. Berki | 9985d9a | 2021-11-04 11:47:42 +0100 | [diff] [blame] | 55 | bootstrapEpoch = 1 |
Lukacs T. Berki | f8e2428 | 2021-04-14 10:31:00 +0200 | [diff] [blame] | 56 | ) |
| 57 | |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 58 | func writeEnvironmentFile(_ Context, envFile string, envDeps map[string]string) error { |
Lukacs T. Berki | 7690c09 | 2021-02-26 14:27:36 +0100 | [diff] [blame] | 59 | data, err := shared.EnvFileContents(envDeps) |
| 60 | if err != nil { |
| 61 | return err |
| 62 | } |
| 63 | |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 64 | return os.WriteFile(envFile, data, 0644) |
Lukacs T. Berki | 7690c09 | 2021-02-26 14:27:36 +0100 | [diff] [blame] | 65 | } |
| 66 | |
Rupert Shuttleworth | b7d9710 | 2020-11-25 10:19:29 +0000 | [diff] [blame] | 67 | // This uses Android.bp files and various tools to generate <builddir>/build.ninja. |
| 68 | // |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 69 | // However, the execution of <builddir>/build.ninja happens later in |
| 70 | // build/soong/ui/build/build.go#Build() |
Rupert Shuttleworth | b7d9710 | 2020-11-25 10:19:29 +0000 | [diff] [blame] | 71 | // |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 72 | // We want to rely on as few prebuilts as possible, so we need to bootstrap |
| 73 | // Soong. The process is as follows: |
Rupert Shuttleworth | b7d9710 | 2020-11-25 10:19:29 +0000 | [diff] [blame] | 74 | // |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 75 | // 1. We use "Microfactory", a simple tool to compile Go code, to build |
| 76 | // first itself, then soong_ui from soong_ui.bash. This binary contains |
| 77 | // parts of soong_build that are needed to build itself. |
| 78 | // 2. This simplified version of soong_build then reads the Blueprint files |
| 79 | // that describe itself and emits .bootstrap/build.ninja that describes |
| 80 | // how to build its full version and use that to produce the final Ninja |
| 81 | // file Soong emits. |
| 82 | // 3. soong_ui executes .bootstrap/build.ninja |
Rupert Shuttleworth | b7d9710 | 2020-11-25 10:19:29 +0000 | [diff] [blame] | 83 | // |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 84 | // (After this, Kati is executed to parse the Makefiles, but that's not part of |
| 85 | // bootstrapping Soong) |
| 86 | |
| 87 | // A tiny struct used to tell Blueprint that it's in bootstrap mode. It would |
| 88 | // probably be nicer to use a flag in bootstrap.Args instead. |
| 89 | type BlueprintConfig struct { |
Lukacs T. Berki | ea1a31c | 2021-09-02 09:58:09 +0200 | [diff] [blame] | 90 | toolDir string |
| 91 | soongOutDir string |
| 92 | outDir string |
| 93 | runGoTests bool |
Lukacs T. Berki | ea1a31c | 2021-09-02 09:58:09 +0200 | [diff] [blame] | 94 | debugCompilation bool |
| 95 | subninjas []string |
| 96 | primaryBuilderInvocations []bootstrap.PrimaryBuilderInvocation |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 97 | } |
| 98 | |
Lukacs T. Berki | a806e41 | 2021-09-01 08:57:48 +0200 | [diff] [blame] | 99 | func (c BlueprintConfig) HostToolDir() string { |
| 100 | return c.toolDir |
| 101 | } |
| 102 | |
Lukacs T. Berki | 9f6c24a | 2021-08-26 15:07:24 +0200 | [diff] [blame] | 103 | func (c BlueprintConfig) SoongOutDir() string { |
| 104 | return c.soongOutDir |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 105 | } |
| 106 | |
Lukacs T. Berki | 9f6c24a | 2021-08-26 15:07:24 +0200 | [diff] [blame] | 107 | func (c BlueprintConfig) OutDir() string { |
| 108 | return c.outDir |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 109 | } |
| 110 | |
Lukacs T. Berki | ea1a31c | 2021-09-02 09:58:09 +0200 | [diff] [blame] | 111 | func (c BlueprintConfig) RunGoTests() bool { |
| 112 | return c.runGoTests |
| 113 | } |
| 114 | |
Lukacs T. Berki | 5f6cb1d | 2021-03-17 15:03:14 +0100 | [diff] [blame] | 115 | func (c BlueprintConfig) DebugCompilation() bool { |
| 116 | return c.debugCompilation |
| 117 | } |
| 118 | |
Lukacs T. Berki | ea1a31c | 2021-09-02 09:58:09 +0200 | [diff] [blame] | 119 | func (c BlueprintConfig) Subninjas() []string { |
| 120 | return c.subninjas |
| 121 | } |
| 122 | |
| 123 | func (c BlueprintConfig) PrimaryBuilderInvocations() []bootstrap.PrimaryBuilderInvocation { |
| 124 | return c.primaryBuilderInvocations |
| 125 | } |
| 126 | |
Lukacs T. Berki | e1df43f | 2021-09-08 15:31:14 +0200 | [diff] [blame] | 127 | func environmentArgs(config Config, tag string) []string { |
Lukacs T. Berki | f8e2428 | 2021-04-14 10:31:00 +0200 | [diff] [blame] | 128 | return []string{ |
| 129 | "--available_env", shared.JoinPath(config.SoongOutDir(), availableEnvFile), |
Lukacs T. Berki | e1df43f | 2021-09-08 15:31:14 +0200 | [diff] [blame] | 130 | "--used_env", config.UsedEnvFile(tag), |
Lukacs T. Berki | f8e2428 | 2021-04-14 10:31:00 +0200 | [diff] [blame] | 131 | } |
| 132 | } |
Spandan Das | 8f99ae6 | 2021-06-11 16:48:06 +0000 | [diff] [blame] | 133 | |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 134 | func writeEmptyFile(ctx Context, path string) { |
Spandan Das | 8f99ae6 | 2021-06-11 16:48:06 +0000 | [diff] [blame] | 135 | err := os.MkdirAll(filepath.Dir(path), 0777) |
| 136 | if err != nil { |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 137 | ctx.Fatalf("Failed to create parent directories of empty file '%s': %s", path, err) |
Spandan Das | 8f99ae6 | 2021-06-11 16:48:06 +0000 | [diff] [blame] | 138 | } |
| 139 | |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 140 | if exists, err := fileExists(path); err != nil { |
| 141 | ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err) |
| 142 | } else if !exists { |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 143 | err = os.WriteFile(path, nil, 0666) |
Spandan Das | 8f99ae6 | 2021-06-11 16:48:06 +0000 | [diff] [blame] | 144 | if err != nil { |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 145 | ctx.Fatalf("Failed to create empty file '%s': %s", path, err) |
Spandan Das | 8f99ae6 | 2021-06-11 16:48:06 +0000 | [diff] [blame] | 146 | } |
| 147 | } |
| 148 | } |
| 149 | |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 150 | func fileExists(path string) (bool, error) { |
| 151 | if _, err := os.Stat(path); os.IsNotExist(err) { |
| 152 | return false, nil |
| 153 | } else if err != nil { |
| 154 | return false, err |
| 155 | } |
| 156 | return true, nil |
| 157 | } |
| 158 | |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 159 | type PrimaryBuilderFactory struct { |
| 160 | name string |
| 161 | description string |
| 162 | config Config |
| 163 | output string |
| 164 | specificArgs []string |
| 165 | debugPort string |
| 166 | } |
| 167 | |
| 168 | func (pb PrimaryBuilderFactory) primaryBuilderInvocation() bootstrap.PrimaryBuilderInvocation { |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 169 | commonArgs := make([]string, 0, 0) |
| 170 | |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 171 | if !pb.config.skipSoongTests { |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 172 | commonArgs = append(commonArgs, "-t") |
| 173 | } |
| 174 | |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 175 | commonArgs = append(commonArgs, "-l", filepath.Join(pb.config.FileListDir(), "Android.bp.list")) |
Lukacs T. Berki | 1364427 | 2022-01-05 10:29:56 +0100 | [diff] [blame] | 176 | invocationEnv := make(map[string]string) |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 177 | if pb.debugPort != "" { |
usta | fb67fd1 | 2022-08-19 19:26:00 -0400 | [diff] [blame] | 178 | //debug mode |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 179 | commonArgs = append(commonArgs, "--delve_listen", pb.debugPort, |
| 180 | "--delve_path", shared.ResolveDelveBinary()) |
Lukacs T. Berki | 1364427 | 2022-01-05 10:29:56 +0100 | [diff] [blame] | 181 | // GODEBUG=asyncpreemptoff=1 disables the preemption of goroutines. This |
| 182 | // is useful because the preemption happens by sending SIGURG to the OS |
| 183 | // thread hosting the goroutine in question and each signal results in |
| 184 | // work that needs to be done by Delve; it uses ptrace to debug the Go |
| 185 | // process and the tracer process must deal with every signal (it is not |
| 186 | // possible to selectively ignore SIGURG). This makes debugging slower, |
| 187 | // sometimes by an order of magnitude depending on luck. |
| 188 | // The original reason for adding async preemption to Go is here: |
| 189 | // https://github.com/golang/proposal/blob/master/design/24543-non-cooperative-preemption.md |
| 190 | invocationEnv["GODEBUG"] = "asyncpreemptoff=1" |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 191 | } |
| 192 | |
usta | fb67fd1 | 2022-08-19 19:26:00 -0400 | [diff] [blame] | 193 | var allArgs []string |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 194 | allArgs = append(allArgs, pb.specificArgs...) |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 195 | allArgs = append(allArgs, |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 196 | "--globListDir", pb.name, |
| 197 | "--globFile", pb.config.NamedGlobFile(pb.name)) |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 198 | |
| 199 | allArgs = append(allArgs, commonArgs...) |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 200 | allArgs = append(allArgs, environmentArgs(pb.config, pb.name)...) |
Sasha Smundak | faa97b7 | 2022-11-18 15:32:49 -0800 | [diff] [blame] | 201 | if profileCpu := os.Getenv("SOONG_PROFILE_CPU"); profileCpu != "" { |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 202 | allArgs = append(allArgs, "--cpuprofile", profileCpu+"."+pb.name) |
Sasha Smundak | faa97b7 | 2022-11-18 15:32:49 -0800 | [diff] [blame] | 203 | } |
| 204 | if profileMem := os.Getenv("SOONG_PROFILE_MEM"); profileMem != "" { |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 205 | allArgs = append(allArgs, "--memprofile", profileMem+"."+pb.name) |
Sasha Smundak | faa97b7 | 2022-11-18 15:32:49 -0800 | [diff] [blame] | 206 | } |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 207 | allArgs = append(allArgs, "Android.bp") |
| 208 | |
| 209 | return bootstrap.PrimaryBuilderInvocation{ |
Jingwen Chen | 78fd87f | 2021-12-06 13:27:43 +0000 | [diff] [blame] | 210 | Inputs: []string{"Android.bp"}, |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 211 | Outputs: []string{pb.output}, |
Jingwen Chen | 78fd87f | 2021-12-06 13:27:43 +0000 | [diff] [blame] | 212 | Args: allArgs, |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 213 | Description: pb.description, |
Lukacs T. Berki | 2fad341 | 2022-01-04 14:40:13 +0100 | [diff] [blame] | 214 | // NB: Changing the value of this environment variable will not result in a |
| 215 | // rebuild. The bootstrap Ninja file will change, but apparently Ninja does |
| 216 | // not consider changing the pool specified in a statement a change that's |
| 217 | // worth rebuilding for. |
| 218 | Console: os.Getenv("SOONG_UNBUFFERED_OUTPUT") == "1", |
Lukacs T. Berki | 1364427 | 2022-01-05 10:29:56 +0100 | [diff] [blame] | 219 | Env: invocationEnv, |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 220 | } |
| 221 | } |
| 222 | |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 223 | // bootstrapEpochCleanup deletes files used by bootstrap during incremental builds across |
| 224 | // incompatible changes. Incompatible changes are marked by incrementing the bootstrapEpoch |
| 225 | // constant. A tree is considered out of date for the current epoch of the |
| 226 | // .soong.bootstrap.epoch.<epoch> file doesn't exist. |
| 227 | func bootstrapEpochCleanup(ctx Context, config Config) { |
| 228 | epochFile := fmt.Sprintf(".soong.bootstrap.epoch.%d", bootstrapEpoch) |
| 229 | epochPath := filepath.Join(config.SoongOutDir(), epochFile) |
| 230 | if exists, err := fileExists(epochPath); err != nil { |
| 231 | ctx.Fatalf("failed to check if bootstrap epoch file %q exists: %q", epochPath, err) |
| 232 | } else if !exists { |
| 233 | // The tree is out of date for the current epoch, delete files used by bootstrap |
| 234 | // and force the primary builder to rerun. |
| 235 | os.Remove(filepath.Join(config.SoongOutDir(), "build.ninja")) |
| 236 | for _, globFile := range bootstrapGlobFileList(config) { |
| 237 | os.Remove(globFile) |
| 238 | } |
| 239 | |
| 240 | // Mark the tree as up to date with the current epoch by writing the epoch marker file. |
| 241 | writeEmptyFile(ctx, epochPath) |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | func bootstrapGlobFileList(config Config) []string { |
| 246 | return []string{ |
| 247 | config.NamedGlobFile(soongBuildTag), |
Lukacs T. Berki | c541cd2 | 2022-10-26 07:26:50 +0000 | [diff] [blame] | 248 | config.NamedGlobFile(bp2buildFilesTag), |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 249 | config.NamedGlobFile(jsonModuleGraphTag), |
| 250 | config.NamedGlobFile(queryviewTag), |
Spandan Das | 5af0bd3 | 2022-09-28 20:43:08 +0000 | [diff] [blame] | 251 | config.NamedGlobFile(apiBp2buildTag), |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 252 | config.NamedGlobFile(soongDocsTag), |
| 253 | } |
| 254 | } |
| 255 | |
Lukacs T. Berki | 56ebaf3 | 2021-08-12 14:03:55 +0200 | [diff] [blame] | 256 | func bootstrapBlueprint(ctx Context, config Config) { |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 257 | ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap") |
| 258 | defer ctx.EndTrace() |
| 259 | |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 260 | // Clean up some files for incremental builds across incompatible changes. |
| 261 | bootstrapEpochCleanup(ctx, config) |
| 262 | |
Cole Faust | 65f298c | 2021-10-28 16:05:13 -0700 | [diff] [blame] | 263 | mainSoongBuildExtraArgs := []string{"-o", config.SoongNinjaFile()} |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 264 | if config.EmptyNinjaFile() { |
| 265 | mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file") |
Lukacs T. Berki | 745380c | 2021-04-12 12:07:44 +0200 | [diff] [blame] | 266 | } |
Chris Parsons | ef615e5 | 2022-08-18 22:04:11 -0400 | [diff] [blame] | 267 | if config.bazelProdMode { |
| 268 | mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode") |
| 269 | } |
| 270 | if config.bazelDevMode { |
| 271 | mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-dev") |
| 272 | } |
MarkDacek | b78465d | 2022-10-18 20:10:16 +0000 | [diff] [blame] | 273 | if config.bazelStagingMode { |
| 274 | mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-mode-staging") |
| 275 | } |
MarkDacek | d06db5d | 2022-11-29 00:47:59 +0000 | [diff] [blame^] | 276 | if len(config.bazelForceEnabledModules) > 0 { |
| 277 | mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--bazel-force-enabled-modules="+config.bazelForceEnabledModules) |
| 278 | } |
| 279 | |
Jingwen Chen | 78fd87f | 2021-12-06 13:27:43 +0000 | [diff] [blame] | 280 | queryviewDir := filepath.Join(config.SoongOutDir(), "queryview") |
Spandan Das | 5af0bd3 | 2022-09-28 20:43:08 +0000 | [diff] [blame] | 281 | // The BUILD files will be generated in out/soong/.api_bp2build (no symlinks to src files) |
| 282 | // The final workspace will be generated in out/soong/api_bp2build |
| 283 | apiBp2buildDir := filepath.Join(config.SoongOutDir(), ".api_bp2build") |
Spandan Das | 5af0bd3 | 2022-09-28 20:43:08 +0000 | [diff] [blame] | 284 | |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 285 | pbfs := []PrimaryBuilderFactory{ |
| 286 | { |
| 287 | name: soongBuildTag, |
| 288 | description: fmt.Sprintf("analyzing Android.bp files and generating ninja file at %s", config.SoongNinjaFile()), |
| 289 | config: config, |
| 290 | output: config.SoongNinjaFile(), |
| 291 | specificArgs: mainSoongBuildExtraArgs, |
Jingwen Chen | 78fd87f | 2021-12-06 13:27:43 +0000 | [diff] [blame] | 292 | }, |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 293 | { |
| 294 | name: bp2buildFilesTag, |
| 295 | description: fmt.Sprintf("converting Android.bp files to BUILD files at %s/bp2build", config.SoongOutDir()), |
| 296 | config: config, |
| 297 | output: config.Bp2BuildFilesMarkerFile(), |
| 298 | specificArgs: []string{"--bp2build_marker", config.Bp2BuildFilesMarkerFile()}, |
| 299 | }, |
| 300 | { |
| 301 | name: bp2buildWorkspaceTag, |
| 302 | description: "Creating Bazel symlink forest", |
| 303 | config: config, |
| 304 | output: config.Bp2BuildWorkspaceMarkerFile(), |
| 305 | specificArgs: []string{"--symlink_forest_marker", config.Bp2BuildWorkspaceMarkerFile()}, |
| 306 | }, |
| 307 | { |
| 308 | name: jsonModuleGraphTag, |
| 309 | description: fmt.Sprintf("generating the Soong module graph at %s", config.ModuleGraphFile()), |
| 310 | config: config, |
| 311 | output: config.ModuleGraphFile(), |
| 312 | specificArgs: []string{ |
| 313 | "--module_graph_file", config.ModuleGraphFile(), |
| 314 | "--module_actions_file", config.ModuleActionsFile(), |
| 315 | }, |
| 316 | }, |
| 317 | { |
| 318 | name: queryviewTag, |
| 319 | description: fmt.Sprintf("generating the Soong module graph as a Bazel workspace at %s", queryviewDir), |
| 320 | config: config, |
| 321 | output: config.QueryviewMarkerFile(), |
| 322 | specificArgs: []string{"--bazel_queryview_dir", queryviewDir}, |
| 323 | }, |
| 324 | { |
| 325 | name: apiBp2buildTag, |
| 326 | description: fmt.Sprintf("generating BUILD files for API contributions at %s", apiBp2buildDir), |
| 327 | config: config, |
| 328 | output: config.ApiBp2buildMarkerFile(), |
| 329 | specificArgs: []string{"--bazel_api_bp2build_dir", apiBp2buildDir}, |
| 330 | }, |
| 331 | { |
| 332 | name: soongDocsTag, |
| 333 | description: fmt.Sprintf("generating Soong docs at %s", config.SoongDocsHtml()), |
| 334 | config: config, |
| 335 | output: config.SoongDocsHtml(), |
| 336 | specificArgs: []string{"--soong_docs", config.SoongDocsHtml()}, |
| 337 | }, |
| 338 | } |
| 339 | |
| 340 | // Figure out which invocations will be run under the debugger: |
| 341 | // * SOONG_DELVE if set specifies listening port |
| 342 | // * SOONG_DELVE_STEPS if set specifies specific invocations to be debugged, otherwise all are |
| 343 | debuggedInvocations := make(map[string]bool) |
| 344 | delvePort := os.Getenv("SOONG_DELVE") |
| 345 | if delvePort != "" { |
| 346 | if steps := os.Getenv("SOONG_DELVE_STEPS"); steps != "" { |
| 347 | var validSteps []string |
| 348 | for _, pbf := range pbfs { |
| 349 | debuggedInvocations[pbf.name] = false |
| 350 | validSteps = append(validSteps, pbf.name) |
| 351 | |
| 352 | } |
| 353 | for _, step := range strings.Split(steps, ",") { |
| 354 | if _, ok := debuggedInvocations[step]; ok { |
| 355 | debuggedInvocations[step] = true |
| 356 | } else { |
| 357 | ctx.Fatalf("SOONG_DELVE_STEPS contains unknown soong_build step %s\n"+ |
| 358 | "Valid steps are %v", step, validSteps) |
| 359 | } |
| 360 | } |
| 361 | } else { |
| 362 | // SOONG_DELVE_STEPS is not set, run all steps in the debugger |
| 363 | for _, pbf := range pbfs { |
| 364 | debuggedInvocations[pbf.name] = true |
| 365 | } |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | var invocations []bootstrap.PrimaryBuilderInvocation |
| 370 | for _, pbf := range pbfs { |
| 371 | if debuggedInvocations[pbf.name] { |
| 372 | pbf.debugPort = delvePort |
| 373 | } |
| 374 | pbi := pbf.primaryBuilderInvocation() |
| 375 | // Some invocations require adjustment: |
| 376 | switch pbf.name { |
| 377 | case soongBuildTag: |
| 378 | if config.BazelBuildEnabled() { |
| 379 | // Mixed builds call Bazel from soong_build and they therefore need the |
| 380 | // Bazel workspace to be available. Make that so by adding a dependency on |
| 381 | // the bp2build marker file to the action that invokes soong_build . |
| 382 | pbi.OrderOnlyInputs = append(pbi.OrderOnlyInputs, config.Bp2BuildWorkspaceMarkerFile()) |
| 383 | } |
| 384 | case bp2buildWorkspaceTag: |
| 385 | pbi.Inputs = append(pbi.Inputs, |
| 386 | config.Bp2BuildFilesMarkerFile(), |
| 387 | filepath.Join(config.FileListDir(), "bazel.list")) |
| 388 | } |
| 389 | invocations = append(invocations, pbi) |
| 390 | } |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 391 | |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 392 | // The glob .ninja files are subninja'd. However, they are generated during |
| 393 | // the build itself so we write an empty file if the file does not exist yet |
| 394 | // so that the subninja doesn't fail on clean builds |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 395 | for _, globFile := range bootstrapGlobFileList(config) { |
| 396 | writeEmptyFile(ctx, globFile) |
Lukacs T. Berki | f8e2428 | 2021-04-14 10:31:00 +0200 | [diff] [blame] | 397 | } |
| 398 | |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 399 | blueprintArgs := bootstrap.Args{ |
| 400 | ModuleListFile: filepath.Join(config.FileListDir(), "Android.bp.list"), |
| 401 | OutFile: shared.JoinPath(config.SoongOutDir(), "bootstrap.ninja"), |
| 402 | EmptyNinjaFile: false, |
| 403 | } |
Lukacs T. Berki | f8e2428 | 2021-04-14 10:31:00 +0200 | [diff] [blame] | 404 | |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 405 | blueprintCtx := blueprint.NewContext() |
| 406 | blueprintCtx.SetIgnoreUnknownModuleTypes(true) |
| 407 | blueprintConfig := BlueprintConfig{ |
Lukacs T. Berki | e1df43f | 2021-09-08 15:31:14 +0200 | [diff] [blame] | 408 | soongOutDir: config.SoongOutDir(), |
| 409 | toolDir: config.HostToolDir(), |
| 410 | outDir: config.OutDir(), |
| 411 | runGoTests: !config.skipSoongTests, |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 412 | // If we want to debug soong_build, we need to compile it for debugging |
Sasha Smundak | 4cbe83a | 2022-11-28 17:02:40 -0800 | [diff] [blame] | 413 | debugCompilation: delvePort != "", |
| 414 | subninjas: bootstrapGlobFileList(config), |
| 415 | primaryBuilderInvocations: invocations, |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 416 | } |
| 417 | |
usta | 5bb4a5d | 2022-08-24 12:53:46 -0400 | [diff] [blame] | 418 | // since `bootstrap.ninja` is regenerated unconditionally, we ignore the deps, i.e. little |
| 419 | // reason to write a `bootstrap.ninja.d` file |
| 420 | _ = bootstrap.RunBlueprint(blueprintArgs, bootstrap.DoEverything, blueprintCtx, blueprintConfig) |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 421 | } |
| 422 | |
Lukacs T. Berki | f8e2428 | 2021-04-14 10:31:00 +0200 | [diff] [blame] | 423 | func checkEnvironmentFile(currentEnv *Environment, envFile string) { |
| 424 | getenv := func(k string) string { |
| 425 | v, _ := currentEnv.Get(k) |
| 426 | return v |
| 427 | } |
Lukacs T. Berki | e1df43f | 2021-09-08 15:31:14 +0200 | [diff] [blame] | 428 | |
Lukacs T. Berki | f8e2428 | 2021-04-14 10:31:00 +0200 | [diff] [blame] | 429 | if stale, _ := shared.StaleEnvFile(envFile, getenv); stale { |
| 430 | os.Remove(envFile) |
| 431 | } |
| 432 | } |
| 433 | |
Dan Willemsen | 1e70446 | 2016-08-21 15:17:17 -0700 | [diff] [blame] | 434 | func runSoong(ctx Context, config Config) { |
Nan Zhang | 17f2767 | 2018-12-12 16:01:49 -0800 | [diff] [blame] | 435 | ctx.BeginTrace(metrics.RunSoong, "soong") |
Dan Willemsen | d9f6fa2 | 2016-08-21 15:17:17 -0700 | [diff] [blame] | 436 | defer ctx.EndTrace() |
| 437 | |
Lukacs T. Berki | 7690c09 | 2021-02-26 14:27:36 +0100 | [diff] [blame] | 438 | // We have two environment files: .available is the one with every variable, |
| 439 | // .used with the ones that were actually used. The latter is used to |
| 440 | // determine whether Soong needs to be re-run since why re-run it if only |
| 441 | // unused variables were changed? |
Lukacs T. Berki | f8e2428 | 2021-04-14 10:31:00 +0200 | [diff] [blame] | 442 | envFile := filepath.Join(config.SoongOutDir(), availableEnvFile) |
Lukacs T. Berki | 7690c09 | 2021-02-26 14:27:36 +0100 | [diff] [blame] | 443 | |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 444 | // This is done unconditionally, but does not take a measurable amount of time |
Lukacs T. Berki | 56ebaf3 | 2021-08-12 14:03:55 +0200 | [diff] [blame] | 445 | bootstrapBlueprint(ctx, config) |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 446 | |
Lukacs T. Berki | 7690c09 | 2021-02-26 14:27:36 +0100 | [diff] [blame] | 447 | soongBuildEnv := config.Environment().Copy() |
| 448 | soongBuildEnv.Set("TOP", os.Getenv("TOP")) |
Lukacs T. Berki | 7690c09 | 2021-02-26 14:27:36 +0100 | [diff] [blame] | 449 | // For Bazel mixed builds. |
Joe Onorato | ba29f38 | 2022-10-24 06:38:11 -0700 | [diff] [blame] | 450 | soongBuildEnv.Set("BAZEL_PATH", "./build/bazel/bin/bazel") |
Chris Parsons | a9bef14 | 2022-09-28 15:07:46 -0400 | [diff] [blame] | 451 | // Bazel's HOME var is set to an output subdirectory which doesn't exist. This |
| 452 | // prevents Bazel from file I/O in the actual user HOME directory. |
| 453 | soongBuildEnv.Set("BAZEL_HOME", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazelhome"))) |
Liz Kammer | 2af5ea8 | 2022-11-11 14:21:03 -0500 | [diff] [blame] | 454 | soongBuildEnv.Set("BAZEL_OUTPUT_BASE", config.bazelOutputBase()) |
Lukacs T. Berki | 7690c09 | 2021-02-26 14:27:36 +0100 | [diff] [blame] | 455 | soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, ".")) |
| 456 | soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir()) |
Alex Márquez Pérez MuñÃz DÃaz Púras Thaureaux | 947fdbf | 2021-11-10 09:55:20 -0500 | [diff] [blame] | 457 | soongBuildEnv.Set("LOG_DIR", config.LogsDir()) |
Jingwen Chen | 3b13b61 | 2022-10-17 12:14:26 +0000 | [diff] [blame] | 458 | soongBuildEnv.Set("BAZEL_DEPS_FILE", absPath(ctx, filepath.Join(config.BazelOutDir(), "bazel.list"))) |
Lukacs T. Berki | 7690c09 | 2021-02-26 14:27:36 +0100 | [diff] [blame] | 459 | |
Lukacs T. Berki | d1e3f1f | 2021-03-16 08:55:23 +0100 | [diff] [blame] | 460 | // For Soong bootstrapping tests |
| 461 | if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" { |
| 462 | soongBuildEnv.Set("ALLOW_MISSING_DEPENDENCIES", "true") |
| 463 | } |
| 464 | |
Paul Duffin | 5e85c66 | 2021-03-05 12:26:14 +0000 | [diff] [blame] | 465 | err := writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap()) |
| 466 | if err != nil { |
| 467 | ctx.Fatalf("failed to write environment file %s: %s", envFile, err) |
| 468 | } |
Lukacs T. Berki | 7690c09 | 2021-02-26 14:27:36 +0100 | [diff] [blame] | 469 | |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 470 | func() { |
Nan Zhang | 17f2767 | 2018-12-12 16:01:49 -0800 | [diff] [blame] | 471 | ctx.BeginTrace(metrics.RunSoong, "environment check") |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 472 | defer ctx.EndTrace() |
| 473 | |
Lukacs T. Berki | e1df43f | 2021-09-08 15:31:14 +0200 | [diff] [blame] | 474 | checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(soongBuildTag)) |
Lukacs T. Berki | f8e2428 | 2021-04-14 10:31:00 +0200 | [diff] [blame] | 475 | |
Chris Parsons | ef615e5 | 2022-08-18 22:04:11 -0400 | [diff] [blame] | 476 | if config.BazelBuildEnabled() || config.Bp2Build() { |
Lukacs T. Berki | c541cd2 | 2022-10-26 07:26:50 +0000 | [diff] [blame] | 477 | checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(bp2buildFilesTag)) |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 478 | } |
| 479 | |
| 480 | if config.JsonModuleGraph() { |
Lukacs T. Berki | e1df43f | 2021-09-08 15:31:14 +0200 | [diff] [blame] | 481 | checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(jsonModuleGraphTag)) |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 482 | } |
| 483 | |
| 484 | if config.Queryview() { |
Lukacs T. Berki | e1df43f | 2021-09-08 15:31:14 +0200 | [diff] [blame] | 485 | checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(queryviewTag)) |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 486 | } |
| 487 | |
Spandan Das | 5af0bd3 | 2022-09-28 20:43:08 +0000 | [diff] [blame] | 488 | if config.ApiBp2build() { |
| 489 | checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(apiBp2buildTag)) |
| 490 | } |
| 491 | |
Lukacs T. Berki | 89fcdcb | 2021-09-07 09:10:33 +0200 | [diff] [blame] | 492 | if config.SoongDocs() { |
Lukacs T. Berki | e1df43f | 2021-09-08 15:31:14 +0200 | [diff] [blame] | 493 | checkEnvironmentFile(soongBuildEnv, config.UsedEnvFile(soongDocsTag)) |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 494 | } |
| 495 | }() |
| 496 | |
Colin Cross | 9191df2 | 2021-10-29 13:11:32 -0700 | [diff] [blame] | 497 | runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob", |
Sasha Smundak | 7ae80a7 | 2021-04-09 12:03:51 -0700 | [diff] [blame] | 498 | map[string]string{"github.com/google/blueprint": "build/blueprint"}) |
Dan Willemsen | 5af1cbe | 2018-07-05 21:46:51 -0700 | [diff] [blame] | 499 | |
Lukacs T. Berki | 56ebaf3 | 2021-08-12 14:03:55 +0200 | [diff] [blame] | 500 | ninja := func(name, ninjaFile string, targets ...string) { |
Nan Zhang | 17f2767 | 2018-12-12 16:01:49 -0800 | [diff] [blame] | 501 | ctx.BeginTrace(metrics.RunSoong, name) |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 502 | defer ctx.EndTrace() |
| 503 | |
Dan Willemsen | b82471a | 2018-05-17 16:37:09 -0700 | [diff] [blame] | 504 | fifo := filepath.Join(config.OutDir(), ".ninja_fifo") |
Colin Cross | b98d3bc | 2019-03-21 16:02:58 -0700 | [diff] [blame] | 505 | nr := status.NewNinjaReader(ctx, ctx.Status.StartTool(), fifo) |
| 506 | defer nr.Close() |
Dan Willemsen | b82471a | 2018-05-17 16:37:09 -0700 | [diff] [blame] | 507 | |
Lukacs T. Berki | 56ebaf3 | 2021-08-12 14:03:55 +0200 | [diff] [blame] | 508 | ninjaArgs := []string{ |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 509 | "-d", "keepdepfile", |
Dan Willemsen | 0821822 | 2020-05-18 14:02:02 -0700 | [diff] [blame] | 510 | "-d", "stats", |
Dan Willemsen | 6587bed | 2020-04-18 20:25:59 -0700 | [diff] [blame] | 511 | "-o", "usesphonyoutputs=yes", |
| 512 | "-o", "preremoveoutputs=yes", |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 513 | "-w", "dupbuild=err", |
Dan Willemsen | 6587bed | 2020-04-18 20:25:59 -0700 | [diff] [blame] | 514 | "-w", "outputdir=err", |
| 515 | "-w", "missingoutfile=err", |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 516 | "-j", strconv.Itoa(config.Parallel()), |
Dan Willemsen | 0273667 | 2018-07-17 17:54:31 -0700 | [diff] [blame] | 517 | "--frontend_file", fifo, |
Lukacs T. Berki | 56ebaf3 | 2021-08-12 14:03:55 +0200 | [diff] [blame] | 518 | "-f", filepath.Join(config.SoongOutDir(), ninjaFile), |
| 519 | } |
| 520 | |
Usta Shrestha | 8dc8b0a | 2022-08-10 17:39:37 -0400 | [diff] [blame] | 521 | if extra, ok := config.Environment().Get("SOONG_UI_NINJA_ARGS"); ok { |
| 522 | ctx.Printf(`CAUTION: arguments in $SOONG_UI_NINJA_ARGS=%q, e.g. "-n", can make soong_build FAIL or INCORRECT`, extra) |
| 523 | ninjaArgs = append(ninjaArgs, strings.Fields(extra)...) |
| 524 | } |
| 525 | |
Lukacs T. Berki | 56ebaf3 | 2021-08-12 14:03:55 +0200 | [diff] [blame] | 526 | ninjaArgs = append(ninjaArgs, targets...) |
| 527 | cmd := Command(ctx, config, "soong "+name, |
| 528 | config.PrebuiltBuildTool("ninja"), ninjaArgs...) |
Jingwen Chen | 7c6089a | 2020-11-02 02:56:20 -0500 | [diff] [blame] | 529 | |
Lukacs T. Berki | b14ad7b | 2021-03-09 10:43:57 +0100 | [diff] [blame] | 530 | var ninjaEnv Environment |
Lukacs T. Berki | 73ab928 | 2021-03-10 10:48:39 +0100 | [diff] [blame] | 531 | |
| 532 | // This is currently how the command line to invoke soong_build finds the |
| 533 | // root of the source tree and the output root |
Lukacs T. Berki | b14ad7b | 2021-03-09 10:43:57 +0100 | [diff] [blame] | 534 | ninjaEnv.Set("TOP", os.Getenv("TOP")) |
Lukacs T. Berki | 7d613bf | 2021-03-02 10:09:41 +0100 | [diff] [blame] | 535 | |
Lukacs T. Berki | b14ad7b | 2021-03-09 10:43:57 +0100 | [diff] [blame] | 536 | cmd.Environment = &ninjaEnv |
Dan Willemsen | 99a75cd | 2017-08-04 16:04:04 -0700 | [diff] [blame] | 537 | cmd.Sandbox = soongSandbox |
Colin Cross | 7b97ecd | 2019-06-19 13:17:59 -0700 | [diff] [blame] | 538 | cmd.RunAndStreamOrFatal() |
Dan Willemsen | 1e70446 | 2016-08-21 15:17:17 -0700 | [diff] [blame] | 539 | } |
Lukacs T. Berki | 56ebaf3 | 2021-08-12 14:03:55 +0200 | [diff] [blame] | 540 | |
Lukacs T. Berki | a1b9372 | 2021-09-02 17:23:06 +0200 | [diff] [blame] | 541 | targets := make([]string, 0, 0) |
Lukacs T. Berki | 56ebaf3 | 2021-08-12 14:03:55 +0200 | [diff] [blame] | 542 | |
Lukacs T. Berki | a1b9372 | 2021-09-02 17:23:06 +0200 | [diff] [blame] | 543 | if config.JsonModuleGraph() { |
| 544 | targets = append(targets, config.ModuleGraphFile()) |
Lukacs T. Berki | 56ebaf3 | 2021-08-12 14:03:55 +0200 | [diff] [blame] | 545 | } |
| 546 | |
Lukacs T. Berki | a1b9372 | 2021-09-02 17:23:06 +0200 | [diff] [blame] | 547 | if config.Bp2Build() { |
Lukacs T. Berki | c541cd2 | 2022-10-26 07:26:50 +0000 | [diff] [blame] | 548 | targets = append(targets, config.Bp2BuildWorkspaceMarkerFile()) |
Lukacs T. Berki | a1b9372 | 2021-09-02 17:23:06 +0200 | [diff] [blame] | 549 | } |
| 550 | |
Lukacs T. Berki | 3a82169 | 2021-09-06 17:08:02 +0200 | [diff] [blame] | 551 | if config.Queryview() { |
| 552 | targets = append(targets, config.QueryviewMarkerFile()) |
| 553 | } |
| 554 | |
Spandan Das | 5af0bd3 | 2022-09-28 20:43:08 +0000 | [diff] [blame] | 555 | if config.ApiBp2build() { |
| 556 | targets = append(targets, config.ApiBp2buildMarkerFile()) |
| 557 | } |
| 558 | |
Lukacs T. Berki | c6012f3 | 2021-09-06 18:31:46 +0200 | [diff] [blame] | 559 | if config.SoongDocs() { |
| 560 | targets = append(targets, config.SoongDocsHtml()) |
| 561 | } |
| 562 | |
Lukacs T. Berki | a1b9372 | 2021-09-02 17:23:06 +0200 | [diff] [blame] | 563 | if config.SoongBuildInvocationNeeded() { |
| 564 | // This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build(). |
Cole Faust | 65f298c | 2021-10-28 16:05:13 -0700 | [diff] [blame] | 565 | targets = append(targets, config.SoongNinjaFile()) |
Lukacs T. Berki | a1b9372 | 2021-09-02 17:23:06 +0200 | [diff] [blame] | 566 | } |
| 567 | |
Jingwen Chen | eb76c43 | 2021-01-28 08:22:12 -0500 | [diff] [blame] | 568 | if shouldCollectBuildSoongMetrics(config) { |
usta | 7f56eaa | 2022-10-27 23:01:02 -0400 | [diff] [blame] | 569 | soongBuildMetricsFile := filepath.Join(config.LogsDir(), "soong_build_metrics.pb") |
| 570 | if err := os.Remove(soongBuildMetricsFile); err != nil && !os.IsNotExist(err) { |
| 571 | ctx.Verbosef("Failed to remove %s", soongBuildMetricsFile) |
Dan Willemsen | de36044 | 2022-04-20 21:21:55 -0700 | [diff] [blame] | 572 | } |
usta | 7f56eaa | 2022-10-27 23:01:02 -0400 | [diff] [blame] | 573 | defer func() { |
| 574 | soongBuildMetrics := loadSoongBuildMetrics(ctx, soongBuildMetricsFile) |
| 575 | if soongBuildMetrics != nil { |
| 576 | logSoongBuildMetrics(ctx, soongBuildMetrics) |
| 577 | if ctx.Metrics != nil { |
| 578 | ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics) |
| 579 | } |
| 580 | } |
| 581 | }() |
Jingwen Chen | eb76c43 | 2021-01-28 08:22:12 -0500 | [diff] [blame] | 582 | } |
usta | 7f56eaa | 2022-10-27 23:01:02 -0400 | [diff] [blame] | 583 | ninja("bootstrap", "bootstrap.ninja", targets...) |
Colin Cross | b72c909 | 2020-02-10 11:23:49 -0800 | [diff] [blame] | 584 | |
Colin Cross | 8ba7d47 | 2020-06-25 11:27:52 -0700 | [diff] [blame] | 585 | distGzipFile(ctx, config, config.SoongNinjaFile(), "soong") |
Jihoon Kang | 9f4f8a3 | 2022-08-16 00:57:30 +0000 | [diff] [blame] | 586 | distFile(ctx, config, config.SoongVarsFile(), "soong") |
Colin Cross | 8ba7d47 | 2020-06-25 11:27:52 -0700 | [diff] [blame] | 587 | |
Anton Hansson | 5e5c48b | 2020-11-27 12:35:20 +0000 | [diff] [blame] | 588 | if !config.SkipKati() { |
Colin Cross | 8ba7d47 | 2020-06-25 11:27:52 -0700 | [diff] [blame] | 589 | distGzipFile(ctx, config, config.SoongAndroidMk(), "soong") |
| 590 | distGzipFile(ctx, config, config.SoongMakeVarsMk(), "soong") |
| 591 | } |
| 592 | |
Rob Seymour | 33cd10d | 2022-03-02 23:10:25 +0000 | [diff] [blame] | 593 | if config.JsonModuleGraph() { |
| 594 | distGzipFile(ctx, config, config.ModuleGraphFile(), "soong") |
| 595 | } |
Colin Cross | b72c909 | 2020-02-10 11:23:49 -0800 | [diff] [blame] | 596 | } |
| 597 | |
Lukacs T. Berki | 90b4334 | 2021-11-02 14:42:04 +0100 | [diff] [blame] | 598 | func runMicrofactory(ctx Context, config Config, name string, pkg string, mapping map[string]string) { |
Sasha Smundak | 7ae80a7 | 2021-04-09 12:03:51 -0700 | [diff] [blame] | 599 | ctx.BeginTrace(metrics.RunSoong, name) |
| 600 | defer ctx.EndTrace() |
| 601 | cfg := microfactory.Config{TrimPath: absPath(ctx, ".")} |
| 602 | for pkgPrefix, pathPrefix := range mapping { |
| 603 | cfg.Map(pkgPrefix, pathPrefix) |
| 604 | } |
| 605 | |
Lukacs T. Berki | 90b4334 | 2021-11-02 14:42:04 +0100 | [diff] [blame] | 606 | exePath := filepath.Join(config.SoongOutDir(), name) |
Sasha Smundak | 7ae80a7 | 2021-04-09 12:03:51 -0700 | [diff] [blame] | 607 | dir := filepath.Dir(exePath) |
| 608 | if err := os.MkdirAll(dir, 0777); err != nil { |
| 609 | ctx.Fatalf("cannot create %s: %s", dir, err) |
| 610 | } |
| 611 | if _, err := microfactory.Build(&cfg, exePath, pkg); err != nil { |
| 612 | ctx.Fatalf("failed to build %s: %s", name, err) |
| 613 | } |
| 614 | } |
| 615 | |
Jingwen Chen | eb76c43 | 2021-01-28 08:22:12 -0500 | [diff] [blame] | 616 | func shouldCollectBuildSoongMetrics(config Config) bool { |
Jingwen Chen | dd9725c | 2021-06-24 08:41:16 +0000 | [diff] [blame] | 617 | // Do not collect metrics protobuf if the soong_build binary ran as the |
| 618 | // bp2build converter or the JSON graph dump. |
Lukacs T. Berki | a1b9372 | 2021-09-02 17:23:06 +0200 | [diff] [blame] | 619 | return config.SoongBuildInvocationNeeded() |
Jingwen Chen | eb76c43 | 2021-01-28 08:22:12 -0500 | [diff] [blame] | 620 | } |
| 621 | |
usta | 7f56eaa | 2022-10-27 23:01:02 -0400 | [diff] [blame] | 622 | func loadSoongBuildMetrics(ctx Context, soongBuildMetricsFile string) *soong_metrics_proto.SoongBuildMetrics { |
Dan Willemsen | de36044 | 2022-04-20 21:21:55 -0700 | [diff] [blame] | 623 | buf, err := os.ReadFile(soongBuildMetricsFile) |
| 624 | if errors.Is(err, fs.ErrNotExist) { |
| 625 | // Soong may not have run during this invocation |
Usta Shrestha | db46a9b | 2022-07-11 11:29:56 -0400 | [diff] [blame] | 626 | ctx.Verbosef("Failed to read metrics file, %s: %s", soongBuildMetricsFile, err) |
Dan Willemsen | de36044 | 2022-04-20 21:21:55 -0700 | [diff] [blame] | 627 | return nil |
| 628 | } else if err != nil { |
Colin Cross | b72c909 | 2020-02-10 11:23:49 -0800 | [diff] [blame] | 629 | ctx.Fatalf("Failed to load %s: %s", soongBuildMetricsFile, err) |
| 630 | } |
| 631 | soongBuildMetrics := &soong_metrics_proto.SoongBuildMetrics{} |
| 632 | err = proto.Unmarshal(buf, soongBuildMetrics) |
| 633 | if err != nil { |
| 634 | ctx.Fatalf("Failed to unmarshal %s: %s", soongBuildMetricsFile, err) |
| 635 | } |
| 636 | return soongBuildMetrics |
| 637 | } |
| 638 | |
| 639 | func logSoongBuildMetrics(ctx Context, metrics *soong_metrics_proto.SoongBuildMetrics) { |
| 640 | ctx.Verbosef("soong_build metrics:") |
| 641 | ctx.Verbosef(" modules: %v", metrics.GetModules()) |
| 642 | ctx.Verbosef(" variants: %v", metrics.GetVariants()) |
| 643 | ctx.Verbosef(" max heap size: %v MB", metrics.GetMaxHeapSize()/1e6) |
| 644 | ctx.Verbosef(" total allocation count: %v", metrics.GetTotalAllocCount()) |
| 645 | ctx.Verbosef(" total allocation size: %v MB", metrics.GetTotalAllocSize()/1e6) |
| 646 | |
Dan Willemsen | 1e70446 | 2016-08-21 15:17:17 -0700 | [diff] [blame] | 647 | } |