blob: f80868c33440905a448513078a5cb5c5c5206a65 [file] [log] [blame]
Dan Willemsen1e704462016-08-21 15:17:17 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
Kousik Kumar3ff037e2022-01-25 22:11:01 -050018 "encoding/json"
Jeongik Chaa87506f2023-06-01 23:16:41 +090019 "errors"
Ramy Medhat0fc67eb2020-08-12 01:26:23 -040020 "fmt"
Kousik Kumar3ff037e2022-01-25 22:11:01 -050021 "io/ioutil"
Kousik Kumar4c180ad2022-05-27 07:48:37 -040022 "math/rand"
Dan Willemsenc2af0be2017-01-20 14:10:01 -080023 "os"
Kousik Kumar84bd5bf2022-01-26 23:32:22 -050024 "os/exec"
Cole Faust583dfb42023-09-28 13:56:30 -070025 "os/user"
Dan Willemsen1e704462016-08-21 15:17:17 -070026 "path/filepath"
27 "runtime"
28 "strconv"
29 "strings"
Kousik Kumar4c180ad2022-05-27 07:48:37 -040030 "syscall"
Nan Zhang2e6a4ff2018-02-14 13:27:26 -080031 "time"
Jeff Gastonefc1b412017-03-29 17:29:06 -070032
33 "android/soong/shared"
Kousik Kumarec478642020-09-21 13:39:24 -040034
Dan Willemsen4591b642021-05-24 14:24:12 -070035 "google.golang.org/protobuf/proto"
Patrice Arruda96850362020-08-11 20:41:11 +000036
37 smpb "android/soong/ui/metrics/metrics_proto"
Dan Willemsen1e704462016-08-21 15:17:17 -070038)
39
Kousik Kumar3ff037e2022-01-25 22:11:01 -050040const (
Chris Parsons53f68ae2022-03-03 12:01:40 -050041 envConfigDir = "vendor/google/tools/soong_config"
42 jsonSuffix = "json"
Kousik Kumar3ff037e2022-01-25 22:11:01 -050043)
44
Kousik Kumar4c180ad2022-05-27 07:48:37 -040045var (
Kevin Dagostino096ab2f2023-03-03 19:47:17 +000046 rbeRandPrefix int
47 googleProdCredsExistCache bool
Kousik Kumar4c180ad2022-05-27 07:48:37 -040048)
49
50func init() {
51 rand.Seed(time.Now().UnixNano())
52 rbeRandPrefix = rand.Intn(1000)
53}
54
Dan Willemsen1e704462016-08-21 15:17:17 -070055type Config struct{ *configImpl }
56
57type configImpl struct {
Lukacs T. Berkia1b93722021-09-02 17:23:06 +020058 // Some targets that are implemented in soong_build
59 // (bp2build, json-module-graph) are not here and have their own bits below.
Colin Cross28f527c2019-11-26 16:19:04 -080060 arguments []string
61 goma bool
62 environ *Environment
63 distDir string
64 buildDateTime string
MarkDacek6614d9c2022-12-07 21:57:38 +000065 logsPrefix string
Dan Willemsen1e704462016-08-21 15:17:17 -070066
67 // From the arguments
MarkDacekf47e1422023-04-19 16:47:36 +000068 parallel int
69 keepGoing int
70 verbose bool
71 checkbuild bool
72 dist bool
73 jsonModuleGraph bool
MarkDacekf47e1422023-04-19 16:47:36 +000074 bp2build bool
75 queryview bool
76 reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
77 soongDocs bool
78 multitreeBuild bool // This is a multitree build.
79 skipConfig bool
80 skipKati bool
81 skipKatiNinja bool
82 skipSoong bool
83 skipNinja bool
84 skipSoongTests bool
85 searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces
86 skipMetricsUpload bool
87 buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
88 buildFromTextStub bool
MarkDacek396491e2023-06-14 19:41:18 +000089 ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
90 bazelExitCode int32 // For b runs - necessary for updating NonZeroExit
91 besId string // For b runs, to identify the BuildEventService logs
Dan Willemsen1e704462016-08-21 15:17:17 -070092
93 // From the product config
Dan Willemsen6ab79db2018-05-02 00:06:28 -070094 katiArgs []string
95 ninjaArgs []string
96 katiSuffix string
97 targetDevice string
98 targetDeviceDir string
Spandan Dasa3639e62021-05-25 19:14:02 +000099 sandboxConfig *SandboxConfig
Dan Willemsen3d60b112018-04-04 22:25:56 -0700100
Dan Willemsen2bb82d02019-12-27 09:35:42 -0800101 // Autodetected
102 totalRAM uint64
103
Dan Willemsene3336352020-01-02 19:10:38 -0800104 brokenDupRules bool
105 brokenUsesNetwork bool
106 brokenNinjaEnvVars []string
Dan Willemsen18490112018-05-25 16:30:04 -0700107
108 pathReplaced bool
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +0000109
MarkDacekb78465d2022-10-18 20:10:16 +0000110 bazelProdMode bool
MarkDacekb78465d2022-10-18 20:10:16 +0000111 bazelStagingMode bool
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +0000112
Colin Crossf3bdbcb2021-06-01 11:43:55 -0700113 // Set by multiproduct_kati
114 emptyNinjaFile bool
Yu Liu6e13b402021-07-27 14:29:06 -0700115
116 metricsUploader string
MarkDacekd06db5d2022-11-29 00:47:59 +0000117
118 bazelForceEnabledModules string
Spandan Dasc5763832022-11-08 18:42:16 +0000119
Sam Delmerico98a73292023-02-21 11:50:29 -0500120 includeTags []string
121 sourceRootDirs []string
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900122
123 // Data source to write ninja weight list
124 ninjaWeightListSource NinjaWeightListSource
Dan Willemsen1e704462016-08-21 15:17:17 -0700125}
126
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900127type NinjaWeightListSource uint
128
129const (
130 // ninja doesn't use weight list.
131 NOT_USED NinjaWeightListSource = iota
132 // ninja uses weight list based on previous builds by ninja log
133 NINJA_LOG
134 // ninja thinks every task has the same weight.
135 EVENLY_DISTRIBUTED
Jeongik Cha518f3ea2023-03-19 00:12:39 +0900136 // ninja uses an external custom weight list
137 EXTERNAL_FILE
Jeongik Chae114e602023-03-19 00:12:39 +0900138 // ninja uses a prioritized module list from Soong
139 HINT_FROM_SOONG
Jeongik Chaa87506f2023-06-01 23:16:41 +0900140 // If ninja log exists, use NINJA_LOG, if not, use HINT_FROM_SOONG instead.
141 // We can assume it is an incremental build if ninja log exists.
142 DEFAULT
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900143)
Dan Willemsenc2af0be2017-01-20 14:10:01 -0800144const srcDirFileCheck = "build/soong/root.bp"
145
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700146var buildFiles = []string{"Android.mk", "Android.bp"}
147
Patrice Arruda13848222019-04-22 17:12:02 -0700148type BuildAction uint
149
150const (
151 // Builds all of the modules and their dependencies of a specified directory, relative to the root
152 // directory of the source tree.
153 BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota
154
155 // Builds all of the modules and their dependencies of a list of specified directories. All specified
156 // directories are relative to the root directory of the source tree.
157 BUILD_MODULES_IN_DIRECTORIES
Patrice Arruda39282062019-06-20 16:35:12 -0700158
159 // Build a list of specified modules. If none was specified, simply build the whole source tree.
160 BUILD_MODULES
Patrice Arruda13848222019-04-22 17:12:02 -0700161)
162
163// checkTopDir validates that the current directory is at the root directory of the source tree.
164func checkTopDir(ctx Context) {
165 if _, err := os.Stat(srcDirFileCheck); err != nil {
166 if os.IsNotExist(err) {
167 ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck)
168 }
169 ctx.Fatalln("Error verifying tree state:", err)
170 }
171}
172
MarkDacek7901e582023-01-09 19:48:01 +0000173func loadEnvConfig(ctx Context, config *configImpl, bc string) error {
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500174 if bc == "" {
175 return nil
176 }
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500177
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500178 configDirs := []string{
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500179 config.OutDir(),
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500180 os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"),
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500181 envConfigDir,
182 }
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500183 for _, dir := range configDirs {
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500184 cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix))
185 envVarsJSON, err := ioutil.ReadFile(cfgFile)
186 if err != nil {
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500187 continue
188 }
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500189 ctx.Verbosef("Loading config file %v\n", cfgFile)
190 var envVars map[string]map[string]string
191 if err := json.Unmarshal(envVarsJSON, &envVars); err != nil {
192 fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error())
193 continue
194 }
195 for k, v := range envVars["env"] {
196 if os.Getenv(k) != "" {
197 continue
198 }
199 config.environ.Set(k, v)
200 }
201 ctx.Verbosef("Finished loading config file %v\n", cfgFile)
202 break
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500203 }
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500204
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500205 return nil
206}
207
Chris Parsonsb6e96902022-10-31 20:08:45 -0400208func defaultBazelProdMode(cfg *configImpl) bool {
MarkDacekd06db5d2022-11-29 00:47:59 +0000209 // Environment flag to disable Bazel for users which experience
Chris Parsonsb6e96902022-10-31 20:08:45 -0400210 // broken bazel-handled builds, or significant performance regressions.
211 if cfg.IsBazelMixedBuildForceDisabled() {
212 return false
213 }
214 // Darwin-host builds are currently untested with Bazel.
215 if runtime.GOOS == "darwin" {
216 return false
217 }
Chris Parsons035e03a2022-11-01 14:25:45 -0400218 return true
Chris Parsonsb6e96902022-10-31 20:08:45 -0400219}
220
MarkDacekd33c2fd2023-05-04 20:40:04 +0000221func UploadOnlyConfig(ctx Context, args ...string) Config {
MarkDacek6614d9c2022-12-07 21:57:38 +0000222 ret := &configImpl{
223 environ: OsEnvironment(),
224 sandboxConfig: &SandboxConfig{},
225 }
MarkDacekd33c2fd2023-05-04 20:40:04 +0000226 ret.parseArgs(ctx, args)
MarkDacek7901e582023-01-09 19:48:01 +0000227 srcDir := absPath(ctx, ".")
228 bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
229 if err := loadEnvConfig(ctx, ret, bc); err != nil {
230 ctx.Fatalln("Failed to parse env config files: %v", err)
231 }
232 ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
MarkDacek6614d9c2022-12-07 21:57:38 +0000233 return Config{ret}
234}
235
Dan Willemsen1e704462016-08-21 15:17:17 -0700236func NewConfig(ctx Context, args ...string) Config {
237 ret := &configImpl{
Jeongik Chaf2ecf762023-05-19 14:03:45 +0900238 environ: OsEnvironment(),
239 sandboxConfig: &SandboxConfig{},
Jeongik Chaa87506f2023-06-01 23:16:41 +0900240 ninjaWeightListSource: DEFAULT,
Dan Willemsen1e704462016-08-21 15:17:17 -0700241 }
242
Patrice Arruda90109172020-07-28 18:07:27 +0000243 // Default matching ninja
Dan Willemsen9b587492017-07-10 22:13:00 -0700244 ret.parallel = runtime.NumCPU() + 2
245 ret.keepGoing = 1
246
Dan Willemsen2bb82d02019-12-27 09:35:42 -0800247 ret.totalRAM = detectTotalRAM(ctx)
Dan Willemsen9b587492017-07-10 22:13:00 -0700248 ret.parseArgs(ctx, args)
Jeongik Chae114e602023-03-19 00:12:39 +0900249
250 if ret.ninjaWeightListSource == HINT_FROM_SOONG {
Jeongik Chaa87506f2023-06-01 23:16:41 +0900251 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "always")
252 } else if ret.ninjaWeightListSource == DEFAULT {
253 defaultNinjaWeightListSource := NINJA_LOG
254 if _, err := os.Stat(filepath.Join(ret.OutDir(), ninjaLogFileName)); errors.Is(err, os.ErrNotExist) {
255 ctx.Verboseln("$OUT/.ninja_log doesn't exist, use HINT_FROM_SOONG instead")
256 defaultNinjaWeightListSource = HINT_FROM_SOONG
257 } else {
258 ctx.Verboseln("$OUT/.ninja_log exist, use NINJA_LOG")
259 }
260 ret.ninjaWeightListSource = defaultNinjaWeightListSource
261 // soong_build generates ninja hint depending on ninja log existence.
262 // Set it "depend" to avoid soong re-run due to env variable change.
263 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "depend")
Jeongik Chae114e602023-03-19 00:12:39 +0900264 }
Jeongik Chaa87506f2023-06-01 23:16:41 +0900265
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800266 // Make sure OUT_DIR is set appropriately
Dan Willemsen02f3add2017-05-12 13:50:19 -0700267 if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
268 ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
269 } else {
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800270 outDir := "out"
271 if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok {
272 if wd, err := os.Getwd(); err != nil {
273 ctx.Fatalln("Failed to get working directory:", err)
274 } else {
275 outDir = filepath.Join(baseDir, filepath.Base(wd))
276 }
277 }
278 ret.environ.Set("OUT_DIR", outDir)
279 }
280
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500281 // loadEnvConfig needs to know what the OUT_DIR is, so it should
282 // be called after we determine the appropriate out directory.
MarkDacek7901e582023-01-09 19:48:01 +0000283 bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
284
285 if bc != "" {
Kousik Kumarc8818332023-01-16 16:33:05 +0000286 if err := loadEnvConfig(ctx, ret, bc); err != nil {
MarkDacek7901e582023-01-09 19:48:01 +0000287 ctx.Fatalln("Failed to parse env config files: %v", err)
288 }
Kousik Kumar6d1e3482023-07-24 03:44:16 +0000289 if !ret.canSupportRBE() {
290 // Explicitly set USE_RBE env variable to false when we cannot run
291 // an RBE build to avoid ninja local execution pool issues.
292 ret.environ.Set("USE_RBE", "false")
293 }
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500294 }
295
Dan Willemsen2d31a442018-10-20 21:33:41 -0700296 if distDir, ok := ret.environ.Get("DIST_DIR"); ok {
297 ret.distDir = filepath.Clean(distDir)
298 } else {
299 ret.distDir = filepath.Join(ret.OutDir(), "dist")
300 }
Dan Willemsend50e89f2018-10-16 17:49:25 -0700301
Spandan Das05063612021-06-25 01:39:04 +0000302 if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok {
303 ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false")
304 }
305
Dan Willemsen1e704462016-08-21 15:17:17 -0700306 ret.environ.Unset(
307 // We're already using it
308 "USE_SOONG_UI",
309
310 // We should never use GOROOT/GOPATH from the shell environment
311 "GOROOT",
312 "GOPATH",
313
314 // These should only come from Soong, not the environment.
315 "CLANG",
316 "CLANG_CXX",
317 "CCC_CC",
318 "CCC_CXX",
319
320 // Used by the goma compiler wrapper, but should only be set by
321 // gomacc
322 "GOMACC_PATH",
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800323
324 // We handle this above
325 "OUT_DIR_COMMON_BASE",
Dan Willemsen68a09852017-04-18 13:56:57 -0700326
Dan Willemsen2d31a442018-10-20 21:33:41 -0700327 // This is handled above too, and set for individual commands later
328 "DIST_DIR",
329
Dan Willemsen68a09852017-04-18 13:56:57 -0700330 // Variables that have caused problems in the past
Dan Willemsen1c504d92019-11-18 19:13:53 +0000331 "BASH_ENV",
Dan Willemsenebfe33a2018-05-01 10:07:50 -0700332 "CDPATH",
Dan Willemsen68a09852017-04-18 13:56:57 -0700333 "DISPLAY",
334 "GREP_OPTIONS",
Nathan Egge7b067fb2023-02-17 17:54:31 +0000335 "JAVAC",
Dan Willemsenebfe33a2018-05-01 10:07:50 -0700336 "NDK_ROOT",
Dan Willemsen00fcb262018-08-15 15:35:38 -0700337 "POSIXLY_CORRECT",
Dan Willemsenc40e10b2017-07-11 14:30:00 -0700338
339 // Drop make flags
340 "MAKEFLAGS",
341 "MAKELEVEL",
342 "MFLAGS",
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700343
344 // Set in envsetup.sh, reset in makefiles
345 "ANDROID_JAVA_TOOLCHAIN",
Colin Cross7f09c402018-07-11 14:49:31 -0700346
347 // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional
348 "ANDROID_BUILD_TOP",
349 "ANDROID_HOST_OUT",
350 "ANDROID_PRODUCT_OUT",
351 "ANDROID_HOST_OUT_TESTCASES",
352 "ANDROID_TARGET_OUT_TESTCASES",
353 "ANDROID_TOOLCHAIN",
354 "ANDROID_TOOLCHAIN_2ND_ARCH",
355 "ANDROID_DEV_SCRIPTS",
356 "ANDROID_EMULATOR_PREBUILTS",
357 "ANDROID_PRE_BUILD_PATHS",
Dan Willemsen1e704462016-08-21 15:17:17 -0700358 )
359
Kousik Kumarb328f6d2020-10-19 01:45:46 -0400360 if ret.UseGoma() || ret.ForceUseGoma() {
361 ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.")
362 ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.")
Kousik Kumarec478642020-09-21 13:39:24 -0400363 }
364
Dan Willemsen1e704462016-08-21 15:17:17 -0700365 // Tell python not to spam the source tree with .pyc files.
366 ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
367
Ramy Medhatca1e44c2020-07-16 12:18:37 -0400368 tmpDir := absPath(ctx, ret.TempDir())
369 ret.environ.Set("TMPDIR", tmpDir)
Dan Willemsen32a669b2018-03-08 19:42:00 -0800370
Dan Willemsen70c1ff82019-08-21 14:56:13 -0700371 // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes
372 symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(),
373 "llvm-binutils-stable/llvm-symbolizer")
374 ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath))
375
Dan Willemsenc2af0be2017-01-20 14:10:01 -0800376 // Precondition: the current directory is the top of the source tree
Patrice Arruda13848222019-04-22 17:12:02 -0700377 checkTopDir(ctx)
Dan Willemsenc2af0be2017-01-20 14:10:01 -0800378
Yu Liu6e13b402021-07-27 14:29:06 -0700379 srcDir := absPath(ctx, ".")
380 if strings.ContainsRune(srcDir, ' ') {
Colin Cross1f6faeb2019-09-23 15:52:40 -0700381 ctx.Println("You are building in a directory whose absolute path contains a space character:")
382 ctx.Println()
383 ctx.Printf("%q\n", srcDir)
384 ctx.Println()
385 ctx.Fatalln("Directory names containing spaces are not supported")
Dan Willemsendb8457c2017-05-12 16:38:17 -0700386 }
387
Yu Liu6e13b402021-07-27 14:29:06 -0700388 ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
389
Dan Willemsendb8457c2017-05-12 16:38:17 -0700390 if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') {
Colin Cross1f6faeb2019-09-23 15:52:40 -0700391 ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:")
392 ctx.Println()
393 ctx.Printf("%q\n", outDir)
394 ctx.Println()
395 ctx.Fatalln("Directory names containing spaces are not supported")
Dan Willemsendb8457c2017-05-12 16:38:17 -0700396 }
397
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +0000398 if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') {
Colin Cross1f6faeb2019-09-23 15:52:40 -0700399 ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:")
400 ctx.Println()
401 ctx.Printf("%q\n", distDir)
402 ctx.Println()
403 ctx.Fatalln("Directory names containing spaces are not supported")
Dan Willemsendb8457c2017-05-12 16:38:17 -0700404 }
405
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700406 // Configure Java-related variables, including adding it to $PATH
Tobias Thierere59aeff2017-12-20 22:40:39 +0000407 java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag())
408 java9Home := filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag())
Pete Gillin1f52e932019-10-09 17:10:08 +0100409 java11Home := filepath.Join("prebuilts/jdk/jdk11", ret.HostPrebuiltTag())
Colin Cross59c1e6a2022-03-04 13:37:19 -0800410 java17Home := filepath.Join("prebuilts/jdk/jdk17", ret.HostPrebuiltTag())
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700411 javaHome := func() string {
412 if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok {
413 return override
414 }
Pete Gillina7a3d642019-11-07 18:58:42 +0000415 if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" {
416 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.")
Pete Gillin1f52e932019-10-09 17:10:08 +0100417 }
Sorin Basca7e094b32022-10-05 08:20:12 +0000418 if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" {
419 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 17 toolchain is now the global default.")
420 }
421 return java17Home
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700422 }()
423 absJavaHome := absPath(ctx, javaHome)
424
Dan Willemsened869522018-01-08 14:58:46 -0800425 ret.configureLocale(ctx)
426
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700427 newPath := []string{filepath.Join(absJavaHome, "bin")}
428 if path, ok := ret.environ.Get("PATH"); ok && path != "" {
429 newPath = append(newPath, path)
430 }
Pete Gillin1f52e932019-10-09 17:10:08 +0100431
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700432 ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME")
433 ret.environ.Set("JAVA_HOME", absJavaHome)
434 ret.environ.Set("ANDROID_JAVA_HOME", javaHome)
Tobias Thierere59aeff2017-12-20 22:40:39 +0000435 ret.environ.Set("ANDROID_JAVA8_HOME", java8Home)
436 ret.environ.Set("ANDROID_JAVA9_HOME", java9Home)
Pete Gillin1f52e932019-10-09 17:10:08 +0100437 ret.environ.Set("ANDROID_JAVA11_HOME", java11Home)
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700438 ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator)))
439
Colin Crossfe5ed4d2023-07-28 09:27:23 -0700440 // b/286885495, https://bugzilla.redhat.com/show_bug.cgi?id=2227130: some versions of Fedora include patches
441 // to unzip to enable zipbomb detection that incorrectly handle zip64 and data descriptors and fail on large
442 // zip files produced by soong_zip. Disable zipbomb detection.
443 ret.environ.Set("UNZIP_DISABLE_ZIPBOMB_DETECTION", "TRUE")
444
LaMont Jones52a72432023-03-09 18:19:35 +0000445 if ret.MultitreeBuild() {
446 ret.environ.Set("MULTITREE_BUILD", "true")
447 }
448
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800449 outDir := ret.OutDir()
450 buildDateTimeFile := filepath.Join(outDir, "build_date.txt")
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800451 if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" {
Colin Cross28f527c2019-11-26 16:19:04 -0800452 ret.buildDateTime = buildDateTime
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800453 } else {
Colin Cross28f527c2019-11-26 16:19:04 -0800454 ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10)
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800455 }
Colin Cross28f527c2019-11-26 16:19:04 -0800456
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800457 ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile)
458
Cole Faust583dfb42023-09-28 13:56:30 -0700459 if _, ok := ret.environ.Get("BUILD_USERNAME"); !ok {
460 username := "unknown"
461 if u, err := user.Current(); err == nil {
462 username = u.Username
463 } else {
464 ctx.Println("Failed to get current user:", err)
465 }
466 ret.environ.Set("BUILD_USERNAME", username)
467 }
468
Ramy Medhatca1e44c2020-07-16 12:18:37 -0400469 if ret.UseRBE() {
Ramy Medhat0fc67eb2020-08-12 01:26:23 -0400470 for k, v := range getRBEVars(ctx, Config{ret}) {
Ramy Medhatca1e44c2020-07-16 12:18:37 -0400471 ret.environ.Set(k, v)
472 }
473 }
474
Jihoon Kang1bff0342023-01-17 20:40:22 +0000475 if ret.BuildFromTextStub() {
476 // TODO(b/271443071): support hidden api check for from-text stub build
477 ret.environ.Set("UNSAFE_DISABLE_HIDDENAPI_FLAGS", "true")
478 }
479
Patrice Arruda83842d72020-12-08 19:42:08 +0000480 bpd := ret.BazelMetricsDir()
Patrice Arrudaaf880da2020-11-13 08:41:26 -0800481 if err := os.RemoveAll(bpd); err != nil {
482 ctx.Fatalf("Unable to remove bazel profile directory %q: %v", bpd, err)
483 }
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +0000484
Patrice Arruda96850362020-08-11 20:41:11 +0000485 c := Config{ret}
486 storeConfigMetrics(ctx, c)
487 return c
Dan Willemsen9b587492017-07-10 22:13:00 -0700488}
489
Patrice Arruda13848222019-04-22 17:12:02 -0700490// NewBuildActionConfig returns a build configuration based on the build action. The arguments are
491// processed based on the build action and extracts any arguments that belongs to the build action.
Dan Willemsence41e942019-07-29 23:39:30 -0700492func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config {
493 return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...)
Patrice Arruda13848222019-04-22 17:12:02 -0700494}
495
Patrice Arruda96850362020-08-11 20:41:11 +0000496// storeConfigMetrics selects a set of configuration information and store in
497// the metrics system for further analysis.
498func storeConfigMetrics(ctx Context, config Config) {
499 if ctx.Metrics == nil {
500 return
501 }
502
Liz Kammerca9cb2e2021-07-14 15:29:57 -0400503 ctx.Metrics.BuildConfig(buildConfig(config))
Patrice Arruda3edfd482020-10-13 23:58:41 +0000504
505 s := &smpb.SystemResourceInfo{
506 TotalPhysicalMemory: proto.Uint64(config.TotalRAM()),
507 AvailableCpus: proto.Int32(int32(runtime.NumCPU())),
508 }
509 ctx.Metrics.SystemResourceInfo(s)
Patrice Arruda96850362020-08-11 20:41:11 +0000510}
511
Jeongik Cha8d63d562023-03-17 03:52:13 +0900512func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource {
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900513 switch s {
514 case NINJA_LOG:
Jeongik Cha8d63d562023-03-17 03:52:13 +0900515 return smpb.BuildConfig_NINJA_LOG.Enum()
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900516 case EVENLY_DISTRIBUTED:
Jeongik Cha8d63d562023-03-17 03:52:13 +0900517 return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum()
Jeongik Cha518f3ea2023-03-19 00:12:39 +0900518 case EXTERNAL_FILE:
519 return smpb.BuildConfig_EXTERNAL_FILE.Enum()
Jeongik Chae114e602023-03-19 00:12:39 +0900520 case HINT_FROM_SOONG:
521 return smpb.BuildConfig_HINT_FROM_SOONG.Enum()
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900522 default:
Jeongik Cha8d63d562023-03-17 03:52:13 +0900523 return smpb.BuildConfig_NOT_USED.Enum()
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900524 }
525}
526
Liz Kammerca9cb2e2021-07-14 15:29:57 -0400527func buildConfig(config Config) *smpb.BuildConfig {
Yu Liue737a992021-10-04 13:21:41 -0700528 c := &smpb.BuildConfig{
Romain Jobredeaux0a7529b2022-10-26 12:56:41 -0400529 ForceUseGoma: proto.Bool(config.ForceUseGoma()),
530 UseGoma: proto.Bool(config.UseGoma()),
531 UseRbe: proto.Bool(config.UseRBE()),
532 BazelMixedBuild: proto.Bool(config.BazelBuildEnabled()),
533 ForceDisableBazelMixedBuild: proto.Bool(config.IsBazelMixedBuildForceDisabled()),
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900534 NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
Liz Kammerca9cb2e2021-07-14 15:29:57 -0400535 }
Yu Liue737a992021-10-04 13:21:41 -0700536 c.Targets = append(c.Targets, config.arguments...)
537
538 return c
Liz Kammerca9cb2e2021-07-14 15:29:57 -0400539}
540
Patrice Arruda13848222019-04-22 17:12:02 -0700541// getConfigArgs processes the command arguments based on the build action and creates a set of new
542// arguments to be accepted by Config.
Dan Willemsence41e942019-07-29 23:39:30 -0700543func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string {
Patrice Arruda13848222019-04-22 17:12:02 -0700544 // The next block of code verifies that the current directory is the root directory of the source
545 // tree. It then finds the relative path of dir based on the root directory of the source tree
546 // and verify that dir is inside of the source tree.
547 checkTopDir(ctx)
548 topDir, err := os.Getwd()
549 if err != nil {
550 ctx.Fatalf("Error retrieving top directory: %v", err)
551 }
Patrice Arrudababa9a92019-07-03 10:47:34 -0700552 dir, err = filepath.EvalSymlinks(dir)
553 if err != nil {
554 ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err)
555 }
Patrice Arruda13848222019-04-22 17:12:02 -0700556 dir, err = filepath.Abs(dir)
557 if err != nil {
558 ctx.Fatalf("Unable to find absolute path %s: %v", dir, err)
559 }
560 relDir, err := filepath.Rel(topDir, dir)
561 if err != nil {
562 ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err)
563 }
564 // If there are ".." in the path, it's not in the source tree.
565 if strings.Contains(relDir, "..") {
566 ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir)
567 }
568
569 configArgs := args[:]
570
571 // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to
572 // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules.
573 targetNamePrefix := "MODULES-IN-"
574 if inList("GET-INSTALL-PATH", configArgs) {
575 targetNamePrefix = "GET-INSTALL-PATH-IN-"
576 configArgs = removeFromList("GET-INSTALL-PATH", configArgs)
577 }
578
Patrice Arruda13848222019-04-22 17:12:02 -0700579 var targets []string
580
581 switch action {
Patrice Arruda39282062019-06-20 16:35:12 -0700582 case BUILD_MODULES:
583 // No additional processing is required when building a list of specific modules or all modules.
Patrice Arruda13848222019-04-22 17:12:02 -0700584 case BUILD_MODULES_IN_A_DIRECTORY:
585 // If dir is the root source tree, all the modules are built of the source tree are built so
586 // no need to find the build file.
587 if topDir == dir {
588 break
589 }
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700590
Patrice Arruda13848222019-04-22 17:12:02 -0700591 buildFile := findBuildFile(ctx, relDir)
592 if buildFile == "" {
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700593 ctx.Fatalf("Build file not found for %s directory", relDir)
Patrice Arruda13848222019-04-22 17:12:02 -0700594 }
Patrice Arruda13848222019-04-22 17:12:02 -0700595 targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
596 case BUILD_MODULES_IN_DIRECTORIES:
597 newConfigArgs, dirs := splitArgs(configArgs)
598 configArgs = newConfigArgs
Dan Willemsence41e942019-07-29 23:39:30 -0700599 targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix)
Patrice Arruda13848222019-04-22 17:12:02 -0700600 }
601
602 // Tidy only override all other specified targets.
603 tidyOnly := os.Getenv("WITH_TIDY_ONLY")
604 if tidyOnly == "true" || tidyOnly == "1" {
605 configArgs = append(configArgs, "tidy_only")
606 } else {
607 configArgs = append(configArgs, targets...)
608 }
609
610 return configArgs
611}
612
613// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name.
614func convertToTarget(dir string, targetNamePrefix string) string {
615 return targetNamePrefix + strings.ReplaceAll(dir, "/", "-")
616}
617
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700618// hasBuildFile returns true if dir contains an Android build file.
619func hasBuildFile(ctx Context, dir string) bool {
620 for _, buildFile := range buildFiles {
621 _, err := os.Stat(filepath.Join(dir, buildFile))
622 if err == nil {
623 return true
624 }
625 if !os.IsNotExist(err) {
626 ctx.Fatalf("Error retrieving the build file stats: %v", err)
627 }
628 }
629 return false
630}
631
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700632// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file
633// in the current and any sub directory of dir. If a build file is not found, traverse the path
634// up by one directory and repeat again until either a build file is found or reached to the root
635// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank
636// string is returned.
Patrice Arruda13848222019-04-22 17:12:02 -0700637func findBuildFile(ctx Context, dir string) string {
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700638 // If the string is empty or ".", assume it is top directory of the source tree.
639 if dir == "" || dir == "." {
Patrice Arruda13848222019-04-22 17:12:02 -0700640 return ""
641 }
642
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700643 found := false
644 for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) {
645 err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error {
646 if err != nil {
647 return err
648 }
649 if found {
650 return filepath.SkipDir
651 }
652 if info.IsDir() {
653 return nil
654 }
655 for _, buildFile := range buildFiles {
656 if info.Name() == buildFile {
657 found = true
658 return filepath.SkipDir
659 }
660 }
661 return nil
662 })
663 if err != nil {
664 ctx.Fatalf("Error finding Android build file: %v", err)
665 }
666
667 if found {
668 return filepath.Join(buildDir, "Android.mk")
Patrice Arruda13848222019-04-22 17:12:02 -0700669 }
670 }
671
672 return ""
673}
674
675// splitArgs iterates over the arguments list and splits into two lists: arguments and directories.
676func splitArgs(args []string) (newArgs []string, dirs []string) {
677 specialArgs := map[string]bool{
678 "showcommands": true,
679 "snod": true,
680 "dist": true,
681 "checkbuild": true,
682 }
683
684 newArgs = []string{}
685 dirs = []string{}
686
687 for _, arg := range args {
688 // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory.
689 if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 {
690 newArgs = append(newArgs, arg)
691 continue
692 }
693
694 if _, ok := specialArgs[arg]; ok {
695 newArgs = append(newArgs, arg)
696 continue
697 }
698
699 dirs = append(dirs, arg)
700 }
701
702 return newArgs, dirs
703}
704
705// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a
706// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the
707// source root tree where the build action command was invoked. Each directory is validated if the
708// build file can be found and follows the format "dir1:target1,target2,...". Target is optional.
Dan Willemsence41e942019-07-29 23:39:30 -0700709func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) {
Patrice Arruda13848222019-04-22 17:12:02 -0700710 for _, dir := range dirs {
711 // The directory may have specified specific modules to build. ":" is the separator to separate
712 // the directory and the list of modules.
713 s := strings.Split(dir, ":")
714 l := len(s)
715 if l > 2 { // more than one ":" was specified.
716 ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir)
717 }
718
719 dir = filepath.Join(relDir, s[0])
720 if _, err := os.Stat(dir); err != nil {
721 ctx.Fatalf("couldn't find directory %s", dir)
722 }
723
724 // Verify that if there are any targets specified after ":". Each target is separated by ",".
725 var newTargets []string
726 if l == 2 && s[1] != "" {
727 newTargets = strings.Split(s[1], ",")
728 if inList("", newTargets) {
729 ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir)
730 }
731 }
732
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700733 // If there are specified targets to build in dir, an android build file must exist for the one
734 // shot build. For the non-targets case, find the appropriate build file and build all the
735 // modules in dir (or the closest one in the dir path).
Patrice Arruda13848222019-04-22 17:12:02 -0700736 if len(newTargets) > 0 {
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700737 if !hasBuildFile(ctx, dir) {
Patrice Arruda13848222019-04-22 17:12:02 -0700738 ctx.Fatalf("Couldn't locate a build file from %s directory", dir)
739 }
740 } else {
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700741 buildFile := findBuildFile(ctx, dir)
742 if buildFile == "" {
743 ctx.Fatalf("Build file not found for %s directory", dir)
744 }
745 newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
Patrice Arruda13848222019-04-22 17:12:02 -0700746 }
747
Patrice Arruda13848222019-04-22 17:12:02 -0700748 targets = append(targets, newTargets...)
749 }
750
Dan Willemsence41e942019-07-29 23:39:30 -0700751 return targets
Patrice Arruda13848222019-04-22 17:12:02 -0700752}
753
Dan Willemsen9b587492017-07-10 22:13:00 -0700754func (c *configImpl) parseArgs(ctx Context, args []string) {
755 for i := 0; i < len(args); i++ {
756 arg := strings.TrimSpace(args[i])
Anton Hansson5a7861a2021-06-04 10:09:01 +0100757 if arg == "showcommands" {
Dan Willemsen9b587492017-07-10 22:13:00 -0700758 c.verbose = true
Lukacs T. Berkicef87b62021-08-10 15:01:13 +0200759 } else if arg == "--empty-ninja-file" {
760 c.emptyNinjaFile = true
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100761 } else if arg == "--skip-ninja" {
762 c.skipNinja = true
Dan Willemsene0879fc2017-08-04 15:06:27 -0700763 } else if arg == "--skip-make" {
Colin Cross30e444b2021-06-18 11:26:19 -0700764 // TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati,
765 // but it does run a Kati ninja file if the .kati_enabled marker file was created
766 // by a previous build.
Anton Hansson5e5c48b2020-11-27 12:35:20 +0000767 c.skipConfig = true
768 c.skipKati = true
Anton Hansson0b55bdb2021-06-04 10:08:08 +0100769 } else if arg == "--soong-only" {
770 c.skipKati = true
771 c.skipKatiNinja = true
Lukacs T. Berkicef87b62021-08-10 15:01:13 +0200772 } else if arg == "--config-only" {
773 c.skipKati = true
774 c.skipKatiNinja = true
775 c.skipSoong = true
Colin Cross30e444b2021-06-18 11:26:19 -0700776 } else if arg == "--skip-config" {
777 c.skipConfig = true
Colin Cross00a8a3f2020-10-29 14:08:31 -0700778 } else if arg == "--skip-soong-tests" {
779 c.skipSoongTests = true
MarkDacekd0e7cd32022-12-02 22:22:40 +0000780 } else if arg == "--skip-metrics-upload" {
781 c.skipMetricsUpload = true
Chris Parsons53f68ae2022-03-03 12:01:40 -0500782 } else if arg == "--mk-metrics" {
783 c.reportMkMetrics = true
LaMont Jones52a72432023-03-09 18:19:35 +0000784 } else if arg == "--multitree-build" {
785 c.multitreeBuild = true
Chris Parsonsef615e52022-08-18 22:04:11 -0400786 } else if arg == "--bazel-mode" {
787 c.bazelProdMode = true
MarkDacekb78465d2022-10-18 20:10:16 +0000788 } else if arg == "--bazel-mode-staging" {
789 c.bazelStagingMode = true
Spandan Das394aa322022-11-03 17:02:10 +0000790 } else if arg == "--search-api-dir" {
791 c.searchApiDir = true
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900792 } else if strings.HasPrefix(arg, "--ninja_weight_source=") {
793 source := strings.TrimPrefix(arg, "--ninja_weight_source=")
794 if source == "ninja_log" {
795 c.ninjaWeightListSource = NINJA_LOG
796 } else if source == "evenly_distributed" {
797 c.ninjaWeightListSource = EVENLY_DISTRIBUTED
798 } else if source == "not_used" {
799 c.ninjaWeightListSource = NOT_USED
Jeongik Chae114e602023-03-19 00:12:39 +0900800 } else if source == "soong" {
801 c.ninjaWeightListSource = HINT_FROM_SOONG
Jeongik Cha518f3ea2023-03-19 00:12:39 +0900802 } else if strings.HasPrefix(source, "file,") {
803 c.ninjaWeightListSource = EXTERNAL_FILE
804 filePath := strings.TrimPrefix(source, "file,")
805 err := validateNinjaWeightList(filePath)
806 if err != nil {
807 ctx.Fatalf("Malformed weight list from %s: %s", filePath, err)
808 }
809 _, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list"))
810 if err != nil {
811 ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err)
812 }
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900813 } else {
814 ctx.Fatalf("unknown option for ninja_weight_source: %s", source)
815 }
Jihoon Kang1bff0342023-01-17 20:40:22 +0000816 } else if arg == "--build-from-text-stub" {
817 c.buildFromTextStub = true
MarkDacekb96561e2022-12-02 04:34:43 +0000818 } else if strings.HasPrefix(arg, "--build-command=") {
819 buildCmd := strings.TrimPrefix(arg, "--build-command=")
820 // remove quotations
821 buildCmd = strings.TrimPrefix(buildCmd, "\"")
822 buildCmd = strings.TrimSuffix(buildCmd, "\"")
823 ctx.Metrics.SetBuildCommand([]string{buildCmd})
MarkDacekd06db5d2022-11-29 00:47:59 +0000824 } else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
825 c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
MarkDacek6614d9c2022-12-07 21:57:38 +0000826 } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
827 buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
828 val, err := strconv.ParseInt(buildTimeStr, 10, 64)
829 if err == nil {
830 c.buildStartedTime = val
831 } else {
832 ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
833 }
MarkDacekf47e1422023-04-19 16:47:36 +0000834 } else if arg == "--ensure-allowlist-integrity" {
835 c.ensureAllowlistIntegrity = true
MarkDacekd33c2fd2023-05-04 20:40:04 +0000836 } else if strings.HasPrefix(arg, "--bazel-exit-code=") {
837 bazelExitCodeStr := strings.TrimPrefix(arg, "--bazel-exit-code=")
838 val, err := strconv.Atoi(bazelExitCodeStr)
839 if err == nil {
840 c.bazelExitCode = int32(val)
841 } else {
842 ctx.Fatalf("Error parsing bazel-exit-code", err)
843 }
MarkDacek396491e2023-06-14 19:41:18 +0000844 } else if strings.HasPrefix(arg, "--bes-id=") {
845 c.besId = strings.TrimPrefix(arg, "--bes-id=")
Dan Willemsen6ac63ef2017-10-17 20:35:34 -0700846 } else if len(arg) > 0 && arg[0] == '-' {
Dan Willemsen9b587492017-07-10 22:13:00 -0700847 parseArgNum := func(def int) int {
848 if len(arg) > 2 {
849 p, err := strconv.ParseUint(arg[2:], 10, 31)
850 if err != nil {
851 ctx.Fatalf("Failed to parse %q: %v", arg, err)
852 }
853 return int(p)
854 } else if i+1 < len(args) {
855 p, err := strconv.ParseUint(args[i+1], 10, 31)
856 if err == nil {
857 i++
858 return int(p)
859 }
860 }
861 return def
862 }
863
Dan Willemsen6ac63ef2017-10-17 20:35:34 -0700864 if len(arg) > 1 && arg[1] == 'j' {
Dan Willemsen9b587492017-07-10 22:13:00 -0700865 c.parallel = parseArgNum(c.parallel)
Dan Willemsen6ac63ef2017-10-17 20:35:34 -0700866 } else if len(arg) > 1 && arg[1] == 'k' {
Dan Willemsen9b587492017-07-10 22:13:00 -0700867 c.keepGoing = parseArgNum(0)
Dan Willemsen1e704462016-08-21 15:17:17 -0700868 } else {
869 ctx.Fatalln("Unknown option:", arg)
870 }
Dan Willemsen091525e2017-07-11 14:17:50 -0700871 } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 {
Dan Willemsen6dfe30a2018-09-10 12:41:10 -0700872 if k == "OUT_DIR" {
873 ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.")
874 }
Dan Willemsen091525e2017-07-11 14:17:50 -0700875 c.environ.Set(k, v)
Dan Willemsen2d31a442018-10-20 21:33:41 -0700876 } else if arg == "dist" {
877 c.dist = true
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200878 } else if arg == "json-module-graph" {
879 c.jsonModuleGraph = true
880 } else if arg == "bp2build" {
881 c.bp2build = true
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200882 } else if arg == "queryview" {
883 c.queryview = true
Lukacs T. Berkic6012f32021-09-06 18:31:46 +0200884 } else if arg == "soong_docs" {
885 c.soongDocs = true
Dan Willemsen1e704462016-08-21 15:17:17 -0700886 } else {
Dan Willemsen2d31a442018-10-20 21:33:41 -0700887 if arg == "checkbuild" {
Colin Cross37193492017-11-16 17:55:00 -0800888 c.checkbuild = true
Dan Willemsene0879fc2017-08-04 15:06:27 -0700889 }
Dan Willemsen9b587492017-07-10 22:13:00 -0700890 c.arguments = append(c.arguments, arg)
Dan Willemsen1e704462016-08-21 15:17:17 -0700891 }
892 }
Chris Parsons21f80272023-06-15 04:02:28 +0000893 if (!c.bazelProdMode) && (!c.bazelStagingMode) {
Chris Parsonsb6e96902022-10-31 20:08:45 -0400894 c.bazelProdMode = defaultBazelProdMode(c)
895 }
Dan Willemsen1e704462016-08-21 15:17:17 -0700896}
897
Jeongik Cha518f3ea2023-03-19 00:12:39 +0900898func validateNinjaWeightList(weightListFilePath string) (err error) {
899 data, err := os.ReadFile(weightListFilePath)
900 if err != nil {
901 return
902 }
903 lines := strings.Split(strings.TrimSpace(string(data)), "\n")
904 for _, line := range lines {
905 fields := strings.Split(line, ",")
906 if len(fields) != 2 {
907 return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line)
908 }
909 _, err = strconv.Atoi(fields[1])
910 if err != nil {
911 return
912 }
913 }
914 return
915}
916
Dan Willemsened869522018-01-08 14:58:46 -0800917func (c *configImpl) configureLocale(ctx Context) {
918 cmd := Command(ctx, Config{c}, "locale", "locale", "-a")
919 output, err := cmd.Output()
920
921 var locales []string
922 if err == nil {
923 locales = strings.Split(string(output), "\n")
924 } else {
925 // If we're unable to list the locales, let's assume en_US.UTF-8
926 locales = []string{"en_US.UTF-8"}
927 ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales)
928 }
929
930 // gettext uses LANGUAGE, which is passed directly through
931
932 // For LANG and LC_*, only preserve the evaluated version of
933 // LC_MESSAGES
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800934 userLang := ""
Dan Willemsened869522018-01-08 14:58:46 -0800935 if lc_all, ok := c.environ.Get("LC_ALL"); ok {
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800936 userLang = lc_all
Dan Willemsened869522018-01-08 14:58:46 -0800937 } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok {
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800938 userLang = lc_messages
Dan Willemsened869522018-01-08 14:58:46 -0800939 } else if lang, ok := c.environ.Get("LANG"); ok {
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800940 userLang = lang
Dan Willemsened869522018-01-08 14:58:46 -0800941 }
942
943 c.environ.UnsetWithPrefix("LC_")
944
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800945 if userLang != "" {
946 c.environ.Set("LC_MESSAGES", userLang)
Dan Willemsened869522018-01-08 14:58:46 -0800947 }
948
949 // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed
950 // for others)
951 if inList("C.UTF-8", locales) {
952 c.environ.Set("LANG", "C.UTF-8")
Aaron Klingd236e0e2018-08-07 19:21:36 -0500953 } else if inList("C.utf8", locales) {
954 // These normalize to the same thing
955 c.environ.Set("LANG", "C.UTF-8")
Dan Willemsened869522018-01-08 14:58:46 -0800956 } else if inList("en_US.UTF-8", locales) {
957 c.environ.Set("LANG", "en_US.UTF-8")
958 } else if inList("en_US.utf8", locales) {
959 // These normalize to the same thing
960 c.environ.Set("LANG", "en_US.UTF-8")
961 } else {
962 ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8")
963 }
964}
965
Dan Willemsen1e704462016-08-21 15:17:17 -0700966func (c *configImpl) Environment() *Environment {
967 return c.environ
968}
969
970func (c *configImpl) Arguments() []string {
971 return c.arguments
972}
973
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200974func (c *configImpl) SoongBuildInvocationNeeded() bool {
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200975 if len(c.Arguments()) > 0 {
976 // Explicit targets requested that are not special targets like b2pbuild
977 // or the JSON module graph
978 return true
979 }
980
Chris Parsons73f411b2023-06-20 21:46:57 +0000981 if !c.JsonModuleGraph() && !c.Bp2Build() && !c.Queryview() && !c.SoongDocs() {
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200982 // Command line was empty, the default Ninja target is built
983 return true
984 }
985
Liz Kammer88677422021-12-15 15:03:19 -0500986 // bp2build + dist may be used to dist bp2build logs but does not require SoongBuildInvocation
987 if c.Dist() && !c.Bp2Build() {
988 return true
989 }
990
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200991 // build.ninja doesn't need to be generated
992 return false
993}
994
Dan Willemsen1e704462016-08-21 15:17:17 -0700995func (c *configImpl) OutDir() string {
996 if outDir, ok := c.environ.Get("OUT_DIR"); ok {
Patrice Arruda19bd53e2019-07-08 17:26:47 -0700997 return outDir
Dan Willemsen1e704462016-08-21 15:17:17 -0700998 }
999 return "out"
1000}
1001
Dan Willemsen8a073a82017-02-04 17:30:44 -08001002func (c *configImpl) DistDir() string {
Chris Parsons19ab9a42022-08-30 13:15:04 -04001003 return c.distDir
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +00001004}
1005
1006func (c *configImpl) RealDistDir() string {
Dan Willemsen2d31a442018-10-20 21:33:41 -07001007 return c.distDir
Dan Willemsen8a073a82017-02-04 17:30:44 -08001008}
1009
Dan Willemsen1e704462016-08-21 15:17:17 -07001010func (c *configImpl) NinjaArgs() []string {
Anton Hansson5e5c48b2020-11-27 12:35:20 +00001011 if c.skipKati {
Dan Willemsene0879fc2017-08-04 15:06:27 -07001012 return c.arguments
1013 }
Dan Willemsen1e704462016-08-21 15:17:17 -07001014 return c.ninjaArgs
1015}
1016
Jingwen Chen7c6089a2020-11-02 02:56:20 -05001017func (c *configImpl) BazelOutDir() string {
1018 return filepath.Join(c.OutDir(), "bazel")
1019}
1020
Liz Kammer2af5ea82022-11-11 14:21:03 -05001021func (c *configImpl) bazelOutputBase() string {
1022 return filepath.Join(c.BazelOutDir(), "output")
1023}
1024
Dan Willemsen1e704462016-08-21 15:17:17 -07001025func (c *configImpl) SoongOutDir() string {
1026 return filepath.Join(c.OutDir(), "soong")
1027}
1028
Spandan Das394aa322022-11-03 17:02:10 +00001029func (c *configImpl) ApiSurfacesOutDir() string {
1030 return filepath.Join(c.OutDir(), "api_surfaces")
1031}
1032
Lukacs T. Berkia806e412021-09-01 08:57:48 +02001033func (c *configImpl) PrebuiltOS() string {
1034 switch runtime.GOOS {
1035 case "linux":
1036 return "linux-x86"
1037 case "darwin":
1038 return "darwin-x86"
1039 default:
1040 panic("Unknown GOOS")
1041 }
1042}
Lukacs T. Berki90b43342021-11-02 14:42:04 +01001043
Lukacs T. Berkia806e412021-09-01 08:57:48 +02001044func (c *configImpl) HostToolDir() string {
Colin Crossacfcc1f2021-10-25 15:40:32 -07001045 if c.SkipKatiNinja() {
1046 return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin")
1047 } else {
1048 return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin")
1049 }
Lukacs T. Berkia806e412021-09-01 08:57:48 +02001050}
1051
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +02001052func (c *configImpl) NamedGlobFile(name string) string {
Lukacs T. Berki90b43342021-11-02 14:42:04 +01001053 return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +02001054}
1055
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +02001056func (c *configImpl) UsedEnvFile(tag string) string {
Kiyoung Kimeaa55a82023-06-05 16:56:49 +09001057 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
1058 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+"."+tag)
1059 }
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +02001060 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
1061}
1062
Lukacs T. Berkic541cd22022-10-26 07:26:50 +00001063func (c *configImpl) Bp2BuildFilesMarkerFile() string {
1064 return shared.JoinPath(c.SoongOutDir(), "bp2build_files_marker")
1065}
1066
1067func (c *configImpl) Bp2BuildWorkspaceMarkerFile() string {
Lukacs T. Berki90b43342021-11-02 14:42:04 +01001068 return shared.JoinPath(c.SoongOutDir(), "bp2build_workspace_marker")
Lukacs T. Berki56ebaf32021-08-12 14:03:55 +02001069}
1070
Lukacs T. Berkic6012f32021-09-06 18:31:46 +02001071func (c *configImpl) SoongDocsHtml() string {
1072 return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
1073}
1074
Lukacs T. Berki3a821692021-09-06 17:08:02 +02001075func (c *configImpl) QueryviewMarkerFile() string {
1076 return shared.JoinPath(c.SoongOutDir(), "queryview.marker")
1077}
1078
Lukacs T. Berkie571dc32021-08-25 14:14:13 +02001079func (c *configImpl) ModuleGraphFile() string {
1080 return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
1081}
1082
kgui67007242022-01-25 13:50:25 +08001083func (c *configImpl) ModuleActionsFile() string {
1084 return shared.JoinPath(c.SoongOutDir(), "module-actions.json")
1085}
1086
Jeff Gastonefc1b412017-03-29 17:29:06 -07001087func (c *configImpl) TempDir() string {
1088 return shared.TempDirForOutDir(c.SoongOutDir())
1089}
1090
Jeff Gastonb64fc1c2017-08-04 12:30:12 -07001091func (c *configImpl) FileListDir() string {
1092 return filepath.Join(c.OutDir(), ".module_paths")
1093}
1094
Dan Willemsen1e704462016-08-21 15:17:17 -07001095func (c *configImpl) KatiSuffix() string {
1096 if c.katiSuffix != "" {
1097 return c.katiSuffix
1098 }
1099 panic("SetKatiSuffix has not been called")
1100}
1101
Colin Cross37193492017-11-16 17:55:00 -08001102// Checkbuild returns true if "checkbuild" was one of the build goals, which means that the
1103// user is interested in additional checks at the expense of build time.
1104func (c *configImpl) Checkbuild() bool {
1105 return c.checkbuild
1106}
1107
Dan Willemsen8a073a82017-02-04 17:30:44 -08001108func (c *configImpl) Dist() bool {
1109 return c.dist
1110}
1111
Lukacs T. Berkia1b93722021-09-02 17:23:06 +02001112func (c *configImpl) JsonModuleGraph() bool {
1113 return c.jsonModuleGraph
1114}
1115
1116func (c *configImpl) Bp2Build() bool {
1117 return c.bp2build
1118}
1119
Lukacs T. Berki3a821692021-09-06 17:08:02 +02001120func (c *configImpl) Queryview() bool {
1121 return c.queryview
1122}
1123
Lukacs T. Berkic6012f32021-09-06 18:31:46 +02001124func (c *configImpl) SoongDocs() bool {
1125 return c.soongDocs
1126}
1127
Dan Willemsen1e704462016-08-21 15:17:17 -07001128func (c *configImpl) IsVerbose() bool {
1129 return c.verbose
1130}
1131
LaMont Jones52a72432023-03-09 18:19:35 +00001132func (c *configImpl) MultitreeBuild() bool {
1133 return c.multitreeBuild
1134}
1135
Jeongik Cha0cf44d52023-03-15 00:10:45 +09001136func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource {
1137 return c.ninjaWeightListSource
1138}
1139
Anton Hansson5e5c48b2020-11-27 12:35:20 +00001140func (c *configImpl) SkipKati() bool {
1141 return c.skipKati
1142}
1143
Anton Hansson0b55bdb2021-06-04 10:08:08 +01001144func (c *configImpl) SkipKatiNinja() bool {
1145 return c.skipKatiNinja
1146}
1147
Lukacs T. Berkicef87b62021-08-10 15:01:13 +02001148func (c *configImpl) SkipSoong() bool {
1149 return c.skipSoong
1150}
1151
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +01001152func (c *configImpl) SkipNinja() bool {
1153 return c.skipNinja
1154}
1155
Anton Hansson5a7861a2021-06-04 10:09:01 +01001156func (c *configImpl) SetSkipNinja(v bool) {
1157 c.skipNinja = v
1158}
1159
Anton Hansson5e5c48b2020-11-27 12:35:20 +00001160func (c *configImpl) SkipConfig() bool {
1161 return c.skipConfig
Dan Willemsene0879fc2017-08-04 15:06:27 -07001162}
1163
Jihoon Kang1bff0342023-01-17 20:40:22 +00001164func (c *configImpl) BuildFromTextStub() bool {
1165 return c.buildFromTextStub
1166}
1167
Dan Willemsen1e704462016-08-21 15:17:17 -07001168func (c *configImpl) TargetProduct() string {
1169 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
1170 return v
1171 }
1172 panic("TARGET_PRODUCT is not defined")
1173}
1174
Kiyoung Kima37d9ba2023-04-19 13:13:45 +09001175func (c *configImpl) TargetProductOrErr() (string, error) {
1176 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
1177 return v, nil
1178 }
1179 return "", fmt.Errorf("TARGET_PRODUCT is not defined")
1180}
1181
Dan Willemsen02781d52017-05-12 19:28:13 -07001182func (c *configImpl) TargetDevice() string {
1183 return c.targetDevice
1184}
1185
1186func (c *configImpl) SetTargetDevice(device string) {
1187 c.targetDevice = device
1188}
1189
1190func (c *configImpl) TargetBuildVariant() string {
1191 if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok {
1192 return v
1193 }
1194 panic("TARGET_BUILD_VARIANT is not defined")
1195}
1196
Dan Willemsen1e704462016-08-21 15:17:17 -07001197func (c *configImpl) KatiArgs() []string {
1198 return c.katiArgs
1199}
1200
1201func (c *configImpl) Parallel() int {
1202 return c.parallel
1203}
1204
Sam Delmerico98a73292023-02-21 11:50:29 -05001205func (c *configImpl) GetSourceRootDirs() []string {
1206 return c.sourceRootDirs
1207}
1208
1209func (c *configImpl) SetSourceRootDirs(i []string) {
1210 c.sourceRootDirs = i
1211}
1212
Spandan Dasc5763832022-11-08 18:42:16 +00001213func (c *configImpl) GetIncludeTags() []string {
1214 return c.includeTags
1215}
1216
1217func (c *configImpl) SetIncludeTags(i []string) {
1218 c.includeTags = i
1219}
1220
MarkDacek6614d9c2022-12-07 21:57:38 +00001221func (c *configImpl) GetLogsPrefix() string {
1222 return c.logsPrefix
1223}
1224
1225func (c *configImpl) SetLogsPrefix(prefix string) {
1226 c.logsPrefix = prefix
1227}
1228
Colin Cross8b8bec32019-11-15 13:18:43 -08001229func (c *configImpl) HighmemParallel() int {
1230 if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
1231 return i
1232 }
1233
1234 const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024
1235 parallel := c.Parallel()
1236 if c.UseRemoteBuild() {
1237 // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism
1238 // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs
1239 // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention.
1240 // Return 1/16th of the size of the local pool, rounding up.
1241 return (parallel + 15) / 16
1242 } else if c.totalRAM == 0 {
1243 // Couldn't detect the total RAM, don't restrict highmem processes.
1244 return parallel
Dan Willemsen570a2922020-05-26 23:02:29 -07001245 } else if c.totalRAM <= 16*1024*1024*1024 {
1246 // Less than 16GB of ram, restrict to 1 highmem processes
1247 return 1
Colin Cross8b8bec32019-11-15 13:18:43 -08001248 } else if c.totalRAM <= 32*1024*1024*1024 {
1249 // Less than 32GB of ram, restrict to 2 highmem processes
1250 return 2
1251 } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel {
1252 // If less than 8GB total RAM per process, reduce the number of highmem processes
1253 return p
1254 }
1255 // No restriction on highmem processes
1256 return parallel
1257}
1258
Dan Willemsen2bb82d02019-12-27 09:35:42 -08001259func (c *configImpl) TotalRAM() uint64 {
1260 return c.totalRAM
1261}
1262
Kousik Kumarec478642020-09-21 13:39:24 -04001263// ForceUseGoma determines whether we should override Goma deprecation
1264// and use Goma for the current build or not.
1265func (c *configImpl) ForceUseGoma() bool {
1266 if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok {
1267 v = strings.TrimSpace(v)
1268 if v != "" && v != "false" {
1269 return true
1270 }
1271 }
1272 return false
1273}
1274
Dan Willemsen1e704462016-08-21 15:17:17 -07001275func (c *configImpl) UseGoma() bool {
1276 if v, ok := c.environ.Get("USE_GOMA"); ok {
1277 v = strings.TrimSpace(v)
1278 if v != "" && v != "false" {
1279 return true
1280 }
1281 }
1282 return false
1283}
1284
Yoshisato Yanagisawa2cb0e5d2019-01-10 10:14:16 +09001285func (c *configImpl) StartGoma() bool {
1286 if !c.UseGoma() {
1287 return false
1288 }
1289
1290 if v, ok := c.environ.Get("NOSTART_GOMA"); ok {
1291 v = strings.TrimSpace(v)
1292 if v != "" && v != "false" {
1293 return false
1294 }
1295 }
1296 return true
1297}
1298
Kousik Kumar6d1e3482023-07-24 03:44:16 +00001299func (c *configImpl) canSupportRBE() bool {
1300 // Do not use RBE with prod credentials in scenarios when stubby doesn't exist, since
1301 // its unlikely that we will be able to obtain necessary creds without stubby.
1302 authType, _ := c.rbeAuth()
1303 if !c.StubbyExists() && strings.Contains(authType, "use_google_prod_creds") {
1304 return false
1305 }
1306 return true
1307}
1308
Ramy Medhatbbf25672019-07-17 12:30:04 +00001309func (c *configImpl) UseRBE() bool {
Jingwen Chend7ccde12023-06-28 07:19:26 +00001310 // These alternate modes of running Soong do not use RBE / reclient.
Chris Parsons73f411b2023-06-20 21:46:57 +00001311 if c.Bp2Build() || c.Queryview() || c.JsonModuleGraph() {
Jingwen Chend7ccde12023-06-28 07:19:26 +00001312 return false
1313 }
1314
Kousik Kumar6d1e3482023-07-24 03:44:16 +00001315 if !c.canSupportRBE() {
Kousik Kumar67ad4342023-06-06 15:09:27 -04001316 return false
1317 }
Kousik Kumar6d1e3482023-07-24 03:44:16 +00001318
Kousik Kumar3ff037e2022-01-25 22:11:01 -05001319 if v, ok := c.Environment().Get("USE_RBE"); ok {
Ramy Medhatbbf25672019-07-17 12:30:04 +00001320 v = strings.TrimSpace(v)
1321 if v != "" && v != "false" {
1322 return true
1323 }
1324 }
1325 return false
1326}
1327
Chris Parsonsef615e52022-08-18 22:04:11 -04001328func (c *configImpl) BazelBuildEnabled() bool {
Chris Parsons21f80272023-06-15 04:02:28 +00001329 return c.bazelProdMode || c.bazelStagingMode
Chris Parsonsec1a3dc2021-04-20 15:32:07 -04001330}
1331
Ramy Medhatbbf25672019-07-17 12:30:04 +00001332func (c *configImpl) StartRBE() bool {
1333 if !c.UseRBE() {
1334 return false
1335 }
1336
1337 if v, ok := c.environ.Get("NOSTART_RBE"); ok {
1338 v = strings.TrimSpace(v)
1339 if v != "" && v != "false" {
1340 return false
1341 }
1342 }
1343 return true
1344}
1345
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001346func (c *configImpl) rbeProxyLogsDir() string {
1347 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
Kousik Kumar0d15a722020-09-23 02:54:11 -04001348 if v, ok := c.environ.Get(f); ok {
1349 return v
1350 }
1351 }
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001352 buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir())
1353 return filepath.Join(buildTmpDir, "rbe")
Ramy Medhat0fc67eb2020-08-12 01:26:23 -04001354}
1355
Ramy Medhatc8f6cc22023-03-31 09:50:34 -04001356func (c *configImpl) rbeCacheDir() string {
1357 for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} {
1358 if v, ok := c.environ.Get(f); ok {
1359 return v
1360 }
1361 }
1362 return shared.JoinPath(c.SoongOutDir(), "rbe")
1363}
1364
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001365func (c *configImpl) shouldCleanupRBELogsDir() bool {
1366 // Perform a log directory cleanup only when the log directory
1367 // is auto created by the build rather than user-specified.
1368 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
1369 if _, ok := c.environ.Get(f); ok {
1370 return false
Patrice Arruda62f1bf22020-07-07 12:48:26 +00001371 }
1372 }
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001373 return true
Ramy Medhat0fc67eb2020-08-12 01:26:23 -04001374}
1375
1376func (c *configImpl) rbeExecRoot() string {
1377 for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} {
1378 if v, ok := c.environ.Get(f); ok {
1379 return v
1380 }
1381 }
1382 wd, err := os.Getwd()
1383 if err != nil {
1384 return ""
1385 }
1386 return wd
1387}
1388
1389func (c *configImpl) rbeDir() string {
1390 if v, ok := c.environ.Get("RBE_DIR"); ok {
1391 return v
1392 }
1393 return "prebuilts/remoteexecution-client/live/"
1394}
1395
1396func (c *configImpl) rbeReproxy() string {
1397 for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} {
1398 if v, ok := c.environ.Get(f); ok {
1399 return v
1400 }
1401 }
1402 return filepath.Join(c.rbeDir(), "reproxy")
1403}
1404
1405func (c *configImpl) rbeAuth() (string, string) {
Kousik Kumar93d192c2022-03-18 01:39:56 -04001406 credFlags := []string{
1407 "use_application_default_credentials",
1408 "use_gce_credentials",
1409 "credential_file",
1410 "use_google_prod_creds",
1411 }
Ramy Medhat0fc67eb2020-08-12 01:26:23 -04001412 for _, cf := range credFlags {
1413 for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} {
1414 if v, ok := c.environ.Get(f); ok {
1415 v = strings.TrimSpace(v)
1416 if v != "" && v != "false" && v != "0" {
1417 return "RBE_" + cf, v
1418 }
1419 }
1420 }
1421 }
1422 return "RBE_use_application_default_credentials", "true"
Patrice Arruda62f1bf22020-07-07 12:48:26 +00001423}
1424
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001425func (c *configImpl) rbeSockAddr(dir string) (string, error) {
1426 maxNameLen := len(syscall.RawSockaddrUnix{}.Path)
1427 base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix)
1428
1429 name := filepath.Join(dir, base)
1430 if len(name) < maxNameLen {
1431 return name, nil
1432 }
1433
1434 name = filepath.Join("/tmp", base)
1435 if len(name) < maxNameLen {
1436 return name, nil
1437 }
1438
1439 return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen)
1440}
1441
Kousik Kumar7bc78192022-04-27 14:52:56 -04001442// IsGooglerEnvironment returns true if the current build is running
1443// on a Google developer machine and false otherwise.
1444func (c *configImpl) IsGooglerEnvironment() bool {
1445 cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG"
1446 if v, ok := c.environ.Get(cf); ok {
1447 return v == "googler"
1448 }
1449 return false
1450}
1451
1452// GoogleProdCredsExist determine whether credentials exist on the
1453// Googler machine to use remote execution.
1454func (c *configImpl) GoogleProdCredsExist() bool {
Kevin Dagostino096ab2f2023-03-03 19:47:17 +00001455 if googleProdCredsExistCache {
1456 return googleProdCredsExistCache
1457 }
andusyu0b3dc032023-06-21 17:29:32 -04001458 if _, err := exec.Command("/usr/bin/gcertstatus", "-nocheck_ssh").Output(); err != nil {
Kousik Kumar7bc78192022-04-27 14:52:56 -04001459 return false
1460 }
Kevin Dagostino096ab2f2023-03-03 19:47:17 +00001461 googleProdCredsExistCache = true
Kousik Kumar7bc78192022-04-27 14:52:56 -04001462 return true
1463}
1464
1465// UseRemoteBuild indicates whether to use a remote build acceleration system
1466// to speed up the build.
Colin Cross9016b912019-11-11 14:57:42 -08001467func (c *configImpl) UseRemoteBuild() bool {
1468 return c.UseGoma() || c.UseRBE()
1469}
1470
Kousik Kumar7bc78192022-04-27 14:52:56 -04001471// StubbyExists checks whether the stubby binary exists on the machine running
1472// the build.
1473func (c *configImpl) StubbyExists() bool {
1474 if _, err := exec.LookPath("stubby"); err != nil {
1475 return false
1476 }
1477 return true
1478}
1479
Dan Willemsen1e704462016-08-21 15:17:17 -07001480// RemoteParallel controls how many remote jobs (i.e., commands which contain
Jeff Gastonefc1b412017-03-29 17:29:06 -07001481// gomacc) are run in parallel. Note the parallelism of all other jobs is
Dan Willemsen1e704462016-08-21 15:17:17 -07001482// still limited by Parallel()
1483func (c *configImpl) RemoteParallel() int {
Colin Cross8b8bec32019-11-15 13:18:43 -08001484 if !c.UseRemoteBuild() {
1485 return 0
1486 }
1487 if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok {
1488 return i
Dan Willemsen1e704462016-08-21 15:17:17 -07001489 }
1490 return 500
1491}
1492
1493func (c *configImpl) SetKatiArgs(args []string) {
1494 c.katiArgs = args
1495}
1496
1497func (c *configImpl) SetNinjaArgs(args []string) {
1498 c.ninjaArgs = args
1499}
1500
1501func (c *configImpl) SetKatiSuffix(suffix string) {
1502 c.katiSuffix = suffix
1503}
1504
Dan Willemsene0879fc2017-08-04 15:06:27 -07001505func (c *configImpl) LastKatiSuffixFile() string {
1506 return filepath.Join(c.OutDir(), "last_kati_suffix")
1507}
1508
1509func (c *configImpl) HasKatiSuffix() bool {
1510 return c.katiSuffix != ""
1511}
1512
Dan Willemsen1e704462016-08-21 15:17:17 -07001513func (c *configImpl) KatiEnvFile() string {
1514 return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
1515}
1516
Dan Willemsen29971232018-09-26 14:58:30 -07001517func (c *configImpl) KatiBuildNinjaFile() string {
1518 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja")
Dan Willemsen1e704462016-08-21 15:17:17 -07001519}
1520
Dan Willemsenfb1271a2018-09-26 15:00:42 -07001521func (c *configImpl) KatiPackageNinjaFile() string {
1522 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja")
1523}
1524
Jihoon Kang9f4f8a32022-08-16 00:57:30 +00001525func (c *configImpl) SoongVarsFile() string {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +09001526 targetProduct, err := c.TargetProductOrErr()
1527 if err != nil {
1528 return filepath.Join(c.SoongOutDir(), "soong.variables")
1529 } else {
1530 return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+".variables")
1531 }
Jihoon Kang9f4f8a32022-08-16 00:57:30 +00001532}
1533
Dan Willemsen1e704462016-08-21 15:17:17 -07001534func (c *configImpl) SoongNinjaFile() string {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +09001535 targetProduct, err := c.TargetProductOrErr()
1536 if err != nil {
1537 return filepath.Join(c.SoongOutDir(), "build.ninja")
1538 } else {
1539 return filepath.Join(c.SoongOutDir(), "build."+targetProduct+".ninja")
1540 }
Dan Willemsen1e704462016-08-21 15:17:17 -07001541}
1542
1543func (c *configImpl) CombinedNinjaFile() string {
Dan Willemsene0879fc2017-08-04 15:06:27 -07001544 if c.katiSuffix == "" {
1545 return filepath.Join(c.OutDir(), "combined.ninja")
1546 }
Dan Willemsen1e704462016-08-21 15:17:17 -07001547 return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
1548}
1549
1550func (c *configImpl) SoongAndroidMk() string {
1551 return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk")
1552}
1553
1554func (c *configImpl) SoongMakeVarsMk() string {
1555 return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk")
1556}
1557
Dan Willemsenf052f782017-05-18 15:29:04 -07001558func (c *configImpl) ProductOut() string {
Dan Willemsen4dc4e142017-09-08 14:35:43 -07001559 return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
Dan Willemsenf052f782017-05-18 15:29:04 -07001560}
1561
Dan Willemsen02781d52017-05-12 19:28:13 -07001562func (c *configImpl) DevicePreviousProductConfig() string {
Dan Willemsenf052f782017-05-18 15:29:04 -07001563 return filepath.Join(c.ProductOut(), "previous_build_config.mk")
1564}
1565
Dan Willemsenfb1271a2018-09-26 15:00:42 -07001566func (c *configImpl) KatiPackageMkDir() string {
1567 return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging")
1568}
1569
Dan Willemsenf052f782017-05-18 15:29:04 -07001570func (c *configImpl) hostOutRoot() string {
Dan Willemsen4dc4e142017-09-08 14:35:43 -07001571 return filepath.Join(c.OutDir(), "host")
Dan Willemsenf052f782017-05-18 15:29:04 -07001572}
1573
1574func (c *configImpl) HostOut() string {
1575 return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag())
1576}
1577
1578// This probably needs to be multi-valued, so not exporting it for now
1579func (c *configImpl) hostCrossOut() string {
1580 if runtime.GOOS == "linux" {
1581 return filepath.Join(c.hostOutRoot(), "windows-x86")
1582 } else {
1583 return ""
1584 }
Dan Willemsen02781d52017-05-12 19:28:13 -07001585}
1586
Dan Willemsen1e704462016-08-21 15:17:17 -07001587func (c *configImpl) HostPrebuiltTag() string {
1588 if runtime.GOOS == "linux" {
1589 return "linux-x86"
1590 } else if runtime.GOOS == "darwin" {
1591 return "darwin-x86"
1592 } else {
1593 panic("Unsupported OS")
1594 }
1595}
Dan Willemsenf173d592017-04-27 14:28:00 -07001596
Dan Willemsen8122bd52017-10-12 20:20:41 -07001597func (c *configImpl) PrebuiltBuildTool(name string) string {
Dan Willemsenf173d592017-04-27 14:28:00 -07001598 if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
1599 if sanitize := strings.Fields(v); inList("address", sanitize) {
Dan Willemsen8122bd52017-10-12 20:20:41 -07001600 asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
1601 if _, err := os.Stat(asan); err == nil {
1602 return asan
1603 }
Dan Willemsenf173d592017-04-27 14:28:00 -07001604 }
1605 }
1606 return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
1607}
Dan Willemsen3d60b112018-04-04 22:25:56 -07001608
1609func (c *configImpl) SetBuildBrokenDupRules(val bool) {
1610 c.brokenDupRules = val
1611}
1612
1613func (c *configImpl) BuildBrokenDupRules() bool {
1614 return c.brokenDupRules
1615}
Dan Willemsen6ab79db2018-05-02 00:06:28 -07001616
Dan Willemsen25e6f092019-04-09 10:22:43 -07001617func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) {
1618 c.brokenUsesNetwork = val
1619}
1620
1621func (c *configImpl) BuildBrokenUsesNetwork() bool {
1622 return c.brokenUsesNetwork
1623}
1624
Dan Willemsene3336352020-01-02 19:10:38 -08001625func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) {
1626 c.brokenNinjaEnvVars = val
1627}
1628
1629func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string {
1630 return c.brokenNinjaEnvVars
1631}
1632
Dan Willemsen6ab79db2018-05-02 00:06:28 -07001633func (c *configImpl) SetTargetDeviceDir(dir string) {
1634 c.targetDeviceDir = dir
1635}
1636
1637func (c *configImpl) TargetDeviceDir() string {
1638 return c.targetDeviceDir
1639}
Dan Willemsenfa42f3c2018-06-15 21:54:47 -07001640
Patrice Arruda219eef32020-06-01 17:29:30 +00001641func (c *configImpl) BuildDateTime() string {
1642 return c.buildDateTime
1643}
1644
1645func (c *configImpl) MetricsUploaderApp() string {
Yu Liu6e13b402021-07-27 14:29:06 -07001646 return c.metricsUploader
Patrice Arruda219eef32020-06-01 17:29:30 +00001647}
Patrice Arruda83842d72020-12-08 19:42:08 +00001648
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -05001649// LogsDir returns the absolute path to the logs directory where build log and
1650// metrics files are located. By default, the logs directory is the out
Patrice Arruda83842d72020-12-08 19:42:08 +00001651// directory. If the argument dist is specified, the logs directory
1652// is <dist_dir>/logs.
1653func (c *configImpl) LogsDir() string {
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -05001654 dir := c.OutDir()
Patrice Arruda83842d72020-12-08 19:42:08 +00001655 if c.Dist() {
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +00001656 // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -05001657 dir = filepath.Join(c.RealDistDir(), "logs")
Patrice Arruda83842d72020-12-08 19:42:08 +00001658 }
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -05001659 absDir, err := filepath.Abs(dir)
1660 if err != nil {
1661 fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error())
1662 os.Exit(1)
1663 }
1664 return absDir
Patrice Arruda83842d72020-12-08 19:42:08 +00001665}
1666
1667// BazelMetricsDir returns the <logs dir>/bazel_metrics directory
1668// where the bazel profiles are located.
1669func (c *configImpl) BazelMetricsDir() string {
1670 return filepath.Join(c.LogsDir(), "bazel_metrics")
1671}
Colin Crossf3bdbcb2021-06-01 11:43:55 -07001672
Chris Parsons53f68ae2022-03-03 12:01:40 -05001673// MkFileMetrics returns the file path for make-related metrics.
1674func (c *configImpl) MkMetrics() string {
1675 return filepath.Join(c.LogsDir(), "mk_metrics.pb")
1676}
1677
Colin Crossf3bdbcb2021-06-01 11:43:55 -07001678func (c *configImpl) SetEmptyNinjaFile(v bool) {
1679 c.emptyNinjaFile = v
1680}
1681
1682func (c *configImpl) EmptyNinjaFile() bool {
1683 return c.emptyNinjaFile
1684}
Yu Liu6e13b402021-07-27 14:29:06 -07001685
Romain Jobredeaux0a7529b2022-10-26 12:56:41 -04001686func (c *configImpl) IsBazelMixedBuildForceDisabled() bool {
1687 return c.Environment().IsEnvTrue("BUILD_BROKEN_DISABLE_BAZEL")
1688}
1689
Chris Parsons9402ca82023-02-23 17:28:06 -05001690func (c *configImpl) IsPersistentBazelEnabled() bool {
1691 return c.Environment().IsEnvTrue("USE_PERSISTENT_BAZEL")
1692}
1693
Chris Parsonsc83398f2023-05-31 18:41:41 +00001694// GetBazeliskBazelVersion returns the Bazel version to use for this build,
1695// or the empty string if the current canonical prod Bazel should be used.
1696// This environment variable should only be set to debug the build system.
1697// The Bazel version, if set, will be passed to Bazelisk, and Bazelisk will
1698// handle downloading and invoking the correct Bazel binary.
1699func (c *configImpl) GetBazeliskBazelVersion() string {
1700 value, _ := c.Environment().Get("USE_BAZEL_VERSION")
1701 return value
1702}
1703
MarkDacekd06db5d2022-11-29 00:47:59 +00001704func (c *configImpl) BazelModulesForceEnabledByFlag() string {
1705 return c.bazelForceEnabledModules
1706}
1707
MarkDacekd0e7cd32022-12-02 22:22:40 +00001708func (c *configImpl) SkipMetricsUpload() bool {
1709 return c.skipMetricsUpload
1710}
1711
MarkDacekf47e1422023-04-19 16:47:36 +00001712func (c *configImpl) EnsureAllowlistIntegrity() bool {
1713 return c.ensureAllowlistIntegrity
1714}
1715
MarkDacek6614d9c2022-12-07 21:57:38 +00001716// Returns a Time object if one was passed via a command-line flag.
1717// Otherwise returns the passed default.
1718func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
1719 if c.buildStartedTime == 0 {
1720 return defaultTime
1721 }
1722 return time.UnixMilli(c.buildStartedTime)
1723}
1724
MarkDacekd33c2fd2023-05-04 20:40:04 +00001725func (c *configImpl) BazelExitCode() int32 {
1726 return c.bazelExitCode
1727}
1728
Yu Liu6e13b402021-07-27 14:29:06 -07001729func GetMetricsUploader(topDir string, env *Environment) string {
1730 if p, ok := env.Get("METRICS_UPLOADER"); ok {
1731 metricsUploader := filepath.Join(topDir, p)
1732 if _, err := os.Stat(metricsUploader); err == nil {
1733 return metricsUploader
1734 }
1735 }
1736
1737 return ""
1738}