blob: 711f3256cfb796a4f144446e5269e71903961958 [file] [log] [blame]
Dan Willemsen1e704462016-08-21 15:17:17 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
Kousik Kumar3ff037e2022-01-25 22:11:01 -050018 "encoding/json"
Jeongik Chaa87506f2023-06-01 23:16:41 +090019 "errors"
Ramy Medhat0fc67eb2020-08-12 01:26:23 -040020 "fmt"
Kousik Kumar3ff037e2022-01-25 22:11:01 -050021 "io/ioutil"
Kousik Kumar4c180ad2022-05-27 07:48:37 -040022 "math/rand"
Dan Willemsenc2af0be2017-01-20 14:10:01 -080023 "os"
Kousik Kumar84bd5bf2022-01-26 23:32:22 -050024 "os/exec"
Cole Faust583dfb42023-09-28 13:56:30 -070025 "os/user"
Dan Willemsen1e704462016-08-21 15:17:17 -070026 "path/filepath"
27 "runtime"
28 "strconv"
29 "strings"
Kousik Kumar4c180ad2022-05-27 07:48:37 -040030 "syscall"
Nan Zhang2e6a4ff2018-02-14 13:27:26 -080031 "time"
Jeff Gastonefc1b412017-03-29 17:29:06 -070032
33 "android/soong/shared"
LaMont Jones9a912862023-11-06 22:11:08 +000034 "android/soong/ui/metrics"
Kousik Kumarec478642020-09-21 13:39:24 -040035
Dan Willemsen4591b642021-05-24 14:24:12 -070036 "google.golang.org/protobuf/proto"
Patrice Arruda96850362020-08-11 20:41:11 +000037
38 smpb "android/soong/ui/metrics/metrics_proto"
Dan Willemsen1e704462016-08-21 15:17:17 -070039)
40
Kousik Kumar3ff037e2022-01-25 22:11:01 -050041const (
Chris Parsons53f68ae2022-03-03 12:01:40 -050042 envConfigDir = "vendor/google/tools/soong_config"
43 jsonSuffix = "json"
Taylor Santiago8b0bed72024-09-03 13:30:22 -070044 abfsSrcDir = "/src"
Kousik Kumar3ff037e2022-01-25 22:11:01 -050045)
46
Kousik Kumar4c180ad2022-05-27 07:48:37 -040047var (
Kevin Dagostino096ab2f2023-03-03 19:47:17 +000048 rbeRandPrefix int
49 googleProdCredsExistCache bool
Kousik Kumar4c180ad2022-05-27 07:48:37 -040050)
51
52func init() {
53 rand.Seed(time.Now().UnixNano())
54 rbeRandPrefix = rand.Intn(1000)
55}
56
Dan Willemsen1e704462016-08-21 15:17:17 -070057type Config struct{ *configImpl }
58
59type configImpl struct {
Lukacs T. Berkia1b93722021-09-02 17:23:06 +020060 // Some targets that are implemented in soong_build
61 // (bp2build, json-module-graph) are not here and have their own bits below.
Colin Cross28f527c2019-11-26 16:19:04 -080062 arguments []string
63 goma bool
64 environ *Environment
65 distDir string
66 buildDateTime string
MarkDacek6614d9c2022-12-07 21:57:38 +000067 logsPrefix string
Dan Willemsen1e704462016-08-21 15:17:17 -070068
69 // From the arguments
MarkDacekf47e1422023-04-19 16:47:36 +000070 parallel int
71 keepGoing int
72 verbose bool
73 checkbuild bool
74 dist bool
75 jsonModuleGraph bool
MarkDacekf47e1422023-04-19 16:47:36 +000076 queryview bool
77 reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
78 soongDocs bool
MarkDacekf47e1422023-04-19 16:47:36 +000079 skipConfig bool
80 skipKati bool
81 skipKatiNinja bool
82 skipSoong bool
83 skipNinja bool
84 skipSoongTests bool
85 searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces
86 skipMetricsUpload bool
87 buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
Jihoon Kang2a929ad2023-06-08 19:02:07 +000088 buildFromSourceStub bool
Yu Liufa297642024-06-11 00:13:02 +000089 incrementalBuildActions bool
Colin Cross8d411ff2023-12-07 10:31:24 -080090 ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
Dan Willemsen1e704462016-08-21 15:17:17 -070091
92 // From the product config
Dan Willemsen6ab79db2018-05-02 00:06:28 -070093 katiArgs []string
94 ninjaArgs []string
95 katiSuffix string
96 targetDevice string
97 targetDeviceDir string
Spandan Dasa3639e62021-05-25 19:14:02 +000098 sandboxConfig *SandboxConfig
Dan Willemsen3d60b112018-04-04 22:25:56 -070099
Dan Willemsen2bb82d02019-12-27 09:35:42 -0800100 // Autodetected
101 totalRAM uint64
102
Spandan Das28a6f192024-07-01 21:00:25 +0000103 brokenDupRules bool
104 brokenUsesNetwork bool
105 brokenNinjaEnvVars []string
106 brokenMissingOutputs bool
Dan Willemsen18490112018-05-25 16:30:04 -0700107
108 pathReplaced bool
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +0000109
Colin Crossf3bdbcb2021-06-01 11:43:55 -0700110 // Set by multiproduct_kati
111 emptyNinjaFile bool
Yu Liu6e13b402021-07-27 14:29:06 -0700112
113 metricsUploader string
MarkDacekd06db5d2022-11-29 00:47:59 +0000114
Sam Delmerico98a73292023-02-21 11:50:29 -0500115 includeTags []string
116 sourceRootDirs []string
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900117
118 // Data source to write ninja weight list
119 ninjaWeightListSource NinjaWeightListSource
Joe Onoratoe5ed3472024-02-02 14:52:05 -0800120
121 // This file is a detailed dump of all soong-defined modules for debugging purposes.
122 // There's quite a bit of overlap with module-info.json and soong module graph. We
123 // could consider merging them.
124 moduleDebugFile string
Cole Faustbee030d2024-01-03 13:45:48 -0800125
126 // Whether to use n2 instead of ninja. This is controlled with the
127 // environment variable SOONG_USE_N2
128 useN2 bool
Dan Willemsen1e704462016-08-21 15:17:17 -0700129}
130
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900131type NinjaWeightListSource uint
132
133const (
134 // ninja doesn't use weight list.
135 NOT_USED NinjaWeightListSource = iota
136 // ninja uses weight list based on previous builds by ninja log
137 NINJA_LOG
138 // ninja thinks every task has the same weight.
139 EVENLY_DISTRIBUTED
Jeongik Cha518f3ea2023-03-19 00:12:39 +0900140 // ninja uses an external custom weight list
141 EXTERNAL_FILE
Jeongik Chae114e602023-03-19 00:12:39 +0900142 // ninja uses a prioritized module list from Soong
143 HINT_FROM_SOONG
Jeongik Chaa87506f2023-06-01 23:16:41 +0900144 // If ninja log exists, use NINJA_LOG, if not, use HINT_FROM_SOONG instead.
145 // We can assume it is an incremental build if ninja log exists.
146 DEFAULT
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900147)
Dan Willemsenc2af0be2017-01-20 14:10:01 -0800148const srcDirFileCheck = "build/soong/root.bp"
149
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700150var buildFiles = []string{"Android.mk", "Android.bp"}
151
Patrice Arruda13848222019-04-22 17:12:02 -0700152type BuildAction uint
153
154const (
155 // Builds all of the modules and their dependencies of a specified directory, relative to the root
156 // directory of the source tree.
157 BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota
158
159 // Builds all of the modules and their dependencies of a list of specified directories. All specified
160 // directories are relative to the root directory of the source tree.
161 BUILD_MODULES_IN_DIRECTORIES
Patrice Arruda39282062019-06-20 16:35:12 -0700162
163 // Build a list of specified modules. If none was specified, simply build the whole source tree.
164 BUILD_MODULES
Patrice Arruda13848222019-04-22 17:12:02 -0700165)
166
167// checkTopDir validates that the current directory is at the root directory of the source tree.
168func checkTopDir(ctx Context) {
169 if _, err := os.Stat(srcDirFileCheck); err != nil {
170 if os.IsNotExist(err) {
171 ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck)
172 }
173 ctx.Fatalln("Error verifying tree state:", err)
174 }
175}
176
MarkDacek7901e582023-01-09 19:48:01 +0000177func loadEnvConfig(ctx Context, config *configImpl, bc string) error {
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500178 if bc == "" {
179 return nil
180 }
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500181
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500182 configDirs := []string{
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500183 config.OutDir(),
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500184 os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"),
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500185 envConfigDir,
186 }
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500187 for _, dir := range configDirs {
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500188 cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix))
189 envVarsJSON, err := ioutil.ReadFile(cfgFile)
190 if err != nil {
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500191 continue
192 }
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500193 ctx.Verbosef("Loading config file %v\n", cfgFile)
194 var envVars map[string]map[string]string
195 if err := json.Unmarshal(envVarsJSON, &envVars); err != nil {
196 fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error())
197 continue
198 }
199 for k, v := range envVars["env"] {
200 if os.Getenv(k) != "" {
201 continue
202 }
203 config.environ.Set(k, v)
204 }
205 ctx.Verbosef("Finished loading config file %v\n", cfgFile)
206 break
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500207 }
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500208
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500209 return nil
210}
211
Dan Willemsen1e704462016-08-21 15:17:17 -0700212func NewConfig(ctx Context, args ...string) Config {
213 ret := &configImpl{
Jeongik Chaf2ecf762023-05-19 14:03:45 +0900214 environ: OsEnvironment(),
215 sandboxConfig: &SandboxConfig{},
Jeongik Chaa87506f2023-06-01 23:16:41 +0900216 ninjaWeightListSource: DEFAULT,
Dan Willemsen1e704462016-08-21 15:17:17 -0700217 }
Taylor Santiago8b0bed72024-09-03 13:30:22 -0700218 wd, err := os.Getwd()
219 if err != nil {
220 ctx.Fatalln("Failed to get working directory:", err)
221 }
Dan Willemsen1e704462016-08-21 15:17:17 -0700222
Colin Cross106d6ef2023-10-24 10:34:56 -0700223 // Skip soong tests by default on Linux
224 if runtime.GOOS == "linux" {
225 ret.skipSoongTests = true
226 }
227
Patrice Arruda90109172020-07-28 18:07:27 +0000228 // Default matching ninja
Dan Willemsen9b587492017-07-10 22:13:00 -0700229 ret.parallel = runtime.NumCPU() + 2
230 ret.keepGoing = 1
231
Dan Willemsen2bb82d02019-12-27 09:35:42 -0800232 ret.totalRAM = detectTotalRAM(ctx)
Dan Willemsen9b587492017-07-10 22:13:00 -0700233 ret.parseArgs(ctx, args)
Jeongik Chae114e602023-03-19 00:12:39 +0900234
235 if ret.ninjaWeightListSource == HINT_FROM_SOONG {
Jeongik Chaa87506f2023-06-01 23:16:41 +0900236 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "always")
237 } else if ret.ninjaWeightListSource == DEFAULT {
238 defaultNinjaWeightListSource := NINJA_LOG
239 if _, err := os.Stat(filepath.Join(ret.OutDir(), ninjaLogFileName)); errors.Is(err, os.ErrNotExist) {
240 ctx.Verboseln("$OUT/.ninja_log doesn't exist, use HINT_FROM_SOONG instead")
241 defaultNinjaWeightListSource = HINT_FROM_SOONG
242 } else {
243 ctx.Verboseln("$OUT/.ninja_log exist, use NINJA_LOG")
244 }
245 ret.ninjaWeightListSource = defaultNinjaWeightListSource
246 // soong_build generates ninja hint depending on ninja log existence.
247 // Set it "depend" to avoid soong re-run due to env variable change.
248 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "depend")
Jeongik Chae114e602023-03-19 00:12:39 +0900249 }
Jeongik Chaa87506f2023-06-01 23:16:41 +0900250
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800251 // Make sure OUT_DIR is set appropriately
Dan Willemsen02f3add2017-05-12 13:50:19 -0700252 if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
Taylor Santiago8b0bed72024-09-03 13:30:22 -0700253 ret.environ.Set("OUT_DIR", ret.sandboxPath(wd, filepath.Clean(outDir)))
Dan Willemsen02f3add2017-05-12 13:50:19 -0700254 } else {
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800255 outDir := "out"
256 if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok {
Taylor Santiago8b0bed72024-09-03 13:30:22 -0700257 outDir = filepath.Join(baseDir, filepath.Base(wd))
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800258 }
Taylor Santiago8b0bed72024-09-03 13:30:22 -0700259 ret.environ.Set("OUT_DIR", ret.sandboxPath(wd, outDir))
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800260 }
261
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500262 // loadEnvConfig needs to know what the OUT_DIR is, so it should
263 // be called after we determine the appropriate out directory.
MarkDacek7901e582023-01-09 19:48:01 +0000264 bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
265
266 if bc != "" {
Kousik Kumarc8818332023-01-16 16:33:05 +0000267 if err := loadEnvConfig(ctx, ret, bc); err != nil {
MarkDacek7901e582023-01-09 19:48:01 +0000268 ctx.Fatalln("Failed to parse env config files: %v", err)
269 }
Kousik Kumar6d1e3482023-07-24 03:44:16 +0000270 if !ret.canSupportRBE() {
271 // Explicitly set USE_RBE env variable to false when we cannot run
272 // an RBE build to avoid ninja local execution pool issues.
273 ret.environ.Set("USE_RBE", "false")
274 }
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500275 }
276
Dan Willemsen2d31a442018-10-20 21:33:41 -0700277 if distDir, ok := ret.environ.Get("DIST_DIR"); ok {
278 ret.distDir = filepath.Clean(distDir)
279 } else {
280 ret.distDir = filepath.Join(ret.OutDir(), "dist")
281 }
Dan Willemsend50e89f2018-10-16 17:49:25 -0700282
Spandan Das05063612021-06-25 01:39:04 +0000283 if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok {
284 ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false")
285 }
286
Joe Onoratoe5ed3472024-02-02 14:52:05 -0800287 if os.Getenv("GENERATE_SOONG_DEBUG") == "true" {
288 ret.moduleDebugFile, _ = filepath.Abs(shared.JoinPath(ret.SoongOutDir(), "soong-debug-info.json"))
289 }
290
Cole Faustbee030d2024-01-03 13:45:48 -0800291 if os.Getenv("SOONG_USE_N2") == "true" {
292 ret.useN2 = true
293 }
294
Dan Willemsen1e704462016-08-21 15:17:17 -0700295 ret.environ.Unset(
296 // We're already using it
297 "USE_SOONG_UI",
298
299 // We should never use GOROOT/GOPATH from the shell environment
300 "GOROOT",
301 "GOPATH",
302
303 // These should only come from Soong, not the environment.
304 "CLANG",
305 "CLANG_CXX",
306 "CCC_CC",
307 "CCC_CXX",
308
309 // Used by the goma compiler wrapper, but should only be set by
310 // gomacc
311 "GOMACC_PATH",
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800312
313 // We handle this above
314 "OUT_DIR_COMMON_BASE",
Dan Willemsen68a09852017-04-18 13:56:57 -0700315
Dan Willemsen2d31a442018-10-20 21:33:41 -0700316 // This is handled above too, and set for individual commands later
317 "DIST_DIR",
318
Dan Willemsen68a09852017-04-18 13:56:57 -0700319 // Variables that have caused problems in the past
Dan Willemsen1c504d92019-11-18 19:13:53 +0000320 "BASH_ENV",
Dan Willemsenebfe33a2018-05-01 10:07:50 -0700321 "CDPATH",
Dan Willemsen68a09852017-04-18 13:56:57 -0700322 "DISPLAY",
323 "GREP_OPTIONS",
Nathan Egge7b067fb2023-02-17 17:54:31 +0000324 "JAVAC",
Nathan Egge978c9342024-07-03 21:13:03 +0000325 "LEX",
Dan Willemsenebfe33a2018-05-01 10:07:50 -0700326 "NDK_ROOT",
Dan Willemsen00fcb262018-08-15 15:35:38 -0700327 "POSIXLY_CORRECT",
Dan Willemsenc40e10b2017-07-11 14:30:00 -0700328
329 // Drop make flags
330 "MAKEFLAGS",
331 "MAKELEVEL",
332 "MFLAGS",
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700333
334 // Set in envsetup.sh, reset in makefiles
335 "ANDROID_JAVA_TOOLCHAIN",
Colin Cross7f09c402018-07-11 14:49:31 -0700336
337 // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional
338 "ANDROID_BUILD_TOP",
339 "ANDROID_HOST_OUT",
340 "ANDROID_PRODUCT_OUT",
341 "ANDROID_HOST_OUT_TESTCASES",
342 "ANDROID_TARGET_OUT_TESTCASES",
343 "ANDROID_TOOLCHAIN",
344 "ANDROID_TOOLCHAIN_2ND_ARCH",
345 "ANDROID_DEV_SCRIPTS",
346 "ANDROID_EMULATOR_PREBUILTS",
347 "ANDROID_PRE_BUILD_PATHS",
Joe Onoratoe5ed3472024-02-02 14:52:05 -0800348
349 // We read it here already, don't let others share in the fun
350 "GENERATE_SOONG_DEBUG",
Cole Faustbee030d2024-01-03 13:45:48 -0800351
352 // Use config.useN2 instead.
353 "SOONG_USE_N2",
Taylor Santiago8b0bed72024-09-03 13:30:22 -0700354
355 // Leaks usernames into environment.
356 "HOME",
Dan Willemsen1e704462016-08-21 15:17:17 -0700357 )
358
Kousik Kumarb328f6d2020-10-19 01:45:46 -0400359 if ret.UseGoma() || ret.ForceUseGoma() {
360 ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.")
361 ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.")
Kousik Kumarec478642020-09-21 13:39:24 -0400362 }
363
Dan Willemsen1e704462016-08-21 15:17:17 -0700364 // Tell python not to spam the source tree with .pyc files.
365 ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
366
Ramy Medhatca1e44c2020-07-16 12:18:37 -0400367 tmpDir := absPath(ctx, ret.TempDir())
Taylor Santiago8b0bed72024-09-03 13:30:22 -0700368 ret.environ.Set("TMPDIR", ret.sandboxPath(wd, tmpDir))
Dan Willemsen32a669b2018-03-08 19:42:00 -0800369
Dan Willemsen70c1ff82019-08-21 14:56:13 -0700370 // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes
371 symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(),
372 "llvm-binutils-stable/llvm-symbolizer")
Taylor Santiago8b0bed72024-09-03 13:30:22 -0700373 ret.environ.Set("ASAN_SYMBOLIZER_PATH", ret.sandboxPath(wd, absPath(ctx, symbolizerPath)))
Dan Willemsen70c1ff82019-08-21 14:56:13 -0700374
Dan Willemsenc2af0be2017-01-20 14:10:01 -0800375 // Precondition: the current directory is the top of the source tree
Patrice Arruda13848222019-04-22 17:12:02 -0700376 checkTopDir(ctx)
Dan Willemsenc2af0be2017-01-20 14:10:01 -0800377
Yu Liu6e13b402021-07-27 14:29:06 -0700378 srcDir := absPath(ctx, ".")
379 if strings.ContainsRune(srcDir, ' ') {
Colin Cross1f6faeb2019-09-23 15:52:40 -0700380 ctx.Println("You are building in a directory whose absolute path contains a space character:")
381 ctx.Println()
382 ctx.Printf("%q\n", srcDir)
383 ctx.Println()
384 ctx.Fatalln("Directory names containing spaces are not supported")
Dan Willemsendb8457c2017-05-12 16:38:17 -0700385 }
386
Yu Liu6e13b402021-07-27 14:29:06 -0700387 ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
388
Dan Willemsendb8457c2017-05-12 16:38:17 -0700389 if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') {
Colin Cross1f6faeb2019-09-23 15:52:40 -0700390 ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:")
391 ctx.Println()
392 ctx.Printf("%q\n", outDir)
393 ctx.Println()
394 ctx.Fatalln("Directory names containing spaces are not supported")
Dan Willemsendb8457c2017-05-12 16:38:17 -0700395 }
396
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +0000397 if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') {
Colin Cross1f6faeb2019-09-23 15:52:40 -0700398 ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:")
399 ctx.Println()
400 ctx.Printf("%q\n", distDir)
401 ctx.Println()
402 ctx.Fatalln("Directory names containing spaces are not supported")
Dan Willemsendb8457c2017-05-12 16:38:17 -0700403 }
404
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700405 // Configure Java-related variables, including adding it to $PATH
Tobias Thierere59aeff2017-12-20 22:40:39 +0000406 java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag())
Sorin Basca0760c892023-11-29 19:13:55 +0000407 java21Home := filepath.Join("prebuilts/jdk/jdk21", ret.HostPrebuiltTag())
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700408 javaHome := func() string {
409 if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok {
410 return override
411 }
Pete Gillina7a3d642019-11-07 18:58:42 +0000412 if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" {
Sorin Basca5dfa2382024-03-11 17:23:06 +0000413 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.")
Pete Gillin1f52e932019-10-09 17:10:08 +0100414 }
Sorin Basca7e094b32022-10-05 08:20:12 +0000415 if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" {
Sorin Basca5dfa2382024-03-11 17:23:06 +0000416 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.")
Sorin Basca7e094b32022-10-05 08:20:12 +0000417 }
Sorin Basca5dfa2382024-03-11 17:23:06 +0000418 if toolchain21, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN"); ok && toolchain21 != "true" {
419 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.")
420 }
421 return java21Home
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700422 }()
423 absJavaHome := absPath(ctx, javaHome)
424
Dan Willemsened869522018-01-08 14:58:46 -0800425 ret.configureLocale(ctx)
426
PODISHETTY KUMAR (xWF)9543d192024-09-02 03:54:36 +0000427 newPath := []string{filepath.Join(absJavaHome, "bin")}
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700428 if path, ok := ret.environ.Get("PATH"); ok && path != "" {
PODISHETTY KUMAR (xWF)9543d192024-09-02 03:54:36 +0000429 newPath = append(newPath, path)
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700430 }
Pete Gillin1f52e932019-10-09 17:10:08 +0100431
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700432 ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME")
Taylor Santiago8b0bed72024-09-03 13:30:22 -0700433 ret.environ.Set("JAVA_HOME", ret.sandboxPath(wd, absJavaHome))
434 ret.environ.Set("ANDROID_JAVA_HOME", ret.sandboxPath(wd, javaHome))
435 ret.environ.Set("ANDROID_JAVA8_HOME", ret.sandboxPath(wd, java8Home))
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700436 ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator)))
437
Colin Crossfe5ed4d2023-07-28 09:27:23 -0700438 // b/286885495, https://bugzilla.redhat.com/show_bug.cgi?id=2227130: some versions of Fedora include patches
439 // to unzip to enable zipbomb detection that incorrectly handle zip64 and data descriptors and fail on large
440 // zip files produced by soong_zip. Disable zipbomb detection.
441 ret.environ.Set("UNZIP_DISABLE_ZIPBOMB_DETECTION", "TRUE")
442
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800443 outDir := ret.OutDir()
444 buildDateTimeFile := filepath.Join(outDir, "build_date.txt")
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800445 if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" {
Colin Cross28f527c2019-11-26 16:19:04 -0800446 ret.buildDateTime = buildDateTime
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800447 } else {
Colin Cross28f527c2019-11-26 16:19:04 -0800448 ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10)
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800449 }
Colin Cross28f527c2019-11-26 16:19:04 -0800450
Taylor Santiago8b0bed72024-09-03 13:30:22 -0700451 ret.environ.Set("BUILD_DATETIME_FILE", ret.sandboxPath(wd, buildDateTimeFile))
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800452
Cole Faust583dfb42023-09-28 13:56:30 -0700453 if _, ok := ret.environ.Get("BUILD_USERNAME"); !ok {
454 username := "unknown"
455 if u, err := user.Current(); err == nil {
456 username = u.Username
457 } else {
458 ctx.Println("Failed to get current user:", err)
459 }
460 ret.environ.Set("BUILD_USERNAME", username)
461 }
Taylor Santiago8b0bed72024-09-03 13:30:22 -0700462 ret.environ.Set("PWD", ret.sandboxPath(wd, wd))
Cole Faust583dfb42023-09-28 13:56:30 -0700463
Ramy Medhatca1e44c2020-07-16 12:18:37 -0400464 if ret.UseRBE() {
Ramy Medhat0fc67eb2020-08-12 01:26:23 -0400465 for k, v := range getRBEVars(ctx, Config{ret}) {
Ramy Medhatca1e44c2020-07-16 12:18:37 -0400466 ret.environ.Set(k, v)
467 }
468 }
469
Patrice Arruda96850362020-08-11 20:41:11 +0000470 c := Config{ret}
471 storeConfigMetrics(ctx, c)
472 return c
Dan Willemsen9b587492017-07-10 22:13:00 -0700473}
474
Patrice Arruda13848222019-04-22 17:12:02 -0700475// NewBuildActionConfig returns a build configuration based on the build action. The arguments are
476// processed based on the build action and extracts any arguments that belongs to the build action.
Dan Willemsence41e942019-07-29 23:39:30 -0700477func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config {
478 return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...)
Patrice Arruda13848222019-04-22 17:12:02 -0700479}
480
LaMont Jones9a912862023-11-06 22:11:08 +0000481// Prepare for getting make variables. For them to be accurate, we need to have
482// obtained PRODUCT_RELEASE_CONFIG_MAPS.
483//
484// Returns:
485//
486// Whether config should be called again.
487//
488// TODO: when converting product config to a declarative language, make sure
489// that PRODUCT_RELEASE_CONFIG_MAPS is properly handled as a separate step in
490// that process.
491func SetProductReleaseConfigMaps(ctx Context, config Config) bool {
492 ctx.BeginTrace(metrics.RunKati, "SetProductReleaseConfigMaps")
493 defer ctx.EndTrace()
494
495 if config.SkipConfig() {
496 // This duplicates the logic from Build to skip product config
497 // if the user has explicitly said to.
498 return false
499 }
500
501 releaseConfigVars := []string{
502 "PRODUCT_RELEASE_CONFIG_MAPS",
503 }
504
505 origValue, _ := config.environ.Get("PRODUCT_RELEASE_CONFIG_MAPS")
506 // Get the PRODUCT_RELEASE_CONFIG_MAPS for this product, to avoid polluting the environment
507 // when we run product config to get the rest of the make vars.
508 releaseMapVars, err := dumpMakeVars(ctx, config, nil, releaseConfigVars, false, "")
509 if err != nil {
510 ctx.Fatalln("Error getting PRODUCT_RELEASE_CONFIG_MAPS:", err)
511 }
512 productReleaseConfigMaps := releaseMapVars["PRODUCT_RELEASE_CONFIG_MAPS"]
513 os.Setenv("PRODUCT_RELEASE_CONFIG_MAPS", productReleaseConfigMaps)
514 return origValue != productReleaseConfigMaps
515}
516
Patrice Arruda96850362020-08-11 20:41:11 +0000517// storeConfigMetrics selects a set of configuration information and store in
518// the metrics system for further analysis.
519func storeConfigMetrics(ctx Context, config Config) {
520 if ctx.Metrics == nil {
521 return
522 }
523
Liz Kammerca9cb2e2021-07-14 15:29:57 -0400524 ctx.Metrics.BuildConfig(buildConfig(config))
Patrice Arruda3edfd482020-10-13 23:58:41 +0000525
526 s := &smpb.SystemResourceInfo{
527 TotalPhysicalMemory: proto.Uint64(config.TotalRAM()),
528 AvailableCpus: proto.Int32(int32(runtime.NumCPU())),
529 }
530 ctx.Metrics.SystemResourceInfo(s)
Patrice Arruda96850362020-08-11 20:41:11 +0000531}
532
Jeongik Cha8d63d562023-03-17 03:52:13 +0900533func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource {
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900534 switch s {
535 case NINJA_LOG:
Jeongik Cha8d63d562023-03-17 03:52:13 +0900536 return smpb.BuildConfig_NINJA_LOG.Enum()
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900537 case EVENLY_DISTRIBUTED:
Jeongik Cha8d63d562023-03-17 03:52:13 +0900538 return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum()
Jeongik Cha518f3ea2023-03-19 00:12:39 +0900539 case EXTERNAL_FILE:
540 return smpb.BuildConfig_EXTERNAL_FILE.Enum()
Jeongik Chae114e602023-03-19 00:12:39 +0900541 case HINT_FROM_SOONG:
542 return smpb.BuildConfig_HINT_FROM_SOONG.Enum()
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900543 default:
Jeongik Cha8d63d562023-03-17 03:52:13 +0900544 return smpb.BuildConfig_NOT_USED.Enum()
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900545 }
546}
547
Liz Kammerca9cb2e2021-07-14 15:29:57 -0400548func buildConfig(config Config) *smpb.BuildConfig {
Yu Liue737a992021-10-04 13:21:41 -0700549 c := &smpb.BuildConfig{
Colin Cross8d411ff2023-12-07 10:31:24 -0800550 ForceUseGoma: proto.Bool(config.ForceUseGoma()),
551 UseGoma: proto.Bool(config.UseGoma()),
552 UseRbe: proto.Bool(config.UseRBE()),
553 NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
Liz Kammerca9cb2e2021-07-14 15:29:57 -0400554 }
Yu Liue737a992021-10-04 13:21:41 -0700555 c.Targets = append(c.Targets, config.arguments...)
556
557 return c
Liz Kammerca9cb2e2021-07-14 15:29:57 -0400558}
559
Patrice Arruda13848222019-04-22 17:12:02 -0700560// getConfigArgs processes the command arguments based on the build action and creates a set of new
561// arguments to be accepted by Config.
Dan Willemsence41e942019-07-29 23:39:30 -0700562func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string {
Patrice Arruda13848222019-04-22 17:12:02 -0700563 // The next block of code verifies that the current directory is the root directory of the source
564 // tree. It then finds the relative path of dir based on the root directory of the source tree
565 // and verify that dir is inside of the source tree.
566 checkTopDir(ctx)
567 topDir, err := os.Getwd()
568 if err != nil {
569 ctx.Fatalf("Error retrieving top directory: %v", err)
570 }
Patrice Arrudababa9a92019-07-03 10:47:34 -0700571 dir, err = filepath.EvalSymlinks(dir)
572 if err != nil {
573 ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err)
574 }
Patrice Arruda13848222019-04-22 17:12:02 -0700575 dir, err = filepath.Abs(dir)
576 if err != nil {
577 ctx.Fatalf("Unable to find absolute path %s: %v", dir, err)
578 }
579 relDir, err := filepath.Rel(topDir, dir)
580 if err != nil {
581 ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err)
582 }
583 // If there are ".." in the path, it's not in the source tree.
584 if strings.Contains(relDir, "..") {
585 ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir)
586 }
587
588 configArgs := args[:]
589
590 // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to
591 // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules.
592 targetNamePrefix := "MODULES-IN-"
593 if inList("GET-INSTALL-PATH", configArgs) {
594 targetNamePrefix = "GET-INSTALL-PATH-IN-"
595 configArgs = removeFromList("GET-INSTALL-PATH", configArgs)
596 }
597
Patrice Arruda13848222019-04-22 17:12:02 -0700598 var targets []string
599
600 switch action {
Patrice Arruda39282062019-06-20 16:35:12 -0700601 case BUILD_MODULES:
602 // No additional processing is required when building a list of specific modules or all modules.
Patrice Arruda13848222019-04-22 17:12:02 -0700603 case BUILD_MODULES_IN_A_DIRECTORY:
604 // If dir is the root source tree, all the modules are built of the source tree are built so
605 // no need to find the build file.
606 if topDir == dir {
607 break
608 }
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700609
Patrice Arruda13848222019-04-22 17:12:02 -0700610 buildFile := findBuildFile(ctx, relDir)
611 if buildFile == "" {
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700612 ctx.Fatalf("Build file not found for %s directory", relDir)
Patrice Arruda13848222019-04-22 17:12:02 -0700613 }
Patrice Arruda13848222019-04-22 17:12:02 -0700614 targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
615 case BUILD_MODULES_IN_DIRECTORIES:
616 newConfigArgs, dirs := splitArgs(configArgs)
617 configArgs = newConfigArgs
Dan Willemsence41e942019-07-29 23:39:30 -0700618 targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix)
Patrice Arruda13848222019-04-22 17:12:02 -0700619 }
620
621 // Tidy only override all other specified targets.
622 tidyOnly := os.Getenv("WITH_TIDY_ONLY")
623 if tidyOnly == "true" || tidyOnly == "1" {
624 configArgs = append(configArgs, "tidy_only")
625 } else {
626 configArgs = append(configArgs, targets...)
627 }
628
629 return configArgs
630}
631
632// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name.
633func convertToTarget(dir string, targetNamePrefix string) string {
634 return targetNamePrefix + strings.ReplaceAll(dir, "/", "-")
635}
636
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700637// hasBuildFile returns true if dir contains an Android build file.
638func hasBuildFile(ctx Context, dir string) bool {
639 for _, buildFile := range buildFiles {
640 _, err := os.Stat(filepath.Join(dir, buildFile))
641 if err == nil {
642 return true
643 }
644 if !os.IsNotExist(err) {
645 ctx.Fatalf("Error retrieving the build file stats: %v", err)
646 }
647 }
648 return false
649}
650
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700651// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file
652// in the current and any sub directory of dir. If a build file is not found, traverse the path
653// up by one directory and repeat again until either a build file is found or reached to the root
654// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank
655// string is returned.
Patrice Arruda13848222019-04-22 17:12:02 -0700656func findBuildFile(ctx Context, dir string) string {
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700657 // If the string is empty or ".", assume it is top directory of the source tree.
658 if dir == "" || dir == "." {
Patrice Arruda13848222019-04-22 17:12:02 -0700659 return ""
660 }
661
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700662 found := false
663 for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) {
664 err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error {
665 if err != nil {
666 return err
667 }
668 if found {
669 return filepath.SkipDir
670 }
671 if info.IsDir() {
672 return nil
673 }
674 for _, buildFile := range buildFiles {
675 if info.Name() == buildFile {
676 found = true
677 return filepath.SkipDir
678 }
679 }
680 return nil
681 })
682 if err != nil {
683 ctx.Fatalf("Error finding Android build file: %v", err)
684 }
685
686 if found {
687 return filepath.Join(buildDir, "Android.mk")
Patrice Arruda13848222019-04-22 17:12:02 -0700688 }
689 }
690
691 return ""
692}
693
694// splitArgs iterates over the arguments list and splits into two lists: arguments and directories.
695func splitArgs(args []string) (newArgs []string, dirs []string) {
696 specialArgs := map[string]bool{
697 "showcommands": true,
698 "snod": true,
699 "dist": true,
700 "checkbuild": true,
701 }
702
703 newArgs = []string{}
704 dirs = []string{}
705
706 for _, arg := range args {
707 // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory.
708 if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 {
709 newArgs = append(newArgs, arg)
710 continue
711 }
712
713 if _, ok := specialArgs[arg]; ok {
714 newArgs = append(newArgs, arg)
715 continue
716 }
717
718 dirs = append(dirs, arg)
719 }
720
721 return newArgs, dirs
722}
723
724// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a
725// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the
726// source root tree where the build action command was invoked. Each directory is validated if the
727// build file can be found and follows the format "dir1:target1,target2,...". Target is optional.
Dan Willemsence41e942019-07-29 23:39:30 -0700728func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) {
Patrice Arruda13848222019-04-22 17:12:02 -0700729 for _, dir := range dirs {
730 // The directory may have specified specific modules to build. ":" is the separator to separate
731 // the directory and the list of modules.
732 s := strings.Split(dir, ":")
733 l := len(s)
734 if l > 2 { // more than one ":" was specified.
735 ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir)
736 }
737
738 dir = filepath.Join(relDir, s[0])
739 if _, err := os.Stat(dir); err != nil {
740 ctx.Fatalf("couldn't find directory %s", dir)
741 }
742
743 // Verify that if there are any targets specified after ":". Each target is separated by ",".
744 var newTargets []string
745 if l == 2 && s[1] != "" {
746 newTargets = strings.Split(s[1], ",")
747 if inList("", newTargets) {
748 ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir)
749 }
750 }
751
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700752 // If there are specified targets to build in dir, an android build file must exist for the one
753 // shot build. For the non-targets case, find the appropriate build file and build all the
754 // modules in dir (or the closest one in the dir path).
Patrice Arruda13848222019-04-22 17:12:02 -0700755 if len(newTargets) > 0 {
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700756 if !hasBuildFile(ctx, dir) {
Patrice Arruda13848222019-04-22 17:12:02 -0700757 ctx.Fatalf("Couldn't locate a build file from %s directory", dir)
758 }
759 } else {
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700760 buildFile := findBuildFile(ctx, dir)
761 if buildFile == "" {
762 ctx.Fatalf("Build file not found for %s directory", dir)
763 }
764 newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
Patrice Arruda13848222019-04-22 17:12:02 -0700765 }
766
Patrice Arruda13848222019-04-22 17:12:02 -0700767 targets = append(targets, newTargets...)
768 }
769
Dan Willemsence41e942019-07-29 23:39:30 -0700770 return targets
Patrice Arruda13848222019-04-22 17:12:02 -0700771}
772
Dan Willemsen9b587492017-07-10 22:13:00 -0700773func (c *configImpl) parseArgs(ctx Context, args []string) {
774 for i := 0; i < len(args); i++ {
775 arg := strings.TrimSpace(args[i])
Anton Hansson5a7861a2021-06-04 10:09:01 +0100776 if arg == "showcommands" {
Dan Willemsen9b587492017-07-10 22:13:00 -0700777 c.verbose = true
Lukacs T. Berkicef87b62021-08-10 15:01:13 +0200778 } else if arg == "--empty-ninja-file" {
779 c.emptyNinjaFile = true
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100780 } else if arg == "--skip-ninja" {
781 c.skipNinja = true
Dan Willemsene0879fc2017-08-04 15:06:27 -0700782 } else if arg == "--skip-make" {
Colin Cross30e444b2021-06-18 11:26:19 -0700783 // TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati,
784 // but it does run a Kati ninja file if the .kati_enabled marker file was created
785 // by a previous build.
Anton Hansson5e5c48b2020-11-27 12:35:20 +0000786 c.skipConfig = true
787 c.skipKati = true
Anton Hansson0b55bdb2021-06-04 10:08:08 +0100788 } else if arg == "--soong-only" {
789 c.skipKati = true
790 c.skipKatiNinja = true
Lukacs T. Berkicef87b62021-08-10 15:01:13 +0200791 } else if arg == "--config-only" {
792 c.skipKati = true
793 c.skipKatiNinja = true
794 c.skipSoong = true
Colin Cross30e444b2021-06-18 11:26:19 -0700795 } else if arg == "--skip-config" {
796 c.skipConfig = true
Colin Cross00a8a3f2020-10-29 14:08:31 -0700797 } else if arg == "--skip-soong-tests" {
798 c.skipSoongTests = true
Colin Cross106d6ef2023-10-24 10:34:56 -0700799 } else if arg == "--no-skip-soong-tests" {
800 c.skipSoongTests = false
MarkDacekd0e7cd32022-12-02 22:22:40 +0000801 } else if arg == "--skip-metrics-upload" {
802 c.skipMetricsUpload = true
Chris Parsons53f68ae2022-03-03 12:01:40 -0500803 } else if arg == "--mk-metrics" {
804 c.reportMkMetrics = true
Spandan Das394aa322022-11-03 17:02:10 +0000805 } else if arg == "--search-api-dir" {
806 c.searchApiDir = true
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900807 } else if strings.HasPrefix(arg, "--ninja_weight_source=") {
808 source := strings.TrimPrefix(arg, "--ninja_weight_source=")
809 if source == "ninja_log" {
810 c.ninjaWeightListSource = NINJA_LOG
811 } else if source == "evenly_distributed" {
812 c.ninjaWeightListSource = EVENLY_DISTRIBUTED
813 } else if source == "not_used" {
814 c.ninjaWeightListSource = NOT_USED
Jeongik Chae114e602023-03-19 00:12:39 +0900815 } else if source == "soong" {
816 c.ninjaWeightListSource = HINT_FROM_SOONG
Jeongik Cha518f3ea2023-03-19 00:12:39 +0900817 } else if strings.HasPrefix(source, "file,") {
818 c.ninjaWeightListSource = EXTERNAL_FILE
819 filePath := strings.TrimPrefix(source, "file,")
820 err := validateNinjaWeightList(filePath)
821 if err != nil {
822 ctx.Fatalf("Malformed weight list from %s: %s", filePath, err)
823 }
824 _, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list"))
825 if err != nil {
826 ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err)
827 }
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900828 } else {
829 ctx.Fatalf("unknown option for ninja_weight_source: %s", source)
830 }
Jihoon Kang2a929ad2023-06-08 19:02:07 +0000831 } else if arg == "--build-from-source-stub" {
832 c.buildFromSourceStub = true
Yu Liufa297642024-06-11 00:13:02 +0000833 } else if arg == "--incremental-build-actions" {
834 c.incrementalBuildActions = true
MarkDacekb96561e2022-12-02 04:34:43 +0000835 } else if strings.HasPrefix(arg, "--build-command=") {
836 buildCmd := strings.TrimPrefix(arg, "--build-command=")
837 // remove quotations
838 buildCmd = strings.TrimPrefix(buildCmd, "\"")
839 buildCmd = strings.TrimSuffix(buildCmd, "\"")
840 ctx.Metrics.SetBuildCommand([]string{buildCmd})
MarkDacek6614d9c2022-12-07 21:57:38 +0000841 } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
842 buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
843 val, err := strconv.ParseInt(buildTimeStr, 10, 64)
844 if err == nil {
845 c.buildStartedTime = val
846 } else {
847 ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
848 }
MarkDacekf47e1422023-04-19 16:47:36 +0000849 } else if arg == "--ensure-allowlist-integrity" {
850 c.ensureAllowlistIntegrity = true
Dan Willemsen6ac63ef2017-10-17 20:35:34 -0700851 } else if len(arg) > 0 && arg[0] == '-' {
Dan Willemsen9b587492017-07-10 22:13:00 -0700852 parseArgNum := func(def int) int {
853 if len(arg) > 2 {
854 p, err := strconv.ParseUint(arg[2:], 10, 31)
855 if err != nil {
856 ctx.Fatalf("Failed to parse %q: %v", arg, err)
857 }
858 return int(p)
859 } else if i+1 < len(args) {
860 p, err := strconv.ParseUint(args[i+1], 10, 31)
861 if err == nil {
862 i++
863 return int(p)
864 }
865 }
866 return def
867 }
868
Dan Willemsen6ac63ef2017-10-17 20:35:34 -0700869 if len(arg) > 1 && arg[1] == 'j' {
Dan Willemsen9b587492017-07-10 22:13:00 -0700870 c.parallel = parseArgNum(c.parallel)
Dan Willemsen6ac63ef2017-10-17 20:35:34 -0700871 } else if len(arg) > 1 && arg[1] == 'k' {
Dan Willemsen9b587492017-07-10 22:13:00 -0700872 c.keepGoing = parseArgNum(0)
Dan Willemsen1e704462016-08-21 15:17:17 -0700873 } else {
874 ctx.Fatalln("Unknown option:", arg)
875 }
Dan Willemsen091525e2017-07-11 14:17:50 -0700876 } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 {
Dan Willemsen6dfe30a2018-09-10 12:41:10 -0700877 if k == "OUT_DIR" {
878 ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.")
879 }
Dan Willemsen091525e2017-07-11 14:17:50 -0700880 c.environ.Set(k, v)
Dan Willemsen2d31a442018-10-20 21:33:41 -0700881 } else if arg == "dist" {
882 c.dist = true
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200883 } else if arg == "json-module-graph" {
884 c.jsonModuleGraph = true
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200885 } else if arg == "queryview" {
886 c.queryview = true
Lukacs T. Berkic6012f32021-09-06 18:31:46 +0200887 } else if arg == "soong_docs" {
888 c.soongDocs = true
Dan Willemsen1e704462016-08-21 15:17:17 -0700889 } else {
Dan Willemsen2d31a442018-10-20 21:33:41 -0700890 if arg == "checkbuild" {
Colin Cross37193492017-11-16 17:55:00 -0800891 c.checkbuild = true
Dan Willemsene0879fc2017-08-04 15:06:27 -0700892 }
Dan Willemsen9b587492017-07-10 22:13:00 -0700893 c.arguments = append(c.arguments, arg)
Dan Willemsen1e704462016-08-21 15:17:17 -0700894 }
895 }
Dan Willemsen1e704462016-08-21 15:17:17 -0700896}
897
Jeongik Cha518f3ea2023-03-19 00:12:39 +0900898func validateNinjaWeightList(weightListFilePath string) (err error) {
899 data, err := os.ReadFile(weightListFilePath)
900 if err != nil {
901 return
902 }
903 lines := strings.Split(strings.TrimSpace(string(data)), "\n")
904 for _, line := range lines {
905 fields := strings.Split(line, ",")
906 if len(fields) != 2 {
907 return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line)
908 }
909 _, err = strconv.Atoi(fields[1])
910 if err != nil {
911 return
912 }
913 }
914 return
915}
916
Dan Willemsened869522018-01-08 14:58:46 -0800917func (c *configImpl) configureLocale(ctx Context) {
918 cmd := Command(ctx, Config{c}, "locale", "locale", "-a")
919 output, err := cmd.Output()
920
921 var locales []string
922 if err == nil {
923 locales = strings.Split(string(output), "\n")
924 } else {
925 // If we're unable to list the locales, let's assume en_US.UTF-8
926 locales = []string{"en_US.UTF-8"}
927 ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales)
928 }
929
930 // gettext uses LANGUAGE, which is passed directly through
931
932 // For LANG and LC_*, only preserve the evaluated version of
933 // LC_MESSAGES
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800934 userLang := ""
Dan Willemsened869522018-01-08 14:58:46 -0800935 if lc_all, ok := c.environ.Get("LC_ALL"); ok {
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800936 userLang = lc_all
Dan Willemsened869522018-01-08 14:58:46 -0800937 } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok {
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800938 userLang = lc_messages
Dan Willemsened869522018-01-08 14:58:46 -0800939 } else if lang, ok := c.environ.Get("LANG"); ok {
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800940 userLang = lang
Dan Willemsened869522018-01-08 14:58:46 -0800941 }
942
943 c.environ.UnsetWithPrefix("LC_")
944
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800945 if userLang != "" {
946 c.environ.Set("LC_MESSAGES", userLang)
Dan Willemsened869522018-01-08 14:58:46 -0800947 }
948
949 // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed
950 // for others)
951 if inList("C.UTF-8", locales) {
952 c.environ.Set("LANG", "C.UTF-8")
Aaron Klingd236e0e2018-08-07 19:21:36 -0500953 } else if inList("C.utf8", locales) {
954 // These normalize to the same thing
955 c.environ.Set("LANG", "C.UTF-8")
Dan Willemsened869522018-01-08 14:58:46 -0800956 } else if inList("en_US.UTF-8", locales) {
957 c.environ.Set("LANG", "en_US.UTF-8")
958 } else if inList("en_US.utf8", locales) {
959 // These normalize to the same thing
960 c.environ.Set("LANG", "en_US.UTF-8")
961 } else {
962 ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8")
963 }
964}
965
Dan Willemsen1e704462016-08-21 15:17:17 -0700966func (c *configImpl) Environment() *Environment {
967 return c.environ
968}
969
970func (c *configImpl) Arguments() []string {
971 return c.arguments
972}
973
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200974func (c *configImpl) SoongBuildInvocationNeeded() bool {
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200975 if len(c.Arguments()) > 0 {
976 // Explicit targets requested that are not special targets like b2pbuild
977 // or the JSON module graph
978 return true
979 }
980
Colin Cross8d411ff2023-12-07 10:31:24 -0800981 if !c.JsonModuleGraph() && !c.Queryview() && !c.SoongDocs() {
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200982 // Command line was empty, the default Ninja target is built
983 return true
984 }
985
Colin Cross8d411ff2023-12-07 10:31:24 -0800986 if c.Dist() {
Liz Kammer88677422021-12-15 15:03:19 -0500987 return true
988 }
989
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200990 // build.ninja doesn't need to be generated
991 return false
992}
993
Dan Willemsen1e704462016-08-21 15:17:17 -0700994func (c *configImpl) OutDir() string {
995 if outDir, ok := c.environ.Get("OUT_DIR"); ok {
Patrice Arruda19bd53e2019-07-08 17:26:47 -0700996 return outDir
Dan Willemsen1e704462016-08-21 15:17:17 -0700997 }
998 return "out"
999}
1000
Dan Willemsen8a073a82017-02-04 17:30:44 -08001001func (c *configImpl) DistDir() string {
Chris Parsons19ab9a42022-08-30 13:15:04 -04001002 return c.distDir
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +00001003}
1004
1005func (c *configImpl) RealDistDir() string {
Dan Willemsen2d31a442018-10-20 21:33:41 -07001006 return c.distDir
Dan Willemsen8a073a82017-02-04 17:30:44 -08001007}
1008
Dan Willemsen1e704462016-08-21 15:17:17 -07001009func (c *configImpl) NinjaArgs() []string {
Anton Hansson5e5c48b2020-11-27 12:35:20 +00001010 if c.skipKati {
Dan Willemsene0879fc2017-08-04 15:06:27 -07001011 return c.arguments
1012 }
Dan Willemsen1e704462016-08-21 15:17:17 -07001013 return c.ninjaArgs
1014}
1015
1016func (c *configImpl) SoongOutDir() string {
1017 return filepath.Join(c.OutDir(), "soong")
1018}
1019
Spandan Das394aa322022-11-03 17:02:10 +00001020func (c *configImpl) ApiSurfacesOutDir() string {
1021 return filepath.Join(c.OutDir(), "api_surfaces")
1022}
1023
Lukacs T. Berkia806e412021-09-01 08:57:48 +02001024func (c *configImpl) PrebuiltOS() string {
1025 switch runtime.GOOS {
1026 case "linux":
1027 return "linux-x86"
1028 case "darwin":
1029 return "darwin-x86"
1030 default:
1031 panic("Unknown GOOS")
1032 }
1033}
Lukacs T. Berki90b43342021-11-02 14:42:04 +01001034
Lukacs T. Berkia806e412021-09-01 08:57:48 +02001035func (c *configImpl) HostToolDir() string {
Colin Crossacfcc1f2021-10-25 15:40:32 -07001036 if c.SkipKatiNinja() {
1037 return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin")
1038 } else {
1039 return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin")
1040 }
Lukacs T. Berkia806e412021-09-01 08:57:48 +02001041}
1042
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +02001043func (c *configImpl) NamedGlobFile(name string) string {
Lukacs T. Berki90b43342021-11-02 14:42:04 +01001044 return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +02001045}
1046
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +02001047func (c *configImpl) UsedEnvFile(tag string) string {
Kiyoung Kimeaa55a82023-06-05 16:56:49 +09001048 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
Qing Shen713c5422024-08-23 04:09:18 +00001049 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+c.CoverageSuffix()+"."+tag)
Kiyoung Kimeaa55a82023-06-05 16:56:49 +09001050 }
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +02001051 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
1052}
1053
Lukacs T. Berkic6012f32021-09-06 18:31:46 +02001054func (c *configImpl) SoongDocsHtml() string {
1055 return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
1056}
1057
Lukacs T. Berki3a821692021-09-06 17:08:02 +02001058func (c *configImpl) QueryviewMarkerFile() string {
1059 return shared.JoinPath(c.SoongOutDir(), "queryview.marker")
1060}
1061
Lukacs T. Berkie571dc32021-08-25 14:14:13 +02001062func (c *configImpl) ModuleGraphFile() string {
1063 return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
1064}
1065
kgui67007242022-01-25 13:50:25 +08001066func (c *configImpl) ModuleActionsFile() string {
1067 return shared.JoinPath(c.SoongOutDir(), "module-actions.json")
1068}
1069
Jeff Gastonefc1b412017-03-29 17:29:06 -07001070func (c *configImpl) TempDir() string {
1071 return shared.TempDirForOutDir(c.SoongOutDir())
1072}
1073
Jeff Gastonb64fc1c2017-08-04 12:30:12 -07001074func (c *configImpl) FileListDir() string {
1075 return filepath.Join(c.OutDir(), ".module_paths")
1076}
1077
Dan Willemsen1e704462016-08-21 15:17:17 -07001078func (c *configImpl) KatiSuffix() string {
1079 if c.katiSuffix != "" {
1080 return c.katiSuffix
1081 }
1082 panic("SetKatiSuffix has not been called")
1083}
1084
Colin Cross37193492017-11-16 17:55:00 -08001085// Checkbuild returns true if "checkbuild" was one of the build goals, which means that the
1086// user is interested in additional checks at the expense of build time.
1087func (c *configImpl) Checkbuild() bool {
1088 return c.checkbuild
1089}
1090
Dan Willemsen8a073a82017-02-04 17:30:44 -08001091func (c *configImpl) Dist() bool {
1092 return c.dist
1093}
1094
Lukacs T. Berkia1b93722021-09-02 17:23:06 +02001095func (c *configImpl) JsonModuleGraph() bool {
1096 return c.jsonModuleGraph
1097}
1098
Lukacs T. Berki3a821692021-09-06 17:08:02 +02001099func (c *configImpl) Queryview() bool {
1100 return c.queryview
1101}
1102
Lukacs T. Berkic6012f32021-09-06 18:31:46 +02001103func (c *configImpl) SoongDocs() bool {
1104 return c.soongDocs
1105}
1106
Dan Willemsen1e704462016-08-21 15:17:17 -07001107func (c *configImpl) IsVerbose() bool {
1108 return c.verbose
1109}
1110
Jeongik Cha0cf44d52023-03-15 00:10:45 +09001111func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource {
1112 return c.ninjaWeightListSource
1113}
1114
Anton Hansson5e5c48b2020-11-27 12:35:20 +00001115func (c *configImpl) SkipKati() bool {
1116 return c.skipKati
1117}
1118
Anton Hansson0b55bdb2021-06-04 10:08:08 +01001119func (c *configImpl) SkipKatiNinja() bool {
1120 return c.skipKatiNinja
1121}
1122
Lukacs T. Berkicef87b62021-08-10 15:01:13 +02001123func (c *configImpl) SkipSoong() bool {
1124 return c.skipSoong
1125}
1126
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +01001127func (c *configImpl) SkipNinja() bool {
1128 return c.skipNinja
1129}
1130
Anton Hansson5a7861a2021-06-04 10:09:01 +01001131func (c *configImpl) SetSkipNinja(v bool) {
1132 c.skipNinja = v
1133}
1134
Anton Hansson5e5c48b2020-11-27 12:35:20 +00001135func (c *configImpl) SkipConfig() bool {
1136 return c.skipConfig
Dan Willemsene0879fc2017-08-04 15:06:27 -07001137}
1138
Jihoon Kang1bff0342023-01-17 20:40:22 +00001139func (c *configImpl) BuildFromTextStub() bool {
Jihoon Kang2a929ad2023-06-08 19:02:07 +00001140 return !c.buildFromSourceStub
Jihoon Kang1bff0342023-01-17 20:40:22 +00001141}
1142
Dan Willemsen1e704462016-08-21 15:17:17 -07001143func (c *configImpl) TargetProduct() string {
1144 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
1145 return v
1146 }
1147 panic("TARGET_PRODUCT is not defined")
1148}
1149
Kiyoung Kima37d9ba2023-04-19 13:13:45 +09001150func (c *configImpl) TargetProductOrErr() (string, error) {
1151 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
1152 return v, nil
1153 }
1154 return "", fmt.Errorf("TARGET_PRODUCT is not defined")
1155}
1156
Qing Shen713c5422024-08-23 04:09:18 +00001157func (c *configImpl) CoverageSuffix() string {
1158 if v := c.environ.IsEnvTrue("EMMA_INSTRUMENT"); v {
1159 return ".coverage"
1160 }
1161 return ""
1162}
1163
Dan Willemsen02781d52017-05-12 19:28:13 -07001164func (c *configImpl) TargetDevice() string {
1165 return c.targetDevice
1166}
1167
1168func (c *configImpl) SetTargetDevice(device string) {
1169 c.targetDevice = device
1170}
1171
1172func (c *configImpl) TargetBuildVariant() string {
1173 if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok {
1174 return v
1175 }
1176 panic("TARGET_BUILD_VARIANT is not defined")
1177}
1178
Dan Willemsen1e704462016-08-21 15:17:17 -07001179func (c *configImpl) KatiArgs() []string {
1180 return c.katiArgs
1181}
1182
1183func (c *configImpl) Parallel() int {
1184 return c.parallel
1185}
1186
Sam Delmerico98a73292023-02-21 11:50:29 -05001187func (c *configImpl) GetSourceRootDirs() []string {
1188 return c.sourceRootDirs
1189}
1190
1191func (c *configImpl) SetSourceRootDirs(i []string) {
1192 c.sourceRootDirs = i
1193}
1194
MarkDacek6614d9c2022-12-07 21:57:38 +00001195func (c *configImpl) GetLogsPrefix() string {
1196 return c.logsPrefix
1197}
1198
1199func (c *configImpl) SetLogsPrefix(prefix string) {
1200 c.logsPrefix = prefix
1201}
1202
Colin Cross8b8bec32019-11-15 13:18:43 -08001203func (c *configImpl) HighmemParallel() int {
1204 if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
1205 return i
1206 }
1207
1208 const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024
1209 parallel := c.Parallel()
1210 if c.UseRemoteBuild() {
1211 // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism
1212 // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs
1213 // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention.
1214 // Return 1/16th of the size of the local pool, rounding up.
1215 return (parallel + 15) / 16
1216 } else if c.totalRAM == 0 {
1217 // Couldn't detect the total RAM, don't restrict highmem processes.
1218 return parallel
Dan Willemsen570a2922020-05-26 23:02:29 -07001219 } else if c.totalRAM <= 16*1024*1024*1024 {
1220 // Less than 16GB of ram, restrict to 1 highmem processes
1221 return 1
Colin Cross8b8bec32019-11-15 13:18:43 -08001222 } else if c.totalRAM <= 32*1024*1024*1024 {
1223 // Less than 32GB of ram, restrict to 2 highmem processes
1224 return 2
1225 } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel {
1226 // If less than 8GB total RAM per process, reduce the number of highmem processes
1227 return p
1228 }
1229 // No restriction on highmem processes
1230 return parallel
1231}
1232
Dan Willemsen2bb82d02019-12-27 09:35:42 -08001233func (c *configImpl) TotalRAM() uint64 {
1234 return c.totalRAM
1235}
1236
Kousik Kumarec478642020-09-21 13:39:24 -04001237// ForceUseGoma determines whether we should override Goma deprecation
1238// and use Goma for the current build or not.
1239func (c *configImpl) ForceUseGoma() bool {
1240 if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok {
1241 v = strings.TrimSpace(v)
1242 if v != "" && v != "false" {
1243 return true
1244 }
1245 }
1246 return false
1247}
1248
Dan Willemsen1e704462016-08-21 15:17:17 -07001249func (c *configImpl) UseGoma() bool {
1250 if v, ok := c.environ.Get("USE_GOMA"); ok {
1251 v = strings.TrimSpace(v)
1252 if v != "" && v != "false" {
1253 return true
1254 }
1255 }
1256 return false
1257}
1258
Yoshisato Yanagisawa2cb0e5d2019-01-10 10:14:16 +09001259func (c *configImpl) StartGoma() bool {
1260 if !c.UseGoma() {
1261 return false
1262 }
1263
1264 if v, ok := c.environ.Get("NOSTART_GOMA"); ok {
1265 v = strings.TrimSpace(v)
1266 if v != "" && v != "false" {
1267 return false
1268 }
1269 }
1270 return true
1271}
1272
Kousik Kumar6d1e3482023-07-24 03:44:16 +00001273func (c *configImpl) canSupportRBE() bool {
Joe Onorato86f50e72024-06-24 14:28:25 -07001274 // Only supported on linux
1275 if runtime.GOOS != "linux" {
1276 return false
1277 }
1278
Kousik Kumar6d1e3482023-07-24 03:44:16 +00001279 // Do not use RBE with prod credentials in scenarios when stubby doesn't exist, since
1280 // its unlikely that we will be able to obtain necessary creds without stubby.
1281 authType, _ := c.rbeAuth()
1282 if !c.StubbyExists() && strings.Contains(authType, "use_google_prod_creds") {
1283 return false
1284 }
Taylor Santiago3c16e612024-05-30 14:41:31 -07001285 if c.UseABFS() {
1286 return false
1287 }
Kousik Kumar6d1e3482023-07-24 03:44:16 +00001288 return true
1289}
1290
Taylor Santiago3c16e612024-05-30 14:41:31 -07001291func (c *configImpl) UseABFS() bool {
1292 if v, ok := c.environ.Get("NO_ABFS"); ok {
1293 v = strings.ToLower(strings.TrimSpace(v))
1294 if v == "true" || v == "1" {
1295 return false
1296 }
1297 }
1298
1299 abfsBox := c.PrebuiltBuildTool("abfsbox")
1300 err := exec.Command(abfsBox, "hash", srcDirFileCheck).Run()
1301 return err == nil
1302}
1303
Taylor Santiago8b0bed72024-09-03 13:30:22 -07001304func (c *configImpl) sandboxPath(base, in string) string {
1305 if !c.UseABFS() {
1306 return in
1307 }
1308
1309 rel, err := filepath.Rel(base, in)
1310 if err != nil {
1311 return in
1312 }
1313
1314 return filepath.Join(abfsSrcDir, rel)
1315}
1316
Ramy Medhatbbf25672019-07-17 12:30:04 +00001317func (c *configImpl) UseRBE() bool {
Jingwen Chend7ccde12023-06-28 07:19:26 +00001318 // These alternate modes of running Soong do not use RBE / reclient.
Colin Cross8d411ff2023-12-07 10:31:24 -08001319 if c.Queryview() || c.JsonModuleGraph() {
Jingwen Chend7ccde12023-06-28 07:19:26 +00001320 return false
1321 }
1322
Kousik Kumar6d1e3482023-07-24 03:44:16 +00001323 if !c.canSupportRBE() {
Kousik Kumar67ad4342023-06-06 15:09:27 -04001324 return false
1325 }
Kousik Kumar6d1e3482023-07-24 03:44:16 +00001326
Kousik Kumar3ff037e2022-01-25 22:11:01 -05001327 if v, ok := c.Environment().Get("USE_RBE"); ok {
Ramy Medhatbbf25672019-07-17 12:30:04 +00001328 v = strings.TrimSpace(v)
1329 if v != "" && v != "false" {
1330 return true
1331 }
1332 }
1333 return false
1334}
1335
1336func (c *configImpl) StartRBE() bool {
1337 if !c.UseRBE() {
1338 return false
1339 }
1340
1341 if v, ok := c.environ.Get("NOSTART_RBE"); ok {
1342 v = strings.TrimSpace(v)
1343 if v != "" && v != "false" {
1344 return false
1345 }
1346 }
1347 return true
1348}
1349
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001350func (c *configImpl) rbeProxyLogsDir() string {
1351 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
Kousik Kumar0d15a722020-09-23 02:54:11 -04001352 if v, ok := c.environ.Get(f); ok {
1353 return v
1354 }
1355 }
Ramy Medhatbc061762023-10-10 18:36:59 +00001356 return c.rbeTmpDir()
1357}
1358
1359func (c *configImpl) rbeDownloadTmpDir() string {
Cole Faust06ea5312023-10-18 17:38:40 -07001360 for _, f := range []string{"RBE_download_tmp_dir", "FLAG_download_tmp_dir"} {
Ramy Medhatbc061762023-10-10 18:36:59 +00001361 if v, ok := c.environ.Get(f); ok {
1362 return v
1363 }
1364 }
1365 return c.rbeTmpDir()
1366}
1367
1368func (c *configImpl) rbeTmpDir() string {
Yaowen Meid4da2662024-07-24 08:25:12 +00001369 return filepath.Join(c.SoongOutDir(), "rbe")
Ramy Medhat0fc67eb2020-08-12 01:26:23 -04001370}
1371
Ramy Medhatc8f6cc22023-03-31 09:50:34 -04001372func (c *configImpl) rbeCacheDir() string {
1373 for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} {
1374 if v, ok := c.environ.Get(f); ok {
1375 return v
1376 }
1377 }
1378 return shared.JoinPath(c.SoongOutDir(), "rbe")
1379}
1380
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001381func (c *configImpl) shouldCleanupRBELogsDir() bool {
1382 // Perform a log directory cleanup only when the log directory
1383 // is auto created by the build rather than user-specified.
1384 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
1385 if _, ok := c.environ.Get(f); ok {
1386 return false
Patrice Arruda62f1bf22020-07-07 12:48:26 +00001387 }
1388 }
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001389 return true
Ramy Medhat0fc67eb2020-08-12 01:26:23 -04001390}
1391
1392func (c *configImpl) rbeExecRoot() string {
1393 for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} {
1394 if v, ok := c.environ.Get(f); ok {
1395 return v
1396 }
1397 }
1398 wd, err := os.Getwd()
1399 if err != nil {
1400 return ""
1401 }
1402 return wd
1403}
1404
1405func (c *configImpl) rbeDir() string {
1406 if v, ok := c.environ.Get("RBE_DIR"); ok {
1407 return v
1408 }
1409 return "prebuilts/remoteexecution-client/live/"
1410}
1411
1412func (c *configImpl) rbeReproxy() string {
1413 for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} {
1414 if v, ok := c.environ.Get(f); ok {
1415 return v
1416 }
1417 }
1418 return filepath.Join(c.rbeDir(), "reproxy")
1419}
1420
1421func (c *configImpl) rbeAuth() (string, string) {
Kousik Kumar93d192c2022-03-18 01:39:56 -04001422 credFlags := []string{
1423 "use_application_default_credentials",
1424 "use_gce_credentials",
1425 "credential_file",
1426 "use_google_prod_creds",
1427 }
Ramy Medhat0fc67eb2020-08-12 01:26:23 -04001428 for _, cf := range credFlags {
1429 for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} {
1430 if v, ok := c.environ.Get(f); ok {
1431 v = strings.TrimSpace(v)
1432 if v != "" && v != "false" && v != "0" {
1433 return "RBE_" + cf, v
1434 }
1435 }
1436 }
1437 }
1438 return "RBE_use_application_default_credentials", "true"
Patrice Arruda62f1bf22020-07-07 12:48:26 +00001439}
1440
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001441func (c *configImpl) rbeSockAddr(dir string) (string, error) {
Andus Yuc917eb82024-03-06 21:54:15 +00001442 // Absolute path socket addresses have a prefix of //. This should
1443 // be included in the length limit.
1444 maxNameLen := len(syscall.RawSockaddrUnix{}.Path) - 2
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001445 base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix)
1446
1447 name := filepath.Join(dir, base)
1448 if len(name) < maxNameLen {
1449 return name, nil
1450 }
1451
1452 name = filepath.Join("/tmp", base)
1453 if len(name) < maxNameLen {
1454 return name, nil
1455 }
1456
1457 return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen)
1458}
1459
Kousik Kumar7bc78192022-04-27 14:52:56 -04001460// IsGooglerEnvironment returns true if the current build is running
1461// on a Google developer machine and false otherwise.
1462func (c *configImpl) IsGooglerEnvironment() bool {
1463 cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG"
1464 if v, ok := c.environ.Get(cf); ok {
1465 return v == "googler"
1466 }
1467 return false
1468}
1469
1470// GoogleProdCredsExist determine whether credentials exist on the
1471// Googler machine to use remote execution.
1472func (c *configImpl) GoogleProdCredsExist() bool {
Kevin Dagostino096ab2f2023-03-03 19:47:17 +00001473 if googleProdCredsExistCache {
1474 return googleProdCredsExistCache
1475 }
andusyu0b3dc032023-06-21 17:29:32 -04001476 if _, err := exec.Command("/usr/bin/gcertstatus", "-nocheck_ssh").Output(); err != nil {
Kousik Kumar7bc78192022-04-27 14:52:56 -04001477 return false
1478 }
Kevin Dagostino096ab2f2023-03-03 19:47:17 +00001479 googleProdCredsExistCache = true
Kousik Kumar7bc78192022-04-27 14:52:56 -04001480 return true
1481}
1482
1483// UseRemoteBuild indicates whether to use a remote build acceleration system
1484// to speed up the build.
Colin Cross9016b912019-11-11 14:57:42 -08001485func (c *configImpl) UseRemoteBuild() bool {
1486 return c.UseGoma() || c.UseRBE()
1487}
1488
Kousik Kumar7bc78192022-04-27 14:52:56 -04001489// StubbyExists checks whether the stubby binary exists on the machine running
1490// the build.
1491func (c *configImpl) StubbyExists() bool {
1492 if _, err := exec.LookPath("stubby"); err != nil {
1493 return false
1494 }
1495 return true
1496}
1497
Dan Willemsen1e704462016-08-21 15:17:17 -07001498// RemoteParallel controls how many remote jobs (i.e., commands which contain
Jeff Gastonefc1b412017-03-29 17:29:06 -07001499// gomacc) are run in parallel. Note the parallelism of all other jobs is
Dan Willemsen1e704462016-08-21 15:17:17 -07001500// still limited by Parallel()
1501func (c *configImpl) RemoteParallel() int {
Colin Cross8b8bec32019-11-15 13:18:43 -08001502 if !c.UseRemoteBuild() {
1503 return 0
1504 }
1505 if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok {
1506 return i
Dan Willemsen1e704462016-08-21 15:17:17 -07001507 }
1508 return 500
1509}
1510
1511func (c *configImpl) SetKatiArgs(args []string) {
1512 c.katiArgs = args
1513}
1514
1515func (c *configImpl) SetNinjaArgs(args []string) {
1516 c.ninjaArgs = args
1517}
1518
1519func (c *configImpl) SetKatiSuffix(suffix string) {
1520 c.katiSuffix = suffix
1521}
1522
Dan Willemsene0879fc2017-08-04 15:06:27 -07001523func (c *configImpl) LastKatiSuffixFile() string {
1524 return filepath.Join(c.OutDir(), "last_kati_suffix")
1525}
1526
1527func (c *configImpl) HasKatiSuffix() bool {
1528 return c.katiSuffix != ""
1529}
1530
Dan Willemsen1e704462016-08-21 15:17:17 -07001531func (c *configImpl) KatiEnvFile() string {
1532 return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
1533}
1534
Dan Willemsen29971232018-09-26 14:58:30 -07001535func (c *configImpl) KatiBuildNinjaFile() string {
1536 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja")
Dan Willemsen1e704462016-08-21 15:17:17 -07001537}
1538
Dan Willemsenfb1271a2018-09-26 15:00:42 -07001539func (c *configImpl) KatiPackageNinjaFile() string {
1540 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja")
1541}
1542
Jihoon Kang9f4f8a32022-08-16 00:57:30 +00001543func (c *configImpl) SoongVarsFile() string {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +09001544 targetProduct, err := c.TargetProductOrErr()
1545 if err != nil {
1546 return filepath.Join(c.SoongOutDir(), "soong.variables")
1547 } else {
Qing Shen713c5422024-08-23 04:09:18 +00001548 return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+c.CoverageSuffix()+".variables")
Kiyoung Kima37d9ba2023-04-19 13:13:45 +09001549 }
Jihoon Kang9f4f8a32022-08-16 00:57:30 +00001550}
1551
Inseob Kim58c802f2024-06-11 10:59:00 +09001552func (c *configImpl) SoongExtraVarsFile() string {
1553 targetProduct, err := c.TargetProductOrErr()
1554 if err != nil {
1555 return filepath.Join(c.SoongOutDir(), "soong.extra.variables")
1556 } else {
Qing Shen713c5422024-08-23 04:09:18 +00001557 return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+c.CoverageSuffix()+".extra.variables")
Inseob Kim58c802f2024-06-11 10:59:00 +09001558 }
1559}
1560
Dan Willemsen1e704462016-08-21 15:17:17 -07001561func (c *configImpl) SoongNinjaFile() string {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +09001562 targetProduct, err := c.TargetProductOrErr()
1563 if err != nil {
1564 return filepath.Join(c.SoongOutDir(), "build.ninja")
1565 } else {
Qing Shen713c5422024-08-23 04:09:18 +00001566 return filepath.Join(c.SoongOutDir(), "build."+targetProduct+c.CoverageSuffix()+".ninja")
Kiyoung Kima37d9ba2023-04-19 13:13:45 +09001567 }
Dan Willemsen1e704462016-08-21 15:17:17 -07001568}
1569
1570func (c *configImpl) CombinedNinjaFile() string {
Dan Willemsene0879fc2017-08-04 15:06:27 -07001571 if c.katiSuffix == "" {
1572 return filepath.Join(c.OutDir(), "combined.ninja")
1573 }
Dan Willemsen1e704462016-08-21 15:17:17 -07001574 return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
1575}
1576
1577func (c *configImpl) SoongAndroidMk() string {
Qing Shen713c5422024-08-23 04:09:18 +00001578 return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+c.CoverageSuffix()+".mk")
Dan Willemsen1e704462016-08-21 15:17:17 -07001579}
1580
1581func (c *configImpl) SoongMakeVarsMk() string {
Qing Shen713c5422024-08-23 04:09:18 +00001582 return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+c.CoverageSuffix()+".mk")
Dan Willemsen1e704462016-08-21 15:17:17 -07001583}
1584
Colin Crossaa9a2732023-10-27 10:54:27 -07001585func (c *configImpl) SoongBuildMetrics() string {
Colin Crossb67b0612023-10-31 10:02:45 -07001586 return filepath.Join(c.LogsDir(), "soong_build_metrics.pb")
Colin Crossaa9a2732023-10-27 10:54:27 -07001587}
1588
Dan Willemsenf052f782017-05-18 15:29:04 -07001589func (c *configImpl) ProductOut() string {
Dan Willemsen4dc4e142017-09-08 14:35:43 -07001590 return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
Dan Willemsenf052f782017-05-18 15:29:04 -07001591}
1592
Dan Willemsen02781d52017-05-12 19:28:13 -07001593func (c *configImpl) DevicePreviousProductConfig() string {
Dan Willemsenf052f782017-05-18 15:29:04 -07001594 return filepath.Join(c.ProductOut(), "previous_build_config.mk")
1595}
1596
Dan Willemsenfb1271a2018-09-26 15:00:42 -07001597func (c *configImpl) KatiPackageMkDir() string {
1598 return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging")
1599}
1600
Dan Willemsenf052f782017-05-18 15:29:04 -07001601func (c *configImpl) hostOutRoot() string {
Dan Willemsen4dc4e142017-09-08 14:35:43 -07001602 return filepath.Join(c.OutDir(), "host")
Dan Willemsenf052f782017-05-18 15:29:04 -07001603}
1604
1605func (c *configImpl) HostOut() string {
1606 return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag())
1607}
1608
1609// This probably needs to be multi-valued, so not exporting it for now
1610func (c *configImpl) hostCrossOut() string {
1611 if runtime.GOOS == "linux" {
1612 return filepath.Join(c.hostOutRoot(), "windows-x86")
1613 } else {
1614 return ""
1615 }
Dan Willemsen02781d52017-05-12 19:28:13 -07001616}
1617
Dan Willemsen1e704462016-08-21 15:17:17 -07001618func (c *configImpl) HostPrebuiltTag() string {
1619 if runtime.GOOS == "linux" {
1620 return "linux-x86"
1621 } else if runtime.GOOS == "darwin" {
1622 return "darwin-x86"
1623 } else {
1624 panic("Unsupported OS")
1625 }
1626}
Dan Willemsenf173d592017-04-27 14:28:00 -07001627
Taylor Santiago3c16e612024-05-30 14:41:31 -07001628func (c *configImpl) KatiBin() string {
1629 binName := "ckati"
1630 if c.UseABFS() {
1631 binName = "ckati-wrap"
1632 }
1633
1634 return c.PrebuiltBuildTool(binName)
1635}
1636
1637func (c *configImpl) NinjaBin() string {
1638 binName := "ninja"
1639 if c.UseABFS() {
1640 binName = "ninjago"
1641 }
1642 return c.PrebuiltBuildTool(binName)
1643}
1644
Cole Faust4e58bba2024-08-22 14:27:03 -07001645func (c *configImpl) N2Bin() string {
1646 path := c.PrebuiltBuildTool("n2")
1647 // Use musl instead of glibc because glibc on the build server is old and has bugs
1648 return strings.ReplaceAll(path, "/linux-x86/", "/linux_musl-x86/")
1649}
1650
Dan Willemsen8122bd52017-10-12 20:20:41 -07001651func (c *configImpl) PrebuiltBuildTool(name string) string {
Dan Willemsenf173d592017-04-27 14:28:00 -07001652 if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
1653 if sanitize := strings.Fields(v); inList("address", sanitize) {
Dan Willemsen8122bd52017-10-12 20:20:41 -07001654 asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
1655 if _, err := os.Stat(asan); err == nil {
1656 return asan
1657 }
Dan Willemsenf173d592017-04-27 14:28:00 -07001658 }
1659 }
1660 return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
1661}
Dan Willemsen3d60b112018-04-04 22:25:56 -07001662
1663func (c *configImpl) SetBuildBrokenDupRules(val bool) {
1664 c.brokenDupRules = val
1665}
1666
1667func (c *configImpl) BuildBrokenDupRules() bool {
1668 return c.brokenDupRules
1669}
Dan Willemsen6ab79db2018-05-02 00:06:28 -07001670
Dan Willemsen25e6f092019-04-09 10:22:43 -07001671func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) {
1672 c.brokenUsesNetwork = val
1673}
1674
1675func (c *configImpl) BuildBrokenUsesNetwork() bool {
1676 return c.brokenUsesNetwork
1677}
1678
Dan Willemsene3336352020-01-02 19:10:38 -08001679func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) {
1680 c.brokenNinjaEnvVars = val
1681}
1682
1683func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string {
1684 return c.brokenNinjaEnvVars
1685}
1686
Spandan Das28a6f192024-07-01 21:00:25 +00001687func (c *configImpl) SetBuildBrokenMissingOutputs(val bool) {
1688 c.brokenMissingOutputs = val
1689}
1690
1691func (c *configImpl) BuildBrokenMissingOutputs() bool {
1692 return c.brokenMissingOutputs
1693}
1694
Dan Willemsen6ab79db2018-05-02 00:06:28 -07001695func (c *configImpl) SetTargetDeviceDir(dir string) {
1696 c.targetDeviceDir = dir
1697}
1698
1699func (c *configImpl) TargetDeviceDir() string {
1700 return c.targetDeviceDir
1701}
Dan Willemsenfa42f3c2018-06-15 21:54:47 -07001702
Patrice Arruda219eef32020-06-01 17:29:30 +00001703func (c *configImpl) BuildDateTime() string {
1704 return c.buildDateTime
1705}
1706
1707func (c *configImpl) MetricsUploaderApp() string {
Yu Liu6e13b402021-07-27 14:29:06 -07001708 return c.metricsUploader
Patrice Arruda219eef32020-06-01 17:29:30 +00001709}
Patrice Arruda83842d72020-12-08 19:42:08 +00001710
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -05001711// LogsDir returns the absolute path to the logs directory where build log and
1712// metrics files are located. By default, the logs directory is the out
Patrice Arruda83842d72020-12-08 19:42:08 +00001713// directory. If the argument dist is specified, the logs directory
1714// is <dist_dir>/logs.
1715func (c *configImpl) LogsDir() string {
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -05001716 dir := c.OutDir()
Patrice Arruda83842d72020-12-08 19:42:08 +00001717 if c.Dist() {
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +00001718 // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -05001719 dir = filepath.Join(c.RealDistDir(), "logs")
Patrice Arruda83842d72020-12-08 19:42:08 +00001720 }
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -05001721 absDir, err := filepath.Abs(dir)
1722 if err != nil {
1723 fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error())
1724 os.Exit(1)
1725 }
1726 return absDir
Patrice Arruda83842d72020-12-08 19:42:08 +00001727}
1728
Chris Parsons53f68ae2022-03-03 12:01:40 -05001729// MkFileMetrics returns the file path for make-related metrics.
1730func (c *configImpl) MkMetrics() string {
1731 return filepath.Join(c.LogsDir(), "mk_metrics.pb")
1732}
1733
Colin Crossf3bdbcb2021-06-01 11:43:55 -07001734func (c *configImpl) SetEmptyNinjaFile(v bool) {
1735 c.emptyNinjaFile = v
1736}
1737
1738func (c *configImpl) EmptyNinjaFile() bool {
1739 return c.emptyNinjaFile
1740}
Yu Liu6e13b402021-07-27 14:29:06 -07001741
MarkDacekd0e7cd32022-12-02 22:22:40 +00001742func (c *configImpl) SkipMetricsUpload() bool {
Taylor Santiago8b0bed72024-09-03 13:30:22 -07001743 // b/362625275 - Metrics upload sometimes prevents abfs unmount
1744 if c.UseABFS() {
1745 return true
1746 }
1747
MarkDacekd0e7cd32022-12-02 22:22:40 +00001748 return c.skipMetricsUpload
1749}
1750
MarkDacekf47e1422023-04-19 16:47:36 +00001751func (c *configImpl) EnsureAllowlistIntegrity() bool {
1752 return c.ensureAllowlistIntegrity
1753}
1754
MarkDacek6614d9c2022-12-07 21:57:38 +00001755// Returns a Time object if one was passed via a command-line flag.
1756// Otherwise returns the passed default.
1757func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
1758 if c.buildStartedTime == 0 {
1759 return defaultTime
1760 }
1761 return time.UnixMilli(c.buildStartedTime)
1762}
1763
Yu Liu6e13b402021-07-27 14:29:06 -07001764func GetMetricsUploader(topDir string, env *Environment) string {
1765 if p, ok := env.Get("METRICS_UPLOADER"); ok {
1766 metricsUploader := filepath.Join(topDir, p)
1767 if _, err := os.Stat(metricsUploader); err == nil {
1768 return metricsUploader
1769 }
1770 }
1771
1772 return ""
1773}