blob: 1a2539793b741ef81d4901b31856d3cce73233a0 [file] [log] [blame]
Dan Willemsen1e704462016-08-21 15:17:17 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
Kousik Kumar3ff037e2022-01-25 22:11:01 -050018 "encoding/json"
Jeongik Chaa87506f2023-06-01 23:16:41 +090019 "errors"
Ramy Medhat0fc67eb2020-08-12 01:26:23 -040020 "fmt"
Kousik Kumar3ff037e2022-01-25 22:11:01 -050021 "io/ioutil"
Kousik Kumar4c180ad2022-05-27 07:48:37 -040022 "math/rand"
Dan Willemsenc2af0be2017-01-20 14:10:01 -080023 "os"
Kousik Kumar84bd5bf2022-01-26 23:32:22 -050024 "os/exec"
Cole Faust583dfb42023-09-28 13:56:30 -070025 "os/user"
Dan Willemsen1e704462016-08-21 15:17:17 -070026 "path/filepath"
27 "runtime"
28 "strconv"
29 "strings"
Kousik Kumar4c180ad2022-05-27 07:48:37 -040030 "syscall"
Nan Zhang2e6a4ff2018-02-14 13:27:26 -080031 "time"
Jeff Gastonefc1b412017-03-29 17:29:06 -070032
33 "android/soong/shared"
LaMont Jones9a912862023-11-06 22:11:08 +000034 "android/soong/ui/metrics"
Kousik Kumarec478642020-09-21 13:39:24 -040035
Dan Willemsen4591b642021-05-24 14:24:12 -070036 "google.golang.org/protobuf/proto"
Patrice Arruda96850362020-08-11 20:41:11 +000037
38 smpb "android/soong/ui/metrics/metrics_proto"
Dan Willemsen1e704462016-08-21 15:17:17 -070039)
40
Kousik Kumar3ff037e2022-01-25 22:11:01 -050041const (
Chris Parsons53f68ae2022-03-03 12:01:40 -050042 envConfigDir = "vendor/google/tools/soong_config"
43 jsonSuffix = "json"
Kousik Kumar3ff037e2022-01-25 22:11:01 -050044)
45
Kousik Kumar4c180ad2022-05-27 07:48:37 -040046var (
Kevin Dagostino096ab2f2023-03-03 19:47:17 +000047 rbeRandPrefix int
48 googleProdCredsExistCache bool
Kousik Kumar4c180ad2022-05-27 07:48:37 -040049)
50
51func init() {
52 rand.Seed(time.Now().UnixNano())
53 rbeRandPrefix = rand.Intn(1000)
54}
55
Dan Willemsen1e704462016-08-21 15:17:17 -070056type Config struct{ *configImpl }
57
58type configImpl struct {
Lukacs T. Berkia1b93722021-09-02 17:23:06 +020059 // Some targets that are implemented in soong_build
60 // (bp2build, json-module-graph) are not here and have their own bits below.
Colin Cross28f527c2019-11-26 16:19:04 -080061 arguments []string
62 goma bool
63 environ *Environment
64 distDir string
65 buildDateTime string
MarkDacek6614d9c2022-12-07 21:57:38 +000066 logsPrefix string
Dan Willemsen1e704462016-08-21 15:17:17 -070067
68 // From the arguments
MarkDacekf47e1422023-04-19 16:47:36 +000069 parallel int
70 keepGoing int
71 verbose bool
72 checkbuild bool
73 dist bool
74 jsonModuleGraph bool
MarkDacekf47e1422023-04-19 16:47:36 +000075 queryview bool
76 reportMkMetrics bool // Collect and report mk2bp migration progress metrics.
77 soongDocs bool
MarkDacekf47e1422023-04-19 16:47:36 +000078 skipConfig bool
79 skipKati bool
80 skipKatiNinja bool
81 skipSoong bool
82 skipNinja bool
83 skipSoongTests bool
84 searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces
85 skipMetricsUpload bool
86 buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time
Jihoon Kang2a929ad2023-06-08 19:02:07 +000087 buildFromSourceStub bool
Colin Cross8d411ff2023-12-07 10:31:24 -080088 ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built
Dan Willemsen1e704462016-08-21 15:17:17 -070089
90 // From the product config
Dan Willemsen6ab79db2018-05-02 00:06:28 -070091 katiArgs []string
92 ninjaArgs []string
93 katiSuffix string
94 targetDevice string
95 targetDeviceDir string
Spandan Dasa3639e62021-05-25 19:14:02 +000096 sandboxConfig *SandboxConfig
Dan Willemsen3d60b112018-04-04 22:25:56 -070097
Dan Willemsen2bb82d02019-12-27 09:35:42 -080098 // Autodetected
99 totalRAM uint64
100
Dan Willemsene3336352020-01-02 19:10:38 -0800101 brokenDupRules bool
102 brokenUsesNetwork bool
103 brokenNinjaEnvVars []string
Dan Willemsen18490112018-05-25 16:30:04 -0700104
105 pathReplaced bool
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +0000106
Colin Crossf3bdbcb2021-06-01 11:43:55 -0700107 // Set by multiproduct_kati
108 emptyNinjaFile bool
Yu Liu6e13b402021-07-27 14:29:06 -0700109
110 metricsUploader string
MarkDacekd06db5d2022-11-29 00:47:59 +0000111
Sam Delmerico98a73292023-02-21 11:50:29 -0500112 includeTags []string
113 sourceRootDirs []string
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900114
115 // Data source to write ninja weight list
116 ninjaWeightListSource NinjaWeightListSource
Joe Onoratoe5ed3472024-02-02 14:52:05 -0800117
118 // This file is a detailed dump of all soong-defined modules for debugging purposes.
119 // There's quite a bit of overlap with module-info.json and soong module graph. We
120 // could consider merging them.
121 moduleDebugFile string
Dan Willemsen1e704462016-08-21 15:17:17 -0700122}
123
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900124type NinjaWeightListSource uint
125
126const (
127 // ninja doesn't use weight list.
128 NOT_USED NinjaWeightListSource = iota
129 // ninja uses weight list based on previous builds by ninja log
130 NINJA_LOG
131 // ninja thinks every task has the same weight.
132 EVENLY_DISTRIBUTED
Jeongik Cha518f3ea2023-03-19 00:12:39 +0900133 // ninja uses an external custom weight list
134 EXTERNAL_FILE
Jeongik Chae114e602023-03-19 00:12:39 +0900135 // ninja uses a prioritized module list from Soong
136 HINT_FROM_SOONG
Jeongik Chaa87506f2023-06-01 23:16:41 +0900137 // If ninja log exists, use NINJA_LOG, if not, use HINT_FROM_SOONG instead.
138 // We can assume it is an incremental build if ninja log exists.
139 DEFAULT
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900140)
Dan Willemsenc2af0be2017-01-20 14:10:01 -0800141const srcDirFileCheck = "build/soong/root.bp"
142
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700143var buildFiles = []string{"Android.mk", "Android.bp"}
144
Patrice Arruda13848222019-04-22 17:12:02 -0700145type BuildAction uint
146
147const (
148 // Builds all of the modules and their dependencies of a specified directory, relative to the root
149 // directory of the source tree.
150 BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota
151
152 // Builds all of the modules and their dependencies of a list of specified directories. All specified
153 // directories are relative to the root directory of the source tree.
154 BUILD_MODULES_IN_DIRECTORIES
Patrice Arruda39282062019-06-20 16:35:12 -0700155
156 // Build a list of specified modules. If none was specified, simply build the whole source tree.
157 BUILD_MODULES
Patrice Arruda13848222019-04-22 17:12:02 -0700158)
159
160// checkTopDir validates that the current directory is at the root directory of the source tree.
161func checkTopDir(ctx Context) {
162 if _, err := os.Stat(srcDirFileCheck); err != nil {
163 if os.IsNotExist(err) {
164 ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck)
165 }
166 ctx.Fatalln("Error verifying tree state:", err)
167 }
168}
169
MarkDacek7901e582023-01-09 19:48:01 +0000170func loadEnvConfig(ctx Context, config *configImpl, bc string) error {
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500171 if bc == "" {
172 return nil
173 }
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500174
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500175 configDirs := []string{
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500176 config.OutDir(),
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500177 os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"),
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500178 envConfigDir,
179 }
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500180 for _, dir := range configDirs {
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500181 cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix))
182 envVarsJSON, err := ioutil.ReadFile(cfgFile)
183 if err != nil {
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500184 continue
185 }
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500186 ctx.Verbosef("Loading config file %v\n", cfgFile)
187 var envVars map[string]map[string]string
188 if err := json.Unmarshal(envVarsJSON, &envVars); err != nil {
189 fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error())
190 continue
191 }
192 for k, v := range envVars["env"] {
193 if os.Getenv(k) != "" {
194 continue
195 }
196 config.environ.Set(k, v)
197 }
198 ctx.Verbosef("Finished loading config file %v\n", cfgFile)
199 break
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500200 }
Kousik Kumar84bd5bf2022-01-26 23:32:22 -0500201
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500202 return nil
203}
204
Dan Willemsen1e704462016-08-21 15:17:17 -0700205func NewConfig(ctx Context, args ...string) Config {
206 ret := &configImpl{
Jeongik Chaf2ecf762023-05-19 14:03:45 +0900207 environ: OsEnvironment(),
208 sandboxConfig: &SandboxConfig{},
Jeongik Chaa87506f2023-06-01 23:16:41 +0900209 ninjaWeightListSource: DEFAULT,
Dan Willemsen1e704462016-08-21 15:17:17 -0700210 }
211
Colin Cross106d6ef2023-10-24 10:34:56 -0700212 // Skip soong tests by default on Linux
213 if runtime.GOOS == "linux" {
214 ret.skipSoongTests = true
215 }
216
Patrice Arruda90109172020-07-28 18:07:27 +0000217 // Default matching ninja
Dan Willemsen9b587492017-07-10 22:13:00 -0700218 ret.parallel = runtime.NumCPU() + 2
219 ret.keepGoing = 1
220
Dan Willemsen2bb82d02019-12-27 09:35:42 -0800221 ret.totalRAM = detectTotalRAM(ctx)
Dan Willemsen9b587492017-07-10 22:13:00 -0700222 ret.parseArgs(ctx, args)
Jeongik Chae114e602023-03-19 00:12:39 +0900223
224 if ret.ninjaWeightListSource == HINT_FROM_SOONG {
Jeongik Chaa87506f2023-06-01 23:16:41 +0900225 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "always")
226 } else if ret.ninjaWeightListSource == DEFAULT {
227 defaultNinjaWeightListSource := NINJA_LOG
228 if _, err := os.Stat(filepath.Join(ret.OutDir(), ninjaLogFileName)); errors.Is(err, os.ErrNotExist) {
229 ctx.Verboseln("$OUT/.ninja_log doesn't exist, use HINT_FROM_SOONG instead")
230 defaultNinjaWeightListSource = HINT_FROM_SOONG
231 } else {
232 ctx.Verboseln("$OUT/.ninja_log exist, use NINJA_LOG")
233 }
234 ret.ninjaWeightListSource = defaultNinjaWeightListSource
235 // soong_build generates ninja hint depending on ninja log existence.
236 // Set it "depend" to avoid soong re-run due to env variable change.
237 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "depend")
Jeongik Chae114e602023-03-19 00:12:39 +0900238 }
Jeongik Chaa87506f2023-06-01 23:16:41 +0900239
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800240 // Make sure OUT_DIR is set appropriately
Dan Willemsen02f3add2017-05-12 13:50:19 -0700241 if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
242 ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
243 } else {
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800244 outDir := "out"
245 if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok {
246 if wd, err := os.Getwd(); err != nil {
247 ctx.Fatalln("Failed to get working directory:", err)
248 } else {
249 outDir = filepath.Join(baseDir, filepath.Base(wd))
250 }
251 }
252 ret.environ.Set("OUT_DIR", outDir)
253 }
254
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500255 // loadEnvConfig needs to know what the OUT_DIR is, so it should
256 // be called after we determine the appropriate out directory.
MarkDacek7901e582023-01-09 19:48:01 +0000257 bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG")
258
259 if bc != "" {
Kousik Kumarc8818332023-01-16 16:33:05 +0000260 if err := loadEnvConfig(ctx, ret, bc); err != nil {
MarkDacek7901e582023-01-09 19:48:01 +0000261 ctx.Fatalln("Failed to parse env config files: %v", err)
262 }
Kousik Kumar6d1e3482023-07-24 03:44:16 +0000263 if !ret.canSupportRBE() {
264 // Explicitly set USE_RBE env variable to false when we cannot run
265 // an RBE build to avoid ninja local execution pool issues.
266 ret.environ.Set("USE_RBE", "false")
267 }
Kousik Kumar3ff037e2022-01-25 22:11:01 -0500268 }
269
Dan Willemsen2d31a442018-10-20 21:33:41 -0700270 if distDir, ok := ret.environ.Get("DIST_DIR"); ok {
271 ret.distDir = filepath.Clean(distDir)
272 } else {
273 ret.distDir = filepath.Join(ret.OutDir(), "dist")
274 }
Dan Willemsend50e89f2018-10-16 17:49:25 -0700275
Spandan Das05063612021-06-25 01:39:04 +0000276 if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok {
277 ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false")
278 }
279
Joe Onoratoe5ed3472024-02-02 14:52:05 -0800280 if os.Getenv("GENERATE_SOONG_DEBUG") == "true" {
281 ret.moduleDebugFile, _ = filepath.Abs(shared.JoinPath(ret.SoongOutDir(), "soong-debug-info.json"))
282 }
283
Dan Willemsen1e704462016-08-21 15:17:17 -0700284 ret.environ.Unset(
285 // We're already using it
286 "USE_SOONG_UI",
287
288 // We should never use GOROOT/GOPATH from the shell environment
289 "GOROOT",
290 "GOPATH",
291
292 // These should only come from Soong, not the environment.
293 "CLANG",
294 "CLANG_CXX",
295 "CCC_CC",
296 "CCC_CXX",
297
298 // Used by the goma compiler wrapper, but should only be set by
299 // gomacc
300 "GOMACC_PATH",
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800301
302 // We handle this above
303 "OUT_DIR_COMMON_BASE",
Dan Willemsen68a09852017-04-18 13:56:57 -0700304
Dan Willemsen2d31a442018-10-20 21:33:41 -0700305 // This is handled above too, and set for individual commands later
306 "DIST_DIR",
307
Dan Willemsen68a09852017-04-18 13:56:57 -0700308 // Variables that have caused problems in the past
Dan Willemsen1c504d92019-11-18 19:13:53 +0000309 "BASH_ENV",
Dan Willemsenebfe33a2018-05-01 10:07:50 -0700310 "CDPATH",
Dan Willemsen68a09852017-04-18 13:56:57 -0700311 "DISPLAY",
312 "GREP_OPTIONS",
Nathan Egge7b067fb2023-02-17 17:54:31 +0000313 "JAVAC",
Dan Willemsenebfe33a2018-05-01 10:07:50 -0700314 "NDK_ROOT",
Dan Willemsen00fcb262018-08-15 15:35:38 -0700315 "POSIXLY_CORRECT",
Dan Willemsenc40e10b2017-07-11 14:30:00 -0700316
317 // Drop make flags
318 "MAKEFLAGS",
319 "MAKELEVEL",
320 "MFLAGS",
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700321
322 // Set in envsetup.sh, reset in makefiles
323 "ANDROID_JAVA_TOOLCHAIN",
Colin Cross7f09c402018-07-11 14:49:31 -0700324
325 // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional
326 "ANDROID_BUILD_TOP",
327 "ANDROID_HOST_OUT",
328 "ANDROID_PRODUCT_OUT",
329 "ANDROID_HOST_OUT_TESTCASES",
330 "ANDROID_TARGET_OUT_TESTCASES",
331 "ANDROID_TOOLCHAIN",
332 "ANDROID_TOOLCHAIN_2ND_ARCH",
333 "ANDROID_DEV_SCRIPTS",
334 "ANDROID_EMULATOR_PREBUILTS",
335 "ANDROID_PRE_BUILD_PATHS",
Joe Onoratoe5ed3472024-02-02 14:52:05 -0800336
337 // We read it here already, don't let others share in the fun
338 "GENERATE_SOONG_DEBUG",
Dan Willemsen1e704462016-08-21 15:17:17 -0700339 )
340
Kousik Kumarb328f6d2020-10-19 01:45:46 -0400341 if ret.UseGoma() || ret.ForceUseGoma() {
342 ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.")
343 ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.")
Kousik Kumarec478642020-09-21 13:39:24 -0400344 }
345
Dan Willemsen1e704462016-08-21 15:17:17 -0700346 // Tell python not to spam the source tree with .pyc files.
347 ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
348
Ramy Medhatca1e44c2020-07-16 12:18:37 -0400349 tmpDir := absPath(ctx, ret.TempDir())
350 ret.environ.Set("TMPDIR", tmpDir)
Dan Willemsen32a669b2018-03-08 19:42:00 -0800351
Dan Willemsen70c1ff82019-08-21 14:56:13 -0700352 // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes
353 symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(),
354 "llvm-binutils-stable/llvm-symbolizer")
355 ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath))
356
Dan Willemsenc2af0be2017-01-20 14:10:01 -0800357 // Precondition: the current directory is the top of the source tree
Patrice Arruda13848222019-04-22 17:12:02 -0700358 checkTopDir(ctx)
Dan Willemsenc2af0be2017-01-20 14:10:01 -0800359
Yu Liu6e13b402021-07-27 14:29:06 -0700360 srcDir := absPath(ctx, ".")
361 if strings.ContainsRune(srcDir, ' ') {
Colin Cross1f6faeb2019-09-23 15:52:40 -0700362 ctx.Println("You are building in a directory whose absolute path contains a space character:")
363 ctx.Println()
364 ctx.Printf("%q\n", srcDir)
365 ctx.Println()
366 ctx.Fatalln("Directory names containing spaces are not supported")
Dan Willemsendb8457c2017-05-12 16:38:17 -0700367 }
368
Yu Liu6e13b402021-07-27 14:29:06 -0700369 ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ)
370
Dan Willemsendb8457c2017-05-12 16:38:17 -0700371 if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') {
Colin Cross1f6faeb2019-09-23 15:52:40 -0700372 ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:")
373 ctx.Println()
374 ctx.Printf("%q\n", outDir)
375 ctx.Println()
376 ctx.Fatalln("Directory names containing spaces are not supported")
Dan Willemsendb8457c2017-05-12 16:38:17 -0700377 }
378
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +0000379 if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') {
Colin Cross1f6faeb2019-09-23 15:52:40 -0700380 ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:")
381 ctx.Println()
382 ctx.Printf("%q\n", distDir)
383 ctx.Println()
384 ctx.Fatalln("Directory names containing spaces are not supported")
Dan Willemsendb8457c2017-05-12 16:38:17 -0700385 }
386
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700387 // Configure Java-related variables, including adding it to $PATH
Tobias Thierere59aeff2017-12-20 22:40:39 +0000388 java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag())
Colin Cross59c1e6a2022-03-04 13:37:19 -0800389 java17Home := filepath.Join("prebuilts/jdk/jdk17", ret.HostPrebuiltTag())
Sorin Basca0760c892023-11-29 19:13:55 +0000390 java21Home := filepath.Join("prebuilts/jdk/jdk21", ret.HostPrebuiltTag())
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700391 javaHome := func() string {
392 if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok {
393 return override
394 }
Sorin Basca0760c892023-11-29 19:13:55 +0000395 if ret.environ.IsEnvTrue("EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN") {
396 return java21Home
397 }
Pete Gillina7a3d642019-11-07 18:58:42 +0000398 if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" {
399 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.")
Pete Gillin1f52e932019-10-09 17:10:08 +0100400 }
Sorin Basca7e094b32022-10-05 08:20:12 +0000401 if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" {
402 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 17 toolchain is now the global default.")
403 }
404 return java17Home
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700405 }()
406 absJavaHome := absPath(ctx, javaHome)
407
Dan Willemsened869522018-01-08 14:58:46 -0800408 ret.configureLocale(ctx)
409
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700410 newPath := []string{filepath.Join(absJavaHome, "bin")}
411 if path, ok := ret.environ.Get("PATH"); ok && path != "" {
412 newPath = append(newPath, path)
413 }
Pete Gillin1f52e932019-10-09 17:10:08 +0100414
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700415 ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME")
416 ret.environ.Set("JAVA_HOME", absJavaHome)
417 ret.environ.Set("ANDROID_JAVA_HOME", javaHome)
Tobias Thierere59aeff2017-12-20 22:40:39 +0000418 ret.environ.Set("ANDROID_JAVA8_HOME", java8Home)
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700419 ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator)))
420
Colin Crossfe5ed4d2023-07-28 09:27:23 -0700421 // b/286885495, https://bugzilla.redhat.com/show_bug.cgi?id=2227130: some versions of Fedora include patches
422 // to unzip to enable zipbomb detection that incorrectly handle zip64 and data descriptors and fail on large
423 // zip files produced by soong_zip. Disable zipbomb detection.
424 ret.environ.Set("UNZIP_DISABLE_ZIPBOMB_DETECTION", "TRUE")
425
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800426 outDir := ret.OutDir()
427 buildDateTimeFile := filepath.Join(outDir, "build_date.txt")
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800428 if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" {
Colin Cross28f527c2019-11-26 16:19:04 -0800429 ret.buildDateTime = buildDateTime
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800430 } else {
Colin Cross28f527c2019-11-26 16:19:04 -0800431 ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10)
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800432 }
Colin Cross28f527c2019-11-26 16:19:04 -0800433
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800434 ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile)
435
Cole Faust583dfb42023-09-28 13:56:30 -0700436 if _, ok := ret.environ.Get("BUILD_USERNAME"); !ok {
437 username := "unknown"
438 if u, err := user.Current(); err == nil {
439 username = u.Username
440 } else {
441 ctx.Println("Failed to get current user:", err)
442 }
443 ret.environ.Set("BUILD_USERNAME", username)
444 }
445
Ramy Medhatca1e44c2020-07-16 12:18:37 -0400446 if ret.UseRBE() {
Ramy Medhat0fc67eb2020-08-12 01:26:23 -0400447 for k, v := range getRBEVars(ctx, Config{ret}) {
Ramy Medhatca1e44c2020-07-16 12:18:37 -0400448 ret.environ.Set(k, v)
449 }
450 }
451
Patrice Arruda96850362020-08-11 20:41:11 +0000452 c := Config{ret}
453 storeConfigMetrics(ctx, c)
454 return c
Dan Willemsen9b587492017-07-10 22:13:00 -0700455}
456
Patrice Arruda13848222019-04-22 17:12:02 -0700457// NewBuildActionConfig returns a build configuration based on the build action. The arguments are
458// processed based on the build action and extracts any arguments that belongs to the build action.
Dan Willemsence41e942019-07-29 23:39:30 -0700459func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config {
460 return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...)
Patrice Arruda13848222019-04-22 17:12:02 -0700461}
462
LaMont Jones9a912862023-11-06 22:11:08 +0000463// Prepare for getting make variables. For them to be accurate, we need to have
464// obtained PRODUCT_RELEASE_CONFIG_MAPS.
465//
466// Returns:
467//
468// Whether config should be called again.
469//
470// TODO: when converting product config to a declarative language, make sure
471// that PRODUCT_RELEASE_CONFIG_MAPS is properly handled as a separate step in
472// that process.
473func SetProductReleaseConfigMaps(ctx Context, config Config) bool {
474 ctx.BeginTrace(metrics.RunKati, "SetProductReleaseConfigMaps")
475 defer ctx.EndTrace()
476
477 if config.SkipConfig() {
478 // This duplicates the logic from Build to skip product config
479 // if the user has explicitly said to.
480 return false
481 }
482
483 releaseConfigVars := []string{
484 "PRODUCT_RELEASE_CONFIG_MAPS",
485 }
486
487 origValue, _ := config.environ.Get("PRODUCT_RELEASE_CONFIG_MAPS")
488 // Get the PRODUCT_RELEASE_CONFIG_MAPS for this product, to avoid polluting the environment
489 // when we run product config to get the rest of the make vars.
490 releaseMapVars, err := dumpMakeVars(ctx, config, nil, releaseConfigVars, false, "")
491 if err != nil {
492 ctx.Fatalln("Error getting PRODUCT_RELEASE_CONFIG_MAPS:", err)
493 }
494 productReleaseConfigMaps := releaseMapVars["PRODUCT_RELEASE_CONFIG_MAPS"]
495 os.Setenv("PRODUCT_RELEASE_CONFIG_MAPS", productReleaseConfigMaps)
496 return origValue != productReleaseConfigMaps
497}
498
Patrice Arruda96850362020-08-11 20:41:11 +0000499// storeConfigMetrics selects a set of configuration information and store in
500// the metrics system for further analysis.
501func storeConfigMetrics(ctx Context, config Config) {
502 if ctx.Metrics == nil {
503 return
504 }
505
Liz Kammerca9cb2e2021-07-14 15:29:57 -0400506 ctx.Metrics.BuildConfig(buildConfig(config))
Patrice Arruda3edfd482020-10-13 23:58:41 +0000507
508 s := &smpb.SystemResourceInfo{
509 TotalPhysicalMemory: proto.Uint64(config.TotalRAM()),
510 AvailableCpus: proto.Int32(int32(runtime.NumCPU())),
511 }
512 ctx.Metrics.SystemResourceInfo(s)
Patrice Arruda96850362020-08-11 20:41:11 +0000513}
514
Jeongik Cha8d63d562023-03-17 03:52:13 +0900515func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource {
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900516 switch s {
517 case NINJA_LOG:
Jeongik Cha8d63d562023-03-17 03:52:13 +0900518 return smpb.BuildConfig_NINJA_LOG.Enum()
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900519 case EVENLY_DISTRIBUTED:
Jeongik Cha8d63d562023-03-17 03:52:13 +0900520 return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum()
Jeongik Cha518f3ea2023-03-19 00:12:39 +0900521 case EXTERNAL_FILE:
522 return smpb.BuildConfig_EXTERNAL_FILE.Enum()
Jeongik Chae114e602023-03-19 00:12:39 +0900523 case HINT_FROM_SOONG:
524 return smpb.BuildConfig_HINT_FROM_SOONG.Enum()
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900525 default:
Jeongik Cha8d63d562023-03-17 03:52:13 +0900526 return smpb.BuildConfig_NOT_USED.Enum()
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900527 }
528}
529
Liz Kammerca9cb2e2021-07-14 15:29:57 -0400530func buildConfig(config Config) *smpb.BuildConfig {
Yu Liue737a992021-10-04 13:21:41 -0700531 c := &smpb.BuildConfig{
Colin Cross8d411ff2023-12-07 10:31:24 -0800532 ForceUseGoma: proto.Bool(config.ForceUseGoma()),
533 UseGoma: proto.Bool(config.UseGoma()),
534 UseRbe: proto.Bool(config.UseRBE()),
535 NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()),
Liz Kammerca9cb2e2021-07-14 15:29:57 -0400536 }
Yu Liue737a992021-10-04 13:21:41 -0700537 c.Targets = append(c.Targets, config.arguments...)
538
539 return c
Liz Kammerca9cb2e2021-07-14 15:29:57 -0400540}
541
Patrice Arruda13848222019-04-22 17:12:02 -0700542// getConfigArgs processes the command arguments based on the build action and creates a set of new
543// arguments to be accepted by Config.
Dan Willemsence41e942019-07-29 23:39:30 -0700544func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string {
Patrice Arruda13848222019-04-22 17:12:02 -0700545 // The next block of code verifies that the current directory is the root directory of the source
546 // tree. It then finds the relative path of dir based on the root directory of the source tree
547 // and verify that dir is inside of the source tree.
548 checkTopDir(ctx)
549 topDir, err := os.Getwd()
550 if err != nil {
551 ctx.Fatalf("Error retrieving top directory: %v", err)
552 }
Patrice Arrudababa9a92019-07-03 10:47:34 -0700553 dir, err = filepath.EvalSymlinks(dir)
554 if err != nil {
555 ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err)
556 }
Patrice Arruda13848222019-04-22 17:12:02 -0700557 dir, err = filepath.Abs(dir)
558 if err != nil {
559 ctx.Fatalf("Unable to find absolute path %s: %v", dir, err)
560 }
561 relDir, err := filepath.Rel(topDir, dir)
562 if err != nil {
563 ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err)
564 }
565 // If there are ".." in the path, it's not in the source tree.
566 if strings.Contains(relDir, "..") {
567 ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir)
568 }
569
570 configArgs := args[:]
571
572 // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to
573 // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules.
574 targetNamePrefix := "MODULES-IN-"
575 if inList("GET-INSTALL-PATH", configArgs) {
576 targetNamePrefix = "GET-INSTALL-PATH-IN-"
577 configArgs = removeFromList("GET-INSTALL-PATH", configArgs)
578 }
579
Patrice Arruda13848222019-04-22 17:12:02 -0700580 var targets []string
581
582 switch action {
Patrice Arruda39282062019-06-20 16:35:12 -0700583 case BUILD_MODULES:
584 // No additional processing is required when building a list of specific modules or all modules.
Patrice Arruda13848222019-04-22 17:12:02 -0700585 case BUILD_MODULES_IN_A_DIRECTORY:
586 // If dir is the root source tree, all the modules are built of the source tree are built so
587 // no need to find the build file.
588 if topDir == dir {
589 break
590 }
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700591
Patrice Arruda13848222019-04-22 17:12:02 -0700592 buildFile := findBuildFile(ctx, relDir)
593 if buildFile == "" {
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700594 ctx.Fatalf("Build file not found for %s directory", relDir)
Patrice Arruda13848222019-04-22 17:12:02 -0700595 }
Patrice Arruda13848222019-04-22 17:12:02 -0700596 targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
597 case BUILD_MODULES_IN_DIRECTORIES:
598 newConfigArgs, dirs := splitArgs(configArgs)
599 configArgs = newConfigArgs
Dan Willemsence41e942019-07-29 23:39:30 -0700600 targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix)
Patrice Arruda13848222019-04-22 17:12:02 -0700601 }
602
603 // Tidy only override all other specified targets.
604 tidyOnly := os.Getenv("WITH_TIDY_ONLY")
605 if tidyOnly == "true" || tidyOnly == "1" {
606 configArgs = append(configArgs, "tidy_only")
607 } else {
608 configArgs = append(configArgs, targets...)
609 }
610
611 return configArgs
612}
613
614// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name.
615func convertToTarget(dir string, targetNamePrefix string) string {
616 return targetNamePrefix + strings.ReplaceAll(dir, "/", "-")
617}
618
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700619// hasBuildFile returns true if dir contains an Android build file.
620func hasBuildFile(ctx Context, dir string) bool {
621 for _, buildFile := range buildFiles {
622 _, err := os.Stat(filepath.Join(dir, buildFile))
623 if err == nil {
624 return true
625 }
626 if !os.IsNotExist(err) {
627 ctx.Fatalf("Error retrieving the build file stats: %v", err)
628 }
629 }
630 return false
631}
632
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700633// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file
634// in the current and any sub directory of dir. If a build file is not found, traverse the path
635// up by one directory and repeat again until either a build file is found or reached to the root
636// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank
637// string is returned.
Patrice Arruda13848222019-04-22 17:12:02 -0700638func findBuildFile(ctx Context, dir string) string {
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700639 // If the string is empty or ".", assume it is top directory of the source tree.
640 if dir == "" || dir == "." {
Patrice Arruda13848222019-04-22 17:12:02 -0700641 return ""
642 }
643
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700644 found := false
645 for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) {
646 err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error {
647 if err != nil {
648 return err
649 }
650 if found {
651 return filepath.SkipDir
652 }
653 if info.IsDir() {
654 return nil
655 }
656 for _, buildFile := range buildFiles {
657 if info.Name() == buildFile {
658 found = true
659 return filepath.SkipDir
660 }
661 }
662 return nil
663 })
664 if err != nil {
665 ctx.Fatalf("Error finding Android build file: %v", err)
666 }
667
668 if found {
669 return filepath.Join(buildDir, "Android.mk")
Patrice Arruda13848222019-04-22 17:12:02 -0700670 }
671 }
672
673 return ""
674}
675
676// splitArgs iterates over the arguments list and splits into two lists: arguments and directories.
677func splitArgs(args []string) (newArgs []string, dirs []string) {
678 specialArgs := map[string]bool{
679 "showcommands": true,
680 "snod": true,
681 "dist": true,
682 "checkbuild": true,
683 }
684
685 newArgs = []string{}
686 dirs = []string{}
687
688 for _, arg := range args {
689 // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory.
690 if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 {
691 newArgs = append(newArgs, arg)
692 continue
693 }
694
695 if _, ok := specialArgs[arg]; ok {
696 newArgs = append(newArgs, arg)
697 continue
698 }
699
700 dirs = append(dirs, arg)
701 }
702
703 return newArgs, dirs
704}
705
706// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a
707// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the
708// source root tree where the build action command was invoked. Each directory is validated if the
709// build file can be found and follows the format "dir1:target1,target2,...". Target is optional.
Dan Willemsence41e942019-07-29 23:39:30 -0700710func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) {
Patrice Arruda13848222019-04-22 17:12:02 -0700711 for _, dir := range dirs {
712 // The directory may have specified specific modules to build. ":" is the separator to separate
713 // the directory and the list of modules.
714 s := strings.Split(dir, ":")
715 l := len(s)
716 if l > 2 { // more than one ":" was specified.
717 ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir)
718 }
719
720 dir = filepath.Join(relDir, s[0])
721 if _, err := os.Stat(dir); err != nil {
722 ctx.Fatalf("couldn't find directory %s", dir)
723 }
724
725 // Verify that if there are any targets specified after ":". Each target is separated by ",".
726 var newTargets []string
727 if l == 2 && s[1] != "" {
728 newTargets = strings.Split(s[1], ",")
729 if inList("", newTargets) {
730 ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir)
731 }
732 }
733
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700734 // If there are specified targets to build in dir, an android build file must exist for the one
735 // shot build. For the non-targets case, find the appropriate build file and build all the
736 // modules in dir (or the closest one in the dir path).
Patrice Arruda13848222019-04-22 17:12:02 -0700737 if len(newTargets) > 0 {
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700738 if !hasBuildFile(ctx, dir) {
Patrice Arruda13848222019-04-22 17:12:02 -0700739 ctx.Fatalf("Couldn't locate a build file from %s directory", dir)
740 }
741 } else {
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700742 buildFile := findBuildFile(ctx, dir)
743 if buildFile == "" {
744 ctx.Fatalf("Build file not found for %s directory", dir)
745 }
746 newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
Patrice Arruda13848222019-04-22 17:12:02 -0700747 }
748
Patrice Arruda13848222019-04-22 17:12:02 -0700749 targets = append(targets, newTargets...)
750 }
751
Dan Willemsence41e942019-07-29 23:39:30 -0700752 return targets
Patrice Arruda13848222019-04-22 17:12:02 -0700753}
754
Dan Willemsen9b587492017-07-10 22:13:00 -0700755func (c *configImpl) parseArgs(ctx Context, args []string) {
756 for i := 0; i < len(args); i++ {
757 arg := strings.TrimSpace(args[i])
Anton Hansson5a7861a2021-06-04 10:09:01 +0100758 if arg == "showcommands" {
Dan Willemsen9b587492017-07-10 22:13:00 -0700759 c.verbose = true
Lukacs T. Berkicef87b62021-08-10 15:01:13 +0200760 } else if arg == "--empty-ninja-file" {
761 c.emptyNinjaFile = true
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100762 } else if arg == "--skip-ninja" {
763 c.skipNinja = true
Dan Willemsene0879fc2017-08-04 15:06:27 -0700764 } else if arg == "--skip-make" {
Colin Cross30e444b2021-06-18 11:26:19 -0700765 // TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati,
766 // but it does run a Kati ninja file if the .kati_enabled marker file was created
767 // by a previous build.
Anton Hansson5e5c48b2020-11-27 12:35:20 +0000768 c.skipConfig = true
769 c.skipKati = true
Anton Hansson0b55bdb2021-06-04 10:08:08 +0100770 } else if arg == "--soong-only" {
771 c.skipKati = true
772 c.skipKatiNinja = true
Lukacs T. Berkicef87b62021-08-10 15:01:13 +0200773 } else if arg == "--config-only" {
774 c.skipKati = true
775 c.skipKatiNinja = true
776 c.skipSoong = true
Colin Cross30e444b2021-06-18 11:26:19 -0700777 } else if arg == "--skip-config" {
778 c.skipConfig = true
Colin Cross00a8a3f2020-10-29 14:08:31 -0700779 } else if arg == "--skip-soong-tests" {
780 c.skipSoongTests = true
Colin Cross106d6ef2023-10-24 10:34:56 -0700781 } else if arg == "--no-skip-soong-tests" {
782 c.skipSoongTests = false
MarkDacekd0e7cd32022-12-02 22:22:40 +0000783 } else if arg == "--skip-metrics-upload" {
784 c.skipMetricsUpload = true
Chris Parsons53f68ae2022-03-03 12:01:40 -0500785 } else if arg == "--mk-metrics" {
786 c.reportMkMetrics = true
Spandan Das394aa322022-11-03 17:02:10 +0000787 } else if arg == "--search-api-dir" {
788 c.searchApiDir = true
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900789 } else if strings.HasPrefix(arg, "--ninja_weight_source=") {
790 source := strings.TrimPrefix(arg, "--ninja_weight_source=")
791 if source == "ninja_log" {
792 c.ninjaWeightListSource = NINJA_LOG
793 } else if source == "evenly_distributed" {
794 c.ninjaWeightListSource = EVENLY_DISTRIBUTED
795 } else if source == "not_used" {
796 c.ninjaWeightListSource = NOT_USED
Jeongik Chae114e602023-03-19 00:12:39 +0900797 } else if source == "soong" {
798 c.ninjaWeightListSource = HINT_FROM_SOONG
Jeongik Cha518f3ea2023-03-19 00:12:39 +0900799 } else if strings.HasPrefix(source, "file,") {
800 c.ninjaWeightListSource = EXTERNAL_FILE
801 filePath := strings.TrimPrefix(source, "file,")
802 err := validateNinjaWeightList(filePath)
803 if err != nil {
804 ctx.Fatalf("Malformed weight list from %s: %s", filePath, err)
805 }
806 _, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list"))
807 if err != nil {
808 ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err)
809 }
Jeongik Cha0cf44d52023-03-15 00:10:45 +0900810 } else {
811 ctx.Fatalf("unknown option for ninja_weight_source: %s", source)
812 }
Jihoon Kang2a929ad2023-06-08 19:02:07 +0000813 } else if arg == "--build-from-source-stub" {
814 c.buildFromSourceStub = true
MarkDacekb96561e2022-12-02 04:34:43 +0000815 } else if strings.HasPrefix(arg, "--build-command=") {
816 buildCmd := strings.TrimPrefix(arg, "--build-command=")
817 // remove quotations
818 buildCmd = strings.TrimPrefix(buildCmd, "\"")
819 buildCmd = strings.TrimSuffix(buildCmd, "\"")
820 ctx.Metrics.SetBuildCommand([]string{buildCmd})
MarkDacek6614d9c2022-12-07 21:57:38 +0000821 } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
822 buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
823 val, err := strconv.ParseInt(buildTimeStr, 10, 64)
824 if err == nil {
825 c.buildStartedTime = val
826 } else {
827 ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
828 }
MarkDacekf47e1422023-04-19 16:47:36 +0000829 } else if arg == "--ensure-allowlist-integrity" {
830 c.ensureAllowlistIntegrity = true
Dan Willemsen6ac63ef2017-10-17 20:35:34 -0700831 } else if len(arg) > 0 && arg[0] == '-' {
Dan Willemsen9b587492017-07-10 22:13:00 -0700832 parseArgNum := func(def int) int {
833 if len(arg) > 2 {
834 p, err := strconv.ParseUint(arg[2:], 10, 31)
835 if err != nil {
836 ctx.Fatalf("Failed to parse %q: %v", arg, err)
837 }
838 return int(p)
839 } else if i+1 < len(args) {
840 p, err := strconv.ParseUint(args[i+1], 10, 31)
841 if err == nil {
842 i++
843 return int(p)
844 }
845 }
846 return def
847 }
848
Dan Willemsen6ac63ef2017-10-17 20:35:34 -0700849 if len(arg) > 1 && arg[1] == 'j' {
Dan Willemsen9b587492017-07-10 22:13:00 -0700850 c.parallel = parseArgNum(c.parallel)
Dan Willemsen6ac63ef2017-10-17 20:35:34 -0700851 } else if len(arg) > 1 && arg[1] == 'k' {
Dan Willemsen9b587492017-07-10 22:13:00 -0700852 c.keepGoing = parseArgNum(0)
Dan Willemsen1e704462016-08-21 15:17:17 -0700853 } else {
854 ctx.Fatalln("Unknown option:", arg)
855 }
Dan Willemsen091525e2017-07-11 14:17:50 -0700856 } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 {
Dan Willemsen6dfe30a2018-09-10 12:41:10 -0700857 if k == "OUT_DIR" {
858 ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.")
859 }
Dan Willemsen091525e2017-07-11 14:17:50 -0700860 c.environ.Set(k, v)
Dan Willemsen2d31a442018-10-20 21:33:41 -0700861 } else if arg == "dist" {
862 c.dist = true
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200863 } else if arg == "json-module-graph" {
864 c.jsonModuleGraph = true
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200865 } else if arg == "queryview" {
866 c.queryview = true
Lukacs T. Berkic6012f32021-09-06 18:31:46 +0200867 } else if arg == "soong_docs" {
868 c.soongDocs = true
Dan Willemsen1e704462016-08-21 15:17:17 -0700869 } else {
Dan Willemsen2d31a442018-10-20 21:33:41 -0700870 if arg == "checkbuild" {
Colin Cross37193492017-11-16 17:55:00 -0800871 c.checkbuild = true
Dan Willemsene0879fc2017-08-04 15:06:27 -0700872 }
Dan Willemsen9b587492017-07-10 22:13:00 -0700873 c.arguments = append(c.arguments, arg)
Dan Willemsen1e704462016-08-21 15:17:17 -0700874 }
875 }
Dan Willemsen1e704462016-08-21 15:17:17 -0700876}
877
Jeongik Cha518f3ea2023-03-19 00:12:39 +0900878func validateNinjaWeightList(weightListFilePath string) (err error) {
879 data, err := os.ReadFile(weightListFilePath)
880 if err != nil {
881 return
882 }
883 lines := strings.Split(strings.TrimSpace(string(data)), "\n")
884 for _, line := range lines {
885 fields := strings.Split(line, ",")
886 if len(fields) != 2 {
887 return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line)
888 }
889 _, err = strconv.Atoi(fields[1])
890 if err != nil {
891 return
892 }
893 }
894 return
895}
896
Dan Willemsened869522018-01-08 14:58:46 -0800897func (c *configImpl) configureLocale(ctx Context) {
898 cmd := Command(ctx, Config{c}, "locale", "locale", "-a")
899 output, err := cmd.Output()
900
901 var locales []string
902 if err == nil {
903 locales = strings.Split(string(output), "\n")
904 } else {
905 // If we're unable to list the locales, let's assume en_US.UTF-8
906 locales = []string{"en_US.UTF-8"}
907 ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales)
908 }
909
910 // gettext uses LANGUAGE, which is passed directly through
911
912 // For LANG and LC_*, only preserve the evaluated version of
913 // LC_MESSAGES
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800914 userLang := ""
Dan Willemsened869522018-01-08 14:58:46 -0800915 if lc_all, ok := c.environ.Get("LC_ALL"); ok {
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800916 userLang = lc_all
Dan Willemsened869522018-01-08 14:58:46 -0800917 } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok {
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800918 userLang = lc_messages
Dan Willemsened869522018-01-08 14:58:46 -0800919 } else if lang, ok := c.environ.Get("LANG"); ok {
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800920 userLang = lang
Dan Willemsened869522018-01-08 14:58:46 -0800921 }
922
923 c.environ.UnsetWithPrefix("LC_")
924
Jaewoong Jung18aefc12020-12-21 09:11:10 -0800925 if userLang != "" {
926 c.environ.Set("LC_MESSAGES", userLang)
Dan Willemsened869522018-01-08 14:58:46 -0800927 }
928
929 // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed
930 // for others)
931 if inList("C.UTF-8", locales) {
932 c.environ.Set("LANG", "C.UTF-8")
Aaron Klingd236e0e2018-08-07 19:21:36 -0500933 } else if inList("C.utf8", locales) {
934 // These normalize to the same thing
935 c.environ.Set("LANG", "C.UTF-8")
Dan Willemsened869522018-01-08 14:58:46 -0800936 } else if inList("en_US.UTF-8", locales) {
937 c.environ.Set("LANG", "en_US.UTF-8")
938 } else if inList("en_US.utf8", locales) {
939 // These normalize to the same thing
940 c.environ.Set("LANG", "en_US.UTF-8")
941 } else {
942 ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8")
943 }
944}
945
Dan Willemsen1e704462016-08-21 15:17:17 -0700946func (c *configImpl) Environment() *Environment {
947 return c.environ
948}
949
950func (c *configImpl) Arguments() []string {
951 return c.arguments
952}
953
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200954func (c *configImpl) SoongBuildInvocationNeeded() bool {
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200955 if len(c.Arguments()) > 0 {
956 // Explicit targets requested that are not special targets like b2pbuild
957 // or the JSON module graph
958 return true
959 }
960
Colin Cross8d411ff2023-12-07 10:31:24 -0800961 if !c.JsonModuleGraph() && !c.Queryview() && !c.SoongDocs() {
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200962 // Command line was empty, the default Ninja target is built
963 return true
964 }
965
Colin Cross8d411ff2023-12-07 10:31:24 -0800966 if c.Dist() {
Liz Kammer88677422021-12-15 15:03:19 -0500967 return true
968 }
969
Lukacs T. Berkia1b93722021-09-02 17:23:06 +0200970 // build.ninja doesn't need to be generated
971 return false
972}
973
Dan Willemsen1e704462016-08-21 15:17:17 -0700974func (c *configImpl) OutDir() string {
975 if outDir, ok := c.environ.Get("OUT_DIR"); ok {
Patrice Arruda19bd53e2019-07-08 17:26:47 -0700976 return outDir
Dan Willemsen1e704462016-08-21 15:17:17 -0700977 }
978 return "out"
979}
980
Dan Willemsen8a073a82017-02-04 17:30:44 -0800981func (c *configImpl) DistDir() string {
Chris Parsons19ab9a42022-08-30 13:15:04 -0400982 return c.distDir
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +0000983}
984
985func (c *configImpl) RealDistDir() string {
Dan Willemsen2d31a442018-10-20 21:33:41 -0700986 return c.distDir
Dan Willemsen8a073a82017-02-04 17:30:44 -0800987}
988
Dan Willemsen1e704462016-08-21 15:17:17 -0700989func (c *configImpl) NinjaArgs() []string {
Anton Hansson5e5c48b2020-11-27 12:35:20 +0000990 if c.skipKati {
Dan Willemsene0879fc2017-08-04 15:06:27 -0700991 return c.arguments
992 }
Dan Willemsen1e704462016-08-21 15:17:17 -0700993 return c.ninjaArgs
994}
995
996func (c *configImpl) SoongOutDir() string {
997 return filepath.Join(c.OutDir(), "soong")
998}
999
Spandan Das394aa322022-11-03 17:02:10 +00001000func (c *configImpl) ApiSurfacesOutDir() string {
1001 return filepath.Join(c.OutDir(), "api_surfaces")
1002}
1003
Lukacs T. Berkia806e412021-09-01 08:57:48 +02001004func (c *configImpl) PrebuiltOS() string {
1005 switch runtime.GOOS {
1006 case "linux":
1007 return "linux-x86"
1008 case "darwin":
1009 return "darwin-x86"
1010 default:
1011 panic("Unknown GOOS")
1012 }
1013}
Lukacs T. Berki90b43342021-11-02 14:42:04 +01001014
Lukacs T. Berkia806e412021-09-01 08:57:48 +02001015func (c *configImpl) HostToolDir() string {
Colin Crossacfcc1f2021-10-25 15:40:32 -07001016 if c.SkipKatiNinja() {
1017 return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin")
1018 } else {
1019 return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin")
1020 }
Lukacs T. Berkia806e412021-09-01 08:57:48 +02001021}
1022
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +02001023func (c *configImpl) NamedGlobFile(name string) string {
Lukacs T. Berki90b43342021-11-02 14:42:04 +01001024 return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja")
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +02001025}
1026
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +02001027func (c *configImpl) UsedEnvFile(tag string) string {
Kiyoung Kimeaa55a82023-06-05 16:56:49 +09001028 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
1029 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+"."+tag)
1030 }
Lukacs T. Berkie1df43f2021-09-08 15:31:14 +02001031 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag)
1032}
1033
Lukacs T. Berkic6012f32021-09-06 18:31:46 +02001034func (c *configImpl) SoongDocsHtml() string {
1035 return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html")
1036}
1037
Lukacs T. Berki3a821692021-09-06 17:08:02 +02001038func (c *configImpl) QueryviewMarkerFile() string {
1039 return shared.JoinPath(c.SoongOutDir(), "queryview.marker")
1040}
1041
Lukacs T. Berkie571dc32021-08-25 14:14:13 +02001042func (c *configImpl) ModuleGraphFile() string {
1043 return shared.JoinPath(c.SoongOutDir(), "module-graph.json")
1044}
1045
kgui67007242022-01-25 13:50:25 +08001046func (c *configImpl) ModuleActionsFile() string {
1047 return shared.JoinPath(c.SoongOutDir(), "module-actions.json")
1048}
1049
Jeff Gastonefc1b412017-03-29 17:29:06 -07001050func (c *configImpl) TempDir() string {
1051 return shared.TempDirForOutDir(c.SoongOutDir())
1052}
1053
Jeff Gastonb64fc1c2017-08-04 12:30:12 -07001054func (c *configImpl) FileListDir() string {
1055 return filepath.Join(c.OutDir(), ".module_paths")
1056}
1057
Dan Willemsen1e704462016-08-21 15:17:17 -07001058func (c *configImpl) KatiSuffix() string {
1059 if c.katiSuffix != "" {
1060 return c.katiSuffix
1061 }
1062 panic("SetKatiSuffix has not been called")
1063}
1064
Colin Cross37193492017-11-16 17:55:00 -08001065// Checkbuild returns true if "checkbuild" was one of the build goals, which means that the
1066// user is interested in additional checks at the expense of build time.
1067func (c *configImpl) Checkbuild() bool {
1068 return c.checkbuild
1069}
1070
Dan Willemsen8a073a82017-02-04 17:30:44 -08001071func (c *configImpl) Dist() bool {
1072 return c.dist
1073}
1074
Lukacs T. Berkia1b93722021-09-02 17:23:06 +02001075func (c *configImpl) JsonModuleGraph() bool {
1076 return c.jsonModuleGraph
1077}
1078
Lukacs T. Berki3a821692021-09-06 17:08:02 +02001079func (c *configImpl) Queryview() bool {
1080 return c.queryview
1081}
1082
Lukacs T. Berkic6012f32021-09-06 18:31:46 +02001083func (c *configImpl) SoongDocs() bool {
1084 return c.soongDocs
1085}
1086
Dan Willemsen1e704462016-08-21 15:17:17 -07001087func (c *configImpl) IsVerbose() bool {
1088 return c.verbose
1089}
1090
Jeongik Cha0cf44d52023-03-15 00:10:45 +09001091func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource {
1092 return c.ninjaWeightListSource
1093}
1094
Anton Hansson5e5c48b2020-11-27 12:35:20 +00001095func (c *configImpl) SkipKati() bool {
1096 return c.skipKati
1097}
1098
Anton Hansson0b55bdb2021-06-04 10:08:08 +01001099func (c *configImpl) SkipKatiNinja() bool {
1100 return c.skipKatiNinja
1101}
1102
Lukacs T. Berkicef87b62021-08-10 15:01:13 +02001103func (c *configImpl) SkipSoong() bool {
1104 return c.skipSoong
1105}
1106
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +01001107func (c *configImpl) SkipNinja() bool {
1108 return c.skipNinja
1109}
1110
Anton Hansson5a7861a2021-06-04 10:09:01 +01001111func (c *configImpl) SetSkipNinja(v bool) {
1112 c.skipNinja = v
1113}
1114
Anton Hansson5e5c48b2020-11-27 12:35:20 +00001115func (c *configImpl) SkipConfig() bool {
1116 return c.skipConfig
Dan Willemsene0879fc2017-08-04 15:06:27 -07001117}
1118
Jihoon Kang1bff0342023-01-17 20:40:22 +00001119func (c *configImpl) BuildFromTextStub() bool {
Jihoon Kang2a929ad2023-06-08 19:02:07 +00001120 return !c.buildFromSourceStub
Jihoon Kang1bff0342023-01-17 20:40:22 +00001121}
1122
Dan Willemsen1e704462016-08-21 15:17:17 -07001123func (c *configImpl) TargetProduct() string {
1124 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
1125 return v
1126 }
1127 panic("TARGET_PRODUCT is not defined")
1128}
1129
Kiyoung Kima37d9ba2023-04-19 13:13:45 +09001130func (c *configImpl) TargetProductOrErr() (string, error) {
1131 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
1132 return v, nil
1133 }
1134 return "", fmt.Errorf("TARGET_PRODUCT is not defined")
1135}
1136
Dan Willemsen02781d52017-05-12 19:28:13 -07001137func (c *configImpl) TargetDevice() string {
1138 return c.targetDevice
1139}
1140
1141func (c *configImpl) SetTargetDevice(device string) {
1142 c.targetDevice = device
1143}
1144
1145func (c *configImpl) TargetBuildVariant() string {
1146 if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok {
1147 return v
1148 }
1149 panic("TARGET_BUILD_VARIANT is not defined")
1150}
1151
Dan Willemsen1e704462016-08-21 15:17:17 -07001152func (c *configImpl) KatiArgs() []string {
1153 return c.katiArgs
1154}
1155
1156func (c *configImpl) Parallel() int {
1157 return c.parallel
1158}
1159
Sam Delmerico98a73292023-02-21 11:50:29 -05001160func (c *configImpl) GetSourceRootDirs() []string {
1161 return c.sourceRootDirs
1162}
1163
1164func (c *configImpl) SetSourceRootDirs(i []string) {
1165 c.sourceRootDirs = i
1166}
1167
Spandan Dasc5763832022-11-08 18:42:16 +00001168func (c *configImpl) GetIncludeTags() []string {
1169 return c.includeTags
1170}
1171
1172func (c *configImpl) SetIncludeTags(i []string) {
1173 c.includeTags = i
1174}
1175
MarkDacek6614d9c2022-12-07 21:57:38 +00001176func (c *configImpl) GetLogsPrefix() string {
1177 return c.logsPrefix
1178}
1179
1180func (c *configImpl) SetLogsPrefix(prefix string) {
1181 c.logsPrefix = prefix
1182}
1183
Colin Cross8b8bec32019-11-15 13:18:43 -08001184func (c *configImpl) HighmemParallel() int {
1185 if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
1186 return i
1187 }
1188
1189 const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024
1190 parallel := c.Parallel()
1191 if c.UseRemoteBuild() {
1192 // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism
1193 // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs
1194 // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention.
1195 // Return 1/16th of the size of the local pool, rounding up.
1196 return (parallel + 15) / 16
1197 } else if c.totalRAM == 0 {
1198 // Couldn't detect the total RAM, don't restrict highmem processes.
1199 return parallel
Dan Willemsen570a2922020-05-26 23:02:29 -07001200 } else if c.totalRAM <= 16*1024*1024*1024 {
1201 // Less than 16GB of ram, restrict to 1 highmem processes
1202 return 1
Colin Cross8b8bec32019-11-15 13:18:43 -08001203 } else if c.totalRAM <= 32*1024*1024*1024 {
1204 // Less than 32GB of ram, restrict to 2 highmem processes
1205 return 2
1206 } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel {
1207 // If less than 8GB total RAM per process, reduce the number of highmem processes
1208 return p
1209 }
1210 // No restriction on highmem processes
1211 return parallel
1212}
1213
Dan Willemsen2bb82d02019-12-27 09:35:42 -08001214func (c *configImpl) TotalRAM() uint64 {
1215 return c.totalRAM
1216}
1217
Kousik Kumarec478642020-09-21 13:39:24 -04001218// ForceUseGoma determines whether we should override Goma deprecation
1219// and use Goma for the current build or not.
1220func (c *configImpl) ForceUseGoma() bool {
1221 if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok {
1222 v = strings.TrimSpace(v)
1223 if v != "" && v != "false" {
1224 return true
1225 }
1226 }
1227 return false
1228}
1229
Dan Willemsen1e704462016-08-21 15:17:17 -07001230func (c *configImpl) UseGoma() bool {
1231 if v, ok := c.environ.Get("USE_GOMA"); ok {
1232 v = strings.TrimSpace(v)
1233 if v != "" && v != "false" {
1234 return true
1235 }
1236 }
1237 return false
1238}
1239
Yoshisato Yanagisawa2cb0e5d2019-01-10 10:14:16 +09001240func (c *configImpl) StartGoma() bool {
1241 if !c.UseGoma() {
1242 return false
1243 }
1244
1245 if v, ok := c.environ.Get("NOSTART_GOMA"); ok {
1246 v = strings.TrimSpace(v)
1247 if v != "" && v != "false" {
1248 return false
1249 }
1250 }
1251 return true
1252}
1253
Kousik Kumar6d1e3482023-07-24 03:44:16 +00001254func (c *configImpl) canSupportRBE() bool {
1255 // Do not use RBE with prod credentials in scenarios when stubby doesn't exist, since
1256 // its unlikely that we will be able to obtain necessary creds without stubby.
1257 authType, _ := c.rbeAuth()
1258 if !c.StubbyExists() && strings.Contains(authType, "use_google_prod_creds") {
1259 return false
1260 }
1261 return true
1262}
1263
Ramy Medhatbbf25672019-07-17 12:30:04 +00001264func (c *configImpl) UseRBE() bool {
Jingwen Chend7ccde12023-06-28 07:19:26 +00001265 // These alternate modes of running Soong do not use RBE / reclient.
Colin Cross8d411ff2023-12-07 10:31:24 -08001266 if c.Queryview() || c.JsonModuleGraph() {
Jingwen Chend7ccde12023-06-28 07:19:26 +00001267 return false
1268 }
1269
Kousik Kumar6d1e3482023-07-24 03:44:16 +00001270 if !c.canSupportRBE() {
Kousik Kumar67ad4342023-06-06 15:09:27 -04001271 return false
1272 }
Kousik Kumar6d1e3482023-07-24 03:44:16 +00001273
Kousik Kumar3ff037e2022-01-25 22:11:01 -05001274 if v, ok := c.Environment().Get("USE_RBE"); ok {
Ramy Medhatbbf25672019-07-17 12:30:04 +00001275 v = strings.TrimSpace(v)
1276 if v != "" && v != "false" {
1277 return true
1278 }
1279 }
1280 return false
1281}
1282
1283func (c *configImpl) StartRBE() bool {
1284 if !c.UseRBE() {
1285 return false
1286 }
1287
1288 if v, ok := c.environ.Get("NOSTART_RBE"); ok {
1289 v = strings.TrimSpace(v)
1290 if v != "" && v != "false" {
1291 return false
1292 }
1293 }
1294 return true
1295}
1296
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001297func (c *configImpl) rbeProxyLogsDir() string {
1298 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
Kousik Kumar0d15a722020-09-23 02:54:11 -04001299 if v, ok := c.environ.Get(f); ok {
1300 return v
1301 }
1302 }
Ramy Medhatbc061762023-10-10 18:36:59 +00001303 return c.rbeTmpDir()
1304}
1305
1306func (c *configImpl) rbeDownloadTmpDir() string {
Cole Faust06ea5312023-10-18 17:38:40 -07001307 for _, f := range []string{"RBE_download_tmp_dir", "FLAG_download_tmp_dir"} {
Ramy Medhatbc061762023-10-10 18:36:59 +00001308 if v, ok := c.environ.Get(f); ok {
1309 return v
1310 }
1311 }
1312 return c.rbeTmpDir()
1313}
1314
1315func (c *configImpl) rbeTmpDir() string {
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001316 buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir())
1317 return filepath.Join(buildTmpDir, "rbe")
Ramy Medhat0fc67eb2020-08-12 01:26:23 -04001318}
1319
Ramy Medhatc8f6cc22023-03-31 09:50:34 -04001320func (c *configImpl) rbeCacheDir() string {
1321 for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} {
1322 if v, ok := c.environ.Get(f); ok {
1323 return v
1324 }
1325 }
1326 return shared.JoinPath(c.SoongOutDir(), "rbe")
1327}
1328
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001329func (c *configImpl) shouldCleanupRBELogsDir() bool {
1330 // Perform a log directory cleanup only when the log directory
1331 // is auto created by the build rather than user-specified.
1332 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} {
1333 if _, ok := c.environ.Get(f); ok {
1334 return false
Patrice Arruda62f1bf22020-07-07 12:48:26 +00001335 }
1336 }
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001337 return true
Ramy Medhat0fc67eb2020-08-12 01:26:23 -04001338}
1339
1340func (c *configImpl) rbeExecRoot() string {
1341 for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} {
1342 if v, ok := c.environ.Get(f); ok {
1343 return v
1344 }
1345 }
1346 wd, err := os.Getwd()
1347 if err != nil {
1348 return ""
1349 }
1350 return wd
1351}
1352
1353func (c *configImpl) rbeDir() string {
1354 if v, ok := c.environ.Get("RBE_DIR"); ok {
1355 return v
1356 }
1357 return "prebuilts/remoteexecution-client/live/"
1358}
1359
1360func (c *configImpl) rbeReproxy() string {
1361 for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} {
1362 if v, ok := c.environ.Get(f); ok {
1363 return v
1364 }
1365 }
1366 return filepath.Join(c.rbeDir(), "reproxy")
1367}
1368
1369func (c *configImpl) rbeAuth() (string, string) {
Kousik Kumar93d192c2022-03-18 01:39:56 -04001370 credFlags := []string{
1371 "use_application_default_credentials",
1372 "use_gce_credentials",
1373 "credential_file",
1374 "use_google_prod_creds",
1375 }
Ramy Medhat0fc67eb2020-08-12 01:26:23 -04001376 for _, cf := range credFlags {
1377 for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} {
1378 if v, ok := c.environ.Get(f); ok {
1379 v = strings.TrimSpace(v)
1380 if v != "" && v != "false" && v != "0" {
1381 return "RBE_" + cf, v
1382 }
1383 }
1384 }
1385 }
1386 return "RBE_use_application_default_credentials", "true"
Patrice Arruda62f1bf22020-07-07 12:48:26 +00001387}
1388
Kousik Kumar4c180ad2022-05-27 07:48:37 -04001389func (c *configImpl) rbeSockAddr(dir string) (string, error) {
1390 maxNameLen := len(syscall.RawSockaddrUnix{}.Path)
1391 base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix)
1392
1393 name := filepath.Join(dir, base)
1394 if len(name) < maxNameLen {
1395 return name, nil
1396 }
1397
1398 name = filepath.Join("/tmp", base)
1399 if len(name) < maxNameLen {
1400 return name, nil
1401 }
1402
1403 return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen)
1404}
1405
Kousik Kumar7bc78192022-04-27 14:52:56 -04001406// IsGooglerEnvironment returns true if the current build is running
1407// on a Google developer machine and false otherwise.
1408func (c *configImpl) IsGooglerEnvironment() bool {
1409 cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG"
1410 if v, ok := c.environ.Get(cf); ok {
1411 return v == "googler"
1412 }
1413 return false
1414}
1415
1416// GoogleProdCredsExist determine whether credentials exist on the
1417// Googler machine to use remote execution.
1418func (c *configImpl) GoogleProdCredsExist() bool {
Kevin Dagostino096ab2f2023-03-03 19:47:17 +00001419 if googleProdCredsExistCache {
1420 return googleProdCredsExistCache
1421 }
andusyu0b3dc032023-06-21 17:29:32 -04001422 if _, err := exec.Command("/usr/bin/gcertstatus", "-nocheck_ssh").Output(); err != nil {
Kousik Kumar7bc78192022-04-27 14:52:56 -04001423 return false
1424 }
Kevin Dagostino096ab2f2023-03-03 19:47:17 +00001425 googleProdCredsExistCache = true
Kousik Kumar7bc78192022-04-27 14:52:56 -04001426 return true
1427}
1428
1429// UseRemoteBuild indicates whether to use a remote build acceleration system
1430// to speed up the build.
Colin Cross9016b912019-11-11 14:57:42 -08001431func (c *configImpl) UseRemoteBuild() bool {
1432 return c.UseGoma() || c.UseRBE()
1433}
1434
Kousik Kumar7bc78192022-04-27 14:52:56 -04001435// StubbyExists checks whether the stubby binary exists on the machine running
1436// the build.
1437func (c *configImpl) StubbyExists() bool {
1438 if _, err := exec.LookPath("stubby"); err != nil {
1439 return false
1440 }
1441 return true
1442}
1443
Dan Willemsen1e704462016-08-21 15:17:17 -07001444// RemoteParallel controls how many remote jobs (i.e., commands which contain
Jeff Gastonefc1b412017-03-29 17:29:06 -07001445// gomacc) are run in parallel. Note the parallelism of all other jobs is
Dan Willemsen1e704462016-08-21 15:17:17 -07001446// still limited by Parallel()
1447func (c *configImpl) RemoteParallel() int {
Colin Cross8b8bec32019-11-15 13:18:43 -08001448 if !c.UseRemoteBuild() {
1449 return 0
1450 }
1451 if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok {
1452 return i
Dan Willemsen1e704462016-08-21 15:17:17 -07001453 }
1454 return 500
1455}
1456
1457func (c *configImpl) SetKatiArgs(args []string) {
1458 c.katiArgs = args
1459}
1460
1461func (c *configImpl) SetNinjaArgs(args []string) {
1462 c.ninjaArgs = args
1463}
1464
1465func (c *configImpl) SetKatiSuffix(suffix string) {
1466 c.katiSuffix = suffix
1467}
1468
Dan Willemsene0879fc2017-08-04 15:06:27 -07001469func (c *configImpl) LastKatiSuffixFile() string {
1470 return filepath.Join(c.OutDir(), "last_kati_suffix")
1471}
1472
1473func (c *configImpl) HasKatiSuffix() bool {
1474 return c.katiSuffix != ""
1475}
1476
Dan Willemsen1e704462016-08-21 15:17:17 -07001477func (c *configImpl) KatiEnvFile() string {
1478 return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
1479}
1480
Dan Willemsen29971232018-09-26 14:58:30 -07001481func (c *configImpl) KatiBuildNinjaFile() string {
1482 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja")
Dan Willemsen1e704462016-08-21 15:17:17 -07001483}
1484
Dan Willemsenfb1271a2018-09-26 15:00:42 -07001485func (c *configImpl) KatiPackageNinjaFile() string {
1486 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja")
1487}
1488
Jihoon Kang9f4f8a32022-08-16 00:57:30 +00001489func (c *configImpl) SoongVarsFile() string {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +09001490 targetProduct, err := c.TargetProductOrErr()
1491 if err != nil {
1492 return filepath.Join(c.SoongOutDir(), "soong.variables")
1493 } else {
1494 return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+".variables")
1495 }
Jihoon Kang9f4f8a32022-08-16 00:57:30 +00001496}
1497
Dan Willemsen1e704462016-08-21 15:17:17 -07001498func (c *configImpl) SoongNinjaFile() string {
Kiyoung Kima37d9ba2023-04-19 13:13:45 +09001499 targetProduct, err := c.TargetProductOrErr()
1500 if err != nil {
1501 return filepath.Join(c.SoongOutDir(), "build.ninja")
1502 } else {
1503 return filepath.Join(c.SoongOutDir(), "build."+targetProduct+".ninja")
1504 }
Dan Willemsen1e704462016-08-21 15:17:17 -07001505}
1506
1507func (c *configImpl) CombinedNinjaFile() string {
Dan Willemsene0879fc2017-08-04 15:06:27 -07001508 if c.katiSuffix == "" {
1509 return filepath.Join(c.OutDir(), "combined.ninja")
1510 }
Dan Willemsen1e704462016-08-21 15:17:17 -07001511 return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
1512}
1513
1514func (c *configImpl) SoongAndroidMk() string {
1515 return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk")
1516}
1517
1518func (c *configImpl) SoongMakeVarsMk() string {
1519 return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk")
1520}
1521
Colin Crossaa9a2732023-10-27 10:54:27 -07001522func (c *configImpl) SoongBuildMetrics() string {
Colin Crossb67b0612023-10-31 10:02:45 -07001523 return filepath.Join(c.LogsDir(), "soong_build_metrics.pb")
Colin Crossaa9a2732023-10-27 10:54:27 -07001524}
1525
Dan Willemsenf052f782017-05-18 15:29:04 -07001526func (c *configImpl) ProductOut() string {
Dan Willemsen4dc4e142017-09-08 14:35:43 -07001527 return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
Dan Willemsenf052f782017-05-18 15:29:04 -07001528}
1529
Dan Willemsen02781d52017-05-12 19:28:13 -07001530func (c *configImpl) DevicePreviousProductConfig() string {
Dan Willemsenf052f782017-05-18 15:29:04 -07001531 return filepath.Join(c.ProductOut(), "previous_build_config.mk")
1532}
1533
Dan Willemsenfb1271a2018-09-26 15:00:42 -07001534func (c *configImpl) KatiPackageMkDir() string {
1535 return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging")
1536}
1537
Dan Willemsenf052f782017-05-18 15:29:04 -07001538func (c *configImpl) hostOutRoot() string {
Dan Willemsen4dc4e142017-09-08 14:35:43 -07001539 return filepath.Join(c.OutDir(), "host")
Dan Willemsenf052f782017-05-18 15:29:04 -07001540}
1541
1542func (c *configImpl) HostOut() string {
1543 return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag())
1544}
1545
1546// This probably needs to be multi-valued, so not exporting it for now
1547func (c *configImpl) hostCrossOut() string {
1548 if runtime.GOOS == "linux" {
1549 return filepath.Join(c.hostOutRoot(), "windows-x86")
1550 } else {
1551 return ""
1552 }
Dan Willemsen02781d52017-05-12 19:28:13 -07001553}
1554
Dan Willemsen1e704462016-08-21 15:17:17 -07001555func (c *configImpl) HostPrebuiltTag() string {
1556 if runtime.GOOS == "linux" {
1557 return "linux-x86"
1558 } else if runtime.GOOS == "darwin" {
1559 return "darwin-x86"
1560 } else {
1561 panic("Unsupported OS")
1562 }
1563}
Dan Willemsenf173d592017-04-27 14:28:00 -07001564
Dan Willemsen8122bd52017-10-12 20:20:41 -07001565func (c *configImpl) PrebuiltBuildTool(name string) string {
Dan Willemsenf173d592017-04-27 14:28:00 -07001566 if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
1567 if sanitize := strings.Fields(v); inList("address", sanitize) {
Dan Willemsen8122bd52017-10-12 20:20:41 -07001568 asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
1569 if _, err := os.Stat(asan); err == nil {
1570 return asan
1571 }
Dan Willemsenf173d592017-04-27 14:28:00 -07001572 }
1573 }
1574 return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
1575}
Dan Willemsen3d60b112018-04-04 22:25:56 -07001576
1577func (c *configImpl) SetBuildBrokenDupRules(val bool) {
1578 c.brokenDupRules = val
1579}
1580
1581func (c *configImpl) BuildBrokenDupRules() bool {
1582 return c.brokenDupRules
1583}
Dan Willemsen6ab79db2018-05-02 00:06:28 -07001584
Dan Willemsen25e6f092019-04-09 10:22:43 -07001585func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) {
1586 c.brokenUsesNetwork = val
1587}
1588
1589func (c *configImpl) BuildBrokenUsesNetwork() bool {
1590 return c.brokenUsesNetwork
1591}
1592
Dan Willemsene3336352020-01-02 19:10:38 -08001593func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) {
1594 c.brokenNinjaEnvVars = val
1595}
1596
1597func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string {
1598 return c.brokenNinjaEnvVars
1599}
1600
Dan Willemsen6ab79db2018-05-02 00:06:28 -07001601func (c *configImpl) SetTargetDeviceDir(dir string) {
1602 c.targetDeviceDir = dir
1603}
1604
1605func (c *configImpl) TargetDeviceDir() string {
1606 return c.targetDeviceDir
1607}
Dan Willemsenfa42f3c2018-06-15 21:54:47 -07001608
Patrice Arruda219eef32020-06-01 17:29:30 +00001609func (c *configImpl) BuildDateTime() string {
1610 return c.buildDateTime
1611}
1612
1613func (c *configImpl) MetricsUploaderApp() string {
Yu Liu6e13b402021-07-27 14:29:06 -07001614 return c.metricsUploader
Patrice Arruda219eef32020-06-01 17:29:30 +00001615}
Patrice Arruda83842d72020-12-08 19:42:08 +00001616
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -05001617// LogsDir returns the absolute path to the logs directory where build log and
1618// metrics files are located. By default, the logs directory is the out
Patrice Arruda83842d72020-12-08 19:42:08 +00001619// directory. If the argument dist is specified, the logs directory
1620// is <dist_dir>/logs.
1621func (c *configImpl) LogsDir() string {
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -05001622 dir := c.OutDir()
Patrice Arruda83842d72020-12-08 19:42:08 +00001623 if c.Dist() {
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +00001624 // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -05001625 dir = filepath.Join(c.RealDistDir(), "logs")
Patrice Arruda83842d72020-12-08 19:42:08 +00001626 }
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -05001627 absDir, err := filepath.Abs(dir)
1628 if err != nil {
1629 fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error())
1630 os.Exit(1)
1631 }
1632 return absDir
Patrice Arruda83842d72020-12-08 19:42:08 +00001633}
1634
Chris Parsons53f68ae2022-03-03 12:01:40 -05001635// MkFileMetrics returns the file path for make-related metrics.
1636func (c *configImpl) MkMetrics() string {
1637 return filepath.Join(c.LogsDir(), "mk_metrics.pb")
1638}
1639
Colin Crossf3bdbcb2021-06-01 11:43:55 -07001640func (c *configImpl) SetEmptyNinjaFile(v bool) {
1641 c.emptyNinjaFile = v
1642}
1643
1644func (c *configImpl) EmptyNinjaFile() bool {
1645 return c.emptyNinjaFile
1646}
Yu Liu6e13b402021-07-27 14:29:06 -07001647
MarkDacekd0e7cd32022-12-02 22:22:40 +00001648func (c *configImpl) SkipMetricsUpload() bool {
1649 return c.skipMetricsUpload
1650}
1651
MarkDacekf47e1422023-04-19 16:47:36 +00001652func (c *configImpl) EnsureAllowlistIntegrity() bool {
1653 return c.ensureAllowlistIntegrity
1654}
1655
MarkDacek6614d9c2022-12-07 21:57:38 +00001656// Returns a Time object if one was passed via a command-line flag.
1657// Otherwise returns the passed default.
1658func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
1659 if c.buildStartedTime == 0 {
1660 return defaultTime
1661 }
1662 return time.UnixMilli(c.buildStartedTime)
1663}
1664
Yu Liu6e13b402021-07-27 14:29:06 -07001665func GetMetricsUploader(topDir string, env *Environment) string {
1666 if p, ok := env.Get("METRICS_UPLOADER"); ok {
1667 metricsUploader := filepath.Join(topDir, p)
1668 if _, err := os.Stat(metricsUploader); err == nil {
1669 return metricsUploader
1670 }
1671 }
1672
1673 return ""
1674}