blob: bd3ad0c2b40b01692fce5aedd64471ea625ee5eb [file] [log] [blame]
Dan Willemsen1e704462016-08-21 15:17:17 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
Dan Willemsenc2af0be2017-01-20 14:10:01 -080018 "os"
Dan Willemsen1e704462016-08-21 15:17:17 -070019 "path/filepath"
20 "runtime"
21 "strconv"
22 "strings"
Nan Zhang2e6a4ff2018-02-14 13:27:26 -080023 "time"
Jeff Gastonefc1b412017-03-29 17:29:06 -070024
25 "android/soong/shared"
Patrice Arruda96850362020-08-11 20:41:11 +000026 "github.com/golang/protobuf/proto"
27
28 smpb "android/soong/ui/metrics/metrics_proto"
Dan Willemsen1e704462016-08-21 15:17:17 -070029)
30
31type Config struct{ *configImpl }
32
33type configImpl struct {
34 // From the environment
Colin Cross28f527c2019-11-26 16:19:04 -080035 arguments []string
36 goma bool
37 environ *Environment
38 distDir string
39 buildDateTime string
Dan Willemsen1e704462016-08-21 15:17:17 -070040
41 // From the arguments
Colin Cross37193492017-11-16 17:55:00 -080042 parallel int
43 keepGoing int
44 verbose bool
45 checkbuild bool
46 dist bool
47 skipMake bool
Dan Willemsen1e704462016-08-21 15:17:17 -070048
49 // From the product config
Dan Willemsen6ab79db2018-05-02 00:06:28 -070050 katiArgs []string
51 ninjaArgs []string
52 katiSuffix string
53 targetDevice string
54 targetDeviceDir string
Dan Willemsen3d60b112018-04-04 22:25:56 -070055
Dan Willemsen2bb82d02019-12-27 09:35:42 -080056 // Autodetected
57 totalRAM uint64
58
Dan Willemsene3336352020-01-02 19:10:38 -080059 brokenDupRules bool
60 brokenUsesNetwork bool
61 brokenNinjaEnvVars []string
Dan Willemsen18490112018-05-25 16:30:04 -070062
63 pathReplaced bool
Dan Willemsen1e704462016-08-21 15:17:17 -070064}
65
Dan Willemsenc2af0be2017-01-20 14:10:01 -080066const srcDirFileCheck = "build/soong/root.bp"
67
Patrice Arruda9450d0b2019-07-08 11:06:46 -070068var buildFiles = []string{"Android.mk", "Android.bp"}
69
Patrice Arruda13848222019-04-22 17:12:02 -070070type BuildAction uint
71
72const (
73 // Builds all of the modules and their dependencies of a specified directory, relative to the root
74 // directory of the source tree.
75 BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota
76
77 // Builds all of the modules and their dependencies of a list of specified directories. All specified
78 // directories are relative to the root directory of the source tree.
79 BUILD_MODULES_IN_DIRECTORIES
Patrice Arruda39282062019-06-20 16:35:12 -070080
81 // Build a list of specified modules. If none was specified, simply build the whole source tree.
82 BUILD_MODULES
Patrice Arruda13848222019-04-22 17:12:02 -070083)
84
85// checkTopDir validates that the current directory is at the root directory of the source tree.
86func checkTopDir(ctx Context) {
87 if _, err := os.Stat(srcDirFileCheck); err != nil {
88 if os.IsNotExist(err) {
89 ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck)
90 }
91 ctx.Fatalln("Error verifying tree state:", err)
92 }
93}
94
Dan Willemsen1e704462016-08-21 15:17:17 -070095func NewConfig(ctx Context, args ...string) Config {
96 ret := &configImpl{
97 environ: OsEnvironment(),
98 }
99
Patrice Arruda90109172020-07-28 18:07:27 +0000100 // Default matching ninja
Dan Willemsen9b587492017-07-10 22:13:00 -0700101 ret.parallel = runtime.NumCPU() + 2
102 ret.keepGoing = 1
103
Dan Willemsen2bb82d02019-12-27 09:35:42 -0800104 ret.totalRAM = detectTotalRAM(ctx)
105
Dan Willemsen9b587492017-07-10 22:13:00 -0700106 ret.parseArgs(ctx, args)
107
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800108 // Make sure OUT_DIR is set appropriately
Dan Willemsen02f3add2017-05-12 13:50:19 -0700109 if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
110 ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
111 } else {
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800112 outDir := "out"
113 if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok {
114 if wd, err := os.Getwd(); err != nil {
115 ctx.Fatalln("Failed to get working directory:", err)
116 } else {
117 outDir = filepath.Join(baseDir, filepath.Base(wd))
118 }
119 }
120 ret.environ.Set("OUT_DIR", outDir)
121 }
122
Dan Willemsen2d31a442018-10-20 21:33:41 -0700123 if distDir, ok := ret.environ.Get("DIST_DIR"); ok {
124 ret.distDir = filepath.Clean(distDir)
125 } else {
126 ret.distDir = filepath.Join(ret.OutDir(), "dist")
127 }
Dan Willemsend50e89f2018-10-16 17:49:25 -0700128
Dan Willemsen1e704462016-08-21 15:17:17 -0700129 ret.environ.Unset(
130 // We're already using it
131 "USE_SOONG_UI",
132
133 // We should never use GOROOT/GOPATH from the shell environment
134 "GOROOT",
135 "GOPATH",
136
137 // These should only come from Soong, not the environment.
138 "CLANG",
139 "CLANG_CXX",
140 "CCC_CC",
141 "CCC_CXX",
142
143 // Used by the goma compiler wrapper, but should only be set by
144 // gomacc
145 "GOMACC_PATH",
Dan Willemsen0c3919e2017-03-02 15:49:10 -0800146
147 // We handle this above
148 "OUT_DIR_COMMON_BASE",
Dan Willemsen68a09852017-04-18 13:56:57 -0700149
Dan Willemsen2d31a442018-10-20 21:33:41 -0700150 // This is handled above too, and set for individual commands later
151 "DIST_DIR",
152
Dan Willemsen68a09852017-04-18 13:56:57 -0700153 // Variables that have caused problems in the past
Dan Willemsen1c504d92019-11-18 19:13:53 +0000154 "BASH_ENV",
Dan Willemsenebfe33a2018-05-01 10:07:50 -0700155 "CDPATH",
Dan Willemsen68a09852017-04-18 13:56:57 -0700156 "DISPLAY",
157 "GREP_OPTIONS",
Dan Willemsenebfe33a2018-05-01 10:07:50 -0700158 "NDK_ROOT",
Dan Willemsen00fcb262018-08-15 15:35:38 -0700159 "POSIXLY_CORRECT",
Dan Willemsenc40e10b2017-07-11 14:30:00 -0700160
161 // Drop make flags
162 "MAKEFLAGS",
163 "MAKELEVEL",
164 "MFLAGS",
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700165
166 // Set in envsetup.sh, reset in makefiles
167 "ANDROID_JAVA_TOOLCHAIN",
Colin Cross7f09c402018-07-11 14:49:31 -0700168
169 // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional
170 "ANDROID_BUILD_TOP",
171 "ANDROID_HOST_OUT",
172 "ANDROID_PRODUCT_OUT",
173 "ANDROID_HOST_OUT_TESTCASES",
174 "ANDROID_TARGET_OUT_TESTCASES",
175 "ANDROID_TOOLCHAIN",
176 "ANDROID_TOOLCHAIN_2ND_ARCH",
177 "ANDROID_DEV_SCRIPTS",
178 "ANDROID_EMULATOR_PREBUILTS",
179 "ANDROID_PRE_BUILD_PATHS",
Dan Willemsenf99915f2018-10-25 22:04:42 -0700180
181 // Only set in multiproduct_kati after config generation
182 "EMPTY_NINJA_FILE",
Dan Willemsen1e704462016-08-21 15:17:17 -0700183 )
184
185 // Tell python not to spam the source tree with .pyc files.
186 ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1")
187
Ramy Medhatca1e44c2020-07-16 12:18:37 -0400188 tmpDir := absPath(ctx, ret.TempDir())
189 ret.environ.Set("TMPDIR", tmpDir)
Dan Willemsen32a669b2018-03-08 19:42:00 -0800190
Dan Willemsen70c1ff82019-08-21 14:56:13 -0700191 // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes
192 symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(),
193 "llvm-binutils-stable/llvm-symbolizer")
194 ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath))
195
Dan Willemsenc2af0be2017-01-20 14:10:01 -0800196 // Precondition: the current directory is the top of the source tree
Patrice Arruda13848222019-04-22 17:12:02 -0700197 checkTopDir(ctx)
Dan Willemsenc2af0be2017-01-20 14:10:01 -0800198
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700199 if srcDir := absPath(ctx, "."); strings.ContainsRune(srcDir, ' ') {
Colin Cross1f6faeb2019-09-23 15:52:40 -0700200 ctx.Println("You are building in a directory whose absolute path contains a space character:")
201 ctx.Println()
202 ctx.Printf("%q\n", srcDir)
203 ctx.Println()
204 ctx.Fatalln("Directory names containing spaces are not supported")
Dan Willemsendb8457c2017-05-12 16:38:17 -0700205 }
206
207 if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') {
Colin Cross1f6faeb2019-09-23 15:52:40 -0700208 ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:")
209 ctx.Println()
210 ctx.Printf("%q\n", outDir)
211 ctx.Println()
212 ctx.Fatalln("Directory names containing spaces are not supported")
Dan Willemsendb8457c2017-05-12 16:38:17 -0700213 }
214
215 if distDir := ret.DistDir(); strings.ContainsRune(distDir, ' ') {
Colin Cross1f6faeb2019-09-23 15:52:40 -0700216 ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:")
217 ctx.Println()
218 ctx.Printf("%q\n", distDir)
219 ctx.Println()
220 ctx.Fatalln("Directory names containing spaces are not supported")
Dan Willemsendb8457c2017-05-12 16:38:17 -0700221 }
222
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700223 // Configure Java-related variables, including adding it to $PATH
Tobias Thierere59aeff2017-12-20 22:40:39 +0000224 java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag())
225 java9Home := filepath.Join("prebuilts/jdk/jdk9", ret.HostPrebuiltTag())
Pete Gillin1f52e932019-10-09 17:10:08 +0100226 java11Home := filepath.Join("prebuilts/jdk/jdk11", ret.HostPrebuiltTag())
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700227 javaHome := func() string {
228 if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok {
229 return override
230 }
Pete Gillina7a3d642019-11-07 18:58:42 +0000231 if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" {
232 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 11 toolchain is now the global default.")
Pete Gillin1f52e932019-10-09 17:10:08 +0100233 }
Pete Gillinabbcdda2019-10-28 16:15:33 +0000234 return java11Home
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700235 }()
236 absJavaHome := absPath(ctx, javaHome)
237
Dan Willemsened869522018-01-08 14:58:46 -0800238 ret.configureLocale(ctx)
239
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700240 newPath := []string{filepath.Join(absJavaHome, "bin")}
241 if path, ok := ret.environ.Get("PATH"); ok && path != "" {
242 newPath = append(newPath, path)
243 }
Pete Gillin1f52e932019-10-09 17:10:08 +0100244
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700245 ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME")
246 ret.environ.Set("JAVA_HOME", absJavaHome)
247 ret.environ.Set("ANDROID_JAVA_HOME", javaHome)
Tobias Thierere59aeff2017-12-20 22:40:39 +0000248 ret.environ.Set("ANDROID_JAVA8_HOME", java8Home)
249 ret.environ.Set("ANDROID_JAVA9_HOME", java9Home)
Pete Gillin1f52e932019-10-09 17:10:08 +0100250 ret.environ.Set("ANDROID_JAVA11_HOME", java11Home)
Dan Willemsend9e8f0a2017-10-30 13:42:06 -0700251 ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator)))
252
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800253 outDir := ret.OutDir()
254 buildDateTimeFile := filepath.Join(outDir, "build_date.txt")
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800255 if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" {
Colin Cross28f527c2019-11-26 16:19:04 -0800256 ret.buildDateTime = buildDateTime
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800257 } else {
Colin Cross28f527c2019-11-26 16:19:04 -0800258 ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10)
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800259 }
Colin Cross28f527c2019-11-26 16:19:04 -0800260
Nan Zhang2e6a4ff2018-02-14 13:27:26 -0800261 ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile)
262
Ramy Medhatca1e44c2020-07-16 12:18:37 -0400263 if ret.UseRBE() {
264 for k, v := range getRBEVars(ctx, tmpDir) {
265 ret.environ.Set(k, v)
266 }
267 }
268
Patrice Arruda96850362020-08-11 20:41:11 +0000269 c := Config{ret}
270 storeConfigMetrics(ctx, c)
271 return c
Dan Willemsen9b587492017-07-10 22:13:00 -0700272}
273
Patrice Arruda13848222019-04-22 17:12:02 -0700274// NewBuildActionConfig returns a build configuration based on the build action. The arguments are
275// processed based on the build action and extracts any arguments that belongs to the build action.
Dan Willemsence41e942019-07-29 23:39:30 -0700276func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config {
277 return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...)
Patrice Arruda13848222019-04-22 17:12:02 -0700278}
279
Patrice Arruda96850362020-08-11 20:41:11 +0000280// storeConfigMetrics selects a set of configuration information and store in
281// the metrics system for further analysis.
282func storeConfigMetrics(ctx Context, config Config) {
283 if ctx.Metrics == nil {
284 return
285 }
286
287 b := &smpb.BuildConfig{
288 UseGoma: proto.Bool(config.UseGoma()),
289 UseRbe: proto.Bool(config.UseRBE()),
290 }
291 ctx.Metrics.BuildConfig(b)
292}
293
Patrice Arruda13848222019-04-22 17:12:02 -0700294// getConfigArgs processes the command arguments based on the build action and creates a set of new
295// arguments to be accepted by Config.
Dan Willemsence41e942019-07-29 23:39:30 -0700296func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string {
Patrice Arruda13848222019-04-22 17:12:02 -0700297 // The next block of code verifies that the current directory is the root directory of the source
298 // tree. It then finds the relative path of dir based on the root directory of the source tree
299 // and verify that dir is inside of the source tree.
300 checkTopDir(ctx)
301 topDir, err := os.Getwd()
302 if err != nil {
303 ctx.Fatalf("Error retrieving top directory: %v", err)
304 }
Patrice Arrudababa9a92019-07-03 10:47:34 -0700305 dir, err = filepath.EvalSymlinks(dir)
306 if err != nil {
307 ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err)
308 }
Patrice Arruda13848222019-04-22 17:12:02 -0700309 dir, err = filepath.Abs(dir)
310 if err != nil {
311 ctx.Fatalf("Unable to find absolute path %s: %v", dir, err)
312 }
313 relDir, err := filepath.Rel(topDir, dir)
314 if err != nil {
315 ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err)
316 }
317 // If there are ".." in the path, it's not in the source tree.
318 if strings.Contains(relDir, "..") {
319 ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir)
320 }
321
322 configArgs := args[:]
323
324 // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to
325 // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules.
326 targetNamePrefix := "MODULES-IN-"
327 if inList("GET-INSTALL-PATH", configArgs) {
328 targetNamePrefix = "GET-INSTALL-PATH-IN-"
329 configArgs = removeFromList("GET-INSTALL-PATH", configArgs)
330 }
331
Patrice Arruda13848222019-04-22 17:12:02 -0700332 var targets []string
333
334 switch action {
Patrice Arruda39282062019-06-20 16:35:12 -0700335 case BUILD_MODULES:
336 // No additional processing is required when building a list of specific modules or all modules.
Patrice Arruda13848222019-04-22 17:12:02 -0700337 case BUILD_MODULES_IN_A_DIRECTORY:
338 // If dir is the root source tree, all the modules are built of the source tree are built so
339 // no need to find the build file.
340 if topDir == dir {
341 break
342 }
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700343
Patrice Arruda13848222019-04-22 17:12:02 -0700344 buildFile := findBuildFile(ctx, relDir)
345 if buildFile == "" {
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700346 ctx.Fatalf("Build file not found for %s directory", relDir)
Patrice Arruda13848222019-04-22 17:12:02 -0700347 }
Patrice Arruda13848222019-04-22 17:12:02 -0700348 targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
349 case BUILD_MODULES_IN_DIRECTORIES:
350 newConfigArgs, dirs := splitArgs(configArgs)
351 configArgs = newConfigArgs
Dan Willemsence41e942019-07-29 23:39:30 -0700352 targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix)
Patrice Arruda13848222019-04-22 17:12:02 -0700353 }
354
355 // Tidy only override all other specified targets.
356 tidyOnly := os.Getenv("WITH_TIDY_ONLY")
357 if tidyOnly == "true" || tidyOnly == "1" {
358 configArgs = append(configArgs, "tidy_only")
359 } else {
360 configArgs = append(configArgs, targets...)
361 }
362
363 return configArgs
364}
365
366// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name.
367func convertToTarget(dir string, targetNamePrefix string) string {
368 return targetNamePrefix + strings.ReplaceAll(dir, "/", "-")
369}
370
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700371// hasBuildFile returns true if dir contains an Android build file.
372func hasBuildFile(ctx Context, dir string) bool {
373 for _, buildFile := range buildFiles {
374 _, err := os.Stat(filepath.Join(dir, buildFile))
375 if err == nil {
376 return true
377 }
378 if !os.IsNotExist(err) {
379 ctx.Fatalf("Error retrieving the build file stats: %v", err)
380 }
381 }
382 return false
383}
384
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700385// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file
386// in the current and any sub directory of dir. If a build file is not found, traverse the path
387// up by one directory and repeat again until either a build file is found or reached to the root
388// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank
389// string is returned.
Patrice Arruda13848222019-04-22 17:12:02 -0700390func findBuildFile(ctx Context, dir string) string {
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700391 // If the string is empty or ".", assume it is top directory of the source tree.
392 if dir == "" || dir == "." {
Patrice Arruda13848222019-04-22 17:12:02 -0700393 return ""
394 }
395
Patrice Arruda0dcf27f2019-07-08 17:03:33 -0700396 found := false
397 for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) {
398 err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error {
399 if err != nil {
400 return err
401 }
402 if found {
403 return filepath.SkipDir
404 }
405 if info.IsDir() {
406 return nil
407 }
408 for _, buildFile := range buildFiles {
409 if info.Name() == buildFile {
410 found = true
411 return filepath.SkipDir
412 }
413 }
414 return nil
415 })
416 if err != nil {
417 ctx.Fatalf("Error finding Android build file: %v", err)
418 }
419
420 if found {
421 return filepath.Join(buildDir, "Android.mk")
Patrice Arruda13848222019-04-22 17:12:02 -0700422 }
423 }
424
425 return ""
426}
427
428// splitArgs iterates over the arguments list and splits into two lists: arguments and directories.
429func splitArgs(args []string) (newArgs []string, dirs []string) {
430 specialArgs := map[string]bool{
431 "showcommands": true,
432 "snod": true,
433 "dist": true,
434 "checkbuild": true,
435 }
436
437 newArgs = []string{}
438 dirs = []string{}
439
440 for _, arg := range args {
441 // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory.
442 if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 {
443 newArgs = append(newArgs, arg)
444 continue
445 }
446
447 if _, ok := specialArgs[arg]; ok {
448 newArgs = append(newArgs, arg)
449 continue
450 }
451
452 dirs = append(dirs, arg)
453 }
454
455 return newArgs, dirs
456}
457
458// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a
459// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the
460// source root tree where the build action command was invoked. Each directory is validated if the
461// build file can be found and follows the format "dir1:target1,target2,...". Target is optional.
Dan Willemsence41e942019-07-29 23:39:30 -0700462func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) {
Patrice Arruda13848222019-04-22 17:12:02 -0700463 for _, dir := range dirs {
464 // The directory may have specified specific modules to build. ":" is the separator to separate
465 // the directory and the list of modules.
466 s := strings.Split(dir, ":")
467 l := len(s)
468 if l > 2 { // more than one ":" was specified.
469 ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir)
470 }
471
472 dir = filepath.Join(relDir, s[0])
473 if _, err := os.Stat(dir); err != nil {
474 ctx.Fatalf("couldn't find directory %s", dir)
475 }
476
477 // Verify that if there are any targets specified after ":". Each target is separated by ",".
478 var newTargets []string
479 if l == 2 && s[1] != "" {
480 newTargets = strings.Split(s[1], ",")
481 if inList("", newTargets) {
482 ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir)
483 }
484 }
485
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700486 // If there are specified targets to build in dir, an android build file must exist for the one
487 // shot build. For the non-targets case, find the appropriate build file and build all the
488 // modules in dir (or the closest one in the dir path).
Patrice Arruda13848222019-04-22 17:12:02 -0700489 if len(newTargets) > 0 {
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700490 if !hasBuildFile(ctx, dir) {
Patrice Arruda13848222019-04-22 17:12:02 -0700491 ctx.Fatalf("Couldn't locate a build file from %s directory", dir)
492 }
493 } else {
Patrice Arruda9450d0b2019-07-08 11:06:46 -0700494 buildFile := findBuildFile(ctx, dir)
495 if buildFile == "" {
496 ctx.Fatalf("Build file not found for %s directory", dir)
497 }
498 newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)}
Patrice Arruda13848222019-04-22 17:12:02 -0700499 }
500
Patrice Arruda13848222019-04-22 17:12:02 -0700501 targets = append(targets, newTargets...)
502 }
503
Dan Willemsence41e942019-07-29 23:39:30 -0700504 return targets
Patrice Arruda13848222019-04-22 17:12:02 -0700505}
506
Dan Willemsen9b587492017-07-10 22:13:00 -0700507func (c *configImpl) parseArgs(ctx Context, args []string) {
508 for i := 0; i < len(args); i++ {
509 arg := strings.TrimSpace(args[i])
Dan Willemsen1e704462016-08-21 15:17:17 -0700510 if arg == "--make-mode" {
Dan Willemsen1e704462016-08-21 15:17:17 -0700511 } else if arg == "showcommands" {
Dan Willemsen9b587492017-07-10 22:13:00 -0700512 c.verbose = true
Dan Willemsene0879fc2017-08-04 15:06:27 -0700513 } else if arg == "--skip-make" {
514 c.skipMake = true
Dan Willemsen6ac63ef2017-10-17 20:35:34 -0700515 } else if len(arg) > 0 && arg[0] == '-' {
Dan Willemsen9b587492017-07-10 22:13:00 -0700516 parseArgNum := func(def int) int {
517 if len(arg) > 2 {
518 p, err := strconv.ParseUint(arg[2:], 10, 31)
519 if err != nil {
520 ctx.Fatalf("Failed to parse %q: %v", arg, err)
521 }
522 return int(p)
523 } else if i+1 < len(args) {
524 p, err := strconv.ParseUint(args[i+1], 10, 31)
525 if err == nil {
526 i++
527 return int(p)
528 }
529 }
530 return def
531 }
532
Dan Willemsen6ac63ef2017-10-17 20:35:34 -0700533 if len(arg) > 1 && arg[1] == 'j' {
Dan Willemsen9b587492017-07-10 22:13:00 -0700534 c.parallel = parseArgNum(c.parallel)
Dan Willemsen6ac63ef2017-10-17 20:35:34 -0700535 } else if len(arg) > 1 && arg[1] == 'k' {
Dan Willemsen9b587492017-07-10 22:13:00 -0700536 c.keepGoing = parseArgNum(0)
Dan Willemsen1e704462016-08-21 15:17:17 -0700537 } else {
538 ctx.Fatalln("Unknown option:", arg)
539 }
Dan Willemsen091525e2017-07-11 14:17:50 -0700540 } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 {
Dan Willemsen6dfe30a2018-09-10 12:41:10 -0700541 if k == "OUT_DIR" {
542 ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.")
543 }
Dan Willemsen091525e2017-07-11 14:17:50 -0700544 c.environ.Set(k, v)
Dan Willemsen2d31a442018-10-20 21:33:41 -0700545 } else if arg == "dist" {
546 c.dist = true
Dan Willemsen1e704462016-08-21 15:17:17 -0700547 } else {
Dan Willemsen2d31a442018-10-20 21:33:41 -0700548 if arg == "checkbuild" {
Colin Cross37193492017-11-16 17:55:00 -0800549 c.checkbuild = true
Dan Willemsene0879fc2017-08-04 15:06:27 -0700550 }
Dan Willemsen9b587492017-07-10 22:13:00 -0700551 c.arguments = append(c.arguments, arg)
Dan Willemsen1e704462016-08-21 15:17:17 -0700552 }
553 }
Dan Willemsen1e704462016-08-21 15:17:17 -0700554}
555
Dan Willemsened869522018-01-08 14:58:46 -0800556func (c *configImpl) configureLocale(ctx Context) {
557 cmd := Command(ctx, Config{c}, "locale", "locale", "-a")
558 output, err := cmd.Output()
559
560 var locales []string
561 if err == nil {
562 locales = strings.Split(string(output), "\n")
563 } else {
564 // If we're unable to list the locales, let's assume en_US.UTF-8
565 locales = []string{"en_US.UTF-8"}
566 ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales)
567 }
568
569 // gettext uses LANGUAGE, which is passed directly through
570
571 // For LANG and LC_*, only preserve the evaluated version of
572 // LC_MESSAGES
573 user_lang := ""
574 if lc_all, ok := c.environ.Get("LC_ALL"); ok {
575 user_lang = lc_all
576 } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok {
577 user_lang = lc_messages
578 } else if lang, ok := c.environ.Get("LANG"); ok {
579 user_lang = lang
580 }
581
582 c.environ.UnsetWithPrefix("LC_")
583
584 if user_lang != "" {
585 c.environ.Set("LC_MESSAGES", user_lang)
586 }
587
588 // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed
589 // for others)
590 if inList("C.UTF-8", locales) {
591 c.environ.Set("LANG", "C.UTF-8")
Aaron Klingd236e0e2018-08-07 19:21:36 -0500592 } else if inList("C.utf8", locales) {
593 // These normalize to the same thing
594 c.environ.Set("LANG", "C.UTF-8")
Dan Willemsened869522018-01-08 14:58:46 -0800595 } else if inList("en_US.UTF-8", locales) {
596 c.environ.Set("LANG", "en_US.UTF-8")
597 } else if inList("en_US.utf8", locales) {
598 // These normalize to the same thing
599 c.environ.Set("LANG", "en_US.UTF-8")
600 } else {
601 ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8")
602 }
603}
604
Dan Willemsen1e704462016-08-21 15:17:17 -0700605// Lunch configures the environment for a specific product similarly to the
606// `lunch` bash function.
607func (c *configImpl) Lunch(ctx Context, product, variant string) {
608 if variant != "eng" && variant != "userdebug" && variant != "user" {
609 ctx.Fatalf("Invalid variant %q. Must be one of 'user', 'userdebug' or 'eng'", variant)
610 }
611
612 c.environ.Set("TARGET_PRODUCT", product)
613 c.environ.Set("TARGET_BUILD_VARIANT", variant)
614 c.environ.Set("TARGET_BUILD_TYPE", "release")
615 c.environ.Unset("TARGET_BUILD_APPS")
Martin Stjernholm08802332020-06-04 17:00:01 +0100616 c.environ.Unset("TARGET_BUILD_UNBUNDLED")
Dan Willemsen1e704462016-08-21 15:17:17 -0700617}
618
619// Tapas configures the environment to build one or more unbundled apps,
620// similarly to the `tapas` bash function.
621func (c *configImpl) Tapas(ctx Context, apps []string, arch, variant string) {
622 if len(apps) == 0 {
623 apps = []string{"all"}
624 }
625 if variant == "" {
626 variant = "eng"
627 }
628
629 if variant != "eng" && variant != "userdebug" && variant != "user" {
630 ctx.Fatalf("Invalid variant %q. Must be one of 'user', 'userdebug' or 'eng'", variant)
631 }
632
633 var product string
634 switch arch {
Dan Willemsen1e704462016-08-21 15:17:17 -0700635 case "arm", "":
636 product = "aosp_arm"
637 case "arm64":
638 product = "aosm_arm64"
Dan Willemsen1e704462016-08-21 15:17:17 -0700639 case "x86":
640 product = "aosp_x86"
641 case "x86_64":
642 product = "aosp_x86_64"
643 default:
644 ctx.Fatalf("Invalid architecture: %q", arch)
645 }
646
647 c.environ.Set("TARGET_PRODUCT", product)
648 c.environ.Set("TARGET_BUILD_VARIANT", variant)
649 c.environ.Set("TARGET_BUILD_TYPE", "release")
650 c.environ.Set("TARGET_BUILD_APPS", strings.Join(apps, " "))
651}
652
653func (c *configImpl) Environment() *Environment {
654 return c.environ
655}
656
657func (c *configImpl) Arguments() []string {
658 return c.arguments
659}
660
661func (c *configImpl) OutDir() string {
662 if outDir, ok := c.environ.Get("OUT_DIR"); ok {
Patrice Arruda19bd53e2019-07-08 17:26:47 -0700663 return outDir
Dan Willemsen1e704462016-08-21 15:17:17 -0700664 }
665 return "out"
666}
667
Dan Willemsen8a073a82017-02-04 17:30:44 -0800668func (c *configImpl) DistDir() string {
Dan Willemsen2d31a442018-10-20 21:33:41 -0700669 return c.distDir
Dan Willemsen8a073a82017-02-04 17:30:44 -0800670}
671
Dan Willemsen1e704462016-08-21 15:17:17 -0700672func (c *configImpl) NinjaArgs() []string {
Dan Willemsene0879fc2017-08-04 15:06:27 -0700673 if c.skipMake {
674 return c.arguments
675 }
Dan Willemsen1e704462016-08-21 15:17:17 -0700676 return c.ninjaArgs
677}
678
679func (c *configImpl) SoongOutDir() string {
680 return filepath.Join(c.OutDir(), "soong")
681}
682
Jeff Gastonefc1b412017-03-29 17:29:06 -0700683func (c *configImpl) TempDir() string {
684 return shared.TempDirForOutDir(c.SoongOutDir())
685}
686
Jeff Gastonb64fc1c2017-08-04 12:30:12 -0700687func (c *configImpl) FileListDir() string {
688 return filepath.Join(c.OutDir(), ".module_paths")
689}
690
Dan Willemsen1e704462016-08-21 15:17:17 -0700691func (c *configImpl) KatiSuffix() string {
692 if c.katiSuffix != "" {
693 return c.katiSuffix
694 }
695 panic("SetKatiSuffix has not been called")
696}
697
Colin Cross37193492017-11-16 17:55:00 -0800698// Checkbuild returns true if "checkbuild" was one of the build goals, which means that the
699// user is interested in additional checks at the expense of build time.
700func (c *configImpl) Checkbuild() bool {
701 return c.checkbuild
702}
703
Dan Willemsen8a073a82017-02-04 17:30:44 -0800704func (c *configImpl) Dist() bool {
705 return c.dist
706}
707
Dan Willemsen1e704462016-08-21 15:17:17 -0700708func (c *configImpl) IsVerbose() bool {
709 return c.verbose
710}
711
Dan Willemsene0879fc2017-08-04 15:06:27 -0700712func (c *configImpl) SkipMake() bool {
713 return c.skipMake
714}
715
Dan Willemsen1e704462016-08-21 15:17:17 -0700716func (c *configImpl) TargetProduct() string {
717 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok {
718 return v
719 }
720 panic("TARGET_PRODUCT is not defined")
721}
722
Dan Willemsen02781d52017-05-12 19:28:13 -0700723func (c *configImpl) TargetDevice() string {
724 return c.targetDevice
725}
726
727func (c *configImpl) SetTargetDevice(device string) {
728 c.targetDevice = device
729}
730
731func (c *configImpl) TargetBuildVariant() string {
732 if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok {
733 return v
734 }
735 panic("TARGET_BUILD_VARIANT is not defined")
736}
737
Dan Willemsen1e704462016-08-21 15:17:17 -0700738func (c *configImpl) KatiArgs() []string {
739 return c.katiArgs
740}
741
742func (c *configImpl) Parallel() int {
743 return c.parallel
744}
745
Colin Cross8b8bec32019-11-15 13:18:43 -0800746func (c *configImpl) HighmemParallel() int {
747 if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
748 return i
749 }
750
751 const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024
752 parallel := c.Parallel()
753 if c.UseRemoteBuild() {
754 // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism
755 // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs
756 // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention.
757 // Return 1/16th of the size of the local pool, rounding up.
758 return (parallel + 15) / 16
759 } else if c.totalRAM == 0 {
760 // Couldn't detect the total RAM, don't restrict highmem processes.
761 return parallel
Dan Willemsen570a2922020-05-26 23:02:29 -0700762 } else if c.totalRAM <= 16*1024*1024*1024 {
763 // Less than 16GB of ram, restrict to 1 highmem processes
764 return 1
Colin Cross8b8bec32019-11-15 13:18:43 -0800765 } else if c.totalRAM <= 32*1024*1024*1024 {
766 // Less than 32GB of ram, restrict to 2 highmem processes
767 return 2
768 } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel {
769 // If less than 8GB total RAM per process, reduce the number of highmem processes
770 return p
771 }
772 // No restriction on highmem processes
773 return parallel
774}
775
Dan Willemsen2bb82d02019-12-27 09:35:42 -0800776func (c *configImpl) TotalRAM() uint64 {
777 return c.totalRAM
778}
779
Dan Willemsen1e704462016-08-21 15:17:17 -0700780func (c *configImpl) UseGoma() bool {
781 if v, ok := c.environ.Get("USE_GOMA"); ok {
782 v = strings.TrimSpace(v)
783 if v != "" && v != "false" {
784 return true
785 }
786 }
787 return false
788}
789
Yoshisato Yanagisawa2cb0e5d2019-01-10 10:14:16 +0900790func (c *configImpl) StartGoma() bool {
791 if !c.UseGoma() {
792 return false
793 }
794
795 if v, ok := c.environ.Get("NOSTART_GOMA"); ok {
796 v = strings.TrimSpace(v)
797 if v != "" && v != "false" {
798 return false
799 }
800 }
801 return true
802}
803
Ramy Medhatbbf25672019-07-17 12:30:04 +0000804func (c *configImpl) UseRBE() bool {
805 if v, ok := c.environ.Get("USE_RBE"); ok {
806 v = strings.TrimSpace(v)
807 if v != "" && v != "false" {
808 return true
809 }
810 }
811 return false
812}
813
814func (c *configImpl) StartRBE() bool {
815 if !c.UseRBE() {
816 return false
817 }
818
819 if v, ok := c.environ.Get("NOSTART_RBE"); ok {
820 v = strings.TrimSpace(v)
821 if v != "" && v != "false" {
822 return false
823 }
824 }
825 return true
826}
827
Patrice Arruda62f1bf22020-07-07 12:48:26 +0000828func (c *configImpl) RBEStatsOutputDir() string {
829 for _, f := range []string{"RBE_output_dir", "FLAG_output_dir"} {
830 if v, ok := c.environ.Get(f); ok {
831 return v
832 }
833 }
834 return ""
835}
836
Colin Cross9016b912019-11-11 14:57:42 -0800837func (c *configImpl) UseRemoteBuild() bool {
838 return c.UseGoma() || c.UseRBE()
839}
840
Dan Willemsen1e704462016-08-21 15:17:17 -0700841// RemoteParallel controls how many remote jobs (i.e., commands which contain
Jeff Gastonefc1b412017-03-29 17:29:06 -0700842// gomacc) are run in parallel. Note the parallelism of all other jobs is
Dan Willemsen1e704462016-08-21 15:17:17 -0700843// still limited by Parallel()
844func (c *configImpl) RemoteParallel() int {
Colin Cross8b8bec32019-11-15 13:18:43 -0800845 if !c.UseRemoteBuild() {
846 return 0
847 }
848 if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok {
849 return i
Dan Willemsen1e704462016-08-21 15:17:17 -0700850 }
851 return 500
852}
853
854func (c *configImpl) SetKatiArgs(args []string) {
855 c.katiArgs = args
856}
857
858func (c *configImpl) SetNinjaArgs(args []string) {
859 c.ninjaArgs = args
860}
861
862func (c *configImpl) SetKatiSuffix(suffix string) {
863 c.katiSuffix = suffix
864}
865
Dan Willemsene0879fc2017-08-04 15:06:27 -0700866func (c *configImpl) LastKatiSuffixFile() string {
867 return filepath.Join(c.OutDir(), "last_kati_suffix")
868}
869
870func (c *configImpl) HasKatiSuffix() bool {
871 return c.katiSuffix != ""
872}
873
Dan Willemsen1e704462016-08-21 15:17:17 -0700874func (c *configImpl) KatiEnvFile() string {
875 return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh")
876}
877
Dan Willemsen29971232018-09-26 14:58:30 -0700878func (c *configImpl) KatiBuildNinjaFile() string {
879 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja")
Dan Willemsen1e704462016-08-21 15:17:17 -0700880}
881
Dan Willemsenfb1271a2018-09-26 15:00:42 -0700882func (c *configImpl) KatiPackageNinjaFile() string {
883 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja")
884}
885
Dan Willemsen1e704462016-08-21 15:17:17 -0700886func (c *configImpl) SoongNinjaFile() string {
887 return filepath.Join(c.SoongOutDir(), "build.ninja")
888}
889
890func (c *configImpl) CombinedNinjaFile() string {
Dan Willemsene0879fc2017-08-04 15:06:27 -0700891 if c.katiSuffix == "" {
892 return filepath.Join(c.OutDir(), "combined.ninja")
893 }
Dan Willemsen1e704462016-08-21 15:17:17 -0700894 return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja")
895}
896
897func (c *configImpl) SoongAndroidMk() string {
898 return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk")
899}
900
901func (c *configImpl) SoongMakeVarsMk() string {
902 return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk")
903}
904
Dan Willemsenf052f782017-05-18 15:29:04 -0700905func (c *configImpl) ProductOut() string {
Dan Willemsen4dc4e142017-09-08 14:35:43 -0700906 return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice())
Dan Willemsenf052f782017-05-18 15:29:04 -0700907}
908
Dan Willemsen02781d52017-05-12 19:28:13 -0700909func (c *configImpl) DevicePreviousProductConfig() string {
Dan Willemsenf052f782017-05-18 15:29:04 -0700910 return filepath.Join(c.ProductOut(), "previous_build_config.mk")
911}
912
Dan Willemsenfb1271a2018-09-26 15:00:42 -0700913func (c *configImpl) KatiPackageMkDir() string {
914 return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging")
915}
916
Dan Willemsenf052f782017-05-18 15:29:04 -0700917func (c *configImpl) hostOutRoot() string {
Dan Willemsen4dc4e142017-09-08 14:35:43 -0700918 return filepath.Join(c.OutDir(), "host")
Dan Willemsenf052f782017-05-18 15:29:04 -0700919}
920
921func (c *configImpl) HostOut() string {
922 return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag())
923}
924
925// This probably needs to be multi-valued, so not exporting it for now
926func (c *configImpl) hostCrossOut() string {
927 if runtime.GOOS == "linux" {
928 return filepath.Join(c.hostOutRoot(), "windows-x86")
929 } else {
930 return ""
931 }
Dan Willemsen02781d52017-05-12 19:28:13 -0700932}
933
Dan Willemsen1e704462016-08-21 15:17:17 -0700934func (c *configImpl) HostPrebuiltTag() string {
935 if runtime.GOOS == "linux" {
936 return "linux-x86"
937 } else if runtime.GOOS == "darwin" {
938 return "darwin-x86"
939 } else {
940 panic("Unsupported OS")
941 }
942}
Dan Willemsenf173d592017-04-27 14:28:00 -0700943
Dan Willemsen8122bd52017-10-12 20:20:41 -0700944func (c *configImpl) PrebuiltBuildTool(name string) string {
Dan Willemsenf173d592017-04-27 14:28:00 -0700945 if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
946 if sanitize := strings.Fields(v); inList("address", sanitize) {
Dan Willemsen8122bd52017-10-12 20:20:41 -0700947 asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
948 if _, err := os.Stat(asan); err == nil {
949 return asan
950 }
Dan Willemsenf173d592017-04-27 14:28:00 -0700951 }
952 }
953 return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
954}
Dan Willemsen3d60b112018-04-04 22:25:56 -0700955
956func (c *configImpl) SetBuildBrokenDupRules(val bool) {
957 c.brokenDupRules = val
958}
959
960func (c *configImpl) BuildBrokenDupRules() bool {
961 return c.brokenDupRules
962}
Dan Willemsen6ab79db2018-05-02 00:06:28 -0700963
Dan Willemsen25e6f092019-04-09 10:22:43 -0700964func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) {
965 c.brokenUsesNetwork = val
966}
967
968func (c *configImpl) BuildBrokenUsesNetwork() bool {
969 return c.brokenUsesNetwork
970}
971
Dan Willemsene3336352020-01-02 19:10:38 -0800972func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) {
973 c.brokenNinjaEnvVars = val
974}
975
976func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string {
977 return c.brokenNinjaEnvVars
978}
979
Dan Willemsen6ab79db2018-05-02 00:06:28 -0700980func (c *configImpl) SetTargetDeviceDir(dir string) {
981 c.targetDeviceDir = dir
982}
983
984func (c *configImpl) TargetDeviceDir() string {
985 return c.targetDeviceDir
986}
Dan Willemsenfa42f3c2018-06-15 21:54:47 -0700987
Patrice Arruda219eef32020-06-01 17:29:30 +0000988func (c *configImpl) BuildDateTime() string {
989 return c.buildDateTime
990}
991
992func (c *configImpl) MetricsUploaderApp() string {
993 if p, ok := c.environ.Get("ANDROID_ENABLE_METRICS_UPLOAD"); ok {
994 return p
995 }
996 return ""
997}