blob: ec6670e654a8168452e907420555cb6f2a56ae51 [file] [log] [blame]
Dan Willemsen1e704462016-08-21 15:17:17 -07001// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18 "context"
Dan Willemsen051133b2017-07-14 11:29:29 -070019 "flag"
20 "fmt"
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +000021 "io/ioutil"
Dan Willemsen1e704462016-08-21 15:17:17 -070022 "os"
23 "path/filepath"
24 "strconv"
25 "strings"
Liz Kammera7541782022-02-07 13:38:52 -050026 "syscall"
Dan Willemsen1e704462016-08-21 15:17:17 -070027 "time"
28
Lukacs T. Berki7d613bf2021-03-02 10:09:41 +010029 "android/soong/shared"
Dan Willemsen1e704462016-08-21 15:17:17 -070030 "android/soong/ui/build"
31 "android/soong/ui/logger"
Nan Zhang17f27672018-12-12 16:01:49 -080032 "android/soong/ui/metrics"
Lukacs T. Berkif656b842021-08-11 11:10:28 +020033 "android/soong/ui/signal"
Dan Willemsenb82471a2018-05-17 16:37:09 -070034 "android/soong/ui/status"
35 "android/soong/ui/terminal"
Dan Willemsend9f6fa22016-08-21 15:17:17 -070036 "android/soong/ui/tracer"
Dan Willemsen1e704462016-08-21 15:17:17 -070037)
38
Patrice Arrudaa5c25422019-04-09 18:49:49 -070039// A command represents an operation to be executed in the soong build
40// system.
41type command struct {
Patrice Arrudaf445ba12020-07-28 17:49:01 +000042 // The flag name (must have double dashes).
Patrice Arrudaa5c25422019-04-09 18:49:49 -070043 flag string
44
Patrice Arrudaf445ba12020-07-28 17:49:01 +000045 // Description for the flag (to display when running help).
Patrice Arrudaa5c25422019-04-09 18:49:49 -070046 description string
47
Patrice Arrudaf445ba12020-07-28 17:49:01 +000048 // Stream the build status output into the simple terminal mode.
49 simpleOutput bool
Colin Crossc0b9f6b2019-09-23 12:44:54 -070050
51 // Sets a prefix string to use for filenames of log files.
52 logsPrefix string
53
Patrice Arrudaa5c25422019-04-09 18:49:49 -070054 // Creates the build configuration based on the args and build context.
55 config func(ctx build.Context, args ...string) build.Config
56
57 // Returns what type of IO redirection this Command requires.
58 stdio func() terminal.StdioInterface
59
60 // run the command
MarkDacek6614d9c2022-12-07 21:57:38 +000061 run func(ctx build.Context, config build.Config, args []string)
Patrice Arrudaa5c25422019-04-09 18:49:49 -070062}
63
Patrice Arrudaa5c25422019-04-09 18:49:49 -070064// list of supported commands (flags) supported by soong ui
Usta6feae382021-12-13 12:31:50 -050065var commands = []command{
Patrice Arrudaa5c25422019-04-09 18:49:49 -070066 {
Anton Hansson5a7861a2021-06-04 10:09:01 +010067 flag: "--make-mode",
Patrice Arrudaa5c25422019-04-09 18:49:49 -070068 description: "build the modules by the target name (i.e. soong_docs)",
Usta Shrestha59417a12022-08-05 17:14:49 -040069 config: build.NewConfig,
70 stdio: stdio,
71 run: runMake,
Patrice Arrudaa5c25422019-04-09 18:49:49 -070072 }, {
Patrice Arrudaf445ba12020-07-28 17:49:01 +000073 flag: "--dumpvar-mode",
74 description: "print the value of the legacy make variable VAR to stdout",
75 simpleOutput: true,
76 logsPrefix: "dumpvars-",
77 config: dumpVarConfig,
78 stdio: customStdio,
79 run: dumpVar,
Patrice Arrudaa5c25422019-04-09 18:49:49 -070080 }, {
Patrice Arrudaf445ba12020-07-28 17:49:01 +000081 flag: "--dumpvars-mode",
82 description: "dump the values of one or more legacy make variables, in shell syntax",
83 simpleOutput: true,
84 logsPrefix: "dumpvars-",
85 config: dumpVarConfig,
86 stdio: customStdio,
87 run: dumpVars,
Patrice Arrudab7b22822019-05-21 17:46:23 -070088 }, {
89 flag: "--build-mode",
90 description: "build modules based on the specified build action",
91 config: buildActionConfig,
92 stdio: stdio,
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +010093 run: runMake,
MarkDacek6614d9c2022-12-07 21:57:38 +000094 }, {
95 flag: "--upload-metrics-only",
96 description: "upload metrics without building anything",
97 config: uploadOnlyConfig,
98 stdio: stdio,
99 // Upload-only mode mostly skips to the metrics-uploading phase of soong_ui.
100 // However, this invocation marks the true "end of the build", and thus we
101 // need to update the total runtime of the build to include this upload step.
102 run: updateTotalRealTime,
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700103 },
104}
105
106// indexList returns the index of first found s. -1 is return if s is not
107// found.
Dan Willemsen1e704462016-08-21 15:17:17 -0700108func indexList(s string, list []string) int {
109 for i, l := range list {
110 if l == s {
111 return i
112 }
113 }
Dan Willemsen1e704462016-08-21 15:17:17 -0700114 return -1
115}
116
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700117// inList returns true if one or more of s is in the list.
Dan Willemsen1e704462016-08-21 15:17:17 -0700118func inList(s string, list []string) bool {
119 return indexList(s, list) != -1
120}
121
Jason Wucc166a72022-12-19 11:53:12 -0500122func deleteStaleMetrics(metricsFilePathSlice []string) error {
123 for _, metricsFilePath := range metricsFilePathSlice {
124 if err := os.Remove(metricsFilePath); err != nil && !os.IsNotExist(err) {
125 return fmt.Errorf("Failed to remove %s\nError message: %w", metricsFilePath, err)
126 }
127 }
128 return nil
129}
130
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700131// Main execution of soong_ui. The command format is as follows:
132//
Usta Shrestha59417a12022-08-05 17:14:49 -0400133// soong_ui <command> [<arg 1> <arg 2> ... <arg n>]
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700134//
135// Command is the type of soong_ui execution. Only one type of
136// execution is specified. The args are specific to the command.
Dan Willemsen1e704462016-08-21 15:17:17 -0700137func main() {
Lukacs T. Berki7d613bf2021-03-02 10:09:41 +0100138 shared.ReexecWithDelveMaybe(os.Getenv("SOONG_UI_DELVE"), shared.ResolveDelveBinary())
139
Patrice Arruda73c790f2020-07-13 23:01:18 +0000140 buildStarted := time.Now()
Patrice Arruda219eef32020-06-01 17:29:30 +0000141
Liz Kammer0e7993e2020-10-15 11:07:13 -0700142 c, args, err := getCommand(os.Args)
143 if err != nil {
144 fmt.Fprintf(os.Stderr, "Error parsing `soong` args: %s.\n", err)
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700145 os.Exit(1)
Dan Willemsenc35b3812018-07-16 19:59:10 -0700146 }
147
Jaewoong Jung9f98d3f2020-11-17 18:20:14 -0800148 // Create a terminal output that mimics Ninja's.
Patrice Arrudaf445ba12020-07-28 17:49:01 +0000149 output := terminal.NewStatusOutput(c.stdio().Stdout(), os.Getenv("NINJA_STATUS"), c.simpleOutput,
Colin Cross3c0fe0e2021-02-10 13:11:18 -0800150 build.OsEnvironment().IsEnvTrue("ANDROID_QUIET_BUILD"),
151 build.OsEnvironment().IsEnvTrue("SOONG_UI_ANSI_OUTPUT"))
Dan Willemsenb82471a2018-05-17 16:37:09 -0700152
Liz Kammerf2a80c62022-10-21 10:42:35 -0400153 // Create and start a new metric record.
154 met := metrics.New()
155 met.SetBuildDateTime(buildStarted)
156 met.SetBuildCommand(os.Args)
157
Jaewoong Jung9f98d3f2020-11-17 18:20:14 -0800158 // Attach a new logger instance to the terminal output.
Liz Kammerf2a80c62022-10-21 10:42:35 -0400159 log := logger.NewWithMetrics(output, met)
Dan Willemsen1e704462016-08-21 15:17:17 -0700160 defer log.Cleanup()
161
Jaewoong Jung9f98d3f2020-11-17 18:20:14 -0800162 // Create a context to simplify the program termination process.
Dan Willemsen1e704462016-08-21 15:17:17 -0700163 ctx, cancel := context.WithCancel(context.Background())
164 defer cancel()
165
Jaewoong Jung9f98d3f2020-11-17 18:20:14 -0800166 // Create a new trace file writer, making it log events to the log instance.
Dan Willemsend9f6fa22016-08-21 15:17:17 -0700167 trace := tracer.New(log)
168 defer trace.Close()
Dan Willemsen1e704462016-08-21 15:17:17 -0700169
Jaewoong Jung9f98d3f2020-11-17 18:20:14 -0800170 // Create a new Status instance, which manages action counts and event output channels.
Dan Willemsenb82471a2018-05-17 16:37:09 -0700171 stat := &status.Status{}
172 defer stat.Finish()
Jaewoong Jung9f98d3f2020-11-17 18:20:14 -0800173 // Hook up the terminal output and tracer to Status.
Colin Crosse0df1a32019-06-09 19:40:08 -0700174 stat.AddOutput(output)
Dan Willemsenb82471a2018-05-17 16:37:09 -0700175 stat.AddOutput(trace.StatusTracer())
176
Jaewoong Jung9f98d3f2020-11-17 18:20:14 -0800177 // Set up a cleanup procedure in case the normal termination process doesn't work.
Lukacs T. Berkif656b842021-08-11 11:10:28 +0200178 signal.SetupSignals(log, cancel, func() {
Dan Willemsend9f6fa22016-08-21 15:17:17 -0700179 trace.Close()
180 log.Cleanup()
Dan Willemsenb82471a2018-05-17 16:37:09 -0700181 stat.Finish()
Dan Willemsend9f6fa22016-08-21 15:17:17 -0700182 })
183
Dan Willemsen59339a22018-07-22 21:18:45 -0700184 buildCtx := build.Context{ContextImpl: &build.ContextImpl{
Dan Willemsenb82471a2018-05-17 16:37:09 -0700185 Context: ctx,
186 Logger: log,
Nan Zhang17f27672018-12-12 16:01:49 -0800187 Metrics: met,
Dan Willemsenb82471a2018-05-17 16:37:09 -0700188 Tracer: trace,
Colin Crosse0df1a32019-06-09 19:40:08 -0700189 Writer: output,
Dan Willemsenb82471a2018-05-17 16:37:09 -0700190 Status: stat,
Dan Willemsend9f6fa22016-08-21 15:17:17 -0700191 }}
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700192
Kousik Kumar7b7dca42022-01-14 00:22:32 -0500193 config := c.config(buildCtx, args...)
MarkDacek6614d9c2022-12-07 21:57:38 +0000194 config.SetLogsPrefix(c.logsPrefix)
195 logsDir := config.LogsDir()
196 buildStarted = config.BuildStartedTimeOrDefault(buildStarted)
Kousik Kumar7b7dca42022-01-14 00:22:32 -0500197
MarkDacek6614d9c2022-12-07 21:57:38 +0000198 buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
199 soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
200 rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
201 bp2buildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bp2build_metrics.pb")
MarkDacek00e31522023-01-06 22:15:24 +0000202 bazelMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bazel_metrics.pb")
203
204 //the profile file generated by Bazel"
205 bazelProfileFile := filepath.Join(logsDir, c.logsPrefix+"analyzed_bazel_profile.txt")
MarkDacek6614d9c2022-12-07 21:57:38 +0000206 metricsFiles := []string{
207 buildErrorFile, // build error strings
208 rbeMetricsFile, // high level metrics related to remote build execution.
209 bp2buildMetricsFile, // high level metrics related to bp2build.
210 soongMetricsFile, // high level metrics related to this build system.
MarkDacek00e31522023-01-06 22:15:24 +0000211 bazelMetricsFile, // high level metrics related to bazel execution
MarkDacek6614d9c2022-12-07 21:57:38 +0000212 config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
213 }
214
215 os.MkdirAll(logsDir, 0777)
216
217 log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log"))
218
219 trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace"))
220
221 c.run(buildCtx, config, args)
222
MarkDacek6614d9c2022-12-07 21:57:38 +0000223 if !config.SkipMetricsUpload() {
MarkDacek00e31522023-01-06 22:15:24 +0000224 defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, bazelProfileFile, bazelMetricsFile, metricsFiles...)
MarkDacek6614d9c2022-12-07 21:57:38 +0000225 }
Jason Wu7796b112023-02-07 19:05:55 -0500226 defer met.Dump(soongMetricsFile)
MarkDacek6614d9c2022-12-07 21:57:38 +0000227
228}
229
230func logAndSymlinkSetup(buildCtx build.Context, config build.Config) {
231 log := buildCtx.ContextImpl.Logger
232 logsPrefix := config.GetLogsPrefix()
Dan Willemsend9f6fa22016-08-21 15:17:17 -0700233 build.SetupOutDir(buildCtx, config)
Patrice Arruda83842d72020-12-08 19:42:08 +0000234 logsDir := config.LogsDir()
Dan Willemsen1e704462016-08-21 15:17:17 -0700235
Patrice Arruda40564022020-12-10 00:42:58 +0000236 // Common list of metric file definition.
MarkDacek6614d9c2022-12-07 21:57:38 +0000237 buildErrorFile := filepath.Join(logsDir, logsPrefix+"build_error")
238 rbeMetricsFile := filepath.Join(logsDir, logsPrefix+"rbe_metrics.pb")
239 soongMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_metrics")
240 bp2buildMetricsFile := filepath.Join(logsDir, logsPrefix+"bp2build_metrics.pb")
241 soongBuildMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_build_metrics.pb")
Patrice Arruda40564022020-12-10 00:42:58 +0000242
Jason Wucc166a72022-12-19 11:53:12 -0500243 //Delete the stale metrics files
244 staleFileSlice := []string{buildErrorFile, rbeMetricsFile, soongMetricsFile, bp2buildMetricsFile, soongBuildMetricsFile}
245 if err := deleteStaleMetrics(staleFileSlice); err != nil {
246 log.Fatalln(err)
247 }
248
Kousik Kumara0a44a82020-10-08 02:33:29 -0400249 build.PrintOutDirWarning(buildCtx, config)
Patrice Arruda219eef32020-06-01 17:29:30 +0000250
MarkDacek6614d9c2022-12-07 21:57:38 +0000251 stat := buildCtx.Status
252 stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, logsPrefix+"verbose.log")))
253 stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, logsPrefix+"error.log")))
Patrice Arruda219eef32020-06-01 17:29:30 +0000254 stat.AddOutput(status.NewProtoErrorLog(log, buildErrorFile))
Colin Cross7b624532019-06-21 15:08:30 -0700255 stat.AddOutput(status.NewCriticalPath(log))
MarkDacek6614d9c2022-12-07 21:57:38 +0000256 stat.AddOutput(status.NewBuildProgressLog(log, filepath.Join(logsDir, logsPrefix+"build_progress.pb")))
Dan Willemsenb82471a2018-05-17 16:37:09 -0700257
Colin Cross8b8bec32019-11-15 13:18:43 -0800258 buildCtx.Verbosef("Detected %.3v GB total RAM", float32(config.TotalRAM())/(1024*1024*1024))
259 buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v",
260 config.Parallel(), config.RemoteParallel(), config.HighmemParallel())
261
Liz Kammer4ae119c2022-02-09 10:54:05 -0500262 setMaxFiles(buildCtx)
Liz Kammera7541782022-02-07 13:38:52 -0500263
MarkDacek6614d9c2022-12-07 21:57:38 +0000264 defer build.CheckProdCreds(buildCtx, config)
Nan Zhangd50f53b2019-01-07 20:26:51 -0800265
Jaewoong Jung9f98d3f2020-11-17 18:20:14 -0800266 // Read the time at the starting point.
Dan Willemsen1e704462016-08-21 15:17:17 -0700267 if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
Jaewoong Jung9f98d3f2020-11-17 18:20:14 -0800268 // soong_ui.bash uses the date command's %N (nanosec) flag when getting the start time,
269 // which Darwin doesn't support. Check if it was executed properly before parsing the value.
Dan Willemsen1e704462016-08-21 15:17:17 -0700270 if !strings.HasSuffix(start, "N") {
271 if start_time, err := strconv.ParseUint(start, 10, 64); err == nil {
272 log.Verbosef("Took %dms to start up.",
273 time.Since(time.Unix(0, int64(start_time))).Nanoseconds()/time.Millisecond.Nanoseconds())
Nan Zhang17f27672018-12-12 16:01:49 -0800274 buildCtx.CompleteTrace(metrics.RunSetupTool, "startup", start_time, uint64(time.Now().UnixNano()))
Dan Willemsen1e704462016-08-21 15:17:17 -0700275 }
276 }
Dan Willemsencae59bc2017-07-13 14:27:31 -0700277
278 if executable, err := os.Executable(); err == nil {
MarkDacek6614d9c2022-12-07 21:57:38 +0000279 buildCtx.ContextImpl.Tracer.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace"))
Dan Willemsencae59bc2017-07-13 14:27:31 -0700280 }
Dan Willemsen1e704462016-08-21 15:17:17 -0700281 }
282
Dan Willemsen6b783c82019-03-08 11:42:28 -0800283 // Fix up the source tree due to a repo bug where it doesn't remove
284 // linkfiles that have been removed
285 fixBadDanglingLink(buildCtx, "hardware/qcom/sdm710/Android.bp")
286 fixBadDanglingLink(buildCtx, "hardware/qcom/sdm710/Android.mk")
287
Jaewoong Jung9f98d3f2020-11-17 18:20:14 -0800288 // Create a source finder.
Jeff Gastonb64fc1c2017-08-04 12:30:12 -0700289 f := build.NewSourceFinder(buildCtx, config)
290 defer f.Shutdown()
291 build.FindSources(buildCtx, config, f)
Dan Willemsen051133b2017-07-14 11:29:29 -0700292}
293
Dan Willemsen6b783c82019-03-08 11:42:28 -0800294func fixBadDanglingLink(ctx build.Context, name string) {
295 _, err := os.Lstat(name)
296 if err != nil {
297 return
298 }
299 _, err = os.Stat(name)
300 if os.IsNotExist(err) {
301 err = os.Remove(name)
302 if err != nil {
303 ctx.Fatalf("Failed to remove dangling link %q: %v", name, err)
304 }
305 }
306}
307
MarkDacek6614d9c2022-12-07 21:57:38 +0000308func dumpVar(ctx build.Context, config build.Config, args []string) {
309 logAndSymlinkSetup(ctx, config)
Dan Willemsen051133b2017-07-14 11:29:29 -0700310 flags := flag.NewFlagSet("dumpvar", flag.ExitOnError)
Usta Shrestha675564d2022-08-09 18:03:23 -0400311 flags.SetOutput(ctx.Writer)
312
Dan Willemsen051133b2017-07-14 11:29:29 -0700313 flags.Usage = func() {
Patrice Arrudadb4c2f12019-06-17 17:27:09 -0700314 fmt.Fprintf(ctx.Writer, "usage: %s --dumpvar-mode [--abs] <VAR>\n\n", os.Args[0])
315 fmt.Fprintln(ctx.Writer, "In dumpvar mode, print the value of the legacy make variable VAR to stdout")
316 fmt.Fprintln(ctx.Writer, "")
Dan Willemsen051133b2017-07-14 11:29:29 -0700317
Patrice Arrudadb4c2f12019-06-17 17:27:09 -0700318 fmt.Fprintln(ctx.Writer, "'report_config' is a special case that prints the human-readable config banner")
319 fmt.Fprintln(ctx.Writer, "from the beginning of the build.")
320 fmt.Fprintln(ctx.Writer, "")
Dan Willemsen051133b2017-07-14 11:29:29 -0700321 flags.PrintDefaults()
322 }
323 abs := flags.Bool("abs", false, "Print the absolute path of the value")
324 flags.Parse(args)
325
326 if flags.NArg() != 1 {
327 flags.Usage()
Liz Kammerf2a80c62022-10-21 10:42:35 -0400328 ctx.Fatalf("Invalid usage")
Dan Willemsen051133b2017-07-14 11:29:29 -0700329 }
330
331 varName := flags.Arg(0)
332 if varName == "report_config" {
333 varData, err := build.DumpMakeVars(ctx, config, nil, build.BannerVars)
334 if err != nil {
335 ctx.Fatal(err)
336 }
337
338 fmt.Println(build.Banner(varData))
339 } else {
340 varData, err := build.DumpMakeVars(ctx, config, nil, []string{varName})
341 if err != nil {
342 ctx.Fatal(err)
343 }
344
345 if *abs {
346 var res []string
347 for _, path := range strings.Fields(varData[varName]) {
348 if abs, err := filepath.Abs(path); err == nil {
349 res = append(res, abs)
350 } else {
351 ctx.Fatalln("Failed to get absolute path of", path, err)
352 }
353 }
354 fmt.Println(strings.Join(res, " "))
355 } else {
356 fmt.Println(varData[varName])
357 }
358 }
359}
360
MarkDacek6614d9c2022-12-07 21:57:38 +0000361func dumpVars(ctx build.Context, config build.Config, args []string) {
362 logAndSymlinkSetup(ctx, config)
363
Dan Willemsen051133b2017-07-14 11:29:29 -0700364 flags := flag.NewFlagSet("dumpvars", flag.ExitOnError)
Usta Shrestha675564d2022-08-09 18:03:23 -0400365 flags.SetOutput(ctx.Writer)
366
Dan Willemsen051133b2017-07-14 11:29:29 -0700367 flags.Usage = func() {
Patrice Arrudadb4c2f12019-06-17 17:27:09 -0700368 fmt.Fprintf(ctx.Writer, "usage: %s --dumpvars-mode [--vars=\"VAR VAR ...\"]\n\n", os.Args[0])
369 fmt.Fprintln(ctx.Writer, "In dumpvars mode, dump the values of one or more legacy make variables, in")
370 fmt.Fprintln(ctx.Writer, "shell syntax. The resulting output may be sourced directly into a shell to")
371 fmt.Fprintln(ctx.Writer, "set corresponding shell variables.")
372 fmt.Fprintln(ctx.Writer, "")
Dan Willemsen051133b2017-07-14 11:29:29 -0700373
Patrice Arrudadb4c2f12019-06-17 17:27:09 -0700374 fmt.Fprintln(ctx.Writer, "'report_config' is a special case that dumps a variable containing the")
375 fmt.Fprintln(ctx.Writer, "human-readable config banner from the beginning of the build.")
376 fmt.Fprintln(ctx.Writer, "")
Dan Willemsen051133b2017-07-14 11:29:29 -0700377 flags.PrintDefaults()
378 }
379
380 varsStr := flags.String("vars", "", "Space-separated list of variables to dump")
381 absVarsStr := flags.String("abs-vars", "", "Space-separated list of variables to dump (using absolute paths)")
382
383 varPrefix := flags.String("var-prefix", "", "String to prepend to all variable names when dumping")
384 absVarPrefix := flags.String("abs-var-prefix", "", "String to prepent to all absolute path variable names when dumping")
385
386 flags.Parse(args)
387
388 if flags.NArg() != 0 {
389 flags.Usage()
Liz Kammerf2a80c62022-10-21 10:42:35 -0400390 ctx.Fatalf("Invalid usage")
Dan Willemsen051133b2017-07-14 11:29:29 -0700391 }
392
393 vars := strings.Fields(*varsStr)
394 absVars := strings.Fields(*absVarsStr)
395
396 allVars := append([]string{}, vars...)
397 allVars = append(allVars, absVars...)
398
399 if i := indexList("report_config", allVars); i != -1 {
400 allVars = append(allVars[:i], allVars[i+1:]...)
401 allVars = append(allVars, build.BannerVars...)
402 }
403
404 if len(allVars) == 0 {
405 return
406 }
407
408 varData, err := build.DumpMakeVars(ctx, config, nil, allVars)
409 if err != nil {
410 ctx.Fatal(err)
411 }
412
413 for _, name := range vars {
414 if name == "report_config" {
415 fmt.Printf("%sreport_config='%s'\n", *varPrefix, build.Banner(varData))
416 } else {
417 fmt.Printf("%s%s='%s'\n", *varPrefix, name, varData[name])
418 }
419 }
420 for _, name := range absVars {
421 var res []string
422 for _, path := range strings.Fields(varData[name]) {
423 abs, err := filepath.Abs(path)
424 if err != nil {
425 ctx.Fatalln("Failed to get absolute path of", path, err)
426 }
427 res = append(res, abs)
428 }
429 fmt.Printf("%s%s='%s'\n", *absVarPrefix, name, strings.Join(res, " "))
430 }
Dan Willemsen1e704462016-08-21 15:17:17 -0700431}
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700432
Patrice Arrudab7b22822019-05-21 17:46:23 -0700433func stdio() terminal.StdioInterface {
434 return terminal.StdioImpl{}
435}
436
Jaewoong Jung9f98d3f2020-11-17 18:20:14 -0800437// dumpvar and dumpvars use stdout to output variable values, so use stderr instead of stdout when
438// reporting events to keep stdout clean from noise.
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700439func customStdio() terminal.StdioInterface {
440 return terminal.NewCustomStdio(os.Stdin, os.Stderr, os.Stderr)
441}
442
443// dumpVarConfig does not require any arguments to be parsed by the NewConfig.
444func dumpVarConfig(ctx build.Context, args ...string) build.Config {
445 return build.NewConfig(ctx)
446}
447
MarkDacek6614d9c2022-12-07 21:57:38 +0000448// uploadOnlyConfig explicitly requires no arguments.
449func uploadOnlyConfig(ctx build.Context, args ...string) build.Config {
450 if len(args) > 0 {
451 fmt.Printf("--upload-only does not require arguments.")
452 }
453 return build.UploadOnlyConfig(ctx)
454}
455
Patrice Arrudab7b22822019-05-21 17:46:23 -0700456func buildActionConfig(ctx build.Context, args ...string) build.Config {
457 flags := flag.NewFlagSet("build-mode", flag.ContinueOnError)
Usta Shrestha675564d2022-08-09 18:03:23 -0400458 flags.SetOutput(ctx.Writer)
459
Patrice Arrudab7b22822019-05-21 17:46:23 -0700460 flags.Usage = func() {
461 fmt.Fprintf(ctx.Writer, "usage: %s --build-mode --dir=<path> <build action> [<build arg 1> <build arg 2> ...]\n\n", os.Args[0])
462 fmt.Fprintln(ctx.Writer, "In build mode, build the set of modules based on the specified build")
463 fmt.Fprintln(ctx.Writer, "action. The --dir flag is required to determine what is needed to")
464 fmt.Fprintln(ctx.Writer, "build in the source tree based on the build action. See below for")
465 fmt.Fprintln(ctx.Writer, "the list of acceptable build action flags.")
466 fmt.Fprintln(ctx.Writer, "")
467 flags.PrintDefaults()
468 }
469
470 buildActionFlags := []struct {
Dan Willemsence41e942019-07-29 23:39:30 -0700471 name string
472 description string
473 action build.BuildAction
474 set bool
Patrice Arrudab7b22822019-05-21 17:46:23 -0700475 }{{
Dan Willemsence41e942019-07-29 23:39:30 -0700476 name: "all-modules",
477 description: "Build action: build from the top of the source tree.",
478 action: build.BUILD_MODULES,
Patrice Arrudab7b22822019-05-21 17:46:23 -0700479 }, {
Dan Willemsence41e942019-07-29 23:39:30 -0700480 // This is redirecting to mma build command behaviour. Once it has soaked for a
481 // while, the build command is deleted from here once it has been removed from the
482 // envsetup.sh.
483 name: "modules-in-a-dir-no-deps",
484 description: "Build action: builds all of the modules in the current directory without their dependencies.",
485 action: build.BUILD_MODULES_IN_A_DIRECTORY,
Patrice Arrudab7b22822019-05-21 17:46:23 -0700486 }, {
Dan Willemsence41e942019-07-29 23:39:30 -0700487 // This is redirecting to mmma build command behaviour. Once it has soaked for a
488 // while, the build command is deleted from here once it has been removed from the
489 // envsetup.sh.
490 name: "modules-in-dirs-no-deps",
491 description: "Build action: builds all of the modules in the supplied directories without their dependencies.",
492 action: build.BUILD_MODULES_IN_DIRECTORIES,
Patrice Arrudab7b22822019-05-21 17:46:23 -0700493 }, {
Dan Willemsence41e942019-07-29 23:39:30 -0700494 name: "modules-in-a-dir",
495 description: "Build action: builds all of the modules in the current directory and their dependencies.",
496 action: build.BUILD_MODULES_IN_A_DIRECTORY,
Patrice Arrudab7b22822019-05-21 17:46:23 -0700497 }, {
Dan Willemsence41e942019-07-29 23:39:30 -0700498 name: "modules-in-dirs",
499 description: "Build action: builds all of the modules in the supplied directories and their dependencies.",
500 action: build.BUILD_MODULES_IN_DIRECTORIES,
Patrice Arrudab7b22822019-05-21 17:46:23 -0700501 }}
502 for i, flag := range buildActionFlags {
503 flags.BoolVar(&buildActionFlags[i].set, flag.name, false, flag.description)
504 }
505 dir := flags.String("dir", "", "Directory of the executed build command.")
506
507 // Only interested in the first two args which defines the build action and the directory.
508 // The remaining arguments are passed down to the config.
509 const numBuildActionFlags = 2
510 if len(args) < numBuildActionFlags {
511 flags.Usage()
Usta Shrestha675564d2022-08-09 18:03:23 -0400512 ctx.Fatalln("Improper build action arguments: too few arguments")
Patrice Arrudab7b22822019-05-21 17:46:23 -0700513 }
Usta Shrestha675564d2022-08-09 18:03:23 -0400514 parseError := flags.Parse(args[0:numBuildActionFlags])
Patrice Arrudab7b22822019-05-21 17:46:23 -0700515
516 // The next block of code is to validate that exactly one build action is set and the dir flag
517 // is specified.
Usta Shrestha675564d2022-08-09 18:03:23 -0400518 buildActionFound := false
Patrice Arrudab7b22822019-05-21 17:46:23 -0700519 var buildAction build.BuildAction
Usta Shrestha675564d2022-08-09 18:03:23 -0400520 for _, f := range buildActionFlags {
521 if f.set {
522 if buildActionFound {
523 if parseError == nil {
524 //otherwise Parse() already called Usage()
525 flags.Usage()
526 }
527 ctx.Fatalf("Build action already specified, omit: --%s\n", f.name)
528 }
529 buildActionFound = true
530 buildAction = f.action
Patrice Arrudab7b22822019-05-21 17:46:23 -0700531 }
532 }
Usta Shrestha675564d2022-08-09 18:03:23 -0400533 if !buildActionFound {
534 if parseError == nil {
535 //otherwise Parse() already called Usage()
536 flags.Usage()
537 }
Patrice Arrudab7b22822019-05-21 17:46:23 -0700538 ctx.Fatalln("Build action not defined.")
539 }
540 if *dir == "" {
541 ctx.Fatalln("-dir not specified.")
542 }
543
544 // Remove the build action flags from the args as they are not recognized by the config.
545 args = args[numBuildActionFlags:]
Dan Willemsence41e942019-07-29 23:39:30 -0700546 return build.NewBuildActionConfig(buildAction, *dir, ctx, args...)
Patrice Arrudab7b22822019-05-21 17:46:23 -0700547}
548
MarkDacek6614d9c2022-12-07 21:57:38 +0000549func runMake(ctx build.Context, config build.Config, _ []string) {
550 logAndSymlinkSetup(ctx, config)
551 logsDir := config.LogsDir()
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700552 if config.IsVerbose() {
553 writer := ctx.Writer
Colin Cross097ed2a2019-06-08 21:48:58 -0700554 fmt.Fprintln(writer, "! The argument `showcommands` is no longer supported.")
555 fmt.Fprintln(writer, "! Instead, the verbose log is always written to a compressed file in the output dir:")
556 fmt.Fprintln(writer, "!")
557 fmt.Fprintf(writer, "! gzip -cd %s/verbose.log.gz | less -R\n", logsDir)
558 fmt.Fprintln(writer, "!")
559 fmt.Fprintln(writer, "! Older versions are saved in verbose.log.#.gz files")
560 fmt.Fprintln(writer, "")
Liz Kammerf2a80c62022-10-21 10:42:35 -0400561 ctx.Fatal("Invalid argument")
Dan Willemsenc6360832019-07-25 14:07:36 -0700562 }
563
564 if _, ok := config.Environment().Get("ONE_SHOT_MAKEFILE"); ok {
565 writer := ctx.Writer
Dan Willemsence41e942019-07-29 23:39:30 -0700566 fmt.Fprintln(writer, "! The variable `ONE_SHOT_MAKEFILE` is obsolete.")
Dan Willemsenc6360832019-07-25 14:07:36 -0700567 fmt.Fprintln(writer, "!")
568 fmt.Fprintln(writer, "! If you're using `mm`, you'll need to run `source build/envsetup.sh` to update.")
569 fmt.Fprintln(writer, "!")
570 fmt.Fprintln(writer, "! Otherwise, either specify a module name with m, or use mma / MODULES-IN-...")
571 fmt.Fprintln(writer, "")
Liz Kammerf2a80c62022-10-21 10:42:35 -0400572 ctx.Fatal("Invalid environment")
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700573 }
574
Anton Hansson5a7861a2021-06-04 10:09:01 +0100575 build.Build(ctx, config)
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700576}
577
578// getCommand finds the appropriate command based on args[1] flag. args[0]
579// is the soong_ui filename.
Liz Kammer0e7993e2020-10-15 11:07:13 -0700580func getCommand(args []string) (*command, []string, error) {
Usta Shrestha675564d2022-08-09 18:03:23 -0400581 listFlags := func() []string {
582 flags := make([]string, len(commands))
583 for i, c := range commands {
584 flags[i] = c.flag
585 }
586 return flags
587 }
588
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700589 if len(args) < 2 {
Usta Shrestha675564d2022-08-09 18:03:23 -0400590 return nil, nil, fmt.Errorf("Too few arguments: %q\nUse one of these: %q", args, listFlags())
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700591 }
592
593 for _, c := range commands {
594 if c.flag == args[1] {
Liz Kammer0e7993e2020-10-15 11:07:13 -0700595 return &c, args[2:], nil
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700596 }
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700597 }
Usta Shrestha675564d2022-08-09 18:03:23 -0400598 return nil, nil, fmt.Errorf("Command not found: %q\nDid you mean one of these: %q", args[1], listFlags())
Patrice Arrudaa5c25422019-04-09 18:49:49 -0700599}
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +0000600
601// For Bazel support, this moves files and directories from e.g. out/dist/$f to DIST_DIR/$f if necessary.
602func populateExternalDistDir(ctx build.Context, config build.Config) {
603 // Make sure that internalDistDirPath and externalDistDirPath are both absolute paths, so we can compare them
604 var err error
605 var internalDistDirPath string
606 var externalDistDirPath string
607 if internalDistDirPath, err = filepath.Abs(config.DistDir()); err != nil {
608 ctx.Fatalf("Unable to find absolute path of %s: %s", internalDistDirPath, err)
609 }
610 if externalDistDirPath, err = filepath.Abs(config.RealDistDir()); err != nil {
611 ctx.Fatalf("Unable to find absolute path of %s: %s", externalDistDirPath, err)
612 }
613 if externalDistDirPath == internalDistDirPath {
614 return
615 }
616
Rupert Shuttleworth534f1572020-12-16 23:07:06 +0000617 // Make sure the internal DIST_DIR actually exists before trying to read from it
618 if _, err = os.Stat(internalDistDirPath); os.IsNotExist(err) {
619 ctx.Println("Skipping Bazel dist dir migration - nothing to do!")
620 return
621 }
622
Rupert Shuttleworth3c9f5ac2020-12-10 11:32:38 +0000623 // Make sure the external DIST_DIR actually exists before trying to write to it
624 if err = os.MkdirAll(externalDistDirPath, 0755); err != nil {
625 ctx.Fatalf("Unable to make directory %s: %s", externalDistDirPath, err)
626 }
627
628 ctx.Println("Populating external DIST_DIR...")
629
630 populateExternalDistDirHelper(ctx, config, internalDistDirPath, externalDistDirPath)
631}
632
633func populateExternalDistDirHelper(ctx build.Context, config build.Config, internalDistDirPath string, externalDistDirPath string) {
634 files, err := ioutil.ReadDir(internalDistDirPath)
635 if err != nil {
636 ctx.Fatalf("Can't read internal distdir %s: %s", internalDistDirPath, err)
637 }
638 for _, f := range files {
639 internalFilePath := filepath.Join(internalDistDirPath, f.Name())
640 externalFilePath := filepath.Join(externalDistDirPath, f.Name())
641
642 if f.IsDir() {
643 // Moving a directory - check if there is an existing directory to merge with
644 externalLstat, err := os.Lstat(externalFilePath)
645 if err != nil {
646 if !os.IsNotExist(err) {
647 ctx.Fatalf("Can't lstat external %s: %s", externalDistDirPath, err)
648 }
649 // Otherwise, if the error was os.IsNotExist, that's fine and we fall through to the rename at the bottom
650 } else {
651 if externalLstat.IsDir() {
652 // Existing dir - try to merge the directories?
653 populateExternalDistDirHelper(ctx, config, internalFilePath, externalFilePath)
654 continue
655 } else {
656 // Existing file being replaced with a directory. Delete the existing file...
657 if err := os.RemoveAll(externalFilePath); err != nil {
658 ctx.Fatalf("Unable to remove existing %s: %s", externalFilePath, err)
659 }
660 }
661 }
662 } else {
663 // Moving a file (not a dir) - delete any existing file or directory
664 if err := os.RemoveAll(externalFilePath); err != nil {
665 ctx.Fatalf("Unable to remove existing %s: %s", externalFilePath, err)
666 }
667 }
668
669 // The actual move - do a rename instead of a copy in order to save disk space.
670 if err := os.Rename(internalFilePath, externalFilePath); err != nil {
671 ctx.Fatalf("Unable to rename %s -> %s due to error %s", internalFilePath, externalFilePath, err)
672 }
673 }
674}
Liz Kammer4ae119c2022-02-09 10:54:05 -0500675
676func setMaxFiles(ctx build.Context) {
677 var limits syscall.Rlimit
678
679 err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limits)
680 if err != nil {
681 ctx.Println("Failed to get file limit:", err)
682 return
683 }
684
685 ctx.Verbosef("Current file limits: %d soft, %d hard", limits.Cur, limits.Max)
686 if limits.Cur == limits.Max {
687 return
688 }
689
690 limits.Cur = limits.Max
691 err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limits)
692 if err != nil {
693 ctx.Println("Failed to increase file limit:", err)
694 }
695}
MarkDacek6614d9c2022-12-07 21:57:38 +0000696
697func updateTotalRealTime(ctx build.Context, config build.Config, args []string) {
698 soongMetricsFile := filepath.Join(config.LogsDir(), "soong_metrics")
699
700 //read file into proto
701 data, err := os.ReadFile(soongMetricsFile)
702 if err != nil {
703 ctx.Fatal(err)
704 }
705 met := ctx.ContextImpl.Metrics
706
707 err = met.UpdateTotalRealTime(data)
708 if err != nil {
709 ctx.Fatal(err)
710 }
711}