blob: a4d1fb69bcc55fc3a42c2e6d141d0190195221d7 [file] [log] [blame]
Colin Cross3f40fa42015-01-30 17:27:36 -08001// Copyright 2015 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18 "flag"
19 "fmt"
Lukacs T. Berkif0b3b942021-03-23 11:46:47 +010020 "io/ioutil"
Colin Cross3f40fa42015-01-30 17:27:36 -080021 "os"
22 "path/filepath"
Jingwen Cheneb76c432021-01-28 08:22:12 -050023 "strings"
Lukacs T. Berkic99c9472021-03-24 10:50:06 +010024 "time"
Colin Cross3f40fa42015-01-30 17:27:36 -080025
Dan Willemsen66213a62021-09-21 17:50:30 -070026 "android/soong/android"
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020027 "android/soong/bp2build"
Lukacs T. Berki7690c092021-02-26 14:27:36 +010028 "android/soong/shared"
Chris Parsons715b08f2022-03-22 19:23:40 -040029 "android/soong/ui/metrics/bp2build_metrics_proto"
Lukacs T. Berkie571dc32021-08-25 14:14:13 +020030
Colin Cross70b40592015-03-23 12:57:34 -070031 "github.com/google/blueprint/bootstrap"
Lukacs T. Berkid518e1a2021-04-14 13:49:50 +020032 "github.com/google/blueprint/deptools"
Chris Parsons715b08f2022-03-22 19:23:40 -040033 "github.com/google/blueprint/metrics"
Dan Willemsen66213a62021-09-21 17:50:30 -070034 androidProtobuf "google.golang.org/protobuf/android"
Colin Cross3f40fa42015-01-30 17:27:36 -080035)
36
Colin Crosse87040b2017-12-11 15:52:26 -080037var (
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020038 topDir string
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +020039 outDir string
Lukacs T. Berkid6cee7e2021-09-01 16:25:51 +020040 soongOutDir string
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020041 availableEnvFile string
42 usedEnvFile string
43
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +020044 runGoTests bool
45
Lukacs T. Berki809d2ed2021-08-18 10:55:32 +020046 globFile string
47 globListDir string
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020048 delveListen string
49 delvePath string
50
Lukacs T. Berkie571dc32021-08-25 14:14:13 +020051 moduleGraphFile string
kgui67007242022-01-25 13:50:25 +080052 moduleActionsFile string
Jingwen Chen50f93d22020-11-05 07:42:11 -050053 docFile string
54 bazelQueryViewDir string
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020055 bp2buildMarker string
Lukacs T. Berkif9008072021-08-16 15:24:48 +020056
57 cmdlineArgs bootstrap.Args
Colin Crosse87040b2017-12-11 15:52:26 -080058)
59
60func init() {
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020061 // Flags that make sense in every mode
Lukacs T. Berki7690c092021-02-26 14:27:36 +010062 flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree")
Lukacs T. Berkid6cee7e2021-09-01 16:25:51 +020063 flag.StringVar(&soongOutDir, "soong_out", "", "Soong output directory (usually $TOP/out/soong)")
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020064 flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")
65 flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
Lukacs T. Berkib078ade2021-08-31 10:42:08 +020066 flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")
67 flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +020068 flag.StringVar(&outDir, "out", "", "the ninja builddir directory")
Lukacs T. Berkib078ade2021-08-31 10:42:08 +020069 flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020070
71 // Debug flags
Lukacs T. Berki7d613bf2021-03-02 10:09:41 +010072 flag.StringVar(&delveListen, "delve_listen", "", "Delve port to listen on for debugging")
73 flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set")
Lukacs T. Berkib078ade2021-08-31 10:42:08 +020074 flag.StringVar(&cmdlineArgs.Cpuprofile, "cpuprofile", "", "write cpu profile to file")
75 flag.StringVar(&cmdlineArgs.TraceFile, "trace", "", "write trace to file")
76 flag.StringVar(&cmdlineArgs.Memprofile, "memprofile", "", "write memory profile to file")
77 flag.BoolVar(&cmdlineArgs.NoGC, "nogc", false, "turn off GC for debugging")
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020078
79 // Flags representing various modes soong_build can run in
Lukacs T. Berkie571dc32021-08-25 14:14:13 +020080 flag.StringVar(&moduleGraphFile, "module_graph_file", "", "JSON module graph file to output")
kgui67007242022-01-25 13:50:25 +080081 flag.StringVar(&moduleActionsFile, "module_actions_file", "", "JSON file to output inputs/outputs of actions of modules")
Colin Crosse87040b2017-12-11 15:52:26 -080082 flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output")
Lukacs T. Berki47a9d0c2021-03-08 16:34:09 +010083 flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
Lukacs T. Berkif8e24282021-04-14 10:31:00 +020084 flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit")
Lukacs T. Berkif9008072021-08-16 15:24:48 +020085 flag.StringVar(&cmdlineArgs.OutFile, "o", "build.ninja", "the Ninja file to output")
Lukacs T. Berkib078ade2021-08-31 10:42:08 +020086 flag.BoolVar(&cmdlineArgs.EmptyNinjaFile, "empty-ninja-file", false, "write out a 0-byte ninja file")
Chris Parsonsef615e52022-08-18 22:04:11 -040087 flag.BoolVar(&cmdlineArgs.BazelMode, "bazel-mode", false, "use bazel for analysis of certain modules")
88 flag.BoolVar(&cmdlineArgs.BazelModeDev, "bazel-mode-dev", false, "use bazel for analysis of a large number of modules (less stable)")
Lukacs T. Berkib078ade2021-08-31 10:42:08 +020089
90 // Flags that probably shouldn't be flags of soong_build but we haven't found
91 // the time to remove them yet
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +020092 flag.BoolVar(&runGoTests, "t", false, "build and run go tests during bootstrap")
Dan Willemsen66213a62021-09-21 17:50:30 -070093
94 // Disable deterministic randomization in the protobuf package, so incremental
95 // builds with unrelated Soong changes don't trigger large rebuilds (since we
96 // write out text protos in command lines, and command line changes trigger
97 // rebuilds).
98 androidProtobuf.DisableRand()
Colin Crosse87040b2017-12-11 15:52:26 -080099}
100
Jeff Gaston088e29e2017-11-29 16:47:17 -0800101func newNameResolver(config android.Config) *android.NameResolver {
102 namespacePathsToExport := make(map[string]bool)
103
Dan Willemsen3fb1fae2018-03-12 15:30:26 -0700104 for _, namespaceName := range config.ExportedNamespaces() {
Jeff Gaston088e29e2017-11-29 16:47:17 -0800105 namespacePathsToExport[namespaceName] = true
106 }
107
108 namespacePathsToExport["."] = true // always export the root namespace
109
110 exportFilter := func(namespace *android.Namespace) bool {
111 return namespacePathsToExport[namespace.Path]
112 }
113
114 return android.NewNameResolver(exportFilter)
115}
116
Lukacs T. Berkiffc9e8d2021-09-07 17:54:38 +0200117func newContext(configuration android.Config) *android.Context {
Colin Crossae8600b2020-10-29 17:09:13 -0700118 ctx := android.NewContext(configuration)
Jingwen Chendaa54bc2020-12-14 02:58:54 -0500119 ctx.Register()
Chris Parsonsf3c96ef2020-09-29 02:23:17 -0400120 ctx.SetNameInterface(newNameResolver(configuration))
121 ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
122 return ctx
123}
124
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200125func newConfig(availableEnv map[string]string) android.Config {
Chris Parsonsad876012022-08-20 14:48:32 -0400126 var buildMode android.SoongBuildMode
127
128 if bp2buildMarker != "" {
129 buildMode = android.Bp2build
130 } else if bazelQueryViewDir != "" {
131 buildMode = android.GenerateQueryView
132 } else if moduleGraphFile != "" {
133 buildMode = android.GenerateModuleGraph
134 } else if docFile != "" {
135 buildMode = android.GenerateDocFile
Chris Parsonsef615e52022-08-18 22:04:11 -0400136 } else if cmdlineArgs.BazelModeDev {
137 buildMode = android.BazelDevMode
138 } else if cmdlineArgs.BazelMode {
139 buildMode = android.BazelProdMode
Chris Parsonsad876012022-08-20 14:48:32 -0400140 } else {
141 buildMode = android.AnalysisNoBazel
142 }
143
144 configuration, err := android.NewConfig(cmdlineArgs.ModuleListFile, buildMode, runGoTests, outDir, soongOutDir, availableEnv)
Chris Parsonsf3c96ef2020-09-29 02:23:17 -0400145 if err != nil {
146 fmt.Fprintf(os.Stderr, "%s", err)
147 os.Exit(1)
148 }
149 return configuration
150}
151
Chris Parsonsf874e462022-05-10 13:50:12 -0400152// Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a
153// BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata
154// for modules that should be handled by Bazel.
155func runMixedModeBuild(configuration android.Config, ctx *android.Context, extraNinjaDeps []string) {
156 ctx.EventHandler.Begin("mixed_build")
157 defer ctx.EventHandler.End("mixed_build")
Lukacs T. Berkid518e1a2021-04-14 13:49:50 +0200158
Chris Parsonsf874e462022-05-10 13:50:12 -0400159 bazelHook := func() error {
160 ctx.EventHandler.Begin("bazel")
161 defer ctx.EventHandler.End("bazel")
Yu Liu8d82ac52022-05-17 15:13:28 -0700162 return configuration.BazelContext.InvokeBazel(configuration)
Lukacs T. Berki6790ebc2021-04-01 17:55:58 +0200163 }
Chris Parsonsf874e462022-05-10 13:50:12 -0400164 ctx.SetBeforePrepareBuildActionsHook(bazelHook)
Chris Parsons715b08f2022-03-22 19:23:40 -0400165
Chris Parsonsf874e462022-05-10 13:50:12 -0400166 ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, ctx.Context, configuration)
Chris Parsons027881c2022-05-24 15:38:38 -0400167 ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
Lukacs T. Berki809d2ed2021-08-18 10:55:32 +0200168
Chris Parsonsf874e462022-05-10 13:50:12 -0400169 globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration)
Lukacs T. Berki809d2ed2021-08-18 10:55:32 +0200170 ninjaDeps = append(ninjaDeps, globListFiles...)
171
Chris Parsonsf874e462022-05-10 13:50:12 -0400172 writeDepFile(cmdlineArgs.OutFile, *ctx.EventHandler, ninjaDeps)
Lukacs T. Berki6790ebc2021-04-01 17:55:58 +0200173}
174
175// Run the code-generation phase to convert BazelTargetModules to BUILD files.
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200176func runQueryView(queryviewDir, queryviewMarker string, configuration android.Config, ctx *android.Context) {
Chris Parsons715b08f2022-03-22 19:23:40 -0400177 ctx.EventHandler.Begin("queryview")
178 defer ctx.EventHandler.End("queryview")
Lukacs T. Berki6790ebc2021-04-01 17:55:58 +0200179 codegenContext := bp2build.NewCodegenContext(configuration, *ctx, bp2build.QueryView)
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200180 absoluteQueryViewDir := shared.JoinPath(topDir, queryviewDir)
Lukacs T. Berki6790ebc2021-04-01 17:55:58 +0200181 if err := createBazelQueryView(codegenContext, absoluteQueryViewDir); err != nil {
182 fmt.Fprintf(os.Stderr, "%s", err)
183 os.Exit(1)
184 }
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200185
186 touch(shared.JoinPath(topDir, queryviewMarker))
Lukacs T. Berki6790ebc2021-04-01 17:55:58 +0200187}
188
Dan Willemsenccf36aa2022-04-20 23:11:43 -0700189func writeMetrics(configuration android.Config, eventHandler metrics.EventHandler, metricsDir string) {
Chris Parsons715b08f2022-03-22 19:23:40 -0400190 if len(metricsDir) < 1 {
191 fmt.Fprintf(os.Stderr, "\nMissing required env var for generating soong metrics: LOG_DIR\n")
192 os.Exit(1)
193 }
194 metricsFile := filepath.Join(metricsDir, "soong_build_metrics.pb")
195 err := android.WriteMetrics(configuration, eventHandler, metricsFile)
Lukacs T. Berki6790ebc2021-04-01 17:55:58 +0200196 if err != nil {
197 fmt.Fprintf(os.Stderr, "error writing soong_build metrics %s: %s", metricsFile, err)
198 os.Exit(1)
199 }
200}
201
kgui67007242022-01-25 13:50:25 +0800202func writeJsonModuleGraphAndActions(ctx *android.Context, graphPath string, actionsPath string) {
203 graphFile, graphErr := os.Create(shared.JoinPath(topDir, graphPath))
204 actionsFile, actionsErr := os.Create(shared.JoinPath(topDir, actionsPath))
205 if graphErr != nil || actionsErr != nil {
206 fmt.Fprintf(os.Stderr, "Graph err: %s, actions err: %s", graphErr, actionsErr)
Lukacs T. Berki97bb9f12021-04-01 18:28:45 +0200207 os.Exit(1)
208 }
209
kgui67007242022-01-25 13:50:25 +0800210 defer graphFile.Close()
211 defer actionsFile.Close()
212 ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile)
Lukacs T. Berki97bb9f12021-04-01 18:28:45 +0200213}
214
Chris Parsons715b08f2022-03-22 19:23:40 -0400215func writeBuildGlobsNinjaFile(ctx *android.Context, buildDir string, config interface{}) []string {
216 ctx.EventHandler.Begin("globs_ninja_file")
217 defer ctx.EventHandler.End("globs_ninja_file")
218
Lukacs T. Berki809d2ed2021-08-18 10:55:32 +0200219 globDir := bootstrap.GlobDirectory(buildDir, globListDir)
220 bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{
Chris Parsons715b08f2022-03-22 19:23:40 -0400221 GlobLister: ctx.Globs,
Lukacs T. Berki809d2ed2021-08-18 10:55:32 +0200222 GlobFile: globFile,
223 GlobDir: globDir,
Chris Parsons715b08f2022-03-22 19:23:40 -0400224 SrcDir: ctx.SrcDir(),
Lukacs T. Berki809d2ed2021-08-18 10:55:32 +0200225 }, config)
226 return bootstrap.GlobFileListFiles(globDir)
227}
228
Chris Parsons715b08f2022-03-22 19:23:40 -0400229func writeDepFile(outputFile string, eventHandler metrics.EventHandler, ninjaDeps []string) {
230 eventHandler.Begin("ninja_deps")
231 defer eventHandler.End("ninja_deps")
Lukacs T. Berkie571dc32021-08-25 14:14:13 +0200232 depFile := shared.JoinPath(topDir, outputFile+".d")
233 err := deptools.WriteDepFile(depFile, outputFile, ninjaDeps)
234 if err != nil {
235 fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", depFile, err)
236 os.Exit(1)
237 }
238}
239
Jingwen Chen4fabaf52021-05-25 02:40:29 +0000240// doChosenActivity runs Soong for a specific activity, like bp2build, queryview
241// or the actual Soong build for the build.ninja file. Returns the top level
242// output file of the specific activity.
ustacd245f72022-08-24 11:45:19 -0400243func doChosenActivity(ctx *android.Context, configuration android.Config, extraNinjaDeps []string) string {
Chris Parsonsad876012022-08-20 14:48:32 -0400244 if configuration.BuildMode == android.Bp2build {
Lukacs T. Berki6790ebc2021-04-01 17:55:58 +0200245 // Run the alternate pipeline of bp2build mutators and singleton to convert
246 // Blueprint to BUILD files before everything else.
247 runBp2Build(configuration, extraNinjaDeps)
Jingwen Chen4fabaf52021-05-25 02:40:29 +0000248 return bp2buildMarker
Chris Parsonsad876012022-08-20 14:48:32 -0400249 } else if configuration.IsMixedBuildsEnabled() {
Lukacs T. Berki6790ebc2021-04-01 17:55:58 +0200250 runMixedModeBuild(configuration, ctx, extraNinjaDeps)
251 } else {
Chris Parsons715b08f2022-03-22 19:23:40 -0400252 var stopBefore bootstrap.StopBefore
Chris Parsonsad876012022-08-20 14:48:32 -0400253 if configuration.BuildMode == android.GenerateModuleGraph {
Chris Parsons715b08f2022-03-22 19:23:40 -0400254 stopBefore = bootstrap.StopBeforeWriteNinja
Chris Parsonsad876012022-08-20 14:48:32 -0400255 } else if configuration.BuildMode == android.GenerateQueryView || configuration.BuildMode == android.GenerateDocFile {
Chris Parsons715b08f2022-03-22 19:23:40 -0400256 stopBefore = bootstrap.StopBeforePrepareBuildActions
257 } else {
258 stopBefore = bootstrap.DoEverything
259 }
260
Chris Parsonsad876012022-08-20 14:48:32 -0400261 ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, stopBefore, ctx.Context, configuration)
Lukacs T. Berki6124c9b2021-04-15 15:06:40 +0200262 ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
Lukacs T. Berki809d2ed2021-08-18 10:55:32 +0200263
Chris Parsons715b08f2022-03-22 19:23:40 -0400264 globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration)
Lukacs T. Berki809d2ed2021-08-18 10:55:32 +0200265 ninjaDeps = append(ninjaDeps, globListFiles...)
266
Lukacs T. Berkie571dc32021-08-25 14:14:13 +0200267 // Convert the Soong module graph into Bazel BUILD files.
Chris Parsonsad876012022-08-20 14:48:32 -0400268 if configuration.BuildMode == android.GenerateQueryView {
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200269 queryviewMarkerFile := bazelQueryViewDir + ".marker"
270 runQueryView(bazelQueryViewDir, queryviewMarkerFile, configuration, ctx)
Chris Parsons715b08f2022-03-22 19:23:40 -0400271 writeDepFile(queryviewMarkerFile, *ctx.EventHandler, ninjaDeps)
Lukacs T. Berki3a821692021-09-06 17:08:02 +0200272 return queryviewMarkerFile
Chris Parsonsad876012022-08-20 14:48:32 -0400273 } else if configuration.BuildMode == android.GenerateModuleGraph {
kgui67007242022-01-25 13:50:25 +0800274 writeJsonModuleGraphAndActions(ctx, moduleGraphFile, moduleActionsFile)
Chris Parsons715b08f2022-03-22 19:23:40 -0400275 writeDepFile(moduleGraphFile, *ctx.EventHandler, ninjaDeps)
Lukacs T. Berkie571dc32021-08-25 14:14:13 +0200276 return moduleGraphFile
Chris Parsonsad876012022-08-20 14:48:32 -0400277 } else if configuration.BuildMode == android.GenerateDocFile {
Lukacs T. Berkic6012f32021-09-06 18:31:46 +0200278 // TODO: we could make writeDocs() return the list of documentation files
279 // written and add them to the .d file. Then soong_docs would be re-run
280 // whenever one is deleted.
281 if err := writeDocs(ctx, shared.JoinPath(topDir, docFile)); err != nil {
282 fmt.Fprintf(os.Stderr, "error building Soong documentation: %s\n", err)
283 os.Exit(1)
284 }
Chris Parsons715b08f2022-03-22 19:23:40 -0400285 writeDepFile(docFile, *ctx.EventHandler, ninjaDeps)
Lukacs T. Berkic6012f32021-09-06 18:31:46 +0200286 return docFile
Lukacs T. Berkie571dc32021-08-25 14:14:13 +0200287 } else {
288 // The actual output (build.ninja) was written in the RunBlueprint() call
289 // above
Chris Parsons715b08f2022-03-22 19:23:40 -0400290 writeDepFile(cmdlineArgs.OutFile, *ctx.EventHandler, ninjaDeps)
Lukacs T. Berkid518e1a2021-04-14 13:49:50 +0200291 }
Lukacs T. Berki6790ebc2021-04-01 17:55:58 +0200292 }
293
Lukacs T. Berkif9008072021-08-16 15:24:48 +0200294 return cmdlineArgs.OutFile
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200295}
296
297// soong_ui dumps the available environment variables to
298// soong.environment.available . Then soong_build itself is run with an empty
299// environment so that the only way environment variables can be accessed is
300// using Config, which tracks access to them.
301
302// At the end of the build, a file called soong.environment.used is written
303// containing the current value of all used environment variables. The next
304// time soong_ui is run, it checks whether any environment variables that was
305// used had changed and if so, it deletes soong.environment.used to cause a
306// rebuild.
307//
308// The dependency of build.ninja on soong.environment.used is declared in
309// build.ninja.d
310func parseAvailableEnv() map[string]string {
311 if availableEnvFile == "" {
312 fmt.Fprintf(os.Stderr, "--available_env not set\n")
313 os.Exit(1)
314 }
315
316 result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile))
317 if err != nil {
318 fmt.Fprintf(os.Stderr, "error reading available environment file '%s': %s\n", availableEnvFile, err)
319 os.Exit(1)
320 }
321
322 return result
Lukacs T. Berki6790ebc2021-04-01 17:55:58 +0200323}
324
Colin Cross3f40fa42015-01-30 17:27:36 -0800325func main() {
326 flag.Parse()
327
Lukacs T. Berki7d613bf2021-03-02 10:09:41 +0100328 shared.ReexecWithDelveMaybe(delveListen, delvePath)
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100329 android.InitSandbox(topDir)
Lukacs T. Berki7690c092021-02-26 14:27:36 +0100330
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200331 availableEnv := parseAvailableEnv()
Lukacs T. Berki53b2f362021-04-12 14:04:24 +0200332
Lukacs T. Berki89fcdcb2021-09-07 09:10:33 +0200333 configuration := newConfig(availableEnv)
Lukacs T. Berkif0b3b942021-03-23 11:46:47 +0100334 extraNinjaDeps := []string{
335 configuration.ProductVariablesFileName,
Lukacs T. Berkid518e1a2021-04-14 13:49:50 +0200336 usedEnvFile,
Lukacs T. Berkif0b3b942021-03-23 11:46:47 +0100337 }
Colin Crossaa812d12019-06-19 13:33:24 -0700338
Lukacs T. Berkid1e3f1f2021-03-16 08:55:23 +0100339 if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
340 configuration.SetAllowMissingDependencies()
341 }
342
Lukacs T. Berki7d613bf2021-03-02 10:09:41 +0100343 if shared.IsDebugging() {
Colin Crossaa812d12019-06-19 13:33:24 -0700344 // Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is
345 // enabled even if it completed successfully.
Lukacs T. Berki9f6c24a2021-08-26 15:07:24 +0200346 extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))
Colin Crossaa812d12019-06-19 13:33:24 -0700347 }
Jingwen Chendaa54bc2020-12-14 02:58:54 -0500348
Dan Willemsenccf36aa2022-04-20 23:11:43 -0700349 // Bypass configuration.Getenv, as LOG_DIR does not need to be dependency tracked. By definition, it will
350 // change between every CI build, so tracking it would require re-running Soong for every build.
351 logDir := availableEnv["LOG_DIR"]
352
Joe Onorato2e5e4012022-06-07 17:16:08 -0700353 ctx := newContext(configuration)
354 ctx.EventHandler.Begin("soong_build")
355
ustacd245f72022-08-24 11:45:19 -0400356 finalOutputFile := doChosenActivity(ctx, configuration, extraNinjaDeps)
Joe Onorato2e5e4012022-06-07 17:16:08 -0700357
358 ctx.EventHandler.End("soong_build")
359 writeMetrics(configuration, *ctx.EventHandler, logDir)
Chris Parsons715b08f2022-03-22 19:23:40 -0400360
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200361 writeUsedEnvironmentFile(configuration, finalOutputFile)
Lukacs T. Berkif0b3b942021-03-23 11:46:47 +0100362}
363
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200364func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile string) {
365 if usedEnvFile == "" {
366 return
367 }
368
369 path := shared.JoinPath(topDir, usedEnvFile)
Lukacs T. Berkif0b3b942021-03-23 11:46:47 +0100370 data, err := shared.EnvFileContents(configuration.EnvDeps())
371 if err != nil {
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200372 fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err)
Lukacs T. Berkif0b3b942021-03-23 11:46:47 +0100373 os.Exit(1)
374 }
375
376 err = ioutil.WriteFile(path, data, 0666)
377 if err != nil {
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200378 fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err)
Lukacs T. Berkic99c9472021-03-24 10:50:06 +0100379 os.Exit(1)
380 }
381
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200382 // Touch the output file so that it's not older than the file we just
Lukacs T. Berkic99c9472021-03-24 10:50:06 +0100383 // wrote. We can't write the environment file earlier because one an access
384 // new environment variables while writing it.
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200385 touch(shared.JoinPath(topDir, finalOutputFile))
Colin Cross3f40fa42015-01-30 17:27:36 -0800386}
Jingwen Chen5ba7e472020-07-15 10:06:41 +0000387
Lukacs T. Berkif8e24282021-04-14 10:31:00 +0200388func touch(path string) {
389 f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
390 if err != nil {
391 fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err)
392 os.Exit(1)
393 }
394
395 err = f.Close()
396 if err != nil {
397 fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err)
398 os.Exit(1)
399 }
400
401 currentTime := time.Now().Local()
402 err = os.Chtimes(path, currentTime, currentTime)
403 if err != nil {
404 fmt.Fprintf(os.Stderr, "error touching '%s': %s\n", path, err)
405 os.Exit(1)
406 }
407}
408
Cole Faust324a92e2022-08-23 15:29:05 -0700409// Find BUILD files in the srcDir which are not in the allowlist
410// (android.Bp2BuildConversionAllowlist#ShouldKeepExistingBuildFileForDir)
411// and return their paths so they can be left out of the Bazel workspace dir (i.e. ignored)
412func getPathsToIgnoredBuildFiles(config android.Bp2BuildConversionAllowlist, topDir string, srcDirBazelFiles []string, verbose bool) []string {
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400413 paths := make([]string, 0)
414
Rupert Shuttleworthe03bb612021-05-20 19:34:50 -0400415 for _, srcDirBazelFileRelativePath := range srcDirBazelFiles {
416 srcDirBazelFileFullPath := shared.JoinPath(topDir, srcDirBazelFileRelativePath)
417 fileInfo, err := os.Stat(srcDirBazelFileFullPath)
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400418 if err != nil {
Rupert Shuttleworthe03bb612021-05-20 19:34:50 -0400419 // Warn about error, but continue trying to check files
420 fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", srcDirBazelFileFullPath, err)
421 continue
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400422 }
Rupert Shuttleworthe03bb612021-05-20 19:34:50 -0400423 if fileInfo.IsDir() {
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400424 // Don't ignore entire directories
Rupert Shuttleworthe03bb612021-05-20 19:34:50 -0400425 continue
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400426 }
Cole Faust324a92e2022-08-23 15:29:05 -0700427 if fileInfo.Name() != "BUILD" && fileInfo.Name() != "BUILD.bazel" {
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400428 // Don't ignore this file - it is not a build file
Rupert Shuttleworthe03bb612021-05-20 19:34:50 -0400429 continue
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400430 }
Cole Faust324a92e2022-08-23 15:29:05 -0700431 if config.ShouldKeepExistingBuildFileForDir(filepath.Dir(srcDirBazelFileRelativePath)) {
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400432 // Don't ignore this existing build file
Rupert Shuttleworthe03bb612021-05-20 19:34:50 -0400433 continue
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400434 }
Liz Kammer1aefc9d2021-10-18 16:59:00 -0400435 if verbose {
436 fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", srcDirBazelFileRelativePath)
437 }
Rupert Shuttleworthe03bb612021-05-20 19:34:50 -0400438 paths = append(paths, srcDirBazelFileRelativePath)
439 }
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400440
Rupert Shuttleworthe03bb612021-05-20 19:34:50 -0400441 return paths
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400442}
443
444// Returns temporary symlink forest excludes necessary for bazel build //external/... (and bazel build //frameworks/...) to work
445func getTemporaryExcludes() []string {
446 excludes := make([]string, 0)
447
448 // FIXME: 'autotest_lib' is a symlink back to external/autotest, and this causes an infinite symlink expansion error for Bazel
449 excludes = append(excludes, "external/autotest/venv/autotest_lib")
Jingwen Chenc0c50212022-06-07 10:07:22 +0000450 excludes = append(excludes, "external/autotest/autotest_lib")
451 excludes = append(excludes, "external/autotest/client/autotest_lib/client")
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400452
453 // FIXME: The external/google-fruit/extras/bazel_root/third_party/fruit dir is poison
454 // It contains several symlinks back to real source dirs, and those source dirs contain BUILD files we want to ignore
455 excludes = append(excludes, "external/google-fruit/extras/bazel_root/third_party/fruit")
456
457 // FIXME: 'frameworks/compile/slang' has a filegroup error due to an escaping issue
458 excludes = append(excludes, "frameworks/compile/slang")
459
460 return excludes
461}
462
Rupert Shuttleworthe03bb612021-05-20 19:34:50 -0400463// Read the bazel.list file that the Soong Finder already dumped earlier (hopefully)
464// It contains the locations of BUILD files, BUILD.bazel files, etc. in the source dir
465func getExistingBazelRelatedFiles(topDir string) ([]string, error) {
Lukacs T. Berkif9008072021-08-16 15:24:48 +0200466 bazelFinderFile := filepath.Join(filepath.Dir(cmdlineArgs.ModuleListFile), "bazel.list")
Rupert Shuttleworthe03bb612021-05-20 19:34:50 -0400467 if !filepath.IsAbs(bazelFinderFile) {
468 // Assume this was a relative path under topDir
469 bazelFinderFile = filepath.Join(topDir, bazelFinderFile)
470 }
471 data, err := ioutil.ReadFile(bazelFinderFile)
472 if err != nil {
473 return nil, err
474 }
475 files := strings.Split(strings.TrimSpace(string(data)), "\n")
476 return files, nil
477}
478
Jingwen Chendaa54bc2020-12-14 02:58:54 -0500479// Run Soong in the bp2build mode. This creates a standalone context that registers
480// an alternate pipeline of mutators and singletons specifically for generating
481// Bazel BUILD files instead of Ninja files.
Lukacs T. Berki6790ebc2021-04-01 17:55:58 +0200482func runBp2Build(configuration android.Config, extraNinjaDeps []string) {
Usta Shresthaa117c582022-10-04 12:13:25 -0400483 var codegenMetrics bp2build.CodegenMetrics
Chris Parsons715b08f2022-03-22 19:23:40 -0400484 eventHandler := metrics.EventHandler{}
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400485 eventHandler.Do("bp2build", func() {
Chris Parsons715b08f2022-03-22 19:23:40 -0400486
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400487 // Register an alternate set of singletons and mutators for bazel
488 // conversion for Bazel conversion.
489 bp2buildCtx := android.NewContext(configuration)
Jingwen Cheneb76c432021-01-28 08:22:12 -0500490
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400491 // Propagate "allow misssing dependencies" bit. This is normally set in
492 // newContext(), but we create bp2buildCtx without calling that method.
493 bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
494 bp2buildCtx.SetNameInterface(newNameResolver(configuration))
495 bp2buildCtx.RegisterForBazelConversion()
Jingwen Cheneb76c432021-01-28 08:22:12 -0500496
Usta Shresthaa117c582022-10-04 12:13:25 -0400497 var ninjaDeps []string
498 ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
499
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400500 // The bp2build process is a purely functional process that only depends on
501 // Android.bp files. It must not depend on the values of per-build product
502 // configurations or variables, since those will generate different BUILD
503 // files based on how the user has configured their tree.
504 bp2buildCtx.SetModuleListFile(cmdlineArgs.ModuleListFile)
Usta Shresthaa117c582022-10-04 12:13:25 -0400505 if modulePaths, err := bp2buildCtx.ListModulePaths("."); err != nil {
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400506 panic(err)
Usta Shresthaa117c582022-10-04 12:13:25 -0400507 } else {
508 ninjaDeps = append(ninjaDeps, modulePaths...)
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400509 }
Jingwen Chen7dcc4fc2021-02-05 01:28:44 -0500510
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400511 // Run the loading and analysis pipeline to prepare the graph of regular
512 // Modules parsed from Android.bp files, and the BazelTargetModules mapped
513 // from the regular Modules.
Usta Shresthaa117c582022-10-04 12:13:25 -0400514 eventHandler.Do("bootstrap", func() {
515 blueprintArgs := cmdlineArgs
516 bootstrapDeps := bootstrap.RunBlueprint(blueprintArgs, bootstrap.StopBeforePrepareBuildActions, bp2buildCtx.Context, configuration)
517 ninjaDeps = append(ninjaDeps, bootstrapDeps...)
518 })
Lukacs T. Berkid518e1a2021-04-14 13:49:50 +0200519
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400520 globListFiles := writeBuildGlobsNinjaFile(bp2buildCtx, configuration.SoongOutDir(), configuration)
521 ninjaDeps = append(ninjaDeps, globListFiles...)
Lukacs T. Berkid518e1a2021-04-14 13:49:50 +0200522
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400523 // Run the code-generation phase to convert BazelTargetModules to BUILD files
Usta Shresthaa117c582022-10-04 12:13:25 -0400524 // and print conversion codegenMetrics to the user.
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400525 codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx, bp2build.Bp2Build)
Usta Shresthaa117c582022-10-04 12:13:25 -0400526 eventHandler.Do("codegen", func() {
527 codegenMetrics = bp2build.Codegen(codegenContext)
528 })
Lukacs T. Berkib353cca2021-04-16 13:47:36 +0200529
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400530 generatedRoot := shared.JoinPath(configuration.SoongOutDir(), "bp2build")
531 workspaceRoot := shared.JoinPath(configuration.SoongOutDir(), "workspace")
Lukacs T. Berkib353cca2021-04-16 13:47:36 +0200532
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400533 excludes := []string{
534 "bazel-bin",
535 "bazel-genfiles",
536 "bazel-out",
537 "bazel-testlogs",
538 "bazel-" + filepath.Base(topDir),
539 }
Lukacs T. Berkib353cca2021-04-16 13:47:36 +0200540
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400541 if outDir[0] != '/' {
542 excludes = append(excludes, outDir)
543 }
Lukacs T. Berkib353cca2021-04-16 13:47:36 +0200544
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400545 existingBazelRelatedFiles, err := getExistingBazelRelatedFiles(topDir)
546 if err != nil {
547 fmt.Fprintf(os.Stderr, "Error determining existing Bazel-related files: %s\n", err)
548 os.Exit(1)
549 }
Rupert Shuttleworthe03bb612021-05-20 19:34:50 -0400550
Cole Faust324a92e2022-08-23 15:29:05 -0700551 pathsToIgnoredBuildFiles := getPathsToIgnoredBuildFiles(configuration.Bp2buildPackageConfig, topDir, existingBazelRelatedFiles, configuration.IsEnvTrue("BP2BUILD_VERBOSE"))
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400552 excludes = append(excludes, pathsToIgnoredBuildFiles...)
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400553
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400554 excludes = append(excludes, getTemporaryExcludes()...)
Rupert Shuttleworth00960792021-05-12 21:20:13 -0400555
Usta Shresthaa117c582022-10-04 12:13:25 -0400556 // PlantSymlinkForest() returns all the directories that were readdir()'ed.
557 // Such a directory SHOULD be added to `ninjaDeps` so that a child directory
558 // or file created/deleted under it would trigger an update of the symlink
559 // forest.
560 eventHandler.Do("symlink_forest", func() {
561 symlinkForestDeps := bp2build.PlantSymlinkForest(
562 configuration, topDir, workspaceRoot, generatedRoot, ".", excludes)
563 ninjaDeps = append(ninjaDeps, symlinkForestDeps...)
564 })
Lukacs T. Berkib353cca2021-04-16 13:47:36 +0200565
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400566 ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
Chris Parsons715b08f2022-03-22 19:23:40 -0400567
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400568 writeDepFile(bp2buildMarker, eventHandler, ninjaDeps)
Chris Parsons715b08f2022-03-22 19:23:40 -0400569
Usta Shrestha5c6b9482022-05-26 12:22:17 -0400570 // Create an empty bp2build marker file.
571 touch(shared.JoinPath(topDir, bp2buildMarker))
572 })
Chris Parsons715b08f2022-03-22 19:23:40 -0400573
Lukacs T. Berkib353cca2021-04-16 13:47:36 +0200574 // Only report metrics when in bp2build mode. The metrics aren't relevant
575 // for queryview, since that's a total repo-wide conversion and there's a
576 // 1:1 mapping for each module.
Sasha Smundak0fd93e02022-05-19 19:34:31 -0700577 if configuration.IsEnvTrue("BP2BUILD_VERBOSE") {
Usta Shresthaa117c582022-10-04 12:13:25 -0400578 codegenMetrics.Print()
Sasha Smundak0fd93e02022-05-19 19:34:31 -0700579 }
Usta Shresthaa117c582022-10-04 12:13:25 -0400580 writeBp2BuildMetrics(&codegenMetrics, configuration, eventHandler)
Jingwen Chendaa54bc2020-12-14 02:58:54 -0500581}
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -0500582
583// Write Bp2Build metrics into $LOG_DIR
Chris Parsons715b08f2022-03-22 19:23:40 -0400584func writeBp2BuildMetrics(codegenMetrics *bp2build.CodegenMetrics,
585 configuration android.Config, eventHandler metrics.EventHandler) {
586 for _, event := range eventHandler.CompletedEvents() {
587 codegenMetrics.Events = append(codegenMetrics.Events,
588 &bp2build_metrics_proto.Event{
589 Name: event.Id,
590 StartTime: uint64(event.Start.UnixNano()),
591 RealTime: event.RuntimeNanoseconds(),
592 })
593 }
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -0500594 metricsDir := configuration.Getenv("LOG_DIR")
595 if len(metricsDir) < 1 {
596 fmt.Fprintf(os.Stderr, "\nMissing required env var for generating bp2build metrics: LOG_DIR\n")
597 os.Exit(1)
598 }
Chris Parsons715b08f2022-03-22 19:23:40 -0400599 codegenMetrics.Write(metricsDir)
Alex Márquez Pérez Muñíz Díaz Púras Thaureaux947fdbf2021-11-10 09:55:20 -0500600}