blob: 25507d2d600822ff7b214f1455b14ffdb79ecaca [file] [log] [blame]
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001// Copyright 2021 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// The application to convert product configuration makefiles to Starlark.
16// Converts either given list of files (and optionally the dependent files
17// of the same kind), or all all product configuration makefiles in the
18// given source tree.
19// Previous version of a converted file can be backed up.
20// Optionally prints detailed statistics at the end.
21package main
22
23import (
Sasha Smundak6609ba72021-07-22 18:32:56 -070024 "bufio"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080025 "flag"
26 "fmt"
27 "io/ioutil"
28 "os"
Sasha Smundak6609ba72021-07-22 18:32:56 -070029 "os/exec"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080030 "path/filepath"
31 "regexp"
32 "runtime/debug"
Sasha Smundak38802792021-08-01 14:38:07 -070033 "runtime/pprof"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080034 "sort"
35 "strings"
36 "time"
37
38 "android/soong/androidmk/parser"
39 "android/soong/mk2rbc"
40)
41
42var (
43 rootDir = flag.String("root", ".", "the value of // for load paths")
44 // TODO(asmundak): remove this option once there is a consensus on suffix
45 suffix = flag.String("suffix", ".rbc", "generated files' suffix")
46 dryRun = flag.Bool("dry_run", false, "dry run")
47 recurse = flag.Bool("convert_dependents", false, "convert all dependent files")
48 mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`)
49 warn = flag.Bool("warnings", false, "warn about partially failed conversions")
50 verbose = flag.Bool("v", false, "print summary")
51 errstat = flag.Bool("error_stat", false, "print error statistics")
52 traceVar = flag.String("trace", "", "comma-separated list of variables to trace")
53 // TODO(asmundak): this option is for debugging
54 allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //")
55 outputTop = flag.String("outdir", "", "write output files into this directory hierarchy")
Cole Faust6ed7cb42021-10-07 17:08:46 -070056 launcher = flag.String("launcher", "", "generated launcher path.")
57 boardlauncher = flag.String("boardlauncher", "", "generated board configuration launcher path.")
Sasha Smundakb051c4e2020-11-05 20:45:07 -080058 printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit")
Sasha Smundak38802792021-08-01 14:38:07 -070059 cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file")
Sasha Smundakb051c4e2020-11-05 20:45:07 -080060 traceCalls = flag.Bool("trace_calls", false, "trace function calls")
Cole Faust6ed7cb42021-10-07 17:08:46 -070061 inputVariables = flag.String("input_variables", "", "starlark file containing product config and global variables")
Sasha Smundakb051c4e2020-11-05 20:45:07 -080062)
63
64func init() {
Joe Onoratob4638c12021-10-27 15:47:06 -070065 // Simplistic flag aliasing: works, but the usage string is ugly and
Sasha Smundakb051c4e2020-11-05 20:45:07 -080066 // both flag and its alias can be present on the command line
67 flagAlias := func(target string, alias string) {
68 if f := flag.Lookup(target); f != nil {
69 flag.Var(f.Value, alias, "alias for --"+f.Name)
70 return
71 }
72 quit("cannot alias unknown flag " + target)
73 }
74 flagAlias("suffix", "s")
75 flagAlias("root", "d")
76 flagAlias("dry_run", "n")
77 flagAlias("convert_dependents", "r")
78 flagAlias("warnings", "w")
79 flagAlias("error_stat", "e")
80}
81
82var backupSuffix string
83var tracedVariables []string
84var errorLogger = errorsByType{data: make(map[string]datum)}
Sasha Smundak6609ba72021-07-22 18:32:56 -070085var makefileFinder = &LinuxMakefileFinder{}
Sasha Smundakd7d07ad2021-09-10 15:42:34 -070086var versionDefaultsMk = filepath.Join("build", "make", "core", "version_defaults.mk")
Sasha Smundakb051c4e2020-11-05 20:45:07 -080087
88func main() {
89 flag.Usage = func() {
90 cmd := filepath.Base(os.Args[0])
91 fmt.Fprintf(flag.CommandLine.Output(),
Cole Faust6ed7cb42021-10-07 17:08:46 -070092 "Usage: %[1]s flags file...\n", cmd)
Sasha Smundakb051c4e2020-11-05 20:45:07 -080093 flag.PrintDefaults()
94 }
95 flag.Parse()
96
97 // Delouse
98 if *suffix == ".mk" {
99 quit("cannot use .mk as generated file suffix")
100 }
101 if *suffix == "" {
102 quit("suffix cannot be empty")
103 }
104 if *outputTop != "" {
105 if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil {
106 quit(err)
107 }
108 s, err := filepath.Abs(*outputTop)
109 if err != nil {
110 quit(err)
111 }
112 *outputTop = s
113 }
114 if *allInSource && len(flag.Args()) > 0 {
115 quit("file list cannot be specified when -all is present")
116 }
117 if *allInSource && *launcher != "" {
118 quit("--all and --launcher are mutually exclusive")
119 }
120
121 // Flag-driven adjustments
122 if (*suffix)[0] != '.' {
123 *suffix = "." + *suffix
124 }
125 if *mode == "backup" {
126 backupSuffix = time.Now().Format("20060102150405")
127 }
128 if *traceVar != "" {
129 tracedVariables = strings.Split(*traceVar, ",")
130 }
131
Sasha Smundak38802792021-08-01 14:38:07 -0700132 if *cpuProfile != "" {
133 f, err := os.Create(*cpuProfile)
134 if err != nil {
135 quit(err)
136 }
137 pprof.StartCPUProfile(f)
138 defer pprof.StopCPUProfile()
139 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800140 // Find out global variables
141 getConfigVariables()
142 getSoongVariables()
143
144 if *printProductConfigMap {
145 productConfigMap := buildProductConfigMap()
146 var products []string
147 for p := range productConfigMap {
148 products = append(products, p)
149 }
150 sort.Strings(products)
151 for _, p := range products {
152 fmt.Println(p, productConfigMap[p])
153 }
154 os.Exit(0)
155 }
Sasha Smundakb2ac8592021-07-26 09:27:22 -0700156
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800157 // Convert!
Cole Faust07ea5032021-09-30 16:11:16 -0700158 files := flag.Args()
Cole Faust07ea5032021-09-30 16:11:16 -0700159 if *allInSource {
Cole Faustebf79bf2021-10-05 11:05:02 -0700160 productConfigMap := buildProductConfigMap()
Cole Faust07ea5032021-09-30 16:11:16 -0700161 for _, path := range productConfigMap {
162 files = append(files, path)
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800163 }
Cole Faust07ea5032021-09-30 16:11:16 -0700164 }
Cole Faust07ea5032021-09-30 16:11:16 -0700165 ok := true
166 for _, mkFile := range files {
167 ok = convertOne(mkFile) && ok
168 }
169
170 if *launcher != "" {
171 if len(files) != 1 {
172 quit(fmt.Errorf("a launcher can be generated only for a single product"))
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800173 }
Sasha Smundakd7d07ad2021-09-10 15:42:34 -0700174 versionDefaults, err := generateVersionDefaults()
175 if err != nil {
176 quit(err)
177 }
Sasha Smundakd7d07ad2021-09-10 15:42:34 -0700178 versionDefaultsPath := outputFilePath(versionDefaultsMk)
179 err = writeGenerated(versionDefaultsPath, versionDefaults)
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800180 if err != nil {
Cole Faust6ed7cb42021-10-07 17:08:46 -0700181 fmt.Fprintf(os.Stderr, "%s: %s", files[0], err)
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800182 ok = false
183 }
184
Cole Faust07ea5032021-09-30 16:11:16 -0700185 err = writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(files[0]), versionDefaultsPath,
186 mk2rbc.MakePath2ModuleName(files[0])))
Sasha Smundakd7d07ad2021-09-10 15:42:34 -0700187 if err != nil {
Cole Faust6ed7cb42021-10-07 17:08:46 -0700188 fmt.Fprintf(os.Stderr, "%s: %s", files[0], err)
189 ok = false
190 }
191 }
192 if *boardlauncher != "" {
193 if len(files) != 1 {
194 quit(fmt.Errorf("a launcher can be generated only for a single product"))
195 }
196 if *inputVariables == "" {
197 quit(fmt.Errorf("the board launcher requires an input variables file"))
198 }
199 if !convertOne(*inputVariables) {
200 quit(fmt.Errorf("the board launcher input variables file failed to convert"))
201 }
202 err := writeGenerated(*boardlauncher, mk2rbc.BoardLauncher(
203 outputFilePath(files[0]), outputFilePath(*inputVariables)))
204 if err != nil {
205 fmt.Fprintf(os.Stderr, "%s: %s", files[0], err)
Sasha Smundakd7d07ad2021-09-10 15:42:34 -0700206 ok = false
207 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800208 }
209
210 printStats()
211 if *errstat {
212 errorLogger.printStatistics()
213 }
214 if !ok {
215 os.Exit(1)
216 }
217}
218
Sasha Smundakd7d07ad2021-09-10 15:42:34 -0700219func generateVersionDefaults() (string, error) {
220 versionSettings, err := mk2rbc.ParseVersionDefaults(filepath.Join(*rootDir, versionDefaultsMk))
221 if err != nil {
222 return "", err
223 }
224 return mk2rbc.VersionDefaults(versionSettings), nil
225
226}
227
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800228func quit(s interface{}) {
229 fmt.Fprintln(os.Stderr, s)
230 os.Exit(2)
231}
232
233func buildProductConfigMap() map[string]string {
234 const androidProductsMk = "AndroidProducts.mk"
235 // Build the list of AndroidProducts.mk files: it's
Sasha Smundak70f17452021-09-01 19:14:24 -0700236 // build/make/target/product/AndroidProducts.mk + device/**/AndroidProducts.mk plus + vendor/**/AndroidProducts.mk
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800237 targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk)
238 if _, err := os.Stat(targetAndroidProductsFile); err != nil {
239 fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n",
240 targetAndroidProductsFile, err, *rootDir)
241 }
242 productConfigMap := make(map[string]string)
243 if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil {
244 fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
245 }
Sasha Smundak70f17452021-09-01 19:14:24 -0700246 for _, t := range []string{"device", "vendor"} {
247 _ = filepath.WalkDir(filepath.Join(*rootDir, t),
248 func(path string, d os.DirEntry, err error) error {
249 if err != nil || d.IsDir() || filepath.Base(path) != androidProductsMk {
250 return nil
251 }
252 if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil {
253 fmt.Fprintf(os.Stderr, "%s: %s\n", path, err)
254 // Keep going, we want to find all such errors in a single run
255 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800256 return nil
Sasha Smundak70f17452021-09-01 19:14:24 -0700257 })
258 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800259 return productConfigMap
260}
261
262func getConfigVariables() {
263 path := filepath.Join(*rootDir, "build", "make", "core", "product.mk")
264 if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil {
265 quit(fmt.Errorf("%s\n(check --root[=%s], it should point to the source root)",
266 err, *rootDir))
267 }
268}
269
270// Implements mkparser.Scope, to be used by mkparser.Value.Value()
271type fileNameScope struct {
272 mk2rbc.ScopeBase
273}
274
275func (s fileNameScope) Get(name string) string {
276 if name != "BUILD_SYSTEM" {
277 return fmt.Sprintf("$(%s)", name)
278 }
279 return filepath.Join(*rootDir, "build", "make", "core")
280}
281
282func getSoongVariables() {
283 path := filepath.Join(*rootDir, "build", "make", "core", "soong_config.mk")
284 err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables)
285 if err != nil {
286 quit(err)
287 }
288}
289
290var converted = make(map[string]*mk2rbc.StarlarkScript)
291
292//goland:noinspection RegExpRepeatedSpace
293var cpNormalizer = regexp.MustCompile(
294 "# Copyright \\(C\\) 20.. The Android Open Source Project")
295
296const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project"
297const copyright = `#
298# Copyright (C) 20xx The Android Open Source Project
299#
300# Licensed under the Apache License, Version 2.0 (the "License");
301# you may not use this file except in compliance with the License.
302# You may obtain a copy of the License at
303#
304# http://www.apache.org/licenses/LICENSE-2.0
305#
306# Unless required by applicable law or agreed to in writing, software
307# distributed under the License is distributed on an "AS IS" BASIS,
308# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
309# See the License for the specific language governing permissions and
310# limitations under the License.
311#
312`
313
314// Convert a single file.
315// Write the result either to the same directory, to the same place in
316// the output hierarchy, or to the stdout.
317// Optionally, recursively convert the files this one includes by
318// $(call inherit-product) or an include statement.
319func convertOne(mkFile string) (ok bool) {
320 if v, ok := converted[mkFile]; ok {
321 return v != nil
322 }
323 converted[mkFile] = nil
324 defer func() {
325 if r := recover(); r != nil {
326 ok = false
327 fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack())
328 }
329 }()
330
331 mk2starRequest := mk2rbc.Request{
332 MkFile: mkFile,
333 Reader: nil,
334 RootDir: *rootDir,
335 OutputDir: *outputTop,
336 OutputSuffix: *suffix,
337 TracedVariables: tracedVariables,
338 TraceCalls: *traceCalls,
339 WarnPartialSuccess: *warn,
Sasha Smundak6609ba72021-07-22 18:32:56 -0700340 SourceFS: os.DirFS(*rootDir),
341 MakefileFinder: makefileFinder,
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800342 }
343 if *errstat {
344 mk2starRequest.ErrorLogger = errorLogger
345 }
346 ss, err := mk2rbc.Convert(mk2starRequest)
347 if err != nil {
348 fmt.Fprintln(os.Stderr, mkFile, ": ", err)
349 return false
350 }
351 script := ss.String()
352 outputPath := outputFilePath(mkFile)
353
354 if *dryRun {
355 fmt.Printf("==== %s ====\n", outputPath)
356 // Print generated script after removing the copyright header
357 outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright)
358 fmt.Println(strings.TrimPrefix(outText, copyright))
359 } else {
360 if err := maybeBackup(outputPath); err != nil {
361 fmt.Fprintln(os.Stderr, err)
362 return false
363 }
364 if err := writeGenerated(outputPath, script); err != nil {
365 fmt.Fprintln(os.Stderr, err)
366 return false
367 }
368 }
369 ok = true
370 if *recurse {
371 for _, sub := range ss.SubConfigFiles() {
372 // File may be absent if it is a conditional load
373 if _, err := os.Stat(sub); os.IsNotExist(err) {
374 continue
375 }
376 ok = convertOne(sub) && ok
377 }
378 }
379 converted[mkFile] = ss
380 return ok
381}
382
383// Optionally saves the previous version of the generated file
384func maybeBackup(filename string) error {
385 stat, err := os.Stat(filename)
386 if os.IsNotExist(err) {
387 return nil
388 }
389 if !stat.Mode().IsRegular() {
390 return fmt.Errorf("%s exists and is not a regular file", filename)
391 }
392 switch *mode {
393 case "backup":
394 return os.Rename(filename, filename+backupSuffix)
395 case "write":
396 return os.Remove(filename)
397 default:
398 return fmt.Errorf("%s already exists, use --mode option", filename)
399 }
400}
401
402func outputFilePath(mkFile string) string {
403 path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix
404 if *outputTop != "" {
405 path = filepath.Join(*outputTop, path)
406 }
407 return path
408}
409
410func writeGenerated(path string, contents string) error {
411 if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
412 return err
413 }
414 if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil {
415 return err
416 }
417 return nil
418}
419
420func printStats() {
421 var sortedFiles []string
422 if !*warn && !*verbose {
423 return
424 }
425 for p := range converted {
426 sortedFiles = append(sortedFiles, p)
427 }
428 sort.Strings(sortedFiles)
429
430 nOk, nPartial, nFailed := 0, 0, 0
431 for _, f := range sortedFiles {
432 if converted[f] == nil {
433 nFailed++
434 } else if converted[f].HasErrors() {
435 nPartial++
436 } else {
437 nOk++
438 }
439 }
440 if *warn {
441 if nPartial > 0 {
442 fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n")
443 for _, f := range sortedFiles {
444 if ss := converted[f]; ss != nil && ss.HasErrors() {
445 fmt.Fprintln(os.Stderr, " ", f)
446 }
447 }
448 }
449
450 if nFailed > 0 {
451 fmt.Fprintf(os.Stderr, "Conversion failed for files:\n")
452 for _, f := range sortedFiles {
453 if converted[f] == nil {
454 fmt.Fprintln(os.Stderr, " ", f)
455 }
456 }
457 }
458 }
459 if *verbose {
460 fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Succeeded:", nOk)
461 fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Partial:", nPartial)
462 fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Failed:", nFailed)
463 }
464}
465
466type datum struct {
467 count int
468 formattingArgs []string
469}
470
471type errorsByType struct {
472 data map[string]datum
473}
474
475func (ebt errorsByType) NewError(message string, node parser.Node, args ...interface{}) {
476 v, exists := ebt.data[message]
477 if exists {
478 v.count++
479 } else {
480 v = datum{1, nil}
481 }
482 if strings.Contains(message, "%s") {
483 var newArg1 string
484 if len(args) == 0 {
485 panic(fmt.Errorf(`%s has %%s but args are missing`, message))
486 }
487 newArg1 = fmt.Sprint(args[0])
488 if message == "unsupported line" {
489 newArg1 = node.Dump()
490 } else if message == "unsupported directive %s" {
491 if newArg1 == "include" || newArg1 == "-include" {
492 newArg1 = node.Dump()
493 }
494 }
495 v.formattingArgs = append(v.formattingArgs, newArg1)
496 }
497 ebt.data[message] = v
498}
499
500func (ebt errorsByType) printStatistics() {
501 if len(ebt.data) > 0 {
502 fmt.Fprintln(os.Stderr, "Error counts:")
503 }
504 for message, data := range ebt.data {
505 if len(data.formattingArgs) == 0 {
506 fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message)
507 continue
508 }
509 itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30)
510 fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count)
511 fmt.Fprintln(os.Stderr, " ", itemsByFreq)
512 }
513}
514
515func stringsWithFreq(items []string, topN int) (string, int) {
516 freq := make(map[string]int)
517 for _, item := range items {
518 freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++
519 }
520 var sorted []string
521 for item := range freq {
522 sorted = append(sorted, item)
523 }
524 sort.Slice(sorted, func(i int, j int) bool {
525 return freq[sorted[i]] > freq[sorted[j]]
526 })
527 sep := ""
528 res := ""
529 for i, item := range sorted {
530 if i >= topN {
531 res += " ..."
532 break
533 }
534 count := freq[item]
535 if count > 1 {
536 res += fmt.Sprintf("%s%s(%d)", sep, item, count)
537 } else {
538 res += fmt.Sprintf("%s%s", sep, item)
539 }
540 sep = ", "
541 }
542 return res, len(sorted)
543}
Sasha Smundak6609ba72021-07-22 18:32:56 -0700544
545type LinuxMakefileFinder struct {
546 cachedRoot string
547 cachedMakefiles []string
548}
549
550func (l *LinuxMakefileFinder) Find(root string) []string {
551 if l.cachedMakefiles != nil && l.cachedRoot == root {
552 return l.cachedMakefiles
553 }
554 l.cachedRoot = root
555 l.cachedMakefiles = make([]string, 0)
556
557 // Return all *.mk files but not in hidden directories.
558
559 // NOTE(asmundak): as it turns out, even the WalkDir (which is an _optimized_ directory tree walker)
560 // is about twice slower than running `find` command (14s vs 6s on the internal Android source tree).
561 common_args := []string{"!", "-type", "d", "-name", "*.mk", "!", "-path", "*/.*/*"}
562 if root != "" {
563 common_args = append([]string{root}, common_args...)
564 }
565 cmd := exec.Command("/usr/bin/find", common_args...)
566 stdout, err := cmd.StdoutPipe()
567 if err == nil {
568 err = cmd.Start()
569 }
570 if err != nil {
571 panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
572 }
573 scanner := bufio.NewScanner(stdout)
574 for scanner.Scan() {
575 l.cachedMakefiles = append(l.cachedMakefiles, strings.TrimPrefix(scanner.Text(), "./"))
576 }
577 stdout.Close()
578 return l.cachedMakefiles
579}