blob: 209e82bbd2eca045cfc0f1bc6b5c88df48b4fa58 [file] [log] [blame]
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001// Copyright 2021 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// The application to convert product configuration makefiles to Starlark.
16// Converts either given list of files (and optionally the dependent files
17// of the same kind), or all all product configuration makefiles in the
18// given source tree.
19// Previous version of a converted file can be backed up.
20// Optionally prints detailed statistics at the end.
21package main
22
23import (
Sasha Smundak6609ba72021-07-22 18:32:56 -070024 "bufio"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080025 "flag"
26 "fmt"
27 "io/ioutil"
28 "os"
Sasha Smundak6609ba72021-07-22 18:32:56 -070029 "os/exec"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080030 "path/filepath"
31 "regexp"
32 "runtime/debug"
Sasha Smundak38802792021-08-01 14:38:07 -070033 "runtime/pprof"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080034 "sort"
35 "strings"
36 "time"
37
38 "android/soong/androidmk/parser"
39 "android/soong/mk2rbc"
40)
41
42var (
43 rootDir = flag.String("root", ".", "the value of // for load paths")
44 // TODO(asmundak): remove this option once there is a consensus on suffix
45 suffix = flag.String("suffix", ".rbc", "generated files' suffix")
46 dryRun = flag.Bool("dry_run", false, "dry run")
47 recurse = flag.Bool("convert_dependents", false, "convert all dependent files")
48 mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`)
49 warn = flag.Bool("warnings", false, "warn about partially failed conversions")
50 verbose = flag.Bool("v", false, "print summary")
51 errstat = flag.Bool("error_stat", false, "print error statistics")
52 traceVar = flag.String("trace", "", "comma-separated list of variables to trace")
53 // TODO(asmundak): this option is for debugging
54 allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //")
55 outputTop = flag.String("outdir", "", "write output files into this directory hierarchy")
56 launcher = flag.String("launcher", "", "generated launcher path. If set, the non-flag argument is _product_name_")
57 printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit")
Sasha Smundak38802792021-08-01 14:38:07 -070058 cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file")
Sasha Smundakb051c4e2020-11-05 20:45:07 -080059 traceCalls = flag.Bool("trace_calls", false, "trace function calls")
60)
61
62func init() {
63 // Poor man's flag aliasing: works, but the usage string is ugly and
64 // both flag and its alias can be present on the command line
65 flagAlias := func(target string, alias string) {
66 if f := flag.Lookup(target); f != nil {
67 flag.Var(f.Value, alias, "alias for --"+f.Name)
68 return
69 }
70 quit("cannot alias unknown flag " + target)
71 }
72 flagAlias("suffix", "s")
73 flagAlias("root", "d")
74 flagAlias("dry_run", "n")
75 flagAlias("convert_dependents", "r")
76 flagAlias("warnings", "w")
77 flagAlias("error_stat", "e")
78}
79
80var backupSuffix string
81var tracedVariables []string
82var errorLogger = errorsByType{data: make(map[string]datum)}
Sasha Smundak6609ba72021-07-22 18:32:56 -070083var makefileFinder = &LinuxMakefileFinder{}
Sasha Smundakb051c4e2020-11-05 20:45:07 -080084
85func main() {
86 flag.Usage = func() {
87 cmd := filepath.Base(os.Args[0])
88 fmt.Fprintf(flag.CommandLine.Output(),
89 "Usage: %[1]s flags file...\n"+
90 "or: %[1]s flags --launcher=PATH PRODUCT\n", cmd)
91 flag.PrintDefaults()
92 }
93 flag.Parse()
94
95 // Delouse
96 if *suffix == ".mk" {
97 quit("cannot use .mk as generated file suffix")
98 }
99 if *suffix == "" {
100 quit("suffix cannot be empty")
101 }
102 if *outputTop != "" {
103 if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil {
104 quit(err)
105 }
106 s, err := filepath.Abs(*outputTop)
107 if err != nil {
108 quit(err)
109 }
110 *outputTop = s
111 }
112 if *allInSource && len(flag.Args()) > 0 {
113 quit("file list cannot be specified when -all is present")
114 }
115 if *allInSource && *launcher != "" {
116 quit("--all and --launcher are mutually exclusive")
117 }
118
119 // Flag-driven adjustments
120 if (*suffix)[0] != '.' {
121 *suffix = "." + *suffix
122 }
123 if *mode == "backup" {
124 backupSuffix = time.Now().Format("20060102150405")
125 }
126 if *traceVar != "" {
127 tracedVariables = strings.Split(*traceVar, ",")
128 }
129
Sasha Smundak38802792021-08-01 14:38:07 -0700130 if *cpuProfile != "" {
131 f, err := os.Create(*cpuProfile)
132 if err != nil {
133 quit(err)
134 }
135 pprof.StartCPUProfile(f)
136 defer pprof.StopCPUProfile()
137 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800138 // Find out global variables
139 getConfigVariables()
140 getSoongVariables()
141
142 if *printProductConfigMap {
143 productConfigMap := buildProductConfigMap()
144 var products []string
145 for p := range productConfigMap {
146 products = append(products, p)
147 }
148 sort.Strings(products)
149 for _, p := range products {
150 fmt.Println(p, productConfigMap[p])
151 }
152 os.Exit(0)
153 }
Sasha Smundakb2ac8592021-07-26 09:27:22 -0700154
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800155 // Convert!
156 ok := true
157 if *launcher != "" {
158 if len(flag.Args()) != 1 {
159 quit(fmt.Errorf("a launcher can be generated only for a single product"))
160 }
161 product := flag.Args()[0]
162 productConfigMap := buildProductConfigMap()
163 path, found := productConfigMap[product]
164 if !found {
165 quit(fmt.Errorf("cannot generate configuration launcher for %s, it is not a known product",
166 product))
167 }
168 ok = convertOne(path) && ok
169 err := writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(path), mk2rbc.MakePath2ModuleName(path)))
170 if err != nil {
171 fmt.Fprintf(os.Stderr, "%s:%s", path, err)
172 ok = false
173 }
174
175 } else {
176 files := flag.Args()
177 if *allInSource {
178 productConfigMap := buildProductConfigMap()
179 for _, path := range productConfigMap {
180 files = append(files, path)
181 }
182 }
183 for _, mkFile := range files {
184 ok = convertOne(mkFile) && ok
185 }
186 }
187
188 printStats()
189 if *errstat {
190 errorLogger.printStatistics()
191 }
192 if !ok {
193 os.Exit(1)
194 }
195}
196
197func quit(s interface{}) {
198 fmt.Fprintln(os.Stderr, s)
199 os.Exit(2)
200}
201
202func buildProductConfigMap() map[string]string {
203 const androidProductsMk = "AndroidProducts.mk"
204 // Build the list of AndroidProducts.mk files: it's
Sasha Smundak70f17452021-09-01 19:14:24 -0700205 // build/make/target/product/AndroidProducts.mk + device/**/AndroidProducts.mk plus + vendor/**/AndroidProducts.mk
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800206 targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk)
207 if _, err := os.Stat(targetAndroidProductsFile); err != nil {
208 fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n",
209 targetAndroidProductsFile, err, *rootDir)
210 }
211 productConfigMap := make(map[string]string)
212 if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil {
213 fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
214 }
Sasha Smundak70f17452021-09-01 19:14:24 -0700215 for _, t := range []string{"device", "vendor"} {
216 _ = filepath.WalkDir(filepath.Join(*rootDir, t),
217 func(path string, d os.DirEntry, err error) error {
218 if err != nil || d.IsDir() || filepath.Base(path) != androidProductsMk {
219 return nil
220 }
221 if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil {
222 fmt.Fprintf(os.Stderr, "%s: %s\n", path, err)
223 // Keep going, we want to find all such errors in a single run
224 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800225 return nil
Sasha Smundak70f17452021-09-01 19:14:24 -0700226 })
227 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800228 return productConfigMap
229}
230
231func getConfigVariables() {
232 path := filepath.Join(*rootDir, "build", "make", "core", "product.mk")
233 if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil {
234 quit(fmt.Errorf("%s\n(check --root[=%s], it should point to the source root)",
235 err, *rootDir))
236 }
237}
238
239// Implements mkparser.Scope, to be used by mkparser.Value.Value()
240type fileNameScope struct {
241 mk2rbc.ScopeBase
242}
243
244func (s fileNameScope) Get(name string) string {
245 if name != "BUILD_SYSTEM" {
246 return fmt.Sprintf("$(%s)", name)
247 }
248 return filepath.Join(*rootDir, "build", "make", "core")
249}
250
251func getSoongVariables() {
252 path := filepath.Join(*rootDir, "build", "make", "core", "soong_config.mk")
253 err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables)
254 if err != nil {
255 quit(err)
256 }
257}
258
259var converted = make(map[string]*mk2rbc.StarlarkScript)
260
261//goland:noinspection RegExpRepeatedSpace
262var cpNormalizer = regexp.MustCompile(
263 "# Copyright \\(C\\) 20.. The Android Open Source Project")
264
265const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project"
266const copyright = `#
267# Copyright (C) 20xx The Android Open Source Project
268#
269# Licensed under the Apache License, Version 2.0 (the "License");
270# you may not use this file except in compliance with the License.
271# You may obtain a copy of the License at
272#
273# http://www.apache.org/licenses/LICENSE-2.0
274#
275# Unless required by applicable law or agreed to in writing, software
276# distributed under the License is distributed on an "AS IS" BASIS,
277# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
278# See the License for the specific language governing permissions and
279# limitations under the License.
280#
281`
282
283// Convert a single file.
284// Write the result either to the same directory, to the same place in
285// the output hierarchy, or to the stdout.
286// Optionally, recursively convert the files this one includes by
287// $(call inherit-product) or an include statement.
288func convertOne(mkFile string) (ok bool) {
289 if v, ok := converted[mkFile]; ok {
290 return v != nil
291 }
292 converted[mkFile] = nil
293 defer func() {
294 if r := recover(); r != nil {
295 ok = false
296 fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack())
297 }
298 }()
299
300 mk2starRequest := mk2rbc.Request{
301 MkFile: mkFile,
302 Reader: nil,
303 RootDir: *rootDir,
304 OutputDir: *outputTop,
305 OutputSuffix: *suffix,
306 TracedVariables: tracedVariables,
307 TraceCalls: *traceCalls,
308 WarnPartialSuccess: *warn,
Sasha Smundak6609ba72021-07-22 18:32:56 -0700309 SourceFS: os.DirFS(*rootDir),
310 MakefileFinder: makefileFinder,
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800311 }
312 if *errstat {
313 mk2starRequest.ErrorLogger = errorLogger
314 }
315 ss, err := mk2rbc.Convert(mk2starRequest)
316 if err != nil {
317 fmt.Fprintln(os.Stderr, mkFile, ": ", err)
318 return false
319 }
320 script := ss.String()
321 outputPath := outputFilePath(mkFile)
322
323 if *dryRun {
324 fmt.Printf("==== %s ====\n", outputPath)
325 // Print generated script after removing the copyright header
326 outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright)
327 fmt.Println(strings.TrimPrefix(outText, copyright))
328 } else {
329 if err := maybeBackup(outputPath); err != nil {
330 fmt.Fprintln(os.Stderr, err)
331 return false
332 }
333 if err := writeGenerated(outputPath, script); err != nil {
334 fmt.Fprintln(os.Stderr, err)
335 return false
336 }
337 }
338 ok = true
339 if *recurse {
340 for _, sub := range ss.SubConfigFiles() {
341 // File may be absent if it is a conditional load
342 if _, err := os.Stat(sub); os.IsNotExist(err) {
343 continue
344 }
345 ok = convertOne(sub) && ok
346 }
347 }
348 converted[mkFile] = ss
349 return ok
350}
351
352// Optionally saves the previous version of the generated file
353func maybeBackup(filename string) error {
354 stat, err := os.Stat(filename)
355 if os.IsNotExist(err) {
356 return nil
357 }
358 if !stat.Mode().IsRegular() {
359 return fmt.Errorf("%s exists and is not a regular file", filename)
360 }
361 switch *mode {
362 case "backup":
363 return os.Rename(filename, filename+backupSuffix)
364 case "write":
365 return os.Remove(filename)
366 default:
367 return fmt.Errorf("%s already exists, use --mode option", filename)
368 }
369}
370
371func outputFilePath(mkFile string) string {
372 path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix
373 if *outputTop != "" {
374 path = filepath.Join(*outputTop, path)
375 }
376 return path
377}
378
379func writeGenerated(path string, contents string) error {
380 if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
381 return err
382 }
383 if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil {
384 return err
385 }
386 return nil
387}
388
389func printStats() {
390 var sortedFiles []string
391 if !*warn && !*verbose {
392 return
393 }
394 for p := range converted {
395 sortedFiles = append(sortedFiles, p)
396 }
397 sort.Strings(sortedFiles)
398
399 nOk, nPartial, nFailed := 0, 0, 0
400 for _, f := range sortedFiles {
401 if converted[f] == nil {
402 nFailed++
403 } else if converted[f].HasErrors() {
404 nPartial++
405 } else {
406 nOk++
407 }
408 }
409 if *warn {
410 if nPartial > 0 {
411 fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n")
412 for _, f := range sortedFiles {
413 if ss := converted[f]; ss != nil && ss.HasErrors() {
414 fmt.Fprintln(os.Stderr, " ", f)
415 }
416 }
417 }
418
419 if nFailed > 0 {
420 fmt.Fprintf(os.Stderr, "Conversion failed for files:\n")
421 for _, f := range sortedFiles {
422 if converted[f] == nil {
423 fmt.Fprintln(os.Stderr, " ", f)
424 }
425 }
426 }
427 }
428 if *verbose {
429 fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Succeeded:", nOk)
430 fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Partial:", nPartial)
431 fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Failed:", nFailed)
432 }
433}
434
435type datum struct {
436 count int
437 formattingArgs []string
438}
439
440type errorsByType struct {
441 data map[string]datum
442}
443
444func (ebt errorsByType) NewError(message string, node parser.Node, args ...interface{}) {
445 v, exists := ebt.data[message]
446 if exists {
447 v.count++
448 } else {
449 v = datum{1, nil}
450 }
451 if strings.Contains(message, "%s") {
452 var newArg1 string
453 if len(args) == 0 {
454 panic(fmt.Errorf(`%s has %%s but args are missing`, message))
455 }
456 newArg1 = fmt.Sprint(args[0])
457 if message == "unsupported line" {
458 newArg1 = node.Dump()
459 } else if message == "unsupported directive %s" {
460 if newArg1 == "include" || newArg1 == "-include" {
461 newArg1 = node.Dump()
462 }
463 }
464 v.formattingArgs = append(v.formattingArgs, newArg1)
465 }
466 ebt.data[message] = v
467}
468
469func (ebt errorsByType) printStatistics() {
470 if len(ebt.data) > 0 {
471 fmt.Fprintln(os.Stderr, "Error counts:")
472 }
473 for message, data := range ebt.data {
474 if len(data.formattingArgs) == 0 {
475 fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message)
476 continue
477 }
478 itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30)
479 fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count)
480 fmt.Fprintln(os.Stderr, " ", itemsByFreq)
481 }
482}
483
484func stringsWithFreq(items []string, topN int) (string, int) {
485 freq := make(map[string]int)
486 for _, item := range items {
487 freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++
488 }
489 var sorted []string
490 for item := range freq {
491 sorted = append(sorted, item)
492 }
493 sort.Slice(sorted, func(i int, j int) bool {
494 return freq[sorted[i]] > freq[sorted[j]]
495 })
496 sep := ""
497 res := ""
498 for i, item := range sorted {
499 if i >= topN {
500 res += " ..."
501 break
502 }
503 count := freq[item]
504 if count > 1 {
505 res += fmt.Sprintf("%s%s(%d)", sep, item, count)
506 } else {
507 res += fmt.Sprintf("%s%s", sep, item)
508 }
509 sep = ", "
510 }
511 return res, len(sorted)
512}
Sasha Smundak6609ba72021-07-22 18:32:56 -0700513
514type LinuxMakefileFinder struct {
515 cachedRoot string
516 cachedMakefiles []string
517}
518
519func (l *LinuxMakefileFinder) Find(root string) []string {
520 if l.cachedMakefiles != nil && l.cachedRoot == root {
521 return l.cachedMakefiles
522 }
523 l.cachedRoot = root
524 l.cachedMakefiles = make([]string, 0)
525
526 // Return all *.mk files but not in hidden directories.
527
528 // NOTE(asmundak): as it turns out, even the WalkDir (which is an _optimized_ directory tree walker)
529 // is about twice slower than running `find` command (14s vs 6s on the internal Android source tree).
530 common_args := []string{"!", "-type", "d", "-name", "*.mk", "!", "-path", "*/.*/*"}
531 if root != "" {
532 common_args = append([]string{root}, common_args...)
533 }
534 cmd := exec.Command("/usr/bin/find", common_args...)
535 stdout, err := cmd.StdoutPipe()
536 if err == nil {
537 err = cmd.Start()
538 }
539 if err != nil {
540 panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
541 }
542 scanner := bufio.NewScanner(stdout)
543 for scanner.Scan() {
544 l.cachedMakefiles = append(l.cachedMakefiles, strings.TrimPrefix(scanner.Text(), "./"))
545 }
546 stdout.Close()
547 return l.cachedMakefiles
548}