blob: 72525c4cdfaebab0755a5f007a4a4e071319f6d3 [file] [log] [blame]
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001// Copyright 2021 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// The application to convert product configuration makefiles to Starlark.
16// Converts either given list of files (and optionally the dependent files
17// of the same kind), or all all product configuration makefiles in the
18// given source tree.
19// Previous version of a converted file can be backed up.
20// Optionally prints detailed statistics at the end.
21package main
22
23import (
Sasha Smundak6609ba72021-07-22 18:32:56 -070024 "bufio"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080025 "flag"
26 "fmt"
27 "io/ioutil"
28 "os"
Sasha Smundak6609ba72021-07-22 18:32:56 -070029 "os/exec"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080030 "path/filepath"
31 "regexp"
32 "runtime/debug"
Sasha Smundak38802792021-08-01 14:38:07 -070033 "runtime/pprof"
Sasha Smundakb051c4e2020-11-05 20:45:07 -080034 "sort"
35 "strings"
36 "time"
37
38 "android/soong/androidmk/parser"
39 "android/soong/mk2rbc"
40)
41
42var (
43 rootDir = flag.String("root", ".", "the value of // for load paths")
44 // TODO(asmundak): remove this option once there is a consensus on suffix
45 suffix = flag.String("suffix", ".rbc", "generated files' suffix")
46 dryRun = flag.Bool("dry_run", false, "dry run")
47 recurse = flag.Bool("convert_dependents", false, "convert all dependent files")
48 mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`)
49 warn = flag.Bool("warnings", false, "warn about partially failed conversions")
50 verbose = flag.Bool("v", false, "print summary")
51 errstat = flag.Bool("error_stat", false, "print error statistics")
52 traceVar = flag.String("trace", "", "comma-separated list of variables to trace")
53 // TODO(asmundak): this option is for debugging
54 allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //")
55 outputTop = flag.String("outdir", "", "write output files into this directory hierarchy")
56 launcher = flag.String("launcher", "", "generated launcher path. If set, the non-flag argument is _product_name_")
57 printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit")
Sasha Smundak38802792021-08-01 14:38:07 -070058 cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file")
Sasha Smundakb051c4e2020-11-05 20:45:07 -080059 traceCalls = flag.Bool("trace_calls", false, "trace function calls")
60)
61
62func init() {
63 // Poor man's flag aliasing: works, but the usage string is ugly and
64 // both flag and its alias can be present on the command line
65 flagAlias := func(target string, alias string) {
66 if f := flag.Lookup(target); f != nil {
67 flag.Var(f.Value, alias, "alias for --"+f.Name)
68 return
69 }
70 quit("cannot alias unknown flag " + target)
71 }
72 flagAlias("suffix", "s")
73 flagAlias("root", "d")
74 flagAlias("dry_run", "n")
75 flagAlias("convert_dependents", "r")
76 flagAlias("warnings", "w")
77 flagAlias("error_stat", "e")
78}
79
80var backupSuffix string
81var tracedVariables []string
82var errorLogger = errorsByType{data: make(map[string]datum)}
Sasha Smundak6609ba72021-07-22 18:32:56 -070083var makefileFinder = &LinuxMakefileFinder{}
Sasha Smundakb051c4e2020-11-05 20:45:07 -080084
85func main() {
86 flag.Usage = func() {
87 cmd := filepath.Base(os.Args[0])
88 fmt.Fprintf(flag.CommandLine.Output(),
89 "Usage: %[1]s flags file...\n"+
90 "or: %[1]s flags --launcher=PATH PRODUCT\n", cmd)
91 flag.PrintDefaults()
92 }
93 flag.Parse()
94
95 // Delouse
96 if *suffix == ".mk" {
97 quit("cannot use .mk as generated file suffix")
98 }
99 if *suffix == "" {
100 quit("suffix cannot be empty")
101 }
102 if *outputTop != "" {
103 if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil {
104 quit(err)
105 }
106 s, err := filepath.Abs(*outputTop)
107 if err != nil {
108 quit(err)
109 }
110 *outputTop = s
111 }
112 if *allInSource && len(flag.Args()) > 0 {
113 quit("file list cannot be specified when -all is present")
114 }
115 if *allInSource && *launcher != "" {
116 quit("--all and --launcher are mutually exclusive")
117 }
118
119 // Flag-driven adjustments
120 if (*suffix)[0] != '.' {
121 *suffix = "." + *suffix
122 }
123 if *mode == "backup" {
124 backupSuffix = time.Now().Format("20060102150405")
125 }
126 if *traceVar != "" {
127 tracedVariables = strings.Split(*traceVar, ",")
128 }
129
Sasha Smundak38802792021-08-01 14:38:07 -0700130 if *cpuProfile != "" {
131 f, err := os.Create(*cpuProfile)
132 if err != nil {
133 quit(err)
134 }
135 pprof.StartCPUProfile(f)
136 defer pprof.StopCPUProfile()
137 }
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800138 // Find out global variables
139 getConfigVariables()
140 getSoongVariables()
141
142 if *printProductConfigMap {
143 productConfigMap := buildProductConfigMap()
144 var products []string
145 for p := range productConfigMap {
146 products = append(products, p)
147 }
148 sort.Strings(products)
149 for _, p := range products {
150 fmt.Println(p, productConfigMap[p])
151 }
152 os.Exit(0)
153 }
Sasha Smundakb2ac8592021-07-26 09:27:22 -0700154
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800155 // Convert!
156 ok := true
157 if *launcher != "" {
158 if len(flag.Args()) != 1 {
159 quit(fmt.Errorf("a launcher can be generated only for a single product"))
160 }
161 product := flag.Args()[0]
162 productConfigMap := buildProductConfigMap()
163 path, found := productConfigMap[product]
164 if !found {
165 quit(fmt.Errorf("cannot generate configuration launcher for %s, it is not a known product",
166 product))
167 }
168 ok = convertOne(path) && ok
169 err := writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(path), mk2rbc.MakePath2ModuleName(path)))
170 if err != nil {
171 fmt.Fprintf(os.Stderr, "%s:%s", path, err)
172 ok = false
173 }
174
175 } else {
176 files := flag.Args()
177 if *allInSource {
178 productConfigMap := buildProductConfigMap()
179 for _, path := range productConfigMap {
180 files = append(files, path)
181 }
182 }
183 for _, mkFile := range files {
184 ok = convertOne(mkFile) && ok
185 }
186 }
187
188 printStats()
189 if *errstat {
190 errorLogger.printStatistics()
191 }
192 if !ok {
193 os.Exit(1)
194 }
195}
196
197func quit(s interface{}) {
198 fmt.Fprintln(os.Stderr, s)
199 os.Exit(2)
200}
201
202func buildProductConfigMap() map[string]string {
203 const androidProductsMk = "AndroidProducts.mk"
204 // Build the list of AndroidProducts.mk files: it's
205 // build/make/target/product/AndroidProducts.mk plus
206 // device/**/AndroidProducts.mk
207 targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk)
208 if _, err := os.Stat(targetAndroidProductsFile); err != nil {
209 fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n",
210 targetAndroidProductsFile, err, *rootDir)
211 }
212 productConfigMap := make(map[string]string)
213 if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil {
214 fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
215 }
216 _ = filepath.Walk(filepath.Join(*rootDir, "device"),
217 func(path string, info os.FileInfo, err error) error {
218 if info.IsDir() || filepath.Base(path) != androidProductsMk {
219 return nil
220 }
221 if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil {
222 fmt.Fprintf(os.Stderr, "%s: %s\n", path, err)
223 // Keep going, we want to find all such errors in a single run
224 }
225 return nil
226 })
227 return productConfigMap
228}
229
230func getConfigVariables() {
231 path := filepath.Join(*rootDir, "build", "make", "core", "product.mk")
232 if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil {
233 quit(fmt.Errorf("%s\n(check --root[=%s], it should point to the source root)",
234 err, *rootDir))
235 }
236}
237
238// Implements mkparser.Scope, to be used by mkparser.Value.Value()
239type fileNameScope struct {
240 mk2rbc.ScopeBase
241}
242
243func (s fileNameScope) Get(name string) string {
244 if name != "BUILD_SYSTEM" {
245 return fmt.Sprintf("$(%s)", name)
246 }
247 return filepath.Join(*rootDir, "build", "make", "core")
248}
249
250func getSoongVariables() {
251 path := filepath.Join(*rootDir, "build", "make", "core", "soong_config.mk")
252 err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables)
253 if err != nil {
254 quit(err)
255 }
256}
257
258var converted = make(map[string]*mk2rbc.StarlarkScript)
259
260//goland:noinspection RegExpRepeatedSpace
261var cpNormalizer = regexp.MustCompile(
262 "# Copyright \\(C\\) 20.. The Android Open Source Project")
263
264const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project"
265const copyright = `#
266# Copyright (C) 20xx The Android Open Source Project
267#
268# Licensed under the Apache License, Version 2.0 (the "License");
269# you may not use this file except in compliance with the License.
270# You may obtain a copy of the License at
271#
272# http://www.apache.org/licenses/LICENSE-2.0
273#
274# Unless required by applicable law or agreed to in writing, software
275# distributed under the License is distributed on an "AS IS" BASIS,
276# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
277# See the License for the specific language governing permissions and
278# limitations under the License.
279#
280`
281
282// Convert a single file.
283// Write the result either to the same directory, to the same place in
284// the output hierarchy, or to the stdout.
285// Optionally, recursively convert the files this one includes by
286// $(call inherit-product) or an include statement.
287func convertOne(mkFile string) (ok bool) {
288 if v, ok := converted[mkFile]; ok {
289 return v != nil
290 }
291 converted[mkFile] = nil
292 defer func() {
293 if r := recover(); r != nil {
294 ok = false
295 fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack())
296 }
297 }()
298
299 mk2starRequest := mk2rbc.Request{
300 MkFile: mkFile,
301 Reader: nil,
302 RootDir: *rootDir,
303 OutputDir: *outputTop,
304 OutputSuffix: *suffix,
305 TracedVariables: tracedVariables,
306 TraceCalls: *traceCalls,
307 WarnPartialSuccess: *warn,
Sasha Smundak6609ba72021-07-22 18:32:56 -0700308 SourceFS: os.DirFS(*rootDir),
309 MakefileFinder: makefileFinder,
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800310 }
311 if *errstat {
312 mk2starRequest.ErrorLogger = errorLogger
313 }
314 ss, err := mk2rbc.Convert(mk2starRequest)
315 if err != nil {
316 fmt.Fprintln(os.Stderr, mkFile, ": ", err)
317 return false
318 }
319 script := ss.String()
320 outputPath := outputFilePath(mkFile)
321
322 if *dryRun {
323 fmt.Printf("==== %s ====\n", outputPath)
324 // Print generated script after removing the copyright header
325 outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright)
326 fmt.Println(strings.TrimPrefix(outText, copyright))
327 } else {
328 if err := maybeBackup(outputPath); err != nil {
329 fmt.Fprintln(os.Stderr, err)
330 return false
331 }
332 if err := writeGenerated(outputPath, script); err != nil {
333 fmt.Fprintln(os.Stderr, err)
334 return false
335 }
336 }
337 ok = true
338 if *recurse {
339 for _, sub := range ss.SubConfigFiles() {
340 // File may be absent if it is a conditional load
341 if _, err := os.Stat(sub); os.IsNotExist(err) {
342 continue
343 }
344 ok = convertOne(sub) && ok
345 }
346 }
347 converted[mkFile] = ss
348 return ok
349}
350
351// Optionally saves the previous version of the generated file
352func maybeBackup(filename string) error {
353 stat, err := os.Stat(filename)
354 if os.IsNotExist(err) {
355 return nil
356 }
357 if !stat.Mode().IsRegular() {
358 return fmt.Errorf("%s exists and is not a regular file", filename)
359 }
360 switch *mode {
361 case "backup":
362 return os.Rename(filename, filename+backupSuffix)
363 case "write":
364 return os.Remove(filename)
365 default:
366 return fmt.Errorf("%s already exists, use --mode option", filename)
367 }
368}
369
370func outputFilePath(mkFile string) string {
371 path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix
372 if *outputTop != "" {
373 path = filepath.Join(*outputTop, path)
374 }
375 return path
376}
377
378func writeGenerated(path string, contents string) error {
379 if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
380 return err
381 }
382 if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil {
383 return err
384 }
385 return nil
386}
387
388func printStats() {
389 var sortedFiles []string
390 if !*warn && !*verbose {
391 return
392 }
393 for p := range converted {
394 sortedFiles = append(sortedFiles, p)
395 }
396 sort.Strings(sortedFiles)
397
398 nOk, nPartial, nFailed := 0, 0, 0
399 for _, f := range sortedFiles {
400 if converted[f] == nil {
401 nFailed++
402 } else if converted[f].HasErrors() {
403 nPartial++
404 } else {
405 nOk++
406 }
407 }
408 if *warn {
409 if nPartial > 0 {
410 fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n")
411 for _, f := range sortedFiles {
412 if ss := converted[f]; ss != nil && ss.HasErrors() {
413 fmt.Fprintln(os.Stderr, " ", f)
414 }
415 }
416 }
417
418 if nFailed > 0 {
419 fmt.Fprintf(os.Stderr, "Conversion failed for files:\n")
420 for _, f := range sortedFiles {
421 if converted[f] == nil {
422 fmt.Fprintln(os.Stderr, " ", f)
423 }
424 }
425 }
426 }
427 if *verbose {
428 fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Succeeded:", nOk)
429 fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Partial:", nPartial)
430 fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Failed:", nFailed)
431 }
432}
433
434type datum struct {
435 count int
436 formattingArgs []string
437}
438
439type errorsByType struct {
440 data map[string]datum
441}
442
443func (ebt errorsByType) NewError(message string, node parser.Node, args ...interface{}) {
444 v, exists := ebt.data[message]
445 if exists {
446 v.count++
447 } else {
448 v = datum{1, nil}
449 }
450 if strings.Contains(message, "%s") {
451 var newArg1 string
452 if len(args) == 0 {
453 panic(fmt.Errorf(`%s has %%s but args are missing`, message))
454 }
455 newArg1 = fmt.Sprint(args[0])
456 if message == "unsupported line" {
457 newArg1 = node.Dump()
458 } else if message == "unsupported directive %s" {
459 if newArg1 == "include" || newArg1 == "-include" {
460 newArg1 = node.Dump()
461 }
462 }
463 v.formattingArgs = append(v.formattingArgs, newArg1)
464 }
465 ebt.data[message] = v
466}
467
468func (ebt errorsByType) printStatistics() {
469 if len(ebt.data) > 0 {
470 fmt.Fprintln(os.Stderr, "Error counts:")
471 }
472 for message, data := range ebt.data {
473 if len(data.formattingArgs) == 0 {
474 fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message)
475 continue
476 }
477 itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30)
478 fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count)
479 fmt.Fprintln(os.Stderr, " ", itemsByFreq)
480 }
481}
482
483func stringsWithFreq(items []string, topN int) (string, int) {
484 freq := make(map[string]int)
485 for _, item := range items {
486 freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++
487 }
488 var sorted []string
489 for item := range freq {
490 sorted = append(sorted, item)
491 }
492 sort.Slice(sorted, func(i int, j int) bool {
493 return freq[sorted[i]] > freq[sorted[j]]
494 })
495 sep := ""
496 res := ""
497 for i, item := range sorted {
498 if i >= topN {
499 res += " ..."
500 break
501 }
502 count := freq[item]
503 if count > 1 {
504 res += fmt.Sprintf("%s%s(%d)", sep, item, count)
505 } else {
506 res += fmt.Sprintf("%s%s", sep, item)
507 }
508 sep = ", "
509 }
510 return res, len(sorted)
511}
Sasha Smundak6609ba72021-07-22 18:32:56 -0700512
513type LinuxMakefileFinder struct {
514 cachedRoot string
515 cachedMakefiles []string
516}
517
518func (l *LinuxMakefileFinder) Find(root string) []string {
519 if l.cachedMakefiles != nil && l.cachedRoot == root {
520 return l.cachedMakefiles
521 }
522 l.cachedRoot = root
523 l.cachedMakefiles = make([]string, 0)
524
525 // Return all *.mk files but not in hidden directories.
526
527 // NOTE(asmundak): as it turns out, even the WalkDir (which is an _optimized_ directory tree walker)
528 // is about twice slower than running `find` command (14s vs 6s on the internal Android source tree).
529 common_args := []string{"!", "-type", "d", "-name", "*.mk", "!", "-path", "*/.*/*"}
530 if root != "" {
531 common_args = append([]string{root}, common_args...)
532 }
533 cmd := exec.Command("/usr/bin/find", common_args...)
534 stdout, err := cmd.StdoutPipe()
535 if err == nil {
536 err = cmd.Start()
537 }
538 if err != nil {
539 panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
540 }
541 scanner := bufio.NewScanner(stdout)
542 for scanner.Scan() {
543 l.cachedMakefiles = append(l.cachedMakefiles, strings.TrimPrefix(scanner.Text(), "./"))
544 }
545 stdout.Close()
546 return l.cachedMakefiles
547}