blob: 4d3d8d8823a8d41d64fcd08c370c4baf268d86f1 [file] [log] [blame]
Sasha Smundakb051c4e2020-11-05 20:45:07 -08001// Copyright 2021 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// The application to convert product configuration makefiles to Starlark.
16// Converts either given list of files (and optionally the dependent files
17// of the same kind), or all all product configuration makefiles in the
18// given source tree.
19// Previous version of a converted file can be backed up.
20// Optionally prints detailed statistics at the end.
21package main
22
23import (
24 "flag"
25 "fmt"
26 "io/ioutil"
27 "os"
28 "path/filepath"
29 "regexp"
30 "runtime/debug"
31 "sort"
32 "strings"
33 "time"
34
35 "android/soong/androidmk/parser"
36 "android/soong/mk2rbc"
37)
38
39var (
40 rootDir = flag.String("root", ".", "the value of // for load paths")
41 // TODO(asmundak): remove this option once there is a consensus on suffix
42 suffix = flag.String("suffix", ".rbc", "generated files' suffix")
43 dryRun = flag.Bool("dry_run", false, "dry run")
44 recurse = flag.Bool("convert_dependents", false, "convert all dependent files")
45 mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`)
46 warn = flag.Bool("warnings", false, "warn about partially failed conversions")
47 verbose = flag.Bool("v", false, "print summary")
48 errstat = flag.Bool("error_stat", false, "print error statistics")
49 traceVar = flag.String("trace", "", "comma-separated list of variables to trace")
50 // TODO(asmundak): this option is for debugging
51 allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //")
52 outputTop = flag.String("outdir", "", "write output files into this directory hierarchy")
53 launcher = flag.String("launcher", "", "generated launcher path. If set, the non-flag argument is _product_name_")
54 printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit")
55 traceCalls = flag.Bool("trace_calls", false, "trace function calls")
56)
57
58func init() {
59 // Poor man's flag aliasing: works, but the usage string is ugly and
60 // both flag and its alias can be present on the command line
61 flagAlias := func(target string, alias string) {
62 if f := flag.Lookup(target); f != nil {
63 flag.Var(f.Value, alias, "alias for --"+f.Name)
64 return
65 }
66 quit("cannot alias unknown flag " + target)
67 }
68 flagAlias("suffix", "s")
69 flagAlias("root", "d")
70 flagAlias("dry_run", "n")
71 flagAlias("convert_dependents", "r")
72 flagAlias("warnings", "w")
73 flagAlias("error_stat", "e")
74}
75
76var backupSuffix string
77var tracedVariables []string
78var errorLogger = errorsByType{data: make(map[string]datum)}
79
80func main() {
81 flag.Usage = func() {
82 cmd := filepath.Base(os.Args[0])
83 fmt.Fprintf(flag.CommandLine.Output(),
84 "Usage: %[1]s flags file...\n"+
85 "or: %[1]s flags --launcher=PATH PRODUCT\n", cmd)
86 flag.PrintDefaults()
87 }
88 flag.Parse()
89
90 // Delouse
91 if *suffix == ".mk" {
92 quit("cannot use .mk as generated file suffix")
93 }
94 if *suffix == "" {
95 quit("suffix cannot be empty")
96 }
97 if *outputTop != "" {
98 if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil {
99 quit(err)
100 }
101 s, err := filepath.Abs(*outputTop)
102 if err != nil {
103 quit(err)
104 }
105 *outputTop = s
106 }
107 if *allInSource && len(flag.Args()) > 0 {
108 quit("file list cannot be specified when -all is present")
109 }
110 if *allInSource && *launcher != "" {
111 quit("--all and --launcher are mutually exclusive")
112 }
113
114 // Flag-driven adjustments
115 if (*suffix)[0] != '.' {
116 *suffix = "." + *suffix
117 }
118 if *mode == "backup" {
119 backupSuffix = time.Now().Format("20060102150405")
120 }
121 if *traceVar != "" {
122 tracedVariables = strings.Split(*traceVar, ",")
123 }
124
125 // Find out global variables
126 getConfigVariables()
127 getSoongVariables()
128
129 if *printProductConfigMap {
130 productConfigMap := buildProductConfigMap()
131 var products []string
132 for p := range productConfigMap {
133 products = append(products, p)
134 }
135 sort.Strings(products)
136 for _, p := range products {
137 fmt.Println(p, productConfigMap[p])
138 }
139 os.Exit(0)
140 }
Sasha Smundakb2ac8592021-07-26 09:27:22 -0700141
Sasha Smundakb051c4e2020-11-05 20:45:07 -0800142 // Convert!
143 ok := true
144 if *launcher != "" {
145 if len(flag.Args()) != 1 {
146 quit(fmt.Errorf("a launcher can be generated only for a single product"))
147 }
148 product := flag.Args()[0]
149 productConfigMap := buildProductConfigMap()
150 path, found := productConfigMap[product]
151 if !found {
152 quit(fmt.Errorf("cannot generate configuration launcher for %s, it is not a known product",
153 product))
154 }
155 ok = convertOne(path) && ok
156 err := writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(path), mk2rbc.MakePath2ModuleName(path)))
157 if err != nil {
158 fmt.Fprintf(os.Stderr, "%s:%s", path, err)
159 ok = false
160 }
161
162 } else {
163 files := flag.Args()
164 if *allInSource {
165 productConfigMap := buildProductConfigMap()
166 for _, path := range productConfigMap {
167 files = append(files, path)
168 }
169 }
170 for _, mkFile := range files {
171 ok = convertOne(mkFile) && ok
172 }
173 }
174
175 printStats()
176 if *errstat {
177 errorLogger.printStatistics()
178 }
179 if !ok {
180 os.Exit(1)
181 }
182}
183
184func quit(s interface{}) {
185 fmt.Fprintln(os.Stderr, s)
186 os.Exit(2)
187}
188
189func buildProductConfigMap() map[string]string {
190 const androidProductsMk = "AndroidProducts.mk"
191 // Build the list of AndroidProducts.mk files: it's
192 // build/make/target/product/AndroidProducts.mk plus
193 // device/**/AndroidProducts.mk
194 targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk)
195 if _, err := os.Stat(targetAndroidProductsFile); err != nil {
196 fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n",
197 targetAndroidProductsFile, err, *rootDir)
198 }
199 productConfigMap := make(map[string]string)
200 if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil {
201 fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err)
202 }
203 _ = filepath.Walk(filepath.Join(*rootDir, "device"),
204 func(path string, info os.FileInfo, err error) error {
205 if info.IsDir() || filepath.Base(path) != androidProductsMk {
206 return nil
207 }
208 if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil {
209 fmt.Fprintf(os.Stderr, "%s: %s\n", path, err)
210 // Keep going, we want to find all such errors in a single run
211 }
212 return nil
213 })
214 return productConfigMap
215}
216
217func getConfigVariables() {
218 path := filepath.Join(*rootDir, "build", "make", "core", "product.mk")
219 if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil {
220 quit(fmt.Errorf("%s\n(check --root[=%s], it should point to the source root)",
221 err, *rootDir))
222 }
223}
224
225// Implements mkparser.Scope, to be used by mkparser.Value.Value()
226type fileNameScope struct {
227 mk2rbc.ScopeBase
228}
229
230func (s fileNameScope) Get(name string) string {
231 if name != "BUILD_SYSTEM" {
232 return fmt.Sprintf("$(%s)", name)
233 }
234 return filepath.Join(*rootDir, "build", "make", "core")
235}
236
237func getSoongVariables() {
238 path := filepath.Join(*rootDir, "build", "make", "core", "soong_config.mk")
239 err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables)
240 if err != nil {
241 quit(err)
242 }
243}
244
245var converted = make(map[string]*mk2rbc.StarlarkScript)
246
247//goland:noinspection RegExpRepeatedSpace
248var cpNormalizer = regexp.MustCompile(
249 "# Copyright \\(C\\) 20.. The Android Open Source Project")
250
251const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project"
252const copyright = `#
253# Copyright (C) 20xx The Android Open Source Project
254#
255# Licensed under the Apache License, Version 2.0 (the "License");
256# you may not use this file except in compliance with the License.
257# You may obtain a copy of the License at
258#
259# http://www.apache.org/licenses/LICENSE-2.0
260#
261# Unless required by applicable law or agreed to in writing, software
262# distributed under the License is distributed on an "AS IS" BASIS,
263# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
264# See the License for the specific language governing permissions and
265# limitations under the License.
266#
267`
268
269// Convert a single file.
270// Write the result either to the same directory, to the same place in
271// the output hierarchy, or to the stdout.
272// Optionally, recursively convert the files this one includes by
273// $(call inherit-product) or an include statement.
274func convertOne(mkFile string) (ok bool) {
275 if v, ok := converted[mkFile]; ok {
276 return v != nil
277 }
278 converted[mkFile] = nil
279 defer func() {
280 if r := recover(); r != nil {
281 ok = false
282 fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack())
283 }
284 }()
285
286 mk2starRequest := mk2rbc.Request{
287 MkFile: mkFile,
288 Reader: nil,
289 RootDir: *rootDir,
290 OutputDir: *outputTop,
291 OutputSuffix: *suffix,
292 TracedVariables: tracedVariables,
293 TraceCalls: *traceCalls,
294 WarnPartialSuccess: *warn,
295 }
296 if *errstat {
297 mk2starRequest.ErrorLogger = errorLogger
298 }
299 ss, err := mk2rbc.Convert(mk2starRequest)
300 if err != nil {
301 fmt.Fprintln(os.Stderr, mkFile, ": ", err)
302 return false
303 }
304 script := ss.String()
305 outputPath := outputFilePath(mkFile)
306
307 if *dryRun {
308 fmt.Printf("==== %s ====\n", outputPath)
309 // Print generated script after removing the copyright header
310 outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright)
311 fmt.Println(strings.TrimPrefix(outText, copyright))
312 } else {
313 if err := maybeBackup(outputPath); err != nil {
314 fmt.Fprintln(os.Stderr, err)
315 return false
316 }
317 if err := writeGenerated(outputPath, script); err != nil {
318 fmt.Fprintln(os.Stderr, err)
319 return false
320 }
321 }
322 ok = true
323 if *recurse {
324 for _, sub := range ss.SubConfigFiles() {
325 // File may be absent if it is a conditional load
326 if _, err := os.Stat(sub); os.IsNotExist(err) {
327 continue
328 }
329 ok = convertOne(sub) && ok
330 }
331 }
332 converted[mkFile] = ss
333 return ok
334}
335
336// Optionally saves the previous version of the generated file
337func maybeBackup(filename string) error {
338 stat, err := os.Stat(filename)
339 if os.IsNotExist(err) {
340 return nil
341 }
342 if !stat.Mode().IsRegular() {
343 return fmt.Errorf("%s exists and is not a regular file", filename)
344 }
345 switch *mode {
346 case "backup":
347 return os.Rename(filename, filename+backupSuffix)
348 case "write":
349 return os.Remove(filename)
350 default:
351 return fmt.Errorf("%s already exists, use --mode option", filename)
352 }
353}
354
355func outputFilePath(mkFile string) string {
356 path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix
357 if *outputTop != "" {
358 path = filepath.Join(*outputTop, path)
359 }
360 return path
361}
362
363func writeGenerated(path string, contents string) error {
364 if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
365 return err
366 }
367 if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil {
368 return err
369 }
370 return nil
371}
372
373func printStats() {
374 var sortedFiles []string
375 if !*warn && !*verbose {
376 return
377 }
378 for p := range converted {
379 sortedFiles = append(sortedFiles, p)
380 }
381 sort.Strings(sortedFiles)
382
383 nOk, nPartial, nFailed := 0, 0, 0
384 for _, f := range sortedFiles {
385 if converted[f] == nil {
386 nFailed++
387 } else if converted[f].HasErrors() {
388 nPartial++
389 } else {
390 nOk++
391 }
392 }
393 if *warn {
394 if nPartial > 0 {
395 fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n")
396 for _, f := range sortedFiles {
397 if ss := converted[f]; ss != nil && ss.HasErrors() {
398 fmt.Fprintln(os.Stderr, " ", f)
399 }
400 }
401 }
402
403 if nFailed > 0 {
404 fmt.Fprintf(os.Stderr, "Conversion failed for files:\n")
405 for _, f := range sortedFiles {
406 if converted[f] == nil {
407 fmt.Fprintln(os.Stderr, " ", f)
408 }
409 }
410 }
411 }
412 if *verbose {
413 fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Succeeded:", nOk)
414 fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Partial:", nPartial)
415 fmt.Fprintf(os.Stderr, "%-16s%5d\n", "Failed:", nFailed)
416 }
417}
418
419type datum struct {
420 count int
421 formattingArgs []string
422}
423
424type errorsByType struct {
425 data map[string]datum
426}
427
428func (ebt errorsByType) NewError(message string, node parser.Node, args ...interface{}) {
429 v, exists := ebt.data[message]
430 if exists {
431 v.count++
432 } else {
433 v = datum{1, nil}
434 }
435 if strings.Contains(message, "%s") {
436 var newArg1 string
437 if len(args) == 0 {
438 panic(fmt.Errorf(`%s has %%s but args are missing`, message))
439 }
440 newArg1 = fmt.Sprint(args[0])
441 if message == "unsupported line" {
442 newArg1 = node.Dump()
443 } else if message == "unsupported directive %s" {
444 if newArg1 == "include" || newArg1 == "-include" {
445 newArg1 = node.Dump()
446 }
447 }
448 v.formattingArgs = append(v.formattingArgs, newArg1)
449 }
450 ebt.data[message] = v
451}
452
453func (ebt errorsByType) printStatistics() {
454 if len(ebt.data) > 0 {
455 fmt.Fprintln(os.Stderr, "Error counts:")
456 }
457 for message, data := range ebt.data {
458 if len(data.formattingArgs) == 0 {
459 fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message)
460 continue
461 }
462 itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30)
463 fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count)
464 fmt.Fprintln(os.Stderr, " ", itemsByFreq)
465 }
466}
467
468func stringsWithFreq(items []string, topN int) (string, int) {
469 freq := make(map[string]int)
470 for _, item := range items {
471 freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++
472 }
473 var sorted []string
474 for item := range freq {
475 sorted = append(sorted, item)
476 }
477 sort.Slice(sorted, func(i int, j int) bool {
478 return freq[sorted[i]] > freq[sorted[j]]
479 })
480 sep := ""
481 res := ""
482 for i, item := range sorted {
483 if i >= topN {
484 res += " ..."
485 break
486 }
487 count := freq[item]
488 if count > 1 {
489 res += fmt.Sprintf("%s%s(%d)", sep, item, count)
490 } else {
491 res += fmt.Sprintf("%s%s", sep, item)
492 }
493 sep = ", "
494 }
495 return res, len(sorted)
496}