Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 1 | // Copyright 2021 Google LLC |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | // The application to convert product configuration makefiles to Starlark. |
| 16 | // Converts either given list of files (and optionally the dependent files |
| 17 | // of the same kind), or all all product configuration makefiles in the |
| 18 | // given source tree. |
| 19 | // Previous version of a converted file can be backed up. |
| 20 | // Optionally prints detailed statistics at the end. |
| 21 | package main |
| 22 | |
| 23 | import ( |
Sasha Smundak | 6609ba7 | 2021-07-22 18:32:56 -0700 | [diff] [blame] | 24 | "bufio" |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 25 | "flag" |
| 26 | "fmt" |
| 27 | "io/ioutil" |
| 28 | "os" |
Sasha Smundak | 6609ba7 | 2021-07-22 18:32:56 -0700 | [diff] [blame] | 29 | "os/exec" |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 30 | "path/filepath" |
| 31 | "regexp" |
| 32 | "runtime/debug" |
Sasha Smundak | 3880279 | 2021-08-01 14:38:07 -0700 | [diff] [blame] | 33 | "runtime/pprof" |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 34 | "sort" |
| 35 | "strings" |
| 36 | "time" |
| 37 | |
| 38 | "android/soong/androidmk/parser" |
| 39 | "android/soong/mk2rbc" |
| 40 | ) |
| 41 | |
| 42 | var ( |
| 43 | rootDir = flag.String("root", ".", "the value of // for load paths") |
| 44 | // TODO(asmundak): remove this option once there is a consensus on suffix |
| 45 | suffix = flag.String("suffix", ".rbc", "generated files' suffix") |
| 46 | dryRun = flag.Bool("dry_run", false, "dry run") |
| 47 | recurse = flag.Bool("convert_dependents", false, "convert all dependent files") |
| 48 | mode = flag.String("mode", "", `"backup" to back up existing files, "write" to overwrite them`) |
Cole Faust | 17cecd7 | 2021-10-25 16:02:37 -0700 | [diff] [blame] | 49 | noWarn = flag.Bool("no_warnings", false, "don't warn about partially failed conversions") |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 50 | verbose = flag.Bool("v", false, "print summary") |
| 51 | errstat = flag.Bool("error_stat", false, "print error statistics") |
| 52 | traceVar = flag.String("trace", "", "comma-separated list of variables to trace") |
| 53 | // TODO(asmundak): this option is for debugging |
| 54 | allInSource = flag.Bool("all", false, "convert all product config makefiles in the tree under //") |
| 55 | outputTop = flag.String("outdir", "", "write output files into this directory hierarchy") |
Cole Faust | 6ed7cb4 | 2021-10-07 17:08:46 -0700 | [diff] [blame] | 56 | launcher = flag.String("launcher", "", "generated launcher path.") |
| 57 | boardlauncher = flag.String("boardlauncher", "", "generated board configuration launcher path.") |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 58 | printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit") |
Sasha Smundak | 3880279 | 2021-08-01 14:38:07 -0700 | [diff] [blame] | 59 | cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file") |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 60 | traceCalls = flag.Bool("trace_calls", false, "trace function calls") |
Cole Faust | 6ed7cb4 | 2021-10-07 17:08:46 -0700 | [diff] [blame] | 61 | inputVariables = flag.String("input_variables", "", "starlark file containing product config and global variables") |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 62 | ) |
| 63 | |
| 64 | func init() { |
Joe Onorato | b4638c1 | 2021-10-27 15:47:06 -0700 | [diff] [blame] | 65 | // Simplistic flag aliasing: works, but the usage string is ugly and |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 66 | // both flag and its alias can be present on the command line |
| 67 | flagAlias := func(target string, alias string) { |
| 68 | if f := flag.Lookup(target); f != nil { |
| 69 | flag.Var(f.Value, alias, "alias for --"+f.Name) |
| 70 | return |
| 71 | } |
| 72 | quit("cannot alias unknown flag " + target) |
| 73 | } |
| 74 | flagAlias("suffix", "s") |
| 75 | flagAlias("root", "d") |
| 76 | flagAlias("dry_run", "n") |
| 77 | flagAlias("convert_dependents", "r") |
Cole Faust | 17cecd7 | 2021-10-25 16:02:37 -0700 | [diff] [blame] | 78 | flagAlias("no_warnings", "w") |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 79 | flagAlias("error_stat", "e") |
| 80 | } |
| 81 | |
| 82 | var backupSuffix string |
| 83 | var tracedVariables []string |
Sasha Smundak | 7d934b9 | 2021-11-10 12:20:01 -0800 | [diff] [blame^] | 84 | var errorLogger = errorSink{data: make(map[string]datum)} |
Sasha Smundak | 6609ba7 | 2021-07-22 18:32:56 -0700 | [diff] [blame] | 85 | var makefileFinder = &LinuxMakefileFinder{} |
Sasha Smundak | d7d07ad | 2021-09-10 15:42:34 -0700 | [diff] [blame] | 86 | var versionDefaultsMk = filepath.Join("build", "make", "core", "version_defaults.mk") |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 87 | |
| 88 | func main() { |
| 89 | flag.Usage = func() { |
| 90 | cmd := filepath.Base(os.Args[0]) |
| 91 | fmt.Fprintf(flag.CommandLine.Output(), |
Cole Faust | 6ed7cb4 | 2021-10-07 17:08:46 -0700 | [diff] [blame] | 92 | "Usage: %[1]s flags file...\n", cmd) |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 93 | flag.PrintDefaults() |
| 94 | } |
| 95 | flag.Parse() |
| 96 | |
| 97 | // Delouse |
| 98 | if *suffix == ".mk" { |
| 99 | quit("cannot use .mk as generated file suffix") |
| 100 | } |
| 101 | if *suffix == "" { |
| 102 | quit("suffix cannot be empty") |
| 103 | } |
| 104 | if *outputTop != "" { |
| 105 | if err := os.MkdirAll(*outputTop, os.ModeDir+os.ModePerm); err != nil { |
| 106 | quit(err) |
| 107 | } |
| 108 | s, err := filepath.Abs(*outputTop) |
| 109 | if err != nil { |
| 110 | quit(err) |
| 111 | } |
| 112 | *outputTop = s |
| 113 | } |
| 114 | if *allInSource && len(flag.Args()) > 0 { |
| 115 | quit("file list cannot be specified when -all is present") |
| 116 | } |
| 117 | if *allInSource && *launcher != "" { |
| 118 | quit("--all and --launcher are mutually exclusive") |
| 119 | } |
| 120 | |
| 121 | // Flag-driven adjustments |
| 122 | if (*suffix)[0] != '.' { |
| 123 | *suffix = "." + *suffix |
| 124 | } |
| 125 | if *mode == "backup" { |
| 126 | backupSuffix = time.Now().Format("20060102150405") |
| 127 | } |
| 128 | if *traceVar != "" { |
| 129 | tracedVariables = strings.Split(*traceVar, ",") |
| 130 | } |
| 131 | |
Sasha Smundak | 3880279 | 2021-08-01 14:38:07 -0700 | [diff] [blame] | 132 | if *cpuProfile != "" { |
| 133 | f, err := os.Create(*cpuProfile) |
| 134 | if err != nil { |
| 135 | quit(err) |
| 136 | } |
| 137 | pprof.StartCPUProfile(f) |
| 138 | defer pprof.StopCPUProfile() |
| 139 | } |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 140 | // Find out global variables |
| 141 | getConfigVariables() |
| 142 | getSoongVariables() |
| 143 | |
| 144 | if *printProductConfigMap { |
| 145 | productConfigMap := buildProductConfigMap() |
| 146 | var products []string |
| 147 | for p := range productConfigMap { |
| 148 | products = append(products, p) |
| 149 | } |
| 150 | sort.Strings(products) |
| 151 | for _, p := range products { |
| 152 | fmt.Println(p, productConfigMap[p]) |
| 153 | } |
| 154 | os.Exit(0) |
| 155 | } |
Sasha Smundak | b2ac859 | 2021-07-26 09:27:22 -0700 | [diff] [blame] | 156 | |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 157 | // Convert! |
Cole Faust | 07ea503 | 2021-09-30 16:11:16 -0700 | [diff] [blame] | 158 | files := flag.Args() |
Cole Faust | 07ea503 | 2021-09-30 16:11:16 -0700 | [diff] [blame] | 159 | if *allInSource { |
Cole Faust | ebf79bf | 2021-10-05 11:05:02 -0700 | [diff] [blame] | 160 | productConfigMap := buildProductConfigMap() |
Cole Faust | 07ea503 | 2021-09-30 16:11:16 -0700 | [diff] [blame] | 161 | for _, path := range productConfigMap { |
| 162 | files = append(files, path) |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 163 | } |
Cole Faust | 07ea503 | 2021-09-30 16:11:16 -0700 | [diff] [blame] | 164 | } |
Cole Faust | 07ea503 | 2021-09-30 16:11:16 -0700 | [diff] [blame] | 165 | ok := true |
| 166 | for _, mkFile := range files { |
| 167 | ok = convertOne(mkFile) && ok |
| 168 | } |
| 169 | |
| 170 | if *launcher != "" { |
| 171 | if len(files) != 1 { |
| 172 | quit(fmt.Errorf("a launcher can be generated only for a single product")) |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 173 | } |
Sasha Smundak | d7d07ad | 2021-09-10 15:42:34 -0700 | [diff] [blame] | 174 | versionDefaults, err := generateVersionDefaults() |
| 175 | if err != nil { |
| 176 | quit(err) |
| 177 | } |
Sasha Smundak | d7d07ad | 2021-09-10 15:42:34 -0700 | [diff] [blame] | 178 | versionDefaultsPath := outputFilePath(versionDefaultsMk) |
| 179 | err = writeGenerated(versionDefaultsPath, versionDefaults) |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 180 | if err != nil { |
Cole Faust | 6ed7cb4 | 2021-10-07 17:08:46 -0700 | [diff] [blame] | 181 | fmt.Fprintf(os.Stderr, "%s: %s", files[0], err) |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 182 | ok = false |
| 183 | } |
| 184 | |
Cole Faust | 07ea503 | 2021-09-30 16:11:16 -0700 | [diff] [blame] | 185 | err = writeGenerated(*launcher, mk2rbc.Launcher(outputFilePath(files[0]), versionDefaultsPath, |
| 186 | mk2rbc.MakePath2ModuleName(files[0]))) |
Sasha Smundak | d7d07ad | 2021-09-10 15:42:34 -0700 | [diff] [blame] | 187 | if err != nil { |
Cole Faust | 6ed7cb4 | 2021-10-07 17:08:46 -0700 | [diff] [blame] | 188 | fmt.Fprintf(os.Stderr, "%s: %s", files[0], err) |
| 189 | ok = false |
| 190 | } |
| 191 | } |
| 192 | if *boardlauncher != "" { |
| 193 | if len(files) != 1 { |
| 194 | quit(fmt.Errorf("a launcher can be generated only for a single product")) |
| 195 | } |
| 196 | if *inputVariables == "" { |
| 197 | quit(fmt.Errorf("the board launcher requires an input variables file")) |
| 198 | } |
| 199 | if !convertOne(*inputVariables) { |
| 200 | quit(fmt.Errorf("the board launcher input variables file failed to convert")) |
| 201 | } |
| 202 | err := writeGenerated(*boardlauncher, mk2rbc.BoardLauncher( |
| 203 | outputFilePath(files[0]), outputFilePath(*inputVariables))) |
| 204 | if err != nil { |
| 205 | fmt.Fprintf(os.Stderr, "%s: %s", files[0], err) |
Sasha Smundak | d7d07ad | 2021-09-10 15:42:34 -0700 | [diff] [blame] | 206 | ok = false |
| 207 | } |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 208 | } |
| 209 | |
| 210 | printStats() |
| 211 | if *errstat { |
| 212 | errorLogger.printStatistics() |
| 213 | } |
| 214 | if !ok { |
| 215 | os.Exit(1) |
| 216 | } |
| 217 | } |
| 218 | |
Sasha Smundak | d7d07ad | 2021-09-10 15:42:34 -0700 | [diff] [blame] | 219 | func generateVersionDefaults() (string, error) { |
| 220 | versionSettings, err := mk2rbc.ParseVersionDefaults(filepath.Join(*rootDir, versionDefaultsMk)) |
| 221 | if err != nil { |
| 222 | return "", err |
| 223 | } |
| 224 | return mk2rbc.VersionDefaults(versionSettings), nil |
| 225 | |
| 226 | } |
| 227 | |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 228 | func quit(s interface{}) { |
| 229 | fmt.Fprintln(os.Stderr, s) |
| 230 | os.Exit(2) |
| 231 | } |
| 232 | |
| 233 | func buildProductConfigMap() map[string]string { |
| 234 | const androidProductsMk = "AndroidProducts.mk" |
| 235 | // Build the list of AndroidProducts.mk files: it's |
Sasha Smundak | 70f1745 | 2021-09-01 19:14:24 -0700 | [diff] [blame] | 236 | // build/make/target/product/AndroidProducts.mk + device/**/AndroidProducts.mk plus + vendor/**/AndroidProducts.mk |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 237 | targetAndroidProductsFile := filepath.Join(*rootDir, "build", "make", "target", "product", androidProductsMk) |
| 238 | if _, err := os.Stat(targetAndroidProductsFile); err != nil { |
| 239 | fmt.Fprintf(os.Stderr, "%s: %s\n(hint: %s is not a source tree root)\n", |
| 240 | targetAndroidProductsFile, err, *rootDir) |
| 241 | } |
| 242 | productConfigMap := make(map[string]string) |
| 243 | if err := mk2rbc.UpdateProductConfigMap(productConfigMap, targetAndroidProductsFile); err != nil { |
| 244 | fmt.Fprintf(os.Stderr, "%s: %s\n", targetAndroidProductsFile, err) |
| 245 | } |
Sasha Smundak | 70f1745 | 2021-09-01 19:14:24 -0700 | [diff] [blame] | 246 | for _, t := range []string{"device", "vendor"} { |
| 247 | _ = filepath.WalkDir(filepath.Join(*rootDir, t), |
| 248 | func(path string, d os.DirEntry, err error) error { |
| 249 | if err != nil || d.IsDir() || filepath.Base(path) != androidProductsMk { |
| 250 | return nil |
| 251 | } |
| 252 | if err2 := mk2rbc.UpdateProductConfigMap(productConfigMap, path); err2 != nil { |
| 253 | fmt.Fprintf(os.Stderr, "%s: %s\n", path, err) |
| 254 | // Keep going, we want to find all such errors in a single run |
| 255 | } |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 256 | return nil |
Sasha Smundak | 70f1745 | 2021-09-01 19:14:24 -0700 | [diff] [blame] | 257 | }) |
| 258 | } |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 259 | return productConfigMap |
| 260 | } |
| 261 | |
| 262 | func getConfigVariables() { |
| 263 | path := filepath.Join(*rootDir, "build", "make", "core", "product.mk") |
| 264 | if err := mk2rbc.FindConfigVariables(path, mk2rbc.KnownVariables); err != nil { |
| 265 | quit(fmt.Errorf("%s\n(check --root[=%s], it should point to the source root)", |
| 266 | err, *rootDir)) |
| 267 | } |
| 268 | } |
| 269 | |
| 270 | // Implements mkparser.Scope, to be used by mkparser.Value.Value() |
| 271 | type fileNameScope struct { |
| 272 | mk2rbc.ScopeBase |
| 273 | } |
| 274 | |
| 275 | func (s fileNameScope) Get(name string) string { |
| 276 | if name != "BUILD_SYSTEM" { |
| 277 | return fmt.Sprintf("$(%s)", name) |
| 278 | } |
| 279 | return filepath.Join(*rootDir, "build", "make", "core") |
| 280 | } |
| 281 | |
| 282 | func getSoongVariables() { |
| 283 | path := filepath.Join(*rootDir, "build", "make", "core", "soong_config.mk") |
| 284 | err := mk2rbc.FindSoongVariables(path, fileNameScope{}, mk2rbc.KnownVariables) |
| 285 | if err != nil { |
| 286 | quit(err) |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | var converted = make(map[string]*mk2rbc.StarlarkScript) |
| 291 | |
| 292 | //goland:noinspection RegExpRepeatedSpace |
| 293 | var cpNormalizer = regexp.MustCompile( |
| 294 | "# Copyright \\(C\\) 20.. The Android Open Source Project") |
| 295 | |
| 296 | const cpNormalizedCopyright = "# Copyright (C) 20xx The Android Open Source Project" |
| 297 | const copyright = `# |
| 298 | # Copyright (C) 20xx The Android Open Source Project |
| 299 | # |
| 300 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 301 | # you may not use this file except in compliance with the License. |
| 302 | # You may obtain a copy of the License at |
| 303 | # |
| 304 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 305 | # |
| 306 | # Unless required by applicable law or agreed to in writing, software |
| 307 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 308 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 309 | # See the License for the specific language governing permissions and |
| 310 | # limitations under the License. |
| 311 | # |
| 312 | ` |
| 313 | |
| 314 | // Convert a single file. |
| 315 | // Write the result either to the same directory, to the same place in |
| 316 | // the output hierarchy, or to the stdout. |
| 317 | // Optionally, recursively convert the files this one includes by |
| 318 | // $(call inherit-product) or an include statement. |
| 319 | func convertOne(mkFile string) (ok bool) { |
| 320 | if v, ok := converted[mkFile]; ok { |
| 321 | return v != nil |
| 322 | } |
| 323 | converted[mkFile] = nil |
| 324 | defer func() { |
| 325 | if r := recover(); r != nil { |
| 326 | ok = false |
| 327 | fmt.Fprintf(os.Stderr, "%s: panic while converting: %s\n%s\n", mkFile, r, debug.Stack()) |
| 328 | } |
| 329 | }() |
| 330 | |
| 331 | mk2starRequest := mk2rbc.Request{ |
| 332 | MkFile: mkFile, |
| 333 | Reader: nil, |
| 334 | RootDir: *rootDir, |
| 335 | OutputDir: *outputTop, |
| 336 | OutputSuffix: *suffix, |
| 337 | TracedVariables: tracedVariables, |
| 338 | TraceCalls: *traceCalls, |
Cole Faust | 17cecd7 | 2021-10-25 16:02:37 -0700 | [diff] [blame] | 339 | WarnPartialSuccess: !*noWarn, |
Sasha Smundak | 6609ba7 | 2021-07-22 18:32:56 -0700 | [diff] [blame] | 340 | SourceFS: os.DirFS(*rootDir), |
| 341 | MakefileFinder: makefileFinder, |
Sasha Smundak | 7d934b9 | 2021-11-10 12:20:01 -0800 | [diff] [blame^] | 342 | ErrorLogger: errorLogger, |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 343 | } |
| 344 | ss, err := mk2rbc.Convert(mk2starRequest) |
| 345 | if err != nil { |
| 346 | fmt.Fprintln(os.Stderr, mkFile, ": ", err) |
| 347 | return false |
| 348 | } |
| 349 | script := ss.String() |
| 350 | outputPath := outputFilePath(mkFile) |
| 351 | |
| 352 | if *dryRun { |
| 353 | fmt.Printf("==== %s ====\n", outputPath) |
| 354 | // Print generated script after removing the copyright header |
| 355 | outText := cpNormalizer.ReplaceAllString(script, cpNormalizedCopyright) |
| 356 | fmt.Println(strings.TrimPrefix(outText, copyright)) |
| 357 | } else { |
| 358 | if err := maybeBackup(outputPath); err != nil { |
| 359 | fmt.Fprintln(os.Stderr, err) |
| 360 | return false |
| 361 | } |
| 362 | if err := writeGenerated(outputPath, script); err != nil { |
| 363 | fmt.Fprintln(os.Stderr, err) |
| 364 | return false |
| 365 | } |
| 366 | } |
| 367 | ok = true |
| 368 | if *recurse { |
| 369 | for _, sub := range ss.SubConfigFiles() { |
| 370 | // File may be absent if it is a conditional load |
| 371 | if _, err := os.Stat(sub); os.IsNotExist(err) { |
| 372 | continue |
| 373 | } |
| 374 | ok = convertOne(sub) && ok |
| 375 | } |
| 376 | } |
| 377 | converted[mkFile] = ss |
| 378 | return ok |
| 379 | } |
| 380 | |
| 381 | // Optionally saves the previous version of the generated file |
| 382 | func maybeBackup(filename string) error { |
| 383 | stat, err := os.Stat(filename) |
| 384 | if os.IsNotExist(err) { |
| 385 | return nil |
| 386 | } |
| 387 | if !stat.Mode().IsRegular() { |
| 388 | return fmt.Errorf("%s exists and is not a regular file", filename) |
| 389 | } |
| 390 | switch *mode { |
| 391 | case "backup": |
| 392 | return os.Rename(filename, filename+backupSuffix) |
| 393 | case "write": |
| 394 | return os.Remove(filename) |
| 395 | default: |
| 396 | return fmt.Errorf("%s already exists, use --mode option", filename) |
| 397 | } |
| 398 | } |
| 399 | |
| 400 | func outputFilePath(mkFile string) string { |
| 401 | path := strings.TrimSuffix(mkFile, filepath.Ext(mkFile)) + *suffix |
| 402 | if *outputTop != "" { |
| 403 | path = filepath.Join(*outputTop, path) |
| 404 | } |
| 405 | return path |
| 406 | } |
| 407 | |
| 408 | func writeGenerated(path string, contents string) error { |
| 409 | if err := os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil { |
| 410 | return err |
| 411 | } |
| 412 | if err := ioutil.WriteFile(path, []byte(contents), 0644); err != nil { |
| 413 | return err |
| 414 | } |
| 415 | return nil |
| 416 | } |
| 417 | |
| 418 | func printStats() { |
| 419 | var sortedFiles []string |
Cole Faust | 17cecd7 | 2021-10-25 16:02:37 -0700 | [diff] [blame] | 420 | if *noWarn && !*verbose { |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 421 | return |
| 422 | } |
| 423 | for p := range converted { |
| 424 | sortedFiles = append(sortedFiles, p) |
| 425 | } |
| 426 | sort.Strings(sortedFiles) |
| 427 | |
| 428 | nOk, nPartial, nFailed := 0, 0, 0 |
| 429 | for _, f := range sortedFiles { |
| 430 | if converted[f] == nil { |
| 431 | nFailed++ |
| 432 | } else if converted[f].HasErrors() { |
| 433 | nPartial++ |
| 434 | } else { |
| 435 | nOk++ |
| 436 | } |
| 437 | } |
Cole Faust | 17cecd7 | 2021-10-25 16:02:37 -0700 | [diff] [blame] | 438 | if !*noWarn { |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 439 | if nPartial > 0 { |
| 440 | fmt.Fprintf(os.Stderr, "Conversion was partially successful for:\n") |
| 441 | for _, f := range sortedFiles { |
| 442 | if ss := converted[f]; ss != nil && ss.HasErrors() { |
| 443 | fmt.Fprintln(os.Stderr, " ", f) |
| 444 | } |
| 445 | } |
| 446 | } |
| 447 | |
| 448 | if nFailed > 0 { |
| 449 | fmt.Fprintf(os.Stderr, "Conversion failed for files:\n") |
| 450 | for _, f := range sortedFiles { |
| 451 | if converted[f] == nil { |
| 452 | fmt.Fprintln(os.Stderr, " ", f) |
| 453 | } |
| 454 | } |
| 455 | } |
| 456 | } |
Sasha Smundak | 7d934b9 | 2021-11-10 12:20:01 -0800 | [diff] [blame^] | 457 | if *verbose && (nPartial > 0 || nFailed > 0) { |
| 458 | fmt.Fprintln(os.Stderr, "Succeeded: ", nOk, " Partial: ", nPartial, " Failed: ", nFailed) |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 459 | } |
| 460 | } |
| 461 | |
| 462 | type datum struct { |
| 463 | count int |
| 464 | formattingArgs []string |
| 465 | } |
| 466 | |
Sasha Smundak | 7d934b9 | 2021-11-10 12:20:01 -0800 | [diff] [blame^] | 467 | type errorSink struct { |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 468 | data map[string]datum |
| 469 | } |
| 470 | |
Sasha Smundak | 7d934b9 | 2021-11-10 12:20:01 -0800 | [diff] [blame^] | 471 | func (ebt errorSink) NewError(sourceFile string, sourceLine int, node parser.Node, message string, args ...interface{}) { |
| 472 | fmt.Fprintf(os.Stderr, "%s:%d ", sourceFile, sourceLine) |
| 473 | fmt.Fprintf(os.Stderr, message, args...) |
| 474 | fmt.Fprintln(os.Stderr) |
| 475 | if !*errstat { |
| 476 | return |
| 477 | } |
| 478 | |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 479 | v, exists := ebt.data[message] |
| 480 | if exists { |
| 481 | v.count++ |
| 482 | } else { |
| 483 | v = datum{1, nil} |
| 484 | } |
| 485 | if strings.Contains(message, "%s") { |
| 486 | var newArg1 string |
| 487 | if len(args) == 0 { |
| 488 | panic(fmt.Errorf(`%s has %%s but args are missing`, message)) |
| 489 | } |
| 490 | newArg1 = fmt.Sprint(args[0]) |
| 491 | if message == "unsupported line" { |
| 492 | newArg1 = node.Dump() |
| 493 | } else if message == "unsupported directive %s" { |
| 494 | if newArg1 == "include" || newArg1 == "-include" { |
| 495 | newArg1 = node.Dump() |
| 496 | } |
| 497 | } |
| 498 | v.formattingArgs = append(v.formattingArgs, newArg1) |
| 499 | } |
| 500 | ebt.data[message] = v |
| 501 | } |
| 502 | |
Sasha Smundak | 7d934b9 | 2021-11-10 12:20:01 -0800 | [diff] [blame^] | 503 | func (ebt errorSink) printStatistics() { |
Sasha Smundak | b051c4e | 2020-11-05 20:45:07 -0800 | [diff] [blame] | 504 | if len(ebt.data) > 0 { |
| 505 | fmt.Fprintln(os.Stderr, "Error counts:") |
| 506 | } |
| 507 | for message, data := range ebt.data { |
| 508 | if len(data.formattingArgs) == 0 { |
| 509 | fmt.Fprintf(os.Stderr, "%4d %s\n", data.count, message) |
| 510 | continue |
| 511 | } |
| 512 | itemsByFreq, count := stringsWithFreq(data.formattingArgs, 30) |
| 513 | fmt.Fprintf(os.Stderr, "%4d %s [%d unique items]:\n", data.count, message, count) |
| 514 | fmt.Fprintln(os.Stderr, " ", itemsByFreq) |
| 515 | } |
| 516 | } |
| 517 | |
| 518 | func stringsWithFreq(items []string, topN int) (string, int) { |
| 519 | freq := make(map[string]int) |
| 520 | for _, item := range items { |
| 521 | freq[strings.TrimPrefix(strings.TrimSuffix(item, "]"), "[")]++ |
| 522 | } |
| 523 | var sorted []string |
| 524 | for item := range freq { |
| 525 | sorted = append(sorted, item) |
| 526 | } |
| 527 | sort.Slice(sorted, func(i int, j int) bool { |
| 528 | return freq[sorted[i]] > freq[sorted[j]] |
| 529 | }) |
| 530 | sep := "" |
| 531 | res := "" |
| 532 | for i, item := range sorted { |
| 533 | if i >= topN { |
| 534 | res += " ..." |
| 535 | break |
| 536 | } |
| 537 | count := freq[item] |
| 538 | if count > 1 { |
| 539 | res += fmt.Sprintf("%s%s(%d)", sep, item, count) |
| 540 | } else { |
| 541 | res += fmt.Sprintf("%s%s", sep, item) |
| 542 | } |
| 543 | sep = ", " |
| 544 | } |
| 545 | return res, len(sorted) |
| 546 | } |
Sasha Smundak | 6609ba7 | 2021-07-22 18:32:56 -0700 | [diff] [blame] | 547 | |
| 548 | type LinuxMakefileFinder struct { |
| 549 | cachedRoot string |
| 550 | cachedMakefiles []string |
| 551 | } |
| 552 | |
| 553 | func (l *LinuxMakefileFinder) Find(root string) []string { |
| 554 | if l.cachedMakefiles != nil && l.cachedRoot == root { |
| 555 | return l.cachedMakefiles |
| 556 | } |
| 557 | l.cachedRoot = root |
| 558 | l.cachedMakefiles = make([]string, 0) |
| 559 | |
| 560 | // Return all *.mk files but not in hidden directories. |
| 561 | |
| 562 | // NOTE(asmundak): as it turns out, even the WalkDir (which is an _optimized_ directory tree walker) |
| 563 | // is about twice slower than running `find` command (14s vs 6s on the internal Android source tree). |
| 564 | common_args := []string{"!", "-type", "d", "-name", "*.mk", "!", "-path", "*/.*/*"} |
| 565 | if root != "" { |
| 566 | common_args = append([]string{root}, common_args...) |
| 567 | } |
| 568 | cmd := exec.Command("/usr/bin/find", common_args...) |
| 569 | stdout, err := cmd.StdoutPipe() |
| 570 | if err == nil { |
| 571 | err = cmd.Start() |
| 572 | } |
| 573 | if err != nil { |
| 574 | panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err)) |
| 575 | } |
| 576 | scanner := bufio.NewScanner(stdout) |
| 577 | for scanner.Scan() { |
| 578 | l.cachedMakefiles = append(l.cachedMakefiles, strings.TrimPrefix(scanner.Text(), "./")) |
| 579 | } |
| 580 | stdout.Close() |
| 581 | return l.cachedMakefiles |
| 582 | } |