Use soong's finder to find included makefiles in mk2rbc
Having soong generate a list of makefiles for mk2rbc
to look through is much faster than having mk2rbc search
itself. Profiling the readLinesFromeFile() function that
reads the list of makefiles shows it takes on the order
of 200 microseconds, much faster than the ~5 seconds
it takes for mk2rbc to search the tree itself.
This CL also allows include statements that are prefixed
with a variable. The concern with this was that there
would be a lot of load statemnts emitted for generic
include statements, causing the generated code to look
ugly, and converting and loading all those files could
cause performance issues. On the performance issues
front, there's already a check that it doesn't result
in over 150 potentially included files. We can lower
that number if necessary, but it's probably good for now.
On the generated code front, while it's true that it's
ugly, it's better to have working but ugly generated
code than refusing to generate anything working at all.
To ensure the soong finder step isn't slowed down due
to having to find a bunch of new makefiles, I profiled
the combination of newSourceFinder and FindSources in main.go:
Baseline incremental:
338.011634ms
340.853335ms
348.541762ms
333.229644ms
349.124824ms
Baseline clean:
1.003836419s
1.006203912s
996.193648ms
1.031005604s
1.03691152s
Modified incremental:
349.029285ms
349.264496ms
351.774948ms
337.63187ms
359.425306ms
Modified clean:
1.028238704s
1.053103506s
1.032757506s
1.016631201s
1.04288108s
So we can see the times are barely affected by this change.
Fixes: 213508006
Test: go test
Change-Id: Iab18bfb127ba3b7e63f2c01f69064805a8398764
diff --git a/mk2rbc/cmd/mk2rbc.go b/mk2rbc/cmd/mk2rbc.go
index d9b4e86..8e23a53 100644
--- a/mk2rbc/cmd/mk2rbc.go
+++ b/mk2rbc/cmd/mk2rbc.go
@@ -57,6 +57,7 @@
cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file")
traceCalls = flag.Bool("trace_calls", false, "trace function calls")
inputVariables = flag.String("input_variables", "", "starlark file containing product config and global variables")
+ makefileList = flag.String("makefile_list", "", "path to a list of all makefiles in the source tree, generated by soong's finder. If not provided, mk2rbc will find the makefiles itself (more slowly than if this flag was provided)")
)
func init() {
@@ -79,7 +80,7 @@
var backupSuffix string
var tracedVariables []string
var errorLogger = errorSink{data: make(map[string]datum)}
-var makefileFinder = &LinuxMakefileFinder{}
+var makefileFinder mk2rbc.MakefileFinder
func main() {
flag.Usage = func() {
@@ -133,6 +134,16 @@
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
+
+ if *makefileList != "" {
+ makefileFinder = &FileListMakefileFinder{
+ cachedMakefiles: nil,
+ filePath: *makefileList,
+ }
+ } else {
+ makefileFinder = &FindCommandMakefileFinder{}
+ }
+
// Find out global variables
getConfigVariables()
getSoongVariables()
@@ -519,17 +530,17 @@
return res, len(sorted)
}
-type LinuxMakefileFinder struct {
+// FindCommandMakefileFinder is an implementation of mk2rbc.MakefileFinder that
+// runs the unix find command to find all the makefiles in the source tree.
+type FindCommandMakefileFinder struct {
cachedRoot string
cachedMakefiles []string
}
-func (l *LinuxMakefileFinder) Find(root string) []string {
+func (l *FindCommandMakefileFinder) Find(root string) []string {
if l.cachedMakefiles != nil && l.cachedRoot == root {
return l.cachedMakefiles
}
- l.cachedRoot = root
- l.cachedMakefiles = make([]string, 0)
// Return all *.mk files but not in hidden directories.
@@ -548,9 +559,60 @@
panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
}
scanner := bufio.NewScanner(stdout)
+ result := make([]string, 0)
for scanner.Scan() {
- l.cachedMakefiles = append(l.cachedMakefiles, strings.TrimPrefix(scanner.Text(), "./"))
+ result = append(result, strings.TrimPrefix(scanner.Text(), "./"))
}
stdout.Close()
+ err = scanner.Err()
+ if err != nil {
+ panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
+ }
+ l.cachedRoot = root
+ l.cachedMakefiles = result
+ return l.cachedMakefiles
+}
+
+// FileListMakefileFinder is an implementation of mk2rbc.MakefileFinder that
+// reads a file containing the list of makefiles in the android source tree.
+// This file is generated by soong's finder, so that it can be computed while
+// soong is already walking the source tree looking for other files. If the root
+// to find makefiles under is not the root of the android source tree, it will
+// fall back to using FindCommandMakefileFinder.
+type FileListMakefileFinder struct {
+ FindCommandMakefileFinder
+ cachedMakefiles []string
+ filePath string
+}
+
+func (l *FileListMakefileFinder) Find(root string) []string {
+ root, err1 := filepath.Abs(root)
+ wd, err2 := filepath.Abs(*rootDir)
+ if root != wd || err1 != nil || err2 != nil {
+ return l.FindCommandMakefileFinder.Find(root)
+ }
+ if l.cachedMakefiles != nil {
+ return l.cachedMakefiles
+ }
+
+ file, err := os.Open(l.filePath)
+ if err != nil {
+ panic(fmt.Errorf("Cannot read makefile list: %s\n", err))
+ }
+ defer file.Close()
+
+ result := make([]string, 0)
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ line := scanner.Text()
+ if len(line) > 0 {
+ result = append(result, line)
+ }
+ }
+
+ if err = scanner.Err(); err != nil {
+ panic(fmt.Errorf("Cannot read makefile list: %s\n", err))
+ }
+ l.cachedMakefiles = result
return l.cachedMakefiles
}
diff --git a/scripts/rbc-run b/scripts/rbc-run
index 7243421..b8a6c0c 100755
--- a/scripts/rbc-run
+++ b/scripts/rbc-run
@@ -9,9 +9,10 @@
declare -r runner="${output_root}/soong/rbcrun"
declare -r converter="${output_root}/soong/mk2rbc"
declare -r launcher="${output_root}/rbc/launcher.rbc"
+declare -r makefile_list="${output_root}/.module_paths/configuration.list"
declare -r makefile="$1"
declare -r input_variables="$2"
shift 2
-"${converter}" -mode=write -r --outdir "${output_root}/rbc" --input_variables "${input_variables}" --launcher="${launcher}" "${makefile}"
+"${converter}" -mode=write -r --outdir "${output_root}/rbc" --input_variables "${input_variables}" --launcher="${launcher}" --makefile_list="${makefile_list}" "${makefile}"
"${runner}" RBC_OUT="make,global" RBC_DEBUG="${RBC_DEBUG:-}" $@ "${launcher}"
diff --git a/ui/build/finder.go b/ui/build/finder.go
index 8f74969..68efe21 100644
--- a/ui/build/finder.go
+++ b/ui/build/finder.go
@@ -87,8 +87,8 @@
// Bazel top-level file to mark a directory as a Bazel workspace.
"WORKSPACE",
},
- // Bazel Starlark configuration files.
- IncludeSuffixes: []string{".bzl"},
+ // Bazel Starlark configuration files and all .mk files for product/board configuration.
+ IncludeSuffixes: []string{".bzl", ".mk"},
}
dumpDir := config.FileListDir()
f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard),
@@ -110,6 +110,19 @@
return entries.DirNames, matches
}
+func findProductAndBoardConfigFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) {
+ matches := []string{}
+ for _, foundName := range entries.FileNames {
+ if foundName != "Android.mk" &&
+ foundName != "AndroidProducts.mk" &&
+ foundName != "CleanSpec.mk" &&
+ strings.HasSuffix(foundName, ".mk") {
+ matches = append(matches, foundName)
+ }
+ }
+ return entries.DirNames, matches
+}
+
// FindSources searches for source files known to <f> and writes them to the filesystem for
// use later.
func FindSources(ctx Context, config Config, f *finder.Finder) {
@@ -172,6 +185,13 @@
ctx.Fatalf("Could not find modules: %v", err)
}
+ // Recursively look for all product/board config files.
+ configurationFiles := f.FindMatching(".", findProductAndBoardConfigFiles)
+ err = dumpListToFile(ctx, config, configurationFiles, filepath.Join(dumpDir, "configuration.list"))
+ if err != nil {
+ ctx.Fatalf("Could not export product/board configuration list: %v", err)
+ }
+
if config.Dist() {
f.WaitForDbDump()
// Dist the files.db plain text database.