Allow dynamically calculated inherit-product path
Bug: 193566316
Test: internal
Change-Id: Iaa7b68cf459f9a694ae9d37a32c9372cf8a8335a
diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go
index ca9431f..1757965 100644
--- a/mk2rbc/mk2rbc.go
+++ b/mk2rbc/mk2rbc.go
@@ -27,6 +27,7 @@
"bytes"
"fmt"
"io"
+ "io/fs"
"io/ioutil"
"os"
"path/filepath"
@@ -104,6 +105,7 @@
"match-prefix": {"!match-prefix", starlarkTypeUnknown}, // internal macro
"match-word": {"!match-word", starlarkTypeUnknown}, // internal macro
"match-word-in-list": {"!match-word-in-list", starlarkTypeUnknown}, // internal macro
+ "my-dir": {"!my-dir", starlarkTypeString},
"patsubst": {baseName + ".mkpatsubst", starlarkTypeString},
"produce_copy_files": {baseName + ".produce_copy_files", starlarkTypeList},
"require-artifacts-in-path": {baseName + ".require_artifacts_in_path", starlarkTypeVoid},
@@ -136,6 +138,8 @@
TracedVariables []string // trace assignment to these variables
TraceCalls bool
WarnPartialSuccess bool
+ SourceFS fs.FS
+ MakefileFinder MakefileFinder
}
// An error sink allowing to gather error statistics.
@@ -149,7 +153,8 @@
func moduleNameForFile(mkFile string) string {
base := strings.TrimSuffix(filepath.Base(mkFile), filepath.Ext(mkFile))
// TODO(asmundak): what else can be in the product file names?
- return strings.ReplaceAll(base, "-", "_")
+ return strings.NewReplacer("-", "_", ".", "_").Replace(base)
+
}
func cloneMakeString(mkString *mkparser.MakeString) *mkparser.MakeString {
@@ -241,7 +246,7 @@
sc.moduleLocalName = m
continue
}
- if !sc.loadAlways {
+ if sc.optional {
uri += "|init"
}
gctx.newLine()
@@ -342,11 +347,13 @@
moduleName string
mkPos scanner.Position
nodes []starlarkNode
- inherited []*inheritedModule
+ inherited []*moduleInfo
hasErrors bool
topDir string
traceCalls bool // print enter/exit each init function
warnPartialSuccess bool
+ sourceFS fs.FS
+ makefileFinder MakefileFinder
}
func (ss *StarlarkScript) newNode(node starlarkNode) {
@@ -379,13 +386,15 @@
receiver nodeReceiver // receptacle for the generated starlarkNode's
receiverStack []nodeReceiver
outputDir string
+ dependentModules map[string]*moduleInfo
}
func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext {
+ topdir, _ := filepath.Split(filepath.Join(ss.topDir, "foo"))
predefined := []struct{ name, value string }{
{"SRC_TARGET_DIR", filepath.Join("build", "make", "target")},
{"LOCAL_PATH", filepath.Dir(ss.mkFile)},
- {"TOPDIR", ss.topDir},
+ {"TOPDIR", topdir},
// TODO(asmundak): maybe read it from build/make/core/envsetup.mk?
{"TARGET_COPY_OUT_SYSTEM", "system"},
{"TARGET_COPY_OUT_SYSTEM_OTHER", "system_other"},
@@ -428,6 +437,7 @@
moduleNameCount: make(map[string]int),
builtinMakeVars: map[string]starlarkExpr{},
variables: make(map[string]variable),
+ dependentModules: make(map[string]*moduleInfo),
}
ctx.pushVarAssignments()
for _, item := range predefined {
@@ -619,16 +629,12 @@
return xConcat
}
-func (ctx *parseContext) newInheritedModule(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) *inheritedModule {
- var path string
- x, _ := pathExpr.eval(ctx.builtinMakeVars)
- s, ok := x.(*stringLiteralExpr)
- if !ok {
- ctx.errorf(v, "inherit-product/include argument is too complex")
- return nil
+func (ctx *parseContext) newDependentModule(path string, optional bool) *moduleInfo {
+ modulePath := ctx.loadedModulePath(path)
+ if mi, ok := ctx.dependentModules[modulePath]; ok {
+ mi.optional = mi.optional || optional
+ return mi
}
-
- path = s.literal
moduleName := moduleNameForFile(path)
moduleLocalName := "_" + moduleName
n, found := ctx.moduleNameCount[moduleName]
@@ -636,27 +642,124 @@
moduleLocalName += fmt.Sprintf("%d", n)
}
ctx.moduleNameCount[moduleName] = n + 1
- ln := &inheritedModule{
- path: ctx.loadedModulePath(path),
+ mi := &moduleInfo{
+ path: modulePath,
originalPath: path,
- moduleName: moduleName,
moduleLocalName: moduleLocalName,
- loadAlways: loadAlways,
+ optional: optional,
}
- ctx.script.inherited = append(ctx.script.inherited, ln)
- return ln
+ ctx.dependentModules[modulePath] = mi
+ ctx.script.inherited = append(ctx.script.inherited, mi)
+ return mi
+}
+
+func (ctx *parseContext) handleSubConfig(
+ v mkparser.Node, pathExpr starlarkExpr, loadAlways bool, processModule func(inheritedModule)) {
+ pathExpr, _ = pathExpr.eval(ctx.builtinMakeVars)
+
+ // In a simple case, the name of a module to inherit/include is known statically.
+ if path, ok := maybeString(pathExpr); ok {
+ if strings.Contains(path, "*") {
+ if paths, err := fs.Glob(ctx.script.sourceFS, path); err == nil {
+ for _, p := range paths {
+ processModule(inheritedStaticModule{ctx.newDependentModule(p, !loadAlways), loadAlways})
+ }
+ } else {
+ ctx.errorf(v, "cannot glob wildcard argument")
+ }
+ } else {
+ processModule(inheritedStaticModule{ctx.newDependentModule(path, !loadAlways), loadAlways})
+ }
+ return
+ }
+
+ // If module path references variables (e.g., $(v1)/foo/$(v2)/device-config.mk), find all the paths in the
+ // source tree that may be a match and the corresponding variable values. For instance, if the source tree
+ // contains vendor1/foo/abc/dev.mk and vendor2/foo/def/dev.mk, the first one will be inherited when
+ // (v1, v2) == ('vendor1', 'abc'), and the second one when (v1, v2) == ('vendor2', 'def').
+ // We then emit the code that loads all of them, e.g.:
+ // load("//vendor1/foo/abc:dev.rbc", _dev1_init="init")
+ // load("//vendor2/foo/def/dev.rbc", _dev2_init="init")
+ // And then inherit it as follows:
+ // _e = {
+ // "vendor1/foo/abc/dev.mk": ("vendor1/foo/abc/dev", _dev1_init),
+ // "vendor2/foo/def/dev.mk": ("vendor2/foo/def/dev", _dev_init2) }.get("%s/foo/%s/dev.mk" % (v1, v2))
+ // if _e:
+ // rblf.inherit(handle, _e[0], _e[1])
+ //
+ var matchingPaths []string
+ varPath, ok := pathExpr.(*interpolateExpr)
+ if !ok {
+ ctx.errorf(v, "inherit-product/include argument is too complex")
+ return
+ }
+
+ pathPattern := []string{varPath.chunks[0]}
+ for _, chunk := range varPath.chunks[1:] {
+ if chunk != "" {
+ pathPattern = append(pathPattern, chunk)
+ }
+ }
+ if pathPattern[0] != "" {
+ matchingPaths = ctx.findMatchingPaths(pathPattern)
+ } else {
+ // Heuristics -- if pattern starts from top, restrict it to the directories where
+ // we know inherit-product uses dynamically calculated path.
+ for _, t := range []string{"vendor/qcom", "vendor/google_devices"} {
+ pathPattern[0] = t
+ matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...)
+ }
+ }
+ // Safeguard against $(call inherit-product,$(PRODUCT_PATH))
+ const maxMatchingFiles = 100
+ if len(matchingPaths) > maxMatchingFiles {
+ ctx.errorf(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)
+ return
+ }
+ res := inheritedDynamicModule{*varPath, []*moduleInfo{}, loadAlways}
+ for _, p := range matchingPaths {
+ // A product configuration files discovered dynamically may attempt to inherit
+ // from another one which does not exist in this source tree. Prevent load errors
+ // by always loading the dynamic files as optional.
+ res.candidateModules = append(res.candidateModules, ctx.newDependentModule(p, true))
+ }
+ processModule(res)
+}
+
+func (ctx *parseContext) findMatchingPaths(pattern []string) []string {
+ files := ctx.script.makefileFinder.Find(ctx.script.topDir)
+ if len(pattern) == 0 {
+ return files
+ }
+
+ // Create regular expression from the pattern
+ s_regexp := "^" + regexp.QuoteMeta(pattern[0])
+ for _, s := range pattern[1:] {
+ s_regexp += ".*" + regexp.QuoteMeta(s)
+ }
+ s_regexp += "$"
+ rex := regexp.MustCompile(s_regexp)
+
+ // Now match
+ var res []string
+ for _, p := range files {
+ if rex.MatchString(p) {
+ res = append(res, p)
+ }
+ }
+ return res
}
func (ctx *parseContext) handleInheritModule(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) {
- if im := ctx.newInheritedModule(v, pathExpr, loadAlways); im != nil {
+ ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) {
ctx.receiver.newNode(&inheritNode{im})
- }
+ })
}
func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) {
- if ln := ctx.newInheritedModule(v, pathExpr, loadAlways); ln != nil {
- ctx.receiver.newNode(&includeNode{ln})
- }
+ ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) {
+ ctx.receiver.newNode(&includeNode{im})
+ })
}
func (ctx *parseContext) handleVariable(v *mkparser.Variable) {
@@ -1091,9 +1194,10 @@
}
expr.name = words[0].Dump()
if len(words) < 2 {
- return expr
+ args = &mkparser.MakeString{}
+ } else {
+ args = words[1]
}
- args = words[1]
}
if kf, found := knownFunctions[expr.name]; found {
expr.returnType = kf.returnType
@@ -1103,6 +1207,8 @@
switch expr.name {
case "word":
return ctx.parseWordFunc(node, args)
+ case "my-dir":
+ return &variableRefExpr{ctx.addVariable("LOCAL_PATH"), true}
case "subst", "patsubst":
return ctx.parseSubstFunc(node, expr.name, args)
default:
@@ -1285,6 +1391,7 @@
}
func (ss *StarlarkScript) SubConfigFiles() []string {
+
var subs []string
for _, src := range ss.inherited {
subs = append(subs, src.originalPath)
@@ -1322,6 +1429,8 @@
topDir: req.RootDir,
traceCalls: req.TraceCalls,
warnPartialSuccess: req.WarnPartialSuccess,
+ sourceFS: req.SourceFS,
+ makefileFinder: req.MakefileFinder,
}
ctx := newParseContext(starScript, nodes)
ctx.outputSuffix = req.OutputSuffix