Merge "Correct isThirdParty check"
diff --git a/mk2rbc/cmd/mk2rbc.go b/mk2rbc/cmd/mk2rbc.go
index 4d3d8d8..72525c4 100644
--- a/mk2rbc/cmd/mk2rbc.go
+++ b/mk2rbc/cmd/mk2rbc.go
@@ -21,13 +21,16 @@
package main
import (
+ "bufio"
"flag"
"fmt"
"io/ioutil"
"os"
+ "os/exec"
"path/filepath"
"regexp"
"runtime/debug"
+ "runtime/pprof"
"sort"
"strings"
"time"
@@ -52,6 +55,7 @@
outputTop = flag.String("outdir", "", "write output files into this directory hierarchy")
launcher = flag.String("launcher", "", "generated launcher path. If set, the non-flag argument is _product_name_")
printProductConfigMap = flag.Bool("print_product_config_map", false, "print product config map and exit")
+ cpuProfile = flag.String("cpu_profile", "", "write cpu profile to file")
traceCalls = flag.Bool("trace_calls", false, "trace function calls")
)
@@ -76,6 +80,7 @@
var backupSuffix string
var tracedVariables []string
var errorLogger = errorsByType{data: make(map[string]datum)}
+var makefileFinder = &LinuxMakefileFinder{}
func main() {
flag.Usage = func() {
@@ -122,6 +127,14 @@
tracedVariables = strings.Split(*traceVar, ",")
}
+ if *cpuProfile != "" {
+ f, err := os.Create(*cpuProfile)
+ if err != nil {
+ quit(err)
+ }
+ pprof.StartCPUProfile(f)
+ defer pprof.StopCPUProfile()
+ }
// Find out global variables
getConfigVariables()
getSoongVariables()
@@ -292,6 +305,8 @@
TracedVariables: tracedVariables,
TraceCalls: *traceCalls,
WarnPartialSuccess: *warn,
+ SourceFS: os.DirFS(*rootDir),
+ MakefileFinder: makefileFinder,
}
if *errstat {
mk2starRequest.ErrorLogger = errorLogger
@@ -494,3 +509,39 @@
}
return res, len(sorted)
}
+
+type LinuxMakefileFinder struct {
+ cachedRoot string
+ cachedMakefiles []string
+}
+
+func (l *LinuxMakefileFinder) Find(root string) []string {
+ if l.cachedMakefiles != nil && l.cachedRoot == root {
+ return l.cachedMakefiles
+ }
+ l.cachedRoot = root
+ l.cachedMakefiles = make([]string, 0)
+
+ // Return all *.mk files but not in hidden directories.
+
+ // NOTE(asmundak): as it turns out, even the WalkDir (which is an _optimized_ directory tree walker)
+ // is about twice slower than running `find` command (14s vs 6s on the internal Android source tree).
+ common_args := []string{"!", "-type", "d", "-name", "*.mk", "!", "-path", "*/.*/*"}
+ if root != "" {
+ common_args = append([]string{root}, common_args...)
+ }
+ cmd := exec.Command("/usr/bin/find", common_args...)
+ stdout, err := cmd.StdoutPipe()
+ if err == nil {
+ err = cmd.Start()
+ }
+ if err != nil {
+ panic(fmt.Errorf("cannot get the output from %s: %s", cmd, err))
+ }
+ scanner := bufio.NewScanner(stdout)
+ for scanner.Scan() {
+ l.cachedMakefiles = append(l.cachedMakefiles, strings.TrimPrefix(scanner.Text(), "./"))
+ }
+ stdout.Close()
+ return l.cachedMakefiles
+}
diff --git a/mk2rbc/expr.go b/mk2rbc/expr.go
index b06ed90..915f69e 100644
--- a/mk2rbc/expr.go
+++ b/mk2rbc/expr.go
@@ -240,22 +240,21 @@
}
func (eq *eqExpr) emit(gctx *generationContext) {
- // Are we checking that a variable is empty?
- var varRef *variableRefExpr
- if s, ok := maybeString(eq.left); ok && s == "" {
- varRef, ok = eq.right.(*variableRefExpr)
- } else if s, ok := maybeString(eq.right); ok && s == "" {
- varRef, ok = eq.left.(*variableRefExpr)
- }
- if varRef != nil {
- // Yes.
+ emitSimple := func(expr starlarkExpr) {
if eq.isEq {
gctx.write("not ")
}
- varRef.emit(gctx)
- return
+ expr.emit(gctx)
}
+ // Are we checking that a variable is empty?
+ if isEmptyString(eq.left) {
+ emitSimple(eq.right)
+ return
+ } else if isEmptyString(eq.right) {
+ emitSimple(eq.left)
+ return
+ }
// General case
eq.left.emit(gctx)
if eq.isEq {
@@ -578,3 +577,8 @@
}
return expr
}
+
+func isEmptyString(expr starlarkExpr) bool {
+ x, ok := expr.(*stringLiteralExpr)
+ return ok && x.literal == ""
+}
diff --git a/mk2rbc/find_mockfs.go b/mk2rbc/find_mockfs.go
new file mode 100644
index 0000000..73eff07
--- /dev/null
+++ b/mk2rbc/find_mockfs.go
@@ -0,0 +1,121 @@
+package mk2rbc
+
+import (
+ "io/fs"
+ "os"
+ "path/filepath"
+ "time"
+)
+
+// Mock FS. Maps a directory name to an array of entries.
+// An entry implements fs.DirEntry, fs.FIleInfo and fs.File interface
+type FindMockFS struct {
+ dirs map[string][]myFileInfo
+}
+
+func (m FindMockFS) locate(name string) (myFileInfo, bool) {
+ if name == "." {
+ return myFileInfo{".", true}, true
+ }
+ dir := filepath.Dir(name)
+ base := filepath.Base(name)
+ if entries, ok := m.dirs[dir]; ok {
+ for _, e := range entries {
+ if e.name == base {
+ return e, true
+ }
+ }
+ }
+ return myFileInfo{}, false
+}
+
+func (m FindMockFS) create(name string, isDir bool) {
+ dir := filepath.Dir(name)
+ m.dirs[dir] = append(m.dirs[dir], myFileInfo{filepath.Base(name), isDir})
+}
+
+func (m FindMockFS) Stat(name string) (fs.FileInfo, error) {
+ if fi, ok := m.locate(name); ok {
+ return fi, nil
+ }
+ return nil, os.ErrNotExist
+}
+
+type myFileInfo struct {
+ name string
+ isDir bool
+}
+
+func (m myFileInfo) Info() (fs.FileInfo, error) {
+ panic("implement me")
+}
+
+func (m myFileInfo) Size() int64 {
+ panic("implement me")
+}
+
+func (m myFileInfo) Mode() fs.FileMode {
+ panic("implement me")
+}
+
+func (m myFileInfo) ModTime() time.Time {
+ panic("implement me")
+}
+
+func (m myFileInfo) Sys() interface{} {
+ return nil
+}
+
+func (m myFileInfo) Stat() (fs.FileInfo, error) {
+ return m, nil
+}
+
+func (m myFileInfo) Read(bytes []byte) (int, error) {
+ panic("implement me")
+}
+
+func (m myFileInfo) Close() error {
+ panic("implement me")
+}
+
+func (m myFileInfo) Name() string {
+ return m.name
+}
+
+func (m myFileInfo) IsDir() bool {
+ return m.isDir
+}
+
+func (m myFileInfo) Type() fs.FileMode {
+ return m.Mode()
+}
+
+func (m FindMockFS) Open(name string) (fs.File, error) {
+ panic("implement me")
+}
+
+func (m FindMockFS) ReadDir(name string) ([]fs.DirEntry, error) {
+ if d, ok := m.dirs[name]; ok {
+ var res []fs.DirEntry
+ for _, e := range d {
+ res = append(res, e)
+ }
+ return res, nil
+ }
+ return nil, os.ErrNotExist
+}
+
+func NewFindMockFS(files []string) FindMockFS {
+ myfs := FindMockFS{make(map[string][]myFileInfo)}
+ for _, f := range files {
+ isDir := false
+ for f != "." {
+ if _, ok := myfs.locate(f); !ok {
+ myfs.create(f, isDir)
+ }
+ isDir = true
+ f = filepath.Dir(f)
+ }
+ }
+ return myfs
+}
diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go
index 86e647d..2173e9b 100644
--- a/mk2rbc/mk2rbc.go
+++ b/mk2rbc/mk2rbc.go
@@ -27,6 +27,7 @@
"bytes"
"fmt"
"io"
+ "io/fs"
"io/ioutil"
"os"
"path/filepath"
@@ -75,11 +76,13 @@
runtimeName string
returnType starlarkType
}{
+ "abspath": {baseName + ".abspath", starlarkTypeString},
fileExistsPhony: {baseName + ".file_exists", starlarkTypeBool},
wildcardExistsPhony: {baseName + ".file_wildcard_exists", starlarkTypeBool},
"add-to-product-copy-files-if-exists": {baseName + ".copy_if_exists", starlarkTypeList},
"addprefix": {baseName + ".addprefix", starlarkTypeList},
"addsuffix": {baseName + ".addsuffix", starlarkTypeList},
+ "dir": {baseName + ".dir", starlarkTypeList},
"enforce-product-packages-exist": {baseName + ".enforce_product_packages_exist", starlarkTypeVoid},
"error": {baseName + ".mkerror", starlarkTypeVoid},
"findstring": {"!findstring", starlarkTypeInt},
@@ -87,6 +90,7 @@
"find-word-in-list": {"!find-word-in-list", starlarkTypeUnknown}, // internal macro
"filter": {baseName + ".filter", starlarkTypeList},
"filter-out": {baseName + ".filter_out", starlarkTypeList},
+ "firstword": {"!firstword", starlarkTypeString},
"get-vendor-board-platforms": {"!get-vendor-board-platforms", starlarkTypeList}, // internal macro, used by is-board-platform, etc.
"info": {baseName + ".mkinfo", starlarkTypeVoid},
"is-android-codename": {"!is-android-codename", starlarkTypeBool}, // unused by product config
@@ -101,9 +105,12 @@
"is-vendor-board-platform": {"!is-vendor-board-platform", starlarkTypeBool},
callLoadAlways: {"!inherit-product", starlarkTypeVoid},
callLoadIf: {"!inherit-product-if-exists", starlarkTypeVoid},
+ "lastword": {"!lastword", starlarkTypeString},
"match-prefix": {"!match-prefix", starlarkTypeUnknown}, // internal macro
"match-word": {"!match-word", starlarkTypeUnknown}, // internal macro
"match-word-in-list": {"!match-word-in-list", starlarkTypeUnknown}, // internal macro
+ "notdir": {baseName + ".notdir", starlarkTypeString},
+ "my-dir": {"!my-dir", starlarkTypeString},
"patsubst": {baseName + ".mkpatsubst", starlarkTypeString},
"produce_copy_files": {baseName + ".produce_copy_files", starlarkTypeList},
"require-artifacts-in-path": {baseName + ".require_artifacts_in_path", starlarkTypeVoid},
@@ -136,6 +143,8 @@
TracedVariables []string // trace assignment to these variables
TraceCalls bool
WarnPartialSuccess bool
+ SourceFS fs.FS
+ MakefileFinder MakefileFinder
}
// An error sink allowing to gather error statistics.
@@ -149,7 +158,8 @@
func moduleNameForFile(mkFile string) string {
base := strings.TrimSuffix(filepath.Base(mkFile), filepath.Ext(mkFile))
// TODO(asmundak): what else can be in the product file names?
- return strings.ReplaceAll(base, "-", "_")
+ return strings.NewReplacer("-", "_", ".", "_").Replace(base)
+
}
func cloneMakeString(mkString *mkparser.MakeString) *mkparser.MakeString {
@@ -241,7 +251,7 @@
sc.moduleLocalName = m
continue
}
- if !sc.loadAlways {
+ if sc.optional {
uri += "|init"
}
gctx.newLine()
@@ -342,11 +352,13 @@
moduleName string
mkPos scanner.Position
nodes []starlarkNode
- inherited []*inheritedModule
+ inherited []*moduleInfo
hasErrors bool
topDir string
traceCalls bool // print enter/exit each init function
warnPartialSuccess bool
+ sourceFS fs.FS
+ makefileFinder MakefileFinder
}
func (ss *StarlarkScript) newNode(node starlarkNode) {
@@ -379,13 +391,15 @@
receiver nodeReceiver // receptacle for the generated starlarkNode's
receiverStack []nodeReceiver
outputDir string
+ dependentModules map[string]*moduleInfo
}
func newParseContext(ss *StarlarkScript, nodes []mkparser.Node) *parseContext {
+ topdir, _ := filepath.Split(filepath.Join(ss.topDir, "foo"))
predefined := []struct{ name, value string }{
{"SRC_TARGET_DIR", filepath.Join("build", "make", "target")},
{"LOCAL_PATH", filepath.Dir(ss.mkFile)},
- {"TOPDIR", ss.topDir},
+ {"TOPDIR", topdir},
// TODO(asmundak): maybe read it from build/make/core/envsetup.mk?
{"TARGET_COPY_OUT_SYSTEM", "system"},
{"TARGET_COPY_OUT_SYSTEM_OTHER", "system_other"},
@@ -428,6 +442,7 @@
moduleNameCount: make(map[string]int),
builtinMakeVars: map[string]starlarkExpr{},
variables: make(map[string]variable),
+ dependentModules: make(map[string]*moduleInfo),
}
ctx.pushVarAssignments()
for _, item := range predefined {
@@ -619,16 +634,12 @@
return xConcat
}
-func (ctx *parseContext) newInheritedModule(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) *inheritedModule {
- var path string
- x, _ := pathExpr.eval(ctx.builtinMakeVars)
- s, ok := x.(*stringLiteralExpr)
- if !ok {
- ctx.errorf(v, "inherit-product/include argument is too complex")
- return nil
+func (ctx *parseContext) newDependentModule(path string, optional bool) *moduleInfo {
+ modulePath := ctx.loadedModulePath(path)
+ if mi, ok := ctx.dependentModules[modulePath]; ok {
+ mi.optional = mi.optional || optional
+ return mi
}
-
- path = s.literal
moduleName := moduleNameForFile(path)
moduleLocalName := "_" + moduleName
n, found := ctx.moduleNameCount[moduleName]
@@ -636,27 +647,124 @@
moduleLocalName += fmt.Sprintf("%d", n)
}
ctx.moduleNameCount[moduleName] = n + 1
- ln := &inheritedModule{
- path: ctx.loadedModulePath(path),
+ mi := &moduleInfo{
+ path: modulePath,
originalPath: path,
- moduleName: moduleName,
moduleLocalName: moduleLocalName,
- loadAlways: loadAlways,
+ optional: optional,
}
- ctx.script.inherited = append(ctx.script.inherited, ln)
- return ln
+ ctx.dependentModules[modulePath] = mi
+ ctx.script.inherited = append(ctx.script.inherited, mi)
+ return mi
+}
+
+func (ctx *parseContext) handleSubConfig(
+ v mkparser.Node, pathExpr starlarkExpr, loadAlways bool, processModule func(inheritedModule)) {
+ pathExpr, _ = pathExpr.eval(ctx.builtinMakeVars)
+
+ // In a simple case, the name of a module to inherit/include is known statically.
+ if path, ok := maybeString(pathExpr); ok {
+ if strings.Contains(path, "*") {
+ if paths, err := fs.Glob(ctx.script.sourceFS, path); err == nil {
+ for _, p := range paths {
+ processModule(inheritedStaticModule{ctx.newDependentModule(p, !loadAlways), loadAlways})
+ }
+ } else {
+ ctx.errorf(v, "cannot glob wildcard argument")
+ }
+ } else {
+ processModule(inheritedStaticModule{ctx.newDependentModule(path, !loadAlways), loadAlways})
+ }
+ return
+ }
+
+ // If module path references variables (e.g., $(v1)/foo/$(v2)/device-config.mk), find all the paths in the
+ // source tree that may be a match and the corresponding variable values. For instance, if the source tree
+ // contains vendor1/foo/abc/dev.mk and vendor2/foo/def/dev.mk, the first one will be inherited when
+ // (v1, v2) == ('vendor1', 'abc'), and the second one when (v1, v2) == ('vendor2', 'def').
+ // We then emit the code that loads all of them, e.g.:
+ // load("//vendor1/foo/abc:dev.rbc", _dev1_init="init")
+ // load("//vendor2/foo/def/dev.rbc", _dev2_init="init")
+ // And then inherit it as follows:
+ // _e = {
+ // "vendor1/foo/abc/dev.mk": ("vendor1/foo/abc/dev", _dev1_init),
+ // "vendor2/foo/def/dev.mk": ("vendor2/foo/def/dev", _dev_init2) }.get("%s/foo/%s/dev.mk" % (v1, v2))
+ // if _e:
+ // rblf.inherit(handle, _e[0], _e[1])
+ //
+ var matchingPaths []string
+ varPath, ok := pathExpr.(*interpolateExpr)
+ if !ok {
+ ctx.errorf(v, "inherit-product/include argument is too complex")
+ return
+ }
+
+ pathPattern := []string{varPath.chunks[0]}
+ for _, chunk := range varPath.chunks[1:] {
+ if chunk != "" {
+ pathPattern = append(pathPattern, chunk)
+ }
+ }
+ if pathPattern[0] != "" {
+ matchingPaths = ctx.findMatchingPaths(pathPattern)
+ } else {
+ // Heuristics -- if pattern starts from top, restrict it to the directories where
+ // we know inherit-product uses dynamically calculated path.
+ for _, t := range []string{"vendor/qcom", "vendor/google_devices"} {
+ pathPattern[0] = t
+ matchingPaths = append(matchingPaths, ctx.findMatchingPaths(pathPattern)...)
+ }
+ }
+ // Safeguard against $(call inherit-product,$(PRODUCT_PATH))
+ const maxMatchingFiles = 100
+ if len(matchingPaths) > maxMatchingFiles {
+ ctx.errorf(v, "there are >%d files matching the pattern, please rewrite it", maxMatchingFiles)
+ return
+ }
+ res := inheritedDynamicModule{*varPath, []*moduleInfo{}, loadAlways}
+ for _, p := range matchingPaths {
+ // A product configuration files discovered dynamically may attempt to inherit
+ // from another one which does not exist in this source tree. Prevent load errors
+ // by always loading the dynamic files as optional.
+ res.candidateModules = append(res.candidateModules, ctx.newDependentModule(p, true))
+ }
+ processModule(res)
+}
+
+func (ctx *parseContext) findMatchingPaths(pattern []string) []string {
+ files := ctx.script.makefileFinder.Find(ctx.script.topDir)
+ if len(pattern) == 0 {
+ return files
+ }
+
+ // Create regular expression from the pattern
+ s_regexp := "^" + regexp.QuoteMeta(pattern[0])
+ for _, s := range pattern[1:] {
+ s_regexp += ".*" + regexp.QuoteMeta(s)
+ }
+ s_regexp += "$"
+ rex := regexp.MustCompile(s_regexp)
+
+ // Now match
+ var res []string
+ for _, p := range files {
+ if rex.MatchString(p) {
+ res = append(res, p)
+ }
+ }
+ return res
}
func (ctx *parseContext) handleInheritModule(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) {
- if im := ctx.newInheritedModule(v, pathExpr, loadAlways); im != nil {
+ ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) {
ctx.receiver.newNode(&inheritNode{im})
- }
+ })
}
func (ctx *parseContext) handleInclude(v mkparser.Node, pathExpr starlarkExpr, loadAlways bool) {
- if ln := ctx.newInheritedModule(v, pathExpr, loadAlways); ln != nil {
- ctx.receiver.newNode(&includeNode{ln})
- }
+ ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) {
+ ctx.receiver.newNode(&includeNode{im})
+ })
}
func (ctx *parseContext) handleVariable(v *mkparser.Variable) {
@@ -938,14 +1046,14 @@
func (ctx *parseContext) parseCompareFilterFuncResult(cond *mkparser.Directive,
filterFuncCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
// We handle:
- // * ifeq/ifneq (,$(filter v1 v2 ..., $(VAR)) becomes if VAR not in/in ["v1", "v2", ...]
- // * ifeq/ifneq (,$(filter $(VAR), v1 v2 ...) becomes if VAR not in/in ["v1", "v2", ...]
+ // * ifeq/ifneq (,$(filter v1 v2 ..., EXPR) becomes if EXPR not in/in ["v1", "v2", ...]
+ // * ifeq/ifneq (,$(filter EXPR, v1 v2 ...) becomes if EXPR not in/in ["v1", "v2", ...]
// * ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...) becomes if VAR in/not in ["v1", "v2"]
// TODO(Asmundak): check the last case works for filter-out, too.
xPattern := filterFuncCall.args[0]
xText := filterFuncCall.args[1]
var xInList *stringLiteralExpr
- var xVar starlarkExpr
+ var expr starlarkExpr
var ok bool
switch x := xValue.(type) {
case *stringLiteralExpr:
@@ -955,34 +1063,42 @@
// Either pattern or text should be const, and the
// non-const one should be varRefExpr
if xInList, ok = xPattern.(*stringLiteralExpr); ok {
- xVar = xText
+ expr = xText
} else if xInList, ok = xText.(*stringLiteralExpr); ok {
- xVar = xPattern
+ expr = xPattern
+ } else {
+ return &callExpr{
+ object: nil,
+ name: filterFuncCall.name,
+ args: filterFuncCall.args,
+ returnType: starlarkTypeBool,
+ }
}
case *variableRefExpr:
if v, ok := xPattern.(*variableRefExpr); ok {
if xInList, ok = xText.(*stringLiteralExpr); ok && v.ref.name() == x.ref.name() {
// ifeq/ifneq ($(VAR),$(filter $(VAR), v1 v2 ...), flip negate,
// it's the opposite to what is done when comparing to empty.
- xVar = xPattern
+ expr = xPattern
negate = !negate
}
}
}
- if xVar != nil && xInList != nil {
- if _, ok := xVar.(*variableRefExpr); ok {
- slExpr := newStringListExpr(strings.Fields(xInList.literal))
- // Generate simpler code for the common cases:
- if xVar.typ() == starlarkTypeList {
- if len(slExpr.items) == 1 {
- // Checking that a string belongs to list
- return &inExpr{isNot: negate, list: xVar, expr: slExpr.items[0]}
- } else {
- // TODO(asmundak):
- panic("TBD")
- }
+ if expr != nil && xInList != nil {
+ slExpr := newStringListExpr(strings.Fields(xInList.literal))
+ // Generate simpler code for the common cases:
+ if expr.typ() == starlarkTypeList {
+ if len(slExpr.items) == 1 {
+ // Checking that a string belongs to list
+ return &inExpr{isNot: negate, list: expr, expr: slExpr.items[0]}
+ } else {
+ // TODO(asmundak):
+ panic("TBD")
}
- return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: xVar}
+ } else if len(slExpr.items) == 1 {
+ return &eqExpr{left: expr, right: slExpr.items[0], isEq: !negate}
+ } else {
+ return &inExpr{isNot: negate, list: newStringListExpr(strings.Fields(xInList.literal)), expr: expr}
}
}
return ctx.newBadExpr(cond, "filter arguments are too complex: %s", cond.Dump())
@@ -990,7 +1106,7 @@
func (ctx *parseContext) parseCompareWildcardFuncResult(directive *mkparser.Directive,
xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
- if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" {
+ if !isEmptyString(xValue) {
return ctx.newBadExpr(directive, "wildcard result can be compared only to empty: %s", xValue)
}
callFunc := wildcardExistsPhony
@@ -1006,19 +1122,19 @@
func (ctx *parseContext) parseCheckFindstringFuncResult(directive *mkparser.Directive,
xCall *callExpr, xValue starlarkExpr, negate bool) starlarkExpr {
- if x, ok := xValue.(*stringLiteralExpr); !ok || x.literal != "" {
- return ctx.newBadExpr(directive, "findstring result can be compared only to empty: %s", xValue)
+ if isEmptyString(xValue) {
+ return &eqExpr{
+ left: &callExpr{
+ object: xCall.args[1],
+ name: "find",
+ args: []starlarkExpr{xCall.args[0]},
+ returnType: starlarkTypeInt,
+ },
+ right: &intLiteralExpr{-1},
+ isEq: !negate,
+ }
}
- return &eqExpr{
- left: &callExpr{
- object: xCall.args[1],
- name: "find",
- args: []starlarkExpr{xCall.args[0]},
- returnType: starlarkTypeInt,
- },
- right: &intLiteralExpr{-1},
- isEq: !negate,
- }
+ return ctx.newBadExpr(directive, "findstring result can be compared only to empty: %s", xValue)
}
func (ctx *parseContext) parseCompareStripFuncResult(directive *mkparser.Directive,
@@ -1083,9 +1199,10 @@
}
expr.name = words[0].Dump()
if len(words) < 2 {
- return expr
+ args = &mkparser.MakeString{}
+ } else {
+ args = words[1]
}
- args = words[1]
}
if kf, found := knownFunctions[expr.name]; found {
expr.returnType = kf.returnType
@@ -1095,6 +1212,10 @@
switch expr.name {
case "word":
return ctx.parseWordFunc(node, args)
+ case "firstword", "lastword":
+ return ctx.parseFirstOrLastwordFunc(node, expr.name, args)
+ case "my-dir":
+ return &variableRefExpr{ctx.addVariable("LOCAL_PATH"), true}
case "subst", "patsubst":
return ctx.parseSubstFunc(node, expr.name, args)
default:
@@ -1165,6 +1286,24 @@
return indexExpr{array, &intLiteralExpr{int(index - 1)}}
}
+func (ctx *parseContext) parseFirstOrLastwordFunc(node mkparser.Node, name string, args *mkparser.MakeString) starlarkExpr {
+ arg := ctx.parseMakeString(node, args)
+ if bad, ok := arg.(*badExpr); ok {
+ return bad
+ }
+ index := &intLiteralExpr{0}
+ if name == "lastword" {
+ if v, ok := arg.(*variableRefExpr); ok && v.ref.name() == "MAKEFILE_LIST" {
+ return &stringLiteralExpr{ctx.script.mkFile}
+ }
+ index.literal = -1
+ }
+ if arg.typ() == starlarkTypeList {
+ return &indexExpr{arg, index}
+ }
+ return &indexExpr{&callExpr{object: arg, name: "split", returnType: starlarkTypeList}, index}
+}
+
func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr {
if mk.Const() {
return &stringLiteralExpr{mk.Dump()}
@@ -1277,6 +1416,7 @@
}
func (ss *StarlarkScript) SubConfigFiles() []string {
+
var subs []string
for _, src := range ss.inherited {
subs = append(subs, src.originalPath)
@@ -1314,6 +1454,8 @@
topDir: req.RootDir,
traceCalls: req.TraceCalls,
warnPartialSuccess: req.WarnPartialSuccess,
+ sourceFS: req.SourceFS,
+ makefileFinder: req.MakefileFinder,
}
ctx := newParseContext(starScript, nodes)
ctx.outputSuffix = req.OutputSuffix
diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go
index 240d0b8..88c9c38 100644
--- a/mk2rbc/mk2rbc_test.go
+++ b/mk2rbc/mk2rbc_test.go
@@ -16,6 +16,8 @@
import (
"bytes"
+ "io/fs"
+ "path/filepath"
"strings"
"testing"
)
@@ -100,10 +102,13 @@
desc: "Unknown function",
mkname: "product.mk",
in: `
-PRODUCT_NAME := $(call foo, bar)
+PRODUCT_NAME := $(call foo1, bar)
+PRODUCT_NAME := $(call foo0)
`,
- expected: `# MK2RBC TRANSLATION ERROR: cannot handle invoking foo
-# PRODUCT_NAME := $(call foo, bar)
+ expected: `# MK2RBC TRANSLATION ERROR: cannot handle invoking foo1
+# PRODUCT_NAME := $(call foo1, bar)
+# MK2RBC TRANSLATION ERROR: cannot handle invoking foo0
+# PRODUCT_NAME := $(call foo0)
load("//build/make/core:product_config.rbc", "rblf")
def init(g, handle):
@@ -130,7 +135,7 @@
rblf.inherit(handle, "part", _part_init)
else:
# Comment
- rblf.inherit(handle, "./part", _part_init)
+ rblf.inherit(handle, "part", _part_init)
`,
},
{
@@ -144,7 +149,7 @@
def init(g, handle):
cfg = rblf.cfg(handle)
- if _part_init != None:
+ if _part_init:
rblf.inherit(handle, "part", _part_init)
`,
},
@@ -160,7 +165,7 @@
endif
`,
expected: `load("//build/make/core:product_config.rbc", "rblf")
-load(":part.star", _part_init = "init")
+load(":part.star|init", _part_init = "init")
def init(g, handle):
cfg = rblf.cfg(handle)
@@ -176,8 +181,7 @@
desc: "Synonymous inherited configurations",
mkname: "path/product.mk",
in: `
-$(call inherit-product, foo/font.mk)
-$(call inherit-product, bar/font.mk)
+$(call inherit-product, */font.mk)
`,
expected: `load("//build/make/core:product_config.rbc", "rblf")
load("//foo:font.star", _font_init = "init")
@@ -254,6 +258,8 @@
in: `
ifdef PRODUCT_NAME
# Comment
+else
+ TARGET_COPY_OUT_VENDOR := foo
endif
`,
expected: `load("//build/make/core:product_config.rbc", "rblf")
@@ -263,6 +269,10 @@
if g.get("PRODUCT_NAME") != None:
# Comment
pass
+ else:
+ # MK2RBC TRANSLATION ERROR: cannot set predefined variable TARGET_COPY_OUT_VENDOR to "foo", its value should be "||VENDOR-PATH-PH||"
+ pass
+ rblf.warning("product.mk", "partially successful conversion")
`,
},
{
@@ -342,6 +352,8 @@
endif
ifeq ($(TARGET_BUILD_VARIANT), $(filter $(TARGET_BUILD_VARIANT), userdebug eng))
endif
+ifneq (,$(filter true, $(v1)$(v2)))
+endif
`,
expected: `load("//build/make/core:product_config.rbc", "rblf")
@@ -349,12 +361,14 @@
cfg = rblf.cfg(handle)
if g["TARGET_BUILD_VARIANT"] not in ["userdebug", "eng"]:
pass
- if g["TARGET_BUILD_VARIANT"] in ["userdebug"]:
+ if g["TARGET_BUILD_VARIANT"] == "userdebug":
pass
if "plaf" in g.get("PLATFORM_LIST", []):
pass
if g["TARGET_BUILD_VARIANT"] in ["userdebug", "eng"]:
pass
+ if "%s%s" % (_v1, _v2) == "true":
+ pass
`,
},
{
@@ -385,11 +399,26 @@
def init(g, handle):
cfg = rblf.cfg(handle)
if g["TARGET_PRODUCT"] not in ["yukawa_gms", "yukawa_sei510_gms"]:
- if g["TARGET_PRODUCT"] in ["yukawa_gms"]:
+ if g["TARGET_PRODUCT"] == "yukawa_gms":
pass
`,
},
{
+ desc: "filter $(V1), $(V2)",
+ mkname: "product.mk",
+ in: `
+ifneq (, $(filter $(PRODUCT_LIST), $(TARGET_PRODUCT)))
+endif
+`,
+ expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+ cfg = rblf.cfg(handle)
+ if rblf.filter(g.get("PRODUCT_LIST", ""), g["TARGET_PRODUCT"]):
+ pass
+`,
+ },
+ {
desc: "ifeq",
mkname: "product.mk",
in: `
@@ -629,6 +658,14 @@
PRODUCT_COPY_FILES := $(addsuffix .sff, a b c)
PRODUCT_NAME := $(word 1, $(subst ., ,$(TARGET_BOARD_PLATFORM)))
$(info $(patsubst %.pub,%,$(PRODUCT_ADB_KEYS)))
+$(info $(dir foo/bar))
+$(info $(firstword $(PRODUCT_COPY_FILES)))
+$(info $(lastword $(PRODUCT_COPY_FILES)))
+$(info $(dir $(lastword $(MAKEFILE_LIST))))
+$(info $(dir $(lastword $(PRODUCT_COPY_FILES))))
+$(info $(dir $(lastword $(foobar))))
+$(info $(abspath foo/bar))
+$(info $(notdir foo/bar))
`,
expected: `load("//build/make/core:product_config.rbc", "rblf")
@@ -639,6 +676,14 @@
cfg["PRODUCT_COPY_FILES"] = rblf.addsuffix(".sff", "a b c")
cfg["PRODUCT_NAME"] = ((g.get("TARGET_BOARD_PLATFORM", "")).replace(".", " ")).split()[0]
rblf.mkinfo("product.mk", rblf.mkpatsubst("%.pub", "%", g.get("PRODUCT_ADB_KEYS", "")))
+ rblf.mkinfo("product.mk", rblf.dir("foo/bar"))
+ rblf.mkinfo("product.mk", cfg["PRODUCT_COPY_FILES"][0])
+ rblf.mkinfo("product.mk", cfg["PRODUCT_COPY_FILES"][-1])
+ rblf.mkinfo("product.mk", rblf.dir("product.mk"))
+ rblf.mkinfo("product.mk", rblf.dir(cfg["PRODUCT_COPY_FILES"][-1]))
+ rblf.mkinfo("product.mk", rblf.dir((_foobar).split()[-1]))
+ rblf.mkinfo("product.mk", rblf.abspath("foo/bar"))
+ rblf.mkinfo("product.mk", rblf.notdir("foo/bar"))
`,
},
{
@@ -779,7 +824,7 @@
def init(g, handle):
cfg = rblf.cfg(handle)
- if rblf.mkstrip(g.get("TARGET_VENDOR", "")) != "":
+ if rblf.mkstrip(g.get("TARGET_VENDOR", "")):
pass
`,
},
@@ -823,6 +868,30 @@
g["V3"] = g["PRODUCT_ADB_KEYS"]
`,
},
+ {
+ desc: "Dynamic inherit path",
+ mkname: "product.mk",
+ in: `
+MY_PATH=foo
+$(call inherit-product,vendor/$(MY_PATH)/cfg.mk)
+`,
+ expected: `load("//build/make/core:product_config.rbc", "rblf")
+load("//vendor/foo1:cfg.star|init", _cfg_init = "init")
+load("//vendor/bar/baz:cfg.star|init", _cfg1_init = "init")
+
+def init(g, handle):
+ cfg = rblf.cfg(handle)
+ g["MY_PATH"] = "foo"
+ _entry = {
+ "vendor/foo1/cfg.mk": ("_cfg", _cfg_init),
+ "vendor/bar/baz/cfg.mk": ("_cfg1", _cfg1_init),
+ }.get("vendor/%s/cfg.mk" % g["MY_PATH"])
+ (_varmod, _varmod_init) = _entry if _entry else (None, None)
+ if not _varmod_init:
+ rblf.mkerror("cannot")
+ rblf.inherit(handle, _varmod, _varmod_init)
+`,
+ },
}
var known_variables = []struct {
@@ -846,10 +915,47 @@
{"PLATFORM_LIST", VarClassSoong, starlarkTypeList}, // TODO(asmundak): make it local instead of soong
}
+type testMakefileFinder struct {
+ fs fs.FS
+ root string
+ files []string
+}
+
+func (t *testMakefileFinder) Find(root string) []string {
+ if t.files != nil || root == t.root {
+ return t.files
+ }
+ t.files = make([]string, 0)
+ fs.WalkDir(t.fs, root, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ base := filepath.Base(path)
+ if base[0] == '.' && len(base) > 1 {
+ return fs.SkipDir
+ }
+ return nil
+ }
+ if strings.HasSuffix(path, ".mk") {
+ t.files = append(t.files, path)
+ }
+ return nil
+ })
+ return t.files
+}
+
func TestGood(t *testing.T) {
for _, v := range known_variables {
KnownVariables.NewVariable(v.name, v.class, v.starlarkType)
}
+ fs := NewFindMockFS([]string{
+ "vendor/foo1/cfg.mk",
+ "vendor/bar/baz/cfg.mk",
+ "part.mk",
+ "foo/font.mk",
+ "bar/font.mk",
+ })
for _, test := range testCases {
t.Run(test.desc,
func(t *testing.T) {
@@ -859,6 +965,8 @@
RootDir: ".",
OutputSuffix: ".star",
WarnPartialSuccess: true,
+ SourceFS: fs,
+ MakefileFinder: &testMakefileFinder{fs: fs},
})
if err != nil {
t.Error(err)
diff --git a/mk2rbc/node.go b/mk2rbc/node.go
index d4b4222..8efe207 100644
--- a/mk2rbc/node.go
+++ b/mk2rbc/node.go
@@ -42,24 +42,85 @@
}
}
-type inheritedModule struct {
+type moduleInfo struct {
path string // Converted Starlark file path
originalPath string // Makefile file path
- moduleName string
moduleLocalName string
- loadAlways bool
+ optional bool
}
-func (im inheritedModule) name() string {
- return MakePath2ModuleName(im.originalPath)
-}
-
-func (im inheritedModule) entryName() string {
+func (im moduleInfo) entryName() string {
return im.moduleLocalName + "_init"
}
+type inheritedModule interface {
+ name() string
+ entryName() string
+ emitSelect(gctx *generationContext)
+ isLoadAlways() bool
+}
+
+type inheritedStaticModule struct {
+ *moduleInfo
+ loadAlways bool
+}
+
+func (im inheritedStaticModule) name() string {
+ return fmt.Sprintf("%q", MakePath2ModuleName(im.originalPath))
+}
+
+func (im inheritedStaticModule) emitSelect(_ *generationContext) {
+}
+
+func (im inheritedStaticModule) isLoadAlways() bool {
+ return im.loadAlways
+}
+
+type inheritedDynamicModule struct {
+ path interpolateExpr
+ candidateModules []*moduleInfo
+ loadAlways bool
+}
+
+func (i inheritedDynamicModule) name() string {
+ return "_varmod"
+}
+
+func (i inheritedDynamicModule) entryName() string {
+ return i.name() + "_init"
+}
+
+func (i inheritedDynamicModule) emitSelect(gctx *generationContext) {
+ gctx.newLine()
+ gctx.writef("_entry = {")
+ gctx.indentLevel++
+ for _, mi := range i.candidateModules {
+ gctx.newLine()
+ gctx.writef(`"%s": (%q, %s),`, mi.originalPath, mi.moduleLocalName, mi.entryName())
+ }
+ gctx.indentLevel--
+ gctx.newLine()
+ gctx.write("}.get(")
+ i.path.emit(gctx)
+ gctx.write(")")
+ gctx.newLine()
+ gctx.writef("(%s, %s) = _entry if _entry else (None, None)", i.name(), i.entryName())
+ if i.loadAlways {
+ gctx.newLine()
+ gctx.writef("if not %s:", i.entryName())
+ gctx.indentLevel++
+ gctx.newLine()
+ gctx.write(`rblf.mkerror("cannot")`)
+ gctx.indentLevel--
+ }
+}
+
+func (i inheritedDynamicModule) isLoadAlways() bool {
+ return i.loadAlways
+}
+
type inheritNode struct {
- *inheritedModule
+ module inheritedModule
}
func (inn *inheritNode) emit(gctx *generationContext) {
@@ -68,32 +129,40 @@
// Conditional case:
// if <module>_init != None:
// same as above
+ inn.module.emitSelect(gctx)
+
+ name := inn.module.name()
+ entry := inn.module.entryName()
gctx.newLine()
- if inn.loadAlways {
- gctx.writef("%s(handle, %q, %s)", cfnInherit, inn.name(), inn.entryName())
+ if inn.module.isLoadAlways() {
+ gctx.writef("%s(handle, %s, %s)", cfnInherit, name, entry)
return
}
- gctx.writef("if %s != None:", inn.entryName())
+
+ gctx.writef("if %s:", entry)
gctx.indentLevel++
gctx.newLine()
- gctx.writef("%s(handle, %q, %s)", cfnInherit, inn.name(), inn.entryName())
+ gctx.writef("%s(handle, %s, %s)", cfnInherit, name, entry)
gctx.indentLevel--
}
type includeNode struct {
- *inheritedModule
+ module inheritedModule
}
func (inn *includeNode) emit(gctx *generationContext) {
+ inn.module.emitSelect(gctx)
+ entry := inn.module.entryName()
gctx.newLine()
- if inn.loadAlways {
- gctx.writef("%s(g, handle)", inn.entryName())
+ if inn.module.isLoadAlways() {
+ gctx.writef("%s(g, handle)", entry)
return
}
- gctx.writef("if %s != None:", inn.entryName())
+
+ gctx.writef("if %s != None:", entry)
gctx.indentLevel++
gctx.newLine()
- gctx.writef("%s(g, handle)", inn.entryName())
+ gctx.writef("%s(g, handle)", entry)
gctx.indentLevel--
}
diff --git a/mk2rbc/types.go b/mk2rbc/types.go
index 1625464..4a6d376 100644
--- a/mk2rbc/types.go
+++ b/mk2rbc/types.go
@@ -58,3 +58,8 @@
func (s ScopeBase) SetFunc(_ string, _ func([]string) []string) {
panic("implement me")
}
+
+// Used to find all makefiles in the source tree
+type MakefileFinder interface {
+ Find(root string) []string
+}
diff --git a/mk2rbc/variable.go b/mk2rbc/variable.go
index a650453..88d63c9 100644
--- a/mk2rbc/variable.go
+++ b/mk2rbc/variable.go
@@ -16,7 +16,6 @@
import (
"fmt"
- "os"
"strings"
)
@@ -222,15 +221,18 @@
pv.value.emit(gctx)
}
-func (pv predefinedVariable) emitSet(_ *generationContext, asgn *assignmentNode) {
+func (pv predefinedVariable) emitSet(gctx *generationContext, asgn *assignmentNode) {
if expectedValue, ok1 := maybeString(pv.value); ok1 {
actualValue, ok2 := maybeString(asgn.value)
if ok2 {
if actualValue == expectedValue {
return
}
- fmt.Fprintf(os.Stderr, "cannot set predefined variable %s to %q, its value should be %q",
+ gctx.writef("# MK2RBC TRANSLATION ERROR: cannot set predefined variable %s to %q, its value should be %q",
pv.name(), actualValue, expectedValue)
+ gctx.newLine()
+ gctx.write("pass")
+ gctx.starScript.hasErrors = true
return
}
}
diff --git a/rust/config/allowed_list.go b/rust/config/allowed_list.go
index ca110a2..e527aea 100644
--- a/rust/config/allowed_list.go
+++ b/rust/config/allowed_list.go
@@ -6,7 +6,6 @@
// for an example.
// TODO(b/160223496): enable rustfmt globally.
RustAllowedPaths = []string{
- "bionic/libc",
"device/google/cuttlefish",
"external/adhd",
"external/crosvm",
@@ -24,6 +23,7 @@
"system/extras/profcollectd",
"system/extras/simpleperf",
"system/hardware/interfaces/keystore2",
+ "system/librustutils",
"system/logging/rust",
"system/security",
"system/tools/aidl",
diff --git a/rust/sanitize.go b/rust/sanitize.go
index 3d14d51..a4ba4bd 100644
--- a/rust/sanitize.go
+++ b/rust/sanitize.go
@@ -47,6 +47,9 @@
"-C llvm-args=-sanitizer-coverage-trace-geps",
"-C llvm-args=-sanitizer-coverage-prune-blocks=0",
+ // See https://github.com/rust-fuzz/cargo-fuzz/pull/193
+ "-C link-dead-code",
+
// Sancov breaks with lto
// TODO: Remove when https://bugs.llvm.org/show_bug.cgi?id=41734 is resolved and sancov works with LTO
"-C lto=no",