Merge "Denylist modules that depend on a rust binary"
diff --git a/android/rule_builder.go b/android/rule_builder.go
index 098c1fc..11da36c 100644
--- a/android/rule_builder.go
+++ b/android/rule_builder.go
@@ -101,12 +101,7 @@
}
// Restat marks the rule as a restat rule, which will be passed to ModuleContext.Rule in BuildParams.Restat.
-//
-// Restat is not compatible with Sbox()
func (r *RuleBuilder) Restat() *RuleBuilder {
- if r.sbox {
- panic("Restat() is not compatible with Sbox()")
- }
r.restat = true
return r
}
@@ -141,8 +136,6 @@
// point to a location where sbox's manifest will be written and must be outside outputDir. sbox
// will ensure that all outputs have been written, and will discard any output files that were not
// specified.
-//
-// Sbox is not compatible with Restat()
func (r *RuleBuilder) Sbox(outputDir WritablePath, manifestPath WritablePath) *RuleBuilder {
if r.sbox {
panic("Sbox() may not be called more than once")
@@ -150,9 +143,6 @@
if len(r.commands) > 0 {
panic("Sbox() may not be called after Command()")
}
- if r.restat {
- panic("Sbox() is not compatible with Restat()")
- }
r.sbox = true
r.outDir = outputDir
r.sboxManifestPath = manifestPath
@@ -636,11 +626,14 @@
ctx: r.ctx,
},
}
- sboxCmd.Text("rm -rf").Output(r.outDir)
- sboxCmd.Text("&&")
sboxCmd.builtToolWithoutDeps("sbox").
- Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(r.ctx).String())).
- Flag("--manifest").Input(r.sboxManifestPath)
+ FlagWithArg("--sandbox-path ", shared.TempDirForOutDir(PathForOutput(r.ctx).String())).
+ FlagWithArg("--output-dir ", r.outDir.String()).
+ FlagWithInput("--manifest ", r.sboxManifestPath)
+
+ if r.restat {
+ sboxCmd.Flag("--write-if-changed")
+ }
// Replace the command string, and add the sbox tool and manifest textproto to the
// dependencies of the final sbox rule.
diff --git a/android/rule_builder_test.go b/android/rule_builder_test.go
index 3766bb0..86647eb 100644
--- a/android/rule_builder_test.go
+++ b/android/rule_builder_test.go
@@ -678,32 +678,32 @@
})
t.Run("sbox", func(t *testing.T) {
outDir := "out/soong/.intermediates/foo_sbox"
- outFile := filepath.Join(outDir, "gen/foo_sbox")
- depFile := filepath.Join(outDir, "gen/foo_sbox.d")
+ sboxOutDir := filepath.Join(outDir, "gen")
+ outFile := filepath.Join(sboxOutDir, "foo_sbox")
+ depFile := filepath.Join(sboxOutDir, "foo_sbox.d")
rspFile := filepath.Join(outDir, "rsp")
rspFile2 := filepath.Join(outDir, "rsp2")
manifest := filepath.Join(outDir, "sbox.textproto")
sbox := filepath.Join("out", "soong", "host", result.Config.PrebuiltOS(), "bin/sbox")
sandboxPath := shared.TempDirForOutDir("out/soong")
- cmd := `rm -rf ` + outDir + `/gen && ` +
- sbox + ` --sandbox-path ` + sandboxPath + ` --manifest ` + manifest
+ cmd := sbox + ` --sandbox-path ` + sandboxPath + ` --output-dir ` + sboxOutDir + ` --manifest ` + manifest
module := result.ModuleForTests("foo_sbox", "")
check(t, module.Output("gen/foo_sbox"), module.Output(rspFile2),
cmd, outFile, depFile, rspFile, rspFile2, false, []string{manifest}, []string{sbox})
})
t.Run("sbox_inputs", func(t *testing.T) {
outDir := "out/soong/.intermediates/foo_sbox_inputs"
- outFile := filepath.Join(outDir, "gen/foo_sbox_inputs")
- depFile := filepath.Join(outDir, "gen/foo_sbox_inputs.d")
+ sboxOutDir := filepath.Join(outDir, "gen")
+ outFile := filepath.Join(sboxOutDir, "foo_sbox_inputs")
+ depFile := filepath.Join(sboxOutDir, "foo_sbox_inputs.d")
rspFile := filepath.Join(outDir, "rsp")
rspFile2 := filepath.Join(outDir, "rsp2")
manifest := filepath.Join(outDir, "sbox.textproto")
sbox := filepath.Join("out", "soong", "host", result.Config.PrebuiltOS(), "bin/sbox")
sandboxPath := shared.TempDirForOutDir("out/soong")
- cmd := `rm -rf ` + outDir + `/gen && ` +
- sbox + ` --sandbox-path ` + sandboxPath + ` --manifest ` + manifest
+ cmd := sbox + ` --sandbox-path ` + sandboxPath + ` --output-dir ` + sboxOutDir + ` --manifest ` + manifest
module := result.ModuleForTests("foo_sbox_inputs", "")
check(t, module.Output("gen/foo_sbox_inputs"), module.Output(rspFile2),
diff --git a/cc/config/global.go b/cc/config/global.go
index fad675a..0f31931 100644
--- a/cc/config/global.go
+++ b/cc/config/global.go
@@ -225,7 +225,6 @@
"-Wno-deprecated-enum-enum-conversion", // http://b/153746563
"-Wno-string-compare", // http://b/153764102
"-Wno-enum-enum-conversion", // http://b/154138986
- "-Wno-enum-float-conversion", // http://b/154255917
"-Wno-pessimizing-move", // http://b/154270751
// New warnings to be fixed after clang-r399163
"-Wno-non-c-typedef-for-linkage", // http://b/161304145
diff --git a/cmd/sbox/sbox.go b/cmd/sbox/sbox.go
index 4fa7486..418826c 100644
--- a/cmd/sbox/sbox.go
+++ b/cmd/sbox/sbox.go
@@ -38,9 +38,11 @@
)
var (
- sandboxesRoot string
- manifestFile string
- keepOutDir bool
+ sandboxesRoot string
+ outputDir string
+ manifestFile string
+ keepOutDir bool
+ writeIfChanged bool
)
const (
@@ -51,10 +53,14 @@
func init() {
flag.StringVar(&sandboxesRoot, "sandbox-path", "",
"root of temp directory to put the sandbox into")
+ flag.StringVar(&outputDir, "output-dir", "",
+ "directory which will contain all output files and only output files")
flag.StringVar(&manifestFile, "manifest", "",
"textproto manifest describing the sandboxed command(s)")
flag.BoolVar(&keepOutDir, "keep-out-dir", false,
"whether to keep the sandbox directory when done")
+ flag.BoolVar(&writeIfChanged, "write-if-changed", false,
+ "only write the output files if they have changed")
}
func usageViolation(violation string) {
@@ -241,6 +247,12 @@
return "", fmt.Errorf("command is required")
}
+ // Remove files from the output directory
+ err = clearOutputDirectory(command.CopyAfter, outputDir, writeType(writeIfChanged))
+ if err != nil {
+ return "", err
+ }
+
pathToTempDirInSbox := tempDir
if command.GetChdir() {
pathToTempDirInSbox = "."
@@ -252,7 +264,7 @@
}
// Copy in any files specified by the manifest.
- err = copyFiles(command.CopyBefore, "", tempDir, false)
+ err = copyFiles(command.CopyBefore, "", tempDir, requireFromExists, alwaysWrite)
if err != nil {
return "", err
}
@@ -306,7 +318,7 @@
// especially useful for linters with baselines that print an error message on failure
// with a command to copy the output lint errors to the new baseline. Use a copy instead of
// a move to leave the sandbox intact for manual inspection
- copyFiles(command.CopyAfter, tempDir, "", true)
+ copyFiles(command.CopyAfter, tempDir, "", allowFromNotExists, writeType(writeIfChanged))
}
// If the command was executed but failed with an error, print a debugging message before
@@ -327,39 +339,16 @@
return "", err
}
- missingOutputErrors := validateOutputFiles(command.CopyAfter, tempDir)
-
- if len(missingOutputErrors) > 0 {
- // find all created files for making a more informative error message
- createdFiles := findAllFilesUnder(tempDir)
-
- // build error message
- errorMessage := "mismatch between declared and actual outputs\n"
- errorMessage += "in sbox command(" + rawCommand + ")\n\n"
- errorMessage += "in sandbox " + tempDir + ",\n"
- errorMessage += fmt.Sprintf("failed to create %v files:\n", len(missingOutputErrors))
- for _, missingOutputError := range missingOutputErrors {
- errorMessage += " " + missingOutputError.Error() + "\n"
- }
- if len(createdFiles) < 1 {
- errorMessage += "created 0 files."
- } else {
- errorMessage += fmt.Sprintf("did create %v files:\n", len(createdFiles))
- creationMessages := createdFiles
- maxNumCreationLines := 10
- if len(creationMessages) > maxNumCreationLines {
- creationMessages = creationMessages[:maxNumCreationLines]
- creationMessages = append(creationMessages, fmt.Sprintf("...%v more", len(createdFiles)-maxNumCreationLines))
- }
- for _, creationMessage := range creationMessages {
- errorMessage += " " + creationMessage + "\n"
- }
- }
-
- return "", errors.New(errorMessage)
+ err = validateOutputFiles(command.CopyAfter, tempDir, outputDir, rawCommand)
+ if err != nil {
+ return "", err
}
+
// the created files match the declared files; now move them
- err = moveFiles(command.CopyAfter, tempDir, "")
+ err = moveFiles(command.CopyAfter, tempDir, "", writeType(writeIfChanged))
+ if err != nil {
+ return "", err
+ }
return depFile, nil
}
@@ -380,8 +369,9 @@
// validateOutputFiles verifies that all files that have a rule to be copied out of the sandbox
// were created by the command.
-func validateOutputFiles(copies []*sbox_proto.Copy, sandboxDir string) []error {
+func validateOutputFiles(copies []*sbox_proto.Copy, sandboxDir, outputDir, rawCommand string) error {
var missingOutputErrors []error
+ var incorrectOutputDirectoryErrors []error
for _, copyPair := range copies {
fromPath := joinPath(sandboxDir, copyPair.GetFrom())
fileInfo, err := os.Stat(fromPath)
@@ -392,17 +382,91 @@
if fileInfo.IsDir() {
missingOutputErrors = append(missingOutputErrors, fmt.Errorf("%s: not a file", fromPath))
}
+
+ toPath := copyPair.GetTo()
+ if rel, err := filepath.Rel(outputDir, toPath); err != nil {
+ return err
+ } else if strings.HasPrefix(rel, "../") {
+ incorrectOutputDirectoryErrors = append(incorrectOutputDirectoryErrors,
+ fmt.Errorf("%s is not under %s", toPath, outputDir))
+ }
}
- return missingOutputErrors
+
+ const maxErrors = 10
+
+ if len(incorrectOutputDirectoryErrors) > 0 {
+ errorMessage := ""
+ more := 0
+ if len(incorrectOutputDirectoryErrors) > maxErrors {
+ more = len(incorrectOutputDirectoryErrors) - maxErrors
+ incorrectOutputDirectoryErrors = incorrectOutputDirectoryErrors[:maxErrors]
+ }
+
+ for _, err := range incorrectOutputDirectoryErrors {
+ errorMessage += err.Error() + "\n"
+ }
+ if more > 0 {
+ errorMessage += fmt.Sprintf("...%v more", more)
+ }
+
+ return errors.New(errorMessage)
+ }
+
+ if len(missingOutputErrors) > 0 {
+ // find all created files for making a more informative error message
+ createdFiles := findAllFilesUnder(sandboxDir)
+
+ // build error message
+ errorMessage := "mismatch between declared and actual outputs\n"
+ errorMessage += "in sbox command(" + rawCommand + ")\n\n"
+ errorMessage += "in sandbox " + sandboxDir + ",\n"
+ errorMessage += fmt.Sprintf("failed to create %v files:\n", len(missingOutputErrors))
+ for _, missingOutputError := range missingOutputErrors {
+ errorMessage += " " + missingOutputError.Error() + "\n"
+ }
+ if len(createdFiles) < 1 {
+ errorMessage += "created 0 files."
+ } else {
+ errorMessage += fmt.Sprintf("did create %v files:\n", len(createdFiles))
+ creationMessages := createdFiles
+ if len(creationMessages) > maxErrors {
+ creationMessages = creationMessages[:maxErrors]
+ creationMessages = append(creationMessages, fmt.Sprintf("...%v more", len(createdFiles)-maxErrors))
+ }
+ for _, creationMessage := range creationMessages {
+ errorMessage += " " + creationMessage + "\n"
+ }
+ }
+
+ return errors.New(errorMessage)
+ }
+
+ return nil
}
-// copyFiles copies files in or out of the sandbox. If allowFromNotExists is true then errors
-// caused by a from path not existing are ignored.
-func copyFiles(copies []*sbox_proto.Copy, fromDir, toDir string, allowFromNotExists bool) error {
+type existsType bool
+
+const (
+ requireFromExists existsType = false
+ allowFromNotExists = true
+)
+
+type writeType bool
+
+const (
+ alwaysWrite writeType = false
+ onlyWriteIfChanged = true
+)
+
+// copyFiles copies files in or out of the sandbox. If exists is allowFromNotExists then errors
+// caused by a from path not existing are ignored. If write is onlyWriteIfChanged then the output
+// file is compared to the input file and not written to if it is the same, avoiding updating
+// the timestamp.
+func copyFiles(copies []*sbox_proto.Copy, fromDir, toDir string, exists existsType, write writeType) error {
for _, copyPair := range copies {
fromPath := joinPath(fromDir, copyPair.GetFrom())
toPath := joinPath(toDir, copyPair.GetTo())
- err := copyOneFile(fromPath, toPath, copyPair.GetExecutable(), allowFromNotExists)
+ err := copyOneFile(fromPath, toPath, copyPair.GetExecutable(), exists, write)
if err != nil {
return fmt.Errorf("error copying %q to %q: %w", fromPath, toPath, err)
}
@@ -411,8 +475,11 @@
}
// copyOneFile copies a file and its permissions. If forceExecutable is true it adds u+x to the
-// permissions. If allowFromNotExists is true it returns nil if the from path doesn't exist.
-func copyOneFile(from string, to string, forceExecutable, allowFromNotExists bool) error {
+// permissions. If exists is allowFromNotExists it returns nil if the from path doesn't exist.
+// If write is onlyWriteIfChanged then the output file is compared to the input file and not written to
+// if it is the same, avoiding updating the timestamp.
+func copyOneFile(from string, to string, forceExecutable bool, exists existsType,
+ write writeType) error {
err := os.MkdirAll(filepath.Dir(to), 0777)
if err != nil {
return err
@@ -420,7 +487,7 @@
stat, err := os.Stat(from)
if err != nil {
- if os.IsNotExist(err) && allowFromNotExists {
+ if os.IsNotExist(err) && exists == allowFromNotExists {
return nil
}
return err
@@ -431,6 +498,10 @@
perm = perm | 0100 // u+x
}
+ if write == onlyWriteIfChanged && filesHaveSameContents(from, to) {
+ return nil
+ }
+
in, err := os.Open(from)
if err != nil {
return err
@@ -504,7 +575,7 @@
to := applyPathMappings(rspFile.PathMappings, from)
// Copy the file into the sandbox.
- err := copyOneFile(from, joinPath(toDir, to), false, false)
+ err := copyOneFile(from, joinPath(toDir, to), false, requireFromExists, alwaysWrite)
if err != nil {
return err
}
@@ -551,9 +622,10 @@
// moveFiles moves files specified by a set of copy rules. It uses os.Rename, so it is restricted
// to moving files where the source and destination are in the same filesystem. This is OK for
-// sbox because the temporary directory is inside the out directory. It updates the timestamp
-// of the new file.
-func moveFiles(copies []*sbox_proto.Copy, fromDir, toDir string) error {
+// sbox because the temporary directory is inside the out directory. If write is onlyWriteIfChanged
+// then the output file is compared to the input file and not written to if it is the same, avoiding
+// updating the timestamp. Otherwise it always updates the timestamp of the new file.
+func moveFiles(copies []*sbox_proto.Copy, fromDir, toDir string, write writeType) error {
for _, copyPair := range copies {
fromPath := joinPath(fromDir, copyPair.GetFrom())
toPath := joinPath(toDir, copyPair.GetTo())
@@ -562,6 +634,10 @@
return err
}
+ if write == onlyWriteIfChanged && filesHaveSameContents(fromPath, toPath) {
+ continue
+ }
+
err = os.Rename(fromPath, toPath)
if err != nil {
return err
@@ -578,6 +654,37 @@
return nil
}
+// clearOutputDirectory removes all files in the output directory if write is alwaysWrite, or
+// any files not listed in copies if write is onlyWriteIfChanged
+func clearOutputDirectory(copies []*sbox_proto.Copy, outputDir string, write writeType) error {
+ if outputDir == "" {
+ return fmt.Errorf("output directory must be set")
+ }
+
+ if write == alwaysWrite {
+ // When writing all the output files remove the whole output directory
+ return os.RemoveAll(outputDir)
+ }
+
+ outputFiles := make(map[string]bool, len(copies))
+ for _, copyPair := range copies {
+ outputFiles[copyPair.GetTo()] = true
+ }
+
+ existingFiles := findAllFilesUnder(outputDir)
+ for _, existingFile := range existingFiles {
+ fullExistingFile := filepath.Join(outputDir, existingFile)
+ if !outputFiles[fullExistingFile] {
+ err := os.Remove(fullExistingFile)
+ if err != nil {
+ return fmt.Errorf("failed to remove obsolete output file %s: %w", fullExistingFile, err)
+ }
+ }
+ }
+
+ return nil
+}
+
// Rewrite one or more depfiles so that it doesn't include the (randomized) sandbox directory
// to an output file.
func rewriteDepFiles(ins []string, out string) error {
@@ -621,6 +728,66 @@
return filepath.Join(dir, file)
}
+// filesHaveSameContents compares the contents if two files, returning true if they are the same
+// and returning false if they are different or any errors occur.
+func filesHaveSameContents(a, b string) bool {
+ // Compare the sizes of the two files
+ statA, err := os.Stat(a)
+ if err != nil {
+ return false
+ }
+ statB, err := os.Stat(b)
+ if err != nil {
+ return false
+ }
+
+ if statA.Size() != statB.Size() {
+ return false
+ }
+
+ // Open the two files
+ fileA, err := os.Open(a)
+ if err != nil {
+ return false
+ }
+ defer fileA.Close()
+ fileB, err := os.Open(a)
+ if err != nil {
+ return false
+ }
+ defer fileB.Close()
+
+ // Compare the files 1MB at a time
+ const bufSize = 1 * 1024 * 1024
+ bufA := make([]byte, bufSize)
+ bufB := make([]byte, bufSize)
+
+ remain := statA.Size()
+ for remain > 0 {
+ toRead := int64(bufSize)
+ if toRead > remain {
+ toRead = remain
+ }
+
+ _, err = io.ReadFull(fileA, bufA[:toRead])
+ if err != nil {
+ return false
+ }
+ _, err = io.ReadFull(fileB, bufB[:toRead])
+ if err != nil {
+ return false
+ }
+
+ if bytes.Compare(bufA[:toRead], bufB[:toRead]) != 0 {
+ return false
+ }
+
+ remain -= toRead
+ }
+
+ return true
+}
+
func makeAbsPathEnv(pathEnv string) (string, error) {
pathEnvElements := filepath.SplitList(pathEnv)
for i, p := range pathEnvElements {
diff --git a/java/droidstubs.go b/java/droidstubs.go
index e7aeeb8..3b1f7c0 100644
--- a/java/droidstubs.go
+++ b/java/droidstubs.go
@@ -433,6 +433,10 @@
}
}
+func metalavaUseRbe(ctx android.ModuleContext) bool {
+ return ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_METALAVA")
+}
+
func metalavaCmd(ctx android.ModuleContext, rule *android.RuleBuilder, javaVersion javaVersion, srcs android.Paths,
srcJarList android.Path, bootclasspath, classpath classpath, homeDir android.WritablePath) *android.RuleBuilderCommand {
rule.Command().Text("rm -rf").Flag(homeDir.String())
@@ -441,7 +445,7 @@
cmd := rule.Command()
cmd.FlagWithArg("ANDROID_PREFS_ROOT=", homeDir.String())
- if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_METALAVA") {
+ if metalavaUseRbe(ctx) {
rule.Remoteable(android.RemoteRuleSupports{RBE: true})
execStrategy := ctx.Config().GetenvWithDefault("RBE_METALAVA_EXEC_STRATEGY", remoteexec.LocalExecStrategy)
labels := map[string]string{"type": "tool", "name": "metalava"}
@@ -665,7 +669,9 @@
}
// TODO(b/183630617): rewrapper doesn't support restat rules
- // rule.Restat()
+ if !metalavaUseRbe(ctx) {
+ rule.Restat()
+ }
zipSyncCleanupCmd(rule, srcJarDir)
diff --git a/mk2rbc/expr.go b/mk2rbc/expr.go
index dc16d1d..54bb6d1 100644
--- a/mk2rbc/expr.go
+++ b/mk2rbc/expr.go
@@ -221,11 +221,9 @@
}
func (xi *interpolateExpr) transform(transformer func(expr starlarkExpr) starlarkExpr) starlarkExpr {
- argsCopy := make([]starlarkExpr, len(xi.args))
- for i, arg := range xi.args {
- argsCopy[i] = arg.transform(transformer)
+ for i := range xi.args {
+ xi.args[i] = xi.args[i].transform(transformer)
}
- xi.args = argsCopy
if replacement := transformer(xi); replacement != nil {
return replacement
} else {
@@ -591,11 +589,9 @@
if cx.object != nil {
cx.object = cx.object.transform(transformer)
}
- argsCopy := make([]starlarkExpr, len(cx.args))
- for i, arg := range cx.args {
- argsCopy[i] = arg.transform(transformer)
+ for i := range cx.args {
+ cx.args[i] = cx.args[i].transform(transformer)
}
- cx.args = argsCopy
if replacement := transformer(cx); replacement != nil {
return replacement
} else {
@@ -769,3 +765,35 @@
x, ok := expr.(*stringLiteralExpr)
return ok && x.literal == ""
}
+
+func negateExpr(expr starlarkExpr) starlarkExpr {
+ switch typedExpr := expr.(type) {
+ case *notExpr:
+ return typedExpr.expr
+ case *inExpr:
+ typedExpr.isNot = !typedExpr.isNot
+ return typedExpr
+ case *eqExpr:
+ typedExpr.isEq = !typedExpr.isEq
+ return typedExpr
+ case *binaryOpExpr:
+ switch typedExpr.op {
+ case ">":
+ typedExpr.op = "<="
+ return typedExpr
+ case "<":
+ typedExpr.op = ">="
+ return typedExpr
+ case ">=":
+ typedExpr.op = "<"
+ return typedExpr
+ case "<=":
+ typedExpr.op = ">"
+ return typedExpr
+ default:
+ return ¬Expr{expr: expr}
+ }
+ default:
+ return ¬Expr{expr: expr}
+ }
+}
diff --git a/mk2rbc/mk2rbc.go b/mk2rbc/mk2rbc.go
index 950a1e5..8807437 100644
--- a/mk2rbc/mk2rbc.go
+++ b/mk2rbc/mk2rbc.go
@@ -86,7 +86,7 @@
"filter": &simpleCallParser{name: baseName + ".filter", returnType: starlarkTypeList},
"filter-out": &simpleCallParser{name: baseName + ".filter_out", returnType: starlarkTypeList},
"firstword": &firstOrLastwordCallParser{isLastWord: false},
- "foreach": &foreachCallPaser{},
+ "foreach": &foreachCallParser{},
"if": &ifCallParser{},
"info": &makeControlFuncParser{name: baseName + ".mkinfo"},
"is-board-platform": &simpleCallParser{name: baseName + ".board_platform_is", returnType: starlarkTypeBool, addGlobals: true},
@@ -117,6 +117,17 @@
"wildcard": &simpleCallParser{name: baseName + ".expand_wildcard", returnType: starlarkTypeList},
}
+// The same as knownFunctions, but returns a []starlarkNode instead of a starlarkExpr
+var knownNodeFunctions = map[string]interface {
+ parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode
+}{
+ "eval": &evalNodeParser{},
+ "if": &ifCallNodeParser{},
+ "inherit-product": &inheritProductCallParser{loadAlways: true},
+ "inherit-product-if-exists": &inheritProductCallParser{loadAlways: false},
+ "foreach": &foreachCallNodeParser{},
+}
+
// These are functions that we don't implement conversions for, but
// we allow seeing their definitions in the product config files.
var ignoredDefines = map[string]bool{
@@ -846,15 +857,19 @@
return res
}
-func (ctx *parseContext) handleInheritModule(v mkparser.Node, args *mkparser.MakeString, loadAlways bool) []starlarkNode {
+type inheritProductCallParser struct {
+ loadAlways bool
+}
+
+func (p *inheritProductCallParser) parse(ctx *parseContext, v mkparser.Node, args *mkparser.MakeString) []starlarkNode {
args.TrimLeftSpaces()
args.TrimRightSpaces()
pathExpr := ctx.parseMakeString(v, args)
if _, ok := pathExpr.(*badExpr); ok {
return []starlarkNode{ctx.newBadNode(v, "Unable to parse argument to inherit")}
}
- return ctx.handleSubConfig(v, pathExpr, loadAlways, func(im inheritedModule) starlarkNode {
- return &inheritNode{im, loadAlways}
+ return ctx.handleSubConfig(v, pathExpr, p.loadAlways, func(im inheritedModule) starlarkNode {
+ return &inheritNode{im, p.loadAlways}
})
}
@@ -873,19 +888,12 @@
// $(error xxx)
// $(call other-custom-functions,...)
- // inherit-product(-if-exists) gets converted to a series of statements,
- // not just a single expression like parseReference returns. So handle it
- // separately at the beginning here.
- if strings.HasPrefix(v.Name.Dump(), "call inherit-product,") {
- args := v.Name.Clone()
- args.ReplaceLiteral("call inherit-product,", "")
- return ctx.handleInheritModule(v, args, true)
+ if name, args, ok := ctx.maybeParseFunctionCall(v, v.Name); ok {
+ if kf, ok := knownNodeFunctions[name]; ok {
+ return kf.parse(ctx, v, args)
+ }
}
- if strings.HasPrefix(v.Name.Dump(), "call inherit-product-if-exists,") {
- args := v.Name.Clone()
- args.ReplaceLiteral("call inherit-product-if-exists,", "")
- return ctx.handleInheritModule(v, args, false)
- }
+
return []starlarkNode{&exprNode{expr: ctx.parseReference(v, v.Name)}}
}
@@ -1030,49 +1038,19 @@
otherOperand = xLeft
}
- not := func(expr starlarkExpr) starlarkExpr {
- switch typedExpr := expr.(type) {
- case *inExpr:
- typedExpr.isNot = !typedExpr.isNot
- return typedExpr
- case *eqExpr:
- typedExpr.isEq = !typedExpr.isEq
- return typedExpr
- case *binaryOpExpr:
- switch typedExpr.op {
- case ">":
- typedExpr.op = "<="
- return typedExpr
- case "<":
- typedExpr.op = ">="
- return typedExpr
- case ">=":
- typedExpr.op = "<"
- return typedExpr
- case "<=":
- typedExpr.op = ">"
- return typedExpr
- default:
- return ¬Expr{expr: expr}
- }
- default:
- return ¬Expr{expr: expr}
- }
- }
-
// If we've identified one of the operands as being a string literal, check
// for some special cases we can do to simplify the resulting expression.
if otherOperand != nil {
if stringOperand == "" {
if isEq {
- return not(otherOperand)
+ return negateExpr(otherOperand)
} else {
return otherOperand
}
}
if stringOperand == "true" && otherOperand.typ() == starlarkTypeBool {
if !isEq {
- return not(otherOperand)
+ return negateExpr(otherOperand)
} else {
return otherOperand
}
@@ -1228,6 +1206,37 @@
right: xValue, isEq: !negate}
}
+func (ctx *parseContext) maybeParseFunctionCall(node mkparser.Node, ref *mkparser.MakeString) (name string, args *mkparser.MakeString, ok bool) {
+ ref.TrimLeftSpaces()
+ ref.TrimRightSpaces()
+
+ words := ref.SplitN(" ", 2)
+ if !words[0].Const() {
+ return "", nil, false
+ }
+
+ name = words[0].Dump()
+ args = mkparser.SimpleMakeString("", words[0].Pos())
+ if len(words) >= 2 {
+ args = words[1]
+ }
+ args.TrimLeftSpaces()
+ if name == "call" {
+ words = args.SplitN(",", 2)
+ if words[0].Empty() || !words[0].Const() {
+ return "", nil, false
+ }
+ name = words[0].Dump()
+ if len(words) < 2 {
+ args = &mkparser.MakeString{}
+ } else {
+ args = words[1]
+ }
+ }
+ ok = true
+ return
+}
+
// parses $(...), returning an expression
func (ctx *parseContext) parseReference(node mkparser.Node, ref *mkparser.MakeString) starlarkExpr {
ref.TrimLeftSpaces()
@@ -1242,7 +1251,7 @@
// If it is a single word, it can be a simple variable
// reference or a function call
- if len(words) == 1 && !isMakeControlFunc(refDump) && refDump != "shell" {
+ if len(words) == 1 && !isMakeControlFunc(refDump) && refDump != "shell" && refDump != "eval" {
if strings.HasPrefix(refDump, soongNsPrefix) {
// TODO (asmundak): if we find many, maybe handle them.
return ctx.newBadExpr(node, "SOONG_CONFIG_ variables cannot be referenced, use soong_config_get instead: %s", refDump)
@@ -1281,28 +1290,14 @@
return ctx.newBadExpr(node, "unknown variable %s", refDump)
}
- expr := &callExpr{name: words[0].Dump(), returnType: starlarkTypeUnknown}
- args := mkparser.SimpleMakeString("", words[0].Pos())
- if len(words) >= 2 {
- args = words[1]
- }
- args.TrimLeftSpaces()
- if expr.name == "call" {
- words = args.SplitN(",", 2)
- if words[0].Empty() || !words[0].Const() {
- return ctx.newBadExpr(node, "cannot handle %s", refDump)
- }
- expr.name = words[0].Dump()
- if len(words) < 2 {
- args = &mkparser.MakeString{}
+ if name, args, ok := ctx.maybeParseFunctionCall(node, ref); ok {
+ if kf, found := knownFunctions[name]; found {
+ return kf.parse(ctx, node, args)
} else {
- args = words[1]
+ return ctx.newBadExpr(node, "cannot handle invoking %s", name)
}
- }
- if kf, found := knownFunctions[expr.name]; found {
- return kf.parse(ctx, node, args)
} else {
- return ctx.newBadExpr(node, "cannot handle invoking %s", expr.name)
+ return ctx.newBadExpr(node, "cannot handle %s", refDump)
}
}
@@ -1486,9 +1481,46 @@
}
}
-type foreachCallPaser struct{}
+type ifCallNodeParser struct{}
-func (p *foreachCallPaser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
+func (p *ifCallNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode {
+ words := args.Split(",")
+ if len(words) != 2 && len(words) != 3 {
+ return []starlarkNode{ctx.newBadNode(node, "if function should have 2 or 3 arguments, found "+strconv.Itoa(len(words)))}
+ }
+
+ ifn := &ifNode{expr: ctx.parseMakeString(node, words[0])}
+ cases := []*switchCase{
+ {
+ gate: ifn,
+ nodes: ctx.parseNodeMakeString(node, words[1]),
+ },
+ }
+ if len(words) == 3 {
+ cases = append(cases, &switchCase{
+ gate: &elseNode{},
+ nodes: ctx.parseNodeMakeString(node, words[2]),
+ })
+ }
+ if len(cases) == 2 {
+ if len(cases[1].nodes) == 0 {
+ // Remove else branch if it has no contents
+ cases = cases[:1]
+ } else if len(cases[0].nodes) == 0 {
+ // If the if branch has no contents but the else does,
+ // move them to the if and negate its condition
+ ifn.expr = negateExpr(ifn.expr)
+ cases[0].nodes = cases[1].nodes
+ cases = cases[:1]
+ }
+ }
+
+ return []starlarkNode{&switchNode{ssCases: cases}}
+}
+
+type foreachCallParser struct{}
+
+func (p *foreachCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
words := args.Split(",")
if len(words) != 3 {
return ctx.newBadExpr(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words)))
@@ -1520,6 +1552,71 @@
}
}
+func transformNode(node starlarkNode, transformer func(expr starlarkExpr) starlarkExpr) {
+ switch a := node.(type) {
+ case *ifNode:
+ a.expr = a.expr.transform(transformer)
+ case *switchCase:
+ transformNode(a.gate, transformer)
+ for _, n := range a.nodes {
+ transformNode(n, transformer)
+ }
+ case *switchNode:
+ for _, n := range a.ssCases {
+ transformNode(n, transformer)
+ }
+ case *exprNode:
+ a.expr = a.expr.transform(transformer)
+ case *assignmentNode:
+ a.value = a.value.transform(transformer)
+ case *foreachNode:
+ a.list = a.list.transform(transformer)
+ for _, n := range a.actions {
+ transformNode(n, transformer)
+ }
+ }
+}
+
+type foreachCallNodeParser struct{}
+
+func (p *foreachCallNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode {
+ words := args.Split(",")
+ if len(words) != 3 {
+ return []starlarkNode{ctx.newBadNode(node, "foreach function should have 3 arguments, found "+strconv.Itoa(len(words)))}
+ }
+ if !words[0].Const() || words[0].Empty() || !identifierFullMatchRegex.MatchString(words[0].Strings[0]) {
+ return []starlarkNode{ctx.newBadNode(node, "first argument to foreach function must be a simple string identifier")}
+ }
+
+ loopVarName := words[0].Strings[0]
+
+ list := ctx.parseMakeString(node, words[1])
+ if list.typ() != starlarkTypeList {
+ list = &callExpr{
+ name: baseName + ".words",
+ returnType: starlarkTypeList,
+ args: []starlarkExpr{list},
+ }
+ }
+
+ actions := ctx.parseNodeMakeString(node, words[2])
+ // TODO(colefaust): Replace transforming code with something more elegant
+ for _, action := range actions {
+ transformNode(action, func(expr starlarkExpr) starlarkExpr {
+ if varRefExpr, ok := expr.(*variableRefExpr); ok && varRefExpr.ref.name() == loopVarName {
+ return &identifierExpr{loopVarName}
+ }
+ return nil
+ })
+ }
+
+ return []starlarkNode{&foreachNode{
+ varName: loopVarName,
+ list: list,
+ actions: actions,
+ }}
+}
+
type wordCallParser struct{}
func (p *wordCallParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) starlarkExpr {
@@ -1630,6 +1727,31 @@
}
}
+type evalNodeParser struct{}
+
+func (p *evalNodeParser) parse(ctx *parseContext, node mkparser.Node, args *mkparser.MakeString) []starlarkNode {
+ parser := mkparser.NewParser("Eval expression", strings.NewReader(args.Dump()))
+ nodes, errs := parser.Parse()
+ if errs != nil {
+ return []starlarkNode{ctx.newBadNode(node, "Unable to parse eval statement")}
+ }
+
+ if len(nodes) == 0 {
+ return []starlarkNode{}
+ } else if len(nodes) == 1 {
+ switch n := nodes[0].(type) {
+ case *mkparser.Assignment:
+ if n.Name.Const() {
+ return ctx.handleAssignment(n)
+ }
+ case *mkparser.Comment:
+ return []starlarkNode{&commentNode{strings.TrimSpace("#" + n.Comment)}}
+ }
+ }
+
+ return []starlarkNode{ctx.newBadNode(node, "Eval expression too complex; only assignments and comments are supported")}
+}
+
func (ctx *parseContext) parseMakeString(node mkparser.Node, mk *mkparser.MakeString) starlarkExpr {
if mk.Const() {
return &stringLiteralExpr{mk.Dump()}
@@ -1654,6 +1776,16 @@
return NewInterpolateExpr(parts)
}
+func (ctx *parseContext) parseNodeMakeString(node mkparser.Node, mk *mkparser.MakeString) []starlarkNode {
+ // Discard any constant values in the make string, as they would be top level
+ // string literals and do nothing.
+ result := make([]starlarkNode, 0, len(mk.Variables))
+ for i := range mk.Variables {
+ result = append(result, ctx.handleVariable(&mk.Variables[i])...)
+ }
+ return result
+}
+
// Handles the statements whose treatment is the same in all contexts: comment,
// assignment, variable (which is a macro call in reality) and all constructs that
// do not handle in any context ('define directive and any unrecognized stuff).
@@ -1698,6 +1830,7 @@
if result == nil {
result = []starlarkNode{}
}
+
return result
}
diff --git a/mk2rbc/mk2rbc_test.go b/mk2rbc/mk2rbc_test.go
index 2b447e3..3698813 100644
--- a/mk2rbc/mk2rbc_test.go
+++ b/mk2rbc/mk2rbc_test.go
@@ -1313,6 +1313,11 @@
FOREACH_WITH_IF := $(foreach module,\
$(BOOT_KERNEL_MODULES_LIST),\
$(if $(filter $(module),foo.ko),,$(error module "$(module)" has an error!)))
+
+# Same as above, but not assigning it to a variable allows it to be converted to statements
+$(foreach module,\
+ $(BOOT_KERNEL_MODULES_LIST),\
+ $(if $(filter $(module),foo.ko),,$(error module "$(module)" has an error!)))
`,
expected: `load("//build/make/core:product_config.rbc", "rblf")
@@ -1324,6 +1329,10 @@
g["BOOT_KERNEL_MODULES_LIST"] += ["bar.ko"]
g["BOOT_KERNEL_MODULES_FILTER_2"] = ["%%/%s" % m for m in g["BOOT_KERNEL_MODULES_LIST"]]
g["FOREACH_WITH_IF"] = [("" if rblf.filter(module, "foo.ko") else rblf.mkerror("product.mk", "module \"%s\" has an error!" % module)) for module in g["BOOT_KERNEL_MODULES_LIST"]]
+ # Same as above, but not assigning it to a variable allows it to be converted to statements
+ for module in g["BOOT_KERNEL_MODULES_LIST"]:
+ if not rblf.filter(module, "foo.ko"):
+ rblf.mkerror("product.mk", "module \"%s\" has an error!" % module)
`,
},
{
@@ -1474,6 +1483,34 @@
`,
},
+ {
+ desc: "Evals",
+ mkname: "product.mk",
+ in: `
+$(eval)
+$(eval MY_VAR := foo)
+$(eval # This is a test of eval functions)
+$(eval $(TOO_COMPLICATED) := bar)
+$(foreach x,$(MY_LIST_VAR), \
+ $(eval PRODUCT_COPY_FILES += foo/bar/$(x):$(TARGET_COPY_OUT_VENDOR)/etc/$(x)) \
+ $(if $(MY_OTHER_VAR),$(eval PRODUCT_COPY_FILES += $(MY_OTHER_VAR):foo/bar/$(x))) \
+)
+
+`,
+ expected: `load("//build/make/core:product_config.rbc", "rblf")
+
+def init(g, handle):
+ cfg = rblf.cfg(handle)
+ g["MY_VAR"] = "foo"
+ # This is a test of eval functions
+ rblf.mk2rbc_error("product.mk:5", "Eval expression too complex; only assignments and comments are supported")
+ for x in rblf.words(g.get("MY_LIST_VAR", "")):
+ rblf.setdefault(handle, "PRODUCT_COPY_FILES")
+ cfg["PRODUCT_COPY_FILES"] += ("foo/bar/%s:%s/etc/%s" % (x, g.get("TARGET_COPY_OUT_VENDOR", ""), x)).split()
+ if g.get("MY_OTHER_VAR", ""):
+ cfg["PRODUCT_COPY_FILES"] += ("%s:foo/bar/%s" % (g.get("MY_OTHER_VAR", ""), x)).split()
+`,
+ },
}
var known_variables = []struct {
diff --git a/mk2rbc/node.go b/mk2rbc/node.go
index 9d5af91..c0c4c98 100644
--- a/mk2rbc/node.go
+++ b/mk2rbc/node.go
@@ -294,3 +294,28 @@
ssCase.emit(gctx)
}
}
+
+type foreachNode struct {
+ varName string
+ list starlarkExpr
+ actions []starlarkNode
+}
+
+func (f *foreachNode) emit(gctx *generationContext) {
+ gctx.newLine()
+ gctx.writef("for %s in ", f.varName)
+ f.list.emit(gctx)
+ gctx.write(":")
+ gctx.indentLevel++
+ hasStatements := false
+ for _, a := range f.actions {
+ if _, ok := a.(*commentNode); !ok {
+ hasStatements = true
+ }
+ a.emit(gctx)
+ }
+ if !hasStatements {
+ gctx.emitPass()
+ }
+ gctx.indentLevel--
+}
diff --git a/rust/config/allowed_list.go b/rust/config/allowed_list.go
index 30700dd..802e1da 100644
--- a/rust/config/allowed_list.go
+++ b/rust/config/allowed_list.go
@@ -28,6 +28,7 @@
"prebuilts/rust",
"system/core/debuggerd/rust",
"system/core/libstats/pull_rust",
+ "system/core/trusty/libtrusty-rs",
"system/extras/profcollectd",
"system/extras/simpleperf",
"system/hardware/interfaces/keystore2",