Move genrule on top of RuleBuilder
In preparation for more complicated sandboxing support in sbox, use
a single implementation of the sbox sandboxing by moving genrule to
use RuleBuilder's sbox support instead of creating an sbox rule
directly.
Also move genrule's input list hash support into RuleBuilder.
Test: genrule_test.go
Test: rule_builder_test.go
Change-Id: I292184d02743c7e6887ebbcd232ba565db2ab0cc
diff --git a/android/rule_builder.go b/android/rule_builder.go
index 8dc9d6a..86418b2 100644
--- a/android/rule_builder.go
+++ b/android/rule_builder.go
@@ -15,7 +15,9 @@
package android
import (
+ "crypto/sha256"
"fmt"
+ "path/filepath"
"sort"
"strings"
@@ -25,6 +27,8 @@
"android/soong/shared"
)
+const sboxOutDir = "__SBOX_OUT_DIR__"
+
// RuleBuilder provides an alternative to ModuleContext.Rule and ModuleContext.Build to add a command line to the build
// graph.
type RuleBuilder struct {
@@ -133,8 +137,8 @@
// race with any call to Build.
func (r *RuleBuilder) Command() *RuleBuilderCommand {
command := &RuleBuilderCommand{
- sbox: r.sbox,
- sboxOutDir: r.sboxOutDir,
+ sbox: r.sbox,
+ outDir: r.sboxOutDir,
}
r.commands = append(r.commands, command)
return command
@@ -163,7 +167,7 @@
}
// Inputs returns the list of paths that were passed to the RuleBuilderCommand methods that take
-// input paths, such as RuleBuilderCommand.Input, RuleBuilderComand.Implicit, or
+// input paths, such as RuleBuilderCommand.Input, RuleBuilderCommand.Implicit, or
// RuleBuilderCommand.FlagWithInput. Inputs to a command that are also outputs of another command
// in the same RuleBuilder are filtered out. The list is sorted and duplicates removed.
func (r *RuleBuilder) Inputs() Paths {
@@ -362,7 +366,7 @@
return commands
}
-// NinjaEscapedCommands returns a slice containin the built command line after ninja escaping for each call to
+// NinjaEscapedCommands returns a slice containing the built command line after ninja escaping for each call to
// RuleBuilder.Command.
func (r *RuleBuilder) NinjaEscapedCommands() []string {
var commands []string
@@ -427,6 +431,7 @@
tools := r.Tools()
commands := r.NinjaEscapedCommands()
outputs := r.Outputs()
+ inputs := r.Inputs()
if len(commands) == 0 {
return
@@ -440,7 +445,7 @@
if r.sbox {
sboxOutputs := make([]string, len(outputs))
for i, output := range outputs {
- sboxOutputs[i] = "__SBOX_OUT_DIR__/" + Rel(ctx, r.sboxOutDir.String(), output.String())
+ sboxOutputs[i] = filepath.Join(sboxOutDir, Rel(ctx, r.sboxOutDir.String(), output.String()))
}
commandString = proptools.ShellEscape(commandString)
@@ -458,10 +463,19 @@
sboxCmd.Flag("--depfile-out").Text(depFile.String())
}
+ // Add a hash of the list of input files to the xbox command line so that ninja reruns
+ // it when the list of input files changes.
+ sboxCmd.FlagWithArg("--input-hash ", hashSrcFiles(inputs))
+
sboxCmd.Flags(sboxOutputs)
commandString = sboxCmd.buf.String()
tools = append(tools, sboxCmd.tools...)
+ } else {
+ // If not using sbox the rule will run the command directly, put the hash of the
+ // list of input files in a comment at the end of the command line to ensure ninja
+ // reruns the rule when the list of input files changes.
+ commandString += " # hash of input list: " + hashSrcFiles(inputs)
}
// Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to
@@ -499,7 +513,7 @@
Pool: pool,
}),
Inputs: rspFileInputs,
- Implicits: r.Inputs(),
+ Implicits: inputs,
Output: output,
ImplicitOutputs: implicitOutputs,
SymlinkOutputs: r.SymlinkOutputs(),
@@ -527,14 +541,16 @@
// spans [start,end) of the command that should not be ninja escaped
unescapedSpans [][2]int
- sbox bool
- sboxOutDir WritablePath
+ sbox bool
+ // outDir is the directory that will contain the output files of the rules. sbox will copy
+ // the output files from the sandbox directory to this directory when it finishes.
+ outDir WritablePath
}
func (c *RuleBuilderCommand) addInput(path Path) string {
if c.sbox {
- if rel, isRel, _ := maybeRelErr(c.sboxOutDir.String(), path.String()); isRel {
- return "__SBOX_OUT_DIR__/" + rel
+ if rel, isRel, _ := maybeRelErr(c.outDir.String(), path.String()); isRel {
+ return filepath.Join(sboxOutDir, rel)
}
}
c.inputs = append(c.inputs, path)
@@ -543,8 +559,8 @@
func (c *RuleBuilderCommand) addImplicit(path Path) string {
if c.sbox {
- if rel, isRel, _ := maybeRelErr(c.sboxOutDir.String(), path.String()); isRel {
- return "__SBOX_OUT_DIR__/" + rel
+ if rel, isRel, _ := maybeRelErr(c.outDir.String(), path.String()); isRel {
+ return filepath.Join(sboxOutDir, rel)
}
}
c.implicits = append(c.implicits, path)
@@ -555,15 +571,22 @@
c.orderOnlys = append(c.orderOnlys, path)
}
-func (c *RuleBuilderCommand) outputStr(path Path) string {
+func (c *RuleBuilderCommand) outputStr(path WritablePath) string {
if c.sbox {
- // Errors will be handled in RuleBuilder.Build where we have a context to report them
- rel, _, _ := maybeRelErr(c.sboxOutDir.String(), path.String())
- return "__SBOX_OUT_DIR__/" + rel
+ return SboxPathForOutput(path, c.outDir)
}
return path.String()
}
+// SboxPathForOutput takes an output path and the out directory passed to RuleBuilder.Sbox(),
+// and returns the corresponding path for the output in the sbox sandbox. This can be used
+// on the RuleBuilder command line to reference the output.
+func SboxPathForOutput(path WritablePath, outDir WritablePath) string {
+ // Errors will be handled in RuleBuilder.Build where we have a context to report them
+ rel, _, _ := maybeRelErr(outDir.String(), path.String())
+ return filepath.Join(sboxOutDir, rel)
+}
+
// Text adds the specified raw text to the command line. The text should not contain input or output paths or the
// rule will not have them listed in its dependencies or outputs.
func (c *RuleBuilderCommand) Text(text string) *RuleBuilderCommand {
@@ -727,7 +750,7 @@
if !c.sbox {
panic("OutputDir only valid with Sbox")
}
- return c.Text("__SBOX_OUT_DIR__")
+ return c.Text(sboxOutDir)
}
// DepFile adds the specified depfile path to the paths returned by RuleBuilder.DepFiles and adds it to the command
@@ -906,3 +929,12 @@
}
return s
}
+
+// hashSrcFiles returns a hash of the list of source files. It is used to ensure the command line
+// or the sbox textproto manifest change even if the input files are not listed on the command line.
+func hashSrcFiles(srcFiles Paths) string {
+ h := sha256.New()
+ srcFileList := strings.Join(srcFiles.Strings(), "\n")
+ h.Write([]byte(srcFileList))
+ return fmt.Sprintf("%x", h.Sum(nil))
+}
diff --git a/android/rule_builder_test.go b/android/rule_builder_test.go
index ca6359d..c1d5521 100644
--- a/android/rule_builder_test.go
+++ b/android/rule_builder_test.go
@@ -18,6 +18,7 @@
"fmt"
"path/filepath"
"reflect"
+ "regexp"
"strings"
"testing"
@@ -441,7 +442,7 @@
type testRuleBuilderModule struct {
ModuleBase
properties struct {
- Src string
+ Srcs []string
Restat bool
Sbox bool
@@ -449,7 +450,7 @@
}
func (t *testRuleBuilderModule) GenerateAndroidBuildActions(ctx ModuleContext) {
- in := PathForSource(ctx, t.properties.Src)
+ in := PathsForSource(ctx, t.properties.Srcs)
out := PathForModuleOut(ctx, ctx.ModuleName())
outDep := PathForModuleOut(ctx, ctx.ModuleName()+".d")
outDir := PathForModuleOut(ctx)
@@ -468,17 +469,17 @@
out := PathForOutput(ctx, "baz")
outDep := PathForOutput(ctx, "baz.d")
outDir := PathForOutput(ctx)
- testRuleBuilder_Build(ctx, in, out, outDep, outDir, true, false)
+ testRuleBuilder_Build(ctx, Paths{in}, out, outDep, outDir, true, false)
}
-func testRuleBuilder_Build(ctx BuilderContext, in Path, out, outDep, outDir WritablePath, restat, sbox bool) {
+func testRuleBuilder_Build(ctx BuilderContext, in Paths, out, outDep, outDir WritablePath, restat, sbox bool) {
rule := NewRuleBuilder()
if sbox {
rule.Sbox(outDir)
}
- rule.Command().Tool(PathForSource(ctx, "cp")).Input(in).Output(out).ImplicitDepFile(outDep)
+ rule.Command().Tool(PathForSource(ctx, "cp")).Inputs(in).Output(out).ImplicitDepFile(outDep)
if restat {
rule.Restat()
@@ -496,12 +497,12 @@
bp := `
rule_builder_test {
name: "foo",
- src: "bar",
+ srcs: ["bar"],
restat: true,
}
rule_builder_test {
name: "foo_sbox",
- src: "bar",
+ srcs: ["bar"],
sbox: true,
}
`
@@ -519,7 +520,10 @@
check := func(t *testing.T, params TestingBuildParams, wantCommand, wantOutput, wantDepfile string, wantRestat bool, extraCmdDeps []string) {
t.Helper()
- if params.RuleParams.Command != wantCommand {
+ command := params.RuleParams.Command
+ re := regexp.MustCompile(" (# hash of input list:|--input-hash) [a-z0-9]*")
+ command = re.ReplaceAllLiteralString(command, "")
+ if command != wantCommand {
t.Errorf("\nwant RuleParams.Command = %q\n got %q", wantCommand, params.RuleParams.Command)
}
@@ -651,3 +655,78 @@
})
}
}
+
+func TestRuleBuilderHashInputs(t *testing.T) {
+ // The basic idea here is to verify that the command (in the case of a
+ // non-sbox rule) or the sbox textproto manifest contain a hash of the
+ // inputs.
+
+ // By including a hash of the inputs, we cause the rule to re-run if
+ // the list of inputs changes because the command line or a dependency
+ // changes.
+
+ bp := `
+ rule_builder_test {
+ name: "hash0",
+ srcs: ["in1.txt", "in2.txt"],
+ }
+ rule_builder_test {
+ name: "hash0_sbox",
+ srcs: ["in1.txt", "in2.txt"],
+ sbox: true,
+ }
+ rule_builder_test {
+ name: "hash1",
+ srcs: ["in1.txt", "in2.txt", "in3.txt"],
+ }
+ rule_builder_test {
+ name: "hash1_sbox",
+ srcs: ["in1.txt", "in2.txt", "in3.txt"],
+ sbox: true,
+ }
+ `
+ testcases := []struct {
+ name string
+ expectedHash string
+ }{
+ {
+ name: "hash0",
+ // sha256 value obtained from: echo -en 'in1.txt\nin2.txt' | sha256sum
+ expectedHash: "18da75b9b1cc74b09e365b4ca2e321b5d618f438cc632b387ad9dc2ab4b20e9d",
+ },
+ {
+ name: "hash1",
+ // sha256 value obtained from: echo -en 'in1.txt\nin2.txt\nin3.txt' | sha256sum
+ expectedHash: "a38d432a4b19df93140e1f1fe26c97ff0387dae01fe506412b47208f0595fb45",
+ },
+ }
+
+ config := TestConfig(buildDir, nil, bp, nil)
+ ctx := NewTestContext(config)
+ ctx.RegisterModuleType("rule_builder_test", testRuleBuilderFactory)
+ ctx.Register()
+
+ _, errs := ctx.ParseFileList(".", []string{"Android.bp"})
+ FailIfErrored(t, errs)
+ _, errs = ctx.PrepareBuildActions(config)
+ FailIfErrored(t, errs)
+
+ for _, test := range testcases {
+ t.Run(test.name, func(t *testing.T) {
+ t.Run("sbox", func(t *testing.T) {
+ gen := ctx.ModuleForTests(test.name+"_sbox", "")
+ command := gen.Output(test.name + "_sbox").RuleParams.Command
+ if g, w := command, " --input-hash "+test.expectedHash; !strings.Contains(g, w) {
+ t.Errorf("Expected command line to end with %q, got %q", w, g)
+ }
+ })
+ t.Run("", func(t *testing.T) {
+ gen := ctx.ModuleForTests(test.name+"", "")
+ command := gen.Output(test.name).RuleParams.Command
+ if g, w := command, " # hash of input list: "+test.expectedHash; !strings.HasSuffix(g, w) {
+ t.Errorf("Expected command line to end with %q, got %q", w, g)
+ }
+ })
+ })
+ }
+}