Pass rsp files into sbox and rewrapper
The current implementation causes inputs listed in an rsp file used with
sbox to be duplicated 3 times in the build.ninja file; once as a
dependency of the rule, once in the rspfile_content field of the rule
with the paths rewritten to be relative to the sandbox, and once in the
rule to write the sbox manifest. When RBE is enabled it also gets a
fourth copy in the list of files to be treated as inputs by rewrapper.
Reduce this to a single copy by using "$in" for the rspfile_content so
that the files only have to be listed in the input dependencies of the
rule, and then add support to sbox to rewrite the rsp file while copying
it into the sandbox, and pass it to rewrapper as well.
Test: m lint-check
Change-Id: I3f46f61119508d39a8bb231c99fc130153fb6f04
diff --git a/android/rule_builder.go b/android/rule_builder.go
index 4b356fa..4813d7a 100644
--- a/android/rule_builder.go
+++ b/android/rule_builder.go
@@ -28,6 +28,7 @@
"android/soong/cmd/sbox/sbox_proto"
"android/soong/remoteexec"
+ "android/soong/response"
"android/soong/shared"
)
@@ -459,26 +460,6 @@
Inputs(depFiles.Paths())
}
-// composeRspFileContent returns a string that will serve as the contents of the rsp file to pass
-// the listed input files to the command running in the sandbox.
-func (r *RuleBuilder) composeRspFileContent(rspFileInputs Paths) string {
- if r.sboxInputs {
- if len(rspFileInputs) > 0 {
- // When SandboxInputs is used the paths need to be rewritten to be relative to the sandbox
- // directory so that they are valid after sbox chdirs into the sandbox directory.
- return proptools.NinjaEscape(strings.Join(r.sboxPathsForInputsRel(rspFileInputs), " "))
- } else {
- // If the list of inputs is empty fall back to "$in" so that the rspfilecontent Ninja
- // variable is set to something non-empty, otherwise ninja will complain. The inputs
- // will be empty (all the non-rspfile inputs are implicits), so $in will evaluate to
- // an empty string.
- return "$in"
- }
- } else {
- return "$in"
- }
-}
-
// Build adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for
// Outputs.
func (r *RuleBuilder) Build(name string, desc string) {
@@ -577,9 +558,19 @@
// If using an rsp file copy it into the sbox directory.
if rspFilePath != nil {
- command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
- From: proto.String(rspFilePath.String()),
- To: proto.String(r.sboxPathForInputRel(rspFilePath)),
+ command.RspFiles = append(command.RspFiles, &sbox_proto.RspFile{
+ File: proto.String(rspFilePath.String()),
+ // These have to match the logic in sboxPathForInputRel
+ PathMappings: []*sbox_proto.PathMapping{
+ {
+ From: proto.String(r.outDir.String()),
+ To: proto.String(sboxOutSubDir),
+ },
+ {
+ From: proto.String(PathForOutput(r.ctx).String()),
+ To: proto.String(sboxOutSubDir),
+ },
+ },
})
}
@@ -641,20 +632,30 @@
inputs = append(inputs, sboxCmd.inputs...)
if r.rbeParams != nil {
- var remoteInputs []string
- remoteInputs = append(remoteInputs, inputs.Strings()...)
- remoteInputs = append(remoteInputs, tools.Strings()...)
- remoteInputs = append(remoteInputs, rspFileInputs.Strings()...)
+ // RBE needs a list of input files to copy to the remote builder. For inputs already
+ // listed in an rsp file, pass the rsp file directly to rewrapper. For the rest,
+ // create a new rsp file to pass to rewrapper.
+ var remoteRspFiles Paths
+ var remoteInputs Paths
+
+ remoteInputs = append(remoteInputs, inputs...)
+ remoteInputs = append(remoteInputs, tools...)
+
if rspFilePath != nil {
- remoteInputs = append(remoteInputs, rspFilePath.String())
+ remoteInputs = append(remoteInputs, rspFilePath)
+ remoteRspFiles = append(remoteRspFiles, rspFilePath)
}
- inputsListFile := r.sboxManifestPath.ReplaceExtension(r.ctx, "rbe_inputs.list")
- inputsListContents := rspFileForInputs(remoteInputs)
- WriteFileRule(r.ctx, inputsListFile, inputsListContents)
- inputs = append(inputs, inputsListFile)
+
+ if len(remoteInputs) > 0 {
+ inputsListFile := r.sboxManifestPath.ReplaceExtension(r.ctx, "rbe_inputs.list")
+ writeRspFileRule(r.ctx, inputsListFile, remoteInputs)
+ remoteRspFiles = append(remoteRspFiles, inputsListFile)
+ // Add the new rsp file as an extra input to the rule.
+ inputs = append(inputs, inputsListFile)
+ }
r.rbeParams.OutputFiles = outputs.Strings()
- r.rbeParams.RSPFiles = []string{inputsListFile.String()}
+ r.rbeParams.RSPFiles = remoteRspFiles.Strings()
rewrapperCommand := r.rbeParams.NoVarTemplate(r.ctx.Config().RBEWrapper())
commandString = rewrapperCommand + " bash -c '" + strings.ReplaceAll(commandString, `'`, `'\''`) + "'"
}
@@ -674,7 +675,10 @@
var rspFile, rspFileContent string
if rspFilePath != nil {
rspFile = rspFilePath.String()
- rspFileContent = r.composeRspFileContent(rspFileInputs)
+ // Use "$in" for rspFileContent to avoid duplicating the list of files in the dependency
+ // list and in the contents of the rsp file. Inputs to the rule that are not in the
+ // rsp file will be listed in Implicits instead of Inputs so they don't show up in "$in".
+ rspFileContent = "$in"
}
var pool blueprint.Pool
@@ -1264,13 +1268,12 @@
}
func (builderContextForTests) Build(PackageContext, BuildParams) {}
-func rspFileForInputs(paths []string) string {
- s := strings.Builder{}
- for i, path := range paths {
- if i != 0 {
- s.WriteByte(' ')
- }
- s.WriteString(proptools.ShellEscape(path))
+func writeRspFileRule(ctx BuilderContext, rspFile WritablePath, paths Paths) {
+ buf := &strings.Builder{}
+ err := response.WriteRspFile(buf, paths.Strings())
+ if err != nil {
+ // There should never be I/O errors writing to a bytes.Buffer.
+ panic(err)
}
- return s.String()
+ WriteFileRule(ctx, rspFile, buf.String())
}