Use rsp files to pass turbine inputs to RBE
Turbine rules that run in RBE fail when they have long lists of
flags. They work locally because the list of flags is placed in
an RSP file, but the list of inputs still appears on the command
line when RBE is enabled to pass them to rewrapper.
When the list of srcjars or classpath arguments are too long,
move the list of files into an rsp file, and pass the rsp file to
RBE instead of the list of files.
Bug; 308016794
Test: builds
Flag: EXEMPT refactor
Change-Id: I5ee610a91302ce94ec77b6f13b28a29bc63fd962
diff --git a/java/builder.go b/java/builder.go
index 5d84d0b..5915235 100644
--- a/java/builder.go
+++ b/java/builder.go
@@ -156,7 +156,7 @@
turbine, turbineRE = pctx.RemoteStaticRules("turbine",
blueprint.RuleParams{
Command: `$reTemplate${config.JavaCmd} ${config.JavaVmFlags} -jar ${config.TurbineJar} $outputFlags ` +
- `--sources @$out.rsp --source_jars $srcJars ` +
+ `--sources @$out.rsp ` +
`--javacopts ${config.CommonJdkFlags} ` +
`$javacFlags -source $javaVersion -target $javaVersion -- $turbineFlags && ` +
`(for o in $outputs; do if cmp -s $${o}.tmp $${o} ; then rm $${o}.tmp ; else mv $${o}.tmp $${o} ; fi; done )`,
@@ -170,13 +170,13 @@
},
&remoteexec.REParams{Labels: map[string]string{"type": "tool", "name": "turbine"},
ExecStrategy: "${config.RETurbineExecStrategy}",
- Inputs: []string{"${config.TurbineJar}", "${out}.rsp", "$implicits"},
- RSPFiles: []string{"${out}.rsp"},
+ Inputs: []string{"${config.TurbineJar}", "${out}.rsp", "$rbeInputs"},
+ RSPFiles: []string{"$out.rsp", "$rspFiles"},
OutputFiles: []string{"$rbeOutputs"},
ToolchainInputs: []string{"${config.JavaCmd}"},
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
},
- []string{"javacFlags", "turbineFlags", "outputFlags", "javaVersion", "outputs", "rbeOutputs", "srcJars"}, []string{"implicits"})
+ []string{"javacFlags", "turbineFlags", "outputFlags", "javaVersion", "outputs", "rbeOutputs"}, []string{"rbeInputs", "rspFiles"})
jar, jarRE = pctx.RemoteStaticRules("jar",
blueprint.RuleParams{
@@ -428,53 +428,72 @@
})
}
-func turbineFlags(ctx android.ModuleContext, flags javaBuilderFlags, dir string) (string, android.Paths) {
- var deps android.Paths
+func turbineFlags(ctx android.ModuleContext, flags javaBuilderFlags, dir string, srcJars android.Paths) (string, android.Paths, android.Paths, android.Paths) {
+ var implicits android.Paths
+ var rbeInputs android.Paths
+ var rspFiles android.Paths
classpath := flags.classpath
- var bootClasspath string
+ srcJarArgs := strings.Join(srcJars.Strings(), " ")
+ implicits = append(implicits, srcJars...)
+ const srcJarArgsLimit = 32 * 1024
+ if len(srcJarArgs) > srcJarArgsLimit {
+ srcJarRspFile := android.PathForModuleOut(ctx, "turbine", "srcjars.rsp")
+ android.WriteFileRule(ctx, srcJarRspFile, srcJarArgs)
+ srcJarArgs = "@" + srcJarRspFile.String()
+ implicits = append(implicits, srcJarRspFile)
+ rbeInputs = append(rbeInputs, srcJarRspFile)
+ } else {
+ rbeInputs = append(rbeInputs, srcJars...)
+ }
+
+ var bootClasspathFlags string
if flags.javaVersion.usesJavaModules() {
var systemModuleDeps android.Paths
- bootClasspath, systemModuleDeps = flags.systemModules.FormTurbineSystemModulesPath(ctx.Device())
- deps = append(deps, systemModuleDeps...)
+ bootClasspathFlags, systemModuleDeps = flags.systemModules.FormTurbineSystemModulesPath(ctx.Device())
+ implicits = append(implicits, systemModuleDeps...)
+ rbeInputs = append(rbeInputs, systemModuleDeps...)
classpath = append(flags.java9Classpath, classpath...)
} else {
- deps = append(deps, flags.bootClasspath...)
+ implicits = append(implicits, flags.bootClasspath...)
+ rbeInputs = append(rbeInputs, flags.bootClasspath...)
if len(flags.bootClasspath) == 0 && ctx.Device() {
// explicitly specify -bootclasspath "" if the bootclasspath is empty to
// ensure turbine does not fall back to the default bootclasspath.
- bootClasspath = `--bootclasspath ""`
+ bootClasspathFlags = `--bootclasspath ""`
} else {
- bootClasspath = flags.bootClasspath.FormTurbineClassPath("--bootclasspath ")
+ bootClasspathFlags = flags.bootClasspath.FormTurbineClassPath("--bootclasspath ")
}
}
- deps = append(deps, classpath...)
- turbineFlags := bootClasspath + " " + classpath.FormTurbineClassPath("--classpath ")
-
- const flagsLimit = 32 * 1024
- if len(turbineFlags) > flagsLimit {
- flagsRspFile := android.PathForModuleOut(ctx, dir, "turbine-flags.rsp")
- android.WriteFileRule(ctx, flagsRspFile, turbineFlags)
- turbineFlags = "@" + flagsRspFile.String()
- deps = append(deps, flagsRspFile)
+ classpathFlags := classpath.FormTurbineClassPath("")
+ implicits = append(implicits, classpath...)
+ const classpathLimit = 32 * 1024
+ if len(classpathFlags) > classpathLimit {
+ classpathRspFile := android.PathForModuleOut(ctx, dir, "classpath.rsp")
+ android.WriteFileRule(ctx, classpathRspFile, classpathFlags)
+ classpathFlags = "@" + classpathRspFile.String()
+ implicits = append(implicits, classpathRspFile)
+ rspFiles = append(rspFiles, classpathRspFile)
+ rbeInputs = append(rbeInputs, classpathRspFile)
+ } else {
+ rbeInputs = append(rbeInputs, classpath...)
}
- return turbineFlags, deps
+ turbineFlags := "--source_jars " + srcJarArgs + " " + bootClasspathFlags + " --classpath " + classpathFlags
+
+ return turbineFlags, implicits, rbeInputs, rspFiles
}
func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android.WritablePath,
srcFiles, srcJars android.Paths, flags javaBuilderFlags) {
- turbineFlags, deps := turbineFlags(ctx, flags, "turbine")
-
- deps = append(deps, srcJars...)
+ turbineFlags, implicits, rbeInputs, rspFiles := turbineFlags(ctx, flags, "turbine", srcJars)
rule := turbine
args := map[string]string{
"javacFlags": flags.javacFlags,
- "srcJars": strings.Join(srcJars.Strings(), " "),
"javaVersion": flags.javaVersion.String(),
"turbineFlags": turbineFlags,
"outputFlags": "--output " + outputFile.String() + ".tmp",
@@ -482,15 +501,16 @@
}
if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_TURBINE") {
rule = turbineRE
- args["implicits"] = strings.Join(deps.Strings(), ",")
+ args["rbeInputs"] = strings.Join(rbeInputs.Strings(), ",")
args["rbeOutputs"] = outputFile.String() + ".tmp"
+ args["rspFiles"] = strings.Join(rspFiles.Strings(), ",")
}
ctx.Build(pctx, android.BuildParams{
Rule: rule,
Description: "turbine",
Output: outputFile,
Inputs: srcFiles,
- Implicits: deps,
+ Implicits: implicits,
Args: args,
})
}
@@ -499,11 +519,10 @@
func TurbineApt(ctx android.ModuleContext, outputSrcJar, outputResJar android.WritablePath,
srcFiles, srcJars android.Paths, flags javaBuilderFlags) {
- turbineFlags, deps := turbineFlags(ctx, flags, "kapt")
+ turbineFlags, implicits, rbeInputs, rspFiles := turbineFlags(ctx, flags, "turbine-apt", srcJars)
- deps = append(deps, srcJars...)
-
- deps = append(deps, flags.processorPath...)
+ implicits = append(implicits, flags.processorPath...)
+ rbeInputs = append(rbeInputs, flags.processorPath...)
turbineFlags += " " + flags.processorPath.FormTurbineClassPath("--processorpath ")
turbineFlags += " --processors " + strings.Join(flags.processors, " ")
@@ -514,7 +533,6 @@
rule := turbine
args := map[string]string{
"javacFlags": flags.javacFlags,
- "srcJars": strings.Join(srcJars.Strings(), " "),
"javaVersion": flags.javaVersion.String(),
"turbineFlags": turbineFlags,
"outputFlags": outputFlags,
@@ -522,8 +540,9 @@
}
if ctx.Config().UseRBE() && ctx.Config().IsEnvTrue("RBE_TURBINE") {
rule = turbineRE
- args["implicits"] = strings.Join(deps.Strings(), ",")
+ args["rbeInputs"] = strings.Join(rbeInputs.Strings(), ",")
args["rbeOutputs"] = outputSrcJar.String() + ".tmp," + outputResJar.String() + ".tmp"
+ args["rspFiles"] = strings.Join(rspFiles.Strings(), ",")
}
ctx.Build(pctx, android.BuildParams{
Rule: rule,
@@ -531,7 +550,7 @@
Output: outputs[0],
ImplicitOutputs: outputs[1:],
Inputs: srcFiles,
- Implicits: deps,
+ Implicits: implicits,
Args: args,
})
}