Add support for remote-execution / caching of jar/zip actions
Test: Ran a build with `RBE_JAR="true" RBE_JAR_EXEC_STRATEGY="remote"
RBE_ZIP="true" RBE_ZIP_EXEC_STRATEGY="remote" ... use_rbe m` and that
succeeded.
Bug: b/156765207
Change-Id: Ia17b532fbb282be414adf879914870082dd0841b
Merged-In: Ia17b532fbb282be414adf879914870082dd0841b
diff --git a/java/app_builder.go b/java/app_builder.go
index acc02f2..97ec269 100644
--- a/java/app_builder.go
+++ b/java/app_builder.go
@@ -237,14 +237,20 @@
"-f", j.path.String())
}
+ rule := zip
+ args := map[string]string{
+ "jarArgs": strings.Join(proptools.NinjaAndShellEscapeList(jarArgs), " "),
+ }
+ if ctx.Config().IsEnvTrue("RBE_ZIP") {
+ rule = zipRE
+ args["implicits"] = strings.Join(deps.Strings(), ",")
+ }
ctx.Build(pctx, android.BuildParams{
- Rule: zip,
+ Rule: rule,
Description: "zip jni libs",
Output: outputFile,
Implicits: deps,
- Args: map[string]string{
- "jarArgs": strings.Join(proptools.NinjaAndShellEscapeList(jarArgs), " "),
- },
+ Args: args,
})
}
diff --git a/java/builder.go b/java/builder.go
index b6cb2ae..a27e5c3 100644
--- a/java/builder.go
+++ b/java/builder.go
@@ -40,17 +40,17 @@
// (if the rule produces .class files) or a .srcjar file (if the rule produces .java files).
// .srcjar files are unzipped into a temporary directory when compiled with javac.
// TODO(b/143658984): goma can't handle the --system argument to javac.
- javac, javacRE = remoteexec.StaticRules(pctx, "javac",
+ javac, javacRE = remoteexec.MultiCommandStaticRules(pctx, "javac",
blueprint.RuleParams{
Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` +
`${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` +
`(if [ -s $srcJarDir/list ] || [ -s $out.rsp ] ; then ` +
- `${config.SoongJavacWrapper} $reTemplate${config.JavacCmd} ` +
+ `${config.SoongJavacWrapper} $javaTemplate${config.JavacCmd} ` +
`${config.JavacHeapFlags} ${config.JavacVmFlags} ${config.CommonJdkFlags} ` +
`$processorpath $processor $javacFlags $bootClasspath $classpath ` +
`-source $javaVersion -target $javaVersion ` +
`-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list ; fi ) && ` +
- `${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir && ` +
+ `$zipTemplate${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir && ` +
`rm -rf "$srcJarDir"`,
CommandDeps: []string{
"${config.JavacCmd}",
@@ -60,10 +60,19 @@
CommandOrderOnly: []string{"${config.SoongJavacWrapper}"},
Rspfile: "$out.rsp",
RspfileContent: "$in",
- }, &remoteexec.REParams{
- Labels: map[string]string{"type": "compile", "lang": "java", "compiler": "javac"},
- ExecStrategy: "${config.REJavacExecStrategy}",
- Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
+ }, map[string]*remoteexec.REParams{
+ "$javaTemplate": &remoteexec.REParams{
+ Labels: map[string]string{"type": "compile", "lang": "java", "compiler": "javac"},
+ ExecStrategy: "${config.REJavacExecStrategy}",
+ Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
+ },
+ "$zipTemplate": &remoteexec.REParams{
+ Labels: map[string]string{"type": "tool", "name": "soong_zip"},
+ Inputs: []string{"${config.SoongZipCmd}", "$outDir"},
+ OutputFiles: []string{"$out"},
+ ExecStrategy: "${config.REJavacExecStrategy}",
+ Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
+ },
}, []string{"javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir",
"outDir", "annoDir", "javaVersion"}, nil)
@@ -144,23 +153,35 @@
Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
}, []string{"javacFlags", "bootClasspath", "classpath", "srcJars", "outDir", "javaVersion"}, []string{"implicits"})
- jar = pctx.AndroidStaticRule("jar",
+ jar, jarRE = remoteexec.StaticRules(pctx, "jar",
blueprint.RuleParams{
- Command: `${config.SoongZipCmd} -jar -o $out @$out.rsp`,
+ Command: `$reTemplate${config.SoongZipCmd} -jar -o $out @$out.rsp`,
CommandDeps: []string{"${config.SoongZipCmd}"},
Rspfile: "$out.rsp",
RspfileContent: "$jarArgs",
},
- "jarArgs")
+ &remoteexec.REParams{
+ ExecStrategy: "${config.REJarExecStrategy}",
+ Inputs: []string{"${config.SoongZipCmd}", "${out}.rsp"},
+ RSPFile: "${out}.rsp",
+ OutputFiles: []string{"$out"},
+ Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
+ }, []string{"jarArgs"}, nil)
- zip = pctx.AndroidStaticRule("zip",
+ zip, zipRE = remoteexec.StaticRules(pctx, "zip",
blueprint.RuleParams{
Command: `${config.SoongZipCmd} -o $out @$out.rsp`,
CommandDeps: []string{"${config.SoongZipCmd}"},
Rspfile: "$out.rsp",
RspfileContent: "$jarArgs",
},
- "jarArgs")
+ &remoteexec.REParams{
+ ExecStrategy: "${config.REZipExecStrategy}",
+ Inputs: []string{"${config.SoongZipCmd}", "${out}.rsp", "$implicits"},
+ RSPFile: "${out}.rsp",
+ OutputFiles: []string{"$out"},
+ Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
+ }, []string{"jarArgs"}, []string{"implicits"})
combineJar = pctx.AndroidStaticRule("combineJar",
blueprint.RuleParams{
@@ -457,8 +478,12 @@
func TransformResourcesToJar(ctx android.ModuleContext, outputFile android.WritablePath,
jarArgs []string, deps android.Paths) {
+ rule := jar
+ if ctx.Config().IsEnvTrue("RBE_JAR") {
+ rule = jarRE
+ }
ctx.Build(pctx, android.BuildParams{
- Rule: jar,
+ Rule: rule,
Description: "jar",
Output: outputFile,
Implicits: deps,
diff --git a/java/config/config.go b/java/config/config.go
index d2550b1..337355d 100644
--- a/java/config/config.go
+++ b/java/config/config.go
@@ -152,6 +152,8 @@
pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
pctx.VariableFunc("RETurbineExecStrategy", remoteexec.EnvOverrideFunc("RBE_TURBINE_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
pctx.VariableFunc("RESignApkExecStrategy", remoteexec.EnvOverrideFunc("RBE_SIGNAPK_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
+ pctx.VariableFunc("REJarExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAR_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
+ pctx.VariableFunc("REZipExecStrategy", remoteexec.EnvOverrideFunc("RBE_ZIP_EXEC_STRATEGY", remoteexec.LocalExecStrategy))
pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar")
diff --git a/java/dex.go b/java/dex.go
index e2a8e7e..9e61e95 100644
--- a/java/dex.go
+++ b/java/dex.go
@@ -24,48 +24,66 @@
"android/soong/remoteexec"
)
-var d8, d8RE = remoteexec.StaticRules(pctx, "d8",
+var d8, d8RE = remoteexec.MultiCommandStaticRules(pctx, "d8",
blueprint.RuleParams{
Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
- `$reTemplate${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` +
- `${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
+ `$d8Template${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` +
+ `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
`${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`,
CommandDeps: []string{
"${config.D8Cmd}",
"${config.SoongZipCmd}",
"${config.MergeZipsCmd}",
},
- }, &remoteexec.REParams{
- Labels: map[string]string{"type": "compile", "compiler": "d8"},
- Inputs: []string{"${config.D8Jar}"},
- ExecStrategy: "${config.RED8ExecStrategy}",
- ToolchainInputs: []string{"${config.JavaCmd}"},
- Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
+ }, map[string]*remoteexec.REParams{
+ "$d8Template": &remoteexec.REParams{
+ Labels: map[string]string{"type": "compile", "compiler": "d8"},
+ Inputs: []string{"${config.D8Jar}"},
+ ExecStrategy: "${config.RED8ExecStrategy}",
+ ToolchainInputs: []string{"${config.JavaCmd}"},
+ Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
+ },
+ "$zipTemplate": &remoteexec.REParams{
+ Labels: map[string]string{"type": "tool", "name": "soong_zip"},
+ Inputs: []string{"${config.SoongZipCmd}", "$outDir"},
+ OutputFiles: []string{"$outDir/classes.dex.jar"},
+ ExecStrategy: "${config.RED8ExecStrategy}",
+ Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
+ },
}, []string{"outDir", "d8Flags", "zipFlags"}, nil)
-var r8, r8RE = remoteexec.StaticRules(pctx, "r8",
+var r8, r8RE = remoteexec.MultiCommandStaticRules(pctx, "r8",
blueprint.RuleParams{
Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
`rm -f "$outDict" && ` +
- `$reTemplate${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` +
+ `$r8Template${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` +
`--force-proguard-compatibility ` +
`--no-data-resources ` +
`-printmapping $outDict ` +
`$r8Flags && ` +
`touch "$outDict" && ` +
- `${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
+ `$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +
`${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`,
CommandDeps: []string{
"${config.R8Cmd}",
"${config.SoongZipCmd}",
"${config.MergeZipsCmd}",
},
- }, &remoteexec.REParams{
- Labels: map[string]string{"type": "compile", "compiler": "r8"},
- Inputs: []string{"$implicits", "${config.R8Jar}"},
- ExecStrategy: "${config.RER8ExecStrategy}",
- ToolchainInputs: []string{"${config.JavaCmd}"},
- Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
+ }, map[string]*remoteexec.REParams{
+ "$r8Template": &remoteexec.REParams{
+ Labels: map[string]string{"type": "compile", "compiler": "r8"},
+ Inputs: []string{"$implicits", "${config.R8Jar}"},
+ ExecStrategy: "${config.RER8ExecStrategy}",
+ ToolchainInputs: []string{"${config.JavaCmd}"},
+ Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
+ },
+ "$zipTemplate": &remoteexec.REParams{
+ Labels: map[string]string{"type": "tool", "name": "soong_zip"},
+ Inputs: []string{"${config.SoongZipCmd}", "$outDir"},
+ OutputFiles: []string{"$outDir/classes.dex.jar"},
+ ExecStrategy: "${config.RER8ExecStrategy}",
+ Platform: map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},
+ },
}, []string{"outDir", "outDict", "r8Flags", "zipFlags"}, []string{"implicits"})
func (j *Module) dexCommonFlags(ctx android.ModuleContext) []string {
diff --git a/java/java.go b/java/java.go
index e01d109..2623d2c 100644
--- a/java/java.go
+++ b/java/java.go
@@ -1462,13 +1462,19 @@
serviceFile := file.String()
zipargs = append(zipargs, "-C", filepath.Dir(serviceFile), "-f", serviceFile)
}
+ rule := zip
+ args := map[string]string{
+ "jarArgs": "-P META-INF/services/ " + strings.Join(proptools.NinjaAndShellEscapeList(zipargs), " "),
+ }
+ if ctx.Config().IsEnvTrue("RBE_ZIP") {
+ rule = zipRE
+ args["implicits"] = strings.Join(services.Strings(), ",")
+ }
ctx.Build(pctx, android.BuildParams{
- Rule: zip,
+ Rule: rule,
Output: servicesJar,
Implicits: services,
- Args: map[string]string{
- "jarArgs": "-P META-INF/services/ " + strings.Join(proptools.NinjaAndShellEscapeList(zipargs), " "),
- },
+ Args: args,
})
jars = append(jars, servicesJar)
}