<Hermetic> Replace Soong Python bootstrap process with embedded
launcher.
For Python2, we bundle embedded launcher as bootstrapper within every
.par file. This feature is only enabled for linux_x86_64 for now. We
provide a user flag: hermetic_enabled within bp file. By default, Pyhon2
still use classic bootstrapping way to construct .par file and relys on
host interpreter. Once embedded_launcher is enabled, launcher will be
used to bootstrap .par file and execute user program.
For Python3, the launcher will be ready soon, and for now it still relys
on classic bootstrapping.
Test: Real example is used to test.
Bug: b/63018041
Change-Id: I28deba413d8ad3af407595e46f77d663e79a3705
diff --git a/python/builder.go b/python/builder.go
index b823fcb..a459d3d 100644
--- a/python/builder.go
+++ b/python/builder.go
@@ -17,6 +17,7 @@
// This file contains Ninja build actions for building Python program.
import (
+ "fmt"
"strings"
"android/soong/android"
@@ -28,7 +29,7 @@
var (
pctx = android.NewPackageContext("android/soong/python")
- par = pctx.AndroidStaticRule("par",
+ host_par = pctx.AndroidStaticRule("host_par",
blueprint.RuleParams{
Command: `touch $initFile && ` +
`sed -e 's/%interpreter%/$interp/g' -e 's/%main%/$main/g' $template > $stub && ` +
@@ -37,6 +38,16 @@
CommandDeps: []string{"$parCmd", "$template"},
},
"initFile", "interp", "main", "template", "stub", "parCmd", "parFile", "parArgs")
+
+ embedded_par = pctx.AndroidStaticRule("embedded_par",
+ blueprint.RuleParams{
+ Command: `touch $initFile && ` +
+ `echo '$main' > $entry_point && ` +
+ `$parCmd -o $parFile $parArgs && cat $launcher | cat - $parFile > $out && ` +
+ `chmod +x $out && (rm -f $initFile; rm -f $entry_point; rm -f $parFile)`,
+ CommandDeps: []string{"$parCmd"},
+ },
+ "initFile", "main", "entry_point", "parCmd", "parFile", "parArgs", "launcher")
)
func init() {
@@ -58,10 +69,10 @@
}
func (p parSpec) soongParArgs() string {
- ret := "-P " + p.rootPrefix
+ ret := `-P ` + p.rootPrefix
for _, spec := range p.fileListSpecs {
- ret += " -C " + spec.relativeRoot + " -l " + spec.fileList.String()
+ ret += ` -C ` + spec.relativeRoot + ` -l ` + spec.fileList.String()
}
return ret
@@ -89,22 +100,17 @@
return fileList
}
-func registerBuildActionForParFile(ctx android.ModuleContext,
- interpreter, main, binName string, newPyPkgs []string, parSpecs []parSpec) android.Path {
+func registerBuildActionForParFile(ctx android.ModuleContext, embedded_launcher bool,
+ launcher_path android.Path, interpreter, main, binName string,
+ newPyPkgs []string, parSpecs []parSpec) android.Path {
- // intermediate output path for __init__.py
+ // .intermediate output path for __init__.py
initFile := android.PathForModuleOut(ctx, initFileName).String()
- // the path of stub_template_host.txt from source tree.
- template := android.PathForSource(ctx, stubTemplateHost)
-
- // intermediate output path for __main__.py
- stub := android.PathForModuleOut(ctx, mainFileName).String()
-
- // intermediate output path for par file.
+ // .intermediate output path for par file.
parFile := android.PathForModuleOut(ctx, binName+parFileExt)
- // intermediate output path for bin executable.
+ // .intermediate output path for bin executable.
binFile := android.PathForModuleOut(ctx, binName)
// implicit dependency for parFile build action.
@@ -116,32 +122,68 @@
}
parArgs := []string{}
- parArgs = append(parArgs, "-C "+strings.TrimSuffix(stub, mainFileName)+" -f "+stub)
- parArgs = append(parArgs, "-C "+strings.TrimSuffix(initFile, initFileName)+" -f "+initFile)
+ parArgs = append(parArgs, `-P "" `+`-C `+strings.TrimSuffix(initFile, initFileName)+` -f `+initFile)
for _, pkg := range newPyPkgs {
- parArgs = append(parArgs, "-P "+pkg+" -f "+initFile)
+ parArgs = append(parArgs, `-P `+pkg+` -f `+initFile)
}
for _, p := range parSpecs {
parArgs = append(parArgs, p.soongParArgs())
}
- ctx.ModuleBuild(pctx, android.ModuleBuildParams{
- Rule: par,
- Description: "python archive",
- Output: binFile,
- Implicits: implicits,
- Args: map[string]string{
- "initFile": initFile,
- // the "\" isn't being interpreted by regex parser, it's being
- // interpreted in the string literal.
- "interp": strings.Replace(interpreter, "/", `\/`, -1),
- "main": strings.Replace(main, "/", `\/`, -1),
- "template": template.String(),
- "stub": stub,
- "parFile": parFile.String(),
- "parArgs": strings.Join(parArgs, " "),
- },
- })
+ if !embedded_launcher {
+ // the path of stub_template_host.txt from source tree.
+ template := android.PathForSource(ctx, stubTemplateHost)
+
+ // intermediate output path for __main__.py
+ stub := android.PathForModuleOut(ctx, mainFileName).String()
+
+ // added stub file to the soong_zip args.
+ parArgs = append(parArgs, `-P "" `+`-C `+strings.TrimSuffix(stub, mainFileName)+` -f `+stub)
+
+ ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ Rule: host_par,
+ Description: "host python archive",
+ Output: binFile,
+ Implicits: implicits,
+ Args: map[string]string{
+ "initFile": initFile,
+ "interp": strings.Replace(interpreter, "/", `\/`, -1),
+ // we need remove "runfiles/" suffix since stub script starts
+ // searching for main file in each sub-dir of "runfiles" directory tree.
+ "main": strings.Replace(strings.TrimPrefix(main, runFiles+"/"),
+ "/", `\/`, -1),
+ "template": template.String(),
+ "stub": stub,
+ "parFile": parFile.String(),
+ "parArgs": strings.Join(parArgs, " "),
+ },
+ })
+ } else {
+ // added launcher_path to the implicits Ninja dependencies.
+ implicits = append(implicits, launcher_path)
+
+ // .intermediate output path for entry_point.txt
+ entryPoint := android.PathForModuleOut(ctx, entryPointFile).String()
+
+ // added entry_point file to the soong_zip args.
+ parArgs = append(parArgs, `-P "" `+`-C `+fmt.Sprintf(
+ "%q", strings.TrimSuffix(entryPoint, entryPointFile))+` -f `+entryPoint)
+
+ ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+ Rule: embedded_par,
+ Description: "embedded python archive",
+ Output: binFile,
+ Implicits: implicits,
+ Args: map[string]string{
+ "initFile": initFile,
+ "main": main,
+ "entry_point": entryPoint,
+ "parFile": parFile.String(),
+ "parArgs": strings.Join(parArgs, " "),
+ "launcher": launcher_path.String(),
+ },
+ })
+ }
return binFile
}