Merge changes from topic "dexpreopt_bootjars"
* changes:
Add more paths helper methods
Add java/testing.go for sysprop_test.go
Move dexpreopting of boot jars into Soong
diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go
index c21da44..68fe259 100644
--- a/cmd/merge_zips/merge_zips.go
+++ b/cmd/merge_zips/merge_zips.go
@@ -66,7 +66,6 @@
stripDirEntries = flag.Bool("D", false, "strip directory entries from the output zip file")
manifest = flag.String("m", "", "manifest file to insert in jar")
pyMain = flag.String("pm", "", "__main__.py file to insert in par")
- entrypoint = flag.String("e", "", "par entrypoint file to insert in par")
prefix = flag.String("prefix", "", "A file to prefix to the zip file")
ignoreDuplicates = flag.Bool("ignore-duplicates", false, "take each entry from the first zip it exists in and don't warn")
)
@@ -79,7 +78,7 @@
func main() {
flag.Usage = func() {
- fmt.Fprintln(os.Stderr, "usage: merge_zips [-jpsD] [-m manifest] [--prefix script] [-e entrypoint] [-pm __main__.py] output [inputs...]")
+ fmt.Fprintln(os.Stderr, "usage: merge_zips [-jpsD] [-m manifest] [--prefix script] [-pm __main__.py] output [inputs...]")
flag.PrintDefaults()
}
@@ -139,16 +138,12 @@
log.Fatal(errors.New("must specify -j when specifying a manifest via -m"))
}
- if *entrypoint != "" && !*emulatePar {
- log.Fatal(errors.New("must specify -p when specifying a entrypoint via -e"))
- }
-
if *pyMain != "" && !*emulatePar {
log.Fatal(errors.New("must specify -p when specifying a Python __main__.py via -pm"))
}
// do merge
- err = mergeZips(readers, writer, *manifest, *entrypoint, *pyMain, *sortEntries, *emulateJar, *emulatePar,
+ err = mergeZips(readers, writer, *manifest, *pyMain, *sortEntries, *emulateJar, *emulatePar,
*stripDirEntries, *ignoreDuplicates, []string(stripFiles), []string(stripDirs), map[string]bool(zipsToNotStrip))
if err != nil {
log.Fatal(err)
@@ -249,7 +244,7 @@
source zipSource
}
-func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest, entrypoint, pyMain string,
+func mergeZips(readers []namedZipReader, writer *zip.Writer, manifest, pyMain string,
sortEntries, emulateJar, emulatePar, stripDirEntries, ignoreDuplicates bool,
stripFiles, stripDirs []string, zipsToNotStrip map[string]bool) error {
@@ -289,22 +284,6 @@
addMapping(jar.ManifestFile, fileSource)
}
- if entrypoint != "" {
- buf, err := ioutil.ReadFile(entrypoint)
- if err != nil {
- return err
- }
- fh := &zip.FileHeader{
- Name: "entry_point.txt",
- Method: zip.Store,
- UncompressedSize64: uint64(len(buf)),
- }
- fh.SetMode(0700)
- fh.SetModTime(jar.DefaultTime)
- fileSource := bufferEntry{fh, buf}
- addMapping("entry_point.txt", fileSource)
- }
-
if pyMain != "" {
buf, err := ioutil.ReadFile(pyMain)
if err != nil {
diff --git a/cmd/merge_zips/merge_zips_test.go b/cmd/merge_zips/merge_zips_test.go
index 19fa5ed..dbde270 100644
--- a/cmd/merge_zips/merge_zips_test.go
+++ b/cmd/merge_zips/merge_zips_test.go
@@ -221,7 +221,7 @@
out := &bytes.Buffer{}
writer := zip.NewWriter(out)
- err := mergeZips(readers, writer, "", "", "",
+ err := mergeZips(readers, writer, "", "",
test.sort, test.jar, false, test.stripDirEntries, test.ignoreDuplicates,
test.stripFiles, test.stripDirs, test.zipsToNotStrip)
diff --git a/java/config/config.go b/java/config/config.go
index 75be9e2..2602e6b 100644
--- a/java/config/config.go
+++ b/java/config/config.go
@@ -33,7 +33,12 @@
DefaultLambdaStubsLibrary = "core-lambda-stubs"
SdkLambdaStubsPath = "prebuilts/sdk/tools/core-lambda-stubs.jar"
- // A list of the jars that provide information about usages of the hidden API.
+ // A list of the non-boot jars that provide hidden APIs, i.e. libraries.
+ HiddenAPIProvidingNonBootJars = []string{
+ "android.test.base",
+ }
+
+ // A list of the non-boot jars that provide information about usages of the hidden API.
HiddenAPIExtraAppUsageJars = []string{
// The core-oj-hiddenapi provides information for the core-oj jar.
"core-oj-hiddenapi",
diff --git a/java/hiddenapi.go b/java/hiddenapi.go
index f199051..01e2c5e 100644
--- a/java/hiddenapi.go
+++ b/java/hiddenapi.go
@@ -59,7 +59,14 @@
if !ctx.Config().IsEnvTrue("UNSAFE_DISABLE_HIDDENAPI_FLAGS") {
isBootJar := inList(ctx.ModuleName(), ctx.Config().BootJars())
- if isBootJar || inList(ctx.ModuleName(), config.HiddenAPIExtraAppUsageJars) {
+ // Check to see if this module provides part of the hiddenapi, i.e. is a boot jar or a white listed
+ // library.
+ isProvidingJar := isBootJar || inList(ctx.ModuleName(), config.HiddenAPIProvidingNonBootJars)
+
+ // If this module provides part of the hiddenapi or is a special module that simply provides information
+ // about the hiddenapi then extract information about the hiddenapi from the UnsupportedAppUsage
+ // annotations compiled into the classes.jar.
+ if isProvidingJar || inList(ctx.ModuleName(), config.HiddenAPIExtraAppUsageJars) {
// Derive the greylist from classes jar.
flagsCSV := android.PathForModuleOut(ctx, "hiddenapi", "flags.csv")
metadataCSV := android.PathForModuleOut(ctx, "hiddenapi", "metadata.csv")
@@ -67,7 +74,10 @@
h.flagsCSVPath = flagsCSV
h.metadataCSVPath = metadataCSV
}
- if isBootJar {
+
+ // If this module provides part of the hiddenapi then encode the information about the hiddenapi into
+ // the dex file created for this module.
+ if isProvidingJar {
hiddenAPIJar := android.PathForModuleOut(ctx, "hiddenapi", ctx.ModuleName()+".jar")
h.bootDexJarPath = dexJar
hiddenAPIEncodeDex(ctx, hiddenAPIJar, dexJar, uncompressDex)
diff --git a/java/prebuilt_apis.go b/java/prebuilt_apis.go
index 49cc931..02b9b45 100644
--- a/java/prebuilt_apis.go
+++ b/java/prebuilt_apis.go
@@ -103,19 +103,25 @@
mctx.CreateModule(android.ModuleFactoryAdaptor(android.FileGroupFactory), &filegroupProps)
}
-func prebuiltSdkStubs(mctx android.TopDownMutatorContext) {
+func getPrebuiltFiles(mctx android.TopDownMutatorContext, name string) []string {
mydir := mctx.ModuleDir() + "/"
- // <apiver>/<scope>/<module>.jar
var files []string
for _, apiver := range mctx.Module().(*prebuiltApis).properties.Api_dirs {
for _, scope := range []string{"public", "system", "test", "core"} {
- vfiles, err := mctx.GlobWithDeps(mydir+apiver+"/"+scope+"*/*.jar", nil)
+ vfiles, err := mctx.GlobWithDeps(mydir+apiver+"/"+scope+"/"+name, nil)
if err != nil {
- mctx.ModuleErrorf("failed to glob jar files under %q: %s", mydir+apiver+"/"+scope, err)
+ mctx.ModuleErrorf("failed to glob %s files under %q: %s", name, mydir+apiver+"/"+scope, err)
}
files = append(files, vfiles...)
}
}
+ return files
+}
+
+func prebuiltSdkStubs(mctx android.TopDownMutatorContext) {
+ mydir := mctx.ModuleDir() + "/"
+ // <apiver>/<scope>/<module>.jar
+ files := getPrebuiltFiles(mctx, "*.jar")
for _, f := range files {
// create a Import module for each jar file
@@ -128,10 +134,8 @@
func prebuiltApiFiles(mctx android.TopDownMutatorContext) {
mydir := mctx.ModuleDir() + "/"
// <apiver>/<scope>/api/<module>.txt
- files, err := mctx.GlobWithDeps(mydir+"*/*/api/*.txt", nil)
- if err != nil {
- mctx.ModuleErrorf("failed to glob api txt files under %q: %s", mydir, err)
- }
+ files := getPrebuiltFiles(mctx, "api/*.txt")
+
if len(files) == 0 {
mctx.ModuleErrorf("no api file found under %q", mydir)
}
@@ -161,6 +165,7 @@
strings.Compare(apiver, info.apiver) > 0) {
info.apiver = apiver
info.path = localPath
+ m[key] = info
}
}
// create filegroups for the latest version of (<module>, <scope>) pairs
diff --git a/python/binary.go b/python/binary.go
index bf9acb4..140f07a 100644
--- a/python/binary.go
+++ b/python/binary.go
@@ -42,6 +42,11 @@
// list of compatibility suites (for example "cts", "vts") that the module should be
// installed into.
Test_suites []string `android:"arch_variant"`
+
+ // whether to use `main` when starting the executable. The default is true, when set to
+ // false it will act much like the normal `python` executable, but with the sources and
+ // libraries automatically included in the PYTHONPATH.
+ Autorun *bool `android:"arch_variant"`
}
type binaryDecorator struct {
@@ -74,6 +79,10 @@
return module.Init()
}
+func (binary *binaryDecorator) autorun() bool {
+ return BoolDefault(binary.binaryProperties.Autorun, true)
+}
+
func (binary *binaryDecorator) bootstrapperProps() []interface{} {
return []interface{}{&binary.binaryProperties}
}
@@ -82,7 +91,10 @@
embeddedLauncher bool, srcsPathMappings []pathMapping, srcsZip android.Path,
depsSrcsZips android.Paths) android.OptionalPath {
- main := binary.getPyMainFile(ctx, srcsPathMappings)
+ main := ""
+ if binary.autorun() {
+ main = binary.getPyMainFile(ctx, srcsPathMappings)
+ }
var launcherPath android.OptionalPath
if embeddedLauncher {
diff --git a/python/builder.go b/python/builder.go
index cbbe56e..e3b490c 100644
--- a/python/builder.go
+++ b/python/builder.go
@@ -54,15 +54,21 @@
embeddedPar = pctx.AndroidStaticRule("embeddedPar",
blueprint.RuleParams{
- // `echo -n` to trim the newline, since the python code just wants the name.
- // /bin/sh (used by ninja) on Mac turns off posix mode, and stops supporting -n.
- // Explicitly use bash instead.
- Command: `/bin/bash -c "echo -n '$main' > $entryPoint" &&` +
- `$mergeParCmd -p --prefix $launcher -e $entryPoint $out $srcsZips && ` +
- `chmod +x $out && (rm -f $entryPoint)`,
+ Command: `rm -f $out.main && ` +
+ `sed 's/ENTRY_POINT/$main/' build/soong/python/scripts/main.py >$out.main &&` +
+ `$mergeParCmd -p -pm $out.main --prefix $launcher $out $srcsZips && ` +
+ `chmod +x $out && rm -rf $out.main`,
+ CommandDeps: []string{"$mergeParCmd", "$parCmd", "build/soong/python/scripts/main.py"},
+ },
+ "main", "srcsZips", "launcher")
+
+ embeddedParNoMain = pctx.AndroidStaticRule("embeddedParNoMain",
+ blueprint.RuleParams{
+ Command: `$mergeParCmd -p --prefix $launcher $out $srcsZips && ` +
+ `chmod +x $out`,
CommandDeps: []string{"$mergeParCmd"},
},
- "main", "entryPoint", "srcsZips", "launcher")
+ "srcsZips", "launcher")
)
func init() {
@@ -108,21 +114,30 @@
// added launcherPath to the implicits Ninja dependencies.
implicits = append(implicits, launcherPath.Path())
- // .intermediate output path for entry_point.txt
- entryPoint := android.PathForModuleOut(ctx, entryPointFile).String()
-
- ctx.Build(pctx, android.BuildParams{
- Rule: embeddedPar,
- Description: "embedded python archive",
- Output: binFile,
- Implicits: implicits,
- Args: map[string]string{
- "main": strings.Replace(strings.TrimSuffix(main, pyExt), "/", ".", -1),
- "entryPoint": entryPoint,
- "srcsZips": strings.Join(srcsZips.Strings(), " "),
- "launcher": launcherPath.String(),
- },
- })
+ if main == "" {
+ ctx.Build(pctx, android.BuildParams{
+ Rule: embeddedParNoMain,
+ Description: "embedded python archive",
+ Output: binFile,
+ Implicits: implicits,
+ Args: map[string]string{
+ "srcsZips": strings.Join(srcsZips.Strings(), " "),
+ "launcher": launcherPath.String(),
+ },
+ })
+ } else {
+ ctx.Build(pctx, android.BuildParams{
+ Rule: embeddedPar,
+ Description: "embedded python archive",
+ Output: binFile,
+ Implicits: implicits,
+ Args: map[string]string{
+ "main": strings.Replace(strings.TrimSuffix(main, pyExt), "/", ".", -1),
+ "srcsZips": strings.Join(srcsZips.Strings(), " "),
+ "launcher": launcherPath.String(),
+ },
+ })
+ }
}
return binFile
diff --git a/python/python.go b/python/python.go
index ddc3f1f..4445f40 100644
--- a/python/python.go
+++ b/python/python.go
@@ -156,6 +156,8 @@
bootstrap(ctx android.ModuleContext, ActualVersion string, embeddedLauncher bool,
srcsPathMappings []pathMapping, srcsZip android.Path,
depsSrcsZips android.Paths) android.OptionalPath
+
+ autorun() bool
}
type installer interface {
@@ -307,9 +309,14 @@
if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion2) {
ctx.AddVariationDependencies(nil, pythonLibTag, "py2-stdlib")
+
+ launcherModule := "py2-launcher"
+ if p.bootstrapper.autorun() {
+ launcherModule = "py2-launcher-autorun"
+ }
ctx.AddFarVariationDependencies([]blueprint.Variation{
{Mutator: "arch", Variation: ctx.Target().String()},
- }, launcherTag, "py2-launcher")
+ }, launcherTag, launcherModule)
// Add py2-launcher shared lib dependencies. Ideally, these should be
// derived from the `shared_libs` property of "py2-launcher". However, we
@@ -422,7 +429,11 @@
p.properties.Actual_version, ctx.ModuleName()))
}
expandedSrcs := ctx.ExpandSources(srcs, exclude_srcs)
- if len(expandedSrcs) == 0 {
+ requiresSrcs := true
+ if p.bootstrapper != nil && !p.bootstrapper.autorun() {
+ requiresSrcs = false
+ }
+ if len(expandedSrcs) == 0 && requiresSrcs {
ctx.ModuleErrorf("doesn't have any source files!")
}
@@ -656,4 +667,5 @@
}
var Bool = proptools.Bool
+var BoolDefault = proptools.BoolDefault
var String = proptools.String
diff --git a/python/scripts/main.py b/python/scripts/main.py
new file mode 100644
index 0000000..225dbe4
--- /dev/null
+++ b/python/scripts/main.py
@@ -0,0 +1,12 @@
+import runpy
+import sys
+
+sys.argv[0] = __loader__.archive
+
+# Set sys.executable to None. The real executable is available as
+# sys.argv[0], and too many things assume sys.executable is a regular Python
+# binary, which isn't available. By setting it to None we get clear errors
+# when people try to use it.
+sys.executable = None
+
+runpy._run_module_as_main("ENTRY_POINT", alter_argv=False)
diff --git a/ui/build/path.go b/ui/build/path.go
index ee72cfd..0e1c02c 100644
--- a/ui/build/path.go
+++ b/ui/build/path.go
@@ -147,11 +147,10 @@
myPath, _ = filepath.Abs(myPath)
- // Use the toybox prebuilts on linux
- if runtime.GOOS == "linux" {
- toyboxPath, _ := filepath.Abs("prebuilts/build-tools/toybox/linux-x86")
- myPath = toyboxPath + string(os.PathListSeparator) + myPath
- }
+ // We put some prebuilts in $PATH, since it's infeasible to add dependencies for all of
+ // them.
+ prebuiltsPath, _ := filepath.Abs("prebuilts/build-tools/path/" + runtime.GOOS + "-x86")
+ myPath = prebuiltsPath + string(os.PathListSeparator) + myPath
config.Environment().Set("PATH", myPath)
config.pathReplaced = true
diff --git a/ui/build/paths/config.go b/ui/build/paths/config.go
index b9713fe..d4922f3 100644
--- a/ui/build/paths/config.go
+++ b/ui/build/paths/config.go
@@ -26,9 +26,9 @@
// Whether to exit with an error instead of invoking the underlying tool.
Error bool
- // Whether we use a toybox prebuilt for this tool. Since we don't have
- // toybox for Darwin, we'll use the host version instead.
- Toybox bool
+ // Whether we use a linux-specific prebuilt for this tool. On Darwin,
+ // we'll allow the host executable instead.
+ LinuxOnlyPrebuilt bool
}
var Allowed = PathConfig{
@@ -59,11 +59,11 @@
Error: true,
}
-var Toybox = PathConfig{
- Symlink: false,
- Log: true,
- Error: true,
- Toybox: true,
+var LinuxOnlyPrebuilt = PathConfig{
+ Symlink: false,
+ Log: true,
+ Error: true,
+ LinuxOnlyPrebuilt: true,
}
func GetConfig(name string) PathConfig {
@@ -101,7 +101,6 @@
"python3": Allowed,
"realpath": Allowed,
"rsync": Allowed,
- "sed": Allowed,
"sh": Allowed,
"tar": Allowed,
"timeout": Allowed,
@@ -126,57 +125,57 @@
"pkg-config": Forbidden,
// On Linux we'll use the toybox versions of these instead.
- "awk": Toybox, // Strictly one-true-awk, but...
- "basename": Toybox,
- "cat": Toybox,
- "chmod": Toybox,
- "cmp": Toybox,
- "cp": Toybox,
- "comm": Toybox,
- "cut": Toybox,
- "dirname": Toybox,
- "du": Toybox,
- "echo": Toybox,
- "env": Toybox,
- "expr": Toybox,
- "head": Toybox,
- "getconf": Toybox,
- "hostname": Toybox,
- "id": Toybox,
- "ln": Toybox,
- "ls": Toybox,
- "md5sum": Toybox,
- "mkdir": Toybox,
- "mktemp": Toybox,
- "mv": Toybox,
- "od": Toybox,
- "paste": Toybox,
- "pgrep": Toybox,
- "pkill": Toybox,
- "ps": Toybox,
- "pwd": Toybox,
- "readlink": Toybox,
- "rm": Toybox,
- "rmdir": Toybox,
- "setsid": Toybox,
- "sha1sum": Toybox,
- "sha256sum": Toybox,
- "sha512sum": Toybox,
- "sleep": Toybox,
- "sort": Toybox,
- "stat": Toybox,
- "tail": Toybox,
- "tee": Toybox,
- "touch": Toybox,
- "true": Toybox,
- "uname": Toybox,
- "uniq": Toybox,
- "unix2dos": Toybox,
- "wc": Toybox,
- "whoami": Toybox,
- "which": Toybox,
- "xargs": Toybox,
- "xxd": Toybox,
+ "basename": LinuxOnlyPrebuilt,
+ "cat": LinuxOnlyPrebuilt,
+ "chmod": LinuxOnlyPrebuilt,
+ "cmp": LinuxOnlyPrebuilt,
+ "cp": LinuxOnlyPrebuilt,
+ "comm": LinuxOnlyPrebuilt,
+ "cut": LinuxOnlyPrebuilt,
+ "dirname": LinuxOnlyPrebuilt,
+ "du": LinuxOnlyPrebuilt,
+ "echo": LinuxOnlyPrebuilt,
+ "env": LinuxOnlyPrebuilt,
+ "expr": LinuxOnlyPrebuilt,
+ "head": LinuxOnlyPrebuilt,
+ "getconf": LinuxOnlyPrebuilt,
+ "hostname": LinuxOnlyPrebuilt,
+ "id": LinuxOnlyPrebuilt,
+ "ln": LinuxOnlyPrebuilt,
+ "ls": LinuxOnlyPrebuilt,
+ "md5sum": LinuxOnlyPrebuilt,
+ "mkdir": LinuxOnlyPrebuilt,
+ "mktemp": LinuxOnlyPrebuilt,
+ "mv": LinuxOnlyPrebuilt,
+ "od": LinuxOnlyPrebuilt,
+ "paste": LinuxOnlyPrebuilt,
+ "pgrep": LinuxOnlyPrebuilt,
+ "pkill": LinuxOnlyPrebuilt,
+ "ps": LinuxOnlyPrebuilt,
+ "pwd": LinuxOnlyPrebuilt,
+ "readlink": LinuxOnlyPrebuilt,
+ "rm": LinuxOnlyPrebuilt,
+ "rmdir": LinuxOnlyPrebuilt,
+ "sed": LinuxOnlyPrebuilt,
+ "setsid": LinuxOnlyPrebuilt,
+ "sha1sum": LinuxOnlyPrebuilt,
+ "sha256sum": LinuxOnlyPrebuilt,
+ "sha512sum": LinuxOnlyPrebuilt,
+ "sleep": LinuxOnlyPrebuilt,
+ "sort": LinuxOnlyPrebuilt,
+ "stat": LinuxOnlyPrebuilt,
+ "tail": LinuxOnlyPrebuilt,
+ "tee": LinuxOnlyPrebuilt,
+ "touch": LinuxOnlyPrebuilt,
+ "true": LinuxOnlyPrebuilt,
+ "uname": LinuxOnlyPrebuilt,
+ "uniq": LinuxOnlyPrebuilt,
+ "unix2dos": LinuxOnlyPrebuilt,
+ "wc": LinuxOnlyPrebuilt,
+ "whoami": LinuxOnlyPrebuilt,
+ "which": LinuxOnlyPrebuilt,
+ "xargs": LinuxOnlyPrebuilt,
+ "xxd": LinuxOnlyPrebuilt,
}
func init() {
@@ -185,10 +184,10 @@
Configuration["sw_vers"] = Allowed
Configuration["xcrun"] = Allowed
- // We don't have toybox prebuilts for darwin, so allow the
- // host versions.
+ // We don't have darwin prebuilts for some tools (like toybox),
+ // so allow the host versions.
for name, config := range Configuration {
- if config.Toybox {
+ if config.LinuxOnlyPrebuilt {
Configuration[name] = Allowed
}
}