dexpreopt/dexpreopt.go

// Copyright 2018 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The dexpreopt package converts a global dexpreopt config and a module dexpreopt config into rules to perform
// dexpreopting.
//
// It is used in two places; in the dexpeopt_gen binary for modules defined in Make, and directly linked into Soong.
//
// For Make modules it is built into the dexpreopt_gen binary, which is executed as a Make rule using global config and
// module config specified in JSON files.  The binary writes out two shell scripts, only updating them if they have
// changed.  One script takes an APK or JAR as an input and produces a zip file containing any outputs of preopting,
// in the location they should be on the device.  The Make build rules will unzip the zip file into $(PRODUCT_OUT) when
// installing the APK, which will install the preopt outputs into $(PRODUCT_OUT)/system or $(PRODUCT_OUT)/system_other
// as necessary.  The zip file may be empty if preopting was disabled for any reason.
//
// The intermediate shell scripts allow changes to this package or to the global config to regenerate the shell scripts
// but only require re-executing preopting if the script has changed.
//
// For Soong modules this package is linked directly into Soong and run from the java package.  It generates the same
// commands as for make, using athe same global config JSON file used by make, but using a module config structure
// provided by Soong.  The generated commands are then converted into Soong rule and written directly to the ninja file,
// with no extra shell scripts involved.
package dexpreopt

import (
	"fmt"
	"path/filepath"
	"runtime"
	"sort"
	"strings"

	"android/soong/android"

	"github.com/google/blueprint/pathtools"
)

const SystemPartition = "/system/"
const SystemOtherPartition = "/system_other/"

var DexpreoptRunningInSoong = false

// GenerateDexpreoptRule generates a set of commands that will preopt a module based on a GlobalConfig and a
// ModuleConfig.  The produced files and their install locations will be available through rule.Installs().
func GenerateDexpreoptRule(ctx android.PathContext, globalSoong *GlobalSoongConfig,
	global *GlobalConfig, module *ModuleConfig) (rule *android.RuleBuilder, err error) {

	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(runtime.Error); ok {
				panic(r)
			} else if e, ok := r.(error); ok {
				err = e
				rule = nil
			} else {
				panic(r)
			}
		}
	}()

	rule = android.NewRuleBuilder()

	generateProfile := module.ProfileClassListing.Valid() && !global.DisableGenerateProfile
	generateBootProfile := module.ProfileBootListing.Valid() && !global.DisableGenerateProfile

	var profile android.WritablePath
	if generateProfile {
		profile = profileCommand(ctx, globalSoong, global, module, rule)
	}
	if generateBootProfile {
		bootProfileCommand(ctx, globalSoong, global, module, rule)
	}

	if !dexpreoptDisabled(ctx, global, module) {
		// Don't preopt individual boot jars, they will be preopted together.
		if !contains(android.GetJarsFromApexJarPairs(ctx, global.BootJars), module.Name) {
			appImage := (generateProfile || module.ForceCreateAppImage || global.DefaultAppImages) &&
				!module.NoCreateAppImage

			generateDM := shouldGenerateDM(module, global)

			for archIdx, _ := range module.Archs {
				dexpreoptCommand(ctx, globalSoong, global, module, rule, archIdx, profile, appImage, generateDM)
			}
		}
	}

	return rule, nil
}

func dexpreoptDisabled(ctx android.PathContext, global *GlobalConfig, module *ModuleConfig) bool {
	if contains(global.DisablePreoptModules, module.Name) {
		return true
	}

	// Don't preopt system server jars that are updatable.
	for _, p := range global.UpdatableSystemServerJars {
		if _, jar := android.SplitApexJarPair(ctx, p); jar == module.Name {
			return true
		}
	}

	// If OnlyPreoptBootImageAndSystemServer=true and module is not in boot class path skip
	// Also preopt system server jars since selinux prevents system server from loading anything from
	// /data. If we don't do this they will need to be extracted which is not favorable for RAM usage
	// or performance. If PreoptExtractedApk is true, we ignore the only preopt boot image options.
	if global.OnlyPreoptBootImageAndSystemServer && !contains(android.GetJarsFromApexJarPairs(ctx, global.BootJars), module.Name) &&
		!contains(global.SystemServerJars, module.Name) && !module.PreoptExtractedApk {
		return true
	}

	return false
}

func profileCommand(ctx android.PathContext, globalSoong *GlobalSoongConfig, global *GlobalConfig,
	module *ModuleConfig, rule *android.RuleBuilder) android.WritablePath {

	profilePath := module.BuildPath.InSameDir(ctx, "profile.prof")
	profileInstalledPath := module.DexLocation + ".prof"

	if !module.ProfileIsTextListing {
		rule.Command().FlagWithOutput("touch ", profilePath)
	}

	cmd := rule.Command().
		Text(`ANDROID_LOG_TAGS="*:e"`).
		Tool(globalSoong.Profman)

	if module.ProfileIsTextListing {
		// The profile is a test listing of classes (used for framework jars).
		// We need to generate the actual binary profile before being able to compile.
		cmd.FlagWithInput("--create-profile-from=", module.ProfileClassListing.Path())
	} else {
		// The profile is binary profile (used for apps). Run it through profman to
		// ensure the profile keys match the apk.
		cmd.
			Flag("--copy-and-update-profile-key").
			FlagWithInput("--profile-file=", module.ProfileClassListing.Path())
	}

	cmd.
		FlagWithInput("--apk=", module.DexPath).
		Flag("--dex-location="+module.DexLocation).
		FlagWithOutput("--reference-profile-file=", profilePath)

	if !module.ProfileIsTextListing {
		cmd.Text(fmt.Sprintf(`|| echo "Profile out of date for %s"`, module.DexPath))
	}
	rule.Install(profilePath, profileInstalledPath)

	return profilePath
}

func bootProfileCommand(ctx android.PathContext, globalSoong *GlobalSoongConfig, global *GlobalConfig,
	module *ModuleConfig, rule *android.RuleBuilder) android.WritablePath {

	profilePath := module.BuildPath.InSameDir(ctx, "profile.bprof")
	profileInstalledPath := module.DexLocation + ".bprof"

	if !module.ProfileIsTextListing {
		rule.Command().FlagWithOutput("touch ", profilePath)
	}

	cmd := rule.Command().
		Text(`ANDROID_LOG_TAGS="*:e"`).
		Tool(globalSoong.Profman)

	// The profile is a test listing of methods.
	// We need to generate the actual binary profile.
	cmd.FlagWithInput("--create-profile-from=", module.ProfileBootListing.Path())

	cmd.
		Flag("--generate-boot-profile").
		FlagWithInput("--apk=", module.DexPath).
		Flag("--dex-location="+module.DexLocation).
		FlagWithOutput("--reference-profile-file=", profilePath)

	if !module.ProfileIsTextListing {
		cmd.Text(fmt.Sprintf(`|| echo "Profile out of date for %s"`, module.DexPath))
	}
	rule.Install(profilePath, profileInstalledPath)

	return profilePath
}

type classLoaderContext struct {
	// The class loader context using paths in the build.
	Host android.Paths

	// The class loader context using paths as they will be on the device.
	Target []string
}

// A map of class loader contexts for each SDK version.
// A map entry for "any" version contains libraries that are unconditionally added to class loader
// context. Map entries for existing versions contains libraries that were in the default classpath
// until that API version, and should be added to class loader context if and only if the
// targetSdkVersion in the manifest or APK is less than that API version.
type classLoaderContextMap map[int]*classLoaderContext

const anySdkVersion int = -1

func (m classLoaderContextMap) getSortedKeys() []int {
	keys := make([]int, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Ints(keys)
	return keys
}

func (m classLoaderContextMap) getValue(sdkVer int) *classLoaderContext {
	if _, ok := m[sdkVer]; !ok {
		m[sdkVer] = &classLoaderContext{}
	}
	return m[sdkVer]
}

func (m classLoaderContextMap) addLibs(sdkVer int, module *ModuleConfig, libs ...string) {
	clc := m.getValue(sdkVer)
	for _, lib := range libs {
		clc.Host = append(clc.Host, pathForLibrary(module, lib))
		clc.Target = append(clc.Target, filepath.Join("/system/framework", lib+".jar"))
	}
}

func (m classLoaderContextMap) addSystemServerLibs(sdkVer int, ctx android.PathContext, module *ModuleConfig, libs ...string) {
	clc := m.getValue(sdkVer)
	for _, lib := range libs {
		clc.Host = append(clc.Host, SystemServerDexJarHostPath(ctx, lib))
		clc.Target = append(clc.Target, filepath.Join("/system/framework", lib+".jar"))
	}
}

func dexpreoptCommand(ctx android.PathContext, globalSoong *GlobalSoongConfig, global *GlobalConfig,
	module *ModuleConfig, rule *android.RuleBuilder, archIdx int, profile android.WritablePath,
	appImage bool, generateDM bool) {

	arch := module.Archs[archIdx]

	// HACK: make soname in Soong-generated .odex files match Make.
	base := filepath.Base(module.DexLocation)
	if filepath.Ext(base) == ".jar" {
		base = "javalib.jar"
	} else if filepath.Ext(base) == ".apk" {
		base = "package.apk"
	}

	toOdexPath := func(path string) string {
		return filepath.Join(
			filepath.Dir(path),
			"oat",
			arch.String(),
			pathtools.ReplaceExtension(filepath.Base(path), "odex"))
	}

	odexPath := module.BuildPath.InSameDir(ctx, "oat", arch.String(), pathtools.ReplaceExtension(base, "odex"))
	odexInstallPath := toOdexPath(module.DexLocation)
	if odexOnSystemOther(module, global) {
		odexInstallPath = filepath.Join(SystemOtherPartition, odexInstallPath)
	}

	vdexPath := odexPath.ReplaceExtension(ctx, "vdex")
	vdexInstallPath := pathtools.ReplaceExtension(odexInstallPath, "vdex")

	invocationPath := odexPath.ReplaceExtension(ctx, "invocation")

	systemServerJars := NonUpdatableSystemServerJars(ctx, global)

	classLoaderContexts := make(classLoaderContextMap)

	// A flag indicating if the '&' class loader context is used.
	unknownClassLoaderContext := false

	if module.EnforceUsesLibraries {
		// Unconditional class loader context.
		usesLibs := append(copyOf(module.UsesLibraries), module.PresentOptionalUsesLibraries...)
		classLoaderContexts.addLibs(anySdkVersion, module, usesLibs...)

		// Conditional class loader context for API version < 28.
		const httpLegacy = "org.apache.http.legacy"
		if !contains(usesLibs, httpLegacy) {
			classLoaderContexts.addLibs(28, module, httpLegacy)
		}

		// Conditional class loader context for API version < 29.
		usesLibs29 := []string{
			"android.hidl.base-V1.0-java",
			"android.hidl.manager-V1.0-java",
		}
		classLoaderContexts.addLibs(29, module, usesLibs29...)

		// Conditional class loader context for API version < 30.
		const testBase = "android.test.base"
		if !contains(usesLibs, testBase) {
			classLoaderContexts.addLibs(30, module, testBase)
		}
	} else if jarIndex := android.IndexList(module.Name, systemServerJars); jarIndex >= 0 {
		// System server jars should be dexpreopted together: class loader context of each jar
		// should include all preceding jars on the system server classpath.
		classLoaderContexts.addSystemServerLibs(anySdkVersion, ctx, module, systemServerJars[:jarIndex]...)

		// Copy the system server jar to a predefined location where dex2oat will find it.
		dexPathHost := SystemServerDexJarHostPath(ctx, module.Name)
		rule.Command().Text("mkdir -p").Flag(filepath.Dir(dexPathHost.String()))
		rule.Command().Text("cp -f").Input(module.DexPath).Output(dexPathHost)
	} else {
		// Pass special class loader context to skip the classpath and collision check.
		// This will get removed once LOCAL_USES_LIBRARIES is enforced.
		// Right now LOCAL_USES_LIBRARIES is opt in, for the case where it's not specified we still default
		// to the &.
		unknownClassLoaderContext = true
	}

	rule.Command().FlagWithArg("mkdir -p ", filepath.Dir(odexPath.String()))
	rule.Command().FlagWithOutput("rm -f ", odexPath)
	// Set values in the environment of the rule.  These may be modified by construct_context.sh.
	if unknownClassLoaderContext {
		rule.Command().
			Text(`class_loader_context_arg=--class-loader-context=\&`).
			Text(`stored_class_loader_context_arg=""`)
	} else {
		clc := classLoaderContexts[anySdkVersion]
		rule.Command().
			Text("class_loader_context_arg=--class-loader-context=PCL[" + strings.Join(clc.Host.Strings(), ":") + "]").
			Implicits(clc.Host).
			Text("stored_class_loader_context_arg=--stored-class-loader-context=PCL[" + strings.Join(clc.Target, ":") + "]")
	}

	if module.EnforceUsesLibraries {
		if module.ManifestPath != nil {
			rule.Command().Text(`target_sdk_version="$(`).
				Tool(globalSoong.ManifestCheck).
				Flag("--extract-target-sdk-version").
				Input(module.ManifestPath).
				Text(`)"`)
		} else {
			// No manifest to extract targetSdkVersion from, hope that DexJar is an APK
			rule.Command().Text(`target_sdk_version="$(`).
				Tool(globalSoong.Aapt).
				Flag("dump badging").
				Input(module.DexPath).
				Text(`| grep "targetSdkVersion" | sed -n "s/targetSdkVersion:'\(.*\)'/\1/p"`).
				Text(`)"`)
		}
		for _, ver := range classLoaderContexts.getSortedKeys() {
			clc := classLoaderContexts.getValue(ver)
			var varHost, varTarget string
			if ver == anySdkVersion {
				varHost = "dex_preopt_host_libraries"
				varTarget = "dex_preopt_target_libraries"
			} else {
				varHost = fmt.Sprintf("conditional_host_libs_%d", ver)
				varTarget = fmt.Sprintf("conditional_target_libs_%d", ver)
			}
			rule.Command().Textf(varHost+`="%s"`, strings.Join(clc.Host.Strings(), " ")).Implicits(clc.Host)
			rule.Command().Textf(varTarget+`="%s"`, strings.Join(clc.Target, " "))
		}
		rule.Command().Text("source").Tool(globalSoong.ConstructContext).Input(module.DexPath)
	}

	// Devices that do not have a product partition use a symlink from /product to /system/product.
	// Because on-device dexopt will see dex locations starting with /product, we change the paths
	// to mimic this behavior.
	dexLocationArg := module.DexLocation
	if strings.HasPrefix(dexLocationArg, "/system/product/") {
		dexLocationArg = strings.TrimPrefix(dexLocationArg, "/system")
	}

	cmd := rule.Command().
		Text(`ANDROID_LOG_TAGS="*:e"`).
		Tool(globalSoong.Dex2oat).
		Flag("--avoid-storing-invocation").
		FlagWithOutput("--write-invocation-to=", invocationPath).ImplicitOutput(invocationPath).
		Flag("--runtime-arg").FlagWithArg("-Xms", global.Dex2oatXms).
		Flag("--runtime-arg").FlagWithArg("-Xmx", global.Dex2oatXmx).
		Flag("--runtime-arg").FlagWithInputList("-Xbootclasspath:", module.PreoptBootClassPathDexFiles, ":").
		Flag("--runtime-arg").FlagWithList("-Xbootclasspath-locations:", module.PreoptBootClassPathDexLocations, ":").
		Flag("${class_loader_context_arg}").
		Flag("${stored_class_loader_context_arg}").
		FlagWithArg("--boot-image=", strings.Join(module.DexPreoptImageLocations, ":")).Implicits(module.DexPreoptImagesDeps[archIdx].Paths()).
		FlagWithInput("--dex-file=", module.DexPath).
		FlagWithArg("--dex-location=", dexLocationArg).
		FlagWithOutput("--oat-file=", odexPath).ImplicitOutput(vdexPath).
		// Pass an empty directory, dex2oat shouldn't be reading arbitrary files
		FlagWithArg("--android-root=", global.EmptyDirectory).
		FlagWithArg("--instruction-set=", arch.String()).
		FlagWithArg("--instruction-set-variant=", global.CpuVariant[arch]).
		FlagWithArg("--instruction-set-features=", global.InstructionSetFeatures[arch]).
		Flag("--no-generate-debug-info").
		Flag("--generate-build-id").
		Flag("--abort-on-hard-verifier-error").
		Flag("--force-determinism").
		FlagWithArg("--no-inline-from=", "core-oj.jar")

	var preoptFlags []string
	if len(module.PreoptFlags) > 0 {
		preoptFlags = module.PreoptFlags
	} else if len(global.PreoptFlags) > 0 {
		preoptFlags = global.PreoptFlags
	}

	if len(preoptFlags) > 0 {
		cmd.Text(strings.Join(preoptFlags, " "))
	}

	if module.UncompressedDex {
		cmd.FlagWithArg("--copy-dex-files=", "false")
	}

	if !android.PrefixInList(preoptFlags, "--compiler-filter=") {
		var compilerFilter string
		if contains(global.SystemServerJars, module.Name) {
			// Jars of system server, use the product option if it is set, speed otherwise.
			if global.SystemServerCompilerFilter != "" {
				compilerFilter = global.SystemServerCompilerFilter
			} else {
				compilerFilter = "speed"
			}
		} else if contains(global.SpeedApps, module.Name) || contains(global.SystemServerApps, module.Name) {
			// Apps loaded into system server, and apps the product default to being compiled with the
			// 'speed' compiler filter.
			compilerFilter = "speed"
		} else if profile != nil {
			// For non system server jars, use speed-profile when we have a profile.
			compilerFilter = "speed-profile"
		} else if global.DefaultCompilerFilter != "" {
			compilerFilter = global.DefaultCompilerFilter
		} else {
			compilerFilter = "quicken"
		}
		cmd.FlagWithArg("--compiler-filter=", compilerFilter)
	}

	if generateDM {
		cmd.FlagWithArg("--copy-dex-files=", "false")
		dmPath := module.BuildPath.InSameDir(ctx, "generated.dm")
		dmInstalledPath := pathtools.ReplaceExtension(module.DexLocation, "dm")
		tmpPath := module.BuildPath.InSameDir(ctx, "primary.vdex")
		rule.Command().Text("cp -f").Input(vdexPath).Output(tmpPath)
		rule.Command().Tool(globalSoong.SoongZip).
			FlagWithArg("-L", "9").
			FlagWithOutput("-o", dmPath).
			Flag("-j").
			Input(tmpPath)
		rule.Install(dmPath, dmInstalledPath)
	}

	// By default, emit debug info.
	debugInfo := true
	if global.NoDebugInfo {
		// If the global setting suppresses mini-debug-info, disable it.
		debugInfo = false
	}

	// PRODUCT_SYSTEM_SERVER_DEBUG_INFO overrides WITH_DEXPREOPT_DEBUG_INFO.
	// PRODUCT_OTHER_JAVA_DEBUG_INFO overrides WITH_DEXPREOPT_DEBUG_INFO.
	if contains(global.SystemServerJars, module.Name) {
		if global.AlwaysSystemServerDebugInfo {
			debugInfo = true
		} else if global.NeverSystemServerDebugInfo {
			debugInfo = false
		}
	} else {
		if global.AlwaysOtherDebugInfo {
			debugInfo = true
		} else if global.NeverOtherDebugInfo {
			debugInfo = false
		}
	}

	// Never enable on eng.
	if global.IsEng {
		debugInfo = false
	}

	if debugInfo {
		cmd.Flag("--generate-mini-debug-info")
	} else {
		cmd.Flag("--no-generate-mini-debug-info")
	}

	// Set the compiler reason to 'prebuilt' to identify the oat files produced
	// during the build, as opposed to compiled on the device.
	cmd.FlagWithArg("--compilation-reason=", "prebuilt")

	if appImage {
		appImagePath := odexPath.ReplaceExtension(ctx, "art")
		appImageInstallPath := pathtools.ReplaceExtension(odexInstallPath, "art")
		cmd.FlagWithOutput("--app-image-file=", appImagePath).
			FlagWithArg("--image-format=", "lz4")
		if !global.DontResolveStartupStrings {
			cmd.FlagWithArg("--resolve-startup-const-strings=", "true")
		}
		rule.Install(appImagePath, appImageInstallPath)
	}

	if profile != nil {
		cmd.FlagWithInput("--profile-file=", profile)
	}

	rule.Install(odexPath, odexInstallPath)
	rule.Install(vdexPath, vdexInstallPath)
}

func shouldGenerateDM(module *ModuleConfig, global *GlobalConfig) bool {
	// Generating DM files only makes sense for verify, avoid doing for non verify compiler filter APKs.
	// No reason to use a dm file if the dex is already uncompressed.
	return global.GenerateDMFiles && !module.UncompressedDex &&
		contains(module.PreoptFlags, "--compiler-filter=verify")
}

func OdexOnSystemOtherByName(name string, dexLocation string, global *GlobalConfig) bool {
	if !global.HasSystemOther {
		return false
	}

	if global.SanitizeLite {
		return false
	}

	if contains(global.SpeedApps, name) || contains(global.SystemServerApps, name) {
		return false
	}

	for _, f := range global.PatternsOnSystemOther {
		if makefileMatch(filepath.Join(SystemPartition, f), dexLocation) {
			return true
		}
	}

	return false
}

func odexOnSystemOther(module *ModuleConfig, global *GlobalConfig) bool {
	return OdexOnSystemOtherByName(module.Name, module.DexLocation, global)
}

// PathToLocation converts .../system/framework/arm64/boot.art to .../system/framework/boot.art
func PathToLocation(path android.Path, arch android.ArchType) string {
	pathArch := filepath.Base(filepath.Dir(path.String()))
	if pathArch != arch.String() {
		panic(fmt.Errorf("last directory in %q must be %q", path, arch.String()))
	}
	return filepath.Join(filepath.Dir(filepath.Dir(path.String())), filepath.Base(path.String()))
}

func pathForLibrary(module *ModuleConfig, lib string) android.Path {
	path, ok := module.LibraryPaths[lib]
	if !ok {
		panic(fmt.Errorf("unknown library path for %q", lib))
	}
	return path
}

func makefileMatch(pattern, s string) bool {
	percent := strings.IndexByte(pattern, '%')
	switch percent {
	case -1:
		return pattern == s
	case len(pattern) - 1:
		return strings.HasPrefix(s, pattern[:len(pattern)-1])
	default:
		panic(fmt.Errorf("unsupported makefile pattern %q", pattern))
	}
}

// Expected format for apexJarValue = <apex name>:<jar name>
func GetJarLocationFromApexJarPair(ctx android.PathContext, apexJarValue string) string {
	apex, jar := android.SplitApexJarPair(ctx, apexJarValue)
	return filepath.Join("/apex", apex, "javalib", jar+".jar")
}

var nonUpdatableSystemServerJarsKey = android.NewOnceKey("nonUpdatableSystemServerJars")

// TODO: eliminate the superficial global config parameter by moving global config definition
// from java subpackage to dexpreopt.
func NonUpdatableSystemServerJars(ctx android.PathContext, global *GlobalConfig) []string {
	return ctx.Config().Once(nonUpdatableSystemServerJarsKey, func() interface{} {
		return android.RemoveListFromList(global.SystemServerJars,
			android.GetJarsFromApexJarPairs(ctx, global.UpdatableSystemServerJars))
	}).([]string)
}

// A predefined location for the system server dex jars. This is needed in order to generate
// class loader context for dex2oat, as the path to the jar in the Soong module may be unknown
// at that time (Soong processes the jars in dependency order, which may be different from the
// the system server classpath order).
func SystemServerDexJarHostPath(ctx android.PathContext, jar string) android.OutputPath {
	if DexpreoptRunningInSoong {
		// Soong module, just use the default output directory $OUT/soong.
		return android.PathForOutput(ctx, "system_server_dexjars", jar+".jar")
	} else {
		// Make module, default output directory is $OUT (passed via the "null config" created
		// by dexpreopt_gen). Append Soong subdirectory to match Soong module paths.
		return android.PathForOutput(ctx, "soong", "system_server_dexjars", jar+".jar")
	}
}

func contains(l []string, s string) bool {
	for _, e := range l {
		if e == s {
			return true
		}
	}
	return false
}

var copyOf = android.CopyOf