Documenting java/dexpreopt_bootjars.go
Test: m nothing
Bug: 173092919
Change-Id: I20018e1df6ad5e21172bb91435ae2c9cad7024e6
diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go
index f9975ba..5df5845 100644
--- a/java/dexpreopt_bootjars.go
+++ b/java/dexpreopt_bootjars.go
@@ -25,11 +25,177 @@
"github.com/google/blueprint/proptools"
)
+// This comment describes:
+// 1. ART boot images in general (their types, structure, file layout, etc.)
+// 2. build system support for boot images
+//
+// 1. ART boot images
+// ------------------
+//
+// A boot image in ART is a set of files that contain AOT-compiled native code and a heap snapshot
+// of AOT-initialized classes for the bootclasspath Java libraries. A boot image is compiled from a
+// set of DEX jars by the dex2oat compiler. A boot image is used for two purposes: 1) it is
+// installed on device and loaded at runtime, and 2) other Java libraries and apps are compiled
+// against it (compilation may take place either on host, known as "dexpreopt", or on device, known
+// as "dexopt").
+//
+// A boot image is not a single file, but a collection of interrelated files. Each boot image has a
+// number of components that correspond to the Java libraries that constitute it. For each component
+// there are multiple files:
+// - *.oat or *.odex file with native code (architecture-specific, one per instruction set)
+// - *.art file with pre-initialized Java classes (architecture-specific, one per instruction set)
+// - *.vdex file with verification metadata for the DEX bytecode (architecture independent)
+//
+// *.vdex files for the boot images do not contain the DEX bytecode itself, because the
+// bootclasspath DEX files are stored on disk in uncompressed and aligned form. Consequently a boot
+// image is not self-contained and cannot be used without its DEX files. To simplify the management
+// of boot image files, ART uses a certain naming scheme and associates the following metadata with
+// each boot image:
+// - A stem, which is a symbolic name that is prepended to boot image file names.
+// - A location (on-device path to the boot image files).
+// - A list of boot image locations (on-device paths to dependency boot images).
+// - A set of DEX locations (on-device paths to the DEX files, one location for one DEX file used
+// to compile the boot image).
+//
+// There are two kinds of boot images:
+// - primary boot images
+// - boot image extensions
+//
+// 1.1. Primary boot images
+// ------------------------
+//
+// A primary boot image is compiled for a core subset of bootclasspath Java libraries. It does not
+// depend on any other images, and other boot images may depend on it.
+//
+// For example, assuming that the stem is "boot", the location is /apex/com.android.art/javalib/,
+// the set of core bootclasspath libraries is A B C, and the boot image is compiled for ARM targets
+// (32 and 64 bits), it will have three components with the following files:
+// - /apex/com.android.art/javalib/{arm,arm64}/boot.{art,oat,vdex}
+// - /apex/com.android.art/javalib/{arm,arm64}/boot-B.{art,oat,vdex}
+// - /apex/com.android.art/javalib/{arm,arm64}/boot-C.{art,oat,vdex}
+//
+// The files of the first component are special: they do not have the component name appended after
+// the stem. This naming convention dates back to the times when the boot image was not split into
+// components, and there were just boot.oat and boot.art. The decision to split was motivated by
+// licensing reasons for one of the bootclasspath libraries.
+//
+// As of November 2020 the only primary boot image in Android is the image in the ART APEX
+// com.android.art. The primary ART boot image contains the Core libraries that are part of the ART
+// module. When the ART module gets updated, the primary boot image will be updated with it, and all
+// dependent images will get invalidated (the checksum of the primary image stored in dependent
+// images will not match), unless they are updated in sync with the ART module.
+//
+// 1.2. Boot image extensions
+// --------------------------
+//
+// A boot image extension is compiled for a subset of bootclasspath Java libraries (in particular,
+// this subset does not include the Core bootclasspath libraries that go into the primary boot
+// image). A boot image extension depends on the primary boot image and optionally some other boot
+// image extensions. Other images may depend on it. In other words, boot image extensions can form
+// acyclic dependency graphs.
+//
+// The motivation for boot image extensions comes from the Mainline project. Consider a situation
+// when the list of bootclasspath libraries is A B C, and both A and B are parts of the Android
+// platform, but C is part of an updatable APEX com.android.C. When the APEX is updated, the Java
+// code for C might have changed compared to the code that was used to compile the boot image.
+// Consequently, the whole boot image is obsolete and invalidated (even though the code for A and B
+// that does not depend on C is up to date). To avoid this, the original monolithic boot image is
+// split in two parts: the primary boot image that contains A B, and the boot image extension that
+// contains C and depends on the primary boot image (extends it).
+//
+// For example, assuming that the stem is "boot", the location is /system/framework, the set of
+// bootclasspath libraries is D E (where D is part of the platform and is located in
+// /system/framework, and E is part of a non-updatable APEX com.android.E and is located in
+// /apex/com.android.E/javalib), and the boot image is compiled for ARM targets (32 and 64 bits),
+// it will have two components with the following files:
+// - /system/framework/{arm,arm64}/boot-D.{art,oat,vdex}
+// - /system/framework/{arm,arm64}/boot-E.{art,oat,vdex}
+//
+// As of November 2020 the only boot image extension in Android is the Framework boot image
+// extension. It extends the primary ART boot image and contains Framework libraries and other
+// bootclasspath libraries from the platform and non-updatable APEXes that are not included in the
+// ART image. The Framework boot image extension is updated together with the platform. In the
+// future other boot image extensions may be added for some updatable modules.
+//
+//
+// 2. Build system support for boot images
+// ---------------------------------------
+//
+// The primary ART boot image needs to be compiled with one dex2oat invocation that depends on DEX
+// jars for the core libraries. Framework boot image extension needs to be compiled with one dex2oat
+// invocation that depends on the primary ART boot image and all bootclasspath DEX jars except the
+// Core libraries.
+//
+// 2.1. Libraries that go in the boot images
+// -----------------------------------------
+//
+// The contents of each boot image are determined by the PRODUCT variables. The primary ART APEX
+// boot image contains libraries listed in the ART_APEX_JARS variable in the AOSP makefiles. The
+// Framework boot image extension contains libraries specified in the PRODUCT_BOOT_JARS and
+// PRODUCT_BOOT_JARS_EXTRA variables. The AOSP makefiles specify some common Framework libraries,
+// but more product-specific libraries can be added in the product makefiles.
+//
+// Each component of the PRODUCT_BOOT_JARS and PRODUCT_BOOT_JARS_EXTRA variables is either a simple
+// name (if the library is a part of the Platform), or a colon-separated pair <apex, name> (if the
+// library is a part of a non-updatable APEX).
+//
+// A related variable PRODUCT_UPDATABLE_BOOT_JARS contains bootclasspath libraries that are in
+// updatable APEXes. They are not included in the boot image.
+//
+// One exception to the above rules are "coverage" builds (a special build flavor which requires
+// setting environment variable EMMA_INSTRUMENT_FRAMEWORK=true). In coverage builds the Java code in
+// boot image libraries is instrumented, which means that the instrumentation library (jacocoagent)
+// needs to be added to the list of bootclasspath DEX jars.
+//
+// In general, there is a requirement that the source code for a boot image library must be
+// available at build time (e.g. it cannot be a stub that has a separate implementation library).
+//
+// 2.2. Static configs
+// -------------------
+//
+// Because boot images are used to dexpreopt other Java modules, the paths to boot image files must
+// be known by the time dexpreopt build rules for the dependent modules are generated. Boot image
+// configs are constructed very early during the build, before build rule generation. The configs
+// provide predefined paths to boot image files (these paths depend only on static build
+// configuration, such as PRODUCT variables, and use hard-coded directory names).
+//
+// 2.3. Singleton
+// --------------
+//
+// Build rules for the boot images are generated with a Soong singleton. Because a singleton has no
+// dependencies on other modules, it has to find the modules for the DEX jars using VisitAllModules.
+// Soong loops through all modules and compares each module against a list of bootclasspath library
+// names. Then it generates build rules that copy DEX jars from their intermediate module-specific
+// locations to the hard-coded locations predefined in the boot image configs.
+//
+// It would be possible to use a module with proper dependencies instead, but that would require
+// changes in the way Soong generates variables for Make: a singleton can use one MakeVars() method
+// that writes variables to out/soong/make_vars-*.mk, which is included early by the main makefile,
+// but module(s) would have to use out/soong/Android-*.mk which has a group of LOCAL_* variables
+// for each module, and is included later.
+//
+// 2.4. Install rules
+// ------------------
+//
+// The primary boot image and the Framework extension are installed in different ways. The primary
+// boot image is part of the ART APEX: it is copied into the APEX intermediate files, packaged
+// together with other APEX contents, extracted and mounted on device. The Framework boot image
+// extension is installed by the rules defined in makefiles (make/core/dex_preopt_libart.mk). Soong
+// writes out a few DEXPREOPT_IMAGE_* variables for Make; these variables contain boot image names,
+// paths and so on.
+//
+// 2.5. JIT-Zygote configuration
+// -----------------------------
+//
+// One special configuration is JIT-Zygote build, when the primary ART image is used for compiling
+// apps instead of the Framework boot image extension (see DEXPREOPT_USE_ART_IMAGE and UseArtImage).
+//
+
func init() {
RegisterDexpreoptBootJarsComponents(android.InitRegistrationContext)
}
-// Target-independent description of pre-compiled boot image.
+// Target-independent description of a boot image.
type bootImageConfig struct {
// If this image is an extension, the image that it extends.
extends *bootImageConfig
@@ -66,7 +232,7 @@
variants []*bootImageVariant
}
-// Target-dependent description of pre-compiled boot image.
+// Target-dependent description of a boot image.
type bootImageVariant struct {
*bootImageConfig
@@ -90,6 +256,7 @@
unstrippedInstalls android.RuleBuilderInstalls
}
+// Get target-specific boot image variant for the given boot image config and target.
func (image bootImageConfig) getVariant(target android.Target) *bootImageVariant {
for _, variant := range image.variants {
if variant.target.Os == target.Os && variant.target.Arch.ArchType == target.Arch.ArchType {
@@ -99,7 +266,7 @@
return nil
}
-// Return any (the first) variant which is for the device (as opposed to for the host)
+// Return any (the first) variant which is for the device (as opposed to for the host).
func (image bootImageConfig) getAnyAndroidVariant() *bootImageVariant {
for _, variant := range image.variants {
if variant.target.Os == android.Android {
@@ -109,10 +276,12 @@
return nil
}
+// Return the name of a boot image module given a boot image config and a component (module) index.
+// A module name is a combination of the Java library name, and the boot image stem (that is stored
+// in the config).
func (image bootImageConfig) moduleName(ctx android.PathContext, idx int) string {
- // Dexpreopt on the boot class path produces multiple files. The first dex file
- // is converted into 'name'.art (to match the legacy assumption that 'name'.art
- // exists), and the rest are converted to 'name'-<jar>.art.
+ // The first module of the primary boot image is special: its module name has only the stem, but
+ // not the library name. All other module names are of the form <stem>-<library name>
m := image.modules.Jar(idx)
name := image.stem
if idx != 0 || image.extends != nil {
@@ -121,6 +290,7 @@
return name
}
+// Return the name of the first boot image module, or stem if the list of modules is empty.
func (image bootImageConfig) firstModuleNameOrStem(ctx android.PathContext) string {
if image.modules.Len() > 0 {
return image.moduleName(ctx, 0)
@@ -129,6 +299,8 @@
}
}
+// Return filenames for the given boot image component, given the output directory and a list of
+// extensions.
func (image bootImageConfig) moduleFiles(ctx android.PathContext, dir android.OutputPath, exts ...string) android.OutputPaths {
ret := make(android.OutputPaths, 0, image.modules.Len()*len(exts))
for i := 0; i < image.modules.Len(); i++ {
@@ -140,17 +312,26 @@
return ret
}
+// Return boot image locations (as a list of symbolic paths).
+//
// The image "location" is a symbolic path that, with multiarchitecture support, doesn't really
// exist on the device. Typically it is /apex/com.android.art/javalib/boot.art and should be the
// same for all supported architectures on the device. The concrete architecture specific files
// actually end up in architecture-specific sub-directory such as arm, arm64, x86, or x86_64.
//
-// For example a physical file
-// "/apex/com.android.art/javalib/x86/boot.art" has "image location"
-// "/apex/com.android.art/javalib/boot.art" (which is not an actual file).
+// For example a physical file /apex/com.android.art/javalib/x86/boot.art has "image location"
+// /apex/com.android.art/javalib/boot.art (which is not an actual file).
+//
+// For a primary boot image the list of locations has a single element.
+//
+// For a boot image extension the list of locations contains a location for all dependency images
+// (including the primary image) and the location of the extension itself. For example, for the
+// Framework boot image extension that depends on the primary ART boot image the list contains two
+// elements.
//
// The location is passed as an argument to the ART tools like dex2oat instead of the real path.
// ART tools will then reconstruct the architecture-specific real path.
+//
func (image *bootImageVariant) imageLocations() (imageLocations []string) {
if image.extends != nil {
imageLocations = image.extends.getVariant(image.target).imageLocations()
@@ -158,18 +339,6 @@
return append(imageLocations, dexpreopt.PathToLocation(image.images, image.target.Arch.ArchType))
}
-func concat(lists ...[]string) []string {
- var size int
- for _, l := range lists {
- size += len(l)
- }
- ret := make([]string, 0, size)
- for _, l := range lists {
- ret = append(ret, l...)
- }
- return ret
-}
-
func dexpreoptBootJarsFactory() android.Singleton {
return &dexpreoptBootJars{}
}
@@ -182,10 +351,21 @@
return dexpreopt.GetGlobalConfig(ctx).DisablePreopt
}
+// Singleton for generating boot image build rules.
type dexpreoptBootJars struct {
+ // Default boot image config (currently always the Framework boot image extension). It should be
+ // noted that JIT-Zygote builds use ART APEX image instead of the Framework boot image extension,
+ // but the switch is handled not here, but in the makefiles (triggered with
+ // DEXPREOPT_USE_ART_IMAGE=true).
defaultBootImage *bootImageConfig
- otherImages []*bootImageConfig
+ // Other boot image configs (currently the list contains only the primary ART APEX image. It
+ // used to contain an experimental JIT-Zygote image (now replaced with the ART APEX image). In
+ // the future other boot image extensions may be added.
+ otherImages []*bootImageConfig
+
+ // Build path to a config file that Soong writes for Make (to be used in makefiles that install
+ // the default boot image).
dexpreoptConfigForMake android.WritablePath
}
@@ -205,7 +385,7 @@
return files
}
-// dexpreoptBoot singleton rules
+// Generate build rules for boot images.
func (d *dexpreoptBootJars) GenerateBuildActions(ctx android.SingletonContext) {
if skipDexpreoptBootJars(ctx) {
return
@@ -334,9 +514,10 @@
}
}
- // The path to bootclasspath dex files needs to be known at module GenerateAndroidBuildAction time, before
- // the bootclasspath modules have been compiled. Copy the dex jars there so the module rules that have
- // already been set up can find them.
+ // The paths to bootclasspath DEX files need to be known at module GenerateAndroidBuildAction
+ // time, before the boot images are built (these paths are used in dexpreopt rule generation for
+ // Java libraries and apps). Generate rules that copy bootclasspath DEX jars to the predefined
+ // paths.
for i := range bootDexJars {
ctx.Build(pctx, android.BuildParams{
Rule: android.Cp,
@@ -371,6 +552,7 @@
return image
}
+// Generate boot image build rules for a specific target.
func buildBootImageVariant(ctx android.SingletonContext, image *bootImageVariant,
profile android.Path, missingDeps []string) android.WritablePaths {
@@ -428,12 +610,15 @@
}
if image.extends != nil {
+ // It is a boot image extension, so it needs the boot image it depends on (in this case the
+ // primary ART APEX image).
artImage := image.primaryImages
cmd.
Flag("--runtime-arg").FlagWithInputList("-Xbootclasspath:", image.dexPathsDeps.Paths(), ":").
Flag("--runtime-arg").FlagWithList("-Xbootclasspath-locations:", image.dexLocationsDeps, ":").
FlagWithArg("--boot-image=", dexpreopt.PathToLocation(artImage, arch)).Implicit(artImage)
} else {
+ // It is a primary image, so it needs a base address.
cmd.FlagWithArg("--base=", ctx.Config().LibartImgDeviceBaseAddress())
}
@@ -717,7 +902,9 @@
android.WriteFileRule(ctx, path, string(data))
}
-// Export paths for default boot image to Make
+// Define Make variables for boot image names, paths, etc. These variables are used in makefiles
+// (make/core/dex_preopt_libart.mk) to generate install rules that copy boot image files to the
+// correct output directories.
func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) {
if d.dexpreoptConfigForMake != nil {
ctx.Strict("DEX_PREOPT_CONFIG_FOR_MAKE", d.dexpreoptConfigForMake.String())
@@ -731,6 +918,11 @@
ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS", strings.Join(image.getAnyAndroidVariant().dexLocationsDeps, " "))
var imageNames []string
+ // TODO: the primary ART boot image should not be exposed to Make, as it is installed in a
+ // different way as a part of the ART APEX. However, there is a special JIT-Zygote build
+ // configuration which uses the primary ART image instead of the Framework boot image
+ // extension, and it relies on the ART image being exposed to Make. To fix this, it is
+ // necessary to rework the logic in makefiles.
for _, current := range append(d.otherImages, image) {
imageNames = append(imageNames, current.name)
for _, variant := range current.variants {