Export dex implementation jars from prebuilt_apex

Dexpreopt and boot jars package check all require access to dex
implementation jars created for java_library and java_sdk_library. They
were available when building from source but not when building from
prebuilts, even though they are embedded within the .apex files that
are referenced from prebuilt_apex.

This changes adds support to prebuilt_apex to export the dex
implementation jars and updates java_import to use those exported dex
implementation jars.

In a source build dexpreopt/boot jars package check access the apex (or
platform) specific variant of a java_library, e.g. core-oj, from which
it retrieves the dex implementation jar path.

After this change in a prebuilt build dexpreopt/boot jars package check
behave in the same way except in this case they retrieve the dex
implementation jar path from the apex (or platform) specific variant of
the java_import, e.g. core-oj.

The work to export files from a `.apex` file for use by other modules
is performed by a new `deapexer` module type. It is not used directly
in an `Android.bp` file but instead is created implicitly by
`prebuilt_apex`,

In order to do that this contains the following changes:
* Adds a new `dexapexer` module type to handle the exporting of files
  from the `.apex` file.
* Adds an exported_java_libs property to prebuilt_apex to specify the
  set of libraries whose dex implementation jars need exporting.
* Creates apex specific variants of the libraries listed in the
  exported_java_libs property.
* Adds the set of exported files to the ApexInfo to make them available
  to the apex specific variants.
* Prevents the prebuilt_apex variants from being merged together as
  they will not be compatible.
* Modifies java_import to use the exported file for variants of a
  prebuilt_apex.
* Adds a ninja rule to unpack (using deapexer) the contents of the
  prebuilt_apex's apex file, verify that the required files are present
  and make them available as outputs for other rules to use.
* Some minor refactorings to support these changes.
* Adds tests to cover prebuilt only, prebuilt with source preferred,
  and prebuilt preferred with source.

Test: m nothing
Bug: 171061220
Change-Id: Ic9bed81fb65b92f0d59f64c0bce168a9ed44cfac
diff --git a/apex/Android.bp b/apex/Android.bp
index e3a547c..77dde72 100644
--- a/apex/Android.bp
+++ b/apex/Android.bp
@@ -18,6 +18,7 @@
         "apex.go",
         "apex_singleton.go",
         "builder.go",
+        "deapexer.go",
         "key.go",
         "prebuilt.go",
         "vndk.go",
diff --git a/apex/apex.go b/apex/apex.go
index 5cd18ed..376811a 100644
--- a/apex/apex.go
+++ b/apex/apex.go
@@ -1501,7 +1501,7 @@
 	return newApexFile(ctx, buildFile, buildFile.Base(), dirInApex, etc, fs)
 }
 
-// WalyPayloadDeps visits dependencies that contributes to the payload of this APEX. For each of the
+// WalkPayloadDeps visits dependencies that contributes to the payload of this APEX. For each of the
 // visited module, the `do` callback is executed. Returning true in the callback continues the visit
 // to the child modules. Returning false makes the visit to continue in the sibling or the parent
 // modules. This is used in check* functions below.
diff --git a/apex/apex_test.go b/apex/apex_test.go
index fc74672..58739b0 100644
--- a/apex/apex_test.go
+++ b/apex/apex_test.go
@@ -190,6 +190,7 @@
 		"testdata/baz":                               nil,
 		"AppSet.apks":                                nil,
 		"foo.rs":                                     nil,
+		"libfoo.jar":                                 nil,
 	}
 
 	cc.GatherRequiredFilesForTest(fs)
@@ -4219,6 +4220,121 @@
 	}
 }
 
+func TestPrebuiltExportDexImplementationJars(t *testing.T) {
+	transform := func(config *dexpreopt.GlobalConfig) {
+		config.BootJars = android.CreateTestConfiguredJarList([]string{"myapex:libfoo"})
+	}
+
+	checkDexJarBuildPath := func(ctx *android.TestContext, name string) {
+		// Make sure the import has been given the correct path to the dex jar.
+		p := ctx.ModuleForTests(name, "android_common_myapex").Module().(java.Dependency)
+		dexJarBuildPath := p.DexJarBuildPath()
+		if expected, actual := ".intermediates/myapex.deapexer/android_common/deapexer/javalib/libfoo.jar", android.NormalizePathForTesting(dexJarBuildPath); actual != expected {
+			t.Errorf("Incorrect DexJarBuildPath value '%s', expected '%s'", actual, expected)
+		}
+	}
+
+	ensureNoSourceVariant := func(ctx *android.TestContext) {
+		// Make sure that an apex variant is not created for the source module.
+		if expected, actual := []string{"android_common"}, ctx.ModuleVariantsForTests("libfoo"); !reflect.DeepEqual(expected, actual) {
+			t.Errorf("invalid set of variants for %q: expected %q, found %q", "libfoo", expected, actual)
+		}
+	}
+
+	t.Run("prebuilt only", func(t *testing.T) {
+		bp := `
+		prebuilt_apex {
+			name: "myapex",
+			arch: {
+				arm64: {
+					src: "myapex-arm64.apex",
+				},
+				arm: {
+					src: "myapex-arm.apex",
+				},
+			},
+			exported_java_libs: ["libfoo"],
+		}
+
+		java_import {
+			name: "libfoo",
+			jars: ["libfoo.jar"],
+		}
+	`
+
+		// Make sure that dexpreopt can access dex implementation files from the prebuilt.
+		ctx := testDexpreoptWithApexes(t, bp, "", transform)
+
+		checkDexJarBuildPath(ctx, "libfoo")
+	})
+
+	t.Run("prebuilt with source preferred", func(t *testing.T) {
+
+		bp := `
+		prebuilt_apex {
+			name: "myapex",
+			arch: {
+				arm64: {
+					src: "myapex-arm64.apex",
+				},
+				arm: {
+					src: "myapex-arm.apex",
+				},
+			},
+			exported_java_libs: ["libfoo"],
+		}
+
+		java_import {
+			name: "libfoo",
+			jars: ["libfoo.jar"],
+		}
+
+		java_library {
+			name: "libfoo",
+		}
+	`
+
+		// Make sure that dexpreopt can access dex implementation files from the prebuilt.
+		ctx := testDexpreoptWithApexes(t, bp, "", transform)
+
+		checkDexJarBuildPath(ctx, "prebuilt_libfoo")
+		ensureNoSourceVariant(ctx)
+	})
+
+	t.Run("prebuilt preferred with source", func(t *testing.T) {
+		bp := `
+		prebuilt_apex {
+			name: "myapex",
+			prefer: true,
+			arch: {
+				arm64: {
+					src: "myapex-arm64.apex",
+				},
+				arm: {
+					src: "myapex-arm.apex",
+				},
+			},
+			exported_java_libs: ["libfoo"],
+		}
+
+		java_import {
+			name: "libfoo",
+			jars: ["libfoo.jar"],
+		}
+
+		java_library {
+			name: "libfoo",
+		}
+	`
+
+		// Make sure that dexpreopt can access dex implementation files from the prebuilt.
+		ctx := testDexpreoptWithApexes(t, bp, "", transform)
+
+		checkDexJarBuildPath(ctx, "prebuilt_libfoo")
+		ensureNoSourceVariant(ctx)
+	})
+}
+
 func TestApexWithTests(t *testing.T) {
 	ctx, config := testApex(t, `
 		apex_test {
@@ -5783,7 +5899,7 @@
 	testDexpreoptWithApexes(t, bp, errmsg, transformDexpreoptConfig)
 }
 
-func testDexpreoptWithApexes(t *testing.T, bp, errmsg string, transformDexpreoptConfig func(*dexpreopt.GlobalConfig)) {
+func testDexpreoptWithApexes(t *testing.T, bp, errmsg string, transformDexpreoptConfig func(*dexpreopt.GlobalConfig)) *android.TestContext {
 	t.Helper()
 
 	bp += cc.GatherRequiredDepsForTest(android.Android)
@@ -5808,6 +5924,7 @@
 	ctx := android.NewTestArchContext(config)
 	ctx.RegisterModuleType("apex", BundleFactory)
 	ctx.RegisterModuleType("apex_key", ApexKeyFactory)
+	ctx.RegisterModuleType("prebuilt_apex", PrebuiltFactory)
 	ctx.RegisterModuleType("filegroup", android.FileGroupFactory)
 	ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators)
 	android.RegisterPrebuiltMutators(ctx)
@@ -5837,10 +5954,11 @@
 		android.FailIfErrored(t, errs)
 	} else if len(errs) > 0 {
 		android.FailIfNoMatchingErrors(t, errmsg, errs)
-		return
 	} else {
 		t.Fatalf("missing expected error %q (0 errors are returned)", errmsg)
 	}
+
+	return ctx
 }
 
 func TestUpdatable_should_set_min_sdk_version(t *testing.T) {
@@ -5939,6 +6057,56 @@
 		}
 		testNoUpdatableJarsInBootImage(t, "", transform)
 	})
+
+}
+
+func TestDexpreoptAccessDexFilesFromPrebuiltApex(t *testing.T) {
+	transform := func(config *dexpreopt.GlobalConfig) {
+		config.BootJars = android.CreateTestConfiguredJarList([]string{"myapex:libfoo"})
+	}
+	t.Run("prebuilt no source", func(t *testing.T) {
+		testDexpreoptWithApexes(t, `
+			prebuilt_apex {
+				name: "myapex" ,
+				arch: {
+					arm64: {
+						src: "myapex-arm64.apex",
+					},
+					arm: {
+						src: "myapex-arm.apex",
+					},
+				},
+			exported_java_libs: ["libfoo"],
+		}
+
+		java_import {
+			name: "libfoo",
+			jars: ["libfoo.jar"],
+		}
+`, "", transform)
+	})
+
+	t.Run("prebuilt no source", func(t *testing.T) {
+		testDexpreoptWithApexes(t, `
+			prebuilt_apex {
+				name: "myapex" ,
+				arch: {
+					arm64: {
+						src: "myapex-arm64.apex",
+					},
+					arm: {
+						src: "myapex-arm.apex",
+					},
+				},
+			exported_java_libs: ["libfoo"],
+		}
+
+		java_import {
+			name: "libfoo",
+			jars: ["libfoo.jar"],
+		}
+`, "", transform)
+	})
 }
 
 func testApexPermittedPackagesRules(t *testing.T, errmsg, bp string, apexBootJars []string, rules []android.Rule) {
diff --git a/apex/deapexer.go b/apex/deapexer.go
new file mode 100644
index 0000000..651cadf
--- /dev/null
+++ b/apex/deapexer.go
@@ -0,0 +1,139 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package apex
+
+import (
+	"android/soong/android"
+)
+
+// Contains 'deapexer' a private module type used by 'prebuilt_apex' to make dex files contained
+// within a .apex file referenced by `prebuilt_apex` available for use by their associated
+// `java_import` modules.
+//
+// An 'apex' module references `java_library` modules from which .dex files are obtained that are
+// stored in the resulting `.apex` file. The resulting `.apex` file is then made available as a
+// prebuilt by referencing it from a `prebuilt_apex`. For each such `java_library` that is used by
+// modules outside the `.apex` file a `java_import` prebuilt is made available referencing a jar
+// that contains the Java classes.
+//
+// When building a Java module type, e.g. `java_module` or `android_app` against such prebuilts the
+// `java_import` provides the classes jar  (jar containing `.class` files) against which the
+// module's `.java` files are compiled. That classes jar usually contains only stub classes. The
+// resulting classes jar is converted into a dex jar (jar containing `.dex` files). Then if
+// necessary the dex jar is further processed by `dexpreopt` to produce an optimized form of the
+// library specific to the current Android version. This process requires access to implementation
+// dex jars for each `java_import`. The `java_import` will obtain the implementation dex jar from
+// the `.apex` file in the associated `prebuilt_apex`.
+//
+// This is intentionally not registered by name as it is not intended to be used from within an
+// `Android.bp` file.
+
+// Properties that are specific to `deapexer` but which need to be provided on the `prebuilt_apex`
+// module.`
+type DeapexerProperties struct {
+	// List of java libraries that are embedded inside this prebuilt APEX bundle and for which this
+	// APEX bundle will provide dex implementation jars for use by dexpreopt and boot jars package
+	// check.
+	Exported_java_libs []string
+}
+
+type Deapexer struct {
+	android.ModuleBase
+	prebuilt android.Prebuilt
+
+	properties         DeapexerProperties
+	apexFileProperties ApexFileProperties
+
+	inputApex android.Path
+}
+
+func privateDeapexerFactory() android.Module {
+	module := &Deapexer{}
+	module.AddProperties(
+		&module.properties,
+		&module.apexFileProperties,
+	)
+	android.InitSingleSourcePrebuiltModule(module, &module.apexFileProperties, "Source")
+	android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon)
+	return module
+}
+
+func (p *Deapexer) Prebuilt() *android.Prebuilt {
+	return &p.prebuilt
+}
+
+func (p *Deapexer) Name() string {
+	return p.prebuilt.Name(p.ModuleBase.Name())
+}
+
+func (p *Deapexer) DepsMutator(ctx android.BottomUpMutatorContext) {
+	if err := p.apexFileProperties.selectSource(ctx); err != nil {
+		ctx.ModuleErrorf("%s", err)
+		return
+	}
+
+	// Add dependencies from the java modules to which this exports files from the `.apex` file onto
+	// this module so that they can access the `DeapexerInfo` object that this provides.
+	for _, lib := range p.properties.Exported_java_libs {
+		dep := prebuiltApexExportedModuleName(ctx, lib)
+		ctx.AddReverseDependency(ctx.Module(), android.DeapexerTag, dep)
+	}
+}
+
+func (p *Deapexer) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+	p.inputApex = p.Prebuilt().SingleSourcePath(ctx)
+
+	// Create and remember the directory into which the .apex file's contents will be unpacked.
+	deapexerOutput := android.PathForModuleOut(ctx, "deapexer")
+
+	exports := make(map[string]android.Path)
+
+	// Create mappings from name+tag to all the required exported paths.
+	for _, l := range p.properties.Exported_java_libs {
+		// Populate the exports that this makes available. The path here must match the path of the
+		// file in the APEX created by apexFileForJavaModule(...).
+		exports[l+"{.dexjar}"] = deapexerOutput.Join(ctx, "javalib", l+".jar")
+	}
+
+	// If the prebuilt_apex exports any files then create a build rule that unpacks the apex using
+	// deapexer and verifies that all the required files were created. Also, make the mapping from
+	// name+tag to path available for other modules.
+	if len(exports) > 0 {
+		// Make the information available for other modules.
+		ctx.SetProvider(android.DeapexerProvider, android.NewDeapexerInfo(exports))
+
+		// Create a sorted list of the files that this exports.
+		exportedPaths := make(android.Paths, 0, len(exports))
+		for _, p := range exports {
+			exportedPaths = append(exportedPaths, p)
+		}
+		exportedPaths = android.SortedUniquePaths(exportedPaths)
+
+		// The apex needs to export some files so create a ninja rule to unpack the apex and check that
+		// the required files are present.
+		builder := android.NewRuleBuilder(pctx, ctx)
+		command := builder.Command()
+		command.
+			Tool(android.PathForSource(ctx, "build/soong/scripts/unpack-prebuilt-apex.sh")).
+			BuiltTool("deapexer").
+			BuiltTool("debugfs").
+			Input(p.inputApex).
+			Text(deapexerOutput.String())
+		for _, p := range exportedPaths {
+			command.Output(p.(android.WritablePath))
+		}
+		builder.Build("deapexer", "deapex "+ctx.ModuleName())
+	}
+}
diff --git a/apex/prebuilt.go b/apex/prebuilt.go
index 50e892e..c72a9eb 100644
--- a/apex/prebuilt.go
+++ b/apex/prebuilt.go
@@ -158,6 +158,7 @@
 
 type PrebuiltProperties struct {
 	ApexFileProperties
+	DeapexerProperties
 
 	Installable *bool
 	// Optional name for the installed apex. If unspecified, name of the
@@ -198,19 +199,152 @@
 }
 
 // prebuilt_apex imports an `.apex` file into the build graph as if it was built with apex.
+//
+// If this needs to make files from within a `.apex` file available for use by other Soong modules,
+// e.g. make dex implementation jars available for java_import modules isted in exported_java_libs,
+// it does so as follows:
+//
+// 1. It creates a `deapexer` module that actually extracts the files from the `.apex` file and
+//    makes them available for use by other modules, at both Soong and ninja levels.
+//
+// 2. It adds a dependency onto those modules and creates an apex specific variant similar to what
+//    an `apex` module does. That ensures that code which looks for specific apex variant, e.g.
+//    dexpreopt, will work the same way from source and prebuilt.
+//
+// 3. The `deapexer` module adds a dependency from the modules that require the exported files onto
+//    itself so that they can retrieve the file paths to those files.
+//
 func PrebuiltFactory() android.Module {
 	module := &Prebuilt{}
 	module.AddProperties(&module.properties)
 	android.InitSingleSourcePrebuiltModule(module, &module.properties, "Source")
 	android.InitAndroidMultiTargetsArchModule(module, android.DeviceSupported, android.MultilibCommon)
+
+	android.AddLoadHook(module, func(ctx android.LoadHookContext) {
+		props := struct {
+			Name *string
+		}{
+			Name: proptools.StringPtr(module.BaseModuleName() + ".deapexer"),
+		}
+		ctx.CreateModule(privateDeapexerFactory,
+			&props,
+			&module.properties.ApexFileProperties,
+			&module.properties.DeapexerProperties,
+		)
+	})
+
 	return module
 }
 
+func prebuiltApexExportedModuleName(ctx android.BottomUpMutatorContext, name string) string {
+	// The prebuilt_apex should be depending on prebuilt modules but as this runs after
+	// prebuilt_rename the prebuilt module may or may not be using the prebuilt_ prefixed named. So,
+	// check to see if the prefixed name is in use first, if it is then use that, otherwise assume
+	// the unprefixed name is the one to use. If the unprefixed one turns out to be a source module
+	// and not a renamed prebuilt module then that will be detected and reported as an error when
+	// processing the dependency in ApexInfoMutator().
+	prebuiltName := "prebuilt_" + name
+	if ctx.OtherModuleExists(prebuiltName) {
+		name = prebuiltName
+	}
+	return name
+}
+
 func (p *Prebuilt) DepsMutator(ctx android.BottomUpMutatorContext) {
 	if err := p.properties.selectSource(ctx); err != nil {
 		ctx.ModuleErrorf("%s", err)
 		return
 	}
+
+	// Add dependencies onto the java modules that represent the java libraries that are provided by
+	// and exported from this prebuilt apex.
+	for _, lib := range p.properties.Exported_java_libs {
+		dep := prebuiltApexExportedModuleName(ctx, lib)
+		ctx.AddFarVariationDependencies(ctx.Config().AndroidCommonTarget.Variations(), javaLibTag, dep)
+	}
+}
+
+var _ ApexInfoMutator = (*Prebuilt)(nil)
+
+// ApexInfoMutator marks any modules for which this apex exports a file as requiring an apex
+// specific variant and checks that they are supported.
+//
+// The apexMutator will ensure that the ApexInfo objects passed to BuildForApex(ApexInfo) are
+// associated with the apex specific variant using the ApexInfoProvider for later retrieval.
+//
+// Unlike the source apex module type the prebuilt_apex module type cannot share compatible variants
+// across prebuilt_apex modules. That is because there is no way to determine whether two
+// prebuilt_apex modules that export files for the same module are compatible. e.g. they could have
+// been built from different source at different times or they could have been built with different
+// build options that affect the libraries.
+//
+// While it may be possible to provide sufficient information to determine whether two prebuilt_apex
+// modules were compatible it would be a lot of work and would not provide much benefit for a couple
+// of reasons:
+// * The number of prebuilt_apex modules that will be exporting files for the same module will be
+//   low as the prebuilt_apex only exports files for the direct dependencies that require it and
+//   very few modules are direct dependencies of multiple prebuilt_apex modules, e.g. there are a
+//   few com.android.art* apex files that contain the same contents and could export files for the
+//   same modules but only one of them needs to do so. Contrast that with source apex modules which
+//   need apex specific variants for every module that contributes code to the apex, whether direct
+//   or indirect.
+// * The build cost of a prebuilt_apex variant is generally low as at worst it will involve some
+//   extra copying of files. Contrast that with source apex modules that has to build each variant
+//   from source.
+func (p *Prebuilt) ApexInfoMutator(mctx android.TopDownMutatorContext) {
+
+	// Collect direct dependencies into contents.
+	contents := make(map[string]android.ApexMembership)
+
+	// Collect the list of dependencies.
+	var dependencies []android.ApexModule
+	mctx.VisitDirectDeps(func(m android.Module) {
+		tag := mctx.OtherModuleDependencyTag(m)
+		if tag == javaLibTag {
+			depName := mctx.OtherModuleName(m)
+
+			// It is an error if the other module is not a prebuilt.
+			if _, ok := m.(android.PrebuiltInterface); !ok {
+				mctx.PropertyErrorf("exported_java_libs", "%q is not a prebuilt module", depName)
+				return
+			}
+
+			// It is an error if the other module is not an ApexModule.
+			if _, ok := m.(android.ApexModule); !ok {
+				mctx.PropertyErrorf("exported_java_libs", "%q is not usable within an apex", depName)
+				return
+			}
+
+			// Strip off the prebuilt_ prefix if present before storing content to ensure consistent
+			// behavior whether there is a corresponding source module present or not.
+			depName = android.RemoveOptionalPrebuiltPrefix(depName)
+
+			// Remember that this module was added as a direct dependency.
+			contents[depName] = contents[depName].Add(true)
+
+			// Add the module to the list of dependencies that need to have an APEX variant.
+			dependencies = append(dependencies, m.(android.ApexModule))
+		}
+	})
+
+	// Create contents for the prebuilt_apex and store it away for later use.
+	apexContents := android.NewApexContents(contents)
+	mctx.SetProvider(ApexBundleInfoProvider, ApexBundleInfo{
+		Contents: apexContents,
+	})
+
+	// Create an ApexInfo for the prebuilt_apex.
+	apexInfo := android.ApexInfo{
+		ApexVariationName: mctx.ModuleName(),
+		InApexes:          []string{mctx.ModuleName()},
+		ApexContents:      []*android.ApexContents{apexContents},
+		ForPrebuiltApex:   true,
+	}
+
+	// Mark the dependencies of this module as requiring a variant for this module.
+	for _, am := range dependencies {
+		am.BuildForApex(apexInfo)
+	}
 }
 
 func (p *Prebuilt) GenerateAndroidBuildActions(ctx android.ModuleContext) {