Move boot jars package check from make
Adds a singleton that traverses the module variants finding the ones
that are in the list (updatable and non-updatable) of boot jars and
add a ninja rule to ensure that they only contain packages from an
allowed list.
Replaces a hack that ignored any prebuilt boot jars supplied as dex
file with an equivalent one to ensure that they are still ignored.
A follow up change that switches to checking dex jars will allow the
hack to be removed.
The boot jars check can be strict or lax. If strict then all the boot
jars listed in the configuration must be found, otherwise it will only
check the ones it finds. It is strict by default unless
TARGET_BUILD_UNBUNDLED=true or ALLOW_MISSING_DEPENDENCIES=true.
Moves the script and data file from build/make.
Test: m check-boot-jars - for failing and passing cases
SKIP_BOOT_JARS_CHECK=true - no check-boot-jars target created
ALLOW_MISSING_DEPENDENCIES=true - not strict
TARGET_BUILD_UNBUNDLED=true - not strict
verified manually that apart from path differences the same
files (same check sum) were checked in both old make checks and
the new Soong ones
EMMA_INSTRUMENT=true EMMA_INSTRUMENT_FRAMEWORK=true m check-boot-jars
Bug: 171479578
Change-Id: I9d81d6650ba64fc0d48d2dab4ba5a3ba8dd03dec
diff --git a/android/apex.go b/android/apex.go
index 3039e79..e70ec4f 100644
--- a/android/apex.go
+++ b/android/apex.go
@@ -67,6 +67,15 @@
return i.ApexVariationName == ""
}
+func (i ApexInfo) InApex(apex string) bool {
+ for _, a := range i.InApexes {
+ if a == apex {
+ return true
+ }
+ }
+ return false
+}
+
// ApexTestForInfo stores the contents of APEXes for which this module is a test and thus has
// access to APEX internals.
type ApexTestForInfo struct {
diff --git a/android/config.go b/android/config.go
index dbae7f7..a499057 100644
--- a/android/config.go
+++ b/android/config.go
@@ -793,6 +793,11 @@
return Bool(c.productVariables.Always_use_prebuilt_sdks)
}
+// Returns true if the boot jars check should be skipped.
+func (c *config) SkipBootJarsCheck() bool {
+ return Bool(c.productVariables.Skip_boot_jars_check)
+}
+
func (c *config) Fuchsia() bool {
return Bool(c.productVariables.Fuchsia)
}
@@ -1341,6 +1346,11 @@
return l.jars[idx]
}
+// Apex component of idx-th pair on the list.
+func (l *ConfiguredJarList) Apex(idx int) string {
+ return l.apexes[idx]
+}
+
// If the list contains a pair with the given jar.
func (l *ConfiguredJarList) ContainsJar(jar string) bool {
return InList(jar, l.jars)
@@ -1538,3 +1548,11 @@
return list
}).([]string)
}
+
+func (c *config) NonUpdatableBootJars() ConfiguredJarList {
+ return c.productVariables.BootJars
+}
+
+func (c *config) UpdatableBootJars() ConfiguredJarList {
+ return c.productVariables.UpdatableBootJars
+}
diff --git a/android/variable.go b/android/variable.go
index 7999f0f..a9a9c87 100644
--- a/android/variable.go
+++ b/android/variable.go
@@ -224,6 +224,7 @@
Unbundled_build *bool `json:",omitempty"`
Unbundled_build_apps *bool `json:",omitempty"`
Always_use_prebuilt_sdks *bool `json:",omitempty"`
+ Skip_boot_jars_check *bool `json:",omitempty"`
Malloc_not_svelte *bool `json:",omitempty"`
Malloc_zero_contents *bool `json:",omitempty"`
Malloc_pattern_fill_contents *bool `json:",omitempty"`
diff --git a/java/Android.bp b/java/Android.bp
index 92e8ca4..9e8dc78 100644
--- a/java/Android.bp
+++ b/java/Android.bp
@@ -22,6 +22,7 @@
"androidmk.go",
"app_builder.go",
"app.go",
+ "boot_jars.go",
"builder.go",
"device_host_converter.go",
"dex.go",
diff --git a/java/androidmk.go b/java/androidmk.go
index c21c83a..e1a661f 100644
--- a/java/androidmk.go
+++ b/java/androidmk.go
@@ -217,10 +217,6 @@
func(entries *android.AndroidMkEntries) {
if prebuilt.dexJarFile != nil {
entries.SetPath("LOCAL_SOONG_DEX_JAR", prebuilt.dexJarFile)
- // TODO(b/125517186): export the dex jar as a classes jar to match some mis-uses in Make until
- // boot_jars_package_check.mk can check dex jars.
- entries.SetPath("LOCAL_SOONG_HEADER_JAR", prebuilt.dexJarFile)
- entries.SetPath("LOCAL_SOONG_CLASSES_JAR", prebuilt.dexJarFile)
}
if len(prebuilt.dexpreopter.builtInstalled) > 0 {
entries.SetString("LOCAL_SOONG_BUILT_INSTALLED", prebuilt.dexpreopter.builtInstalled)
diff --git a/java/boot_jars.go b/java/boot_jars.go
new file mode 100644
index 0000000..900eb7a
--- /dev/null
+++ b/java/boot_jars.go
@@ -0,0 +1,123 @@
+// Copyright 2020 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package java
+
+import (
+ "android/soong/android"
+)
+
+func init() {
+ android.RegisterSingletonType("boot_jars", bootJarsSingletonFactory)
+}
+
+func bootJarsSingletonFactory() android.Singleton {
+ return &bootJarsSingleton{}
+}
+
+type bootJarsSingleton struct{}
+
+func populateMapFromConfiguredJarList(ctx android.SingletonContext, moduleToApex map[string]string, list android.ConfiguredJarList, name string) bool {
+ for i := 0; i < list.Len(); i++ {
+ module := list.Jar(i)
+ // Ignore jacocoagent it is only added when instrumenting and so has no impact on
+ // app compatibility.
+ if module == "jacocoagent" {
+ continue
+ }
+ apex := list.Apex(i)
+ if existing, ok := moduleToApex[module]; ok {
+ ctx.Errorf("Configuration property %q is invalid as it contains multiple references to module (%s) in APEXes (%s and %s)",
+ module, existing, apex)
+ return false
+ }
+
+ moduleToApex[module] = apex
+ }
+
+ return true
+}
+
+func (b *bootJarsSingleton) GenerateBuildActions(ctx android.SingletonContext) {
+ config := ctx.Config()
+ if config.SkipBootJarsCheck() {
+ return
+ }
+
+ // Populate a map from module name to APEX from the boot jars. If there is a problem
+ // such as duplicate modules then fail and return immediately.
+ moduleToApex := make(map[string]string)
+ if !populateMapFromConfiguredJarList(ctx, moduleToApex, config.NonUpdatableBootJars(), "BootJars") ||
+ !populateMapFromConfiguredJarList(ctx, moduleToApex, config.UpdatableBootJars(), "UpdatableBootJars") {
+ return
+ }
+
+ // Map from module name to the correct apex variant.
+ nameToApexVariant := make(map[string]android.Module)
+
+ // Scan all the modules looking for the module/apex variants corresponding to the
+ // boot jars.
+ ctx.VisitAllModules(func(module android.Module) {
+ name := ctx.ModuleName(module)
+ if apex, ok := moduleToApex[name]; ok {
+ apexInfo := ctx.ModuleProvider(module, android.ApexInfoProvider).(android.ApexInfo)
+ if (apex == "platform" && apexInfo.IsForPlatform()) || apexInfo.InApex(apex) {
+ // The module name/apex variant should be unique in the system but double check
+ // just in case something has gone wrong.
+ if existing, ok := nameToApexVariant[name]; ok {
+ ctx.Errorf("found multiple variants matching %s:%s: %q and %q", apex, name, existing, module)
+ }
+ nameToApexVariant[name] = module
+ }
+ }
+ })
+
+ timestamp := android.PathForOutput(ctx, "boot-jars-package-check/stamp")
+
+ rule := android.NewRuleBuilder()
+ checkBootJars := rule.Command().BuiltTool(ctx, "check_boot_jars").
+ Input(android.PathForSource(ctx, "build/soong/scripts/check_boot_jars/package_allowed_list.txt"))
+
+ // If this is not an unbundled build and missing dependencies are not allowed
+ // then all the boot jars listed must have been found.
+ strict := !config.UnbundledBuild() && !config.AllowMissingDependencies()
+
+ // Iterate over the module names on the boot classpath in order
+ for _, name := range android.SortedStringKeys(moduleToApex) {
+ if apexVariant, ok := nameToApexVariant[name]; ok {
+ if dep, ok := apexVariant.(Dependency); ok {
+ // Add the implementation jars for the module to be checked. This uses implementation
+ // and resources jar as that is what the previous make based check uses.
+ for _, jar := range dep.ImplementationAndResourcesJars() {
+ checkBootJars.Input(jar)
+ }
+ } else if _, ok := apexVariant.(*DexImport); ok {
+ // TODO(b/171479578): ignore deximport when doing package check until boot_jars.go can check dex jars.
+ } else {
+ ctx.Errorf("module %q is of type %q which is not supported as a boot jar", name, ctx.ModuleType(apexVariant))
+ }
+ } else if strict {
+ ctx.Errorf("boot jars package check failed as it could not find module %q for apex %q", name, moduleToApex[name])
+ }
+ }
+
+ checkBootJars.Text("&& touch").Output(timestamp)
+ rule.Build(pctx, ctx, "boot_jars_package_check", "check boot jar packages")
+
+ // The check-boot-jars phony target depends on the timestamp created if the check succeeds.
+ ctx.Phony("check-boot-jars", timestamp)
+
+ // The droidcore phony target depends on the check-boot-jars phony target
+ ctx.Phony("droidcore", android.PathForPhony(ctx, "check-boot-jars"))
+}
diff --git a/scripts/Android.bp b/scripts/Android.bp
index 92f5c53..dd03f28 100644
--- a/scripts/Android.bp
+++ b/scripts/Android.bp
@@ -1,4 +1,20 @@
python_binary_host {
+ name: "check_boot_jars",
+ main: "check_boot_jars/check_boot_jars.py",
+ srcs: [
+ "check_boot_jars/check_boot_jars.py",
+ ],
+ version: {
+ py2: {
+ enabled: true,
+ },
+ py3: {
+ enabled: false,
+ },
+ },
+}
+
+python_binary_host {
name: "manifest_fixer",
main: "manifest_fixer.py",
srcs: [
diff --git a/scripts/check_boot_jars/check_boot_jars.py b/scripts/check_boot_jars/check_boot_jars.py
new file mode 100755
index 0000000..cf4ef27
--- /dev/null
+++ b/scripts/check_boot_jars/check_boot_jars.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+
+"""
+Check boot jars.
+
+Usage: check_boot_jars.py <package_allow_list_file> <jar1> <jar2> ...
+"""
+import logging
+import os.path
+import re
+import subprocess
+import sys
+
+
+# The compiled allow list RE.
+allow_list_re = None
+
+
+def LoadAllowList(filename):
+ """ Load and compile allow list regular expressions from filename.
+ """
+ lines = []
+ with open(filename, 'r') as f:
+ for line in f:
+ line = line.strip()
+ if not line or line.startswith('#'):
+ continue
+ lines.append(line)
+ combined_re = r'^(%s)$' % '|'.join(lines)
+ global allow_list_re
+ try:
+ allow_list_re = re.compile(combined_re)
+ except re.error:
+ logging.exception(
+ 'Cannot compile package allow list regular expression: %r',
+ combined_re)
+ allow_list_re = None
+ return False
+ return True
+
+
+def CheckJar(allow_list_path, jar):
+ """Check a jar file.
+ """
+ # Get the list of files inside the jar file.
+ p = subprocess.Popen(args='jar tf %s' % jar,
+ stdout=subprocess.PIPE, shell=True)
+ stdout, _ = p.communicate()
+ if p.returncode != 0:
+ return False
+ items = stdout.split()
+ classes = 0
+ for f in items:
+ if f.endswith('.class'):
+ classes += 1
+ package_name = os.path.dirname(f)
+ package_name = package_name.replace('/', '.')
+ if not package_name or not allow_list_re.match(package_name):
+ print >> sys.stderr, ('Error: %s contains class file %s, whose package name %s is empty or'
+ ' not in the allow list %s of packages allowed on the bootclasspath.'
+ % (jar, f, package_name, allow_list_path))
+ return False
+ if classes == 0:
+ print >> sys.stderr, ('Error: %s does not contain any class files.' % jar)
+ return False
+ return True
+
+
+def main(argv):
+ if len(argv) < 2:
+ print __doc__
+ return 1
+ allow_list_path = argv[0]
+
+ if not LoadAllowList(allow_list_path):
+ return 1
+
+ passed = True
+ for jar in argv[1:]:
+ if not CheckJar(allow_list_path, jar):
+ passed = False
+ if not passed:
+ return 1
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/scripts/check_boot_jars/package_allowed_list.txt b/scripts/check_boot_jars/package_allowed_list.txt
new file mode 100644
index 0000000..18ab427
--- /dev/null
+++ b/scripts/check_boot_jars/package_allowed_list.txt
@@ -0,0 +1,248 @@
+# Boot jar package name allowed list.
+# Each line is interpreted as a regular expression.
+
+###################################################
+# core-libart.jar & core-oj.jar
+java\.awt\.font
+java\.beans
+java\.io
+java\.lang
+java\.lang\.annotation
+java\.lang\.invoke
+java\.lang\.ref
+java\.lang\.reflect
+java\.math
+java\.net
+java\.nio
+java\.nio\.file
+java\.nio\.file\.spi
+java\.nio\.file\.attribute
+java\.nio\.channels
+java\.nio\.channels\.spi
+java\.nio\.charset
+java\.nio\.charset\.spi
+java\.security
+java\.security\.acl
+java\.security\.cert
+java\.security\.interfaces
+java\.security\.spec
+java\.sql
+java\.text
+java\.text\.spi
+java\.time
+java\.time\.chrono
+java\.time\.format
+java\.time\.temporal
+java\.time\.zone
+java\.util
+java\.util\.concurrent
+java\.util\.concurrent\.atomic
+java\.util\.concurrent\.locks
+java\.util\.function
+java\.util\.jar
+java\.util\.logging
+java\.util\.prefs
+java\.util\.regex
+java\.util\.spi
+java\.util\.stream
+java\.util\.zip
+# TODO: Remove javax.annotation.processing if possible, see http://b/132338110:
+javax\.annotation\.processing
+javax\.crypto
+javax\.crypto\.interfaces
+javax\.crypto\.spec
+javax\.net
+javax\.net\.ssl
+javax\.security\.auth
+javax\.security\.auth\.callback
+javax\.security\.auth\.login
+javax\.security\.auth\.x500
+javax\.security\.cert
+javax\.sql
+javax\.xml
+javax\.xml\.datatype
+javax\.xml\.namespace
+javax\.xml\.parsers
+javax\.xml\.transform
+javax\.xml\.transform\.dom
+javax\.xml\.transform\.sax
+javax\.xml\.transform\.stream
+javax\.xml\.validation
+javax\.xml\.xpath
+jdk\.internal\.util
+jdk\.internal\.vm\.annotation
+jdk\.net
+org\.w3c\.dom
+org\.w3c\.dom\.ls
+org\.w3c\.dom\.traversal
+# OpenJdk internal implementation.
+sun\.invoke\.util
+sun\.invoke\.empty
+sun\.misc
+sun\.util.*
+sun\.text.*
+sun\.security.*
+sun\.reflect.*
+sun\.nio.*
+sun\.net.*
+com\.sun\..*
+
+# TODO: Move these internal org.apache.harmony classes to libcore.*
+org\.apache\.harmony\.crypto\.internal
+org\.apache\.harmony\.dalvik
+org\.apache\.harmony\.dalvik\.ddmc
+org\.apache\.harmony\.luni\.internal\.util
+org\.apache\.harmony\.security
+org\.apache\.harmony\.security\.asn1
+org\.apache\.harmony\.security\.fortress
+org\.apache\.harmony\.security\.pkcs10
+org\.apache\.harmony\.security\.pkcs7
+org\.apache\.harmony\.security\.pkcs8
+org\.apache\.harmony\.security\.provider\.crypto
+org\.apache\.harmony\.security\.utils
+org\.apache\.harmony\.security\.x501
+org\.apache\.harmony\.security\.x509
+org\.apache\.harmony\.security\.x509\.tsp
+org\.apache\.harmony\.xml
+org\.apache\.harmony\.xml\.dom
+org\.apache\.harmony\.xml\.parsers
+
+org\.json
+org\.xmlpull\.v1
+org\.xmlpull\.v1\.sax2
+
+# TODO: jarjar org.kxml2.io to com.android org\.kxml2\.io
+org\.kxml2\.io
+org\.xml
+org\.xml\.sax
+org\.xml\.sax\.ext
+org\.xml\.sax\.helpers
+
+dalvik\..*
+libcore\..*
+android\..*
+com\.android\..*
+###################################################
+# android.test.base.jar
+junit\.extensions
+junit\.framework
+android\.test
+android\.test\.suitebuilder\.annotation
+
+
+###################################################
+# ext.jar
+# TODO: jarjar javax.sip to com.android
+javax\.sip
+javax\.sip\.address
+javax\.sip\.header
+javax\.sip\.message
+
+# TODO: jarjar org.apache.commons to com.android
+org\.apache\.commons\.codec
+org\.apache\.commons\.codec\.binary
+org\.apache\.commons\.codec\.language
+org\.apache\.commons\.codec\.net
+org\.apache\.commons\.logging
+org\.apache\.commons\.logging\.impl
+org\.apache\.http
+org\.apache\.http\.auth
+org\.apache\.http\.auth\.params
+org\.apache\.http\.client
+org\.apache\.http\.client\.entity
+org\.apache\.http\.client\.methods
+org\.apache\.http\.client\.params
+org\.apache\.http\.client\.protocol
+org\.apache\.http\.client\.utils
+org\.apache\.http\.conn
+org\.apache\.http\.conn\.params
+org\.apache\.http\.conn\.routing
+org\.apache\.http\.conn\.scheme
+org\.apache\.http\.conn\.ssl
+org\.apache\.http\.conn\.util
+org\.apache\.http\.cookie
+org\.apache\.http\.cookie\.params
+org\.apache\.http\.entity
+org\.apache\.http\.impl
+org\.apache\.http\.impl\.auth
+org\.apache\.http\.impl\.client
+org\.apache\.http\.impl\.client
+org\.apache\.http\.impl\.conn
+org\.apache\.http\.impl\.conn\.tsccm
+org\.apache\.http\.impl\.cookie
+org\.apache\.http\.impl\.entity
+org\.apache\.http\.impl\.io
+org\.apache\.http\.impl\.io
+org\.apache\.http\.io
+org\.apache\.http\.message
+org\.apache\.http\.params
+org\.apache\.http\.protocol
+org\.apache\.http\.util
+
+# TODO: jarjar gov.nist to com.android
+gov\.nist\.core
+gov\.nist\.core\.net
+gov\.nist\.javax\.sip
+gov\.nist\.javax\.sip\.address
+gov\.nist\.javax\.sip\.clientauthutils
+gov\.nist\.javax\.sip\.header
+gov\.nist\.javax\.sip\.header\.extensions
+gov\.nist\.javax\.sip\.header\.ims
+gov\.nist\.javax\.sip\.message
+gov\.nist\.javax\.sip\.parser
+gov\.nist\.javax\.sip\.parser\.extensions
+gov\.nist\.javax\.sip\.parser\.ims
+gov\.nist\.javax\.sip\.stack
+
+org\.ccil\.cowan\.tagsoup
+org\.ccil\.cowan\.tagsoup\.jaxp
+
+###################################################
+# framework.jar
+javax\.microedition\.khronos\.opengles
+javax\.microedition\.khronos\.egl
+
+android
+
+###################################################
+# apache-xml.jar
+org\.apache\.xml\.res
+org\.apache\.xml\.utils
+org\.apache\.xml\.utils\.res
+org\.apache\.xml\.dtm
+org\.apache\.xml\.dtm\.ref
+org\.apache\.xml\.dtm\.ref\.dom2dtm
+org\.apache\.xml\.dtm\.ref\.sax2dtm
+org\.apache\.xml\.serializer
+org\.apache\.xml\.serializer\.utils
+org\.apache\.xml\.serializer\.dom3
+org\.apache\.xpath
+org\.apache\.xpath\.operations
+org\.apache\.xpath\.domapi
+org\.apache\.xpath\.functions
+org\.apache\.xpath\.res
+org\.apache\.xpath\.axes
+org\.apache\.xpath\.objects
+org\.apache\.xpath\.patterns
+org\.apache\.xpath\.jaxp
+org\.apache\.xpath\.compiler
+org\.apache\.xalan
+org\.apache\.xalan\.res
+org\.apache\.xalan\.templates
+org\.apache\.xalan\.serialize
+org\.apache\.xalan\.extensions
+org\.apache\.xalan\.processor
+org\.apache\.xalan\.transformer
+org\.apache\.xalan\.xslt
+
+###################################################
+# Packages in the google namespace across all bootclasspath jars.
+com\.google\.android\..*
+com\.google\.vr\.platform.*
+com\.google\.i18n\.phonenumbers\..*
+com\.google\.i18n\.phonenumbers
+
+###################################################
+# Packages used for Android in Chrome OS
+org\.chromium\.arc
+org\.chromium\.arc\..*