blob: 0ea32b869fc00a33f5d8a1808e27d25ed90d360b [file] [log] [blame]
Colin Cross43f08db2018-11-12 10:13:39 -08001// Copyright 2018 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// The dexpreopt package converts a global dexpreopt config and a module dexpreopt config into rules to perform
16// dexpreopting and to strip the dex files from the APK or JAR.
17//
18// It is used in two places; in the dexpeopt_gen binary for modules defined in Make, and directly linked into Soong.
19//
20// For Make modules it is built into the dexpreopt_gen binary, which is executed as a Make rule using global config and
21// module config specified in JSON files. The binary writes out two shell scripts, only updating them if they have
22// changed. One script takes an APK or JAR as an input and produces a zip file containing any outputs of preopting,
23// in the location they should be on the device. The Make build rules will unzip the zip file into $(PRODUCT_OUT) when
24// installing the APK, which will install the preopt outputs into $(PRODUCT_OUT)/system or $(PRODUCT_OUT)/system_other
25// as necessary. The zip file may be empty if preopting was disabled for any reason. The second script takes an APK or
26// JAR as an input and strips the dex files in it as necessary.
27//
28// The intermediate shell scripts allow changes to this package or to the global config to regenerate the shell scripts
29// but only require re-executing preopting if the script has changed.
30//
31// For Soong modules this package is linked directly into Soong and run from the java package. It generates the same
32// commands as for make, using athe same global config JSON file used by make, but using a module config structure
33// provided by Soong. The generated commands are then converted into Soong rule and written directly to the ninja file,
34// with no extra shell scripts involved.
35package dexpreopt
36
37import (
38 "fmt"
39 "path/filepath"
40 "strings"
41
42 "github.com/google/blueprint/pathtools"
43)
44
45const SystemPartition = "/system/"
46const SystemOtherPartition = "/system_other/"
47
48// GenerateStripRule generates a set of commands that will take an APK or JAR as an input and strip the dex files if
49// they are no longer necessary after preopting.
50func GenerateStripRule(global GlobalConfig, module ModuleConfig) (rule *Rule, err error) {
51 defer func() {
52 if r := recover(); r != nil {
53 if e, ok := r.(error); ok {
54 err = e
55 rule = nil
56 } else {
57 panic(r)
58 }
59 }
60 }()
61
62 tools := global.Tools
63
64 rule = &Rule{}
65
66 strip := shouldStripDex(module, global)
67
68 if strip {
69 // Only strips if the dex files are not already uncompressed
70 rule.Command().
71 Textf(`if (zipinfo %s '*.dex' 2>/dev/null | grep -v ' stor ' >/dev/null) ; then`, module.StripInputPath).
72 Tool(tools.Zip2zip).FlagWithInput("-i ", module.StripInputPath).FlagWithOutput("-o ", module.StripOutputPath).
73 FlagWithArg("-x ", `"classes*.dex"`).
74 Textf(`; else cp -f %s %s; fi`, module.StripInputPath, module.StripOutputPath)
75 } else {
76 rule.Command().Text("cp -f").Input(module.StripInputPath).Output(module.StripOutputPath)
77 }
78
79 return rule, nil
80}
81
82// GenerateDexpreoptRule generates a set of commands that will preopt a module based on a GlobalConfig and a
83// ModuleConfig. The produced files and their install locations will be available through rule.Installs().
84func GenerateDexpreoptRule(global GlobalConfig, module ModuleConfig) (rule *Rule, err error) {
85 defer func() {
86 if r := recover(); r != nil {
87 if e, ok := r.(error); ok {
88 err = e
89 rule = nil
90 } else {
91 panic(r)
92 }
93 }
94 }()
95
96 rule = &Rule{}
97
98 dexpreoptDisabled := contains(global.DisablePreoptModules, module.Name)
99
100 if contains(global.BootJars, module.Name) {
101 // Don't preopt individual boot jars, they will be preopted together
102 dexpreoptDisabled = true
103 }
104
105 // If OnlyPreoptBootImageAndSystemServer=true and module is not in boot class path skip
106 // Also preopt system server jars since selinux prevents system server from loading anything from
107 // /data. If we don't do this they will need to be extracted which is not favorable for RAM usage
108 // or performance. If PreoptExtractedApk is true, we ignore the only preopt boot image options.
109 if global.OnlyPreoptBootImageAndSystemServer && !contains(global.BootJars, module.Name) &&
110 !contains(global.SystemServerJars, module.Name) && !module.PreoptExtractedApk {
111 dexpreoptDisabled = true
112 }
113
114 generateProfile := module.ProfileClassListing != "" && !global.DisableGenerateProfile
115
116 var profile string
117 if generateProfile {
118 profile = profileCommand(global, module, rule)
119 }
120
121 if !dexpreoptDisabled {
122 appImage := (generateProfile || module.ForceCreateAppImage || global.DefaultAppImages) &&
123 !module.NoCreateAppImage
124
125 generateDM := shouldGenerateDM(module, global)
126
127 for _, arch := range module.Archs {
128 imageLocation := module.DexPreoptImageLocation
129 if imageLocation == "" {
130 imageLocation = global.DefaultDexPreoptImageLocation[arch]
131 }
132 dexpreoptCommand(global, module, rule, profile, arch, imageLocation, appImage, generateDM)
133 }
134 }
135
136 return rule, nil
137}
138
139func profileCommand(global GlobalConfig, module ModuleConfig, rule *Rule) string {
140 profilePath := filepath.Join(filepath.Dir(module.BuildPath), "profile.prof")
141 profileInstalledPath := module.DexLocation + ".prof"
142
143 if !module.ProfileIsTextListing {
144 rule.Command().FlagWithOutput("touch ", profilePath)
145 }
146
147 cmd := rule.Command().
148 Text(`ANDROID_LOG_TAGS="*:e"`).
149 Tool(global.Tools.Profman)
150
151 if module.ProfileIsTextListing {
152 // The profile is a test listing of classes (used for framework jars).
153 // We need to generate the actual binary profile before being able to compile.
154 cmd.FlagWithInput("--create-profile-from=", module.ProfileClassListing)
155 } else {
156 // The profile is binary profile (used for apps). Run it through profman to
157 // ensure the profile keys match the apk.
158 cmd.
159 Flag("--copy-and-update-profile-key").
160 FlagWithInput("--profile-file=", module.ProfileClassListing)
161 }
162
163 cmd.
164 FlagWithInput("--apk=", module.DexPath).
165 Flag("--dex-location="+module.DexLocation).
166 FlagWithOutput("--reference-profile-file=", profilePath)
167
168 if !module.ProfileIsTextListing {
169 cmd.Text(fmt.Sprintf(`|| echo "Profile out of date for %s"`, module.DexPath))
170 }
171 rule.Install(profilePath, profileInstalledPath)
172
173 return profilePath
174}
175
176func dexpreoptCommand(global GlobalConfig, module ModuleConfig, rule *Rule, profile, arch, bootImageLocation string,
177 appImage, generateDM bool) {
178
179 // HACK: make soname in Soong-generated .odex files match Make.
180 base := filepath.Base(module.DexLocation)
181 if filepath.Ext(base) == ".jar" {
182 base = "javalib.jar"
183 } else if filepath.Ext(base) == ".apk" {
184 base = "package.apk"
185 }
186
187 toOdexPath := func(path string) string {
188 return filepath.Join(
189 filepath.Dir(path),
190 "oat",
191 arch,
192 pathtools.ReplaceExtension(filepath.Base(path), "odex"))
193 }
194
195 odexPath := toOdexPath(filepath.Join(filepath.Dir(module.BuildPath), base))
196 odexInstallPath := toOdexPath(module.DexLocation)
197 if odexOnSystemOther(module, global) {
198 odexInstallPath = strings.Replace(odexInstallPath, SystemPartition, SystemOtherPartition, 1)
199 }
200
201 vdexPath := pathtools.ReplaceExtension(odexPath, "vdex")
202 vdexInstallPath := pathtools.ReplaceExtension(odexInstallPath, "vdex")
203
204 // bootImageLocation is $OUT/dex_bootjars/system/framework/boot.art, but dex2oat actually reads
205 // $OUT/dex_bootjars/system/framework/arm64/boot.art
206 var bootImagePath string
207 if bootImageLocation != "" {
208 bootImagePath = filepath.Join(filepath.Dir(bootImageLocation), arch, filepath.Base(bootImageLocation))
209 }
210
211 // Lists of used and optional libraries from the build config to be verified against the manifest in the APK
212 var verifyUsesLibs []string
213 var verifyOptionalUsesLibs []string
214
215 // Lists of used and optional libraries from the build config, with optional libraries that are known to not
216 // be present in the current product removed.
217 var filteredUsesLibs []string
218 var filteredOptionalUsesLibs []string
219
220 // The class loader context using paths in the build
221 var classLoaderContextHost []string
222
223 // The class loader context using paths as they will be on the device
224 var classLoaderContextTarget []string
225
226 // Extra paths that will be appended to the class loader if the APK manifest has targetSdkVersion < 28
Nicolas Geoffray05aa7d22018-12-18 14:15:12 +0000227 var conditionalClassLoaderContextHost28 []string
228 var conditionalClassLoaderContextTarget28 []string
229
230 // Extra paths that will be appended to the class loader if the APK manifest has targetSdkVersion < 29
231 var conditionalClassLoaderContextHost29 []string
232 var conditionalClassLoaderContextTarget29 []string
Colin Cross43f08db2018-11-12 10:13:39 -0800233
234 if module.EnforceUsesLibraries {
235 verifyUsesLibs = copyOf(module.UsesLibraries)
236 verifyOptionalUsesLibs = copyOf(module.OptionalUsesLibraries)
237
238 filteredOptionalUsesLibs = filterOut(global.MissingUsesLibraries, module.OptionalUsesLibraries)
239 filteredUsesLibs = append(copyOf(module.UsesLibraries), filteredOptionalUsesLibs...)
240
241 // Create class loader context for dex2oat from uses libraries and filtered optional libraries
242 for _, l := range filteredUsesLibs {
243
244 classLoaderContextHost = append(classLoaderContextHost,
245 pathForLibrary(module, l))
246 classLoaderContextTarget = append(classLoaderContextTarget,
247 filepath.Join("/system/framework", l+".jar"))
248 }
249
250 const httpLegacy = "org.apache.http.legacy"
251 const httpLegacyImpl = "org.apache.http.legacy.impl"
252
253 // Fix up org.apache.http.legacy.impl since it should be org.apache.http.legacy in the manifest.
254 replace(verifyUsesLibs, httpLegacyImpl, httpLegacy)
255 replace(verifyOptionalUsesLibs, httpLegacyImpl, httpLegacy)
256
257 if !contains(verifyUsesLibs, httpLegacy) && !contains(verifyOptionalUsesLibs, httpLegacy) {
Nicolas Geoffray05aa7d22018-12-18 14:15:12 +0000258 conditionalClassLoaderContextHost28 = append(conditionalClassLoaderContextHost28,
Colin Cross43f08db2018-11-12 10:13:39 -0800259 pathForLibrary(module, httpLegacyImpl))
Nicolas Geoffray05aa7d22018-12-18 14:15:12 +0000260 conditionalClassLoaderContextTarget28 = append(conditionalClassLoaderContextTarget28,
Colin Cross43f08db2018-11-12 10:13:39 -0800261 filepath.Join("/system/framework", httpLegacyImpl+".jar"))
262 }
Nicolas Geoffray05aa7d22018-12-18 14:15:12 +0000263
264 const hidlBase = "android.hidl.base-V1.0-java"
265 const hidlManager = "android.hidl.manager-V1.0-java"
266
267 conditionalClassLoaderContextHost29 = append(conditionalClassLoaderContextHost29,
268 pathForLibrary(module, hidlManager))
269 conditionalClassLoaderContextTarget29 = append(conditionalClassLoaderContextTarget29,
270 filepath.Join("/system/framework", hidlManager+".jar"))
271 conditionalClassLoaderContextHost29 = append(conditionalClassLoaderContextHost29,
272 pathForLibrary(module, hidlBase))
273 conditionalClassLoaderContextTarget29 = append(conditionalClassLoaderContextTarget29,
274 filepath.Join("/system/framework", hidlBase+".jar"))
Colin Cross43f08db2018-11-12 10:13:39 -0800275 } else {
276 // Pass special class loader context to skip the classpath and collision check.
277 // This will get removed once LOCAL_USES_LIBRARIES is enforced.
278 // Right now LOCAL_USES_LIBRARIES is opt in, for the case where it's not specified we still default
279 // to the &.
280 classLoaderContextHost = []string{`\&`}
281 }
282
283 rule.Command().FlagWithArg("mkdir -p ", filepath.Dir(odexPath))
284 rule.Command().FlagWithOutput("rm -f ", odexPath)
285 // Set values in the environment of the rule. These may be modified by construct_context.sh.
286 rule.Command().FlagWithArg("class_loader_context_arg=--class-loader-context=",
287 strings.Join(classLoaderContextHost, ":"))
288 rule.Command().Text(`stored_class_loader_context_arg=""`)
289
290 if module.EnforceUsesLibraries {
Colin Cross43f08db2018-11-12 10:13:39 -0800291 rule.Command().Textf(`uses_library_names="%s"`, strings.Join(verifyUsesLibs, " "))
292 rule.Command().Textf(`optional_uses_library_names="%s"`, strings.Join(verifyOptionalUsesLibs, " "))
293 rule.Command().Textf(`aapt_binary="%s"`, global.Tools.Aapt)
Nicolas Geoffray05aa7d22018-12-18 14:15:12 +0000294 rule.Command().Textf(`dex_preopt_host_libraries="%s"`, strings.Join(classLoaderContextHost, " " ))
295 rule.Command().Textf(`dex_preopt_target_libraries="%s"`, strings.Join(classLoaderContextTarget, " "))
296 rule.Command().Textf(`conditional_host_libs_28="%s"`, strings.Join(conditionalClassLoaderContextHost28, " "))
297 rule.Command().Textf(`conditional_target_libs_28="%s"`, strings.Join(conditionalClassLoaderContextTarget28, " "))
298 rule.Command().Textf(`conditional_host_libs_29="%s"`, strings.Join(conditionalClassLoaderContextHost29, " "))
299 rule.Command().Textf(`conditional_target_libs_29="%s"`, strings.Join(conditionalClassLoaderContextTarget29, " "))
Colin Cross43f08db2018-11-12 10:13:39 -0800300 rule.Command().Text("source").Tool(global.Tools.VerifyUsesLibraries).Input(module.DexPath)
Nicolas Geoffray05aa7d22018-12-18 14:15:12 +0000301 rule.Command().Text("source").Tool(global.Tools.ConstructContext)
Colin Cross43f08db2018-11-12 10:13:39 -0800302 }
303
304 cmd := rule.Command().
305 Text(`ANDROID_LOG_TAGS="*:e"`).
306 Tool(global.Tools.Dex2oat).
307 Flag("--avoid-storing-invocation").
308 Flag("--runtime-arg").FlagWithArg("-Xms", global.Dex2oatXms).
309 Flag("--runtime-arg").FlagWithArg("-Xmx", global.Dex2oatXmx).
310 Flag("${class_loader_context_arg}").
311 Flag("${stored_class_loader_context_arg}").
312 FlagWithArg("--boot-image=", bootImageLocation).Implicit(bootImagePath).
313 FlagWithInput("--dex-file=", module.DexPath).
314 FlagWithArg("--dex-location=", module.DexLocation).
315 FlagWithOutput("--oat-file=", odexPath).ImplicitOutput(vdexPath).
316 // Pass an empty directory, dex2oat shouldn't be reading arbitrary files
317 FlagWithArg("--android-root=", global.EmptyDirectory).
318 FlagWithArg("--instruction-set=", arch).
319 FlagWithArg("--instruction-set-variant=", global.CpuVariant[arch]).
320 FlagWithArg("--instruction-set-features=", global.InstructionSetFeatures[arch]).
321 Flag("--no-generate-debug-info").
322 Flag("--generate-build-id").
323 Flag("--abort-on-hard-verifier-error").
324 Flag("--force-determinism").
325 FlagWithArg("--no-inline-from=", "core-oj.jar")
326
327 var preoptFlags []string
328 if len(module.PreoptFlags) > 0 {
329 preoptFlags = module.PreoptFlags
330 } else if len(global.PreoptFlags) > 0 {
331 preoptFlags = global.PreoptFlags
332 }
333
334 if len(preoptFlags) > 0 {
335 cmd.Text(strings.Join(preoptFlags, " "))
336 }
337
338 if module.UncompressedDex {
339 cmd.FlagWithArg("--copy-dex-files=", "false")
340 }
341
342 if !anyHavePrefix(preoptFlags, "--compiler-filter=") {
343 var compilerFilter string
344 if contains(global.SystemServerJars, module.Name) {
345 // Jars of system server, use the product option if it is set, speed otherwise.
346 if global.SystemServerCompilerFilter != "" {
347 compilerFilter = global.SystemServerCompilerFilter
348 } else {
349 compilerFilter = "speed"
350 }
351 } else if contains(global.SpeedApps, module.Name) || contains(global.SystemServerApps, module.Name) {
352 // Apps loaded into system server, and apps the product default to being compiled with the
353 // 'speed' compiler filter.
354 compilerFilter = "speed"
355 } else if profile != "" {
356 // For non system server jars, use speed-profile when we have a profile.
357 compilerFilter = "speed-profile"
358 } else if global.DefaultCompilerFilter != "" {
359 compilerFilter = global.DefaultCompilerFilter
360 } else {
361 compilerFilter = "quicken"
362 }
363 cmd.FlagWithArg("--compiler-filter=", compilerFilter)
364 }
365
366 if generateDM {
367 cmd.FlagWithArg("--copy-dex-files=", "false")
368 dmPath := filepath.Join(filepath.Dir(module.BuildPath), "generated.dm")
369 dmInstalledPath := pathtools.ReplaceExtension(module.DexLocation, "dm")
370 tmpPath := filepath.Join(filepath.Dir(module.BuildPath), "primary.vdex")
371 rule.Command().Text("cp -f").Input(vdexPath).Output(tmpPath)
372 rule.Command().Tool(global.Tools.SoongZip).
373 FlagWithArg("-L", "9").
374 FlagWithOutput("-o", dmPath).
375 Flag("-j").
376 Input(tmpPath)
377 rule.Install(dmPath, dmInstalledPath)
378 }
379
380 // By default, emit debug info.
381 debugInfo := true
382 if global.NoDebugInfo {
383 // If the global setting suppresses mini-debug-info, disable it.
384 debugInfo = false
385 }
386
387 // PRODUCT_SYSTEM_SERVER_DEBUG_INFO overrides WITH_DEXPREOPT_DEBUG_INFO.
388 // PRODUCT_OTHER_JAVA_DEBUG_INFO overrides WITH_DEXPREOPT_DEBUG_INFO.
389 if contains(global.SystemServerJars, module.Name) {
390 if global.AlwaysSystemServerDebugInfo {
391 debugInfo = true
392 } else if global.NeverSystemServerDebugInfo {
393 debugInfo = false
394 }
395 } else {
396 if global.AlwaysOtherDebugInfo {
397 debugInfo = true
398 } else if global.NeverOtherDebugInfo {
399 debugInfo = false
400 }
401 }
402
403 // Never enable on eng.
404 if global.IsEng {
405 debugInfo = false
406 }
407
408 if debugInfo {
409 cmd.Flag("--generate-mini-debug-info")
410 } else {
411 cmd.Flag("--no-generate-mini-debug-info")
412 }
413
414 // Set the compiler reason to 'prebuilt' to identify the oat files produced
415 // during the build, as opposed to compiled on the device.
416 cmd.FlagWithArg("--compilation-reason=", "prebuilt")
417
418 if appImage {
419 appImagePath := pathtools.ReplaceExtension(odexPath, "art")
420 appImageInstallPath := pathtools.ReplaceExtension(odexInstallPath, "art")
421 cmd.FlagWithOutput("--app-image-file=", appImagePath).
422 FlagWithArg("--image-format=", "lz4")
423 rule.Install(appImagePath, appImageInstallPath)
424 }
425
426 if profile != "" {
427 cmd.FlagWithArg("--profile-file=", profile)
428 }
429
430 rule.Install(odexPath, odexInstallPath)
431 rule.Install(vdexPath, vdexInstallPath)
432}
433
434// Return if the dex file in the APK should be stripped. If an APK is found to contain uncompressed dex files at
435// dex2oat time it will not be stripped even if strip=true.
436func shouldStripDex(module ModuleConfig, global GlobalConfig) bool {
437 strip := !global.DefaultNoStripping
438
439 // Don't strip modules that are not on the system partition in case the oat/vdex version in system ROM
440 // doesn't match the one in other partitions. It needs to be able to fall back to the APK for that case.
441 if !strings.HasPrefix(module.DexLocation, SystemPartition) {
442 strip = false
443 }
444
445 // system_other isn't there for an OTA, so don't strip if module is on system, and odex is on system_other.
446 if odexOnSystemOther(module, global) {
447 strip = false
448 }
449
450 if module.HasApkLibraries {
451 strip = false
452 }
453
454 // Don't strip with dex files we explicitly uncompress (dexopt will not store the dex code).
455 if module.UncompressedDex {
456 strip = false
457 }
458
459 if shouldGenerateDM(module, global) {
460 strip = false
461 }
462
463 if module.PresignedPrebuilt {
464 // Only strip out files if we can re-sign the package.
465 strip = false
466 }
467
468 return strip
469}
470
471func shouldGenerateDM(module ModuleConfig, global GlobalConfig) bool {
472 // Generating DM files only makes sense for verify, avoid doing for non verify compiler filter APKs.
473 // No reason to use a dm file if the dex is already uncompressed.
474 return global.GenerateDMFiles && !module.UncompressedDex &&
475 contains(module.PreoptFlags, "--compiler-filter=verify")
476}
477
478func odexOnSystemOther(module ModuleConfig, global GlobalConfig) bool {
479 if !global.HasSystemOther {
480 return false
481 }
482
483 if global.SanitizeLite {
484 return false
485 }
486
487 if contains(global.SpeedApps, module.Name) || contains(global.SystemServerApps, module.Name) {
488 return false
489 }
490
491 for _, f := range global.PatternsOnSystemOther {
492 if makefileMatch(filepath.Join(SystemPartition, f), module.DexLocation) {
493 return true
494 }
495 }
496
497 return false
498}
499
500func pathForLibrary(module ModuleConfig, lib string) string {
501 path := module.LibraryPaths[lib]
502 if path == "" {
503 panic(fmt.Errorf("unknown library path for %q", lib))
504 }
505 return path
506}
507
508func makefileMatch(pattern, s string) bool {
509 percent := strings.IndexByte(pattern, '%')
510 switch percent {
511 case -1:
512 return pattern == s
513 case len(pattern) - 1:
514 return strings.HasPrefix(s, pattern[:len(pattern)-1])
515 default:
516 panic(fmt.Errorf("unsupported makefile pattern %q", pattern))
517 }
518}
519
520func contains(l []string, s string) bool {
521 for _, e := range l {
522 if e == s {
523 return true
524 }
525 }
526 return false
527}
528
529// remove all elements in a from b, returning a new slice
530func filterOut(a []string, b []string) []string {
531 var ret []string
532 for _, x := range b {
533 if !contains(a, x) {
534 ret = append(ret, x)
535 }
536 }
537 return ret
538}
539
540func replace(l []string, from, to string) {
541 for i := range l {
542 if l[i] == from {
543 l[i] = to
544 }
545 }
546}
547
548func copyOf(l []string) []string {
549 return append([]string(nil), l...)
550}
551
552func anyHavePrefix(l []string, prefix string) bool {
553 for _, x := range l {
554 if strings.HasPrefix(x, prefix) {
555 return true
556 }
557 }
558 return false
559}