Ravenizer tool skelton
- It still doesn't do any conversion, but the outer structure is done.
- Also make the "load class structure" step a bit faster by loading
classes in parallel.
Flag: EXEMPT host test change only
Bug: 360390999
Test: $ANDROID_BUILD_TOP/frameworks/base/ravenwood/scripts/run-ravenwood-tests.sh
Change-Id: I0f28ccd7388c310f0f733900fea9ad709e16f1cb
diff --git a/ravenwood/Android.bp b/ravenwood/Android.bp
index 6150343..58cd2e4 100644
--- a/ravenwood/Android.bp
+++ b/ravenwood/Android.bp
@@ -166,6 +166,14 @@
jarjar_rules: ":ravenwood-services-jarjar-rules",
}
+java_device_for_host {
+ name: "ravenwood-junit-impl-for-ravenizer",
+ libs: [
+ "ravenwood-junit-impl",
+ ],
+ visibility: [":__subpackages__"],
+}
+
// Separated out from ravenwood-junit-impl since it needs to compile
// against `module_current`
java_library {
diff --git a/ravenwood/tools/ravenizer-fake/Android.bp b/ravenwood/tools/ravenizer-fake/Android.bp
deleted file mode 100644
index 7e2c407..0000000
--- a/ravenwood/tools/ravenizer-fake/Android.bp
+++ /dev/null
@@ -1,14 +0,0 @@
-package {
- // See: http://go/android-license-faq
- // A large-scale-change added 'default_applicable_licenses' to import
- // all of the 'license_kinds' from "frameworks_base_license"
- // to get the below license kinds:
- // SPDX-license-identifier-Apache-2.0
- default_applicable_licenses: ["frameworks_base_license"],
-}
-
-sh_binary_host {
- name: "ravenizer",
- src: "ravenizer",
- visibility: ["//visibility:public"],
-}
diff --git a/ravenwood/tools/ravenizer-fake/ravenizer b/ravenwood/tools/ravenizer-fake/ravenizer
deleted file mode 100755
index 84b3c8e..0000000
--- a/ravenwood/tools/ravenizer-fake/ravenizer
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-# Copyright (C) 2024 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# "Fake" ravenizer, which just copies the file.
-# We need it to add ravenizer support to Soong on AOSP,
-# when the actual ravenizer is not in AOSP yet.
-
-invalid_arg() {
- echo "Ravenizer(fake): invalid args" 1>&2
- exit 1
-}
-
-(( $# >= 4 )) || invalid_arg
-[[ "$1" == "--in-jar" ]] || invalid_arg
-[[ "$3" == "--out-jar" ]] || invalid_arg
-
-echo "Ravenizer(fake): copiyng $2 to $4"
-
-cp "$2" "$4"
diff --git a/ravenwood/tools/ravenizer/Android.bp b/ravenwood/tools/ravenizer/Android.bp
new file mode 100644
index 0000000..2892d07
--- /dev/null
+++ b/ravenwood/tools/ravenizer/Android.bp
@@ -0,0 +1,25 @@
+package {
+ // See: http://go/android-license-faq
+ // A large-scale-change added 'default_applicable_licenses' to import
+ // all of the 'license_kinds' from "frameworks_base_license"
+ // to get the below license kinds:
+ // SPDX-license-identifier-Apache-2.0
+ default_applicable_licenses: ["frameworks_base_license"],
+}
+
+java_binary_host {
+ name: "ravenizer",
+ main_class: "com.android.platform.test.ravenwood.ravenizer.RavenizerMain",
+ srcs: ["src/**/*.kt"],
+ static_libs: [
+ "hoststubgen-lib",
+ "ow2-asm",
+ "ow2-asm-analysis",
+ "ow2-asm-commons",
+ "ow2-asm-tree",
+ "ow2-asm-util",
+ "junit",
+ "ravenwood-junit-impl-for-ravenizer",
+ ],
+ visibility: ["//visibility:public"],
+}
diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Ravenizer.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Ravenizer.kt
new file mode 100644
index 0000000..da9c7d9
--- /dev/null
+++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Ravenizer.kt
@@ -0,0 +1,212 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.platform.test.ravenwood.ravenizer
+
+import com.android.hoststubgen.GeneralUserErrorException
+import com.android.hoststubgen.asm.ClassNodes
+import com.android.hoststubgen.asm.zipEntryNameToClassName
+import com.android.hoststubgen.executableName
+import com.android.hoststubgen.log
+import com.android.platform.test.ravenwood.ravenizer.adapter.TestRunnerRewritingAdapter
+import org.objectweb.asm.ClassReader
+import org.objectweb.asm.ClassVisitor
+import org.objectweb.asm.ClassWriter
+import org.objectweb.asm.util.CheckClassAdapter
+import java.io.BufferedInputStream
+import java.io.BufferedOutputStream
+import java.io.FileOutputStream
+import java.io.InputStream
+import java.io.OutputStream
+import java.util.zip.ZipEntry
+import java.util.zip.ZipFile
+import java.util.zip.ZipOutputStream
+
+/**
+ * Various stats on Ravenizer.
+ */
+data class RavenizerStats(
+ /** Total end-to-end time. */
+ var totalTime: Double = .0,
+
+ /** Time took to build [ClasNodes] */
+ var loadStructureTime: Double = .0,
+
+ /** Total real time spent for converting the jar file */
+ var totalProcessTime: Double = .0,
+
+ /** Total real time spent for converting class files (except for I/O time). */
+ var totalConversionTime: Double = .0,
+
+ /** Total real time spent for copying class files without modification. */
+ var totalCopyTime: Double = .0,
+
+ /** # of entries in the input jar file */
+ var totalEntiries: Int = 0,
+
+ /** # of *.class files in the input jar file */
+ var totalClasses: Int = 0,
+
+ /** # of *.class files that have been processed. */
+ var processedClasses: Int = 0,
+) {
+ override fun toString(): String {
+ return """
+ RavenizerStats{
+ totalTime=$totalTime,
+ loadStructureTime=$loadStructureTime,
+ totalProcessTime=$totalProcessTime,
+ totalConversionTime=$totalConversionTime,
+ totalCopyTime=$totalCopyTime,
+ totalEntiries=$totalEntiries,
+ totalClasses=$totalClasses,
+ processedClasses=$processedClasses,
+ }
+ """.trimIndent()
+ }
+}
+
+/**
+ * Main class.
+ */
+class Ravenizer(val options: RavenizerOptions) {
+ fun run() {
+ val stats = RavenizerStats()
+ stats.totalTime = log.nTime {
+ process(options.inJar.get, options.outJar.get, stats)
+ }
+ log.i(stats.toString())
+ }
+
+ private fun process(inJar: String, outJar: String, stats: RavenizerStats) {
+ var allClasses = ClassNodes.loadClassStructures(inJar) {
+ time -> stats.loadStructureTime = time
+ }
+
+ stats.totalProcessTime = log.iTime("$executableName processing $inJar") {
+ ZipFile(inJar).use { inZip ->
+ val inEntries = inZip.entries()
+
+ stats.totalEntiries = inZip.size()
+
+ ZipOutputStream(BufferedOutputStream(FileOutputStream(outJar))).use { outZip ->
+ while (inEntries.hasMoreElements()) {
+ val entry = inEntries.nextElement()
+
+ if (entry.name.endsWith(".dex")) {
+ // Seems like it's an ART jar file. We can't process it.
+ // It's a fatal error.
+ throw GeneralUserErrorException(
+ "$inJar is not a desktop jar file. It contains a *.dex file."
+ )
+ }
+
+ val className = zipEntryNameToClassName(entry.name)
+
+ if (className != null) {
+ stats.totalClasses += 1
+ }
+
+ if (className != null && shouldProcessClass(allClasses, className)) {
+ stats.processedClasses += 1
+ processSingleClass(inZip, entry, outZip, allClasses, stats)
+ } else {
+ // Too slow, let's use merge_zips to bring back the original classes.
+ copyZipEntry(inZip, entry, outZip, stats)
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Copy a single ZIP entry to the output.
+ */
+ private fun copyZipEntry(
+ inZip: ZipFile,
+ entry: ZipEntry,
+ out: ZipOutputStream,
+ stats: RavenizerStats,
+ ) {
+ stats.totalCopyTime += log.nTime {
+ inZip.getInputStream(entry).use { ins ->
+ // Copy unknown entries as is to the impl out. (but not to the stub out.)
+ val outEntry = ZipEntry(entry.name)
+ outEntry.method = 0
+ outEntry.size = entry.size
+ outEntry.crc = entry.crc
+ out.putNextEntry(outEntry)
+
+ ins.transferTo(out)
+
+ out.closeEntry()
+ }
+ }
+ }
+
+ private fun processSingleClass(
+ inZip: ZipFile,
+ entry: ZipEntry,
+ outZip: ZipOutputStream,
+ allClasses: ClassNodes,
+ stats: RavenizerStats,
+ ) {
+ val newEntry = ZipEntry(entry.name)
+ outZip.putNextEntry(newEntry)
+
+ BufferedInputStream(inZip.getInputStream(entry)).use { bis ->
+ processSingleClass(entry, bis, outZip, allClasses, stats)
+ }
+ outZip.closeEntry()
+ }
+
+ /**
+ * Whether a class needs to be processed. This must be kept in sync with [processSingleClass].
+ */
+ private fun shouldProcessClass(classes: ClassNodes, classInternalName: String): Boolean {
+ return TestRunnerRewritingAdapter.shouldProcess(classes, classInternalName)
+ }
+
+ private fun processSingleClass(
+ entry: ZipEntry,
+ input: InputStream,
+ output: OutputStream,
+ allClasses: ClassNodes,
+ stats: RavenizerStats,
+ ) {
+ val cr = ClassReader(input)
+
+ lateinit var data: ByteArray
+ stats.totalConversionTime += log.vTime("Modify ${entry.name}") {
+ val flags = ClassWriter.COMPUTE_MAXS
+ val cw = ClassWriter(flags)
+ var outVisitor: ClassVisitor = cw
+
+ val enableChecker = false
+ if (enableChecker) {
+ outVisitor = CheckClassAdapter(outVisitor)
+ }
+
+ // This must be kept in sync with shouldProcessClass.
+ outVisitor = TestRunnerRewritingAdapter(allClasses, outVisitor)
+
+ cr.accept(outVisitor, ClassReader.EXPAND_FRAMES)
+
+ data = cw.toByteArray()
+ }
+ output.write(data)
+ }
+}
diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerMain.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerMain.kt
new file mode 100644
index 0000000..ff41818
--- /dev/null
+++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerMain.kt
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@file:JvmName("RavenizerMain")
+
+package com.android.platform.test.ravenwood.ravenizer
+
+import com.android.hoststubgen.LogLevel
+import com.android.hoststubgen.executableName
+import com.android.hoststubgen.log
+import com.android.hoststubgen.runMainWithBoilerplate
+
+/**
+ * Entry point.
+ */
+fun main(args: Array<String>) {
+ executableName = "Ravenizer"
+ log.setConsoleLogLevel(LogLevel.Info)
+
+ runMainWithBoilerplate {
+ val options = RavenizerOptions.parseArgs(args)
+
+ log.i("$executableName started")
+ log.v("Options: $options")
+
+ // Run.
+ Ravenizer(options).run()
+ }
+}
diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerOptions.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerOptions.kt
new file mode 100644
index 0000000..e85e3be
--- /dev/null
+++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/RavenizerOptions.kt
@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.platform.test.ravenwood.ravenizer
+
+import com.android.hoststubgen.ArgIterator
+import com.android.hoststubgen.ArgumentsException
+import com.android.hoststubgen.SetOnce
+import com.android.hoststubgen.ensureFileExists
+import com.android.hoststubgen.log
+
+class RavenizerOptions(
+ /** Input jar file*/
+ var inJar: SetOnce<String> = SetOnce(""),
+
+ /** Output jar file */
+ var outJar: SetOnce<String> = SetOnce(""),
+) {
+ companion object {
+ fun parseArgs(args: Array<String>): RavenizerOptions {
+ val ret = RavenizerOptions()
+ val ai = ArgIterator.withAtFiles(args)
+
+ while (true) {
+ val arg = ai.nextArgOptional()
+ if (arg == null) {
+ break
+ }
+
+ fun nextArg(): String = ai.nextArgRequired(arg)
+
+ if (log.maybeHandleCommandLineArg(arg) { nextArg() }) {
+ continue
+ }
+ try {
+ when (arg) {
+ // TODO: Write help
+ "-h", "--help" -> TODO("Help is not implemented yet")
+
+ "--in-jar" -> ret.inJar.set(nextArg()).ensureFileExists()
+ "--out-jar" -> ret.outJar.set(nextArg())
+
+ else -> throw ArgumentsException("Unknown option: $arg")
+ }
+ } catch (e: SetOnce.SetMoreThanOnceException) {
+ throw ArgumentsException("Duplicate or conflicting argument found: $arg")
+ }
+ }
+
+ if (!ret.inJar.isSet) {
+ throw ArgumentsException("Required option missing: --in-jar")
+ }
+ if (!ret.outJar.isSet) {
+ throw ArgumentsException("Required option missing: --out-jar")
+ }
+ return ret
+ }
+ }
+
+ override fun toString(): String {
+ return """
+ RavenizerOptions{
+ inJar=$inJar,
+ outJar=$outJar,
+ }
+ """.trimIndent()
+ }
+}
diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Utils.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Utils.kt
new file mode 100644
index 0000000..0018648
--- /dev/null
+++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/Utils.kt
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.platform.test.ravenwood.ravenizer
+
+import com.android.hoststubgen.asm.ClassNodes
+import com.android.hoststubgen.asm.findAnyAnnotation
+import org.objectweb.asm.Type
+
+val junitTestMethodType = Type.getType(org.junit.Test::class.java)
+val junitRunWithType = Type.getType(org.junit.runner.RunWith::class.java)
+
+val junitTestMethodDescriptor = junitTestMethodType.descriptor
+val junitRunWithDescriptor = junitRunWithType.descriptor
+
+val junitTestMethodDescriptors = setOf<String>(junitTestMethodDescriptor)
+val junitRunWithDescriptors = setOf<String>(junitRunWithDescriptor)
+
+/**
+ * Returns true, if a test looks like it's a test class which needs to be processed.
+ */
+fun isTestLookingClass(classes: ClassNodes, className: String): Boolean {
+ // Similar to com.android.tradefed.lite.HostUtils.testLoadClass(), except it's more lenient,
+ // and accept non-public and/or abstract classes.
+ // HostUtils also checks "Suppress" or "SuiteClasses" but this one doesn't.
+ // TODO: SuiteClasses may need to be supported.
+
+ val cn = classes.findClass(className) ?: return false
+
+ if (cn.findAnyAnnotation(junitRunWithDescriptors) != null) {
+ return true
+ }
+ cn.methods?.forEach { method ->
+ if (method.findAnyAnnotation(junitTestMethodDescriptors) != null) {
+ return true
+ }
+ }
+ if (cn.superName == null) {
+ return false
+ }
+ return isTestLookingClass(classes, cn.superName)
+}
diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/TestRunnerRewritingAdapter.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/TestRunnerRewritingAdapter.kt
new file mode 100644
index 0000000..c539908
--- /dev/null
+++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/TestRunnerRewritingAdapter.kt
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.platform.test.ravenwood.ravenizer.adapter
+
+import com.android.hoststubgen.asm.ClassNodes
+import com.android.hoststubgen.visitors.OPCODE_VERSION
+import com.android.platform.test.ravenwood.ravenizer.isTestLookingClass
+import org.objectweb.asm.ClassVisitor
+
+/**
+ * Class visitor to rewrite the test runner for Ravenwood
+ *
+ * TODO: Implement it.
+ */
+class TestRunnerRewritingAdapter(
+ protected val classes: ClassNodes,
+ nextVisitor: ClassVisitor,
+) : ClassVisitor(OPCODE_VERSION, nextVisitor) {
+ companion object {
+ /**
+ * Returns true if a target class is interesting to this adapter.
+ */
+ fun shouldProcess(classes: ClassNodes, className: String): Boolean {
+ return isTestLookingClass(classes, className)
+ }
+ }
+}
diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/Exceptions.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/Exceptions.kt
index 910bf59..f59e143 100644
--- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/Exceptions.kt
+++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/Exceptions.kt
@@ -15,6 +15,8 @@
*/
package com.android.hoststubgen
+import java.io.File
+
/**
* We will not print the stack trace for exceptions implementing it.
*/
@@ -49,4 +51,22 @@
/**
* We use this for general "user" errors.
*/
-class HostStubGenUserErrorException(message: String) : Exception(message), UserErrorException
+class GeneralUserErrorException(message: String) : Exception(message), UserErrorException
+
+/** Base exception class for invalid command line arguments. */
+open class ArgumentsException(message: String?) : Exception(message), UserErrorException
+
+/** Thrown when the same annotation is used with different annotation arguments. */
+class DuplicateAnnotationException(annotationName: String?) :
+ ArgumentsException("Duplicate annotation specified: '$annotationName'")
+
+/** Thrown when an input file does not exist. */
+class InputFileNotFoundException(filename: String) :
+ ArgumentsException("File '$filename' not found")
+
+fun String.ensureFileExists(): String {
+ if (!File(this).exists()) {
+ throw InputFileNotFoundException(this)
+ }
+ return this
+}
diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenErrors.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenErrors.kt
index 6b01d48..a218c55 100644
--- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenErrors.kt
+++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenErrors.kt
@@ -19,6 +19,6 @@
open fun onErrorFound(message: String) {
// TODO: For now, we just throw as soon as any error is found, but eventually we should keep
// all errors and print them at the end.
- throw HostStubGenUserErrorException(message)
+ throw GeneralUserErrorException(message)
}
}
\ No newline at end of file
diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenLogger.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenLogger.kt
index fcdf824..4bcee40 100644
--- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenLogger.kt
+++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenLogger.kt
@@ -89,6 +89,8 @@
addPrinter(StreamPrinter(level, PrintWriter(BufferedOutputStream(
FileOutputStream(logFilename)))))
+ log.i("Log file set: $logFilename for $level")
+
return this
}
@@ -122,6 +124,9 @@
}
fun println(level: LogLevel, message: String) {
+ if (message.isEmpty()) {
+ return // Don't print an empty message.
+ }
printers.forEach {
if (it.logLevel.ordinal >= level.ordinal) {
it.println(level, indent, message)
@@ -185,31 +190,45 @@
println(LogLevel.Debug, format, *args)
}
- inline fun <T> logTime(level: LogLevel, message: String, block: () -> T): T {
+ inline fun <T> logTime(level: LogLevel, message: String, block: () -> T): Double {
+ var ret: Double = -1.0
val start = System.currentTimeMillis()
try {
- return block()
+ block()
} finally {
val end = System.currentTimeMillis()
+ ret = (end - start) / 1000.0
if (isEnabled(level)) {
println(level,
String.format("%s: took %.1f second(s).", message, (end - start) / 1000.0))
}
}
+ return ret
}
- inline fun <T> iTime(message: String, block: () -> T): T {
+ /** Do an "i" log with how long it took. */
+ inline fun <T> iTime(message: String, block: () -> T): Double {
return logTime(LogLevel.Info, message, block)
}
- inline fun <T> vTime(message: String, block: () -> T): T {
+ /** Do a "v" log with how long it took. */
+ inline fun <T> vTime(message: String, block: () -> T): Double {
return logTime(LogLevel.Verbose, message, block)
}
- inline fun <T> dTime(message: String, block: () -> T): T {
+ /** Do a "d" log with how long it took. */
+ inline fun <T> dTime(message: String, block: () -> T): Double {
return logTime(LogLevel.Debug, message, block)
}
+ /**
+ * Similar to the other "xTime" methods, but the message is not supposed to be printed.
+ * It's only used to measure the duration with the same interface as other log methods.
+ */
+ inline fun <T> nTime(block: () -> T): Double {
+ return logTime(LogLevel.Debug, "", block)
+ }
+
inline fun forVerbose(block: () -> Unit) {
if (isEnabled(LogLevel.Verbose)) {
block()
@@ -253,6 +272,21 @@
}
}
}
+
+ /**
+ * Handle log-related command line arguments.
+ */
+ fun maybeHandleCommandLineArg(currentArg: String, nextArgProvider: () -> String): Boolean {
+ when (currentArg) {
+ "-v", "--verbose" -> setConsoleLogLevel(LogLevel.Verbose)
+ "-d", "--debug" -> setConsoleLogLevel(LogLevel.Debug)
+ "-q", "--quiet" -> setConsoleLogLevel(LogLevel.None)
+ "--verbose-log" -> addFilePrinter(LogLevel.Verbose, nextArgProvider())
+ "--debug-log" -> addFilePrinter(LogLevel.Debug, nextArgProvider())
+ else -> return false
+ }
+ return true
+ }
}
private interface LogPrinter {
diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenMain.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenMain.kt
index 45e7e30..8506466 100644
--- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenMain.kt
+++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenMain.kt
@@ -24,20 +24,32 @@
*/
fun main(args: Array<String>) {
executableName = "HostStubGen"
+ runMainWithBoilerplate {
+ // Parse the command line arguments.
+ var clanupOnError = false
+ try {
+ val options = HostStubGenOptions.parseArgs(args)
+ clanupOnError = options.cleanUpOnError.get
+ log.v("$executableName started")
+ log.v("Options: $options")
+
+ // Run.
+ HostStubGen(options).run()
+ } catch (e: Throwable) {
+ if (clanupOnError) {
+ TODO("Remove output jars here")
+ }
+ throw e
+ }
+ }
+}
+
+inline fun runMainWithBoilerplate(realMain: () -> Unit) {
var success = false
- var clanupOnError = false
try {
- // Parse the command line arguments.
- val options = HostStubGenOptions.parseArgs(args)
- clanupOnError = options.cleanUpOnError.get
-
- log.v("$executableName started")
- log.v("Options: $options")
-
- // Run.
- HostStubGen(options).run()
+ realMain()
success = true
} catch (e: Throwable) {
@@ -45,9 +57,6 @@
if (e !is UserErrorException) {
e.printStackTrace(PrintWriter(log.getWriter(LogLevel.Error)))
}
- if (clanupOnError) {
- TODO("Remove output jars here")
- }
} finally {
log.i("$executableName finished")
log.flush()
diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt
index 2f833a8..f88b107 100644
--- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt
+++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt
@@ -17,20 +17,17 @@
import com.android.hoststubgen.filters.FilterPolicy
import java.io.BufferedReader
-import java.io.File
import java.io.FileReader
/**
* A single value that can only set once.
*/
-class SetOnce<T>(
- private var value: T,
-) {
+open class SetOnce<T>(private var value: T) {
class SetMoreThanOnceException : Exception()
private var set = false
- fun set(v: T) {
+ fun set(v: T): T {
if (set) {
throw SetMoreThanOnceException()
}
@@ -39,6 +36,7 @@
}
set = true
value = v
+ return v
}
val get: T
@@ -59,6 +57,16 @@
}
}
+class IntSetOnce(value: Int) : SetOnce<Int>(value) {
+ fun set(v: String): Int {
+ try {
+ return this.set(v.toInt())
+ } catch (e: NumberFormatException) {
+ throw ArgumentsException("Invalid integer $v")
+ }
+ }
+}
+
/**
* Options that can be set from command line arguments.
*/
@@ -113,18 +121,11 @@
var apiListFile: SetOnce<String?> = SetOnce(null),
- var numShards: SetOnce<Int> = SetOnce(1),
- var shard: SetOnce<Int> = SetOnce(0),
+ var numShards: IntSetOnce = IntSetOnce(1),
+ var shard: IntSetOnce = IntSetOnce(0),
) {
companion object {
- private fun String.ensureFileExists(): String {
- if (!File(this).exists()) {
- throw InputFileNotFoundException(this)
- }
- return this
- }
-
private fun parsePackageRedirect(fromColonTo: String): Pair<String, String> {
val colon = fromColonTo.indexOf(':')
if ((colon < 1) || (colon + 1 >= fromColonTo.length)) {
@@ -137,7 +138,7 @@
fun parseArgs(args: Array<String>): HostStubGenOptions {
val ret = HostStubGenOptions()
- val ai = ArgIterator(expandAtFiles(args))
+ val ai = ArgIterator.withAtFiles(args)
var allAnnotations = mutableSetOf<String>()
@@ -148,11 +149,6 @@
return name
}
- fun setLogFile(level: LogLevel, filename: String) {
- log.addFilePrinter(level, filename)
- log.i("$level log file: $filename")
- }
-
while (true) {
val arg = ai.nextArgOptional()
if (arg == null) {
@@ -161,33 +157,23 @@
// Define some shorthands...
fun nextArg(): String = ai.nextArgRequired(arg)
- fun SetOnce<String>.setNextStringArg(): String = nextArg().also { this.set(it) }
- fun SetOnce<String?>.setNextStringArg(): String = nextArg().also { this.set(it) }
fun MutableSet<String>.addUniqueAnnotationArg(): String =
nextArg().also { this += ensureUniqueAnnotation(it) }
- fun SetOnce<Int>.setNextIntArg(): String = nextArg().also {
- try {
- this.set(it.toInt())
- } catch (e: NumberFormatException) {
- throw ArgumentsException("Invalid integer for $arg: $it")
- }
- }
+ if (log.maybeHandleCommandLineArg(arg) { nextArg() }) {
+ continue
+ }
try {
when (arg) {
// TODO: Write help
"-h", "--help" -> TODO("Help is not implemented yet")
- "-v", "--verbose" -> log.setConsoleLogLevel(LogLevel.Verbose)
- "-d", "--debug" -> log.setConsoleLogLevel(LogLevel.Debug)
- "-q", "--quiet" -> log.setConsoleLogLevel(LogLevel.None)
-
- "--in-jar" -> ret.inJar.setNextStringArg().ensureFileExists()
- "--out-stub-jar" -> ret.outStubJar.setNextStringArg()
- "--out-impl-jar" -> ret.outImplJar.setNextStringArg()
+ "--in-jar" -> ret.inJar.set(nextArg()).ensureFileExists()
+ "--out-stub-jar" -> ret.outStubJar.set(nextArg())
+ "--out-impl-jar" -> ret.outImplJar.set(nextArg())
"--policy-override-file" ->
- ret.policyOverrideFile.setNextStringArg().ensureFileExists()
+ ret.policyOverrideFile.set(nextArg())!!.ensureFileExists()
"--clean-up-on-error" -> ret.cleanUpOnError.set(true)
"--no-clean-up-on-error" -> ret.cleanUpOnError.set(false)
@@ -231,19 +217,19 @@
ret.packageRedirects += parsePackageRedirect(nextArg())
"--annotation-allowed-classes-file" ->
- ret.annotationAllowedClassesFile.setNextStringArg()
+ ret.annotationAllowedClassesFile.set(nextArg())
"--default-class-load-hook" ->
- ret.defaultClassLoadHook.setNextStringArg()
+ ret.defaultClassLoadHook.set(nextArg())
"--default-method-call-hook" ->
- ret.defaultMethodCallHook.setNextStringArg()
+ ret.defaultMethodCallHook.set(nextArg())
"--intersect-stub-jar" ->
ret.intersectStubJars += nextArg().ensureFileExists()
"--gen-keep-all-file" ->
- ret.inputJarAsKeepAllFile.setNextStringArg()
+ ret.inputJarAsKeepAllFile.set(nextArg())
// Following options are for debugging.
"--enable-class-checker" -> ret.enableClassChecker.set(true)
@@ -261,16 +247,21 @@
"--no-non-stub-method-check" ->
ret.enableNonStubMethodCallDetection.set(false)
- "--gen-input-dump-file" -> ret.inputJarDumpFile.setNextStringArg()
+ "--gen-input-dump-file" -> ret.inputJarDumpFile.set(nextArg())
- "--verbose-log" -> setLogFile(LogLevel.Verbose, nextArg())
- "--debug-log" -> setLogFile(LogLevel.Debug, nextArg())
+ "--stats-file" -> ret.statsFile.set(nextArg())
+ "--supported-api-list-file" -> ret.apiListFile.set(nextArg())
- "--stats-file" -> ret.statsFile.setNextStringArg()
- "--supported-api-list-file" -> ret.apiListFile.setNextStringArg()
-
- "--num-shards" -> ret.numShards.setNextIntArg()
- "--shard-index" -> ret.shard.setNextIntArg()
+ "--num-shards" -> ret.numShards.set(nextArg()).also {
+ if (it < 1) {
+ throw ArgumentsException("$arg must be positive integer")
+ }
+ }
+ "--shard-index" -> ret.shard.set(nextArg()).also {
+ if (it < 0) {
+ throw ArgumentsException("$arg must be positive integer or zero")
+ }
+ }
else -> throw ArgumentsException("Unknown option: $arg")
}
@@ -286,6 +277,15 @@
log.w("Neither --out-stub-jar nor --out-impl-jar is set." +
" $executableName will not generate jar files.")
}
+ if (ret.numShards.isSet != ret.shard.isSet) {
+ throw ArgumentsException("--num-shards and --shard-index must be used together")
+ }
+
+ if (ret.numShards.isSet) {
+ if (ret.shard.get >= ret.numShards.get) {
+ throw ArgumentsException("--shard-index must be smaller than --num-shards")
+ }
+ }
if (ret.enableNonStubMethodCallDetection.get) {
log.w("--enable-non-stub-method-check is not fully implemented yet." +
@@ -294,87 +294,6 @@
return ret
}
-
- /**
- * Scan the arguments, and if any of them starts with an `@`, then load from the file
- * and use its content as arguments.
- *
- * In this file, each line is treated as a single argument.
- *
- * The file can contain '#' as comments.
- */
- private fun expandAtFiles(args: Array<String>): List<String> {
- val ret = mutableListOf<String>()
-
- args.forEach { arg ->
- if (!arg.startsWith('@')) {
- ret += arg
- return@forEach
- }
- // Read from the file, and add each line to the result.
- val filename = arg.substring(1).ensureFileExists()
-
- log.v("Expanding options file $filename")
-
- BufferedReader(FileReader(filename)).use { reader ->
- while (true) {
- var line = reader.readLine()
- if (line == null) {
- break // EOF
- }
-
- line = normalizeTextLine(line)
- if (line.isNotEmpty()) {
- ret += line
- }
- }
- }
- }
- return ret
- }
- }
-
- open class ArgumentsException(message: String?) : Exception(message), UserErrorException
-
- /** Thrown when the same annotation is used with different annotation arguments. */
- class DuplicateAnnotationException(annotationName: String?) :
- ArgumentsException("Duplicate annotation specified: '$annotationName'")
-
- /** Thrown when an input file does not exist. */
- class InputFileNotFoundException(filename: String) :
- ArgumentsException("File '$filename' not found")
-
- private class ArgIterator(
- private val args: List<String>,
- private var currentIndex: Int = -1
- ) {
- val current: String
- get() = args.get(currentIndex)
-
- /**
- * Get the next argument, or [null] if there's no more arguments.
- */
- fun nextArgOptional(): String? {
- if ((currentIndex + 1) >= args.size) {
- return null
- }
- return args.get(++currentIndex)
- }
-
- /**
- * Get the next argument, or throw if
- */
- fun nextArgRequired(argName: String): String {
- nextArgOptional().let {
- if (it == null) {
- throw ArgumentsException("Missing parameter for option $argName")
- }
- if (it.isEmpty()) {
- throw ArgumentsException("Parameter can't be empty for option $argName")
- }
- return it
- }
- }
}
override fun toString(): String {
@@ -415,3 +334,80 @@
""".trimIndent()
}
}
+
+class ArgIterator(
+ private val args: List<String>,
+ private var currentIndex: Int = -1
+) {
+ val current: String
+ get() = args.get(currentIndex)
+
+ /**
+ * Get the next argument, or [null] if there's no more arguments.
+ */
+ fun nextArgOptional(): String? {
+ if ((currentIndex + 1) >= args.size) {
+ return null
+ }
+ return args.get(++currentIndex)
+ }
+
+ /**
+ * Get the next argument, or throw if
+ */
+ fun nextArgRequired(argName: String): String {
+ nextArgOptional().let {
+ if (it == null) {
+ throw ArgumentsException("Missing parameter for option $argName")
+ }
+ if (it.isEmpty()) {
+ throw ArgumentsException("Parameter can't be empty for option $argName")
+ }
+ return it
+ }
+ }
+
+ companion object {
+ fun withAtFiles(args: Array<String>): ArgIterator {
+ return ArgIterator(expandAtFiles(args))
+ }
+ }
+}
+
+/**
+ * Scan the arguments, and if any of them starts with an `@`, then load from the file
+ * and use its content as arguments.
+ *
+ * In this file, each line is treated as a single argument.
+ *
+ * The file can contain '#' as comments.
+ */
+private fun expandAtFiles(args: Array<String>): List<String> {
+ val ret = mutableListOf<String>()
+
+ args.forEach { arg ->
+ if (!arg.startsWith('@')) {
+ ret += arg
+ return@forEach
+ }
+ // Read from the file, and add each line to the result.
+ val filename = arg.substring(1).ensureFileExists()
+
+ log.v("Expanding options file $filename")
+
+ BufferedReader(FileReader(filename)).use { reader ->
+ while (true) {
+ var line = reader.readLine()
+ if (line == null) {
+ break // EOF
+ }
+
+ line = normalizeTextLine(line)
+ if (line.isNotEmpty()) {
+ ret += line
+ }
+ }
+ }
+ }
+ return ret
+}
diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/AsmUtils.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/AsmUtils.kt
index f219dac..6cf2143 100644
--- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/AsmUtils.kt
+++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/AsmUtils.kt
@@ -58,7 +58,24 @@
return null
}
-fun findAnnotationValueAsString(an: AnnotationNode, propertyName: String): String? {
+fun ClassNode.findAnyAnnotation(set: Set<String>): AnnotationNode? {
+ return findAnyAnnotation(set, this.visibleAnnotations, this.invisibleAnnotations)
+}
+
+fun MethodNode.findAnyAnnotation(set: Set<String>): AnnotationNode? {
+ return findAnyAnnotation(set, this.visibleAnnotations, this.invisibleAnnotations)
+}
+
+fun FieldNode.findAnyAnnotation(set: Set<String>): AnnotationNode? {
+ return findAnyAnnotation(set, this.visibleAnnotations, this.invisibleAnnotations)
+}
+
+fun <T> findAnnotationValueAsObject(
+ an: AnnotationNode,
+ propertyName: String,
+ expectedTypeHumanReadableName: String,
+ converter: (Any?) -> T?,
+): T? {
for (i in 0..(an.values?.size ?: 0) - 2 step 2) {
val name = an.values[i]
@@ -66,16 +83,30 @@
continue
}
val value = an.values[i + 1]
- if (value is String) {
- return value
+ if (value == null) {
+ return null
}
- throw ClassParseException(
- "The type of '$name' in annotation \"${an.desc}\" must be String" +
- ", but is ${value?.javaClass?.canonicalName}")
+
+ try {
+ return converter(value)
+ } catch (e: ClassCastException) {
+ throw ClassParseException(
+ "The type of '$propertyName' in annotation @${an.desc} must be " +
+ "$expectedTypeHumanReadableName, but is ${value?.javaClass?.canonicalName}")
+ }
}
return null
}
+fun findAnnotationValueAsString(an: AnnotationNode, propertyName: String): String? {
+ return findAnnotationValueAsObject(an, propertyName, "String", {it as String})
+}
+
+fun findAnnotationValueAsType(an: AnnotationNode, propertyName: String): Type? {
+ return findAnnotationValueAsObject(an, propertyName, "Class", {it as Type})
+}
+
+
val periodOrSlash = charArrayOf('.', '/')
fun getPackageNameFromFullClassName(fullClassName: String): String {
@@ -125,6 +156,24 @@
return Pair(name.substring(0, pos), name.substring(pos + 1))
}
+fun String.startsWithAny(vararg prefixes: String): Boolean {
+ prefixes.forEach {
+ if (this.startsWith(it)) {
+ return true
+ }
+ }
+ return false
+}
+
+fun String.endsWithAny(vararg suffixes: String): Boolean {
+ suffixes.forEach {
+ if (this.endsWith(it)) {
+ return true
+ }
+ }
+ return false
+}
+
fun String.toJvmClassName(): String {
return this.replace('.', '/')
}
@@ -137,6 +186,14 @@
return this.replace('/', '.')
}
+fun zipEntryNameToClassName(entryFilename: String): String? {
+ val suffix = ".class"
+ if (!entryFilename.endsWith(suffix)) {
+ return null
+ }
+ return entryFilename.substring(0, entryFilename.length - suffix.length)
+}
+
private val numericalInnerClassName = """.*\$\d+$""".toRegex()
fun isAnonymousInnerClass(cn: ClassNode): Boolean {
@@ -278,6 +335,14 @@
return (this.access and Opcodes.ACC_STATIC) != 0
}
+fun MethodNode.isPublic(): Boolean {
+ return (this.access and Opcodes.ACC_PUBLIC) != 0
+}
+
+fun MethodNode.isSpecial(): Boolean {
+ return CTOR_NAME == this.name || CLASS_INITIALIZER_NAME == this.name
+}
+
fun FieldNode.isEnum(): Boolean {
return (this.access and Opcodes.ACC_ENUM) != 0
}
diff --git a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/ClassNodes.kt b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/ClassNodes.kt
index 2607df6..e2647eb 100644
--- a/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/ClassNodes.kt
+++ b/tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/asm/ClassNodes.kt
@@ -27,6 +27,11 @@
import java.io.BufferedInputStream
import java.io.PrintWriter
import java.util.Arrays
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicReference
+import java.util.function.Consumer
+import java.util.zip.ZipEntry
import java.util.zip.ZipFile
/**
@@ -183,10 +188,43 @@
/**
* Load all the classes, without code.
*/
- fun loadClassStructures(inJar: String): ClassNodes {
- log.iTime("Reading class structure from $inJar") {
- val allClasses = ClassNodes()
+ fun loadClassStructures(
+ inJar: String,
+ timeCollector: Consumer<Double>? = null,
+ ): ClassNodes {
+ val allClasses = ClassNodes()
+ // Load classes in parallel.
+ val executor = Executors.newFixedThreadPool(4)
+
+ // First exception defected.
+ val exception = AtomicReference<Throwable>()
+
+ // Called on a BG thread. Read a single jar entry and add it to [allClasses].
+ fun parseClass(inZip: ZipFile, entry: ZipEntry) {
+ try {
+ inZip.getInputStream(entry).use { ins ->
+ val cr = ClassReader(BufferedInputStream(ins))
+ val cn = ClassNode()
+ cr.accept(
+ cn, ClassReader.SKIP_CODE
+ or ClassReader.SKIP_DEBUG
+ or ClassReader.SKIP_FRAMES
+ )
+ synchronized(allClasses) {
+ if (!allClasses.addClass(cn)) {
+ log.w("Duplicate class found: ${cn.name}")
+ }
+ }
+ }
+ } catch (e: Throwable) {
+ log.e("Failed to load class: $e")
+ exception.compareAndSet(null, e)
+ }
+ }
+
+ // Actually open the jar and read it on worker threads.
+ val time = log.iTime("Reading class structure from $inJar") {
log.withIndent {
ZipFile(inJar).use { inZip ->
val inEntries = inZip.entries()
@@ -194,40 +232,42 @@
while (inEntries.hasMoreElements()) {
val entry = inEntries.nextElement()
- BufferedInputStream(inZip.getInputStream(entry)).use { bis ->
- if (entry.name.endsWith(".class")) {
- val cr = ClassReader(bis)
- val cn = ClassNode()
- cr.accept(
- cn, ClassReader.SKIP_CODE
- or ClassReader.SKIP_DEBUG
- or ClassReader.SKIP_FRAMES
- )
- if (!allClasses.addClass(cn)) {
- log.w("Duplicate class found: ${cn.name}")
- }
- } else if (entry.name.endsWith(".dex")) {
- // Seems like it's an ART jar file. We can't process it.
- // It's a fatal error.
- throw InvalidJarFileException(
- "$inJar is not a desktop jar file."
- + " It contains a *.dex file."
- )
- } else {
- // Unknown file type. Skip.
- while (bis.available() > 0) {
- bis.skip((1024 * 1024).toLong())
- }
+ if (entry.name.endsWith(".class")) {
+ executor.submit {
+ parseClass(inZip, entry)
}
+ } else if (entry.name.endsWith(".dex")) {
+ // Seems like it's an ART jar file. We can't process it.
+ // It's a fatal error.
+ throw InvalidJarFileException(
+ "$inJar is not a desktop jar file."
+ + " It contains a *.dex file."
+ )
+ } else {
+ // Unknown file type. Skip.
}
}
+ // Wait for all the work to complete. (must do it before closing the zip)
+ log.i("Waiting for all loaders to finish...")
+ executor.shutdown()
+ executor.awaitTermination(5, TimeUnit.MINUTES)
+ log.i("All loaders to finished.")
}
}
+
+ // If any exception is detected, throw it.
+ exception.get()?.let {
+ throw it
+ }
+
if (allClasses.size == 0) {
log.w("$inJar contains no *.class files.")
+ } else {
+ log.i("Loaded ${allClasses.size} classes from $inJar.")
}
- return allClasses
}
+ timeCollector?.accept(time)
+ return allClasses
}
}
}
\ No newline at end of file