Initial commit of "HostStubGen" (Ravenwood)
See tools/hoststubgen/README.md for the directory structure...
This CL contains:
- The HostGenTool.
- Libraries to build / run host side tets. (helper-*/ directories)
- Currently we expose ArrayMap and Log to the host side, but we also need to
expose a lot more classes that the tests usee.
- Some sample tests. (test-framework/ and test-tiny-framework/)
Sample tests contain very small tests for ArrayMap and Log.
- This version doen't loa JNI code yet. It still uses the Java substitution
for Log's native methods.
This is because `libandroid_runtime` seems to have a lot of obscure dependencies,
and using `libandroid_runtime` could cause obscure build errors when someone
make chages to any of direct/indirect dependencies.
- Current version doesn't use any Java annotations to control what are exposed
on the host side. Instead, we use `framework-policy-override.txt`, which is
easier to change. (because changing the file wouln't require rebuilding
framework-minus-apex.jar.)
- Currently we expose ArrayMap and Log to the host side, but we also need to
expose a lot more classes that the tests usee. See the `framework-policy-override.txt`
file.
Test: ./scripts/run-all-tests.sh
Bug: 292141694
Change-Id: If149e26aa919d17a0b82dacc78f31bd79fbb110b
diff --git a/tools/hoststubgen/scripts/Android.bp b/tools/hoststubgen/scripts/Android.bp
new file mode 100644
index 0000000..5da805e
--- /dev/null
+++ b/tools/hoststubgen/scripts/Android.bp
@@ -0,0 +1,26 @@
+package {
+ // See: http://go/android-license-faq
+ // A large-scale-change added 'default_applicable_licenses' to import
+ // all of the 'license_kinds' from "frameworks_base_license"
+ // to get the below license kinds:
+ // SPDX-license-identifier-Apache-2.0
+ default_applicable_licenses: ["frameworks_base_license"],
+}
+
+sh_binary_host {
+ name: "dump-jar",
+ src: "dump-jar",
+ visibility: ["//visibility:public"],
+}
+
+genrule_defaults {
+ name: "hoststubgen-jar-dump-defaults",
+ tools: ["dump-jar"],
+ cmd: "$(location dump-jar) -s -o $(out) $(in)",
+}
+
+sh_binary_host {
+ name: "run-ravenwood-test",
+ src: "run-ravenwood-test",
+ visibility: ["//visibility:public"],
+}
diff --git a/tools/hoststubgen/scripts/build-framework-hostside-jars-and-extract.sh b/tools/hoststubgen/scripts/build-framework-hostside-jars-and-extract.sh
new file mode 100755
index 0000000..7268123
--- /dev/null
+++ b/tools/hoststubgen/scripts/build-framework-hostside-jars-and-extract.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Script to build `framework-host-stub` and `framework-host-impl`, and copy the
+# generated jars to $out, and unzip them. Useful for looking into the generated files.
+
+source "${0%/*}"/../common.sh
+
+out=framework-all-stub-out
+
+rm -fr $out
+mkdir -p $out
+
+# Build the jars with `m`.
+run m framework-all-hidden-api-host
+
+# Copy the jar to out/ and extract them.
+run cp \
+ $SOONG_INT/frameworks/base/tools/hoststubgen/hoststubgen/framework-all-hidden-api-host/linux_glibc_common/gen/* \
+ $out
+
+extract $out/*.jar
diff --git a/tools/hoststubgen/scripts/build-framework-hostside-jars-without-genrules.sh b/tools/hoststubgen/scripts/build-framework-hostside-jars-without-genrules.sh
new file mode 100755
index 0000000..c3605a9
--- /dev/null
+++ b/tools/hoststubgen/scripts/build-framework-hostside-jars-without-genrules.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Script to build hoststubgen and run it directly (without using the build rules)
+# on framework-all.jar.
+
+
+echo "THIS SCRIPT IS BROKEN DUE TO CHANGES TO FILE PATHS TO DEPENDENT FILES. FIX IT WHEN YOU NEED TO USE IT." 1>&2
+
+exit 99
+
+
+source "${0%/*}"/../common.sh
+
+out=out
+
+mkdir -p $out
+
+# Build the tool and target jar.
+run m hoststubgen framework-all
+
+base_args=(
+ @../hoststubgen/hoststubgen-standard-options.txt
+
+ --in-jar $ANDROID_BUILD_TOP/out/soong/.intermediates/frameworks/base/framework-all/android_common/combined/framework-all.jar
+ --policy-override-file ../hoststubgen/framework-policy-override.txt "${@}"
+
+ # This file will contain all classes as an annotation file, with "keep all" policy.
+ --gen-keep-all-file $out/framework-all-keep-all-policy.txt
+
+ # This file will contains dump of all classes in the input jar.
+ --gen-input-dump-file $out/framework-all-dump.txt
+)
+
+do_it() {
+ local out_file_stem="$1"
+ shift
+ local extra_args=("${@}")
+
+ run hoststubgen \
+ "${base_args[@]}" \
+ "${extra_args[@]}" \
+ --out-stub-jar ${out_file_stem}_stub.jar \
+ --out-impl-jar ${out_file_stem}_impl.jar \
+ $HOSTSTUBGEN_OPTS
+
+ # Extract the jar files, so we can look into them.
+ run extract ${out_file_stem}_*.jar
+}
+
+#-----------------------------------------------------------------------------
+# framework-all, with all hidden APIs.
+#-----------------------------------------------------------------------------
+
+# No extra args.
+do_it $out/framework-all_host
+
+#-----------------------------------------------------------------------------
+# framework-test-api, only public/system/test-APIs in the stub.
+#-----------------------------------------------------------------------------
+
+do_it $out/framework-test-api_host \
+ --intersect-stub-jar $SOONG_INT/frameworks/base/api/android_test_stubs_current/android_common/combined/*.jar
diff --git a/tools/hoststubgen/scripts/dump-jar b/tools/hoststubgen/scripts/dump-jar
new file mode 100755
index 0000000..93729fb
--- /dev/null
+++ b/tools/hoststubgen/scripts/dump-jar
@@ -0,0 +1,163 @@
+#!/bin/bash
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+
+help() {
+ cat <<'EOF'
+
+ dump-jar: Dump java classes in jar files
+
+ Usage:
+ dump-jar [-v] CLASS-FILE [...]
+
+ Dump a *.class file
+
+ dump-jar [-v] [-s] [-o OUTPUT-FILENAME] JAR-FILE[: filename regex] [...]
+
+ Dump a jar file.
+
+ If a filename contains a ':', then the following part
+ will be used to filter files in the jar file.
+
+ For example, "file.jar:/MyClass$" will only dump "MyClass" in file.jar.
+
+ Options:
+ -v: Enable verbose output.
+
+ -s: Simple output mode, used to check HostStubGen output jars.
+
+ -o: Write the output to a specified file.
+EOF
+}
+
+# Parse the options.
+
+verbose=0
+simple=0
+output=""
+while getopts "hvso:" opt; do
+case "$opt" in
+ h)
+ help
+ exit 0
+ ;;
+ v)
+ verbose=1
+ ;;
+ s)
+ simple=1
+ ;;
+ o)
+ output="$OPTARG"
+ ;;
+ '?')
+ help
+ exit 1
+ ;;
+esac
+done
+shift $(($OPTIND - 1))
+
+JAVAP_OPTS="${JAVAP_OPTS:--v -p -s -sysinfo -constants}"
+
+if (( $simple )) ; then
+ JAVAP_OPTS="-p -c -v"
+fi
+
+
+# Normalize a java class name.
+# Convert '.' to '/'
+# Remove the *.class suffix.
+normalize() {
+ local name="$1"
+ name="${name%.class}" # Remove the .class suffix.
+ echo "$name" | tr '.' '/'
+}
+
+# Convert the output for `-s` as needed.
+filter_output() {
+ if (( $simple )) ; then
+ # For "simple output" mode,
+ # - Normalize the constant numbers (replace with "#x")
+ # - Remove the constant pool
+ # - Remove the line number table
+ # - Some other transient lines
+ #
+ # `/PATTERN-1/,/PATTERN-1/{//!d}` is a trick to delete lines between two patterns, without
+ # the start and the end lines.
+ sed -e 's/#[0-9][0-9]*/#x/g' \
+ -e '/^Constant pool:/,/^[^ ]/{//!d}' \
+ -e '/^ *line *[0-9][0-9]*: *[0-9][0-9]*$/d' \
+ -e '/SHA-256 checksum/d' \
+ -e '/Last modified/d' \
+ -e '/^Classfile jar/d'
+ else
+ cat # Print as-is.
+ fi
+}
+
+# Write to the output file (specified with -o) as needed.
+write_to_out() {
+ if [[ -n "$output" ]] ; then
+ cat >"$output"
+ echo "Wrote output to $output" 1>&2
+ else
+ cat # print to stdout
+ fi
+}
+
+for file in "${@}"; do
+
+ # *.class?
+ if echo "$file" | grep -qE '\.class$' ; then
+ echo "# Class: $file" 1>&2
+ javap $dump_code_opt $JAVAP_OPTS $file
+
+ # *.jar?
+ elif echo "$file" | grep -qE '\.jar(:.*)?$' ; then
+ # Take the regex. Remove everything up to : in $file
+ regex=""
+ if [[ "$file" =~ : ]] ; then
+ regex="$(normalize "${file##*:}")"
+ fi
+
+ # Remove everything after ':', inclusively, in $file.
+ file="${file%:*}"
+
+ # Print the filename and the regex.
+ if ! (( $simple )) ; then
+ echo -n "# Jar: $file"
+ if [[ "$regex" != "" ]] ;then
+ echo -n " (regex: $regex)"
+ fi
+ echo
+ fi
+
+ jar tf "$file" | grep '\.class$' | sort | while read -r class ; do
+ if normalize "$class" | grep -q -- "$regex" ; then
+ echo "## Class: $class"
+ javap $dump_code_opt $JAVAP_OPTS -cp $file ${class%.class}
+ else
+ (( $verbose )) && echo "## Skipping class: $class"
+ fi
+ done
+
+ else
+ echo "Unknown file type: $file" 1>&2
+ exit 1
+ fi
+done | filter_output | write_to_out
diff --git a/tools/hoststubgen/scripts/run-all-tests.sh b/tools/hoststubgen/scripts/run-all-tests.sh
new file mode 100755
index 0000000..6bc0ddb
--- /dev/null
+++ b/tools/hoststubgen/scripts/run-all-tests.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source "${0%/*}"/../common.sh
+
+# Move to the top directory of hoststubgen
+cd ..
+
+# These tests are known to pass.
+READY_TEST_MODULES=(
+ HostStubGenTest-framework-all-test-host-test
+ hoststubgen-test-tiny-test
+)
+
+# First, build all the test modules. This shouldn't fail.
+run m run-ravenwood-test ${READY_TEST_MODULES[*]} ${NOT_READY_TEST_MODULES[*]}
+
+# Next, run the golden check. This should always pass too.
+# The following scripts _should_ pass too, but they depend on the internal paths to soong generated
+# files, and they may fail when something changes in the build system.
+run ./hoststubgen/test-tiny-framework/diff-and-update-golden.sh
+
+run ./hoststubgen/test-framework/run-test-without-atest.sh
+run ./hoststubgen/test-tiny-framework/run-test-manually.sh
+run ./scripts/build-framework-hostside-jars-and-extract.sh
+
+# This script is already broken on goog/master
+# run ./scripts/build-framework-hostside-jars-without-genrules.sh
+
+# These tests should all pass.
+run-ravenwood-test ${READY_TEST_MODULES[*]}
+
+echo ""${0##*/}" finished, with no unexpected failures. Ready to submit!"
\ No newline at end of file
diff --git a/tools/hoststubgen/scripts/run-ravenwood-test b/tools/hoststubgen/scripts/run-ravenwood-test
new file mode 100755
index 0000000..9bbb859
--- /dev/null
+++ b/tools/hoststubgen/scripts/run-ravenwood-test
@@ -0,0 +1,129 @@
+#!/bin/bash
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Script to run a "Ravenwood" host side test.
+#
+# A proper way to run these tests is to use `atest`, but `atest` has a known issue of loading
+# unrelated jar files as the class path, so for now we use this script to run host side tests.
+
+# Copy (with some changes) some functions from ../common.sh, so this script can be used without it.
+
+m() {
+ if (( $SKIP_BUILD )) ; then
+ echo "Skipping build: $*" 1>&2
+ return 0
+ fi
+ run ${ANDROID_BUILD_TOP}/build/soong/soong_ui.bash --make-mode "$@"
+}
+
+run() {
+ echo "Running: $*" 1>&2
+ "$@"
+}
+
+run_junit_test_jar() {
+ local jar="$1"
+ echo "Starting test: $jar ..."
+ run cd "${jar%/*}"
+
+ run ${JAVA:-java} $JAVA_OPTS \
+ -cp $jar \
+ org.junit.runner.JUnitCore \
+ com.android.hoststubgen.hosthelper.HostTestSuite || return 1
+ return 0
+}
+
+help() {
+ cat <<'EOF'
+
+ run-ravenwood-test -- Run Ravenwood host tests
+
+ Usage:
+ run-ravenwood-test [options] MODULE-NAME ...
+
+ Options:
+ -h: Help
+ -t: Run test only, without building
+ -b: Build only, without running
+
+ Example:
+ run-ravenwood-test HostStubGenTest-framework-test-host-test
+
+EOF
+}
+
+#-------------------------------------------------------------------------
+# Parse options
+#-------------------------------------------------------------------------
+build=0
+test=0
+
+while getopts "htb" opt; do
+ case "$opt" in
+ h) help; exit 0 ;;
+ t)
+ test=1
+ ;;
+ b)
+ build=1
+ ;;
+ esac
+done
+shift $(($OPTIND - 1))
+
+# If neither -t nor -b is provided, then build and run./
+if (( ( $build + $test ) == 0 )) ; then
+ build=1
+ test=1
+fi
+
+
+modules=("${@}")
+
+if (( "${#modules[@]}" == 0 )); then
+ help
+ exit 1
+fi
+
+#-------------------------------------------------------------------------
+# Build
+#-------------------------------------------------------------------------
+if (( $build )) ; then
+ run m "${modules[@]}"
+fi
+
+#-------------------------------------------------------------------------
+# Run
+#-------------------------------------------------------------------------
+
+failures=0
+if (( test )) ; then
+ for module in "${modules[@]}"; do
+ if run_junit_test_jar "$ANDROID_BUILD_TOP/out/host/linux-x86/testcases/${module}/${module}.jar"; then
+ : # passed.
+ else
+ failures=$(( failures + 1 ))
+ fi
+ done
+
+ if (( $failures > 0 )) ; then
+ echo "$failures test jar(s) failed." 1>&2
+ exit 2
+ fi
+fi
+
+exit 0
\ No newline at end of file