Test_Module_Config soong implementation
New soong module type `TestModuleConfig`
This module type allows to write:
test_module_config {
name: "DerivedFrameworks1",
base: "FrameworksServicesTests",
include_filters: ["com.android.server.accessibility.FingerprintGestureDispatcherTest"],
}
This goal is to put the test options (include_fitler in this case) in
Android.bp rather than TEST_MAPPING to be able to name a set of options
and to allow the CI to more precise about which part of a TestModule
might be failing.
In general, it is recommended to split up tests rather than use this
approach, but that isn't always feasible.
More details on project here: https://docs.google.com/document/d/1MpA4BS6mTr-1D-K9MVmzdHkSufAvIkkM4gPEUglNodk/edit?tab=t.0
Migration plan here: https://docs.google.com/document/d/1x3w-BvKO3E-GbY8YOSBuzJC6-pkLkuhfjSyZ1hJQBYM/edit?tab=t.0
Some details of final output:
* All the [data] files from Derived are the same as base.
* The apk names is changed to be the Module name
* The test options are added to .config file
* test-file-name in the .config file (for base.apk) is changed to be
new apk.
* module-info.json is hand inspected between base and derived.
* I added a .manifest file so future tools know what base is, if they
need it.
We are working a mechanism to use a CAS so the zip of the testcases
won't explode. (however the trident disk image will get bigger)
% tree -ls out/target/product/shiba/testcases/DerivedFrameworks1 out/target/product/shiba/testcases/FrameworksServicesTests
[ 4096] out/target/product/shiba/testcases/DerivedFrameworks1
├── [ 4096] arm64
│ └── [ 72346281] DerivedFrameworks1.apk
├── [ 20887] BstatsTestApp.apk
├── [ 4096] data
│ └── [ 851] broken_shortcut.xml
├── [ 3157] DerivedFrameworks1.config
├── [ 12695] JobTestApp.apk
├── [ 8599] MediaButtonReceiverHolderTestHelperApp.apk
├── [ 16791] SimpleServiceTestApp1.apk
├── [ 16791] SimpleServiceTestApp2.apk
├── [ 16791] SimpleServiceTestApp3.apk
├── [ 1017540] SuspendTestApp.apk
└── [ 36] test_module_config.manifest
[ 4096] out/target/product/shiba/testcases/FrameworksServicesTests
├── [ 4096] arm64
│ └── [ 72346281] FrameworksServicesTests.apk
├── [ 20887] BstatsTestApp.apk
├── [ 4096] data
│ └── [ 851] broken_shortcut.xml
├── [ 2866] FrameworksServicesTests.config
├── [ 12695] JobTestApp.apk
├── [ 8599] MediaButtonReceiverHolderTestHelperApp.apk
├── [ 16791] SimpleServiceTestApp1.apk
├── [ 16791] SimpleServiceTestApp2.apk
├── [ 16791] SimpleServiceTestApp3.apk
└── [ 1017540] SuspendTestApp.apk
Fixes: 314148134
Test: atest DerivedFrameworks1 DerivedFrameworks2
Test: m blueprint_tests
Change-Id: Ib73a3404557e0bd583b065f0cf2fd55fba9ccdbe
diff --git a/tradefed_modules/test_module_config_test.go b/tradefed_modules/test_module_config_test.go
new file mode 100644
index 0000000..ff53043
--- /dev/null
+++ b/tradefed_modules/test_module_config_test.go
@@ -0,0 +1,204 @@
+// Copyright 2024 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package tradefed_modules
+
+import (
+ "android/soong/android"
+ "android/soong/java"
+ "strings"
+ "testing"
+)
+
+const bp = `
+ android_app {
+ name: "foo",
+ srcs: ["a.java"],
+ sdk_version: "current",
+ }
+
+ android_test_helper_app {
+ name: "HelperApp",
+ srcs: ["helper.java"],
+ }
+
+ android_test {
+ name: "base",
+ sdk_version: "current",
+ data: [":HelperApp", "data/testfile"],
+ }
+
+ test_module_config {
+ name: "derived_test",
+ base: "base",
+ exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"],
+ include_annotations: ["android.platform.test.annotations.LargeTest"],
+ }
+
+`
+
+// Ensure we create files needed and set the AndroidMkEntries needed
+func TestModuleConfigAndroidTest(t *testing.T) {
+
+ ctx := android.GroupFixturePreparers(
+ java.PrepareForTestWithJavaDefaultModules,
+ android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
+ ).RunTestWithBp(t, bp)
+
+ derived := ctx.ModuleForTests("derived_test", "android_common")
+ // Assert there are rules to create these files.
+ derived.Output("test_module_config.manifest")
+ derived.Output("test_config_fixer/derived_test.config")
+
+ // Ensure some basic rules exist.
+ ctx.ModuleForTests("base", "android_common").Output("package-res.apk")
+ entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0]
+
+ // Ensure some entries from base are there, specifically support files for data and helper apps.
+ assertEntryPairValues(t, entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"], []string{"HelperApp.apk", "data/testfile"})
+
+ // And some new derived entries are there.
+ android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE_TAGS"], []string{"tests"})
+
+ // And ones we override
+ android.AssertArrayString(t, "", entries.EntryMap["LOCAL_SOONG_JNI_LIBS_SYMBOLS"], []string{""})
+
+ android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], "derived_test/android_common/test_config_fixer/derived_test.config")
+}
+
+// Make sure we call test-config-fixer with the right args.
+func TestModuleConfigOptions(t *testing.T) {
+
+ ctx := android.GroupFixturePreparers(
+ java.PrepareForTestWithJavaDefaultModules,
+ android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
+ ).RunTestWithBp(t, bp)
+
+ // Check that we generate a rule to make a new AndroidTest.xml/Module.config file.
+ derived := ctx.ModuleForTests("derived_test", "android_common")
+ rule_cmd := derived.Rule("fix_test_config").RuleParams.Command
+ android.AssertStringDoesContain(t, "Bad FixConfig rule inputs", rule_cmd,
+ `--test-file-name=derived_test.apk --orig-test-file-name=base.apk --test-runner-options='[{"Name":"exclude-filter","Key":"","Value":"android.test.example.devcodelab.DevCodelabTest#testHelloFail"},{"Name":"include-annotation","Key":"","Value":"android.platform.test.annotations.LargeTest"}]'`)
+}
+
+// Ensure we error for a base we don't support.
+func TestModuleConfigBadBaseShouldFail(t *testing.T) {
+ badBp := `
+ java_test_host {
+ name: "base",
+ srcs: ["a.java"],
+ }
+
+ test_module_config {
+ name: "derived_test",
+ base: "base",
+ exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"],
+ include_annotations: ["android.platform.test.annotations.LargeTest"],
+ }`
+
+ ctx := android.GroupFixturePreparers(
+ java.PrepareForTestWithJavaDefaultModules,
+ android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
+ ).ExtendWithErrorHandler(
+ android.FixtureExpectsAtLeastOneErrorMatchingPattern("does not provide test BaseTestProviderData")).
+ RunTestWithBp(t, badBp)
+
+ ctx.ModuleForTests("derived_test", "android_common")
+}
+
+// Ensure we error for a base we don't support.
+func TestModuleConfigNoFiltersOrAnnotationsShouldFail(t *testing.T) {
+ badBp := `
+ android_test {
+ name: "base",
+ sdk_version: "current",
+ srcs: ["a.java"],
+ }
+
+ test_module_config {
+ name: "derived_test",
+ base: "base",
+ }`
+
+ ctx := android.GroupFixturePreparers(
+ java.PrepareForTestWithJavaDefaultModules,
+ android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
+ ).ExtendWithErrorHandler(
+ android.FixtureExpectsAtLeastOneErrorMatchingPattern("Test options must be given")).
+ RunTestWithBp(t, badBp)
+
+ ctx.ModuleForTests("derived_test", "android_common")
+}
+
+func TestModuleConfigMultipleDerivedTestsWriteDistinctMakeEntries(t *testing.T) {
+ multiBp := `
+ android_test {
+ name: "base",
+ sdk_version: "current",
+ srcs: ["a.java"],
+ }
+
+ test_module_config {
+ name: "derived_test",
+ base: "base",
+ include_annotations: ["android.platform.test.annotations.LargeTest"],
+ }
+
+ test_module_config {
+ name: "another_derived_test",
+ base: "base",
+ include_annotations: ["android.platform.test.annotations.LargeTest"],
+ }`
+
+ ctx := android.GroupFixturePreparers(
+ java.PrepareForTestWithJavaDefaultModules,
+ android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
+ ).RunTestWithBp(t, multiBp)
+
+ {
+ derived := ctx.ModuleForTests("derived_test", "android_common")
+ entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0]
+ // All these should be the same in both derived tests
+ assertEntryPairValues(t, entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"], []string{"HelperApp.apk", "data/testfile"})
+ android.AssertArrayString(t, "", entries.EntryMap["LOCAL_SOONG_JNI_LIBS_SYMBOLS"], []string{""})
+ // Except this one, which points to the updated tradefed xml file.
+ android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], "derived_test/android_common/test_config_fixer/derived_test.config")
+ // And this one, the module name.
+ android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"derived_test"})
+ }
+
+ {
+ derived := ctx.ModuleForTests("another_derived_test", "android_common")
+ entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0]
+ // All these should be the same in both derived tests
+ assertEntryPairValues(t, entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"], []string{"HelperApp.apk", "data/testfile"})
+ android.AssertArrayString(t, "", entries.EntryMap["LOCAL_SOONG_JNI_LIBS_SYMBOLS"], []string{""})
+ // Except this one, which points to the updated tradefed xml file.
+ android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], "another_derived_test/android_common/test_config_fixer/another_derived_test.config")
+ // And this one, the module name.
+ android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"another_derived_test"})
+ }
+}
+
+// Use for situations where the entries map contains pairs: [srcPath:installedPath1, srcPath2:installedPath2]
+// and we want to compare the RHS of the pairs, i.e. installedPath1, installedPath2
+func assertEntryPairValues(t *testing.T, actual []string, expected []string) {
+ for i, e := range actual {
+ parts := strings.Split(e, ":")
+ if len(parts) != 2 {
+ t.Errorf("Expected entry to have a value delimited by :, received: %s", e)
+ return
+ }
+ android.AssertStringEquals(t, "", parts[1], expected[i])
+ }
+}