Generate system diff between KATI and SOONG
There is an effort to generate system image with Soong, but there are
some difference in installed files between Soong defined system image
and KATI defined one. This change generates diffs between installed
files from two sources.
Bug: 346873717
Test: USE_SOONG_DEFINED_SYSTEM_IMAGE=true m -j on aosp_cf_x86_64
generated diff file.
Change-Id: I25c71f88d16a4efb873c21abe70ca9c41c6423ca
Merged-In: I25c71f88d16a4efb873c21abe70ca9c41c6423ca
diff --git a/core/Makefile b/core/Makefile
index dfce50e..39bb5e6 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -3558,6 +3558,24 @@
endef
$(eval $(call write-partition-file-list,$(systemimage_intermediates)/file_list.txt,$(TARGET_OUT),$(FULL_SYSTEMIMAGE_DEPS)))
+
+ifneq ($(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE),)
+file_list_diff := $(HOST_OUT_EXECUTABLES)/file_list_diff$(HOST_EXECUTABLE_SUFFIX)
+system_file_diff_timestamp := $(systemimage_intermediates)/file_diff.timestamp
+
+$(system_file_diff_timestamp): \
+ $(systemimage_intermediates)/file_list.txt \
+ $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \
+ $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) \
+ $(file_list_diff)
+ $(file_list_diff) $(systemimage_intermediates)/file_list.txt \
+ $(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \
+ $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) $(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE)
+ touch $@
+
+$(BUILT_SYSTEMIMAGE): $(system_file_diff_timestamp)
+endif
+
# Used by soong sandwich to request the staging dir be built
$(systemimage_intermediates)/staging_dir.stamp: $(filter $(TARGET_OUT)/%,$(FULL_SYSTEMIMAGE_DEPS))
touch $@
diff --git a/core/base_rules.mk b/core/base_rules.mk
index a8cf67e..a9d6494 100644
--- a/core/base_rules.mk
+++ b/core/base_rules.mk
@@ -1059,6 +1059,11 @@
$(ALL_MODULES.$(my_register_name).ACONFIG_FILES) $(LOCAL_ACONFIG_FILES)
endif
+ifdef LOCAL_FILESYSTEM_FILELIST
+ ALL_MODULES.$(my_register_name).FILESYSTEM_FILELIST := \
+ $(ALL_MODULES.$(my_register_name).FILESYSTEM_FILELIST) $(LOCAL_FILESYSTEM_FILELIST)
+endif
+
ifndef LOCAL_SOONG_MODULE_INFO_JSON
ALL_MAKE_MODULE_INFO_JSON_MODULES += $(my_register_name)
ALL_MODULES.$(my_register_name).SHARED_LIBS := \
diff --git a/core/clear_vars.mk b/core/clear_vars.mk
index 6192690..fed19e6 100644
--- a/core/clear_vars.mk
+++ b/core/clear_vars.mk
@@ -87,6 +87,7 @@
LOCAL_EXTRA_FULL_TEST_CONFIGS:=
LOCAL_EXTRACT_APK:=
LOCAL_EXTRACT_DPI_APK:=
+LOCAL_FILESYSTEM_FILELIST:=
LOCAL_FINDBUGS_FLAGS:=
LOCAL_FORCE_STATIC_EXECUTABLE:=
LOCAL_FULL_CLASSES_JACOCO_JAR:=
diff --git a/core/product.mk b/core/product.mk
index 5ceccbf..ad80ee4 100644
--- a/core/product.mk
+++ b/core/product.mk
@@ -492,6 +492,9 @@
# by this flag.
_product_single_value_vars += PRODUCT_NOT_DEBUGGABLE_IN_USERDEBUG
+# If set, build would generate system image from Soong-defined module.
+_product_single_value_vars += PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE
+
.KATI_READONLY := _product_single_value_vars _product_list_vars
_product_var_list :=$= $(_product_single_value_vars) $(_product_list_vars)
diff --git a/tools/filelistdiff/Android.bp b/tools/filelistdiff/Android.bp
new file mode 100644
index 0000000..632ada3
--- /dev/null
+++ b/tools/filelistdiff/Android.bp
@@ -0,0 +1,27 @@
+// Copyright (C) 2024 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package {
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+python_binary_host {
+ name: "file_list_diff",
+ srcs: ["file_list_diff.py"],
+}
+
+prebuilt_etc_host {
+ name: "system_image_diff_allowlist",
+ src: "allowlist",
+}
diff --git a/tools/filelistdiff/allowlist b/tools/filelistdiff/allowlist
new file mode 100644
index 0000000..0a51d0e
--- /dev/null
+++ b/tools/filelistdiff/allowlist
@@ -0,0 +1,83 @@
+# Known diffs only in the KATI system image
+etc/NOTICE.xml.gz
+etc/compatconfig/TeleService-platform-compat-config.xml
+etc/compatconfig/calendar-provider-compat-config.xml
+etc/compatconfig/contacts-provider-platform-compat-config.xml
+etc/compatconfig/documents-ui-compat-config.xml
+etc/compatconfig/framework-location-compat-config.xml
+etc/compatconfig/framework-platform-compat-config.xml
+etc/compatconfig/icu4j-platform-compat-config.xml
+etc/compatconfig/services-platform-compat-config.xml
+etc/permissions/android.software.credentials.xml
+etc/permissions/android.software.preview_sdk.xml
+etc/permissions/android.software.webview.xml
+etc/permissions/android.software.window_magnification.xml
+etc/permissions/com.android.adservices.sdksandbox.xml
+etc/security/otacerts.zip
+etc/vintf/compatibility_matrix.202404.xml
+etc/vintf/compatibility_matrix.202504.xml
+etc/vintf/compatibility_matrix.5.xml
+etc/vintf/compatibility_matrix.6.xml
+etc/vintf/compatibility_matrix.7.xml
+etc/vintf/compatibility_matrix.8.xml
+etc/vintf/compatibility_matrix.device.xml
+etc/vintf/manifest.xml
+framework/boot-apache-xml.vdex
+framework/boot-apache-xml.vdex.fsv_meta
+framework/boot-bouncycastle.vdex
+framework/boot-bouncycastle.vdex.fsv_meta
+framework/boot-core-icu4j.vdex
+framework/boot-core-icu4j.vdex.fsv_meta
+framework/boot-core-libart.vdex
+framework/boot-core-libart.vdex.fsv_meta
+framework/boot-ext.vdex
+framework/boot-ext.vdex.fsv_meta
+framework/boot-framework-adservices.vdex
+framework/boot-framework-adservices.vdex.fsv_meta
+framework/boot-framework-graphics.vdex
+framework/boot-framework-graphics.vdex.fsv_meta
+framework/boot-framework-location.vdex
+framework/boot-framework-location.vdex.fsv_meta
+framework/boot-framework.vdex
+framework/boot-framework.vdex.fsv_meta
+framework/boot-ims-common.vdex
+framework/boot-ims-common.vdex.fsv_meta
+framework/boot-okhttp.vdex
+framework/boot-okhttp.vdex.fsv_meta
+framework/boot-telephony-common.vdex
+framework/boot-telephony-common.vdex.fsv_meta
+framework/boot-voip-common.vdex
+framework/boot-voip-common.vdex.fsv_meta
+framework/boot.vdex
+framework/boot.vdex.fsv_meta
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex.fsv_meta
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex.fsv_meta
+lib/aaudio-aidl-cpp.so
+lib/android.hardware.biometrics.fingerprint@2.1.so
+lib/android.hardware.radio.config@1.0.so
+lib/android.hardware.radio.deprecated@1.0.so
+lib/android.hardware.radio@1.0.so
+lib/android.hardware.radio@1.1.so
+lib/android.hardware.radio@1.2.so
+lib/android.hardware.radio@1.3.so
+lib/android.hardware.radio@1.4.so
+lib/android.hardware.secure_element@1.0.so
+lib/com.android.media.aaudio-aconfig-cc.so
+lib/heapprofd_client.so
+lib/heapprofd_client_api.so
+lib/libaaudio.so
+lib/libaaudio_internal.so
+lib/libalarm_jni.so
+lib/libamidi.so
+lib/libcups.so
+lib/libjni_deviceAsWebcam.so
+lib/libprintspooler_jni.so
+lib/libvendorsupport.so
+lib/libwfds.so
+lib/libyuv.so
+
+# Known diffs only in the Soong system image
+lib/libhidcommand_jni.so
+lib/libuinputcommand_jni.so
\ No newline at end of file
diff --git a/tools/filelistdiff/file_list_diff.py b/tools/filelistdiff/file_list_diff.py
new file mode 100644
index 0000000..cdc5b2e
--- /dev/null
+++ b/tools/filelistdiff/file_list_diff.py
@@ -0,0 +1,66 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import sys
+
+COLOR_WARNING = '\033[93m'
+COLOR_ERROR = '\033[91m'
+COLOR_NORMAL = '\033[0m'
+
+def find_unique_items(kati_installed_files, soong_installed_files, allowlist, system_module_name):
+ with open(kati_installed_files, 'r') as kati_list_file, \
+ open(soong_installed_files, 'r') as soong_list_file, \
+ open(allowlist, 'r') as allowlist_file:
+ kati_files = set(kati_list_file.read().split())
+ soong_files = set(soong_list_file.read().split())
+ allowed_files = set(filter(lambda x: len(x), map(lambda x: x.lstrip().split('#',1)[0].rstrip() , allowlist_file.read().split('\n'))))
+
+ def is_unknown_diff(filepath):
+ return not filepath in allowed_files
+
+ unique_in_kati = set(filter(is_unknown_diff, kati_files - soong_files))
+ unique_in_soong = set(filter(is_unknown_diff, soong_files - kati_files))
+
+ if unique_in_kati:
+ print(f'{COLOR_ERROR}Please add following modules into system image module {system_module_name}.{COLOR_NORMAL}')
+ print(f'{COLOR_WARNING}KATI only module(s):{COLOR_NORMAL}')
+ for item in sorted(unique_in_kati):
+ print(item)
+
+ if unique_in_soong:
+ if unique_in_kati:
+ print('')
+
+ print(f'{COLOR_ERROR}Please add following modules into build/make/target/product/base_system.mk.{COLOR_NORMAL}')
+ print(f'{COLOR_WARNING}Soong only module(s):{COLOR_NORMAL}')
+ for item in sorted(unique_in_soong):
+ print(item)
+
+ if unique_in_kati or unique_in_soong:
+ print('')
+ print(f'{COLOR_ERROR}FAILED: System image from KATI and SOONG differs from installed file list.{COLOR_NORMAL}')
+ sys.exit(1)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument('kati_installed_file_list')
+ parser.add_argument('soong_installed_file_list')
+ parser.add_argument('allowlist')
+ parser.add_argument('system_module_name')
+ args = parser.parse_args()
+
+ find_unique_items(args.kati_installed_file_list, args.soong_installed_file_list, args.allowlist, args.system_module_name)
\ No newline at end of file