Merge "Only install snapshotctl on debug builds" into main
diff --git a/ci/Android.bp b/ci/Android.bp
index 6d4ac35..0dfbd37 100644
--- a/ci/Android.bp
+++ b/ci/Android.bp
@@ -102,6 +102,7 @@
"optimized_targets.py",
"test_mapping_module_retriever.py",
"build_context.py",
+ "test_discovery_agent.py",
],
}
diff --git a/ci/build_metadata b/ci/build_metadata
index a8eb65d..cd011c8 100755
--- a/ci/build_metadata
+++ b/ci/build_metadata
@@ -20,6 +20,9 @@
export TARGET_RELEASE=trunk_staging
export TARGET_BUILD_VARIANT=eng
-build/soong/bin/m dist \
+TARGETS=(
all_teams
+ release_config_metadata
+)
+build/soong/bin/m dist ${TARGETS[@]}
diff --git a/ci/test_discovery_agent.py b/ci/test_discovery_agent.py
new file mode 100644
index 0000000..89d35d7
--- /dev/null
+++ b/ci/test_discovery_agent.py
@@ -0,0 +1,46 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+class TestDiscoveryAgent:
+ """Test discovery agent."""
+
+ _AOSP_TRADEFED_PREBUILT_JAR_RELATIVE_PATH = (
+ "tools/tradefederation/prebuilts/filegroups/tradefed/"
+ )
+
+ def __init__(
+ self,
+ tradefed_args: list[str],
+ test_mapping_zip_path: str,
+ tradefed_jar_revelant_files_path: str = _AOSP_TRADEFED_PREBUILT_JAR_RELATIVE_PATH,
+ ):
+ self.tradefed_args = tradefed_args
+ self.test_mapping_zip_path = test_mapping_zip_path
+ self.tradefed_jar_relevant_files_path = tradefed_jar_revelant_files_path
+
+ def discover_test_zip_regexes(self) -> list[str]:
+ """Discover test zip regexes from TradeFed.
+
+ Returns:
+ A list of test zip regexes that TF is going to try to pull files from.
+ """
+ return []
+
+ def discover_test_modules(self) -> list[str]:
+ """Discover test modules from TradeFed.
+
+ Returns:
+ A list of test modules that TradeFed is going to execute based on the
+ TradeFed test args.
+ """
+ return []
diff --git a/core/Makefile b/core/Makefile
index 96d0357..638d2b9 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -1676,12 +1676,13 @@
INTERNAL_VENDOR_BOOTIMAGE_ARGS += --vendor_cmdline "$(INTERNAL_KERNEL_CMDLINE)"
endif
-ifdef INTERNAL_BOOTCONFIG
+ifneq (, $(INTERNAL_BOOTCONFIG)$(INTERNAL_BOOTCONFIG_FILE))
INTERNAL_VENDOR_BOOTCONFIG_TARGET := $(PRODUCT_OUT)/vendor-bootconfig.img
$(INTERNAL_VENDOR_BOOTCONFIG_TARGET):
rm -f $@
$(foreach param,$(INTERNAL_BOOTCONFIG), \
printf "%s\n" $(param) >> $@;)
+ cat $(INTERNAL_BOOTCONFIG_FILE) >> $@
INTERNAL_VENDOR_BOOTIMAGE_ARGS += --vendor_bootconfig $(INTERNAL_VENDOR_BOOTCONFIG_TARGET)
endif
@@ -3488,12 +3489,14 @@
# $(2): The partition's staging directory
# $(3): Files to include in the partition
define write-partition-file-list
+$(1): PRIVATE_FILES := $(subst $(2)/,,$(filter $(2)/%,$(3)))
+$(1): PRIVATE_EXTRA_INSTALL_ZIPS := $(call relevant-extra-install-zips,$(filter $(2)/%,$(3)))
$(1): $$(HOST_OUT_EXECUTABLES)/extra_install_zips_file_list $(foreach p,$(call relevant-extra-install-zips,$(filter $(2)/%,$(3))),$(call word-colon,3,$(p)))
@echo Writing $$@
rm -f $$@
echo -n > $$@
- $$(foreach f,$(subst $(2)/,,$(filter $(2)/%,$(3))),echo "$$(f)" >> $$@$$(newline))
- $$(HOST_OUT_EXECUTABLES)/extra_install_zips_file_list $(2) $(call relevant-extra-install-zips,$(filter $(2)/%,$(3))) >> $$@
+ $$(foreach f,$$(PRIVATE_FILES),echo "$$(f)" >> $$@$$(newline))
+ $$(HOST_OUT_EXECUTABLES)/extra_install_zips_file_list $(2) $$(PRIVATE_EXTRA_INSTALL_ZIPS) >> $$@
endef
# -----------------------------------------------------------------
@@ -3567,14 +3570,24 @@
file_list_diff := $(HOST_OUT_EXECUTABLES)/file_list_diff$(HOST_EXECUTABLE_SUFFIX)
system_file_diff_timestamp := $(systemimage_intermediates)/file_diff.timestamp
+# The build configuration to build the REL version may have more files to allow.
+# Use allowlist_next in addition to the allowlist in this case.
+system_file_diff_allowlist_next :=
+ifeq (REL,$(PLATFORM_VERSION_CODENAME))
+system_file_diff_allowlist_next := $(ALL_MODULES.system_image_diff_allowlist_next.INSTALLED)
+$(system_file_diff_timestamp): PRIVATE_ALLOWLIST_NEXT := $(system_file_diff_allowlist_next)
+endif
$(system_file_diff_timestamp): \
$(systemimage_intermediates)/file_list.txt \
$(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \
$(ALL_MODULES.system_image_diff_allowlist.INSTALLED) \
+ $(system_file_diff_allowlist_next) \
$(file_list_diff)
$(file_list_diff) $(systemimage_intermediates)/file_list.txt \
$(ALL_MODULES.$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE).FILESYSTEM_FILELIST) \
- $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) $(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE)
+ $(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE) \
+ --allowlists $(ALL_MODULES.system_image_diff_allowlist.INSTALLED) \
+ $(PRIVATE_ALLOWLIST_NEXT)
touch $@
$(BUILT_SYSTEMIMAGE): $(system_file_diff_timestamp)
@@ -4000,6 +4013,21 @@
$(filter $(TARGET_OUT_PRODUCT)/%,\
$(ALL_DEFAULT_INSTALLED_MODULES))
+# Install product/etc/linker.config.pb with PRODUCT_PRODUCT_LINKER_CONFIG_FRAGMENTS
+product_linker_config_file := $(TARGET_OUT_PRODUCT)/etc/linker.config.pb
+$(product_linker_config_file): private_linker_config_fragments := $(PRODUCT_PRODUCT_LINKER_CONFIG_FRAGMENTS)
+$(product_linker_config_file): $(INTERNAL_PRODUCTIMAGE_FILES) | $(HOST_OUT_EXECUTABLES)/conv_linker_config
+ @echo Creating linker config: $@
+ @mkdir -p $(dir $@)
+ @rm -f $@
+ $(HOST_OUT_EXECUTABLES)/conv_linker_config proto \
+ --source $(call normalize-path-list,$(private_linker_config_fragments)) \
+ --output $@
+$(call define declare-1p-target,$(product_linker_config_file),)
+INTERNAL_PRODUCTIMAGE_FILES += $(product_linker_config_file)
+ALL_DEFAULT_INSTALLED_MODULES += $(product_linker_config_file)
+
+
INSTALLED_FILES_FILE_PRODUCT := $(PRODUCT_OUT)/installed-files-product.txt
INSTALLED_FILES_JSON_PRODUCT := $(INSTALLED_FILES_FILE_PRODUCT:.txt=.json)
$(INSTALLED_FILES_FILE_PRODUCT): .KATI_IMPLICIT_OUTPUTS := $(INSTALLED_FILES_JSON_PRODUCT)
@@ -7975,9 +8003,14 @@
# Desktop pack update image hook.
ifneq (,$(strip $(PACK_DESKTOP_UPDATE_IMAGE)))
PACK_UPDATE_IMAGE_TARGET := $(PRODUCT_OUT)/android-desktop_update_image.bin
+PACK_UPDATE_IMAGE_ARGS := --noarchive --update
+
+ifneq (,$(strip $(PACK_UPDATE_IMAGE_EXPERIMENTAL)))
+PACK_UPDATE_IMAGE_ARGS += --experimental
+endif # PACK_UPDATE_IMAGE_EXPERIMENTAL
$(PACK_UPDATE_IMAGE_TARGET): $(IMAGES) $(PACK_IMAGE_SCRIPT)
- $(PACK_IMAGE_SCRIPT) --out_dir $(PRODUCT_OUT) --noarchive --update
+ $(PACK_IMAGE_SCRIPT) --out_dir $(PRODUCT_OUT) $(PACK_UPDATE_IMAGE_ARGS)
PACKED_UPDATE_IMAGE_ARCHIVE_TARGET := $(PACK_UPDATE_IMAGE_TARGET).gz
diff --git a/core/android_soong_config_vars.mk b/core/android_soong_config_vars.mk
index f66bdd9..10d365c 100644
--- a/core/android_soong_config_vars.mk
+++ b/core/android_soong_config_vars.mk
@@ -30,13 +30,24 @@
$(call soong_config_set_bool,ANDROID,BOARD_USES_RECOVERY_AS_BOOT,$(BOARD_USES_RECOVERY_AS_BOOT))
$(call soong_config_set_bool,ANDROID,BOARD_MOVE_GSI_AVB_KEYS_TO_VENDOR_BOOT,$(BOARD_MOVE_GSI_AVB_KEYS_TO_VENDOR_BOOT))
$(call add_soong_config_var,ANDROID,CHECK_DEV_TYPE_VIOLATIONS)
+$(call soong_config_set_bool,ANDROID,HAS_BOARD_SYSTEM_EXT_PREBUILT_DIR,$(if $(BOARD_SYSTEM_EXT_PREBUILT_DIR),true,false))
+$(call soong_config_set_bool,ANDROID,HAS_BOARD_PRODUCT_PREBUILT_DIR,$(if $(BOARD_PRODUCT_PREBUILT_DIR),true,false))
$(call add_soong_config_var,ANDROID,PLATFORM_SEPOLICY_VERSION)
$(call add_soong_config_var,ANDROID,PLATFORM_SEPOLICY_COMPAT_VERSIONS)
$(call add_soong_config_var,ANDROID,PRODUCT_INSTALL_DEBUG_POLICY_TO_SYSTEM_EXT)
+$(call soong_config_set_bool,ANDROID,RELEASE_BOARD_API_LEVEL_FROZEN,$(RELEASE_BOARD_API_LEVEL_FROZEN))
$(call add_soong_config_var,ANDROID,TARGET_DYNAMIC_64_32_DRMSERVER)
$(call add_soong_config_var,ANDROID,TARGET_ENABLE_MEDIADRM_64)
$(call add_soong_config_var,ANDROID,TARGET_DYNAMIC_64_32_MEDIASERVER)
+$(call add_soong_config_var,ANDROID,ADDITIONAL_M4DEFS,$(if $(BOARD_SEPOLICY_M4DEFS),$(addprefix -D,$(BOARD_SEPOLICY_M4DEFS))))
+
+# For bootable/recovery
+RECOVERY_API_VERSION := 3
+RECOVERY_FSTAB_VERSION := 2
+$(call soong_config_set, recovery, recovery_api_version, $(RECOVERY_API_VERSION))
+$(call soong_config_set, recovery, recovery_fstab_version, $(RECOVERY_FSTAB_VERSION))
+
# For Sanitizers
$(call soong_config_set_bool,ANDROID,ASAN_ENABLED,$(if $(filter address,$(SANITIZE_TARGET)),true,false))
$(call soong_config_set_bool,ANDROID,HWASAN_ENABLED,$(if $(filter hwaddress,$(SANITIZE_TARGET)),true,false))
@@ -91,6 +102,10 @@
$(call add_soong_config_var_value,ANDROID,avf_microdroid_guest_gki_version,$(PRODUCT_AVF_MICRODROID_GUEST_GKI_VERSION))
endif
+ifdef TARGET_BOOTS_16K
+$(call soong_config_set_bool,ANDROID,target_boots_16k,$(filter true,$(TARGET_BOOTS_16K)))
+endif
+
ifdef PRODUCT_MEMCG_V2_FORCE_ENABLED
$(call add_soong_config_var_value,ANDROID,memcg_v2_force_enabled,$(PRODUCT_MEMCG_V2_FORCE_ENABLED))
endif
@@ -204,3 +219,37 @@
# Export target_board_platform to soong for hardware/google/graphics/common/libmemtrack:memtrack.$(TARGET_BOARD_PLATFORM)
$(call soong_config_set,ANDROID,target_board_platform,$(TARGET_BOARD_PLATFORM))
+
+# Export board_uses_scaler_m2m1shot and board_uses_align_restriction to soong for hardware/google/graphics/common/libscaler:libexynosscaler
+$(call soong_config_set_bool,google_graphics,board_uses_scaler_m2m1shot,$(if $(filter true,$(BOARD_USES_SCALER_M2M1SHOT)),true,false))
+$(call soong_config_set_bool,google_graphics,board_uses_align_restriction,$(if $(filter true,$(BOARD_USES_ALIGN_RESTRICTION)),true,false))
+
+# Export video_codec variables to soong for exynos modules
+$(call soong_config_set,video_codec,target_soc_name,$(TARGET_SOC_NAME))
+$(call soong_config_set_bool,video_codec,board_use_codec2_hidl_1_2,$(if $(filter true,$(BOARD_USE_CODEC2_HIDL_1_2)),true,false))
+$(call soong_config_set_bool,video_codec,board_support_mfc_enc_bt2020,$(if $(filter true,$(BOARD_SUPPORT_MFC_ENC_BT2020)),true,false))
+$(call soong_config_set_bool,video_codec,board_support_flexible_p010,$(if $(filter true,$(BOARD_SUPPORT_FLEXIBLE_P010)),true,false))
+$(call soong_config_set_bool,video_codec,board_use_codec2_aidl,$(if $(BOARD_USE_CODEC2_AIDL),true,false))
+$(call soong_config_set,video_codec,board_gpu_type,$(BOARD_GPU_TYPE))
+$(call soong_config_set_bool,video_codec,board_use_small_secure_memory,$(if $(filter true,$(BOARD_USE_SMALL_SECURE_MEMORY)),true,false))
+ifdef BOARD_SUPPORT_MFC_VERSION
+ $(call soong_config_set,video_codec,board_support_mfc_version,$(BOARD_SUPPORT_MFC_VERSION))
+endif
+ifdef BOARD_USE_MAX_SECURE_RESOURCE
+ $(call soong_config_set,video_codec,board_use_max_secure_resource,$(BOARD_USE_MAX_SECURE_RESOURCE))
+endif
+
+# Export related variables to soong for hardware/google/graphics/common/libacryl:libacryl
+ifdef BOARD_LIBACRYL_DEFAULT_COMPOSITOR
+ $(call soong_config_set,acryl,libacryl_default_compositor,$(BOARD_LIBACRYL_DEFAULT_COMPOSITOR))
+endif
+ifdef BOARD_LIBACRYL_DEFAULT_SCALER
+ $(call soong_config_set,acryl,libacryl_default_scaler,$(BOARD_LIBACRYL_DEFAULT_SCALER))
+endif
+ifdef BOARD_LIBACRYL_DEFAULT_BLTER
+ $(call soong_config_set,acryl,libacryl_default_blter,$(BOARD_LIBACRYL_DEFAULT_BLTER))
+endif
+ifdef BOARD_LIBACRYL_G2D_HDR_PLUGIN
+ #BOARD_LIBACRYL_G2D_HDR_PLUGIN is set in each board config
+ $(call soong_config_set_bool,acryl,libacryl_use_g2d_hdr_plugin,true)
+endif
diff --git a/core/binary.mk b/core/binary.mk
index 3481144..ea862be 100644
--- a/core/binary.mk
+++ b/core/binary.mk
@@ -174,7 +174,7 @@
endif
endif
-my_ndk_sysroot_include :=
+my_ndk_sysroot :=
my_ndk_sysroot_lib :=
my_api_level := 10000
@@ -207,11 +207,9 @@
my_built_ndk := $(SOONG_OUT_DIR)/ndk
my_ndk_triple := $($(LOCAL_2ND_ARCH_VAR_PREFIX)TARGET_NDK_TRIPLE)
- my_ndk_sysroot_include := \
- $(my_built_ndk)/sysroot/usr/include \
- $(my_built_ndk)/sysroot/usr/include/$(my_ndk_triple) \
+ my_ndk_sysroot := $(my_built_ndk)/sysroot
- my_ndk_sysroot_lib := $(my_built_ndk)/sysroot/usr/lib/$(my_ndk_triple)/$(my_ndk_api)
+ my_ndk_sysroot_lib := $(my_ndk_sysroot)/usr/lib/$(my_ndk_triple)/$(my_ndk_api)
# The bionic linker now has support for packed relocations and gnu style
# hashes (which are much faster!), but shipping to older devices requires
@@ -1628,19 +1626,6 @@
###########################################################
ifndef LOCAL_IS_HOST_MODULE
-ifeq ($(call module-in-vendor-or-product),true)
- my_target_global_c_includes :=
- my_target_global_c_system_includes := $(TARGET_OUT_HEADERS)
-else ifdef LOCAL_SDK_VERSION
- my_target_global_c_includes :=
- my_target_global_c_system_includes := $(my_ndk_stl_include_path) $(my_ndk_sysroot_include)
-else
- my_target_global_c_includes := $(SRC_HEADERS) \
- $($(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)C_INCLUDES)
- my_target_global_c_system_includes := $(SRC_SYSTEM_HEADERS) \
- $($(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)C_SYSTEM_INCLUDES)
-endif
-
my_target_global_cflags := $($(LOCAL_2ND_ARCH_VAR_PREFIX)CLANG_$(my_prefix)GLOBAL_CFLAGS)
my_target_global_conlyflags := $($(LOCAL_2ND_ARCH_VAR_PREFIX)CLANG_$(my_prefix)GLOBAL_CONLYFLAGS) $(my_c_std_conlyflags)
my_target_global_cppflags := $($(LOCAL_2ND_ARCH_VAR_PREFIX)CLANG_$(my_prefix)GLOBAL_CPPFLAGS) $(my_cpp_std_cppflags)
@@ -1656,6 +1641,22 @@
my_target_global_ldflags := $($(LOCAL_2ND_ARCH_VAR_PREFIX)CLANG_$(my_prefix)GLOBAL_LDFLAGS)
endif # my_use_clang_lld
+ifeq ($(call module-in-vendor-or-product),true)
+ my_target_global_c_includes :=
+ my_target_global_c_system_includes := $(TARGET_OUT_HEADERS)
+ my_target_global_cflags += -nostdlibinc
+else ifdef LOCAL_SDK_VERSION
+ my_target_global_c_includes :=
+ my_target_global_c_system_includes := $(my_ndk_stl_include_path)
+ my_target_global_cflags += --sysroot $(my_ndk_sysroot)
+else
+ my_target_global_c_includes := $(SRC_HEADERS) \
+ $($(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)C_INCLUDES)
+ my_target_global_c_system_includes := $(SRC_SYSTEM_HEADERS) \
+ $($(LOCAL_2ND_ARCH_VAR_PREFIX)$(my_prefix)C_SYSTEM_INCLUDES)
+ my_target_global_cflags += -nostdlibinc
+endif
+
my_target_triple := $($(LOCAL_2ND_ARCH_VAR_PREFIX)CLANG_$(my_prefix)TRIPLE)
ifndef LOCAL_IS_HOST_MODULE
my_target_triple_flag := -target $(my_target_triple)$(my_api_level)
diff --git a/core/board_config.mk b/core/board_config.mk
index 5606964..38baa0a 100644
--- a/core/board_config.mk
+++ b/core/board_config.mk
@@ -27,6 +27,7 @@
_board_strip_readonly_list += BOARD_KERNEL_CMDLINE
_board_strip_readonly_list += BOARD_BOOT_HEADER_VERSION
_board_strip_readonly_list += BOARD_BOOTCONFIG
+_board_strip_readonly_list += BOARD_BOOTCONFIG_FILE
_board_strip_readonly_list += BOARD_KERNEL_BASE
_board_strip_readonly_list += BOARD_USES_GENERIC_AUDIO
_board_strip_readonly_list += BOARD_USES_RECOVERY_AS_BOOT
@@ -311,9 +312,10 @@
.KATI_READONLY := $(_board_strip_readonly_list)
INTERNAL_KERNEL_CMDLINE := $(BOARD_KERNEL_CMDLINE)
-ifneq (,$(BOARD_BOOTCONFIG))
+ifneq (,$(BOARD_BOOTCONFIG)$(BOARD_BOOTCONFIG_FILE))
INTERNAL_KERNEL_CMDLINE += bootconfig
INTERNAL_BOOTCONFIG := $(BOARD_BOOTCONFIG)
+ INTERNAL_BOOTCONFIG_FILE := $(BOARD_BOOTCONFIG_FILE)
endif
ifneq ($(filter %64,$(TARGET_ARCH)),)
diff --git a/core/config.mk b/core/config.mk
index 192c8b2..f9ba38c 100644
--- a/core/config.mk
+++ b/core/config.mk
@@ -432,13 +432,6 @@
endif
.KATI_READONLY := TARGET_MAX_PAGE_SIZE_SUPPORTED
-ifdef PRODUCT_CHECK_PREBUILT_MAX_PAGE_SIZE
- TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE := $(PRODUCT_CHECK_PREBUILT_MAX_PAGE_SIZE)
-else
- TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE := false
-endif
-.KATI_READONLY := TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE
-
# Boolean variable determining if AOSP relies on bionic's PAGE_SIZE macro.
ifdef PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO
TARGET_NO_BIONIC_PAGE_SIZE_MACRO := $(PRODUCT_NO_BIONIC_PAGE_SIZE_MACRO)
@@ -817,6 +810,18 @@
endif
endif
+ifdef PRODUCT_CHECK_PREBUILT_MAX_PAGE_SIZE
+ TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE := $(PRODUCT_CHECK_PREBUILT_MAX_PAGE_SIZE)
+else ifeq (true,$(TARGET_BUILD_UNBUNDLED))
+ # unbundled builds may not have updated build sources
+ TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE := false
+else ifneq ($(call math_gt_or_eq,$(PRODUCT_SHIPPING_API_LEVEL),36),)
+ TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE := true
+else
+ TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE := false
+endif
+.KATI_READONLY := TARGET_CHECK_PREBUILT_MAX_PAGE_SIZE
+
# Set BOARD_SYSTEMSDK_VERSIONS to the latest SystemSDK version starting from P-launching
# devices if unset.
ifndef BOARD_SYSTEMSDK_VERSIONS
@@ -839,12 +844,6 @@
.KATI_READONLY := BOARD_CURRENT_API_LEVEL_FOR_VENDOR_MODULES
ifdef PRODUCT_SHIPPING_API_LEVEL
- board_api_level := $(firstword $(BOARD_API_LEVEL) $(BOARD_SHIPPING_API_LEVEL))
- ifneq (,$(board_api_level))
- min_systemsdk_version := $(call math_min,$(board_api_level),$(PRODUCT_SHIPPING_API_LEVEL))
- else
- min_systemsdk_version := $(PRODUCT_SHIPPING_API_LEVEL)
- endif
ifneq ($(call math_gt_or_eq,$(PRODUCT_SHIPPING_API_LEVEL),29),)
ifneq ($(BOARD_OTA_FRAMEWORK_VBMETA_VERSION_OVERRIDE),)
$(error When PRODUCT_SHIPPING_API_LEVEL >= 29, BOARD_OTA_FRAMEWORK_VBMETA_VERSION_OVERRIDE cannot be set)
diff --git a/core/config_sanitizers.mk b/core/config_sanitizers.mk
index c0f2c68..ab2d5c1 100644
--- a/core/config_sanitizers.mk
+++ b/core/config_sanitizers.mk
@@ -284,9 +284,9 @@
ifneq ($(filter memtag_stack,$(my_sanitize)),)
my_cflags += -fsanitize=memtag-stack
my_ldflags += -fsanitize=memtag-stack
- my_cflags += -Xclang -target-feature -Xclang +mte
- my_ldflags += -Xclang -target-feature -Xclang +mte
- my_asflags += -Xclang -target-feature -Xclang +mte
+ my_cflags += -march=armv8a+memtag
+ my_ldflags += -march=armv8a+memtag
+ my_asflags += -march=armv8a+memtag
my_sanitize := $(filter-out memtag_stack,$(my_sanitize))
endif
diff --git a/core/main.mk b/core/main.mk
index 5bbe1b1..24055e8 100644
--- a/core/main.mk
+++ b/core/main.mk
@@ -290,7 +290,7 @@
$(foreach mk,$(subdir_makefiles),$(info [$(call inc_and_print,subdir_makefiles_inc)/$(subdir_makefiles_total)] including $(mk) ...)$(eval include $(mk)))
# Build bootloader.img/radio.img, and unpack the partitions.
-include $(BUILD_SYSTEM)/tasks/tools/update_bootloader_radio_image.mk
+-include vendor/google/build/tasks/tools/update_bootloader_radio_image.mk
# For an unbundled image, we can skip blueprint_tools because unbundled image
# aims to remove a large number framework projects from the manifest, the
@@ -981,7 +981,9 @@
# Returns modules included automatically as a result of certain BoardConfig
# variables being set.
define auto-included-modules
- llndk_in_system \
+ $(foreach vndk_ver,$(PRODUCT_EXTRA_VNDK_VERSIONS),com.android.vndk.v$(vndk_ver)) \
+ $(filter-out $(LLNDK_MOVED_TO_APEX_LIBRARIES),$(LLNDK_LIBRARIES)) \
+ llndk.libraries.txt \
$(if $(DEVICE_MANIFEST_FILE),vendor_manifest.xml) \
$(if $(DEVICE_MANIFEST_SKUS),$(foreach sku, $(DEVICE_MANIFEST_SKUS),vendor_manifest_$(sku).xml)) \
$(if $(ODM_MANIFEST_FILES),odm_manifest.xml) \
diff --git a/core/product.mk b/core/product.mk
index 93a656d..8fc40f8 100644
--- a/core/product.mk
+++ b/core/product.mk
@@ -422,8 +422,9 @@
# If true, the cgroup v2 hierarchy will be split into apps/system subtrees
_product_single_value_vars += PRODUCT_CGROUP_V2_SYS_APP_ISOLATION_ENABLED
-# List of .json files to be merged/compiled into vendor/etc/linker.config.pb
+# List of .json files to be merged/compiled into vendor/etc/linker.config.pb and product/etc/linker.config.pb
_product_list_vars += PRODUCT_VENDOR_LINKER_CONFIG_FRAGMENTS
+_product_list_vars += PRODUCT_PRODUCT_LINKER_CONFIG_FRAGMENTS
# Whether to use userfaultfd GC.
# Possible values are:
diff --git a/core/product_config.mk b/core/product_config.mk
index 738d4cf..3e1f120 100644
--- a/core/product_config.mk
+++ b/core/product_config.mk
@@ -602,7 +602,12 @@
# Vendors with GRF must define BOARD_SHIPPING_API_LEVEL for the vendor API level.
# In this case, the VSR API level is the minimum of the PRODUCT_SHIPPING_API_LEVEL
# and RELEASE_BOARD_API_LEVEL
- VSR_VENDOR_API_LEVEL := $(call math_min,$(VSR_VENDOR_API_LEVEL),$(RELEASE_BOARD_API_LEVEL))
+ board_api_level := $(RELEASE_BOARD_API_LEVEL)
+ ifdef BOARD_API_LEVEL_PROP_OVERRIDE
+ board_api_level := $(BOARD_API_LEVEL_PROP_OVERRIDE)
+ endif
+ VSR_VENDOR_API_LEVEL := $(call math_min,$(VSR_VENDOR_API_LEVEL),$(board_api_level))
+ board_api_level :=
endif
endif
.KATI_READONLY := VSR_VENDOR_API_LEVEL
diff --git a/core/project_definitions.mk b/core/project_definitions.mk
index 5728b67..184b03e 100644
--- a/core/project_definitions.mk
+++ b/core/project_definitions.mk
@@ -22,3 +22,6 @@
# Include definitions for prebuilt SDK, if present.
#
-include prebuilts/sdk/current/definitions.mk
+
+# SDV-specific config.
+-include system/software_defined_vehicle/platform/config.mk
diff --git a/core/ravenwood_test_config_template.xml b/core/ravenwood_test_config_template.xml
index 2f21bae..9e9dd76 100644
--- a/core/ravenwood_test_config_template.xml
+++ b/core/ravenwood_test_config_template.xml
@@ -22,6 +22,7 @@
<option name="use-ravenwood-resources" value="true" />
<option name="exclude-paths" value="java" />
<option name="null-device" value="true" />
+ <option name="do-not-swallow-runner-errors" value="true" />
{EXTRA_CONFIGS}
diff --git a/core/robolectric_test_config_template.xml b/core/robolectric_test_config_template.xml
index b1d0c2f..257c820 100644
--- a/core/robolectric_test_config_template.xml
+++ b/core/robolectric_test_config_template.xml
@@ -33,5 +33,15 @@
<option name="java-flags" value="--add-opens=java.base/jdk.internal.util.random=ALL-UNNAMED"/>
<!-- b/251387255 -->
<option name="java-flags" value="--add-opens=java.base/java.io=ALL-UNNAMED"/>
+ <option name="java-flags" value="--add-opens=java.base/java.net=ALL-UNNAMED"/>
+ <option name="java-flags" value="--add-opens=java.base/java.nio=ALL-UNNAMED"/> <!-- required for ShadowVMRuntime -->
+ <option name="java-flags" value="--add-opens=java.base/java.security=ALL-UNNAMED"/>
+ <option name="java-flags" value="--add-opens=java.base/java.text=ALL-UNNAMED"/>
+ <option name="java-flags" value="--add-opens=java.base/java.util=ALL-UNNAMED"/>
+ <option name="java-flags" value="--add-opens=java.base/jdk.internal.access=ALL-UNNAMED"/>
+ <option name="java-flags" value="--add-opens=java.desktop/java.awt.font=ALL-UNNAMED"/>
+ <option name="java-flags" value="--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED"/>
+ <option name="java-flags" value="--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED"/>
+ <option name="java-flags" value="--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"/>
</test>
</configuration>
diff --git a/core/soong_config.mk b/core/soong_config.mk
index 0421e19..3c8a48a 100644
--- a/core/soong_config.mk
+++ b/core/soong_config.mk
@@ -150,6 +150,7 @@
$(call add_json_str, BtConfigIncludeDir, $(BOARD_BLUETOOTH_BDROID_BUILDCFG_INCLUDE_DIR))
$(call add_json_list, DeviceKernelHeaders, $(TARGET_DEVICE_KERNEL_HEADERS) $(TARGET_BOARD_KERNEL_HEADERS) $(TARGET_PRODUCT_KERNEL_HEADERS))
$(call add_json_str, VendorApiLevel, $(BOARD_API_LEVEL))
+$(call add_json_str, VendorApiLevelPropOverride, $(BOARD_API_LEVEL_PROP_OVERRIDE))
$(call add_json_list, ExtraVndkVersions, $(PRODUCT_EXTRA_VNDK_VERSIONS))
$(call add_json_list, DeviceSystemSdkVersions, $(BOARD_SYSTEMSDK_VERSIONS))
$(call add_json_list, Platform_systemsdk_versions, $(PLATFORM_SYSTEMSDK_VERSIONS))
@@ -182,8 +183,10 @@
$(call add_json_bool, Uml, $(filter true,$(TARGET_USER_MODE_LINUX)))
$(call add_json_str, VendorPath, $(TARGET_COPY_OUT_VENDOR))
+$(call add_json_bool, BuildingVendorImage, $(BUILDING_VENDOR_IMAGE))
$(call add_json_str, OdmPath, $(TARGET_COPY_OUT_ODM))
$(call add_json_str, ProductPath, $(TARGET_COPY_OUT_PRODUCT))
+$(call add_json_bool, BuildingProductImage, $(BUILDING_PRODUCT_IMAGE))
$(call add_json_str, SystemExtPath, $(TARGET_COPY_OUT_SYSTEM_EXT))
$(call add_json_bool, MinimizeJavaDebugInfo, $(filter true,$(PRODUCT_MINIMIZE_JAVA_DEBUG_INFO)))
@@ -316,6 +319,8 @@
$(call add_json_str, ProductManufacturer, $(PRODUCT_MANUFACTURER))
$(call add_json_str, ProductBrand, $(PRODUCT_BRAND))
+$(call add_json_str, ProductDevice, $(PRODUCT_DEVICE))
+$(call add_json_str, ProductModel, $(PRODUCT_MODEL))
$(call add_json_str, ReleaseVersion, $(_RELEASE_VERSION))
$(call add_json_list, ReleaseAconfigValueSets, $(RELEASE_ACONFIG_VALUE_SETS))
@@ -347,6 +352,9 @@
$(call add_json_list, SystemExtPropFiles, $(TARGET_SYSTEM_EXT_PROP))
$(call add_json_list, ProductPropFiles, $(TARGET_PRODUCT_PROP))
$(call add_json_list, OdmPropFiles, $(TARGET_ODM_PROP))
+$(call add_json_list, VendorPropFiles, $(TARGET_VENDOR_PROP))
+
+$(call add_json_str, ExtraAllowedDepsTxt, $(EXTRA_ALLOWED_DEPS_TXT))
# Do not set ArtTargetIncludeDebugBuild into any value if PRODUCT_ART_TARGET_INCLUDE_DEBUG_BUILD is not set,
# to have the same behavior from runtime_libart.mk.
@@ -364,6 +372,8 @@
$(call add_json_list, BoardAvbSystemAddHashtreeFooterArgs, $(BOARD_AVB_SYSTEM_ADD_HASHTREE_FOOTER_ARGS))
$(call add_json_bool, BoardAvbEnable, $(filter true,$(BOARD_AVB_ENABLE)))
+$(call add_json_str, AdbKeys, $(PRODUCT_ADB_KEYS))
+
$(call add_json_map, PartitionVarsForSoongMigrationOnlyDoNotUse)
$(call add_json_str, ProductDirectory, $(dir $(INTERNAL_PRODUCT)))
@@ -418,6 +428,11 @@
$(call add_json_list, ProductPackages, $(PRODUCT_PACKAGES))
$(call add_json_list, ProductPackagesDebug, $(PRODUCT_PACKAGES_DEBUG))
+ $(call add_json_map, ProductCopyFiles)
+ $(foreach pair,$(PRODUCT_COPY_FILES),\
+ $(call add_json_str,$(word 1,$(subst :, ,$(pair))),$(word 2,$(subst :, ,$(pair)))))
+ $(call end_json_map)
+
$(call end_json_map)
$(call json_end)
diff --git a/core/soong_extra_config.mk b/core/soong_extra_config.mk
index 00b5c0f..2ff83a1 100644
--- a/core/soong_extra_config.mk
+++ b/core/soong_extra_config.mk
@@ -43,6 +43,7 @@
$(call add_json_list, PRODUCT_PRODUCT_PROPERTIES, $(call collapse-prop-pairs,PRODUCT_PRODUCT_PROPERTIES))
$(call add_json_list, PRODUCT_ODM_PROPERTIES, $(call collapse-prop-pairs,PRODUCT_ODM_PROPERTIES))
$(call add_json_list, PRODUCT_PROPERTY_OVERRIDES, $(call collapse-prop-pairs,PRODUCT_PROPERTY_OVERRIDES))
+$(call add_json_list, PRODUCT_DEFAULT_PROPERTY_OVERRIDES, $(call collapse-prop-pairs,PRODUCT_DEFAULT_PROPERTY_OVERRIDES))
$(call add_json_str, BootloaderBoardName, $(TARGET_BOOTLOADER_BOARD_NAME))
diff --git a/core/sysprop_config.mk b/core/sysprop_config.mk
index 6906611..1991503 100644
--- a/core/sysprop_config.mk
+++ b/core/sysprop_config.mk
@@ -91,8 +91,12 @@
# Build system set BOARD_API_LEVEL to show the api level of the vendor API surface.
# This must not be altered outside of build system.
ifdef BOARD_API_LEVEL
-ADDITIONAL_VENDOR_PROPERTIES += \
- ro.board.api_level=$(BOARD_API_LEVEL)
+ ADDITIONAL_VENDOR_PROPERTIES += \
+ ro.board.api_level?=$(BOARD_API_LEVEL)
+ ifdef BOARD_API_LEVEL_PROP_OVERRIDE
+ ADDITIONAL_VENDOR_PROPERTIES += \
+ ro.board.api_level=$(BOARD_API_LEVEL_PROP_OVERRIDE)
+ endif
endif
# RELEASE_BOARD_API_LEVEL_FROZEN is true when the vendor API surface is frozen.
ifdef RELEASE_BOARD_API_LEVEL_FROZEN
diff --git a/target/product/gsi/Android.mk b/core/tasks/check-abi-dump-list.mk
similarity index 71%
rename from target/product/gsi/Android.mk
rename to core/tasks/check-abi-dump-list.mk
index 36897fe..81d549e 100644
--- a/target/product/gsi/Android.mk
+++ b/core/tasks/check-abi-dump-list.mk
@@ -1,4 +1,16 @@
-LOCAL_PATH:= $(call my-dir)
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#####################################################################
# Check the generate list against the latest list stored in the
@@ -109,60 +121,3 @@
$(if $(added_vndk_abi_dumps)$(added_platform_abi_dumps),exit 1)
$(hide) mkdir -p $(dir $@)
$(hide) touch $@
-
-#####################################################################
-# VNDK package and snapshot.
-
-include $(CLEAR_VARS)
-
-LOCAL_MODULE := vndk_apex_snapshot_package
-LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0
-LOCAL_LICENSE_CONDITIONS := notice
-LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE
-LOCAL_REQUIRED_MODULES := $(foreach vndk_ver,$(PRODUCT_EXTRA_VNDK_VERSIONS),com.android.vndk.v$(vndk_ver))
-include $(BUILD_PHONY_PACKAGE)
-
-#####################################################################
-# Define Phony module to install LLNDK modules which are installed in
-# the system image
-include $(CLEAR_VARS)
-LOCAL_MODULE := llndk_in_system
-LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0
-LOCAL_LICENSE_CONDITIONS := notice
-LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE
-
-# Filter LLNDK libs moved to APEX to avoid pulling them into /system/LIB
-LOCAL_REQUIRED_MODULES := \
- $(filter-out $(LLNDK_MOVED_TO_APEX_LIBRARIES),$(LLNDK_LIBRARIES)) \
- llndk.libraries.txt
-
-
-include $(BUILD_PHONY_PACKAGE)
-
-#####################################################################
-# init.gsi.rc, GSI-specific init script.
-
-include $(CLEAR_VARS)
-LOCAL_MODULE := init.gsi.rc
-LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0
-LOCAL_LICENSE_CONDITIONS := notice
-LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE
-LOCAL_SRC_FILES := $(LOCAL_MODULE)
-LOCAL_MODULE_CLASS := ETC
-LOCAL_SYSTEM_EXT_MODULE := true
-LOCAL_MODULE_RELATIVE_PATH := init
-
-include $(BUILD_PREBUILT)
-
-
-include $(CLEAR_VARS)
-LOCAL_MODULE := init.vndk-nodef.rc
-LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0
-LOCAL_LICENSE_CONDITIONS := notice
-LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE
-LOCAL_SRC_FILES := $(LOCAL_MODULE)
-LOCAL_MODULE_CLASS := ETC
-LOCAL_SYSTEM_EXT_MODULE := true
-LOCAL_MODULE_RELATIVE_PATH := gsi
-
-include $(BUILD_PREBUILT)
diff --git a/core/tasks/prebuilt_tradefed.mk b/core/tasks/prebuilt_tradefed.mk
new file mode 100644
index 0000000..96c57d5
--- /dev/null
+++ b/core/tasks/prebuilt_tradefed.mk
@@ -0,0 +1,22 @@
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ifeq (,$(wildcard tools/tradefederation/core))
+.PHONY: tradefed-core
+tradefed-core: tradefed atest_tradefed.sh
+.PHONY: tradefed-all
+tradefed-all: tradefed atest_tradefed.sh
+
+$(call dist-for-goals, tradefed, $(HOST_OUT)/etc/tradefed.zip)
+endif
diff --git a/core/tasks/tools/update_bootloader_radio_image.mk b/core/tasks/tools/update_bootloader_radio_image.mk
deleted file mode 100644
index 0ebf247..0000000
--- a/core/tasks/tools/update_bootloader_radio_image.mk
+++ /dev/null
@@ -1,17 +0,0 @@
-# Copyright (C) 2024 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http:#www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ifeq ($(USES_DEVICE_GOOGLE_ZUMA),true)
- -include vendor/google_devices/zuma/prebuilts/misc_bins/update_bootloader_radio_image.mk
-endif
diff --git a/envsetup.sh b/envsetup.sh
index 3fed5ae..554a220 100644
--- a/envsetup.sh
+++ b/envsetup.sh
@@ -362,7 +362,6 @@
packages/modules/adb/adb.bash
system/core/fastboot/fastboot.bash
tools/asuite/asuite.sh
- prebuilts/bazel/common/bazel-complete.bash
)
# Completion can be disabled selectively to allow users to use non-standard completion.
# e.g.
diff --git a/shell_utils.sh b/shell_utils.sh
index c4a6756..9053c42 100644
--- a/shell_utils.sh
+++ b/shell_utils.sh
@@ -178,11 +178,11 @@
echo -n "${color_failed}#### failed to build some targets "
fi
if [ $hours -gt 0 ] ; then
- printf "(%02g:%02g:%02g (hh:mm:ss))" $hours $mins $secs
+ printf "(%02d:%02d:%02d (hh:mm:ss))" $hours $mins $secs
elif [ $mins -gt 0 ] ; then
- printf "(%02g:%02g (mm:ss))" $mins $secs
+ printf "(%02d:%02d (mm:ss))" $mins $secs
elif [ $secs -gt 0 ] ; then
- printf "(%s seconds)" $secs
+ printf "(%d seconds)" $secs
fi
echo " ####${color_reset}"
echo
diff --git a/target/product/base_system.mk b/target/product/base_system.mk
index 878a1a2..5ce21e2 100644
--- a/target/product/base_system.mk
+++ b/target/product/base_system.mk
@@ -286,6 +286,7 @@
tombstoned \
traced \
traced_probes \
+ tradeinmode \
tune2fs \
uiautomator \
uinput \
diff --git a/target/product/base_vendor.mk b/target/product/base_vendor.mk
index a80e0b3..3f85941 100644
--- a/target/product/base_vendor.mk
+++ b/target/product/base_vendor.mk
@@ -17,7 +17,6 @@
# Base modules and settings for recovery.
PRODUCT_PACKAGES += \
adbd.recovery \
- android.hardware.health@2.0-impl-default.recovery \
build_flag_vendor \
cgroups.recovery.json \
charger.recovery \
diff --git a/target/product/generic/Android.bp b/target/product/generic/Android.bp
index bf82ff1..c980959 100644
--- a/target/product/generic/Android.bp
+++ b/target/product/generic/Android.bp
@@ -72,6 +72,53 @@
target: "/storage/self/primary",
name: "sdcard",
},
+ {
+ target: "/product/etc/security/adb_keys",
+ name: "adb_keys",
+ },
+ // For Treble Generic System Image (GSI), system-as-root GSI needs to work on both devices with
+ // and without /odm partition. Those symlinks are for devices without /odm partition. For
+ // devices with /odm partition, mount odm.img under /odm will hide those symlinks.
+ {
+ target: "/vendor/odm/app",
+ name: "odm/app",
+ },
+ {
+ target: "/vendor/odm/bin",
+ name: "odm/bin",
+ },
+ {
+ target: "/vendor/odm/etc",
+ name: "odm/etc",
+ },
+ {
+ target: "/vendor/odm/firmware",
+ name: "odm/firmware",
+ },
+ {
+ target: "/vendor/odm/framework",
+ name: "odm/framework",
+ },
+ {
+ target: "/vendor/odm/lib",
+ name: "odm/lib",
+ },
+ {
+ target: "/vendor/odm/lib64",
+ name: "odm/lib64",
+ },
+ {
+ target: "/vendor/odm/overlay",
+ name: "odm/overlay",
+ },
+ {
+ target: "/vendor/odm/priv-app",
+ name: "odm/priv-app",
+ },
+ {
+ target: "/vendor/odm/usr",
+ name: "odm/usr",
+ },
]
filegroup {
@@ -314,6 +361,11 @@
],
libs: [":framework-res{.export-package.apk}"],
},
+ type: "erofs",
+ erofs: {
+ compressor: "lz4hc,9",
+ compress_hints: "erofs_compress_hints.txt",
+ },
build_logtags: true,
gen_aconfig_flags_pb: true,
@@ -401,7 +453,6 @@
"init.zygote32.rc",
"init.zygote64.rc",
"init.zygote64_32.rc",
- "init_first_stage", // for boot partition
"initial-package-stopped-states.xml",
"input",
"installd",
@@ -415,7 +466,6 @@
"ld.mc",
"llkd", // base_system
"lmkd", // base_system
- "local_time.default", // handheld_vendo
"locksettings", // base_system
"logcat", // base_system
"logd", // base_system
@@ -477,7 +527,6 @@
"sm", // base_system
"snapshotctl", // base_system
"snapuserd", // base_system
- "snapuserd_ramdisk", // ramdisk
"storaged", // base_system
"surfaceflinger", // base_system
"svc", // base_system
@@ -487,6 +536,7 @@
"tombstoned", // base_system
"traced", // base_system
"traced_probes", // base_system
+ "tradeinmode", // base_system
"tune2fs", // base_system
"uiautomator", // base_system
"uinput", // base_system
@@ -577,6 +627,7 @@
"ContactsProvider", // base_system
"CredentialManager", // handheld_system
"DeviceAsWebcam", // handheld_system
+ "DeviceDiagnostics", // handheld_system - internal
"DocumentsUI", // handheld_system
"DownloadProvider", // base_system
"DownloadProviderUi", // handheld_system
@@ -602,7 +653,6 @@
"PacProcessor", // handheld_system
"PackageInstaller", // base_system
"PartnerBookmarksProvider", // generic_system
- "PhotoTable", // full_base
"PrintRecommendationService", // handheld_system
"PrintSpooler", // handheld_system
"ProxyHandler", // handheld_system
@@ -736,6 +786,7 @@
"libbinder_ndk",
"libbinder_rpc_unstable",
"libcamera2ndk",
+ "libcgrouprc", // llndk library
"libclang_rt.asan",
"libcompiler_rt",
"libcutils", // used by many libs
diff --git a/target/product/generic/erofs_compress_hints.txt b/target/product/generic/erofs_compress_hints.txt
new file mode 100644
index 0000000..8b2a711
--- /dev/null
+++ b/target/product/generic/erofs_compress_hints.txt
@@ -0,0 +1 @@
+0 .*\.apex$
\ No newline at end of file
diff --git a/target/product/generic_system.mk b/target/product/generic_system.mk
index 0a09eb1..b9a623d 100644
--- a/target/product/generic_system.mk
+++ b/target/product/generic_system.mk
@@ -152,4 +152,5 @@
$(call require-artifacts-in-path, $(_my_paths), $(_my_allowed_list))
# Product config map to toggle between sources and prebuilts of required mainline modules
+PRODUCT_RELEASE_CONFIG_MAPS += $(wildcard build/release/gms_mainline/required/release_config_map.textproto)
PRODUCT_RELEASE_CONFIG_MAPS += $(wildcard vendor/google_shared/build/release/gms_mainline/required/release_config_map.textproto)
diff --git a/target/product/go_defaults.mk b/target/product/go_defaults.mk
index c928530..ccc4f36 100644
--- a/target/product/go_defaults.mk
+++ b/target/product/go_defaults.mk
@@ -18,6 +18,7 @@
$(call inherit-product, build/make/target/product/go_defaults_common.mk)
# Product config map to toggle between sources and prebuilts of required mainline modules
+PRODUCT_RELEASE_CONFIG_MAPS += $(wildcard build/release/gms_mainline_go/required/release_config_map.textproto)
PRODUCT_RELEASE_CONFIG_MAPS += $(wildcard vendor/google_shared/build/release/gms_mainline_go/required/release_config_map.textproto)
# Add the system properties.
diff --git a/target/product/gsi/Android.bp b/target/product/gsi/Android.bp
index 45ba143..f18f35a 100644
--- a/target/product/gsi/Android.bp
+++ b/target/product/gsi/Android.bp
@@ -46,3 +46,18 @@
installed_location: "etc/init/config",
symlink_target: "/system/system_ext/etc/init/config",
}
+
+// init.gsi.rc, GSI-specific init script.
+prebuilt_etc {
+ name: "init.gsi.rc",
+ src: "init.gsi.rc",
+ system_ext_specific: true,
+ relative_install_path: "init",
+}
+
+prebuilt_etc {
+ name: "init.vndk-nodef.rc",
+ src: "init.vndk-nodef.rc",
+ system_ext_specific: true,
+ relative_install_path: "gsi",
+}
diff --git a/target/product/media_system_ext.mk b/target/product/media_system_ext.mk
index 30dd2e2..e79a7eb 100644
--- a/target/product/media_system_ext.mk
+++ b/target/product/media_system_ext.mk
@@ -20,9 +20,5 @@
# base_system_ext.mk.
$(call inherit-product, $(SRC_TARGET_DIR)/product/base_system_ext.mk)
-# /system_ext packages
-PRODUCT_PACKAGES += \
- vndk_apex_snapshot_package \
-
# Window Extensions
$(call inherit-product, $(SRC_TARGET_DIR)/product/window_extensions_base.mk)
diff --git a/target/product/security/Android.bp b/target/product/security/Android.bp
index 0d7b35e..69d19a3 100644
--- a/target/product/security/Android.bp
+++ b/target/product/security/Android.bp
@@ -37,3 +37,7 @@
relative_install_path: "security",
filename: "otacerts.zip",
}
+
+adb_keys {
+ name: "adb_keys",
+}
diff --git a/target/product/security/Android.mk b/target/product/security/Android.mk
deleted file mode 100644
index 138e5bb..0000000
--- a/target/product/security/Android.mk
+++ /dev/null
@@ -1,17 +0,0 @@
-LOCAL_PATH:= $(call my-dir)
-
-#######################################
-# adb key, if configured via PRODUCT_ADB_KEYS
-ifdef PRODUCT_ADB_KEYS
- ifneq ($(filter eng userdebug,$(TARGET_BUILD_VARIANT)),)
- include $(CLEAR_VARS)
- LOCAL_MODULE := adb_keys
- LOCAL_LICENSE_KINDS := SPDX-license-identifier-Apache-2.0
- LOCAL_LICENSE_CONDITIONS := notice
- LOCAL_NOTICE_FILE := build/soong/licenses/LICENSE
- LOCAL_MODULE_CLASS := ETC
- LOCAL_MODULE_PATH := $(TARGET_OUT_PRODUCT_ETC)/security
- LOCAL_PREBUILT_MODULE_FILE := $(PRODUCT_ADB_KEYS)
- include $(BUILD_PREBUILT)
- endif
-endif
diff --git a/teams/Android.bp b/teams/Android.bp
index 96d241b..21f5222 100644
--- a/teams/Android.bp
+++ b/teams/Android.bp
@@ -13,6 +13,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+// DON'T ADD NEW RULES HERE. For more details refer to
+// go/new-android-ownership-model
+
package {
default_applicable_licenses: ["Android-Apache-2.0"],
}
@@ -4454,3 +4457,34 @@
// go/trendy/manage/engineers/5121440647577600
trendy_team_id: "5121440647577600",
}
+
+team {
+ name: "trendy_team_ravenwood",
+
+ // go/trendy/manage/engineers/6027181500497920
+ trendy_team_id: "6027181500497920",
+}
+
+team {
+ name: "trendy_team_automotive_cast",
+
+ // go/trendy/manage/engineers/5293683026264064
+ trendy_team_id: "5293683026264064",
+}
+
+team {
+ name: "trendy_team_wear_standalone_kids",
+
+ // go/trendy/manage/engineers/6303298703949824
+ trendy_team_id: "6303298703949824",
+}
+
+team {
+ name: "trendy_team_desktop_stats",
+
+ // go/trendy/manage/engineers/5440764114206720
+ trendy_team_id: "5440764114206720",
+}
+
+// DON'T ADD NEW RULES HERE. For more details refer to
+// go/new-android-ownership-model
diff --git a/tools/aconfig/OWNERS b/tools/aconfig/OWNERS
index 9a76279..c92fc7c 100644
--- a/tools/aconfig/OWNERS
+++ b/tools/aconfig/OWNERS
@@ -1,7 +1,8 @@
-amhk@google.com
dzshen@google.com
-jham@google.com
-joeo@google.com
opg@google.com
tedbauer@google.com
zhidou@google.com
+
+amhk@google.com #{LAST_RESORT_SUGGESTION}
+jham@google.com #{LAST_RESORT_SUGGESTION}
+joeo@google.com #{LAST_RESORT_SUGGESTION}
diff --git a/tools/aconfig/aconfig/Android.bp b/tools/aconfig/aconfig/Android.bp
index 68521af..f4dd103 100644
--- a/tools/aconfig/aconfig/Android.bp
+++ b/tools/aconfig/aconfig/Android.bp
@@ -234,6 +234,7 @@
name: "libaconfig_test_rust_library",
crate_name: "aconfig_test_rust_library",
aconfig_declarations: "aconfig.test.flags",
+ host_supported: true,
}
rust_test {
diff --git a/tools/aconfig/aconfig/src/codegen/java.rs b/tools/aconfig/aconfig/src/codegen/java.rs
index a34166d..47d4042 100644
--- a/tools/aconfig/aconfig/src/codegen/java.rs
+++ b/tools/aconfig/aconfig/src/codegen/java.rs
@@ -531,15 +531,16 @@
private static boolean enabledRw = true;
private void init() {
StorageInternalReader reader = null;
+ boolean foundPackage = true;
try {
reader = new StorageInternalReader("system", "com.android.aconfig.test");
- disabledRw = reader.getBooleanFlagValue(1);
- disabledRwExported = reader.getBooleanFlagValue(2);
- enabledRw = reader.getBooleanFlagValue(8);
- disabledRwInOtherNamespace = reader.getBooleanFlagValue(3);
} catch (Exception e) {
- throw new RuntimeException("Cannot read flag in codegen", e);
+ foundPackage = false;
}
+ disabledRw = foundPackage ? reader.getBooleanFlagValue(1) : false;
+ disabledRwExported = foundPackage ? reader.getBooleanFlagValue(2) : false;
+ enabledRw = foundPackage ? reader.getBooleanFlagValue(8) : true;
+ disabledRwInOtherNamespace = foundPackage ? reader.getBooleanFlagValue(3) : false;
isCached = true;
}
private void load_overrides_aconfig_test() {
diff --git a/tools/aconfig/aconfig/src/codegen/rust.rs b/tools/aconfig/aconfig/src/codegen/rust.rs
index 7bc34d6..d318b96 100644
--- a/tools/aconfig/aconfig/src/codegen/rust.rs
+++ b/tools/aconfig/aconfig/src/codegen/rust.rs
@@ -295,7 +295,10 @@
get_boolean_flag_value(&flag_val_map, offset + 1)
.map_err(|err| format!("failed to get flag: {err}"))
},
- None => Err("no context found for package 'com.android.aconfig.test'".to_string())
+ None => {
+ log!(Level::Error, "no context found for package com.android.aconfig.test");
+ Ok(false)
+ }
}
})
});
@@ -339,7 +342,10 @@
get_boolean_flag_value(&flag_val_map, offset + 2)
.map_err(|err| format!("failed to get flag: {err}"))
},
- None => Err("no context found for package 'com.android.aconfig.test'".to_string())
+ None => {
+ log!(Level::Error, "no context found for package com.android.aconfig.test");
+ Ok(false)
+ }
}
})
});
@@ -383,7 +389,10 @@
get_boolean_flag_value(&flag_val_map, offset + 3)
.map_err(|err| format!("failed to get flag: {err}"))
},
- None => Err("no context found for package 'com.android.aconfig.test'".to_string())
+ None => {
+ log!(Level::Error, "no context found for package com.android.aconfig.test");
+ Ok(false)
+ }
}
})
});
@@ -428,7 +437,10 @@
get_boolean_flag_value(&flag_val_map, offset + 8)
.map_err(|err| format!("failed to get flag: {err}"))
},
- None => Err("no context found for package 'com.android.aconfig.test'".to_string())
+ None => {
+ log!(Level::Error, "no context found for package com.android.aconfig.test");
+ Ok(true)
+ }
}
})
});
diff --git a/tools/aconfig/aconfig/src/main.rs b/tools/aconfig/aconfig/src/main.rs
index edb4fd3..e184efe 100644
--- a/tools/aconfig/aconfig/src/main.rs
+++ b/tools/aconfig/aconfig/src/main.rs
@@ -51,8 +51,7 @@
.subcommand(
Command::new("create-cache")
.arg(Arg::new("package").long("package").required(true))
- // TODO(b/312769710): Make this argument required.
- .arg(Arg::new("container").long("container"))
+ .arg(Arg::new("container").long("container").required(true))
.arg(Arg::new("declarations").long("declarations").action(ArgAction::Append))
.arg(Arg::new("values").long("values").action(ArgAction::Append))
.arg(
diff --git a/tools/aconfig/aconfig/templates/FeatureFlags.java.template b/tools/aconfig/aconfig/templates/FeatureFlags.java.template
index 38c8f13..d2799b2 100644
--- a/tools/aconfig/aconfig/templates/FeatureFlags.java.template
+++ b/tools/aconfig/aconfig/templates/FeatureFlags.java.template
@@ -19,4 +19,4 @@
{{ -endif }}
boolean {item.method_name}();
{{ -endfor }}
-}
\ No newline at end of file
+}
diff --git a/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template
index d1cf191..26d3069 100644
--- a/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template
+++ b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template
@@ -36,18 +36,19 @@
{{ if not library_exported }}
private void init() \{
StorageInternalReader reader = null;
+ boolean foundPackage = true;
try \{
reader = new StorageInternalReader("{container}", "{package_name}");
-{{ for namespace_with_flags in namespace_flags }}
-{{ -for flag in namespace_with_flags.flags }}
-{{ if flag.is_read_write }}
- {flag.method_name} = reader.getBooleanFlagValue({flag.flag_offset});
-{{ endif }}
-{{ -endfor }}
-{{ -endfor }}
} catch (Exception e) \{
- throw new RuntimeException("Cannot read flag in codegen", e);
+ foundPackage = false;
}
+ {{ for namespace_with_flags in namespace_flags }}
+ {{ -for flag in namespace_with_flags.flags }}
+ {{ if flag.is_read_write }}
+ {flag.method_name} = foundPackage ? reader.getBooleanFlagValue({flag.flag_offset}) : {flag.default_value};
+ {{ endif }}
+ {{ -endfor }}
+ {{ -endfor }}
isCached = true;
}
{{ endif }}
diff --git a/tools/aconfig/aconfig/templates/cpp_source_file.template b/tools/aconfig/aconfig/templates/cpp_source_file.template
index 852b905..eaaf86f 100644
--- a/tools/aconfig/aconfig/templates/cpp_source_file.template
+++ b/tools/aconfig/aconfig/templates/cpp_source_file.template
@@ -76,7 +76,8 @@
: boolean_start_index_()
{{ -endif }}
, flag_value_file_(nullptr)
- , read_from_new_storage_(false) \{
+ , read_from_new_storage_(false)
+ , package_exists_in_storage_(true) \{
if (access("/metadata/aconfig/boot/enable_only_new_storage", F_OK) == 0) \{
read_from_new_storage_ = true;
@@ -99,6 +100,11 @@
ALOGE("error: failed to get package read context: %s", context.error().c_str());
}
+ if (!(context->package_exists)) \{
+ package_exists_in_storage_ = false;
+ return;
+ }
+
// cache package boolean flag start index
boolean_start_index_ = context->boolean_start_index;
@@ -126,6 +132,10 @@
if (cache_[{item.readwrite_idx}] == -1) \{
{{ if allow_instrumentation- }}
if (read_from_new_storage_) \{
+ if (!package_exists_in_storage_) \{
+ return {item.default_value};
+ }
+
auto value = aconfig_storage::get_boolean_flag_value(
*flag_value_file_,
boolean_start_index_ + {item.flag_offset});
@@ -168,6 +178,8 @@
std::unique_ptr<aconfig_storage::MappedStorageFile> flag_value_file_;
bool read_from_new_storage_;
+
+ bool package_exists_in_storage_;
{{ -endif }}
{{ -endif }}
diff --git a/tools/aconfig/aconfig/templates/rust.template b/tools/aconfig/aconfig/templates/rust.template
index c2f162f..6456360 100644
--- a/tools/aconfig/aconfig/templates/rust.template
+++ b/tools/aconfig/aconfig/templates/rust.template
@@ -51,7 +51,10 @@
get_boolean_flag_value(&flag_val_map, offset + {flag.flag_offset})
.map_err(|err| format!("failed to get flag: \{err}"))
},
- None => Err("no context found for package '{package}'".to_string())
+ None => \{
+ log!(Level::Error, "no context found for package {package}");
+ Ok({flag.default_value})
+ }
}
})
});
diff --git a/tools/aconfig/aconfig_device_paths/Android.bp b/tools/aconfig/aconfig_device_paths/Android.bp
index dda7a55..bdf96ed 100644
--- a/tools/aconfig/aconfig_device_paths/Android.bp
+++ b/tools/aconfig/aconfig_device_paths/Android.bp
@@ -61,8 +61,12 @@
name: "libaconfig_java_host_device_paths_src",
srcs: ["src/HostDeviceProtosTemplate.java"],
out: ["HostDeviceProtos.java"],
- tool_files: ["partition_aconfig_flags_paths.txt"],
- cmd: "sed -e '/TEMPLATE/{r$(location partition_aconfig_flags_paths.txt)' -e 'd}' $(in) > $(out)",
+ tool_files: [
+ "partition_aconfig_flags_paths.txt",
+ "mainline_aconfig_flags_paths.txt",
+ ],
+ cmd: "sed -e '/TEMPLATE/{r$(location partition_aconfig_flags_paths.txt)' -e 'd}' $(in) > $(out).tmp && " +
+ "sed -e '/MAINLINE_T/{r$(location mainline_aconfig_flags_paths.txt)' -e 'd}' $(out).tmp > $(out)",
}
java_library_host {
diff --git a/tools/aconfig/aconfig_device_paths/mainline_aconfig_flags_paths.txt b/tools/aconfig/aconfig_device_paths/mainline_aconfig_flags_paths.txt
new file mode 100644
index 0000000..af73a84
--- /dev/null
+++ b/tools/aconfig/aconfig_device_paths/mainline_aconfig_flags_paths.txt
@@ -0,0 +1,20 @@
+"/apex/com.android.adservices/etc/aconfig_flags.pb",
+"/apex/com.android.appsearch/etc/aconfig_flags.pb",
+"/apex/com.android.art/etc/aconfig_flags.pb",
+"/apex/com.android.btservices/etc/aconfig_flags.pb",
+"/apex/com.android.cellbroadcast/etc/aconfig_flags.pb",
+"/apex/com.android.configinfrastructure/etc/aconfig_flags.pb",
+"/apex/com.android.conscrypt/etc/aconfig_flags.pb",
+"/apex/com.android.devicelock/etc/aconfig_flags.pb",
+"/apex/com.android.healthfitness/etc/aconfig_flags.pb",
+"/apex/com.android.ipsec/etc/aconfig_flags.pb",
+"/apex/com.android.media/etc/aconfig_flags.pb",
+"/apex/com.android.mediaprovider/etc/aconfig_flags.pb",
+"/apex/com.android.ondevicepersonalization/etc/aconfig_flags.pb",
+"/apex/com.android.os.statsd/etc/aconfig_flags.pb",
+"/apex/com.android.permission/etc/aconfig_flags.pb",
+"/apex/com.android.profiling/etc/aconfig_flags.pb",
+"/apex/com.android.tethering/etc/aconfig_flags.pb",
+"/apex/com.android.uwb/etc/aconfig_flags.pb",
+"/apex/com.android.virt/etc/aconfig_flags.pb",
+"/apex/com.android.wifi/etc/aconfig_flags.pb",
diff --git a/tools/aconfig/aconfig_device_paths/src/HostDeviceProtosTemplate.java b/tools/aconfig/aconfig_device_paths/src/HostDeviceProtosTemplate.java
index e2ad40a..e7d0a76 100644
--- a/tools/aconfig/aconfig_device_paths/src/HostDeviceProtosTemplate.java
+++ b/tools/aconfig/aconfig_device_paths/src/HostDeviceProtosTemplate.java
@@ -40,6 +40,10 @@
TEMPLATE
};
+ static final String[] MAINLINE_PATHS = {
+ MAINLINE_T
+ };
+
private static final String APEX_DIR = "/apex";
private static final String RECURSIVELY_LIST_APEX_DIR_COMMAND =
"shell su 0 find /apex | grep aconfig_flags";
@@ -55,7 +59,8 @@
String adbCommandOutput = adbCommandExecutor.executeAdbCommand(
RECURSIVELY_LIST_APEX_DIR_COMMAND);
- if (adbCommandOutput == null) {
+ if (adbCommandOutput == null || adbCommandOutput.isEmpty()) {
+ paths.addAll(Arrays.asList(MAINLINE_PATHS));
return paths;
}
diff --git a/tools/aconfig/aconfig_storage_file/tests/Android.bp b/tools/aconfig/aconfig_storage_file/tests/Android.bp
index 12e4aca..13d3214 100644
--- a/tools/aconfig/aconfig_storage_file/tests/Android.bp
+++ b/tools/aconfig/aconfig_storage_file/tests/Android.bp
@@ -10,10 +10,10 @@
"libbase",
],
data: [
- "package.map",
- "flag.map",
- "flag.val",
- "flag.info",
+ "data/v1/package.map",
+ "data/v1/flag.map",
+ "data/v1/flag.val",
+ "data/v1/flag.info",
],
test_suites: [
"device-tests",
@@ -35,10 +35,10 @@
test_config: "AndroidStorageJaveTest.xml",
sdk_version: "test_current",
data: [
- "package.map",
- "flag.map",
- "flag.val",
- "flag.info",
+ "data/v1/package.map",
+ "data/v1/flag.map",
+ "data/v1/flag.val",
+ "data/v1/flag.info",
],
test_suites: [
"general-tests",
diff --git a/tools/aconfig/aconfig_storage_file/tests/flag.info b/tools/aconfig/aconfig_storage_file/tests/data/v1/flag.info
similarity index 100%
rename from tools/aconfig/aconfig_storage_file/tests/flag.info
rename to tools/aconfig/aconfig_storage_file/tests/data/v1/flag.info
Binary files differ
diff --git a/tools/aconfig/aconfig_storage_file/tests/flag.map b/tools/aconfig/aconfig_storage_file/tests/data/v1/flag.map
similarity index 100%
rename from tools/aconfig/aconfig_storage_file/tests/flag.map
rename to tools/aconfig/aconfig_storage_file/tests/data/v1/flag.map
Binary files differ
diff --git a/tools/aconfig/aconfig_storage_file/tests/flag.val b/tools/aconfig/aconfig_storage_file/tests/data/v1/flag.val
similarity index 100%
rename from tools/aconfig/aconfig_storage_file/tests/flag.val
rename to tools/aconfig/aconfig_storage_file/tests/data/v1/flag.val
Binary files differ
diff --git a/tools/aconfig/aconfig_storage_file/tests/package.map b/tools/aconfig/aconfig_storage_file/tests/data/v1/package.map
similarity index 100%
rename from tools/aconfig/aconfig_storage_file/tests/package.map
rename to tools/aconfig/aconfig_storage_file/tests/data/v1/package.map
Binary files differ
diff --git a/tools/aconfig/aconfig_storage_file/tests/storage_file_test.cpp b/tools/aconfig/aconfig_storage_file/tests/storage_file_test.cpp
index ebd1dd8..3626f72 100644
--- a/tools/aconfig/aconfig_storage_file/tests/storage_file_test.cpp
+++ b/tools/aconfig/aconfig_storage_file/tests/storage_file_test.cpp
@@ -53,7 +53,8 @@
}
TEST(AconfigStorageFileTest, test_list_flag) {
- auto const test_dir = GetExecutableDirectory();
+ auto const test_base_dir = GetExecutableDirectory();
+ auto const test_dir = test_base_dir + "/data/v1";
auto const package_map = test_dir + "/package.map";
auto const flag_map = test_dir + "/flag.map";
auto const flag_val = test_dir + "/flag.val";
@@ -82,7 +83,8 @@
}
TEST(AconfigStorageFileTest, test_list_flag_with_info) {
- auto const test_dir = GetExecutableDirectory();
+ auto const base_test_dir = GetExecutableDirectory();
+ auto const test_dir = base_test_dir + "/data/v1";
auto const package_map = test_dir + "/package.map";
auto const flag_map = test_dir + "/flag.map";
auto const flag_val = test_dir + "/flag.val";
diff --git a/tools/aconfig/aconfig_storage_read_api/Android.bp b/tools/aconfig/aconfig_storage_read_api/Android.bp
index 6ae34f3..80b8ece 100644
--- a/tools/aconfig/aconfig_storage_read_api/Android.bp
+++ b/tools/aconfig/aconfig_storage_read_api/Android.bp
@@ -36,10 +36,10 @@
"librand",
],
data: [
- "tests/package.map",
- "tests/flag.map",
- "tests/flag.val",
- "tests/flag.info",
+ "tests/data/v1/package.map",
+ "tests/data/v1/flag.map",
+ "tests/data/v1/flag.val",
+ "tests/data/v1/flag.info",
],
}
diff --git a/tools/aconfig/aconfig_storage_read_api/src/lib.rs b/tools/aconfig/aconfig_storage_read_api/src/lib.rs
index 988ce63..5104cd0 100644
--- a/tools/aconfig/aconfig_storage_read_api/src/lib.rs
+++ b/tools/aconfig/aconfig_storage_read_api/src/lib.rs
@@ -44,6 +44,7 @@
pub use aconfig_storage_file::{AconfigStorageError, FlagValueType, StorageFileType};
pub use flag_table_query::FlagReadContext;
+pub use mapped_file::map_file;
pub use package_table_query::PackageReadContext;
use aconfig_storage_file::read_u32_from_bytes;
@@ -114,13 +115,13 @@
/// Get the boolean flag value.
///
-/// \input file: mapped flag file
+/// \input file: a byte slice, can be either &Mmap or &MapMut
/// \input index: boolean flag offset
///
/// \return
/// If the provide offset is valid, it returns the boolean flag value, otherwise it
/// returns the error message.
-pub fn get_boolean_flag_value(file: &Mmap, index: u32) -> Result<bool, AconfigStorageError> {
+pub fn get_boolean_flag_value(file: &[u8], index: u32) -> Result<bool, AconfigStorageError> {
find_boolean_flag_value(file, index)
}
@@ -148,7 +149,7 @@
/// Get the flag attribute.
///
-/// \input file: mapped flag info file
+/// \input file: a byte slice, can be either &Mmap or &MapMut
/// \input flag_type: flag value type
/// \input flag_index: flag index
///
@@ -156,7 +157,7 @@
/// If the provide offset is valid, it returns the flag attribute bitfiled, otherwise it
/// returns the error message.
pub fn get_flag_attribute(
- file: &Mmap,
+ file: &[u8],
flag_type: FlagValueType,
flag_index: u32,
) -> Result<u8, AconfigStorageError> {
@@ -412,10 +413,10 @@
let flag_map = storage_dir.clone() + "/maps/mockup.flag.map";
let flag_val = storage_dir.clone() + "/boot/mockup.val";
let flag_info = storage_dir.clone() + "/boot/mockup.info";
- fs::copy("./tests/package.map", &package_map).unwrap();
- fs::copy("./tests/flag.map", &flag_map).unwrap();
- fs::copy("./tests/flag.val", &flag_val).unwrap();
- fs::copy("./tests/flag.info", &flag_info).unwrap();
+ fs::copy("./tests/data/v1/package.map", &package_map).unwrap();
+ fs::copy("./tests/data/v1/flag.map", &flag_map).unwrap();
+ fs::copy("./tests/data/v1/flag.val", &flag_val).unwrap();
+ fs::copy("./tests/data/v1/flag.info", &flag_info).unwrap();
return storage_dir;
}
@@ -507,9 +508,9 @@
#[test]
// this test point locks down flag storage file version number query api
fn test_storage_version_query() {
- assert_eq!(get_storage_file_version("./tests/package.map").unwrap(), 1);
- assert_eq!(get_storage_file_version("./tests/flag.map").unwrap(), 1);
- assert_eq!(get_storage_file_version("./tests/flag.val").unwrap(), 1);
- assert_eq!(get_storage_file_version("./tests/flag.info").unwrap(), 1);
+ assert_eq!(get_storage_file_version("./tests/data/v1/package.map").unwrap(), 1);
+ assert_eq!(get_storage_file_version("./tests/data/v1/flag.map").unwrap(), 1);
+ assert_eq!(get_storage_file_version("./tests/data/v1/flag.val").unwrap(), 1);
+ assert_eq!(get_storage_file_version("./tests/data/v1/flag.info").unwrap(), 1);
}
}
diff --git a/tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs b/tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs
index 5a16645..2c1884a 100644
--- a/tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs
+++ b/tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs
@@ -28,7 +28,7 @@
/// The memory mapped file may have undefined behavior if there are writes to this
/// file after being mapped. Ensure no writes can happen to this file while this
/// mapping stays alive.
-unsafe fn map_file(file_path: &str) -> Result<Mmap, AconfigStorageError> {
+pub unsafe fn map_file(file_path: &str) -> Result<Mmap, AconfigStorageError> {
let file = File::open(file_path)
.map_err(|errmsg| FileReadFail(anyhow!("Failed to open file {}: {}", file_path, errmsg)))?;
unsafe {
@@ -97,10 +97,10 @@
let flag_map = storage_dir.clone() + "/maps/mockup.flag.map";
let flag_val = storage_dir.clone() + "/boot/mockup.val";
let flag_info = storage_dir.clone() + "/boot/mockup.info";
- fs::copy("./tests/package.map", &package_map).unwrap();
- fs::copy("./tests/flag.map", &flag_map).unwrap();
- fs::copy("./tests/flag.val", &flag_val).unwrap();
- fs::copy("./tests/flag.info", &flag_info).unwrap();
+ fs::copy("./tests/data/v1/package.map", &package_map).unwrap();
+ fs::copy("./tests/data/v1/flag.map", &flag_map).unwrap();
+ fs::copy("./tests/data/v1/flag.val", &flag_val).unwrap();
+ fs::copy("./tests/data/v1/flag.info", &flag_info).unwrap();
return storage_dir;
}
@@ -108,9 +108,9 @@
#[test]
fn test_mapped_file_contents() {
let storage_dir = create_test_storage_files();
- map_and_verify(&storage_dir, StorageFileType::PackageMap, "./tests/package.map");
- map_and_verify(&storage_dir, StorageFileType::FlagMap, "./tests/flag.map");
- map_and_verify(&storage_dir, StorageFileType::FlagVal, "./tests/flag.val");
- map_and_verify(&storage_dir, StorageFileType::FlagInfo, "./tests/flag.info");
+ map_and_verify(&storage_dir, StorageFileType::PackageMap, "./tests/data/v1/package.map");
+ map_and_verify(&storage_dir, StorageFileType::FlagMap, "./tests/data/v1/flag.map");
+ map_and_verify(&storage_dir, StorageFileType::FlagVal, "./tests/data/v1/flag.val");
+ map_and_verify(&storage_dir, StorageFileType::FlagInfo, "./tests/data/v1/flag.info");
}
}
diff --git a/tools/aconfig/aconfig_storage_read_api/tests/Android.bp b/tools/aconfig/aconfig_storage_read_api/tests/Android.bp
index ed0c728..b8e510d 100644
--- a/tools/aconfig/aconfig_storage_read_api/tests/Android.bp
+++ b/tools/aconfig/aconfig_storage_read_api/tests/Android.bp
@@ -1,9 +1,10 @@
filegroup {
name: "read_api_test_storage_files",
- srcs: ["package.map",
- "flag.map",
- "flag.val",
- "flag.info"
+ srcs: [
+ "data/v1/package.map",
+ "data/v1/flag.map",
+ "data/v1/flag.val",
+ "data/v1/flag.info",
],
}
diff --git a/tools/aconfig/aconfig_storage_read_api/tests/flag.info b/tools/aconfig/aconfig_storage_read_api/tests/data/v1/flag.info
similarity index 100%
rename from tools/aconfig/aconfig_storage_read_api/tests/flag.info
rename to tools/aconfig/aconfig_storage_read_api/tests/data/v1/flag.info
Binary files differ
diff --git a/tools/aconfig/aconfig_storage_read_api/tests/flag.map b/tools/aconfig/aconfig_storage_read_api/tests/data/v1/flag.map
similarity index 100%
rename from tools/aconfig/aconfig_storage_read_api/tests/flag.map
rename to tools/aconfig/aconfig_storage_read_api/tests/data/v1/flag.map
Binary files differ
diff --git a/tools/aconfig/aconfig_storage_read_api/tests/flag.val b/tools/aconfig/aconfig_storage_read_api/tests/data/v1/flag.val
similarity index 100%
rename from tools/aconfig/aconfig_storage_read_api/tests/flag.val
rename to tools/aconfig/aconfig_storage_read_api/tests/data/v1/flag.val
Binary files differ
diff --git a/tools/aconfig/aconfig_storage_read_api/tests/package.map b/tools/aconfig/aconfig_storage_read_api/tests/data/v1/package.map
similarity index 100%
rename from tools/aconfig/aconfig_storage_read_api/tests/package.map
rename to tools/aconfig/aconfig_storage_read_api/tests/data/v1/package.map
Binary files differ
diff --git a/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.cpp b/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.cpp
index 6d29045..7537643 100644
--- a/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.cpp
+++ b/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.cpp
@@ -45,7 +45,8 @@
}
void SetUp() override {
- auto const test_dir = android::base::GetExecutableDirectory();
+ auto const test_base_dir = android::base::GetExecutableDirectory();
+ auto const test_dir = test_base_dir + "/data/v1";
storage_dir = std::string(root_dir.path);
auto maps_dir = storage_dir + "/maps";
auto boot_dir = storage_dir + "/boot";
diff --git a/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.rs b/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.rs
index afc44d4..0d943f8 100644
--- a/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.rs
+++ b/tools/aconfig/aconfig_storage_read_api/tests/storage_read_api_test.rs
@@ -26,10 +26,10 @@
let flag_map = storage_dir.clone() + "/maps/mockup.flag.map";
let flag_val = storage_dir.clone() + "/boot/mockup.val";
let flag_info = storage_dir.clone() + "/boot/mockup.info";
- fs::copy("./package.map", package_map).unwrap();
- fs::copy("./flag.map", flag_map).unwrap();
- fs::copy("./flag.val", flag_val).unwrap();
- fs::copy("./flag.info", flag_info).unwrap();
+ fs::copy("./data/v1/package.map", package_map).unwrap();
+ fs::copy("./data/v1/flag.map", flag_map).unwrap();
+ fs::copy("./data/v1/flag.val", flag_val).unwrap();
+ fs::copy("./data/v1/flag.info", flag_info).unwrap();
storage_dir
}
@@ -200,9 +200,9 @@
#[test]
fn test_storage_version_query() {
- assert_eq!(get_storage_file_version("./package.map").unwrap(), 1);
- assert_eq!(get_storage_file_version("./flag.map").unwrap(), 1);
- assert_eq!(get_storage_file_version("./flag.val").unwrap(), 1);
- assert_eq!(get_storage_file_version("./flag.info").unwrap(), 1);
+ assert_eq!(get_storage_file_version("./data/v1/package.map").unwrap(), 1);
+ assert_eq!(get_storage_file_version("./data/v1/flag.map").unwrap(), 1);
+ assert_eq!(get_storage_file_version("./data/v1/flag.val").unwrap(), 1);
+ assert_eq!(get_storage_file_version("./data/v1/flag.info").unwrap(), 1);
}
}
diff --git a/tools/aconfig/aconfig_storage_write_api/Android.bp b/tools/aconfig/aconfig_storage_write_api/Android.bp
index 0f1962c..4c882b4 100644
--- a/tools/aconfig/aconfig_storage_write_api/Android.bp
+++ b/tools/aconfig/aconfig_storage_write_api/Android.bp
@@ -16,6 +16,11 @@
"libaconfig_storage_file",
"libaconfig_storage_read_api",
],
+ min_sdk_version: "34",
+ apex_available: [
+ "//apex_available:anyapex",
+ "//apex_available:platform",
+ ],
}
rust_library {
diff --git a/tools/aconfig/aflags/Android.bp b/tools/aconfig/aflags/Android.bp
index 2040cc6..a7aceee 100644
--- a/tools/aconfig/aflags/Android.bp
+++ b/tools/aconfig/aflags/Android.bp
@@ -12,7 +12,7 @@
"libaconfig_device_paths",
"libaconfig_flags",
"libaconfig_protos",
- "libaconfigd_protos",
+ "libaconfigd_protos_rust",
"libaconfig_storage_read_api",
"libaconfig_storage_file",
"libanyhow",
@@ -20,6 +20,10 @@
"libnix",
"libprotobuf",
"libregex",
+ // TODO: b/371021174 remove this fake dependency once we find a proper strategy to
+ // deal with test aconfig libs are not present in storage because they are never used
+ // by the actual build
+ "libaconfig_test_rust_library",
],
}
diff --git a/tools/aconfig/aflags/Cargo.toml b/tools/aconfig/aflags/Cargo.toml
index 7efce6d..d31e232 100644
--- a/tools/aconfig/aflags/Cargo.toml
+++ b/tools/aconfig/aflags/Cargo.toml
@@ -9,10 +9,10 @@
protobuf = "3.2.0"
regex = "1.10.3"
aconfig_protos = { path = "../aconfig_protos" }
-aconfigd_protos = { version = "0.1.0", path = "../../../../../system/server_configurable_flags/aconfigd"}
+aconfigd_protos = { version = "0.1.0", path = "../../../../../packages/modules/ConfigInfrastructure/aconfigd/proto"}
nix = { version = "0.28.0", features = ["user"] }
aconfig_storage_file = { version = "0.1.0", path = "../aconfig_storage_file" }
aconfig_storage_read_api = { version = "0.1.0", path = "../aconfig_storage_read_api" }
clap = {version = "4.5.2" }
aconfig_device_paths = { version = "0.1.0", path = "../aconfig_device_paths" }
-aconfig_flags = { version = "0.1.0", path = "../aconfig_flags" }
\ No newline at end of file
+aconfig_flags = { version = "0.1.0", path = "../aconfig_flags" }
diff --git a/tools/aconfig/aflags/src/load_protos.rs b/tools/aconfig/aflags/src/load_protos.rs
index 90d8599..c5ac8ff 100644
--- a/tools/aconfig/aflags/src/load_protos.rs
+++ b/tools/aconfig/aflags/src/load_protos.rs
@@ -51,7 +51,10 @@
let paths = aconfig_device_paths::parsed_flags_proto_paths()?;
for path in paths {
- let bytes = fs::read(path.clone())?;
+ let Ok(bytes) = fs::read(&path) else {
+ eprintln!("warning: failed to read {:?}", path);
+ continue;
+ };
let parsed_flags: ProtoParsedFlags = protobuf::Message::parse_from_bytes(&bytes)?;
for flag in parsed_flags.parsed_flag {
// TODO(b/334954748): enforce one-container-per-flag invariant.
@@ -60,3 +63,10 @@
}
Ok(result)
}
+
+pub(crate) fn list_containers() -> Result<Vec<String>> {
+ Ok(aconfig_device_paths::parsed_flags_proto_paths()?
+ .into_iter()
+ .map(|p| infer_container(&p))
+ .collect())
+}
diff --git a/tools/aconfig/aflags/src/main.rs b/tools/aconfig/aflags/src/main.rs
index 07b7243..8173bc2 100644
--- a/tools/aconfig/aflags/src/main.rs
+++ b/tools/aconfig/aflags/src/main.rs
@@ -253,6 +253,14 @@
FlagSourceType::DeviceConfig => DeviceConfigSource::list_flags()?,
FlagSourceType::AconfigStorage => AconfigStorageSource::list_flags()?,
};
+
+ if let Some(ref c) = container {
+ ensure!(
+ load_protos::list_containers()?.contains(c),
+ format!("container '{}' not found", &c)
+ );
+ }
+
let flags = (Filter { container }).apply(&flags_unfiltered);
let padding_info = PaddingInfo {
longest_flag_col: flags.iter().map(|f| f.qualified_name().len()).max().unwrap_or(0),
@@ -298,7 +306,7 @@
Command::List { container } => {
if aconfig_flags::auto_generated::enable_only_new_storage() {
list(FlagSourceType::AconfigStorage, container)
- .map_err(|err| anyhow!("storage may not be enabled: {err}"))
+ .map_err(|err| anyhow!("could not list flags: {err}"))
.map(Some)
} else {
list(FlagSourceType::DeviceConfig, container).map(Some)
diff --git a/tools/compliance/go.work b/tools/compliance/go.work
index a24d2ea..506e619 100644
--- a/tools/compliance/go.work
+++ b/tools/compliance/go.work
@@ -1,4 +1,4 @@
-go 1.22
+go 1.23
use (
.
diff --git a/tools/edit_monitor/Android.bp b/tools/edit_monitor/Android.bp
index 08ca155..e613563 100644
--- a/tools/edit_monitor/Android.bp
+++ b/tools/edit_monitor/Android.bp
@@ -35,6 +35,12 @@
pkg_path: "edit_monitor",
srcs: [
"daemon_manager.py",
+ "edit_monitor.py",
+ ],
+ libs: [
+ "asuite_cc_client",
+ "edit_event_proto",
+ "watchdog",
],
}
@@ -52,3 +58,45 @@
unit_test: true,
},
}
+
+python_test_host {
+ name: "edit_monitor_test",
+ main: "edit_monitor_test.py",
+ pkg_path: "edit_monitor",
+ srcs: [
+ "edit_monitor_test.py",
+ ],
+ libs: [
+ "edit_monitor_lib",
+ ],
+ test_options: {
+ unit_test: true,
+ },
+}
+
+python_test_host {
+ name: "edit_monitor_integration_test",
+ main: "edit_monitor_integration_test.py",
+ pkg_path: "testdata",
+ srcs: [
+ "edit_monitor_integration_test.py",
+ ],
+ test_options: {
+ unit_test: true,
+ },
+ data: [
+ ":edit_monitor",
+ ],
+}
+
+python_binary_host {
+ name: "edit_monitor",
+ pkg_path: "edit_monitor",
+ srcs: [
+ "main.py",
+ ],
+ libs: [
+ "edit_monitor_lib",
+ ],
+ main: "main.py",
+}
diff --git a/tools/edit_monitor/daemon_manager.py b/tools/edit_monitor/daemon_manager.py
index 445d849..c0a57ab 100644
--- a/tools/edit_monitor/daemon_manager.py
+++ b/tools/edit_monitor/daemon_manager.py
@@ -13,24 +13,32 @@
# limitations under the License.
+import getpass
import hashlib
import logging
import multiprocessing
import os
import pathlib
+import platform
import signal
import subprocess
import sys
import tempfile
import time
+from atest.metrics import clearcut_client
+from atest.proto import clientanalytics_pb2
+from proto import edit_event_pb2
-DEFAULT_PROCESS_TERMINATION_TIMEOUT_SECONDS = 1
+DEFAULT_PROCESS_TERMINATION_TIMEOUT_SECONDS = 5
DEFAULT_MONITOR_INTERVAL_SECONDS = 5
-DEFAULT_MEMORY_USAGE_THRESHOLD = 2000
+DEFAULT_MEMORY_USAGE_THRESHOLD = 2 * 1024 # 2GB
DEFAULT_CPU_USAGE_THRESHOLD = 200
DEFAULT_REBOOT_TIMEOUT_SECONDS = 60 * 60 * 24
BLOCK_SIGN_FILE = "edit_monitor_block_sign"
+# Enum of the Clearcut log source defined under
+# /google3/wireless/android/play/playlog/proto/log_source_enum.proto
+LOG_SOURCE = 2524
def default_daemon_target():
@@ -46,11 +54,16 @@
binary_path: str,
daemon_target: callable = default_daemon_target,
daemon_args: tuple = (),
+ cclient: clearcut_client.Clearcut | None = None,
):
self.binary_path = binary_path
self.daemon_target = daemon_target
self.daemon_args = daemon_args
+ self.cclient = cclient or clearcut_client.Clearcut(LOG_SOURCE)
+ self.user_name = getpass.getuser()
+ self.host_name = platform.node()
+ self.source_root = os.environ.get("ANDROID_BUILD_TOP", "")
self.pid = os.getpid()
self.daemon_process = None
@@ -70,9 +83,20 @@
logging.warning("Block sign found, exiting...")
return
- self._stop_any_existing_instance()
- self._write_pid_to_pidfile()
- self._start_daemon_process()
+ if self.binary_path.startswith("/google/cog/"):
+ logging.warning("Edit monitor for cog is not supported, exiting...")
+ return
+
+ try:
+ self._stop_any_existing_instance()
+ self._write_pid_to_pidfile()
+ self._start_daemon_process()
+ except Exception as e:
+ logging.exception("Failed to start daemon manager with error %s", e)
+ self._send_error_event_to_clearcut(
+ edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR
+ )
+ raise e
def monitor_daemon(
self,
@@ -114,6 +138,9 @@
logging.error(
"Daemon process is consuming too much resource, killing..."
),
+ self._send_error_event_to_clearcut(
+ edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE
+ )
self._terminate_process(self.daemon_process.pid)
logging.info(
@@ -127,14 +154,24 @@
def stop(self):
"""Stops the daemon process and removes the pidfile."""
- logging.debug("in daemon manager cleanup.")
+ logging.info("in daemon manager cleanup.")
try:
- if self.daemon_process and self.daemon_process.is_alive():
- self._terminate_process(self.daemon_process.pid)
+ if self.daemon_process:
+ # The daemon process might already in termination process,
+ # wait some time before kill it explicitly.
+ self._wait_for_process_terminate(self.daemon_process.pid, 1)
+ if self.daemon_process.is_alive():
+ self._terminate_process(self.daemon_process.pid)
self._remove_pidfile()
- logging.debug("Successfully stopped daemon manager.")
+ logging.info("Successfully stopped daemon manager.")
except Exception as e:
logging.exception("Failed to stop daemon manager with error %s", e)
+ self._send_error_event_to_clearcut(
+ edit_event_pb2.EditEvent.FAILED_TO_STOP_EDIT_MONITOR
+ )
+ sys.exit(1)
+ finally:
+ self.cclient.flush_events()
def reboot(self):
"""Reboots the current process.
@@ -142,7 +179,7 @@
Stops the current daemon manager and reboots the entire process based on
the binary file. Exits directly If the binary file no longer exists.
"""
- logging.debug("Rebooting process based on binary %s.", self.binary_path)
+ logging.info("Rebooting process based on binary %s.", self.binary_path)
# Stop the current daemon manager first.
self.stop()
@@ -156,6 +193,9 @@
os.execv(self.binary_path, sys.argv)
except OSError as e:
logging.exception("Failed to reboot process with error: %s.", e)
+ self._send_error_event_to_clearcut(
+ edit_event_pb2.EditEvent.FAILED_TO_REBOOT_EDIT_MONITOR
+ )
sys.exit(1) # Indicate an error occurred
def cleanup(self):
@@ -167,6 +207,7 @@
that requires immediate cleanup to prevent damanger to the system.
"""
logging.debug("Start cleaning up all existing instances.")
+ self._send_error_event_to_clearcut(edit_event_pb2.EditEvent.FORCE_CLEANUP)
try:
# First places a block sign to prevent any edit monitor process to start.
@@ -223,6 +264,7 @@
p = multiprocessing.Process(
target=self.daemon_target, args=self.daemon_args
)
+ p.daemon = True
p.start()
logging.info("Start subprocess with PID %d", p.pid)
@@ -295,36 +337,28 @@
return pid_file_path
def _get_process_memory_percent(self, pid: int) -> float:
- try:
- with open(f"/proc/{pid}/stat", "r") as f:
- stat_data = f.readline().split()
- # RSS is the 24th field in /proc/[pid]/stat
- rss_pages = int(stat_data[23])
- return rss_pages * 4 / 1024 # Covert to MB
- except (FileNotFoundError, IndexError, ValueError, IOError) as e:
- logging.exception("Failed to get memory usage.")
- raise e
+ with open(f"/proc/{pid}/stat", "r") as f:
+ stat_data = f.readline().split()
+ # RSS is the 24th field in /proc/[pid]/stat
+ rss_pages = int(stat_data[23])
+ return rss_pages * 4 / 1024 # Covert to MB
def _get_process_cpu_percent(self, pid: int, interval: int = 1) -> float:
- try:
- total_start_time = self._get_total_cpu_time(pid)
- with open("/proc/uptime", "r") as f:
- uptime_start = float(f.readline().split()[0])
+ total_start_time = self._get_total_cpu_time(pid)
+ with open("/proc/uptime", "r") as f:
+ uptime_start = float(f.readline().split()[0])
- time.sleep(interval)
+ time.sleep(interval)
- total_end_time = self._get_total_cpu_time(pid)
- with open("/proc/uptime", "r") as f:
- uptime_end = float(f.readline().split()[0])
+ total_end_time = self._get_total_cpu_time(pid)
+ with open("/proc/uptime", "r") as f:
+ uptime_end = float(f.readline().split()[0])
- return (
- (total_end_time - total_start_time)
- / (uptime_end - uptime_start)
- * 100
- )
- except (FileNotFoundError, IndexError, ValueError, IOError) as e:
- logging.exception("Failed to get CPU usage.")
- raise e
+ return (
+ (total_end_time - total_start_time)
+ / (uptime_end - uptime_start)
+ * 100
+ )
def _get_total_cpu_time(self, pid: int) -> float:
with open(f"/proc/{str(pid)}/stat", "r") as f:
@@ -346,4 +380,19 @@
except (FileNotFoundError, IOError, ValueError, TypeError):
logging.exception("Failed to get pid from file path: %s", file)
- return pids
\ No newline at end of file
+ return pids
+
+ def _send_error_event_to_clearcut(self, error_type):
+ edit_monitor_error_event_proto = edit_event_pb2.EditEvent(
+ user_name=self.user_name,
+ host_name=self.host_name,
+ source_root=self.source_root,
+ )
+ edit_monitor_error_event_proto.edit_monitor_error_event.CopyFrom(
+ edit_event_pb2.EditEvent.EditMonitorErrorEvent(error_type=error_type)
+ )
+ log_event = clientanalytics_pb2.LogEvent(
+ event_time_ms=int(time.time() * 1000),
+ source_extension=edit_monitor_error_event_proto.SerializeToString(),
+ )
+ self.cclient.log(log_event)
diff --git a/tools/edit_monitor/daemon_manager_test.py b/tools/edit_monitor/daemon_manager_test.py
index d62eade..e132000 100644
--- a/tools/edit_monitor/daemon_manager_test.py
+++ b/tools/edit_monitor/daemon_manager_test.py
@@ -26,6 +26,7 @@
import unittest
from unittest import mock
from edit_monitor import daemon_manager
+from proto import edit_event_pb2
TEST_BINARY_FILE = '/path/to/test_binary'
@@ -131,6 +132,14 @@
# Verify no daemon process is started.
self.assertIsNone(dm.daemon_process)
+ def test_start_return_directly_if_in_cog_env(self):
+ dm = daemon_manager.DaemonManager(
+ '/google/cog/cloud/user/workspace/edit_monitor'
+ )
+ dm.start()
+ # Verify no daemon process is started.
+ self.assertIsNone(dm.daemon_process)
+
@mock.patch('os.kill')
def test_start_failed_to_kill_existing_instance(self, mock_kill):
mock_kill.side_effect = OSError('Unknown OSError')
@@ -141,9 +150,13 @@
with open(pid_file_path_dir.joinpath(TEST_PID_FILE_PATH), 'w') as f:
f.write('123456')
- with self.assertRaises(OSError) as error:
- dm = daemon_manager.DaemonManager(TEST_BINARY_FILE)
+ fake_cclient = FakeClearcutClient()
+ with self.assertRaises(OSError):
+ dm = daemon_manager.DaemonManager(TEST_BINARY_FILE, cclient=fake_cclient)
dm.start()
+ self._assert_error_event_logged(
+ fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR
+ )
def test_start_failed_to_write_pidfile(self):
pid_file_path_dir = pathlib.Path(self.working_dir.name).joinpath(
@@ -153,40 +166,63 @@
# Makes the directory read-only so write pidfile will fail.
os.chmod(pid_file_path_dir, 0o555)
- with self.assertRaises(PermissionError) as error:
- dm = daemon_manager.DaemonManager(TEST_BINARY_FILE)
+ fake_cclient = FakeClearcutClient()
+ with self.assertRaises(PermissionError):
+ dm = daemon_manager.DaemonManager(TEST_BINARY_FILE, cclient=fake_cclient)
dm.start()
+ self._assert_error_event_logged(
+ fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR
+ )
def test_start_failed_to_start_daemon_process(self):
- with self.assertRaises(TypeError) as error:
+ fake_cclient = FakeClearcutClient()
+ with self.assertRaises(TypeError):
dm = daemon_manager.DaemonManager(
- TEST_BINARY_FILE, daemon_target='wrong_target', daemon_args=(1)
+ TEST_BINARY_FILE,
+ daemon_target='wrong_target',
+ daemon_args=(1),
+ cclient=fake_cclient,
)
dm.start()
+ self._assert_error_event_logged(
+ fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR
+ )
def test_monitor_daemon_subprocess_killed_high_memory_usage(self):
+ fake_cclient = FakeClearcutClient()
dm = daemon_manager.DaemonManager(
TEST_BINARY_FILE,
daemon_target=memory_consume_daemon_target,
daemon_args=(2,),
+ cclient=fake_cclient,
)
dm.start()
dm.monitor_daemon(interval=1, memory_threshold=2)
self.assertTrue(dm.max_memory_usage >= 2)
self.assert_no_subprocess_running()
+ self._assert_error_event_logged(
+ fake_cclient,
+ edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE,
+ )
def test_monitor_daemon_subprocess_killed_high_cpu_usage(self):
+ fake_cclient = FakeClearcutClient()
dm = daemon_manager.DaemonManager(
TEST_BINARY_FILE,
daemon_target=cpu_consume_daemon_target,
daemon_args=(20,),
+ cclient=fake_cclient,
)
dm.start()
dm.monitor_daemon(interval=1, cpu_threshold=20)
self.assertTrue(dm.max_cpu_usage >= 20)
self.assert_no_subprocess_running()
+ self._assert_error_event_logged(
+ fake_cclient,
+ edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE,
+ )
@mock.patch('subprocess.check_output')
def test_monitor_daemon_failed_does_not_matter(self, mock_output):
@@ -200,7 +236,8 @@
)
dm = daemon_manager.DaemonManager(
- binary_file.name, daemon_target=long_running_daemon
+ binary_file.name,
+ daemon_target=long_running_daemon,
)
dm.start()
dm.monitor_daemon(reboot_timeout=0.5)
@@ -219,27 +256,42 @@
@mock.patch('os.kill')
def test_stop_failed_to_kill_daemon_process(self, mock_kill):
mock_kill.side_effect = OSError('Unknown OSError')
+ fake_cclient = FakeClearcutClient()
dm = daemon_manager.DaemonManager(
- TEST_BINARY_FILE, daemon_target=long_running_daemon
+ TEST_BINARY_FILE,
+ daemon_target=long_running_daemon,
+ cclient=fake_cclient,
)
- dm.start()
- dm.stop()
- self.assertTrue(dm.daemon_process.is_alive())
- self.assertTrue(dm.pid_file_path.exists())
+ with self.assertRaises(SystemExit):
+ dm.start()
+ dm.stop()
+ self.assertTrue(dm.daemon_process.is_alive())
+ self.assertTrue(dm.pid_file_path.exists())
+ self._assert_error_event_logged(
+ fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_STOP_EDIT_MONITOR
+ )
@mock.patch('os.remove')
def test_stop_failed_to_remove_pidfile(self, mock_remove):
mock_remove.side_effect = OSError('Unknown OSError')
+ fake_cclient = FakeClearcutClient()
dm = daemon_manager.DaemonManager(
- TEST_BINARY_FILE, daemon_target=long_running_daemon
+ TEST_BINARY_FILE,
+ daemon_target=long_running_daemon,
+ cclient=fake_cclient,
)
- dm.start()
- dm.stop()
- self.assert_no_subprocess_running()
- self.assertTrue(dm.pid_file_path.exists())
+ with self.assertRaises(SystemExit):
+ dm.start()
+ dm.stop()
+ self.assert_no_subprocess_running()
+ self.assertTrue(dm.pid_file_path.exists())
+
+ self._assert_error_event_logged(
+ fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_STOP_EDIT_MONITOR
+ )
@mock.patch('os.execv')
def test_reboot_success(self, mock_execv):
@@ -266,7 +318,7 @@
)
dm.start()
- with self.assertRaises(SystemExit) as cm:
+ with self.assertRaises(SystemExit):
dm.reboot()
mock_execv.assert_not_called()
self.assertEqual(cm.exception.code, 0)
@@ -274,18 +326,24 @@
@mock.patch('os.execv')
def test_reboot_failed(self, mock_execv):
mock_execv.side_effect = OSError('Unknown OSError')
+ fake_cclient = FakeClearcutClient()
binary_file = tempfile.NamedTemporaryFile(
dir=self.working_dir.name, delete=False
)
dm = daemon_manager.DaemonManager(
- binary_file.name, daemon_target=long_running_daemon
+ binary_file.name,
+ daemon_target=long_running_daemon,
+ cclient=fake_cclient,
)
dm.start()
- with self.assertRaises(SystemExit) as cm:
+ with self.assertRaises(SystemExit):
dm.reboot()
self.assertEqual(cm.exception.code, 1)
+ self._assert_error_event_logged(
+ fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_REBOOT_EDIT_MONITOR
+ )
def assert_run_simple_daemon_success(self):
damone_output_file = tempfile.NamedTemporaryFile(
@@ -367,6 +425,33 @@
f.write(str(p.pid))
return p
+ def _assert_error_event_logged(self, fake_cclient, error_type):
+ error_events = fake_cclient.get_sent_events()
+ self.assertEquals(len(error_events), 1)
+ self.assertEquals(
+ edit_event_pb2.EditEvent.FromString(
+ error_events[0].source_extension
+ ).edit_monitor_error_event.error_type,
+ error_type,
+ )
+
+
+class FakeClearcutClient:
+
+ def __init__(self):
+ self.pending_log_events = []
+ self.sent_log_event = []
+
+ def log(self, log_event):
+ self.pending_log_events.append(log_event)
+
+ def flush_events(self):
+ self.sent_log_event.extend(self.pending_log_events)
+ self.pending_log_events.clear()
+
+ def get_sent_events(self):
+ return self.sent_log_event + self.pending_log_events
+
if __name__ == '__main__':
unittest.main()
diff --git a/tools/edit_monitor/edit_monitor.py b/tools/edit_monitor/edit_monitor.py
new file mode 100644
index 0000000..ab528e8
--- /dev/null
+++ b/tools/edit_monitor/edit_monitor.py
@@ -0,0 +1,220 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import getpass
+import logging
+import multiprocessing.connection
+import os
+import pathlib
+import platform
+import threading
+import time
+
+from atest.metrics import clearcut_client
+from atest.proto import clientanalytics_pb2
+from proto import edit_event_pb2
+from watchdog.events import FileSystemEvent
+from watchdog.events import PatternMatchingEventHandler
+from watchdog.observers import Observer
+
+# Enum of the Clearcut log source defined under
+# /google3/wireless/android/play/playlog/proto/log_source_enum.proto
+LOG_SOURCE = 2524
+DEFAULT_FLUSH_INTERVAL_SECONDS = 5
+DEFAULT_SINGLE_EVENTS_SIZE_THRESHOLD = 100
+
+
+class ClearcutEventHandler(PatternMatchingEventHandler):
+
+ def __init__(
+ self,
+ path: str,
+ flush_interval_sec: int,
+ single_events_size_threshold: int,
+ is_dry_run: bool = False,
+ cclient: clearcut_client.Clearcut | None = None,
+ ):
+
+ super().__init__(patterns=["*"], ignore_directories=True)
+ self.root_monitoring_path = path
+ self.flush_interval_sec = flush_interval_sec
+ self.single_events_size_threshold = single_events_size_threshold
+ self.is_dry_run = is_dry_run
+ self.cclient = cclient or clearcut_client.Clearcut(LOG_SOURCE)
+
+ self.user_name = getpass.getuser()
+ self.host_name = platform.node()
+ self.source_root = os.environ.get("ANDROID_BUILD_TOP", "")
+
+ self.pending_events = []
+ self._scheduled_log_thread = None
+ self._pending_events_lock = threading.Lock()
+
+ def on_moved(self, event: FileSystemEvent):
+ self._log_edit_event(event, edit_event_pb2.EditEvent.MOVE)
+
+ def on_created(self, event: FileSystemEvent):
+ self._log_edit_event(event, edit_event_pb2.EditEvent.CREATE)
+
+ def on_deleted(self, event: FileSystemEvent):
+ self._log_edit_event(event, edit_event_pb2.EditEvent.DELETE)
+
+ def on_modified(self, event: FileSystemEvent):
+ self._log_edit_event(event, edit_event_pb2.EditEvent.MODIFY)
+
+ def flushall(self):
+ logging.info("flushing all pending events.")
+ if self._scheduled_log_thread:
+ logging.info("canceling log thread")
+ self._scheduled_log_thread.cancel()
+ self._scheduled_log_thread = None
+
+ self._log_clearcut_events()
+ self.cclient.flush_events()
+
+ def _log_edit_event(
+ self, event: FileSystemEvent, edit_type: edit_event_pb2.EditEvent.EditType
+ ):
+ try:
+ event_time = time.time()
+
+ if self._is_hidden_file(pathlib.Path(event.src_path)):
+ logging.debug("ignore hidden file: %s.", event.src_path)
+ return
+
+ if not self._is_under_git_project(pathlib.Path(event.src_path)):
+ logging.debug(
+ "ignore file %s which does not belong to a git project",
+ event.src_path,
+ )
+ return
+
+ logging.info("%s: %s", event.event_type, event.src_path)
+
+ event_proto = edit_event_pb2.EditEvent(
+ user_name=self.user_name,
+ host_name=self.host_name,
+ source_root=self.source_root,
+ )
+ event_proto.single_edit_event.CopyFrom(
+ edit_event_pb2.EditEvent.SingleEditEvent(
+ file_path=event.src_path, edit_type=edit_type
+ )
+ )
+ with self._pending_events_lock:
+ self.pending_events.append((event_proto, event_time))
+ if not self._scheduled_log_thread:
+ logging.debug(
+ "Scheduling thread to run in %d seconds", self.flush_interval_sec
+ )
+ self._scheduled_log_thread = threading.Timer(
+ self.flush_interval_sec, self._log_clearcut_events
+ )
+ self._scheduled_log_thread.start()
+
+ except Exception:
+ logging.exception("Failed to log edit event.")
+
+ def _is_hidden_file(self, file_path: pathlib.Path) -> bool:
+ return any(
+ part.startswith(".")
+ for part in file_path.relative_to(self.root_monitoring_path).parts
+ )
+
+ def _is_under_git_project(self, file_path: pathlib.Path) -> bool:
+ root_path = pathlib.Path(self.root_monitoring_path).resolve()
+ return any(
+ root_path.joinpath(dir).joinpath('.git').exists()
+ for dir in file_path.relative_to(root_path).parents
+ )
+
+ def _log_clearcut_events(self):
+ with self._pending_events_lock:
+ self._scheduled_log_thread = None
+ edit_events = self.pending_events
+ self.pending_events = []
+
+ pending_events_size = len(edit_events)
+ if pending_events_size > self.single_events_size_threshold:
+ logging.info(
+ "got %d events in %d seconds, sending aggregated events instead",
+ pending_events_size,
+ self.flush_interval_sec,
+ )
+ aggregated_event_time = edit_events[0][1]
+ aggregated_event_proto = edit_event_pb2.EditEvent(
+ user_name=self.user_name,
+ host_name=self.host_name,
+ source_root=self.source_root,
+ )
+ aggregated_event_proto.aggregated_edit_event.CopyFrom(
+ edit_event_pb2.EditEvent.AggregatedEditEvent(
+ num_edits=pending_events_size
+ )
+ )
+ edit_events = [(aggregated_event_proto, aggregated_event_time)]
+
+ if self.is_dry_run:
+ logging.info("Sent %d edit events in dry run.", len(edit_events))
+ return
+
+ for event_proto, event_time in edit_events:
+ log_event = clientanalytics_pb2.LogEvent(
+ event_time_ms=int(event_time * 1000),
+ source_extension=event_proto.SerializeToString(),
+ )
+ self.cclient.log(log_event)
+
+ logging.info("sent %d edit events", len(edit_events))
+
+
+def start(
+ path: str,
+ is_dry_run: bool = False,
+ flush_interval_sec: int = DEFAULT_FLUSH_INTERVAL_SECONDS,
+ single_events_size_threshold: int = DEFAULT_SINGLE_EVENTS_SIZE_THRESHOLD,
+ cclient: clearcut_client.Clearcut | None = None,
+ pipe_sender: multiprocessing.connection.Connection | None = None,
+):
+ """Method to start the edit monitor.
+
+ This is the entry point to start the edit monitor as a subprocess of
+ the daemon manager.
+
+ params:
+ path: The root path to monitor
+ cclient: The clearcut client to send the edit logs.
+ conn: the sender of the pipe to communicate with the deamon manager.
+ """
+ event_handler = ClearcutEventHandler(
+ path, flush_interval_sec, single_events_size_threshold, is_dry_run, cclient)
+ observer = Observer()
+
+ logging.info("Starting observer on path %s.", path)
+ observer.schedule(event_handler, path, recursive=True)
+ observer.start()
+ logging.info("Observer started.")
+ if pipe_sender:
+ pipe_sender.send("Observer started.")
+
+ try:
+ while True:
+ time.sleep(1)
+ finally:
+ event_handler.flushall()
+ observer.stop()
+ observer.join()
+ if pipe_sender:
+ pipe_sender.close()
diff --git a/tools/edit_monitor/edit_monitor_integration_test.py b/tools/edit_monitor/edit_monitor_integration_test.py
new file mode 100644
index 0000000..d7dc7f1
--- /dev/null
+++ b/tools/edit_monitor/edit_monitor_integration_test.py
@@ -0,0 +1,135 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Integration tests for Edit Monitor."""
+
+import glob
+from importlib import resources
+import logging
+import os
+import pathlib
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+
+
+class EditMonitorIntegrationTest(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+ # Configure to print logging to stdout.
+ logging.basicConfig(filename=None, level=logging.DEBUG)
+ console = logging.StreamHandler(sys.stdout)
+ logging.getLogger("").addHandler(console)
+
+ def setUp(self):
+ super().setUp()
+ self.working_dir = tempfile.TemporaryDirectory()
+ self.root_monitoring_path = pathlib.Path(self.working_dir.name).joinpath(
+ "files"
+ )
+ self.root_monitoring_path.mkdir()
+ self.edit_monitor_binary_path = self._import_executable("edit_monitor")
+
+ def tearDown(self):
+ self.working_dir.cleanup()
+ super().tearDown()
+
+ def test_log_single_edit_event_success(self):
+ p = self._start_edit_monitor_process()
+
+ # Create the .git file under the monitoring dir.
+ self.root_monitoring_path.joinpath(".git").touch()
+
+ # Create and modify a file.
+ test_file = self.root_monitoring_path.joinpath("test.txt")
+ with open(test_file, "w") as f:
+ f.write("something")
+
+ # Move the file.
+ test_file_moved = self.root_monitoring_path.joinpath("new_test.txt")
+ test_file.rename(test_file_moved)
+
+ # Delete the file.
+ test_file_moved.unlink()
+
+ # Give some time for the edit monitor to receive the edit event.
+ time.sleep(1)
+ # Stop the edit monitor and flush all events.
+ os.kill(p.pid, signal.SIGINT)
+ p.communicate()
+
+ self.assertEqual(self._get_logged_events_num(), 4)
+
+ def _start_edit_monitor_process(self):
+ command = f"""
+ export TMPDIR="{self.working_dir.name}"
+ {self.edit_monitor_binary_path} --path={self.root_monitoring_path} --dry_run"""
+ p = subprocess.Popen(
+ command,
+ shell=True,
+ text=True,
+ start_new_session=True,
+ executable="/bin/bash",
+ )
+ self._wait_for_observer_start(time_out=5)
+ return p
+
+ def _wait_for_observer_start(self, time_out):
+ start_time = time.time()
+
+ while time.time() < start_time + time_out:
+ log_files = glob.glob(self.working_dir.name + "/edit_monitor_*/*.log")
+ if log_files:
+ with open(log_files[0], "r") as f:
+ for line in f:
+ logging.debug("initial log: %s", line)
+ if line.rstrip("\n").endswith("Observer started."):
+ return
+ else:
+ time.sleep(1)
+
+ self.fail(f"Observer not started in {time_out} seconds.")
+
+ def _get_logged_events_num(self):
+ log_files = glob.glob(self.working_dir.name + "/edit_monitor_*/*.log")
+ self.assertEqual(len(log_files), 1)
+
+ with open(log_files[0], "r") as f:
+ for line in f:
+ logging.debug("complete log: %s", line)
+ if line.rstrip("\n").endswith("in dry run."):
+ return int(line.split(":")[-1].split(" ")[2])
+
+ return 0
+
+ def _import_executable(self, executable_name: str) -> pathlib.Path:
+ binary_dir = pathlib.Path(self.working_dir.name).joinpath("binary")
+ binary_dir.mkdir()
+ executable_path = binary_dir.joinpath(executable_name)
+ with resources.as_file(
+ resources.files("testdata").joinpath(executable_name)
+ ) as binary:
+ shutil.copy(binary, executable_path)
+ executable_path.chmod(0o755)
+ return executable_path
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tools/edit_monitor/edit_monitor_test.py b/tools/edit_monitor/edit_monitor_test.py
new file mode 100644
index 0000000..64a3871
--- /dev/null
+++ b/tools/edit_monitor/edit_monitor_test.py
@@ -0,0 +1,301 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for Edit Monitor."""
+
+import logging
+import multiprocessing
+import os
+import pathlib
+import signal
+import sys
+import tempfile
+import time
+import unittest
+
+from atest.proto import clientanalytics_pb2
+from edit_monitor import edit_monitor
+from proto import edit_event_pb2
+
+
+class EditMonitorTest(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+ # Configure to print logging to stdout.
+ logging.basicConfig(filename=None, level=logging.DEBUG)
+ console = logging.StreamHandler(sys.stdout)
+ logging.getLogger('').addHandler(console)
+
+ def setUp(self):
+ super().setUp()
+ self.working_dir = tempfile.TemporaryDirectory()
+ self.root_monitoring_path = pathlib.Path(self.working_dir.name).joinpath(
+ 'files'
+ )
+ self.root_monitoring_path.mkdir()
+ self.log_event_dir = pathlib.Path(self.working_dir.name).joinpath('logs')
+ self.log_event_dir.mkdir()
+
+ def tearDown(self):
+ self.working_dir.cleanup()
+ super().tearDown()
+
+ def test_log_single_edit_event_success(self):
+ # Create the .git file under the monitoring dir.
+ self.root_monitoring_path.joinpath('.git').touch()
+ fake_cclient = FakeClearcutClient(
+ log_output_file=self.log_event_dir.joinpath('logs.output')
+ )
+ p = self._start_test_edit_monitor_process(fake_cclient)
+
+ # Create and modify a file.
+ test_file = self.root_monitoring_path.joinpath('test.txt')
+ with open(test_file, 'w') as f:
+ f.write('something')
+ # Move the file.
+ test_file_moved = self.root_monitoring_path.joinpath('new_test.txt')
+ test_file.rename(test_file_moved)
+ # Delete the file.
+ test_file_moved.unlink()
+ # Give some time for the edit monitor to receive the edit event.
+ time.sleep(1)
+ # Stop the edit monitor and flush all events.
+ os.kill(p.pid, signal.SIGINT)
+ p.join()
+
+ logged_events = self._get_logged_events()
+ self.assertEqual(len(logged_events), 4)
+ expected_create_event = edit_event_pb2.EditEvent.SingleEditEvent(
+ file_path=str(
+ self.root_monitoring_path.joinpath('test.txt').resolve()
+ ),
+ edit_type=edit_event_pb2.EditEvent.CREATE,
+ )
+ expected_modify_event = edit_event_pb2.EditEvent.SingleEditEvent(
+ file_path=str(
+ self.root_monitoring_path.joinpath('test.txt').resolve()
+ ),
+ edit_type=edit_event_pb2.EditEvent.MODIFY,
+ )
+ expected_move_event = edit_event_pb2.EditEvent.SingleEditEvent(
+ file_path=str(
+ self.root_monitoring_path.joinpath('test.txt').resolve()
+ ),
+ edit_type=edit_event_pb2.EditEvent.MOVE,
+ )
+ expected_delete_event = edit_event_pb2.EditEvent.SingleEditEvent(
+ file_path=str(
+ self.root_monitoring_path.joinpath('new_test.txt').resolve()
+ ),
+ edit_type=edit_event_pb2.EditEvent.DELETE,
+ )
+ self.assertEqual(
+ expected_create_event,
+ edit_event_pb2.EditEvent.FromString(
+ logged_events[0].source_extension
+ ).single_edit_event,
+ )
+ self.assertEqual(
+ expected_modify_event,
+ edit_event_pb2.EditEvent.FromString(
+ logged_events[1].source_extension
+ ).single_edit_event,
+ )
+ self.assertEqual(
+ expected_move_event,
+ edit_event_pb2.EditEvent.FromString(
+ logged_events[2].source_extension
+ ).single_edit_event,
+ )
+ self.assertEqual(
+ expected_delete_event,
+ edit_event_pb2.EditEvent.FromString(
+ logged_events[3].source_extension
+ ).single_edit_event,
+ )
+
+
+ def test_log_aggregated_edit_event_success(self):
+ # Create the .git file under the monitoring dir.
+ self.root_monitoring_path.joinpath('.git').touch()
+ fake_cclient = FakeClearcutClient(
+ log_output_file=self.log_event_dir.joinpath('logs.output')
+ )
+ p = self._start_test_edit_monitor_process(fake_cclient)
+
+ # Create 6 test files
+ for i in range(6):
+ test_file = self.root_monitoring_path.joinpath('test_' + str(i))
+ test_file.touch()
+
+ # Give some time for the edit monitor to receive the edit event.
+ time.sleep(1)
+ # Stop the edit monitor and flush all events.
+ os.kill(p.pid, signal.SIGINT)
+ p.join()
+
+ logged_events = self._get_logged_events()
+ self.assertEqual(len(logged_events), 1)
+
+ expected_aggregated_edit_event = (
+ edit_event_pb2.EditEvent.AggregatedEditEvent(
+ num_edits=6,
+ )
+ )
+
+ self.assertEqual(
+ expected_aggregated_edit_event,
+ edit_event_pb2.EditEvent.FromString(
+ logged_events[0].source_extension
+ ).aggregated_edit_event,
+ )
+
+ def test_do_not_log_edit_event_for_directory_change(self):
+ # Create the .git file under the monitoring dir.
+ self.root_monitoring_path.joinpath('.git').touch()
+ fake_cclient = FakeClearcutClient(
+ log_output_file=self.log_event_dir.joinpath('logs.output')
+ )
+ p = self._start_test_edit_monitor_process(fake_cclient)
+
+ # Create a sub directory
+ self.root_monitoring_path.joinpath('test_dir').mkdir()
+ # Give some time for the edit monitor to receive the edit event.
+ time.sleep(1)
+ # Stop the edit monitor and flush all events.
+ os.kill(p.pid, signal.SIGINT)
+ p.join()
+
+ logged_events = self._get_logged_events()
+ self.assertEqual(len(logged_events), 0)
+
+ def test_do_not_log_edit_event_for_hidden_file(self):
+ # Create the .git file under the monitoring dir.
+ self.root_monitoring_path.joinpath('.git').touch()
+ fake_cclient = FakeClearcutClient(
+ log_output_file=self.log_event_dir.joinpath('logs.output')
+ )
+ p = self._start_test_edit_monitor_process(fake_cclient)
+
+ # Create a hidden file.
+ self.root_monitoring_path.joinpath('.test.txt').touch()
+ # Create a hidden dir.
+ hidden_dir = self.root_monitoring_path.joinpath('.test')
+ hidden_dir.mkdir()
+ hidden_dir.joinpath('test.txt').touch()
+ # Give some time for the edit monitor to receive the edit event.
+ time.sleep(1)
+ # Stop the edit monitor and flush all events.
+ os.kill(p.pid, signal.SIGINT)
+ p.join()
+
+ logged_events = self._get_logged_events()
+ self.assertEqual(len(logged_events), 0)
+
+ def test_do_not_log_edit_event_for_non_git_project_file(self):
+ fake_cclient = FakeClearcutClient(
+ log_output_file=self.log_event_dir.joinpath('logs.output')
+ )
+ p = self._start_test_edit_monitor_process(fake_cclient)
+
+ # Create a file.
+ self.root_monitoring_path.joinpath('test.txt').touch()
+ # Create a file under a sub dir.
+ sub_dir = self.root_monitoring_path.joinpath('.test')
+ sub_dir.mkdir()
+ sub_dir.joinpath('test.txt').touch()
+ # Give some time for the edit monitor to receive the edit event.
+ time.sleep(1)
+ # Stop the edit monitor and flush all events.
+ os.kill(p.pid, signal.SIGINT)
+ p.join()
+
+ logged_events = self._get_logged_events()
+ self.assertEqual(len(logged_events), 0)
+
+ def test_log_edit_event_fail(self):
+ # Create the .git file under the monitoring dir.
+ self.root_monitoring_path.joinpath('.git').touch()
+ fake_cclient = FakeClearcutClient(
+ log_output_file=self.log_event_dir.joinpath('logs.output'),
+ raise_log_exception=True,
+ )
+ p = self._start_test_edit_monitor_process(fake_cclient)
+
+ # Create a file.
+ self.root_monitoring_path.joinpath('test.txt').touch()
+ # Give some time for the edit monitor to receive the edit event.
+ time.sleep(1)
+ # Stop the edit monitor and flush all events.
+ os.kill(p.pid, signal.SIGINT)
+ p.join()
+
+ logged_events = self._get_logged_events()
+ self.assertEqual(len(logged_events), 0)
+
+ def _start_test_edit_monitor_process(
+ self, cclient
+ ) -> multiprocessing.Process:
+ receiver, sender = multiprocessing.Pipe()
+ # Start edit monitor in a subprocess.
+ p = multiprocessing.Process(
+ target=edit_monitor.start,
+ args=(str(self.root_monitoring_path.resolve()), False, 0.5, 5, cclient, sender),
+ )
+ p.daemon = True
+ p.start()
+
+ # Wait until observer started.
+ received_data = receiver.recv()
+ self.assertEquals(received_data, 'Observer started.')
+
+ receiver.close()
+ return p
+
+ def _get_logged_events(self):
+ with open(self.log_event_dir.joinpath('logs.output'), 'rb') as f:
+ data = f.read()
+
+ return [
+ clientanalytics_pb2.LogEvent.FromString(record)
+ for record in data.split(b'\x00')
+ if record
+ ]
+
+
+class FakeClearcutClient:
+
+ def __init__(self, log_output_file, raise_log_exception=False):
+ self.pending_log_events = []
+ self.raise_log_exception = raise_log_exception
+ self.log_output_file = log_output_file
+
+ def log(self, log_event):
+ if self.raise_log_exception:
+ raise Exception('unknown exception')
+ self.pending_log_events.append(log_event)
+
+ def flush_events(self):
+ delimiter = b'\x00' # Use a null byte as the delimiter
+ with open(self.log_output_file, 'wb') as f:
+ for log_event in self.pending_log_events:
+ f.write(log_event.SerializeToString() + delimiter)
+
+ self.pending_log_events.clear()
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tools/edit_monitor/main.py b/tools/edit_monitor/main.py
new file mode 100644
index 0000000..49385f1
--- /dev/null
+++ b/tools/edit_monitor/main.py
@@ -0,0 +1,118 @@
+# Copyright 2024, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import logging
+import os
+import signal
+import sys
+import tempfile
+
+from edit_monitor import daemon_manager
+from edit_monitor import edit_monitor
+
+
+def create_arg_parser():
+ """Creates an instance of the default arg parser."""
+
+ parser = argparse.ArgumentParser(
+ description=(
+ 'Monitors edits in Android source code and uploads the edit logs.'
+ ),
+ add_help=True,
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ )
+
+ parser.add_argument(
+ '--path',
+ type=str,
+ required=True,
+ help='Root path to monitor the edit events.',
+ )
+
+ parser.add_argument(
+ '--dry_run',
+ action='store_true',
+ help='Dry run the edit monitor. This starts the edit monitor process without actually send the edit logs to clearcut.',
+ )
+
+ parser.add_argument(
+ '--force_cleanup',
+ action='store_true',
+ help=(
+ 'Instead of start a new edit monitor, force stop all existing edit'
+ ' monitors in the system. This option is only used in emergent cases'
+ ' when we want to prevent user damage by the edit monitor.'
+ ),
+ )
+
+ parser.add_argument(
+ '--verbose',
+ action='store_true',
+ help=(
+ 'Log verbose info in the log file for debugging purpose.'
+ ),
+ )
+
+ return parser
+
+
+def configure_logging(verbose=False):
+ root_logging_dir = tempfile.mkdtemp(prefix='edit_monitor_')
+ _, log_path = tempfile.mkstemp(dir=root_logging_dir, suffix='.log')
+
+ log_fmt = '%(asctime)s %(filename)s:%(lineno)s:%(levelname)s: %(message)s'
+ date_fmt = '%Y-%m-%d %H:%M:%S'
+ log_level = logging.DEBUG if verbose else logging.INFO
+
+ logging.basicConfig(
+ filename=log_path, level=log_level, format=log_fmt, datefmt=date_fmt
+ )
+ # Filter out logs from inotify_buff to prevent log pollution.
+ logging.getLogger('watchdog.observers.inotify_buffer').addFilter(
+ lambda record: record.filename != 'inotify_buffer.py')
+ print(f'logging to file {log_path}')
+
+
+def term_signal_handler(_signal_number, _frame):
+ logging.info('Process %d received SIGTERM, Terminating...', os.getpid())
+ sys.exit(0)
+
+
+def main(argv: list[str]):
+ args = create_arg_parser().parse_args(argv[1:])
+ configure_logging(args.verbose)
+ if args.dry_run:
+ logging.info('This is a dry run.')
+ dm = daemon_manager.DaemonManager(
+ binary_path=argv[0],
+ daemon_target=edit_monitor.start,
+ daemon_args=(args.path, args.dry_run),
+ )
+
+ if args.force_cleanup:
+ dm.cleanup()
+
+ try:
+ dm.start()
+ dm.monitor_daemon()
+ except Exception:
+ logging.exception('Unexpected exception raised when run daemon.')
+ finally:
+ dm.stop()
+
+
+if __name__ == '__main__':
+ signal.signal(signal.SIGTERM, term_signal_handler)
+ main(sys.argv)
diff --git a/tools/edit_monitor/proto/edit_event.proto b/tools/edit_monitor/proto/edit_event.proto
index b3630bc..dc3d3f6 100644
--- a/tools/edit_monitor/proto/edit_event.proto
+++ b/tools/edit_monitor/proto/edit_event.proto
@@ -36,8 +36,6 @@
// Event that logs errors happened in the edit monitor.
message EditMonitorErrorEvent {
ErrorType error_type = 1;
- string error_msg = 2;
- string stack_trace = 3;
}
// ------------------------
diff --git a/tools/filelistdiff/Android.bp b/tools/filelistdiff/Android.bp
index ab766d6..3826e50 100644
--- a/tools/filelistdiff/Android.bp
+++ b/tools/filelistdiff/Android.bp
@@ -24,4 +24,9 @@
prebuilt_etc_host {
name: "system_image_diff_allowlist",
src: "allowlist",
-}
\ No newline at end of file
+}
+
+prebuilt_etc_host {
+ name: "system_image_diff_allowlist_next",
+ src: "allowlist_next",
+}
diff --git a/tools/filelistdiff/allowlist b/tools/filelistdiff/allowlist
index ae8a662..eb78587 100644
--- a/tools/filelistdiff/allowlist
+++ b/tools/filelistdiff/allowlist
@@ -1,9 +1,3 @@
-# Known diffs only in the KATI system image
-framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex
-framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex.fsv_meta
-framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex
-framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex.fsv_meta
-
# Known diffs that are installed in either system image with the configuration
# b/353429422
init.environ.rc
diff --git a/tools/filelistdiff/allowlist_next b/tools/filelistdiff/allowlist_next
new file mode 100644
index 0000000..8f91c9f
--- /dev/null
+++ b/tools/filelistdiff/allowlist_next
@@ -0,0 +1,9 @@
+# Allowlist only for the next release configuration.
+# TODO(b/369678122): The list will be cleared when the trunk configurations are
+# available to the next.
+
+# KATI only installed files
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.odex.fsv_meta
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex
+framework/oat/x86_64/apex@com.android.compos@javalib@service-compos.jar@classes.vdex.fsv_meta
diff --git a/tools/filelistdiff/file_list_diff.py b/tools/filelistdiff/file_list_diff.py
index 30ed107..fbbfedf 100644
--- a/tools/filelistdiff/file_list_diff.py
+++ b/tools/filelistdiff/file_list_diff.py
@@ -19,19 +19,26 @@
COLOR_ERROR = '\033[91m'
COLOR_NORMAL = '\033[0m'
-def find_unique_items(kati_installed_files, soong_installed_files, allowlist, system_module_name):
+def find_unique_items(kati_installed_files, soong_installed_files, system_module_name, allowlists):
with open(kati_installed_files, 'r') as kati_list_file, \
- open(soong_installed_files, 'r') as soong_list_file, \
- open(allowlist, 'r') as allowlist_file:
+ open(soong_installed_files, 'r') as soong_list_file:
kati_files = set(kati_list_file.read().split())
soong_files = set(soong_list_file.read().split())
- allowed_files = set(filter(lambda x: len(x), map(lambda x: x.lstrip().split('#',1)[0].rstrip() , allowlist_file.read().split('\n'))))
+
+ allowed_files = set()
+ for allowlist in allowlists:
+ with open(allowlist, 'r') as allowlist_file:
+ allowed_files.update(set(filter(lambda x: len(x), map(lambda x: x.lstrip().split('#',1)[0].rstrip() , allowlist_file.read().split('\n')))))
def is_unknown_diff(filepath):
- return not filepath in allowed_files
+ return filepath not in allowed_files
+
+ def is_unnecessary_allowlist(filepath):
+ return filepath not in kati_files.symmetric_difference(soong_files)
unique_in_kati = set(filter(is_unknown_diff, kati_files - soong_files))
unique_in_soong = set(filter(is_unknown_diff, soong_files - kati_files))
+ unnecessary_allowlists = set(filter(is_unnecessary_allowlist, allowed_files))
if unique_in_kati:
print('')
@@ -50,7 +57,15 @@
for item in sorted(unique_in_soong):
print(' '+item)
- if unique_in_kati or unique_in_soong:
+ if unnecessary_allowlists:
+ print('')
+ print(f'{COLOR_ERROR}Unnecessary files in allowlist.{COLOR_NORMAL}')
+ print('Please remove these entries from build/make/tools/filelistdiff/allowlist')
+ for item in sorted(unnecessary_allowlists):
+ print(' '+item)
+
+
+ if unique_in_kati or unique_in_soong or unnecessary_allowlists:
print('')
sys.exit(1)
@@ -60,8 +75,8 @@
parser.add_argument('kati_installed_file_list')
parser.add_argument('soong_installed_file_list')
- parser.add_argument('allowlist')
parser.add_argument('system_module_name')
+ parser.add_argument('--allowlists', nargs='+')
args = parser.parse_args()
- find_unique_items(args.kati_installed_file_list, args.soong_installed_file_list, args.allowlist, args.system_module_name)
\ No newline at end of file
+ find_unique_items(args.kati_installed_file_list, args.soong_installed_file_list, args.system_module_name, args.allowlists)
\ No newline at end of file
diff --git a/tools/ide_query/ide_query.go b/tools/ide_query/ide_query.go
index 89ac78f..c7cf5ed 100644
--- a/tools/ide_query/ide_query.go
+++ b/tools/ide_query/ide_query.go
@@ -293,11 +293,19 @@
// If a file is covered by multiple modules, the first module is returned.
func findJavaModules(paths []string, modules map[string]*javaModule) map[string]string {
ret := make(map[string]string)
- for name, module := range modules {
+ // A file may be part of multiple modules. To make the result deterministic,
+ // check the modules in sorted order.
+ keys := make([]string, 0, len(modules))
+ for name := range modules {
+ keys = append(keys, name)
+ }
+ slices.Sort(keys)
+ for _, name := range keys {
if strings.HasSuffix(name, ".impl") {
continue
}
+ module := modules[name]
for i, p := range paths {
if slices.Contains(module.Srcs, p) {
ret[p] = name
@@ -341,6 +349,8 @@
Id: moduleName,
Language: pb.Language_LANGUAGE_JAVA,
SourceFilePaths: m.Srcs,
+ GeneratedFiles: genFiles(env, m),
+ DependencyIds: m.Deps,
}
unitsById[u.Id] = u
@@ -355,14 +365,10 @@
continue
}
- var paths []string
- paths = append(paths, mod.Srcs...)
- paths = append(paths, mod.SrcJars...)
- paths = append(paths, mod.Jars...)
unitsById[name] = &pb.BuildableUnit{
Id: name,
SourceFilePaths: mod.Srcs,
- GeneratedFiles: genFiles(env, paths),
+ GeneratedFiles: genFiles(env, mod),
DependencyIds: mod.Deps,
}
@@ -380,8 +386,13 @@
}
// genFiles returns the generated files (paths that start with outDir/) for the
-// given paths. Generated files that do not exist are ignored.
-func genFiles(env Env, paths []string) []*pb.GeneratedFile {
+// given module. Generated files that do not exist are ignored.
+func genFiles(env Env, mod *javaModule) []*pb.GeneratedFile {
+ var paths []string
+ paths = append(paths, mod.Srcs...)
+ paths = append(paths, mod.SrcJars...)
+ paths = append(paths, mod.Jars...)
+
prefix := env.OutDir + "/"
var ret []*pb.GeneratedFile
for _, p := range paths {
diff --git a/tools/metadata/Android.bp b/tools/metadata/Android.bp
deleted file mode 100644
index 77d106d..0000000
--- a/tools/metadata/Android.bp
+++ /dev/null
@@ -1,16 +0,0 @@
-package {
- default_applicable_licenses: ["Android-Apache-2.0"],
-}
-
-blueprint_go_binary {
- name: "metadata",
- deps: [
- "soong-testing-test_spec_proto",
- "soong-testing-code_metadata_proto",
- "soong-testing-code_metadata_internal_proto",
- "golang-protobuf-proto",
- ],
- srcs: [
- "generator.go",
- ]
-}
\ No newline at end of file
diff --git a/tools/metadata/OWNERS b/tools/metadata/OWNERS
deleted file mode 100644
index 03bcdf1..0000000
--- a/tools/metadata/OWNERS
+++ /dev/null
@@ -1,4 +0,0 @@
-dariofreni@google.com
-joeo@google.com
-ronish@google.com
-caditya@google.com
diff --git a/tools/metadata/generator.go b/tools/metadata/generator.go
deleted file mode 100644
index b7668be..0000000
--- a/tools/metadata/generator.go
+++ /dev/null
@@ -1,328 +0,0 @@
-package main
-
-import (
- "flag"
- "fmt"
- "io"
- "log"
- "os"
- "sort"
- "strings"
- "sync"
-
- "android/soong/testing/code_metadata_internal_proto"
- "android/soong/testing/code_metadata_proto"
- "android/soong/testing/test_spec_proto"
- "google.golang.org/protobuf/proto"
-)
-
-type keyToLocksMap struct {
- locks sync.Map
-}
-
-func (kl *keyToLocksMap) GetLockForKey(key string) *sync.Mutex {
- mutex, _ := kl.locks.LoadOrStore(key, &sync.Mutex{})
- return mutex.(*sync.Mutex)
-}
-
-// Define a struct to hold the combination of team ID and multi-ownership flag for validation
-type sourceFileAttributes struct {
- TeamID string
- MultiOwnership bool
- Path string
-}
-
-func getSortedKeys(syncMap *sync.Map) []string {
- var allKeys []string
- syncMap.Range(
- func(key, _ interface{}) bool {
- allKeys = append(allKeys, key.(string))
- return true
- },
- )
-
- sort.Strings(allKeys)
- return allKeys
-}
-
-// writeProtoToFile marshals a protobuf message and writes it to a file
-func writeProtoToFile(outputFile string, message proto.Message) {
- data, err := proto.Marshal(message)
- if err != nil {
- log.Fatal(err)
- }
- file, err := os.Create(outputFile)
- if err != nil {
- log.Fatal(err)
- }
- defer file.Close()
-
- _, err = file.Write(data)
- if err != nil {
- log.Fatal(err)
- }
-}
-
-func readFileToString(filePath string) string {
- file, err := os.Open(filePath)
- if err != nil {
- log.Fatal(err)
- }
- defer file.Close()
-
- data, err := io.ReadAll(file)
- if err != nil {
- log.Fatal(err)
- }
- return string(data)
-}
-
-func writeEmptyOutputProto(outputFile string, metadataRule string) {
- file, err := os.Create(outputFile)
- if err != nil {
- log.Fatal(err)
- }
- var message proto.Message
- if metadataRule == "test_spec" {
- message = &test_spec_proto.TestSpec{}
- } else if metadataRule == "code_metadata" {
- message = &code_metadata_proto.CodeMetadata{}
- }
- data, err := proto.Marshal(message)
- if err != nil {
- log.Fatal(err)
- }
- defer file.Close()
-
- _, err = file.Write([]byte(data))
- if err != nil {
- log.Fatal(err)
- }
-}
-
-func processTestSpecProtobuf(
- filePath string, ownershipMetadataMap *sync.Map, keyLocks *keyToLocksMap,
- errCh chan error, wg *sync.WaitGroup,
-) {
- defer wg.Done()
-
- fileContent := strings.TrimRight(readFileToString(filePath), "\n")
- testData := test_spec_proto.TestSpec{}
- err := proto.Unmarshal([]byte(fileContent), &testData)
- if err != nil {
- errCh <- err
- return
- }
-
- ownershipMetadata := testData.GetOwnershipMetadataList()
- for _, metadata := range ownershipMetadata {
- key := metadata.GetTargetName()
- lock := keyLocks.GetLockForKey(key)
- lock.Lock()
-
- value, loaded := ownershipMetadataMap.LoadOrStore(
- key, []*test_spec_proto.TestSpec_OwnershipMetadata{metadata},
- )
- if loaded {
- existingMetadata := value.([]*test_spec_proto.TestSpec_OwnershipMetadata)
- isDuplicate := false
- for _, existing := range existingMetadata {
- if metadata.GetTrendyTeamId() != existing.GetTrendyTeamId() {
- errCh <- fmt.Errorf(
- "Conflicting trendy team IDs found for %s at:\n%s with teamId"+
- ": %s,\n%s with teamId: %s",
- key,
- metadata.GetPath(), metadata.GetTrendyTeamId(), existing.GetPath(),
- existing.GetTrendyTeamId(),
- )
-
- lock.Unlock()
- return
- }
- if metadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && metadata.GetPath() == existing.GetPath() {
- isDuplicate = true
- break
- }
- }
- if !isDuplicate {
- existingMetadata = append(existingMetadata, metadata)
- ownershipMetadataMap.Store(key, existingMetadata)
- }
- }
-
- lock.Unlock()
- }
-}
-
-// processCodeMetadataProtobuf processes CodeMetadata protobuf files
-func processCodeMetadataProtobuf(
- filePath string, ownershipMetadataMap *sync.Map, sourceFileMetadataMap *sync.Map, keyLocks *keyToLocksMap,
- errCh chan error, wg *sync.WaitGroup,
-) {
- defer wg.Done()
-
- fileContent := strings.TrimRight(readFileToString(filePath), "\n")
- internalCodeData := code_metadata_internal_proto.CodeMetadataInternal{}
- err := proto.Unmarshal([]byte(fileContent), &internalCodeData)
- if err != nil {
- errCh <- err
- return
- }
-
- // Process each TargetOwnership entry
- for _, internalMetadata := range internalCodeData.GetTargetOwnershipList() {
- key := internalMetadata.GetTargetName()
- lock := keyLocks.GetLockForKey(key)
- lock.Lock()
-
- for _, srcFile := range internalMetadata.GetSourceFiles() {
- srcFileKey := srcFile
- srcFileLock := keyLocks.GetLockForKey(srcFileKey)
- srcFileLock.Lock()
- attributes := sourceFileAttributes{
- TeamID: internalMetadata.GetTrendyTeamId(),
- MultiOwnership: internalMetadata.GetMultiOwnership(),
- Path: internalMetadata.GetPath(),
- }
-
- existingAttributes, exists := sourceFileMetadataMap.Load(srcFileKey)
- if exists {
- existing := existingAttributes.(sourceFileAttributes)
- if attributes.TeamID != existing.TeamID && (!attributes.MultiOwnership || !existing.MultiOwnership) {
- errCh <- fmt.Errorf(
- "Conflict found for source file %s covered at %s with team ID: %s. Existing team ID: %s and path: %s."+
- " If multi-ownership is required, multiOwnership should be set to true in all test_spec modules using this target. "+
- "Multiple-ownership in general is discouraged though as it make infrastructure around android relying on this information pick up a random value when it needs only one.",
- srcFile, internalMetadata.GetPath(), attributes.TeamID, existing.TeamID, existing.Path,
- )
- srcFileLock.Unlock()
- lock.Unlock()
- return
- }
- } else {
- // Store the metadata if no conflict
- sourceFileMetadataMap.Store(srcFileKey, attributes)
- }
- srcFileLock.Unlock()
- }
-
- value, loaded := ownershipMetadataMap.LoadOrStore(
- key, []*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership{internalMetadata},
- )
- if loaded {
- existingMetadata := value.([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership)
- isDuplicate := false
- for _, existing := range existingMetadata {
- if internalMetadata.GetTrendyTeamId() == existing.GetTrendyTeamId() && internalMetadata.GetPath() == existing.GetPath() {
- isDuplicate = true
- break
- }
- }
- if !isDuplicate {
- existingMetadata = append(existingMetadata, internalMetadata)
- ownershipMetadataMap.Store(key, existingMetadata)
- }
- }
-
- lock.Unlock()
- }
-}
-
-func main() {
- inputFile := flag.String("inputFile", "", "Input file path")
- outputFile := flag.String("outputFile", "", "Output file path")
- rule := flag.String(
- "rule", "", "Metadata rule (Hint: test_spec or code_metadata)",
- )
- flag.Parse()
-
- if *inputFile == "" || *outputFile == "" || *rule == "" {
- fmt.Println("Usage: metadata -rule <rule> -inputFile <input file path> -outputFile <output file path>")
- os.Exit(1)
- }
-
- inputFileData := strings.TrimRight(readFileToString(*inputFile), "\n")
- filePaths := strings.Split(inputFileData, " ")
- if len(filePaths) == 1 && filePaths[0] == "" {
- writeEmptyOutputProto(*outputFile, *rule)
- return
- }
- ownershipMetadataMap := &sync.Map{}
- keyLocks := &keyToLocksMap{}
- errCh := make(chan error, len(filePaths))
- var wg sync.WaitGroup
-
- switch *rule {
- case "test_spec":
- for _, filePath := range filePaths {
- wg.Add(1)
- go processTestSpecProtobuf(
- filePath, ownershipMetadataMap, keyLocks, errCh, &wg,
- )
- }
-
- wg.Wait()
- close(errCh)
-
- for err := range errCh {
- log.Fatal(err)
- }
-
- allKeys := getSortedKeys(ownershipMetadataMap)
- var allMetadata []*test_spec_proto.TestSpec_OwnershipMetadata
-
- for _, key := range allKeys {
- value, _ := ownershipMetadataMap.Load(key)
- metadataList := value.([]*test_spec_proto.TestSpec_OwnershipMetadata)
- allMetadata = append(allMetadata, metadataList...)
- }
-
- testSpec := &test_spec_proto.TestSpec{
- OwnershipMetadataList: allMetadata,
- }
- writeProtoToFile(*outputFile, testSpec)
- break
- case "code_metadata":
- sourceFileMetadataMap := &sync.Map{}
- for _, filePath := range filePaths {
- wg.Add(1)
- go processCodeMetadataProtobuf(
- filePath, ownershipMetadataMap, sourceFileMetadataMap, keyLocks, errCh, &wg,
- )
- }
-
- wg.Wait()
- close(errCh)
-
- for err := range errCh {
- log.Fatal(err)
- }
-
- sortedKeys := getSortedKeys(ownershipMetadataMap)
- allMetadata := make([]*code_metadata_proto.CodeMetadata_TargetOwnership, 0)
- for _, key := range sortedKeys {
- value, _ := ownershipMetadataMap.Load(key)
- metadata := value.([]*code_metadata_internal_proto.CodeMetadataInternal_TargetOwnership)
- for _, m := range metadata {
- targetName := m.GetTargetName()
- path := m.GetPath()
- trendyTeamId := m.GetTrendyTeamId()
-
- allMetadata = append(allMetadata, &code_metadata_proto.CodeMetadata_TargetOwnership{
- TargetName: &targetName,
- Path: &path,
- TrendyTeamId: &trendyTeamId,
- SourceFiles: m.GetSourceFiles(),
- })
- }
- }
-
- finalMetadata := &code_metadata_proto.CodeMetadata{
- TargetOwnershipList: allMetadata,
- }
- writeProtoToFile(*outputFile, finalMetadata)
- break
- default:
- log.Fatalf("No specific processing implemented for rule '%s'.\n", *rule)
- }
-}
diff --git a/tools/metadata/go.mod b/tools/metadata/go.mod
deleted file mode 100644
index e9d04b1..0000000
--- a/tools/metadata/go.mod
+++ /dev/null
@@ -1,7 +0,0 @@
-module android/soong/tools/metadata
-
-require google.golang.org/protobuf v0.0.0
-
-replace google.golang.org/protobuf v0.0.0 => ../../../external/golang-protobuf
-
-go 1.18
\ No newline at end of file
diff --git a/tools/metadata/go.work b/tools/metadata/go.work
deleted file mode 100644
index f2cdf8e..0000000
--- a/tools/metadata/go.work
+++ /dev/null
@@ -1,11 +0,0 @@
-go 1.18
-
-use (
- .
- ../../../../external/golang-protobuf
- ../../../soong/testing/test_spec_proto
- ../../../soong/testing/code_metadata_proto
- ../../../soong/testing/code_metadata_proto_internal
-)
-
-replace google.golang.org/protobuf v0.0.0 => ../../../../external/golang-protobuf
diff --git a/tools/metadata/testdata/emptyInputFile.txt b/tools/metadata/testdata/emptyInputFile.txt
deleted file mode 100644
index 8b13789..0000000
--- a/tools/metadata/testdata/emptyInputFile.txt
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/tools/metadata/testdata/expectedCodeMetadataOutput.txt b/tools/metadata/testdata/expectedCodeMetadataOutput.txt
deleted file mode 100644
index 755cf40..0000000
--- a/tools/metadata/testdata/expectedCodeMetadataOutput.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-bar
-Android.bp12346"b.java
-
-foo
-Android.bp12345"a.java
\ No newline at end of file
diff --git a/tools/metadata/testdata/expectedOutputFile.txt b/tools/metadata/testdata/expectedOutputFile.txt
deleted file mode 100644
index b0d382f..0000000
--- a/tools/metadata/testdata/expectedOutputFile.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-
-.
-java-test-module-name-one
-Android.bp12345
-.
-java-test-module-name-six
-Android.bp12346
-.
-java-test-module-name-six
-Aqwerty.bp12346
-.
-java-test-module-name-six
-Apoiuyt.bp12346
-.
-java-test-module-name-two
-Android.bp12345
-.
-java-test-module-name-two
-Asdfghj.bp12345
-.
-java-test-module-name-two
-Azxcvbn.bp12345
\ No newline at end of file
diff --git a/tools/metadata/testdata/file1.txt b/tools/metadata/testdata/file1.txt
deleted file mode 100644
index 81beed0..0000000
--- a/tools/metadata/testdata/file1.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-
-.
-java-test-module-name-one
-Android.bp12345
-.
-java-test-module-name-two
-Android.bp12345
-.
-java-test-module-name-two
-Asdfghj.bp12345
-.
-java-test-module-name-two
-Azxcvbn.bp12345
diff --git a/tools/metadata/testdata/file2.txt b/tools/metadata/testdata/file2.txt
deleted file mode 100644
index 32a753f..0000000
--- a/tools/metadata/testdata/file2.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-
-.
-java-test-module-name-one
-Android.bp12345
-.
-java-test-module-name-six
-Android.bp12346
-.
-java-test-module-name-one
-Android.bp12345
-.
-java-test-module-name-six
-Aqwerty.bp12346
-.
-java-test-module-name-six
-Apoiuyt.bp12346
-.
-java-test-module-name-six
-Apoiuyt.bp12346
-.
-java-test-module-name-six
-Apoiuyt.bp12346
-.
-java-test-module-name-six
-Apoiuyt.bp12346
diff --git a/tools/metadata/testdata/file3.txt b/tools/metadata/testdata/file3.txt
deleted file mode 100644
index 81beed0..0000000
--- a/tools/metadata/testdata/file3.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-
-.
-java-test-module-name-one
-Android.bp12345
-.
-java-test-module-name-two
-Android.bp12345
-.
-java-test-module-name-two
-Asdfghj.bp12345
-.
-java-test-module-name-two
-Azxcvbn.bp12345
diff --git a/tools/metadata/testdata/file4.txt b/tools/metadata/testdata/file4.txt
deleted file mode 100644
index 6a75900..0000000
--- a/tools/metadata/testdata/file4.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-
-.
-java-test-module-name-one
-Android.bp12345
-.
-java-test-module-name-six
-Android.bp12346
-.
-java-test-module-name-one
-Android.bp12346
-.
-java-test-module-name-six
-Aqwerty.bp12346
-.
-java-test-module-name-six
-Apoiuyt.bp12346
-.
-java-test-module-name-six
-Apoiuyt.bp12346
-.
-java-test-module-name-six
-Apoiuyt.bp12346
-.
-java-test-module-name-six
-Apoiuyt.bp12346
diff --git a/tools/metadata/testdata/file5.txt b/tools/metadata/testdata/file5.txt
deleted file mode 100644
index d8de064..0000000
--- a/tools/metadata/testdata/file5.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
-foo
-Android.bp12345"a.java
diff --git a/tools/metadata/testdata/file6.txt b/tools/metadata/testdata/file6.txt
deleted file mode 100644
index 9c7cdcd..0000000
--- a/tools/metadata/testdata/file6.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
-bar
-Android.bp12346"b.java
diff --git a/tools/metadata/testdata/file7.txt b/tools/metadata/testdata/file7.txt
deleted file mode 100644
index d8de064..0000000
--- a/tools/metadata/testdata/file7.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
-foo
-Android.bp12345"a.java
diff --git a/tools/metadata/testdata/file8.txt b/tools/metadata/testdata/file8.txt
deleted file mode 100644
index a931690..0000000
--- a/tools/metadata/testdata/file8.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
-foo
-Android.gp12346"a.java
diff --git a/tools/metadata/testdata/generatedCodeMetadataOutput.txt b/tools/metadata/testdata/generatedCodeMetadataOutput.txt
deleted file mode 100644
index 755cf40..0000000
--- a/tools/metadata/testdata/generatedCodeMetadataOutput.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-bar
-Android.bp12346"b.java
-
-foo
-Android.bp12345"a.java
\ No newline at end of file
diff --git a/tools/metadata/testdata/generatedCodeMetadataOutputFile.txt b/tools/metadata/testdata/generatedCodeMetadataOutputFile.txt
deleted file mode 100644
index 755cf40..0000000
--- a/tools/metadata/testdata/generatedCodeMetadataOutputFile.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-bar
-Android.bp12346"b.java
-
-foo
-Android.bp12345"a.java
\ No newline at end of file
diff --git a/tools/metadata/testdata/generatedEmptyOutputFile.txt b/tools/metadata/testdata/generatedEmptyOutputFile.txt
deleted file mode 100644
index e69de29..0000000
--- a/tools/metadata/testdata/generatedEmptyOutputFile.txt
+++ /dev/null
diff --git a/tools/metadata/testdata/generatedOutputFile.txt b/tools/metadata/testdata/generatedOutputFile.txt
deleted file mode 100644
index b0d382f..0000000
--- a/tools/metadata/testdata/generatedOutputFile.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-
-.
-java-test-module-name-one
-Android.bp12345
-.
-java-test-module-name-six
-Android.bp12346
-.
-java-test-module-name-six
-Aqwerty.bp12346
-.
-java-test-module-name-six
-Apoiuyt.bp12346
-.
-java-test-module-name-two
-Android.bp12345
-.
-java-test-module-name-two
-Asdfghj.bp12345
-.
-java-test-module-name-two
-Azxcvbn.bp12345
\ No newline at end of file
diff --git a/tools/metadata/testdata/inputCodeMetadata.txt b/tools/metadata/testdata/inputCodeMetadata.txt
deleted file mode 100644
index 7a81b7d..0000000
--- a/tools/metadata/testdata/inputCodeMetadata.txt
+++ /dev/null
@@ -1 +0,0 @@
-file5.txt file6.txt
\ No newline at end of file
diff --git a/tools/metadata/testdata/inputCodeMetadataNegative.txt b/tools/metadata/testdata/inputCodeMetadataNegative.txt
deleted file mode 100644
index 26668e4..0000000
--- a/tools/metadata/testdata/inputCodeMetadataNegative.txt
+++ /dev/null
@@ -1 +0,0 @@
-file7.txt file8.txt
\ No newline at end of file
diff --git a/tools/metadata/testdata/inputFiles.txt b/tools/metadata/testdata/inputFiles.txt
deleted file mode 100644
index e44bc94..0000000
--- a/tools/metadata/testdata/inputFiles.txt
+++ /dev/null
@@ -1 +0,0 @@
-file1.txt file2.txt
\ No newline at end of file
diff --git a/tools/metadata/testdata/inputFilesNegativeCase.txt b/tools/metadata/testdata/inputFilesNegativeCase.txt
deleted file mode 100644
index a37aa3f..0000000
--- a/tools/metadata/testdata/inputFilesNegativeCase.txt
+++ /dev/null
@@ -1 +0,0 @@
-file3.txt file4.txt
\ No newline at end of file
diff --git a/tools/metadata/testdata/metadata_test.go b/tools/metadata/testdata/metadata_test.go
deleted file mode 100644
index 314add3..0000000
--- a/tools/metadata/testdata/metadata_test.go
+++ /dev/null
@@ -1,119 +0,0 @@
-package main
-
-import (
- "fmt"
- "io/ioutil"
- "os/exec"
- "strings"
- "testing"
-)
-
-func TestMetadata(t *testing.T) {
- cmd := exec.Command(
- "metadata", "-rule", "test_spec", "-inputFile", "./inputFiles.txt", "-outputFile",
- "./generatedOutputFile.txt",
- )
- stderr, err := cmd.CombinedOutput()
- if err != nil {
- t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err)
- }
-
- // Read the contents of the expected output file
- expectedOutput, err := ioutil.ReadFile("./expectedOutputFile.txt")
- if err != nil {
- t.Fatalf("Error reading expected output file: %s", err)
- }
-
- // Read the contents of the generated output file
- generatedOutput, err := ioutil.ReadFile("./generatedOutputFile.txt")
- if err != nil {
- t.Fatalf("Error reading generated output file: %s", err)
- }
-
- fmt.Println()
-
- // Compare the contents
- if string(expectedOutput) != string(generatedOutput) {
- t.Errorf("Generated file contents do not match the expected output")
- }
-}
-
-func TestMetadataNegativeCase(t *testing.T) {
- cmd := exec.Command(
- "metadata", "-rule", "test_spec", "-inputFile", "./inputFilesNegativeCase.txt", "-outputFile",
- "./generatedOutputFileNegativeCase.txt",
- )
- stderr, err := cmd.CombinedOutput()
- if err == nil {
- t.Fatalf(
- "Expected an error, but the metadata command executed successfully. Output: %s",
- stderr,
- )
- }
-
- expectedError := "Conflicting trendy team IDs found for java-test-module" +
- "-name-one at:\nAndroid.bp with teamId: 12346," +
- "\nAndroid.bp with teamId: 12345"
- if !strings.Contains(
- strings.TrimSpace(string(stderr)), strings.TrimSpace(expectedError),
- ) {
- t.Errorf(
- "Unexpected error message. Expected to contain: %s, Got: %s",
- expectedError, stderr,
- )
- }
-}
-
-func TestEmptyInputFile(t *testing.T) {
- cmd := exec.Command(
- "metadata", "-rule", "test_spec", "-inputFile", "./emptyInputFile.txt", "-outputFile",
- "./generatedEmptyOutputFile.txt",
- )
- stderr, err := cmd.CombinedOutput()
- if err != nil {
- t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err)
- }
-
- // Read the contents of the generated output file
- generatedOutput, err := ioutil.ReadFile("./generatedEmptyOutputFile.txt")
- if err != nil {
- t.Fatalf("Error reading generated output file: %s", err)
- }
-
- fmt.Println()
-
- // Compare the contents
- if string(generatedOutput) != "\n" {
- t.Errorf("Generated file contents do not match the expected output")
- }
-}
-
-func TestCodeMetadata(t *testing.T) {
- cmd := exec.Command(
- "metadata", "-rule", "code_metadata", "-inputFile", "./inputCodeMetadata.txt", "-outputFile",
- "./generatedCodeMetadataOutputFile.txt",
- )
- stderr, err := cmd.CombinedOutput()
- if err != nil {
- t.Fatalf("Error running metadata command: %s. Error: %v", stderr, err)
- }
-
- // Read the contents of the expected output file
- expectedOutput, err := ioutil.ReadFile("./expectedCodeMetadataOutput.txt")
- if err != nil {
- t.Fatalf("Error reading expected output file: %s", err)
- }
-
- // Read the contents of the generated output file
- generatedOutput, err := ioutil.ReadFile("./generatedCodeMetadataOutputFile.txt")
- if err != nil {
- t.Fatalf("Error reading generated output file: %s", err)
- }
-
- fmt.Println()
-
- // Compare the contents
- if string(expectedOutput) != string(generatedOutput) {
- t.Errorf("Generated file contents do not match the expected output")
- }
-}
diff --git a/tools/metadata/testdata/outputFile.txt b/tools/metadata/testdata/outputFile.txt
deleted file mode 100644
index b0d382f..0000000
--- a/tools/metadata/testdata/outputFile.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-
-.
-java-test-module-name-one
-Android.bp12345
-.
-java-test-module-name-six
-Android.bp12346
-.
-java-test-module-name-six
-Aqwerty.bp12346
-.
-java-test-module-name-six
-Apoiuyt.bp12346
-.
-java-test-module-name-two
-Android.bp12345
-.
-java-test-module-name-two
-Asdfghj.bp12345
-.
-java-test-module-name-two
-Azxcvbn.bp12345
\ No newline at end of file
diff --git a/tools/releasetools/Android.bp b/tools/releasetools/Android.bp
index 8c71044..e371b23 100644
--- a/tools/releasetools/Android.bp
+++ b/tools/releasetools/Android.bp
@@ -637,6 +637,8 @@
],
data: [
"testdata/**/*",
+ ],
+ device_common_data: [
":com.android.apex.compressed.v1",
":com.android.apex.vendor.foo.with_vintf",
],