Merge "Support sandboxing genrule"
diff --git a/Changes.md b/Changes.md
index daebd52..fc6701d 100644
--- a/Changes.md
+++ b/Changes.md
@@ -1,5 +1,18 @@
# Build System Changes for Android.mk Writers
+## Perform validation of Soong plugins
+
+Each Soong plugin will require manual work to migrate to Bazel. In order to
+minimize the manual work outside of build/soong, we are restricting plugins to
+those that exist today and those in vendor or hardware directories.
+
+If you need to extend the build system via a plugin, please reach out to the
+build team via email android-building@googlegroups.com (external) for any
+questions, or see [go/soong](http://go/soong) (internal).
+
+To omit the validation, `BUILD_BROKEN_PLUGIN_VALIDATION` expects a list of
+plugins to omit from the validation.
+
## Python 2 to 3 migration
The path set when running builds now makes the `python` executable point to python 3,
@@ -15,7 +28,6 @@
variable to `true`.
Python 2 is slated for complete removal in V.
-
## Stop referencing sysprop_library directly from cc modules
For the migration to Bazel, we are no longer mapping sysprop_library targets
@@ -493,6 +505,24 @@
will copy `bar/baz` into `$DIST_DIR/baz` when `m foo dist` is run.
+#### FILE_NAME_TAG {#FILE_NAME_TAG}
+
+To embed the `BUILD_NUMBER` (or for local builds, `eng.${USER}`), include
+`FILE_NAME_TAG_PLACEHOLDER` in the destination:
+
+``` make
+# you can use dist-for-goals-with-filenametag function
+$(call dist-for-goals-with-filenametag,foo,bar.zip)
+# or use FILE_NAME_TAG_PLACEHOLDER manually
+$(call dist-for-goals,foo,bar.zip:baz-FILE_NAME_TAG_PLACEHOLDER.zip)
+```
+
+Which will produce `$DIST_DIR/baz-1234567.zip` on build servers which set
+`BUILD_NUMBER=1234567`, or `$DIST_DIR/baz-eng.builder.zip` for local builds.
+
+If you just want to append `BUILD_NUMBER` at the end of basename, use
+`dist-for-goals-with-filenametag` instead of `dist-for-goals`.
+
#### Renames during copy
Instead of specifying just a file, a destination name can be specified,
diff --git a/core/Makefile b/core/Makefile
index e9340f7..4e5787d 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -17,6 +17,52 @@
SYSTEM_DLKM_NOTICE_DEPS :=
# -----------------------------------------------------------------
+# Release Config Flags
+
+# Create a summary file of build flags for each partition
+# $(1): build flags json file
+# $(2): flag names
+define generate-partition-build-flag-file
+$(eval $(strip $(1)): PRIVATE_OUT := $(strip $(1)))
+$(eval $(strip $(1)): PRIVATE_FLAG_NAMES := $(strip $(2)))
+$(strip $(1)):
+ mkdir -p $$(dir $$(PRIVATE_OUT))
+ ( \
+ echo '{' ; \
+ echo 'flags: [' ; \
+ $$(foreach flag, $$(PRIVATE_FLAG_NAMES), \
+ printf ' { "name": "%s", "value": "%s", ' \
+ '$$(flag)' \
+ '$$(_ALL_RELEASE_FLAGS.$$(flag).VALUE)' \
+ ; \
+ printf '"set": "%s", "default": "%s", "declared": "%s", }' \
+ '$$(_ALL_RELEASE_FLAGS.$$(flag).SET_IN)' \
+ '$$(_ALL_RELEASE_FLAGS.$$(flag).DEFAULT)' \
+ '$$(_ALL_RELEASE_FLAGS.$$(flag).DECLARED_IN)' \
+ ; \
+ printf '$$(if $$(filter $$(lastword $$(PRIVATE_FLAG_NAMES)),$$(flag)),,$$(comma))\n' ; \
+ ) \
+ echo "]" ; \
+ echo "}" \
+ ) >> $$(PRIVATE_OUT)
+endef
+
+$(foreach partition, $(_FLAG_PARTITIONS), \
+ $(eval BUILD_FLAG_SUMMARIES.$(partition) \
+ := $(TARGET_OUT_FLAGS)/$(partition)/etc/build_flags.json) \
+ $(eval $(call generate-partition-build-flag-file, \
+ $(BUILD_FLAG_SUMMARIES.$(partition)), \
+ $(_ALL_RELEASE_FLAGS.PARTITIONS.$(partition)) \
+ ) \
+ ) \
+)
+
+# TODO: Remove
+.PHONY: flag-files
+flag-files: $(foreach partition, $(_FLAG_PARTITIONS), \
+ $(TARGET_OUT_FLAGS)/$(partition)/etc/build_flags.json)
+
+# -----------------------------------------------------------------
# Define rules to copy PRODUCT_COPY_FILES defined by the product.
# PRODUCT_COPY_FILES contains words like <source file>:<dest file>[:<owner>].
# <dest file> is relative to $(PRODUCT_OUT), so it should look like,
@@ -530,6 +576,24 @@
$(call copy-many-files,$(call module-load-list-copy-paths,$(call intermediates-dir-for,PACKAGING,vendor_charger_module_list$(_sep)$(_kver)),$(BOARD_VENDOR_CHARGER_KERNEL_MODULES$(_sep)$(_kver)),$(BOARD_VENDOR_CHARGER_KERNEL_MODULES_LOAD$(_sep)$(_kver)),modules.load.charger,$(TARGET_OUT_VENDOR))))
endef
+# $(1): kernel module directory name (top is an out of band value for no directory)
+define build-vendor-ramdisk-charger-load
+$(if $(filter top,$(1)),\
+ $(eval _kver :=)$(eval _sep :=),\
+ $(eval _kver := $(1))$(eval _sep :=_))\
+ $(if $(BOARD_VENDOR_RAMDISK_CHARGER_KERNEL_MODULES_LOAD$(_sep)$(_kver)),\
+ $(call copy-many-files,$(call module-load-list-copy-paths,$(call intermediates-dir-for,PACKAGING,vendor_ramdisk_charger_module_list$(_sep)$(_kver)),$(BOARD_VENDOR_RAMDISK_KERNEL_MODULES$(_sep)$(_kver)),$(BOARD_VENDOR_RAMDISK_CHARGER_KERNEL_MODULES_LOAD$(_sep)$(_kver)),modules.load.charger,$(TARGET_VENDOR_RAMDISK_OUT))))
+endef
+
+# $(1): kernel module directory name (top is an out of band value for no directory)
+define build-vendor-kernel-ramdisk-charger-load
+$(if $(filter top,$(1)),\
+ $(eval _kver :=)$(eval _sep :=),\
+ $(eval _kver := $(1))$(eval _sep :=_))\
+ $(if $(BOARD_VENDOR_KERNEL_RAMDISK_CHARGER_KERNEL_MODULES_LOAD$(_sep)$(_kver)),\
+ $(call copy-many-files,$(call module-load-list-copy-paths,$(call intermediates-dir-for,PACKAGING,vendor_kernel_ramdisk_charger_module_list$(_sep)$(_kver)),$(BOARD_VENDOR_KERNEL_RAMDISK_KERNEL_MODULES$(_sep)$(_kver)),$(BOARD_VENDOR_KERNEL_RAMDISK_CHARGER_KERNEL_MODULES_LOAD$(_sep)$(_kver)),modules.load.charger,$(TARGET_VENDOR_KERNEL_RAMDISK_OUT))))
+endef
+
ifneq ($(BUILDING_VENDOR_BOOT_IMAGE),true)
# If there is no vendor boot partition, store vendor ramdisk kernel modules in the
# boot ramdisk.
@@ -595,6 +659,8 @@
$(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-vendor-kernel-ramdisk-recovery-load,$(kmd))) \
$(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-image-kernel-modules-dir,VENDOR,$(if $(filter true,$(BOARD_USES_VENDOR_DLKMIMAGE)),$(TARGET_OUT_VENDOR_DLKM),$(TARGET_OUT_VENDOR)),vendor,modules.load,$(VENDOR_STRIPPED_MODULE_STAGING_DIR),$(kmd),$(BOARD_SYSTEM_KERNEL_MODULES),system)) \
$(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-vendor-charger-load,$(kmd))) \
+ $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-vendor-ramdisk-charger-load,$(kmd))) \
+ $(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-vendor-kernel-ramdisk-charger-load,$(kmd))) \
$(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-image-kernel-modules-dir,ODM,$(if $(filter true,$(BOARD_USES_ODM_DLKMIMAGE)),$(TARGET_OUT_ODM_DLKM),$(TARGET_OUT_ODM)),odm,modules.load,,$(kmd))) \
$(eval ALL_DEFAULT_INSTALLED_MODULES += $(call build-image-kernel-modules-dir,SYSTEM,$(if $(filter true,$(BOARD_USES_SYSTEM_DLKMIMAGE)),$(TARGET_OUT_SYSTEM_DLKM),$(TARGET_OUT_SYSTEM)),system,modules.load,,$(kmd))) \
$(if $(filter true,$(BOARD_USES_RECOVERY_AS_BOOT)),\
@@ -645,7 +711,7 @@
ifeq ($(TARGET_BUILD_TYPE),debug)
name := $(name)_debug
endif
-name := $(name)-apkcerts-$(FILE_NAME_TAG)
+name := $(name)-apkcerts
intermediates := \
$(call intermediates-dir-for,PACKAGING,apkcerts)
APKCERTS_FILE := $(intermediates)/$(name).txt
@@ -935,6 +1001,42 @@
BUILT_RAMDISK_TARGET := $(PRODUCT_OUT)/ramdisk.img
+
+ifneq ($(BOARD_KERNEL_MODULES_16K),)
+
+TARGET_OUT_RAMDISK_16K := $(PRODUCT_OUT)/ramdisk_16k
+BUILT_RAMDISK_16K_TARGET := $(PRODUCT_OUT)/ramdisk_16k.img
+RAMDISK_16K_STAGING_DIR := $(call intermediates-dir-for,PACKAGING,depmod_ramdisk_16k)
+
+$(BUILT_RAMDISK_16K_TARGET): $(DEPMOD) $(MKBOOTFS)
+$(BUILT_RAMDISK_16K_TARGET): $(call copy-many-files,$(foreach file,$(BOARD_KERNEL_MODULES_16K),$(file):$(RAMDISK_16K_STAGING_DIR)/lib/modules/0.0/$(notdir $(file))))
+ $(DEPMOD) -b $(RAMDISK_16K_STAGING_DIR) 0.0
+ for MODULE in $(BOARD_KERNEL_MODULES_16K); do \
+ basename $$MODULE >> $(RAMDISK_16K_STAGING_DIR)/lib/modules/0.0/modules.load ; \
+ done;
+ mkdir -p $(TARGET_OUT_RAMDISK_16K)/lib
+ rm -rf $(TARGET_OUT_RAMDISK_16K)/lib/modules
+ cp -r $(RAMDISK_16K_STAGING_DIR)/lib/modules/0.0 $(TARGET_OUT_RAMDISK_16K)/lib/modules
+ $(MKBOOTFS) $(TARGET_OUT_RAMDISK_16K) > $@
+
+# Builds a ramdisk using modules defined in BOARD_KERNEL_MODULES_16K
+ramdisk_16k: $(BUILT_RAMDISK_16K_TARGET)
+.PHONY: ramdisk_16k
+
+endif
+
+ifneq ($(BOARD_KERNEL_PATH_16K),)
+BUILT_KERNEL_16K_TARGET := $(PRODUCT_OUT)/kernel_16k
+
+$(eval $(call copy-one-file,$(BOARD_KERNEL_PATH_16K),$(BUILT_KERNEL_16K_TARGET)))
+
+# Copies BOARD_KERNEL_PATH_16K to output directory as is
+kernel_16k: $(BUILT_KERNEL_16K_TARGET)
+.PHONY: kernel_16k
+
+endif
+
+
ifeq ($(BOARD_RAMDISK_USE_LZ4),true)
# -l enables the legacy format used by the Linux kernel
COMPRESSION_COMMAND_DEPS := $(LZ4)
@@ -5151,6 +5253,7 @@
lz4 \
make_f2fs \
make_f2fs_casefold \
+ merge_ota \
merge_target_files \
minigzip \
mk_combined_img \
@@ -5589,6 +5692,8 @@
.PHONY: fastboot_info
fastboot_info: $(INSTALLED_FASTBOOT_INFO_TARGET)
+droidcore-unbundled: $(INSTALLED_FASTBOOT_INFO_TARGET)
+
$(call declare-0p-target,$(INSTALLED_MISC_INFO_TARGET))
.PHONY: misc_info
@@ -5605,7 +5710,7 @@
ifeq ($(TARGET_BUILD_TYPE),debug)
name := $(name)_debug
endif
-name := $(name)-target_files-$(FILE_NAME_TAG)
+name := $(name)-target_files
intermediates := $(call intermediates-dir-for,PACKAGING,target_files)
BUILT_TARGET_FILES_DIR := $(intermediates)/$(name).zip.list
@@ -5896,6 +6001,8 @@
$(INSTALLED_RAMDISK_TARGET) \
$(INSTALLED_DTBIMAGE_TARGET) \
$(INSTALLED_2NDBOOTLOADER_TARGET) \
+ $(BUILT_RAMDISK_16K_TARGET) \
+ $(BUILT_KERNEL_16K_TARGET) \
$(BOARD_PREBUILT_DTBOIMAGE) \
$(BOARD_PREBUILT_RECOVERY_DTBOIMAGE) \
$(BOARD_RECOVERY_ACPIO) \
@@ -5910,6 +6017,7 @@
$(LPMAKE) \
$(SELINUX_FC) \
$(INSTALLED_MISC_INFO_TARGET) \
+ $(INSTALLED_FASTBOOT_INFO_TARGET) \
$(APKCERTS_FILE) \
$(SOONG_APEX_KEYS_FILE) \
$(SOONG_ZIP) \
@@ -6128,6 +6236,9 @@
$(hide) echo "$(PRODUCT_OTA_PUBLIC_KEYS)" > $(zip_root)/META/otakeys.txt
$(hide) cp $(SELINUX_FC) $(zip_root)/META/file_contexts.bin
$(hide) cp $(INSTALLED_MISC_INFO_TARGET) $(zip_root)/META/misc_info.txt
+ifneq ($(INSTALLED_FASTBOOT_INFO_TARGET),)
+ $(hide) cp $(INSTALLED_FASTBOOT_INFO_TARGET) $(zip_root)/META/fastboot-info.txt
+endif
ifneq ($(PRODUCT_SYSTEM_BASE_FS_PATH),)
$(hide) cp $(PRODUCT_SYSTEM_BASE_FS_PATH) \
$(zip_root)/META/$(notdir $(PRODUCT_SYSTEM_BASE_FS_PATH))
@@ -6235,6 +6346,14 @@
$(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES
$(hide) cp $(INSTALLED_DTBOIMAGE_TARGET) $(zip_root)/PREBUILT_IMAGES/
endif # BOARD_PREBUILT_DTBOIMAGE
+ifdef BUILT_KERNEL_16K_TARGET
+ $(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES
+ $(hide) cp $(BUILT_KERNEL_16K_TARGET) $(zip_root)/PREBUILT_IMAGES/
+endif # BUILT_KERNEL_16K_TARGET
+ifdef BUILT_RAMDISK_16K_TARGET
+ $(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES
+ $(hide) cp $(BUILT_RAMDISK_16K_TARGET) $(zip_root)/PREBUILT_IMAGES/
+endif # BUILT_RAMDISK_16K_TARGET
ifeq ($(BOARD_USES_PVMFWIMAGE),true)
$(hide) mkdir -p $(zip_root)/PREBUILT_IMAGES
$(hide) cp $(INSTALLED_PVMFWIMAGE_TARGET) $(zip_root)/PREBUILT_IMAGES/
@@ -6371,7 +6490,7 @@
$(BUILT_KERNEL_CONFIGS_FILE) \
$(BUILT_KERNEL_VERSION_FILE),$(BUILT_TARGET_FILES_PACKAGE):)
-$(call dist-for-goals, target-files-package, $(BUILT_TARGET_FILES_PACKAGE))
+$(call dist-for-goals-with-filenametag, target-files-package, $(BUILT_TARGET_FILES_PACKAGE))
# -----------------------------------------------------------------
# NDK Sysroot Package
@@ -6406,7 +6525,7 @@
ifeq ($(TARGET_BUILD_TYPE),debug)
product_name := $(product_name)_debug
endif
-name := $(product_name)-ota-$(FILE_NAME_TAG)
+name := $(product_name)-ota
INTERNAL_OTA_PACKAGE_TARGET := $(PRODUCT_OUT)/$(name).zip
INTERNAL_OTA_METADATA := $(PRODUCT_OUT)/ota_metadata
@@ -6426,7 +6545,7 @@
otapackage: $(INTERNAL_OTA_PACKAGE_TARGET)
ifeq ($(BOARD_BUILD_RETROFIT_DYNAMIC_PARTITIONS_OTA_PACKAGE),true)
-name := $(product_name)-ota-retrofit-$(FILE_NAME_TAG)
+name := $(product_name)-ota-retrofit
INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET := $(PRODUCT_OUT)/$(name).zip
$(INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET): KEY_CERT_PAIR := $(DEFAULT_KEY_CERT_PAIR)
@@ -6447,7 +6566,7 @@
endif # BOARD_BUILD_RETROFIT_DYNAMIC_PARTITIONS_OTA_PACKAGE
ifneq ($(BOARD_PARTIAL_OTA_UPDATE_PARTITIONS_LIST),)
-name := $(product_name)-partial-ota-$(FILE_NAME_TAG)
+name := $(product_name)-partial-ota
INTERNAL_OTA_PARTIAL_PACKAGE_TARGET := $(PRODUCT_OUT)/$(name).zip
$(INTERNAL_OTA_PARTIAL_PACKAGE_TARGET): KEY_CERT_PAIR := $(DEFAULT_KEY_CERT_PAIR)
@@ -6547,9 +6666,9 @@
endif
# The path to the zip file containing binaries with symbols.
-SYMBOLS_ZIP := $(PRODUCT_OUT)/$(name)-symbols-$(FILE_NAME_TAG).zip
+SYMBOLS_ZIP := $(PRODUCT_OUT)/$(name)-symbols.zip
# The path to a file containing mappings from elf IDs to filenames.
-SYMBOLS_MAPPING := $(PRODUCT_OUT)/$(name)-symbols-mapping-$(FILE_NAME_TAG).textproto
+SYMBOLS_MAPPING := $(PRODUCT_OUT)/$(name)-symbols-mapping.textproto
.KATI_READONLY := SYMBOLS_ZIP SYMBOLS_MAPPING
# For apps_only build we'll establish the dependency later in build/make/core/main.mk.
ifeq (,$(TARGET_BUILD_UNBUNDLED))
@@ -6624,7 +6743,7 @@
ifeq ($(TARGET_BUILD_TYPE),debug)
name := $(name)_debug
endif
-name := $(name)-apps-$(FILE_NAME_TAG)
+name := $(name)-apps
APPS_ZIP := $(PRODUCT_OUT)/$(name).zip
$(APPS_ZIP): $(FULL_SYSTEMIMAGE_DEPS)
@@ -6675,9 +6794,9 @@
#
# The path to the zip file containing proguard dictionaries.
-PROGUARD_DICT_ZIP := $(PRODUCT_OUT)/$(TARGET_PRODUCT)-proguard-dict-$(FILE_NAME_TAG).zip
+PROGUARD_DICT_ZIP := $(PRODUCT_OUT)/$(TARGET_PRODUCT)-proguard-dict.zip
# The path to the zip file containing mappings from dictionary hashes to filenames.
-PROGUARD_DICT_MAPPING := $(PRODUCT_OUT)/$(TARGET_PRODUCT)-proguard-dict-mapping-$(FILE_NAME_TAG).textproto
+PROGUARD_DICT_MAPPING := $(PRODUCT_OUT)/$(TARGET_PRODUCT)-proguard-dict-mapping.textproto
.KATI_READONLY := PROGUARD_DICT_ZIP PROGUARD_DICT_MAPPING
# For apps_only build we'll establish the dependency later in build/make/core/main.mk.
ifeq (,$(TARGET_BUILD_UNBUNDLED))
@@ -6706,7 +6825,7 @@
#------------------------------------------------------------------
# A zip of Proguard usage files.
#
-PROGUARD_USAGE_ZIP := $(PRODUCT_OUT)/$(TARGET_PRODUCT)-proguard-usage-$(FILE_NAME_TAG).zip
+PROGUARD_USAGE_ZIP := $(PRODUCT_OUT)/$(TARGET_PRODUCT)-proguard-usage.zip
# For apps_only build we'll establish the dependency later in build/make/core/main.mk.
ifeq (,$(TARGET_BUILD_UNBUNDLED))
$(PROGUARD_USAGE_ZIP): \
@@ -6883,7 +7002,7 @@
ifeq ($(TARGET_BUILD_TYPE),debug)
name := $(name)_debug
endif
-name := $(name)-img-$(FILE_NAME_TAG)
+name := $(name)-img
INTERNAL_UPDATE_PACKAGE_TARGET := $(PRODUCT_OUT)/$(name).zip
@@ -6899,7 +7018,7 @@
.PHONY: updatepackage
updatepackage: $(INTERNAL_UPDATE_PACKAGE_TARGET)
-$(call dist-for-goals,updatepackage,$(INTERNAL_UPDATE_PACKAGE_TARGET))
+$(call dist-for-goals-with-filenametag,updatepackage,$(INTERNAL_UPDATE_PACKAGE_TARGET))
# -----------------------------------------------------------------
@@ -7028,7 +7147,7 @@
$(INSTALLED_SYSTEMIMAGE_TARGET) \
$(INSTALLED_USERDATAIMAGE_TARGET)
-name := $(TARGET_PRODUCT)-emulator-$(FILE_NAME_TAG)
+name := $(TARGET_PRODUCT)-emulator
INTERNAL_EMULATOR_PACKAGE_TARGET := $(PRODUCT_OUT)/$(name).zip
@@ -7056,7 +7175,7 @@
ifneq ($(HOST_OS),linux)
$(error Building the monolithic SDK is only supported on Linux)
endif
-sdk_name := android-sdk_$(FILE_NAME_TAG)
+sdk_name := android-sdk
INTERNAL_SDK_HOST_OS_NAME := linux-$(SDK_HOST_ARCH)
sdk_name := $(sdk_name)_$(INTERNAL_SDK_HOST_OS_NAME)
@@ -7230,17 +7349,9 @@
haiku: $(SOONG_FUZZ_PACKAGING_ARCH_MODULES) $(ALL_FUZZ_TARGETS)
$(call dist-for-goals,haiku,$(SOONG_FUZZ_PACKAGING_ARCH_MODULES))
$(call dist-for-goals,haiku,$(PRODUCT_OUT)/module-info.json)
-
-.PHONY: haiku-java-device
-haiku-java-device: $(SOONG_JAVA_FUZZ_DEVICE_PACKAGING_ARCH_MODULES) $(ALL_JAVA_FUZZ_DEVICE_TARGETS)
-$(call dist-for-goals,haiku-java-device,$(SOONG_JAVA_FUZZ_DEVICE_PACKAGING_ARCH_MODULES))
-$(call dist-for-goals,haiku-java-device,$(PRODUCT_OUT)/module-info.json)
-
-.PHONY: haiku-java-host
-haiku-java-host: $(SOONG_JAVA_FUZZ_HOST_PACKAGING_ARCH_MODULES) $(ALL_JAVA_FUZZ_HOST_TARGETS)
-$(call dist-for-goals,haiku-java-host,$(SOONG_JAVA_FUZZ_HOST_PACKAGING_ARCH_MODULES))
-$(call dist-for-goals,haiku-java-host,$(PRODUCT_OUT)/module-info.json)
-
+.PHONY: haiku-java
+haiku-java: $(SOONG_JAVA_FUZZ_PACKAGING_ARCH_MODULES) $(ALL_JAVA_FUZZ_TARGETS)
+$(call dist-for-goals,haiku-java,$(SOONG_JAVA_FUZZ_PACKAGING_ARCH_MODULES))
.PHONY: haiku-rust
haiku-rust: $(SOONG_RUST_FUZZ_PACKAGING_ARCH_MODULES) $(ALL_RUST_FUZZ_TARGETS)
$(call dist-for-goals,haiku-rust,$(SOONG_RUST_FUZZ_PACKAGING_ARCH_MODULES))
diff --git a/core/all_versions.bzl b/core/all_versions.bzl
new file mode 100644
index 0000000..33da673
--- /dev/null
+++ b/core/all_versions.bzl
@@ -0,0 +1,23 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+_all_versions = ["OPR1", "OPD1", "OPD2", "OPM1", "OPM2", "PPR1", "PPD1", "PPD2", "PPM1", "PPM2", "QPR1"] + [
+ version + subversion
+ for version in ["Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"]
+ for subversion in ["P1A", "P1B", "P2A", "P2B", "D1A", "D1B", "D2A", "D2B", "Q1A", "Q1B", "Q2A", "Q2B", "Q3A", "Q3B"]
+]
+
+variables_to_export_to_make = {
+ "ALL_VERSIONS": _all_versions,
+}
diff --git a/core/base_rules.mk b/core/base_rules.mk
index c453469..65e80fb 100644
--- a/core/base_rules.mk
+++ b/core/base_rules.mk
@@ -190,18 +190,6 @@
$(call pretty-error,unusual tags: $(filter-out tests optional samples,$(my_module_tags)))
endif
-# Add implicit tags.
-#
-# If the local directory or one of its parents contains a MODULE_LICENSE_GPL
-# file, tag the module as "gnu". Search for "*_GPL*", "*_LGPL*" and "*_MPL*"
-# so that we can also find files like MODULE_LICENSE_GPL_AND_AFL
-#
-gpl_license_file := $(call find-parent-file,$(LOCAL_PATH),MODULE_LICENSE*_GPL* MODULE_LICENSE*_MPL* MODULE_LICENSE*_LGPL*)
-ifneq ($(gpl_license_file),)
- my_module_tags += gnu
- ALL_GPL_MODULE_LICENSE_FILES += $(gpl_license_file)
-endif
-
LOCAL_MODULE_CLASS := $(strip $(LOCAL_MODULE_CLASS))
ifneq ($(words $(LOCAL_MODULE_CLASS)),1)
$(error $(LOCAL_PATH): LOCAL_MODULE_CLASS must contain exactly one word, not "$(LOCAL_MODULE_CLASS)")
diff --git a/core/board_config.mk b/core/board_config.mk
index 7969b25..c8ec5a9 100644
--- a/core/board_config.mk
+++ b/core/board_config.mk
@@ -174,6 +174,7 @@
_build_broken_var_list := \
+ BUILD_BROKEN_PLUGIN_VALIDATION \
BUILD_BROKEN_CLANG_PROPERTY \
BUILD_BROKEN_CLANG_ASFLAGS \
BUILD_BROKEN_CLANG_CFLAGS \
@@ -256,7 +257,7 @@
endif
$(shell build/soong/scripts/update_out $(OUT_DIR)/rbc/rbc_board_config_results.mk \
- $(OUT_DIR)/rbcrun $(OUT_DIR)/rbc/boardlauncher.rbc)
+ $(OUT_DIR)/rbcrun --mode=rbc $(OUT_DIR)/rbc/boardlauncher.rbc)
ifneq ($(.SHELLSTATUS),0)
$(error board configuration runner failed: $(.SHELLSTATUS))
endif
diff --git a/core/config.mk b/core/config.mk
index 4300800..0e2d271 100644
--- a/core/config.mk
+++ b/core/config.mk
@@ -271,7 +271,7 @@
# Ex: $(call add_soong_config_namespace,acme)
define add_soong_config_namespace
-$(eval SOONG_CONFIG_NAMESPACES += $1) \
+$(eval SOONG_CONFIG_NAMESPACES += $(strip $1)) \
$(eval SOONG_CONFIG_$(strip $1) :=)
endef
@@ -281,8 +281,8 @@
# $1 is the namespace. $2 is the list of variables.
# Ex: $(call add_soong_config_var,acme,COOL_FEATURE_A COOL_FEATURE_B)
define add_soong_config_var
-$(eval SOONG_CONFIG_$(strip $1) += $2) \
-$(foreach v,$(strip $2),$(eval SOONG_CONFIG_$(strip $1)_$v := $($v)))
+$(eval SOONG_CONFIG_$(strip $1) += $(strip $2)) \
+$(foreach v,$(strip $2),$(eval SOONG_CONFIG_$(strip $1)_$v := $(strip $($v))))
endef
# The add_soong_config_var_value function defines a make variable and also adds
@@ -291,7 +291,7 @@
# Ex: $(call add_soong_config_var_value,acme,COOL_FEATURE,true)
define add_soong_config_var_value
-$(eval $2 := $3) \
+$(eval $(strip $2) := $(strip $3)) \
$(call add_soong_config_var,$1,$2)
endef
@@ -299,8 +299,8 @@
#
# internal utility to define a namespace and a variable in it.
define soong_config_define_internal
-$(if $(filter $1,$(SOONG_CONFIG_NAMESPACES)),,$(eval SOONG_CONFIG_NAMESPACES:=$(SOONG_CONFIG_NAMESPACES) $1)) \
-$(if $(filter $2,$(SOONG_CONFIG_$(strip $1))),,$(eval SOONG_CONFIG_$(strip $1):=$(SOONG_CONFIG_$(strip $1)) $2))
+$(if $(filter $1,$(SOONG_CONFIG_NAMESPACES)),,$(eval SOONG_CONFIG_NAMESPACES:=$(SOONG_CONFIG_NAMESPACES) $(strip $1))) \
+$(if $(filter $2,$(SOONG_CONFIG_$(strip $1))),,$(eval SOONG_CONFIG_$(strip $1):=$(SOONG_CONFIG_$(strip $1)) $(strip $2)))
endef
# soong_config_set defines the variable in the given Soong config namespace
@@ -309,7 +309,7 @@
# Ex: $(call soong_config_set,acme,COOL_FEATURE,true)
define soong_config_set
$(call soong_config_define_internal,$1,$2) \
-$(eval SOONG_CONFIG_$(strip $1)_$(strip $2):=$3)
+$(eval SOONG_CONFIG_$(strip $1)_$(strip $2):=$(strip $3))
endef
# soong_config_append appends to the value of the variable in the given Soong
@@ -318,7 +318,7 @@
# $1 is the namespace, $2 is the variable name, $3 is the value
define soong_config_append
$(call soong_config_define_internal,$1,$2) \
-$(eval SOONG_CONFIG_$(strip $1)_$(strip $2):=$(SOONG_CONFIG_$(strip $1)_$(strip $2)) $3)
+$(eval SOONG_CONFIG_$(strip $1)_$(strip $2):=$(SOONG_CONFIG_$(strip $1)_$(strip $2)) $(strip $3))
endef
# soong_config_append gets to the value of the variable in the given Soong
@@ -546,8 +546,10 @@
TARGET_BUILD_USE_PREBUILT_SDKS :=
DISABLE_PREOPT :=
+DISABLE_PREOPT_BOOT_IMAGES :=
ifneq (,$(TARGET_BUILD_APPS)$(TARGET_BUILD_UNBUNDLED_IMAGE))
DISABLE_PREOPT := true
+ DISABLE_PREOPT_BOOT_IMAGES := true
endif
ifeq (true,$(TARGET_BUILD_UNBUNDLED))
ifneq (true,$(UNBUNDLED_BUILD_SDKS_FROM_SOURCE))
@@ -558,6 +560,7 @@
.KATI_READONLY := \
TARGET_BUILD_USE_PREBUILT_SDKS \
DISABLE_PREOPT \
+ DISABLE_PREOPT_BOOT_IMAGES \
prebuilt_sdk_tools := prebuilts/sdk/tools
prebuilt_sdk_tools_bin := $(prebuilt_sdk_tools)/$(HOST_OS)/bin
@@ -853,6 +856,7 @@
.KATI_READONLY := MAINLINE_SEPOLICY_DEV_CERTIFICATES
BUILD_NUMBER_FROM_FILE := $$(cat $(SOONG_OUT_DIR)/build_number.txt)
+BUILD_HOSTNAME_FROM_FILE := $$(cat $(SOONG_OUT_DIR)/build_hostname.txt)
BUILD_DATETIME_FROM_FILE := $$(cat $(BUILD_DATETIME_FILE))
# SEPolicy versions
@@ -1227,16 +1231,7 @@
RSCOMPAT_32BIT_ONLY_API_LEVELS := 8 9 10 11 12 13 14 15 16 17 18 19 20
RSCOMPAT_NO_USAGEIO_API_LEVELS := 8 9 10 11 12 13
-# Add BUILD_NUMBER to apps default version name if it's unbundled build.
-ifdef TARGET_BUILD_APPS
-TARGET_BUILD_WITH_APPS_VERSION_NAME := true
-endif
-
-ifdef TARGET_BUILD_WITH_APPS_VERSION_NAME
-APPS_DEFAULT_VERSION_NAME := $(PLATFORM_VERSION)-$(BUILD_NUMBER_FROM_FILE)
-else
APPS_DEFAULT_VERSION_NAME := $(PLATFORM_VERSION)
-endif
# ANDROID_WARNING_ALLOWED_PROJECTS is generated by build/soong.
define find_warning_allowed_projects
diff --git a/core/definitions.mk b/core/definitions.mk
index e4cee7a..7697211 100644
--- a/core/definitions.mk
+++ b/core/definitions.mk
@@ -75,9 +75,6 @@
# All findbugs xml files
ALL_FINDBUGS_FILES:=
-# GPL module license files
-ALL_GPL_MODULE_LICENSE_FILES:=
-
# Packages with certificate violation
CERTIFICATE_VIOLATION_MODULES :=
@@ -897,7 +894,8 @@
endef
###########################################################
-## Declare license dependencies $(2) for non-module target $(1)
+## Declare license dependencies $(2) with optional colon-separated
+## annotations for non-module target $(1)
###########################################################
define declare-license-deps
$(strip \
@@ -909,7 +907,8 @@
endef
###########################################################
-## Declare license dependencies $(2) for non-module container-type target $(1)
+## Declare license dependencies $(2) with optional colon-separated
+## annotations for non-module container-type target $(1)
##
## Container-type targets are targets like .zip files that
## merely aggregate other files.
diff --git a/core/dex_preopt_config.mk b/core/dex_preopt_config.mk
index e36e2eb..7b9c4db 100644
--- a/core/dex_preopt_config.mk
+++ b/core/dex_preopt_config.mk
@@ -12,9 +12,15 @@
# would result in passing bad arguments to dex2oat and failing the build.
ENABLE_PREOPT :=
ENABLE_PREOPT_BOOT_IMAGES :=
-else ifeq (true,$(DISABLE_PREOPT))
- # Disable dexpreopt for libraries/apps, but do compile boot images.
- ENABLE_PREOPT :=
+else
+ ifeq (true,$(DISABLE_PREOPT))
+ # Disable dexpreopt for libraries/apps, but may compile boot images.
+ ENABLE_PREOPT :=
+ endif
+ ifeq (true,$(DISABLE_PREOPT_BOOT_IMAGES))
+ # Disable dexpreopt for boot images, but may compile libraries/apps.
+ ENABLE_PREOPT_BOOT_IMAGES :=
+ endif
endif
# The default value for LOCAL_DEX_PREOPT
diff --git a/core/dex_preopt_odex_install.mk b/core/dex_preopt_odex_install.mk
index cb16321..7165bea 100644
--- a/core/dex_preopt_odex_install.mk
+++ b/core/dex_preopt_odex_install.mk
@@ -84,12 +84,13 @@
ifndef LOCAL_DEX_PREOPT_GENERATE_PROFILE
# If LOCAL_DEX_PREOPT_GENERATE_PROFILE is not defined, default it based on the existence of the
# profile class listing. TODO: Use product specific directory here.
- my_classes_directory := $(PRODUCT_DEX_PREOPT_PROFILE_DIR)
- LOCAL_DEX_PREOPT_PROFILE := $(my_classes_directory)/$(LOCAL_MODULE).prof
+ ifdef PRODUCT_DEX_PREOPT_PROFILE_DIR
+ LOCAL_DEX_PREOPT_PROFILE := $(PRODUCT_DEX_PREOPT_PROFILE_DIR)/$(LOCAL_MODULE).prof
- ifneq (,$(wildcard $(LOCAL_DEX_PREOPT_PROFILE)))
- my_process_profile := true
- my_profile_is_text_listing :=
+ ifneq (,$(wildcard $(LOCAL_DEX_PREOPT_PROFILE)))
+ my_process_profile := true
+ my_profile_is_text_listing :=
+ endif
endif
else
my_process_profile := $(LOCAL_DEX_PREOPT_GENERATE_PROFILE)
diff --git a/core/distdir.mk b/core/distdir.mk
index bce8e7f..032d1b7 100644
--- a/core/distdir.mk
+++ b/core/distdir.mk
@@ -45,6 +45,18 @@
$(eval _all_dist_goal_output_pairs += $$(goal):$$(dst))))
endef
+define add_file_name_tag_suffix
+$(basename $(notdir $1))-FILE_NAME_TAG_PLACEHOLDER$(suffix $1)
+endef
+
+# This function appends suffix FILE_NAME_TAG_PLACEHOLDER from the input file
+# $(1): a list of goals (e.g. droid, sdk, ndk). These must be PHONY
+# $(2): the dist files to add to those goals.
+define dist-for-goals-with-filenametag
+$(if $(strip $(2)), \
+ $(foreach file,$(2), \
+ $(call dist-for-goals,$(1),$(file):$(call add_file_name_tag_suffix,$(file)))))
+endef
.PHONY: shareprojects
define __share-projects-rule
@@ -209,4 +221,4 @@
fi))
endef
-.KATI_READONLY := dist-for-goals dist-write-file
+.KATI_READONLY := dist-for-goals dist-write-file dist-for-goals-with-filenametag
diff --git a/core/envsetup.mk b/core/envsetup.mk
index 7dd9b12..8887ddc 100644
--- a/core/envsetup.mk
+++ b/core/envsetup.mk
@@ -24,14 +24,30 @@
#$(warning $(call find_and_earlier,A B C,C))
#$(warning $(call find_and_earlier,A B C,D))
-define version-list
-$(1)P1A $(1)P1B $(1)P2A $(1)P2B $(1)D1A $(1)D1B $(1)D2A $(1)D2B $(1)Q1A $(1)Q1B $(1)Q2A $(1)Q2B $(1)Q3A $(1)Q3B
+# Runs the starlark file given in $(1), and sets all the variables in its top-level
+# variables_to_export_to_make variable as make variables.
+#
+# In order to avoid running starlark every time the stamp file is checked, we use
+# $(KATI_shell_no_rerun). Then, to make sure that we actually do rerun kati when
+# modifying the starlark files, we add the starlark files to the kati stamp file with
+# $(KATI_extra_file_deps).
+define run-starlark
+$(eval _starlark_results := $(OUT_DIR)/starlark_results/$(subst /,_,$(1)).mk)
+$(KATI_shell_no_rerun mkdir -p $(OUT_DIR)/starlark_results && $(OUT_DIR)/rbcrun --mode=make $(1) >$(_starlark_results) && touch -t 200001010000 $(_starlark_results))
+$(if $(filter-out 0,$(.SHELLSTATUS)),$(error Starlark failed to run))
+$(eval include $(_starlark_results))
+$(KATI_extra_file_deps $(LOADED_STARLARK_FILES))
+$(eval LOADED_STARLARK_FILES :=)
+$(eval _starlark_results :=)
endef
-PREV_VERSIONS := OPR1 OPD1 OPD2 OPM1 OPM2 PPR1 PPD1 PPD2 PPM1 PPM2 QPR1
-ALL_VERSIONS := Q R S T U V W X Y Z
-ALL_VERSIONS := $(PREV_VERSIONS) $(foreach v,$(ALL_VERSIONS),$(call version-list,$(v)))
-PREV_VERSIONS :=
+# ---------------------------------------------------------------
+# Release config
+include $(BUILD_SYSTEM)/release_config.mk
+
+# ---------------------------------------------------------------
+# defines ALL_VERSIONS
+$(call run-starlark,build/make/core/all_versions.bzl)
# Filters ALL_VERSIONS down to the range [$1, $2], and errors if $1 > $2 or $3 is
# not in [$1, $2]
@@ -339,6 +355,7 @@
RBC_PRODUCT_CONFIG \
RBC_BOARD_CONFIG \
SOONG_% \
+ TARGET_RELEASE \
TOPDIR \
TRACE_BEGIN_SOONG \
USER)
@@ -553,6 +570,8 @@
TARGET_OUT_NOTICE_FILES := $(TARGET_OUT_INTERMEDIATES)/NOTICE_FILES
TARGET_OUT_FAKE := $(PRODUCT_OUT)/fake_packages
TARGET_OUT_TESTCASES := $(PRODUCT_OUT)/testcases
+TARGET_OUT_FLAGS := $(TARGET_OUT_INTERMEDIATES)/FLAGS
+
.KATI_READONLY := \
TARGET_OUT_EXECUTABLES \
TARGET_OUT_OPTIONAL_EXECUTABLES \
@@ -566,7 +585,8 @@
TARGET_OUT_ETC \
TARGET_OUT_NOTICE_FILES \
TARGET_OUT_FAKE \
- TARGET_OUT_TESTCASES
+ TARGET_OUT_TESTCASES \
+ TARGET_OUT_FLAGS
ifeq ($(SANITIZE_LITE),true)
# When using SANITIZE_LITE, APKs must not be packaged with sanitized libraries, as they will not
diff --git a/core/host_java_library.mk b/core/host_java_library.mk
index 89aa53c..d45da48 100644
--- a/core/host_java_library.mk
+++ b/core/host_java_library.mk
@@ -98,9 +98,7 @@
$(full_classes_combined_jar): $(full_classes_compiled_jar) \
$(jar_manifest_file) \
$(full_static_java_libs) | $(MERGE_ZIPS)
- $(if $(PRIVATE_JAR_MANIFEST), $(hide) sed -e "s/%BUILD_NUMBER%/$(BUILD_NUMBER_FROM_FILE)/" \
- $(PRIVATE_JAR_MANIFEST) > $(dir $@)/manifest.mf)
- $(MERGE_ZIPS) -j --ignore-duplicates $(if $(PRIVATE_JAR_MANIFEST),-m $(dir $@)/manifest.mf) \
+ $(MERGE_ZIPS) -j --ignore-duplicates $(if $(PRIVATE_JAR_MANIFEST),-m $(PRIVATE_JAR_MANIFEST)) \
$(if $(PRIVATE_DONT_DELETE_JAR_META_INF),,-stripDir META-INF -zipToNotStrip $<) \
$@ $< $(PRIVATE_STATIC_JAVA_LIBRARIES)
diff --git a/core/java.mk b/core/java.mk
index b13ef4d..842fcbf 100644
--- a/core/java.mk
+++ b/core/java.mk
@@ -296,9 +296,7 @@
$(full_classes_combined_jar): $(full_classes_compiled_jar) \
$(jar_manifest_file) \
$(full_static_java_libs) | $(MERGE_ZIPS)
- $(if $(PRIVATE_JAR_MANIFEST), $(hide) sed -e "s/%BUILD_NUMBER%/$(BUILD_NUMBER_FROM_FILE)/" \
- $(PRIVATE_JAR_MANIFEST) > $(dir $@)/manifest.mf)
- $(MERGE_ZIPS) -j --ignore-duplicates $(if $(PRIVATE_JAR_MANIFEST),-m $(dir $@)/manifest.mf) \
+ $(MERGE_ZIPS) -j --ignore-duplicates $(if $(PRIVATE_JAR_MANIFEST),-m $(PRIVATE_JAR_MANIFEST)) \
$(if $(PRIVATE_DONT_DELETE_JAR_META_INF),,-stripDir META-INF -zipToNotStrip $<) \
$@ $< $(PRIVATE_STATIC_JAVA_LIBRARIES)
diff --git a/core/main.mk b/core/main.mk
index 6a24bd3..903e261 100644
--- a/core/main.mk
+++ b/core/main.mk
@@ -40,31 +40,23 @@
# Write the build number to a file so it can be read back in
# without changing the command line every time. Avoids rebuilds
# when using ninja.
-$(shell mkdir -p $(SOONG_OUT_DIR) && \
- echo -n $(BUILD_NUMBER) > $(SOONG_OUT_DIR)/build_number.tmp; \
- if ! cmp -s $(SOONG_OUT_DIR)/build_number.tmp $(SOONG_OUT_DIR)/build_number.txt; then \
- mv $(SOONG_OUT_DIR)/build_number.tmp $(SOONG_OUT_DIR)/build_number.txt; \
- else \
- rm $(SOONG_OUT_DIR)/build_number.tmp; \
- fi)
BUILD_NUMBER_FILE := $(SOONG_OUT_DIR)/build_number.txt
-.KATI_READONLY := BUILD_NUMBER_FILE
$(KATI_obsolete_var BUILD_NUMBER,See https://android.googlesource.com/platform/build/+/master/Changes.md#BUILD_NUMBER)
+BUILD_HOSTNAME_FILE := $(SOONG_OUT_DIR)/build_hostname.txt
+$(KATI_obsolete_var BUILD_HOSTNAME,Use BUILD_HOSTNAME_FROM_FILE instead)
+$(KATI_obsolete_var FILE_NAME_TAG,https://android.googlesource.com/platform/build/+/master/Changes.md#FILE_NAME_TAG)
+
$(BUILD_NUMBER_FILE):
- touch $@
+ # empty rule to prevent dangling rule error for a file that is written by soong_ui
+$(BUILD_HOSTNAME_FILE):
+ # empty rule to prevent dangling rule error for a file that is written by soong_ui
+
+.KATI_RESTAT: $(BUILD_NUMBER_FILE)
+.KATI_RESTAT: $(BUILD_HOSTNAME_FILE)
DATE_FROM_FILE := date -d @$(BUILD_DATETIME_FROM_FILE)
.KATI_READONLY := DATE_FROM_FILE
-# Pick a reasonable string to use to identify files.
-ifeq ($(strip $(HAS_BUILD_NUMBER)),false)
- # BUILD_NUMBER has a timestamp in it, which means that
- # it will change every time. Pick a stable value.
- FILE_NAME_TAG := eng.$(BUILD_USERNAME)
-else
- FILE_NAME_TAG := $(file <$(BUILD_NUMBER_FILE))
-endif
-.KATI_READONLY := FILE_NAME_TAG
# Make an empty directory, which can be used to make empty jars
EMPTY_DIRECTORY := $(OUT_DIR)/empty
@@ -1390,7 +1382,7 @@
$(CUSTOM_MODULES) \
)
-# Dedpulicate compatibility suite dist files across modules and packages before
+# Deduplicate compatibility suite dist files across modules and packages before
# copying them to their requested locations. Assign the eval result to an unused
# var to prevent Make from trying to make a sense of it.
_unused := $(call copy-many-files, $(sort $(ALL_COMPATIBILITY_DIST_FILES)))
@@ -1751,15 +1743,15 @@
endif
$(PROGUARD_DICT_ZIP) : $(apps_only_installed_files)
- $(call dist-for-goals,apps_only, $(PROGUARD_DICT_ZIP) $(PROGUARD_DICT_MAPPING))
+ $(call dist-for-goals-with-filenametag,apps_only, $(PROGUARD_DICT_ZIP) $(PROGUARD_DICT_ZIP) $(PROGUARD_DICT_MAPPING))
$(call declare-container-license-deps,$(PROGUARD_DICT_ZIP),$(apps_only_installed_files),$(PRODUCT_OUT)/:/)
$(PROGUARD_USAGE_ZIP) : $(apps_only_installed_files)
- $(call dist-for-goals,apps_only, $(PROGUARD_USAGE_ZIP))
+ $(call dist-for-goals-with-filenametag,apps_only, $(PROGUARD_USAGE_ZIP))
$(call declare-container-license-deps,$(PROGUARD_USAGE_ZIP),$(apps_only_installed_files),$(PRODUCT_OUT)/:/)
$(SYMBOLS_ZIP) : $(apps_only_installed_files)
- $(call dist-for-goals,apps_only, $(SYMBOLS_ZIP) $(SYMBOLS_MAPPING))
+ $(call dist-for-goals-with-filenametag,apps_only, $(SYMBOLS_ZIP) $(SYMBOLS_MAPPING))
$(call declare-container-license-deps,$(SYMBOLS_ZIP),$(apps_only_installed_files),$(PRODUCT_OUT)/:/)
$(COVERAGE_ZIP) : $(apps_only_installed_files)
@@ -1805,17 +1797,23 @@
# avoid disting targets that would cause building framework java sources,
# which we want to avoid in an unbundled build.
- $(call dist-for-goals, droidcore-unbundled, \
+ $(call dist-for-goals-with-filenametag, droidcore-unbundled, \
$(INTERNAL_UPDATE_PACKAGE_TARGET) \
$(INTERNAL_OTA_PACKAGE_TARGET) \
- $(INTERNAL_OTA_METADATA) \
$(INTERNAL_OTA_PARTIAL_PACKAGE_TARGET) \
+ $(BUILT_RAMDISK_16K_TARGET) \
+ $(BUILT_KERNEL_16K_TARGET) \
$(INTERNAL_OTA_RETROFIT_DYNAMIC_PARTITIONS_PACKAGE_TARGET) \
$(SYMBOLS_ZIP) \
$(SYMBOLS_MAPPING) \
$(PROGUARD_DICT_ZIP) \
$(PROGUARD_DICT_MAPPING) \
$(PROGUARD_USAGE_ZIP) \
+ $(BUILT_TARGET_FILES_PACKAGE) \
+ )
+
+ $(call dist-for-goals, droidcore-unbundled, \
+ $(INTERNAL_OTA_METADATA) \
$(COVERAGE_ZIP) \
$(INSTALLED_FILES_FILE) \
$(INSTALLED_FILES_JSON) \
@@ -1843,7 +1841,6 @@
$(INSTALLED_ODM_BUILD_PROP_TARGET):build.prop-odm \
$(INSTALLED_SYSTEM_EXT_BUILD_PROP_TARGET):build.prop-system_ext \
$(INSTALLED_RAMDISK_BUILD_PROP_TARGET):build.prop-ramdisk \
- $(BUILT_TARGET_FILES_PACKAGE) \
$(INSTALLED_ANDROID_INFO_TXT_TARGET) \
$(INSTALLED_MISC_INFO_TARGET) \
$(INSTALLED_RAMDISK_TARGET) \
@@ -1855,7 +1852,7 @@
$(call dist-for-goals, droidcore-unbundled, $(f)))
ifneq ($(ANDROID_BUILD_EMBEDDED),true)
- $(call dist-for-goals, droidcore, \
+ $(call dist-for-goals-with-filenametag, droidcore, \
$(APPS_ZIP) \
$(INTERNAL_EMULATOR_PACKAGE_TARGET) \
)
@@ -1955,10 +1952,8 @@
ifeq ($(HOST_OS),linux)
ALL_SDK_TARGETS := $(INTERNAL_SDK_TARGET)
sdk: $(ALL_SDK_TARGETS)
-$(call dist-for-goals,sdk, \
- $(ALL_SDK_TARGETS) \
- $(INSTALLED_BUILD_PROP_TARGET) \
-)
+$(call dist-for-goals-with-filenametag,sdk,$(ALL_SDK_TARGETS))
+$(call dist-for-goals,sdk,$(INSTALLED_BUILD_PROP_TARGET))
endif
# umbrella targets to assit engineers in verifying builds
@@ -2163,10 +2158,11 @@
$(shell rm $(PRODUCT_OUT)/sbom-metadata.csv >/dev/null 2>&1)
$(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files)
rm -f $@
- @echo installed_file$(comma)module_path$(comma)soong_module_type$(comma)is_prebuilt_make_module$(comma)product_copy_files$(comma)kernel_module_copy_files$(comma)is_platform_generated >> $@
+ @echo installed_file$(comma)module_path$(comma)soong_module_type$(comma)is_prebuilt_make_module$(comma)product_copy_files$(comma)kernel_module_copy_files$(comma)is_platform_generated,build_output_path >> $@
$(foreach f,$(installed_files),\
$(eval _module_name := $(ALL_INSTALLED_FILES.$f)) \
$(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$f)) \
+ $(eval _build_output_path := $(PRODUCT_OUT)/$(_path_on_device)) \
$(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH)))) \
$(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE)))) \
$(eval _is_prebuilt_make_module := $(ALL_MODULES.$(_module_name).IS_PREBUILT_MAKE_MODULE)) \
@@ -2184,9 +2180,9 @@
$(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file)),Y)) \
$(eval _is_partition_compat_symlink := $(if $(findstring $f,$(PARTITION_COMPAT_SYMLINKS)),Y)) \
$(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)$(_is_partition_compat_symlink)) \
- @echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated) >> $@ $(newline) \
+ @echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(_build_output_path) >> $@ $(newline) \
$(if $(_post_installed_dexpreopt_zip), \
- for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated) >> $@ ; done $(newline) \
+ for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i >> $@ ; done $(newline) \
) \
)
@@ -2196,14 +2192,14 @@
$(PRODUCT_OUT)/sbom.spdx.json: $(PRODUCT_OUT)/sbom.spdx
$(PRODUCT_OUT)/sbom.spdx: $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM)
rm -rf $@
- $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --product_out_dir=$(PRODUCT_OUT) --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr="$(PRODUCT_MANUFACTURER)" --json
+ $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --json
$(call dist-for-goals,droid,$(PRODUCT_OUT)/sbom.spdx.json:sbom/sbom.spdx.json)
else
apps_only_sbom_files := $(sort $(patsubst %,%.spdx.json,$(filter %.apk,$(apps_only_installed_files))))
$(apps_only_sbom_files): $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM)
rm -rf $@
- $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --product_out_dir=$(PRODUCT_OUT) --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr="$(PRODUCT_MANUFACTURER)" --unbundled
+ $(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --unbundled_apk
sbom: $(apps_only_sbom_files)
diff --git a/core/package_internal.mk b/core/package_internal.mk
index 2d0a569..7cfab5b 100644
--- a/core/package_internal.mk
+++ b/core/package_internal.mk
@@ -111,24 +111,26 @@
# Determine whether auto-RRO is enabled for this package.
enforce_rro_enabled :=
-ifneq (,$(filter *, $(PRODUCT_ENFORCE_RRO_TARGETS)))
- # * means all system and system_ext APKs, so enable conditionally based on module path.
+ifeq (,$(filter tests,$(LOCAL_MODULE_TAGS)))
+ ifneq (,$(filter *, $(PRODUCT_ENFORCE_RRO_TARGETS)))
+ # * means all system and system_ext APKs, so enable conditionally based on module path.
- # Note that base_rules.mk has not yet been included, so it's likely that only
- # one of LOCAL_MODULE_PATH and the LOCAL_X_MODULE flags has been set.
- ifeq (,$(LOCAL_MODULE_PATH))
- non_rro_target_module := $(filter true,\
- $(LOCAL_ODM_MODULE) \
- $(LOCAL_OEM_MODULE) \
- $(LOCAL_PRODUCT_MODULE) \
- $(LOCAL_PROPRIETARY_MODULE) \
- $(LOCAL_VENDOR_MODULE))
- enforce_rro_enabled := $(if $(non_rro_target_module),,true)
- else ifneq ($(filter $(TARGET_OUT)/%,$(LOCAL_MODULE_PATH)),)
+ # Note that base_rules.mk has not yet been included, so it's likely that only
+ # one of LOCAL_MODULE_PATH and the LOCAL_X_MODULE flags has been set.
+ ifeq (,$(LOCAL_MODULE_PATH))
+ non_rro_target_module := $(filter true,\
+ $(LOCAL_ODM_MODULE) \
+ $(LOCAL_OEM_MODULE) \
+ $(LOCAL_PRODUCT_MODULE) \
+ $(LOCAL_PROPRIETARY_MODULE) \
+ $(LOCAL_VENDOR_MODULE))
+ enforce_rro_enabled := $(if $(non_rro_target_module),,true)
+ else ifneq ($(filter $(TARGET_OUT)/%,$(LOCAL_MODULE_PATH)),)
+ enforce_rro_enabled := true
+ endif
+ else ifneq (,$(filter $(LOCAL_PACKAGE_NAME), $(PRODUCT_ENFORCE_RRO_TARGETS)))
enforce_rro_enabled := true
endif
-else ifneq (,$(filter $(LOCAL_PACKAGE_NAME), $(PRODUCT_ENFORCE_RRO_TARGETS)))
- enforce_rro_enabled := true
endif
product_package_overlays := $(strip \
diff --git a/core/product_config.mk b/core/product_config.mk
index 6a27613..01ad030 100644
--- a/core/product_config.mk
+++ b/core/product_config.mk
@@ -247,7 +247,7 @@
endif
$(shell build/soong/scripts/update_out $(OUT_DIR)/rbc/rbc_product_config_results.mk \
- $(OUT_DIR)/rbcrun $(OUT_DIR)/rbc/launcher.rbc)
+ $(OUT_DIR)/rbcrun --mode=rbc $(OUT_DIR)/rbc/launcher.rbc)
ifneq ($(.SHELLSTATUS),0)
$(error product configuration runner failed: $(.SHELLSTATUS))
endif
diff --git a/core/product_config.rbc b/core/product_config.rbc
index e594894..921f068 100644
--- a/core/product_config.rbc
+++ b/core/product_config.rbc
@@ -379,11 +379,7 @@
def _soong_config_set(g, nsname, var, value):
"""Assigns the value to the variable in the namespace."""
_soong_config_namespace(g, nsname)
- if type(value) == "string":
- # Trim right spaces, because in make the variable is set in an $(eval),
- # which will ignore trailing spaces.
- value = value.rstrip(" ")
- g[_soong_config_namespaces_key][nsname][var]=value
+ g[_soong_config_namespaces_key][nsname][var]=_mkstrip(value)
def _soong_config_append(g, nsname, var, value):
"""Appends to the value of the variable in the namespace."""
@@ -391,9 +387,9 @@
ns = g[_soong_config_namespaces_key][nsname]
oldv = ns.get(var)
if oldv == None:
- ns[var] = value
+ ns[var] = _mkstrip(value)
else:
- ns[var] += " " + value
+ ns[var] += " " + _mkstrip(value)
def _soong_config_get(g, nsname, var):
diff --git a/core/release_config.mk b/core/release_config.mk
new file mode 100644
index 0000000..fdfc6a0
--- /dev/null
+++ b/core/release_config.mk
@@ -0,0 +1,222 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Partitions that get build system flag summaries
+_FLAG_PARTITIONS := system vendor system_ext product
+
+# All possible release flags. Defined in the build_flags.mk files
+# throughout the tree
+_ALL_RELEASE_FLAGS :=
+
+# -----------------------------------------------------------------
+# Choose the flag files
+# Do this first, because we're going to unset TARGET_RELEASE before
+# including anyone, so they don't start making conditionals based on it.
+
+# If this is a google source tree, restrict it to only the one file
+# which has OWNERS control. If it isn't let others define their own.
+# TODO: Remove wildcard for build/release one when all branch manifests
+# have updated.
+config_map_files := $(wildcard build/release/release_config_map.mk) \
+ $(if $(wildcard vendor/google/release/release_config_map.mk), \
+ vendor/google/release/release_config_map.mk, \
+ $(sort \
+ $(wildcard device/*/release/release_config_map.mk) \
+ $(wildcard device/*/*/release/release_config_map.mk) \
+ $(wildcard vendor/*/release/release_config_map.mk) \
+ $(wildcard vendor/*/*/release/release_config_map.mk) \
+ ) \
+ )
+
+# $1 config name
+# $2 release config files
+define declare-release-config
+ $(eval # No duplicates)
+ $(if $(filter $(_all_release_configs), $(strip $(1))), \
+ $(error declare-release-config: config $(strip $(1)) declared in: $(_included) Previously declared here: $(_all_release_configs.$(strip $(1)).DECLARED_IN)) \
+ )
+ $(eval # Must have release config files)
+ $(if $(strip $(2)),, \
+ $(error declare-release-config: config $(strip $(1)) must have release config files) \
+ )
+ $(eval _all_release_configs := $(sort $(_all_release_configs) $(strip $(1))))
+ $(eval _all_release_configs.$(strip $(1)).DECLARED_IN := $(_included))
+ $(eval _all_release_configs.$(strip $(1)).FILES := $(strip $(2)))
+endef
+
+# Include the config map files
+$(foreach f, $(config_map_files), \
+ $(eval _included := $(f)) \
+ $(eval include $(f)) \
+)
+
+# If TARGET_RELEASE is set, fail if there is no matching release config
+# If it isn't set, no release config files will be included and all flags
+# will get their default values.
+ifneq ($(TARGET_RELEASE),)
+ifeq ($(filter $(_all_release_configs), $(TARGET_RELEASE)),)
+ $(error No release config found for TARGET_RELEASE: $(TARGET_RELEASE). Available releases are: $(_all_release_configs))
+else
+ # Choose flag files
+ # Don't sort this, use it in the order they gave us.
+ _release_config_files := $(_all_release_configs.$(TARGET_RELEASE).FILES)
+endif
+else
+# Useful for finding scripts etc that aren't passing or setting TARGET_RELEASE
+ifneq ($(FAIL_IF_NO_RELEASE_CONFIG),)
+ $(error FAIL_IF_NO_RELEASE_CONFIG was set and TARGET_RELEASE was not)
+endif
+_release_config_files :=
+endif
+
+# Unset variables so they can't use it
+define declare-release-config
+$(error declare-release-config can only be called from inside release_config_map.mk files)
+endef
+
+# TODO: Remove this check after enough people have sourced lunch that we don't
+# need to worry about it trying to do get_build_vars TARGET_RELEASE. Maybe after ~9/2023
+ifneq ($(CALLED_FROM_SETUP),true)
+define TARGET_RELEASE
+$(error TARGET_RELEASE may not be accessed directly. Use individual flags.)
+endef
+else
+TARGET_RELEASE:=
+endif
+.KATI_READONLY := TARGET_RELEASE
+
+$(foreach config, $(_all_release_configs), \
+ $(eval _all_release_configs.$(config).DECLARED_IN:= ) \
+ $(eval _all_release_configs.$(config).FILES:= ) \
+)
+_all_release_configs:=
+config_map_files:=
+
+# -----------------------------------------------------------------
+# Declare the flags
+
+# $1 partition(s)
+# $2 flag name. Must start with RELEASE_
+# $3 default. True or false
+define declare-build-flag
+ $(if $(filter-out all $(_FLAG_PARTITIONS), $(strip $(1))), \
+ $(error declare-build-flag: invalid partitions: $(strip $(1))) \
+ )
+ $(if $(and $(filter all,$(strip $(1))),$(filter-out all, $(strip $(1)))), \
+ $(error declare-build-flag: "all" can't be combined with other partitions: $(strip $(1))), \
+ $(eval declare-build-flag.partition := $(_FLAG_PARTITIONS)) \
+ )
+ $(if $(filter-out RELEASE_%, $(strip $(2))), \
+ $(error declare-build-flag: Release flag names must start with RELEASE_: $(strip $(2))) \
+ )
+ $(eval _ALL_RELEASE_FLAGS += $(strip $(2)))
+ $(foreach partition, $(declare-build-flag.partition), \
+ $(eval _ALL_RELEASE_FLAGS.PARTITIONS.$(partition) := $(sort \
+ $(_ALL_RELEASE_FLAGS.PARTITIONS.$(partition)) $(strip $(2)))) \
+ )
+ $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).PARTITIONS := $(declare-build-flag.partition))
+ $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).DEFAULT := $(strip $(3)))
+ $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).DECLARED_IN := $(_included))
+ $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).VALUE := $(strip $(3)))
+ $(eval _ALL_RELEASE_FLAGS.$(strip $(2)).SET_IN := $(_included))
+ $(eval declare-build-flag.partition:=)
+endef
+
+
+# Choose the files
+# If this is a google source tree, restrict it to only the one file
+# which has OWNERS control. If it isn't let others define their own.
+flag_declaration_files := $(wildcard build/release/build_flags.mk) \
+ $(if $(wildcard vendor/google/release/build_flags.mk), \
+ vendor/google/release/build_flags.mk, \
+ $(sort \
+ $(wildcard device/*/release/build_flags.mk) \
+ $(wildcard device/*/*/release/build_flags.mk) \
+ $(wildcard vendor/*/release/build_flags.mk) \
+ $(wildcard vendor/*/*/release/build_flags.mk) \
+ ) \
+ )
+
+# Include the files
+$(foreach f, $(flag_declaration_files), \
+ $(eval _included := $(f)) \
+ $(eval include $(f)) \
+)
+
+# Don't let anyone declare build flags after here
+define declare-build-flag
+$(error declare-build-flag can only be called from inside flag definition files.)
+endef
+
+# No more flags from here on
+.KATI_READONLY := _ALL_RELEASE_FLAGS
+
+# -----------------------------------------------------------------
+# Set the flags
+
+# $(1): Flag name. Must start with RELEASE_ and have been defined by declare-build-flag
+# $(2): Value. True or false
+define set-build-flag
+ $(if $(filter-out $(_ALL_RELEASE_FLAGS), $(strip $(1))), \
+ $(error set-build-flag: Undeclared build flag: $(strip $(1))) \
+ )
+ $(eval _ALL_RELEASE_FLAGS.$(strip $(1)).VALUE := $(strip $(2)))
+ $(eval _ALL_RELEASE_FLAGS.$(strip $(1)).SET_IN := $(_included))
+endef
+
+# This writes directly to a file so that the version never exists in make for
+# people to write conditionals upon.
+define set-release-version
+ $(eval _RELEASE_VERSION := $(strip $(1)))
+endef
+
+# Include the files (if there are any)
+ifneq ($(strip $(_release_config_files)),)
+ $(foreach f, $(_release_config_files), \
+ $(eval _included := $(f)) \
+ $(eval include $(f)) \
+ )
+else
+ # No TARGET_RELEASE means release version 0
+ $(call set-release-version, 0)
+endif
+
+
+ifeq ($(_RELEASE_VERSION)),)
+ $(error No release config file called set-release-version. Included files were: $(_release_config_files))
+endif
+
+# Don't let anyone declare build flags after here
+define set-build-flag
+$(error set-build-flag can only be called from inside release config files.)
+endef
+
+# Don't let anyone set the release version after here
+define set-release-version
+$(error set-release-version can only be called from inside release config files.)
+endef
+
+# Set the flag values, and don't allow any one to modify them.
+$(foreach flag, $(_ALL_RELEASE_FLAGS), \
+ $(eval $(flag) := $(_ALL_RELEASE_FLAGS.$(flag).VALUE)) \
+ $(eval .KATI_READONLY := $(flag)) \
+)
+
+
+# -----------------------------------------------------------------
+# Clear out vars
+flag_declaration_files:=
+flag_files:=
+_included:=
+_release_config_files:=
diff --git a/core/soong_app_prebuilt.mk b/core/soong_app_prebuilt.mk
index dd550b5..ccc5449 100644
--- a/core/soong_app_prebuilt.mk
+++ b/core/soong_app_prebuilt.mk
@@ -239,26 +239,28 @@
include $(BUILD_SYSTEM)/link_type.mk
endif # !LOCAL_IS_HOST_MODULE
-ifdef LOCAL_SOONG_DEVICE_RRO_DIRS
- $(call append_enforce_rro_sources, \
- $(my_register_name), \
- false, \
- $(LOCAL_FULL_MANIFEST_FILE), \
- $(if $(LOCAL_EXPORT_PACKAGE_RESOURCES),true,false), \
- $(LOCAL_SOONG_DEVICE_RRO_DIRS), \
- vendor \
- )
-endif
+ifeq (,$(filter tests,$(LOCAL_MODULE_TAGS)))
+ ifdef LOCAL_SOONG_DEVICE_RRO_DIRS
+ $(call append_enforce_rro_sources, \
+ $(my_register_name), \
+ false, \
+ $(LOCAL_FULL_MANIFEST_FILE), \
+ $(if $(LOCAL_EXPORT_PACKAGE_RESOURCES),true,false), \
+ $(LOCAL_SOONG_DEVICE_RRO_DIRS), \
+ vendor \
+ )
+ endif
-ifdef LOCAL_SOONG_PRODUCT_RRO_DIRS
- $(call append_enforce_rro_sources, \
- $(my_register_name), \
- false, \
- $(LOCAL_FULL_MANIFEST_FILE), \
- $(if $(LOCAL_EXPORT_PACKAGE_RESOURCES),true,false), \
- $(LOCAL_SOONG_PRODUCT_RRO_DIRS), \
- product \
- )
+ ifdef LOCAL_SOONG_PRODUCT_RRO_DIRS
+ $(call append_enforce_rro_sources, \
+ $(my_register_name), \
+ false, \
+ $(LOCAL_FULL_MANIFEST_FILE), \
+ $(if $(LOCAL_EXPORT_PACKAGE_RESOURCES),true,false), \
+ $(LOCAL_SOONG_PRODUCT_RRO_DIRS), \
+ product \
+ )
+ endif
endif
ifdef LOCAL_PREBUILT_COVERAGE_ARCHIVE
@@ -273,4 +275,4 @@
###########################################################
## SBOM generation
###########################################################
-include $(BUILD_SBOM_GEN)
\ No newline at end of file
+include $(BUILD_SBOM_GEN)
diff --git a/core/soong_config.mk b/core/soong_config.mk
index c234292..1021650 100644
--- a/core/soong_config.mk
+++ b/core/soong_config.mk
@@ -285,6 +285,7 @@
$(call add_json_str, ShippingApiLevel, $(PRODUCT_SHIPPING_API_LEVEL))
+$(call add_json_list, BuildBrokenPluginValidation, $(BUILD_BROKEN_PLUGIN_VALIDATION))
$(call add_json_bool, BuildBrokenClangProperty, $(filter true,$(BUILD_BROKEN_CLANG_PROPERTY)))
$(call add_json_bool, BuildBrokenClangAsFlags, $(filter true,$(BUILD_BROKEN_CLANG_ASFLAGS)))
$(call add_json_bool, BuildBrokenClangCFlags, $(filter true,$(BUILD_BROKEN_CLANG_CFLAGS)))
@@ -319,6 +320,9 @@
$(call add_json_str, ProductBrand, $(PRODUCT_BRAND))
$(call add_json_list, BuildVersionTags, $(BUILD_VERSION_TAGS))
+$(call add_json_str, ReleaseVersion, $(_RELEASE_VERSION))
+$(call add_json_list, ReleaseDeviceConfigValueSets, $(RELEASE_DEVICE_CONFIG_VALUE_SETS))
+
$(call json_end)
$(file >$(SOONG_VARIABLES).tmp,$(json_contents))
diff --git a/core/sysprop.mk b/core/sysprop.mk
index bd6f3d9..cf4b1f6 100644
--- a/core/sysprop.mk
+++ b/core/sysprop.mk
@@ -174,7 +174,7 @@
ifeq ($(strip $(HAS_BUILD_NUMBER)),false)
BF_BUILD_NUMBER := $(BUILD_USERNAME)$$($(DATE_FROM_FILE) +%m%d%H%M)
else
- BF_BUILD_NUMBER := $(file <$(BUILD_NUMBER_FILE))
+ BF_BUILD_NUMBER := $$(cat $(SOONG_OUT_DIR)/build_hostname.txt)
endif
BUILD_FINGERPRINT := $(PRODUCT_BRAND)/$(TARGET_PRODUCT)/$(TARGET_DEVICE):$(PLATFORM_VERSION)/$(BUILD_ID)/$(BF_BUILD_NUMBER):$(TARGET_BUILD_VARIANT)/$(BUILD_VERSION_TAGS)
endif
@@ -196,6 +196,9 @@
endif
BUILD_THUMBPRINT_FILE := $(PRODUCT_OUT)/build_thumbprint.txt
+ifeq ($(strip $(HAS_BUILD_NUMBER)),true)
+$(BUILD_THUMBPRINT_FILE): $(BUILD_NUMBER_FILE)
+endif
ifneq (,$(shell mkdir -p $(PRODUCT_OUT) && echo $(BUILD_THUMBPRINT) >$(BUILD_THUMBPRINT_FILE) && grep " " $(BUILD_THUMBPRINT_FILE)))
$(error BUILD_THUMBPRINT cannot contain spaces: "$(file <$(BUILD_THUMBPRINT_FILE))")
endif
@@ -260,7 +263,11 @@
endef
gen_from_buildinfo_sh := $(call intermediates-dir-for,PACKAGING,system_build_prop)/buildinfo.prop
-$(gen_from_buildinfo_sh): $(INTERNAL_BUILD_ID_MAKEFILE) $(API_FINGERPRINT) | $(BUILD_DATETIME_FILE) $(BUILD_NUMBER_FILE)
+
+ifeq ($(strip $(HAS_BUILD_NUMBER)),true)
+$(gen_from_buildinfo_sh): $(BUILD_NUMBER_FILE)
+endif
+$(gen_from_buildinfo_sh): $(INTERNAL_BUILD_ID_MAKEFILE) $(API_FINGERPRINT) $(BUILD_HOSTNAME_FILE) | $(BUILD_DATETIME_FILE)
$(hide) TARGET_BUILD_TYPE="$(TARGET_BUILD_VARIANT)" \
TARGET_BUILD_FLAVOR="$(TARGET_BUILD_FLAVOR)" \
TARGET_DEVICE="$(TARGET_DEVICE)" \
@@ -271,7 +278,7 @@
BUILD_DISPLAY_ID="$(BUILD_DISPLAY_ID)" \
DATE="$(DATE_FROM_FILE)" \
BUILD_USERNAME="$(BUILD_USERNAME)" \
- BUILD_HOSTNAME="$(BUILD_HOSTNAME)" \
+ BUILD_HOSTNAME="$(BUILD_HOSTNAME_FROM_FILE)" \
BUILD_NUMBER="$(BUILD_NUMBER_FROM_FILE)" \
BOARD_USE_VBMETA_DIGTEST_IN_FINGERPRINT="$(BOARD_USE_VBMETA_DIGTEST_IN_FINGERPRINT)" \
PLATFORM_VERSION="$(PLATFORM_VERSION)" \
diff --git a/core/tasks/collect_gpl_sources.mk b/core/tasks/collect_gpl_sources.mk
deleted file mode 100644
index 9e9ab8e..0000000
--- a/core/tasks/collect_gpl_sources.mk
+++ /dev/null
@@ -1,29 +0,0 @@
-# Copyright (C) 2011 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The rule below doesn't have dependenices on the files that it copies,
-# so manually generate into a PACKAGING intermediate dir, which is wiped
-# in installclean between incremental builds on build servers.
-gpl_source_tgz := $(call intermediates-dir-for,PACKAGING,gpl_source)/gpl_source.tgz
-
-ALL_GPL_MODULE_LICENSE_FILES := $(sort $(ALL_GPL_MODULE_LICENSE_FILES))
-
-# FORCE since we can't know whether any of the sources changed
-$(gpl_source_tgz): PRIVATE_PATHS := $(sort $(patsubst %/, %, $(dir $(ALL_GPL_MODULE_LICENSE_FILES))))
-$(gpl_source_tgz) : $(ALL_GPL_MODULE_LICENSE_FILES)
- @echo Package GPL sources: $@
- $(hide) tar cfz $@ --exclude ".git*" $(PRIVATE_PATHS)
-
-# Dist the tgz only if we are doing a full build
-$(call dist-for-goals,droidcore-unbundled,$(gpl_source_tgz))
diff --git a/core/tasks/sdk-addon.mk b/core/tasks/sdk-addon.mk
index 5097f12..7acac72 100644
--- a/core/tasks/sdk-addon.mk
+++ b/core/tasks/sdk-addon.mk
@@ -19,12 +19,13 @@
addon_name := $(PRODUCT_SDK_ADDON_NAME)
ifneq ($(addon_name),)
-addon_dir_leaf := $(addon_name)-$(FILE_NAME_TAG)-$(INTERNAL_SDK_HOST_OS_NAME)
-addon_dir_img := $(addon_dir_leaf)-img
-intermediates := $(HOST_OUT_INTERMEDIATES)/SDK_ADDON/$(addon_name)_intermediates
-full_target := $(HOST_OUT_SDK_ADDON)/$(addon_dir_leaf).zip
-full_target_img := $(HOST_OUT_SDK_ADDON)/$(addon_dir_img).zip
-staging := $(intermediates)
+addon_dir_leaf := $(addon_name)-$(INTERNAL_SDK_HOST_OS_NAME)
+addon_dir_img := $(addon_dir_leaf)-img
+intermediates := $(HOST_OUT_INTERMEDIATES)/SDK_ADDON/$(addon_name)_intermediates
+full_target := $(HOST_OUT_SDK_ADDON)/$(addon_dir_leaf).zip
+full_target_dist_name := $(addon_name)-FILE_NAME_TAG_PLACEHOLDER-$(INTERNAL_SDK_HOST_OS_NAME)
+full_target_img := $(HOST_OUT_SDK_ADDON)/$(addon_dir_img).zip
+staging := $(intermediates)
sdk_addon_deps :=
files_to_copy :=
@@ -140,7 +141,7 @@
else
# When not building an sdk_repo, just dist the addon zip file
# as-is.
-$(call dist-for-goals, sdk_addon, $(full_target))
+$(call dist-for-goals, sdk_addon, $(full_target):$(full_target_dist_name))
endif
else # addon_name
diff --git a/core/tasks/tools/build_custom_image.mk b/core/tasks/tools/build_custom_image.mk
index 2626120..ba97e8a 100644
--- a/core/tasks/tools/build_custom_image.mk
+++ b/core/tasks/tools/build_custom_image.mk
@@ -105,6 +105,9 @@
else ifneq (,$(filter true, $(CUSTOM_IMAGE_AVB_HASH_ENABLE) $(CUSTOM_IMAGE_AVB_HASHTREE_ENABLE)))
$(error Cannot set both CUSTOM_IMAGE_AVB_HASH_ENABLE and CUSTOM_IMAGE_AVB_HASHTREE_ENABLE to true)
endif
+ifeq ($(strip $(HAS_BUILD_NUMBER)),true)
+$(my_built_custom_image): $(BUILD_NUMBER_FILE)
+endif
$(my_built_custom_image): $(INTERNAL_USERIMAGES_DEPS) $(my_built_modules) $(my_image_copy_files) $(my_custom_image_modules_dep) \
$(CUSTOM_IMAGE_DICT_FILE)
@echo "Build image $@"
diff --git a/core/tasks/tools/compatibility.mk b/core/tasks/tools/compatibility.mk
index c770b34..dd2305e 100644
--- a/core/tasks/tools/compatibility.mk
+++ b/core/tasks/tools/compatibility.mk
@@ -44,10 +44,16 @@
# The JDK to package into the test suite zip file. Always package the linux JDK.
test_suite_jdk_dir := $(ANDROID_JAVA_HOME)/../linux-x86
+ifndef test_suite_jdk_files
+ # This file gets included many times, so make sure we only run the $(shell) once.
+ # Otherwise it will slow down every build due to all copies of it being rerun when kati
+ # checks the stamp file.
+ test_suite_jdk_files :=$= $(shell find $(test_suite_jdk_dir) -type f | sort)
+endif
test_suite_jdk := $(call intermediates-dir-for,PACKAGING,$(test_suite_name)_jdk,HOST)/jdk.zip
$(test_suite_jdk): PRIVATE_JDK_DIR := $(test_suite_jdk_dir)
$(test_suite_jdk): PRIVATE_SUBDIR := $(test_suite_subdir)
-$(test_suite_jdk): $(shell find $(test_suite_jdk_dir) -type f | sort)
+$(test_suite_jdk): $(test_suite_jdk_files)
$(test_suite_jdk): $(SOONG_ZIP)
$(SOONG_ZIP) -o $@ -P $(PRIVATE_SUBDIR)/jdk -C $(PRIVATE_JDK_DIR) -D $(PRIVATE_JDK_DIR) -sha256
@@ -112,6 +118,9 @@
$(compatibility_zip): PRIVATE_JDK := $(test_suite_jdk)
$(compatibility_zip): PRIVATE_tests_list := $(out_dir)-tests_list
$(compatibility_zip): PRIVATE_tests_list_zip := $(compatibility_tests_list_zip)
+ifeq ($(strip $(HAS_BUILD_NUMBER)),true)
+$(compatibility_zip): $(BUILD_NUMBER_FILE)
+endif
$(compatibility_zip): $(compatibility_zip_deps) | $(ADB) $(ACP)
# Make dir structure
mkdir -p $(PRIVATE_OUT_DIR)/tools $(PRIVATE_OUT_DIR)/testcases
diff --git a/core/tasks/with-license.mk b/core/tasks/with-license.mk
index d41e77a..5ca974a 100644
--- a/core/tasks/with-license.mk
+++ b/core/tasks/with-license.mk
@@ -20,7 +20,8 @@
name := $(name)_debug
endif
-name := $(name)-flashable-$(FILE_NAME_TAG)-with-license
+dist_name := $(name)-flashable-FILE_NAME_TAG_PLACEHOLDER-with-license
+name := $(name)-flashable-with-license
with_license_intermediates := \
$(call intermediates-dir-for,PACKAGING,with_license)
@@ -42,6 +43,7 @@
$(call declare-container-deps,$(license_image_input_zip),$(BUILT_TARGET_FILES_PACKAGE))
with_license_zip := $(PRODUCT_OUT)/$(name).sh
+dist_name := $(dist_name).sh
$(with_license_zip): PRIVATE_NAME := $(name)
$(with_license_zip): PRIVATE_INPUT_ZIP := $(license_image_input_zip)
$(with_license_zip): PRIVATE_VENDOR_BLOBS_LICENSE := $(VENDOR_BLOBS_LICENSE)
@@ -51,7 +53,7 @@
$(HOST_OUT_EXECUTABLES)/generate-self-extracting-archive $@ \
$(PRIVATE_INPUT_ZIP) $(PRIVATE_NAME) $(PRIVATE_VENDOR_BLOBS_LICENSE)
with-license : $(with_license_zip)
-$(call dist-for-goals, with-license, $(with_license_zip))
+$(call dist-for-goals, with-license, $(with_license_zip):$(dist_name))
$(call declare-1p-container,$(with_license_zip),)
$(call declare-container-license-deps,$(with_license_zip),$(license_image_input_zip),$(with_license_zip):)
diff --git a/core/version_util.mk b/core/version_util.mk
index 47883d8..d4ce113 100644
--- a/core/version_util.mk
+++ b/core/version_util.mk
@@ -246,21 +246,10 @@
# to soong_ui.
$(KATI_obsolete_var BUILD_DATETIME,Use BUILD_DATETIME_FROM_FILE)
-HAS_BUILD_NUMBER := true
-ifndef BUILD_NUMBER
- # BUILD_NUMBER should be set to the source control value that
- # represents the current state of the source code. E.g., a
- # perforce changelist number or a git hash. Can be an arbitrary string
- # (to allow for source control that uses something other than numbers),
- # but must be a single word and a valid file name.
- #
- # If no BUILD_NUMBER is set, create a useful "I am an engineering build
- # from this date/time" value. Make it start with a non-digit so that
- # anyone trying to parse it as an integer will probably get "0".
- BUILD_NUMBER := eng.$(shell echo $${BUILD_USERNAME:0:6}).$(shell $(DATE) +%Y%m%d.%H%M%S)
+ifndef HAS_BUILD_NUMBER
HAS_BUILD_NUMBER := false
endif
-.KATI_READONLY := BUILD_NUMBER HAS_BUILD_NUMBER
+.KATI_READONLY := HAS_BUILD_NUMBER
ifndef PLATFORM_MIN_SUPPORTED_TARGET_SDK_VERSION
# Used to set minimum supported target sdk version. Apps targeting sdk
diff --git a/envsetup.sh b/envsetup.sh
index 916344c..d292dbb 100644
--- a/envsetup.sh
+++ b/envsetup.sh
@@ -312,7 +312,7 @@
# would prevent exporting type info from those packages.
#
# http://b/266688086
- export ANDROID_PYTHONPATH=$T/development/python-packages/adb:$T/development/python-packages:
+ export ANDROID_PYTHONPATH=$T/development/python-packages/adb:$T/development/python-packages/gdbrunner:$T/development/python-packages:
if [ -n $VENDOR_PYTHONPATH ]; then
ANDROID_PYTHONPATH=$ANDROID_PYTHONPATH$VENDOR_PYTHONPATH
fi
@@ -842,7 +842,7 @@
export TARGET_PRODUCT=$(get_build_var TARGET_PRODUCT)
export TARGET_BUILD_VARIANT=$(get_build_var TARGET_BUILD_VARIANT)
if [ -n "$release" ]; then
- export TARGET_RELEASE=$(get_build_var TARGET_RELEASE)
+ export TARGET_RELEASE=$release
else
unset TARGET_RELEASE
fi
diff --git a/packaging/distdir.mk b/packaging/distdir.mk
index 264a8b0..c9508af 100644
--- a/packaging/distdir.mk
+++ b/packaging/distdir.mk
@@ -18,10 +18,12 @@
DIST_GOAL_OUTPUT_PAIRS :=
DIST_SRC_DST_PAIRS :=
include $(KATI_PACKAGE_MK_DIR)/dist.mk
+FILE_NAME_TAG := $(file <$(OUT_DIR)/file_name_tag.txt)
+.KATI_READONLY := FILE_NAME_TAG
$(foreach pair,$(DIST_GOAL_OUTPUT_PAIRS), \
$(eval goal := $(call word-colon,1,$(pair))) \
- $(eval output := $(call word-colon,2,$(pair))) \
+ $(eval output := $(subst FILE_NAME_TAG_PLACEHOLDER,$(FILE_NAME_TAG),$(call word-colon,2,$(pair)))) \
$(eval .PHONY: _dist_$$(goal)) \
$(if $(call streq,$(DIST),true),\
$(eval _dist_$$(goal): $$(DIST_DIR)/$$(output)), \
@@ -37,7 +39,7 @@
ifeq ($(DIST),true)
$(foreach pair,$(DIST_SRC_DST_PAIRS), \
$(eval src := $(call word-colon,1,$(pair))) \
- $(eval dst := $(DIST_DIR)/$(call word-colon,2,$(pair))) \
+ $(eval dst := $(subst FILE_NAME_TAG_PLACEHOLDER,$(FILE_NAME_TAG),$(DIST_DIR)/$(call word-colon,2,$(pair)))) \
$(eval $(call copy-one-dist-file,$(src),$(dst))))
endif
diff --git a/target/product/base_system.mk b/target/product/base_system.mk
index 0f5b8a4..d65e5a4 100644
--- a/target/product/base_system.mk
+++ b/target/product/base_system.mk
@@ -226,7 +226,7 @@
mtpd \
ndc \
netd \
- NetworkStackNext \
+ NetworkStack \
odsign \
org.apache.http.legacy \
otacerts \
diff --git a/target/product/gsi/current.txt b/target/product/gsi/current.txt
index 474cb20..2df85e5 100644
--- a/target/product/gsi/current.txt
+++ b/target/product/gsi/current.txt
@@ -21,6 +21,7 @@
LLNDK: libvulkan.so
VNDK-SP: android.hardware.common-V2-ndk.so
VNDK-SP: android.hardware.common.fmq-V1-ndk.so
+VNDK-SP: android.hardware.graphics.allocator-V2-ndk.so
VNDK-SP: android.hardware.graphics.common-V4-ndk.so
VNDK-SP: android.hardware.graphics.common@1.0.so
VNDK-SP: android.hardware.graphics.common@1.1.so
@@ -30,7 +31,6 @@
VNDK-SP: android.hardware.graphics.mapper@2.1.so
VNDK-SP: android.hardware.graphics.mapper@3.0.so
VNDK-SP: android.hardware.graphics.mapper@4.0.so
-VNDK-SP: android.hardware.graphics.allocator-V2-ndk.so
VNDK-SP: android.hardware.renderscript@1.0.so
VNDK-SP: android.hidl.memory.token@1.0.so
VNDK-SP: android.hidl.memory@1.0-impl.so
@@ -90,7 +90,6 @@
VNDK-core: libcrypto.so
VNDK-core: libcrypto_utils.so
VNDK-core: libcurl.so
-VNDK-core: libdiskconfig.so
VNDK-core: libdumpstateutil.so
VNDK-core: libevent.so
VNDK-core: libexif.so
diff --git a/tools/aconfig/Android.bp b/tools/aconfig/Android.bp
index e762f33..9617e0e 100644
--- a/tools/aconfig/Android.bp
+++ b/tools/aconfig/Android.bp
@@ -23,6 +23,7 @@
"libprotobuf",
"libserde",
"libserde_json",
+ "libtinytemplate",
],
}
diff --git a/tools/aconfig/Cargo.toml b/tools/aconfig/Cargo.toml
index b439858..8517dd2 100644
--- a/tools/aconfig/Cargo.toml
+++ b/tools/aconfig/Cargo.toml
@@ -14,6 +14,7 @@
protobuf = "3.2.0"
serde = { version = "1.0.152", features = ["derive"] }
serde_json = "1.0.93"
+tinytemplate = "1.2.1"
[build-dependencies]
protobuf-codegen = "3.2.0"
diff --git a/tools/aconfig/protos/aconfig.proto b/tools/aconfig/protos/aconfig.proto
index 6eac414..9d36a9e 100644
--- a/tools/aconfig/protos/aconfig.proto
+++ b/tools/aconfig/protos/aconfig.proto
@@ -12,46 +12,67 @@
// See the License for the specific language governing permissions and
// limitations under the License
-// This is the schema definition for of Aconfig files. Modifications need to be
-// either backwards compatible, or include updates to all Aconfig files in the
+// This is the schema definition for aconfig files. Modifications need to be
+// either backwards compatible, or include updates to all aconfig files in the
// Android tree.
syntax = "proto2";
package android.aconfig;
+// messages used in both aconfig input and output
+
enum flag_state {
ENABLED = 1;
DISABLED = 2;
}
-enum permission {
+enum flag_permission {
READ_ONLY = 1;
READ_WRITE = 2;
}
-message value {
- required flag_state state = 1;
- required permission permission = 2;
- optional uint32 since = 3;
+// aconfig input messages: flag declarations and values
+
+message flag_declaration {
+ required string name = 1;
+ required string description = 2;
+};
+
+message flag_declarations {
+ required string namespace = 1;
+ repeated flag_declaration flag = 2;
+};
+
+message flag_value {
+ required string namespace = 1;
+ required string name = 2;
+ required flag_state state = 3;
+ required flag_permission permission = 4;
+};
+
+message flag_values {
+ repeated flag_value flag_value = 1;
+};
+
+// aconfig output messages: parsed and verified flag declarations and values
+
+message tracepoint {
+ // path to declaration or value file relative to $TOP
+ required string source = 1;
+ required flag_state state = 2;
+ required flag_permission permission = 3;
}
-message flag {
- required string id = 1;
- required string description = 2;
- repeated value value = 3;
-};
+message parsed_flag {
+ required string namespace = 1;
+ required string name = 2;
+ required string description = 3;
+ required flag_state state = 4;
+ required flag_permission permission = 5;
+ repeated tracepoint trace = 6;
+}
-message android_config {
- repeated flag flag = 1;
-};
-
-message override {
- required string id = 1;
- required flag_state state = 2;
- required permission permission = 3;
-};
-
-message override_config {
- repeated override override = 1;
-};
+message parsed_flags {
+ repeated parsed_flag parsed_flag = 1;
+}
diff --git a/tools/aconfig/src/aconfig.rs b/tools/aconfig/src/aconfig.rs
index f10ca1f..b9fa324 100644
--- a/tools/aconfig/src/aconfig.rs
+++ b/tools/aconfig/src/aconfig.rs
@@ -14,13 +14,14 @@
* limitations under the License.
*/
-use anyhow::{anyhow, Context, Error, Result};
+use anyhow::{anyhow, bail, Context, Error, Result};
use protobuf::{Enum, EnumOrUnknown};
use serde::{Deserialize, Serialize};
+use crate::cache::{Cache, Item, Tracepoint};
use crate::protos::{
- ProtoAndroidConfig, ProtoFlag, ProtoFlagState, ProtoOverride, ProtoOverrideConfig,
- ProtoPermission, ProtoValue,
+ ProtoFlagDeclaration, ProtoFlagDeclarations, ProtoFlagPermission, ProtoFlagState,
+ ProtoFlagValue, ProtoFlagValues, ProtoParsedFlag, ProtoParsedFlags, ProtoTracepoint,
};
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone, Copy)]
@@ -41,164 +42,167 @@
}
}
+impl From<FlagState> for ProtoFlagState {
+ fn from(state: FlagState) -> Self {
+ match state {
+ FlagState::Enabled => ProtoFlagState::ENABLED,
+ FlagState::Disabled => ProtoFlagState::DISABLED,
+ }
+ }
+}
+
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone, Copy)]
pub enum Permission {
ReadOnly,
ReadWrite,
}
-impl TryFrom<EnumOrUnknown<ProtoPermission>> for Permission {
+impl TryFrom<EnumOrUnknown<ProtoFlagPermission>> for Permission {
type Error = Error;
- fn try_from(proto: EnumOrUnknown<ProtoPermission>) -> Result<Self, Self::Error> {
- match ProtoPermission::from_i32(proto.value()) {
- Some(ProtoPermission::READ_ONLY) => Ok(Permission::ReadOnly),
- Some(ProtoPermission::READ_WRITE) => Ok(Permission::ReadWrite),
+ fn try_from(proto: EnumOrUnknown<ProtoFlagPermission>) -> Result<Self, Self::Error> {
+ match ProtoFlagPermission::from_i32(proto.value()) {
+ Some(ProtoFlagPermission::READ_ONLY) => Ok(Permission::ReadOnly),
+ Some(ProtoFlagPermission::READ_WRITE) => Ok(Permission::ReadWrite),
None => Err(anyhow!("unknown permission enum value {}", proto.value())),
}
}
}
-#[derive(Debug, PartialEq, Eq)]
-pub struct Value {
- state: FlagState,
- permission: Permission,
- since: Option<u32>,
-}
-
-#[allow(dead_code)] // only used in unit tests
-impl Value {
- pub fn new(state: FlagState, permission: Permission, since: u32) -> Value {
- Value { state, permission, since: Some(since) }
- }
-
- pub fn default(state: FlagState, permission: Permission) -> Value {
- Value { state, permission, since: None }
- }
-}
-
-impl TryFrom<ProtoValue> for Value {
- type Error = Error;
-
- fn try_from(proto: ProtoValue) -> Result<Self, Self::Error> {
- let Some(proto_state) = proto.state else {
- return Err(anyhow!("missing 'state' field"));
- };
- let state = proto_state.try_into()?;
- let Some(proto_permission) = proto.permission else {
- return Err(anyhow!("missing 'permission' field"));
- };
- let permission = proto_permission.try_into()?;
- Ok(Value { state, permission, since: proto.since })
+impl From<Permission> for ProtoFlagPermission {
+ fn from(permission: Permission) -> Self {
+ match permission {
+ Permission::ReadOnly => ProtoFlagPermission::READ_ONLY,
+ Permission::ReadWrite => ProtoFlagPermission::READ_WRITE,
+ }
}
}
#[derive(Debug, PartialEq, Eq)]
-pub struct Flag {
- pub id: String,
+pub struct FlagDeclaration {
+ pub name: String,
pub description: String,
-
- // ordered by Value.since; guaranteed to contain at least one item (the default value, with
- // since == None)
- pub values: Vec<Value>,
}
-impl Flag {
+impl FlagDeclaration {
#[allow(dead_code)] // only used in unit tests
- pub fn try_from_text_proto(text_proto: &str) -> Result<Flag> {
- let proto: ProtoFlag = crate::protos::try_from_text_proto(text_proto)
+ pub fn try_from_text_proto(text_proto: &str) -> Result<FlagDeclaration> {
+ let proto: ProtoFlagDeclaration = crate::protos::try_from_text_proto(text_proto)
.with_context(|| text_proto.to_owned())?;
proto.try_into()
}
-
- pub fn try_from_text_proto_list(text_proto: &str) -> Result<Vec<Flag>> {
- let proto: ProtoAndroidConfig = crate::protos::try_from_text_proto(text_proto)
- .with_context(|| text_proto.to_owned())?;
- proto.flag.into_iter().map(|proto_flag| proto_flag.try_into()).collect()
- }
-
- pub fn resolve(&self, build_id: u32) -> (FlagState, Permission) {
- let mut state = self.values[0].state;
- let mut permission = self.values[0].permission;
- for candidate in self.values.iter().skip(1) {
- let since = candidate.since.expect("invariant: non-defaults values have Some(since)");
- if since <= build_id {
- state = candidate.state;
- permission = candidate.permission;
- }
- }
- (state, permission)
- }
}
-impl TryFrom<ProtoFlag> for Flag {
+impl TryFrom<ProtoFlagDeclaration> for FlagDeclaration {
type Error = Error;
- fn try_from(proto: ProtoFlag) -> Result<Self, Self::Error> {
- let Some(id) = proto.id else {
- return Err(anyhow!("missing 'id' field"));
+ fn try_from(proto: ProtoFlagDeclaration) -> Result<Self, Self::Error> {
+ let Some(name) = proto.name else {
+ bail!("missing 'name' field");
};
let Some(description) = proto.description else {
- return Err(anyhow!("missing 'description' field"));
+ bail!("missing 'description' field");
};
- if proto.value.is_empty() {
- return Err(anyhow!("missing 'value' field"));
- }
-
- let mut values: Vec<Value> = vec![];
- for proto_value in proto.value.into_iter() {
- let v: Value = proto_value.try_into()?;
- if values.iter().any(|w| v.since == w.since) {
- let msg = match v.since {
- None => format!("flag {}: multiple default values", id),
- Some(x) => format!("flag {}: multiple values for since={}", id, x),
- };
- return Err(anyhow!(msg));
- }
- values.push(v);
- }
- values.sort_by_key(|v| v.since);
-
- Ok(Flag { id, description, values })
+ Ok(FlagDeclaration { name, description })
}
}
#[derive(Debug, PartialEq, Eq)]
-pub struct Override {
- pub id: String,
+pub struct FlagDeclarations {
+ pub namespace: String,
+ pub flags: Vec<FlagDeclaration>,
+}
+
+impl FlagDeclarations {
+ pub fn try_from_text_proto(text_proto: &str) -> Result<FlagDeclarations> {
+ let proto: ProtoFlagDeclarations = crate::protos::try_from_text_proto(text_proto)
+ .with_context(|| text_proto.to_owned())?;
+ let Some(namespace) = proto.namespace else {
+ bail!("missing 'namespace' field");
+ };
+ let mut flags = vec![];
+ for proto_flag in proto.flag.into_iter() {
+ flags.push(proto_flag.try_into()?);
+ }
+ Ok(FlagDeclarations { namespace, flags })
+ }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct FlagValue {
+ pub namespace: String,
+ pub name: String,
pub state: FlagState,
pub permission: Permission,
}
-impl Override {
+impl FlagValue {
#[allow(dead_code)] // only used in unit tests
- pub fn try_from_text_proto(text_proto: &str) -> Result<Override> {
- let proto: ProtoOverride = crate::protos::try_from_text_proto(text_proto)?;
+ pub fn try_from_text_proto(text_proto: &str) -> Result<FlagValue> {
+ let proto: ProtoFlagValue = crate::protos::try_from_text_proto(text_proto)?;
proto.try_into()
}
- pub fn try_from_text_proto_list(text_proto: &str) -> Result<Vec<Override>> {
- let proto: ProtoOverrideConfig = crate::protos::try_from_text_proto(text_proto)?;
- proto.override_.into_iter().map(|proto_flag| proto_flag.try_into()).collect()
+ pub fn try_from_text_proto_list(text_proto: &str) -> Result<Vec<FlagValue>> {
+ let proto: ProtoFlagValues = crate::protos::try_from_text_proto(text_proto)?;
+ proto.flag_value.into_iter().map(|proto_flag| proto_flag.try_into()).collect()
}
}
-impl TryFrom<ProtoOverride> for Override {
+impl TryFrom<ProtoFlagValue> for FlagValue {
type Error = Error;
- fn try_from(proto: ProtoOverride) -> Result<Self, Self::Error> {
- let Some(id) = proto.id else {
- return Err(anyhow!("missing 'id' field"));
+ fn try_from(proto: ProtoFlagValue) -> Result<Self, Self::Error> {
+ let Some(namespace) = proto.namespace else {
+ bail!("missing 'namespace' field");
+ };
+ let Some(name) = proto.name else {
+ bail!("missing 'name' field");
};
let Some(proto_state) = proto.state else {
- return Err(anyhow!("missing 'state' field"));
+ bail!("missing 'state' field");
};
let state = proto_state.try_into()?;
let Some(proto_permission) = proto.permission else {
- return Err(anyhow!("missing 'permission' field"));
+ bail!("missing 'permission' field");
};
let permission = proto_permission.try_into()?;
- Ok(Override { id, state, permission })
+ Ok(FlagValue { namespace, name, state, permission })
+ }
+}
+
+impl From<Cache> for ProtoParsedFlags {
+ fn from(cache: Cache) -> Self {
+ let mut proto = ProtoParsedFlags::new();
+ for item in cache.into_iter() {
+ proto.parsed_flag.push(item.into());
+ }
+ proto
+ }
+}
+
+impl From<Item> for ProtoParsedFlag {
+ fn from(item: Item) -> Self {
+ let mut proto = crate::protos::ProtoParsedFlag::new();
+ proto.set_namespace(item.namespace.to_owned());
+ proto.set_name(item.name.clone());
+ proto.set_description(item.description.clone());
+ proto.set_state(item.state.into());
+ proto.set_permission(item.permission.into());
+ for trace in item.trace.into_iter() {
+ proto.trace.push(trace.into());
+ }
+ proto
+ }
+}
+
+impl From<Tracepoint> for ProtoTracepoint {
+ fn from(tracepoint: Tracepoint) -> Self {
+ let mut proto = ProtoTracepoint::new();
+ proto.set_source(format!("{}", tracepoint.source));
+ proto.set_state(tracepoint.state.into());
+ proto.set_permission(tracepoint.permission.into());
+ proto
}
}
@@ -208,29 +212,16 @@
#[test]
fn test_flag_try_from_text_proto() {
- let expected = Flag {
- id: "1234".to_owned(),
+ let expected = FlagDeclaration {
+ name: "1234".to_owned(),
description: "Description of the flag".to_owned(),
- values: vec![
- Value::default(FlagState::Disabled, Permission::ReadOnly),
- Value::new(FlagState::Enabled, Permission::ReadWrite, 8),
- ],
};
let s = r#"
- id: "1234"
+ name: "1234"
description: "Description of the flag"
- value {
- state: DISABLED
- permission: READ_ONLY
- }
- value {
- state: ENABLED
- permission: READ_WRITE
- since: 8
- }
"#;
- let actual = Flag::try_from_text_proto(s).unwrap();
+ let actual = FlagDeclaration::try_from_text_proto(s).unwrap();
assert_eq!(expected, actual);
}
@@ -238,115 +229,61 @@
#[test]
fn test_flag_try_from_text_proto_bad_input() {
let s = r#"
- id: "a"
- description: "Description of the flag"
+ name: "a"
"#;
- let error = Flag::try_from_text_proto(s).unwrap_err();
- assert_eq!(format!("{:?}", error), "missing 'value' field");
-
- let s = r#"
- description: "Description of the flag"
- value {
- state: ENABLED
- permission: READ_ONLY
- }
- "#;
- let error = Flag::try_from_text_proto(s).unwrap_err();
+ let error = FlagDeclaration::try_from_text_proto(s).unwrap_err();
assert!(format!("{:?}", error).contains("Message not initialized"));
let s = r#"
- id: "a"
description: "Description of the flag"
- value {
- state: ENABLED
- permission: READ_ONLY
- }
- value {
- state: ENABLED
- permission: READ_ONLY
- }
"#;
- let error = Flag::try_from_text_proto(s).unwrap_err();
- assert_eq!(format!("{:?}", error), "flag a: multiple default values");
+ let error = FlagDeclaration::try_from_text_proto(s).unwrap_err();
+ assert!(format!("{:?}", error).contains("Message not initialized"));
}
#[test]
- fn test_flag_try_from_text_proto_list() {
- let expected = vec![
- Flag {
- id: "a".to_owned(),
- description: "A".to_owned(),
- values: vec![Value::default(FlagState::Enabled, Permission::ReadOnly)],
- },
- Flag {
- id: "b".to_owned(),
- description: "B".to_owned(),
- values: vec![Value::default(FlagState::Disabled, Permission::ReadWrite)],
- },
- ];
+ fn test_namespace_try_from_text_proto() {
+ let expected = FlagDeclarations {
+ namespace: "ns".to_owned(),
+ flags: vec![
+ FlagDeclaration { name: "a".to_owned(), description: "A".to_owned() },
+ FlagDeclaration { name: "b".to_owned(), description: "B".to_owned() },
+ ],
+ };
let s = r#"
+ namespace: "ns"
flag {
- id: "a"
+ name: "a"
description: "A"
- value {
- state: ENABLED
- permission: READ_ONLY
- }
}
flag {
- id: "b"
+ name: "b"
description: "B"
- value {
- state: DISABLED
- permission: READ_WRITE
- }
}
"#;
- let actual = Flag::try_from_text_proto_list(s).unwrap();
+ let actual = FlagDeclarations::try_from_text_proto(s).unwrap();
assert_eq!(expected, actual);
}
#[test]
- fn test_override_try_from_text_proto_list() {
- let expected = Override {
- id: "1234".to_owned(),
+ fn test_flag_declaration_try_from_text_proto_list() {
+ let expected = FlagValue {
+ namespace: "ns".to_owned(),
+ name: "1234".to_owned(),
state: FlagState::Enabled,
permission: Permission::ReadOnly,
};
let s = r#"
- id: "1234"
+ namespace: "ns"
+ name: "1234"
state: ENABLED
permission: READ_ONLY
"#;
- let actual = Override::try_from_text_proto(s).unwrap();
+ let actual = FlagValue::try_from_text_proto(s).unwrap();
assert_eq!(expected, actual);
}
-
- #[test]
- fn test_flag_resolve() {
- let flag = Flag {
- id: "a".to_owned(),
- description: "A".to_owned(),
- values: vec![
- Value::default(FlagState::Disabled, Permission::ReadOnly),
- Value::new(FlagState::Disabled, Permission::ReadWrite, 10),
- Value::new(FlagState::Enabled, Permission::ReadOnly, 20),
- Value::new(FlagState::Enabled, Permission::ReadWrite, 30),
- ],
- };
- assert_eq!((FlagState::Disabled, Permission::ReadOnly), flag.resolve(0));
- assert_eq!((FlagState::Disabled, Permission::ReadOnly), flag.resolve(9));
- assert_eq!((FlagState::Disabled, Permission::ReadWrite), flag.resolve(10));
- assert_eq!((FlagState::Disabled, Permission::ReadWrite), flag.resolve(11));
- assert_eq!((FlagState::Disabled, Permission::ReadWrite), flag.resolve(19));
- assert_eq!((FlagState::Enabled, Permission::ReadOnly), flag.resolve(20));
- assert_eq!((FlagState::Enabled, Permission::ReadOnly), flag.resolve(21));
- assert_eq!((FlagState::Enabled, Permission::ReadOnly), flag.resolve(29));
- assert_eq!((FlagState::Enabled, Permission::ReadWrite), flag.resolve(30));
- assert_eq!((FlagState::Enabled, Permission::ReadWrite), flag.resolve(10_000));
- }
}
diff --git a/tools/aconfig/src/cache.rs b/tools/aconfig/src/cache.rs
index 94443d7..30810fa 100644
--- a/tools/aconfig/src/cache.rs
+++ b/tools/aconfig/src/cache.rs
@@ -14,15 +14,18 @@
* limitations under the License.
*/
-use anyhow::{anyhow, Result};
+use anyhow::{bail, ensure, Result};
use serde::{Deserialize, Serialize};
use std::io::{Read, Write};
-use crate::aconfig::{Flag, FlagState, Override, Permission};
+use crate::aconfig::{FlagDeclaration, FlagState, FlagValue, Permission};
use crate::commands::Source;
+const DEFAULT_FLAG_STATE: FlagState = FlagState::Disabled;
+const DEFAULT_FLAG_PERMISSION: Permission = Permission::ReadWrite;
+
#[derive(Serialize, Deserialize, Debug)]
-pub struct TracePoint {
+pub struct Tracepoint {
pub source: Source,
pub state: FlagState,
pub permission: Permission,
@@ -30,162 +33,325 @@
#[derive(Serialize, Deserialize, Debug)]
pub struct Item {
- pub id: String,
+ // TODO: duplicating the Cache.namespace as Item.namespace makes the internal representation
+ // closer to the proto message `parsed_flag`; hopefully this will enable us to replace the Item
+ // struct and use a newtype instead once aconfig has matured. Until then, namespace should
+ // really be a Cow<String>.
+ pub namespace: String,
+ pub name: String,
pub description: String,
pub state: FlagState,
pub permission: Permission,
- pub trace: Vec<TracePoint>,
+ pub trace: Vec<Tracepoint>,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct Cache {
- build_id: u32,
+ namespace: String,
items: Vec<Item>,
}
-impl Cache {
- pub fn new(build_id: u32) -> Cache {
- Cache { build_id, items: vec![] }
+// TODO: replace this function with Iterator.is_sorted_by_key(...)) when that API becomes stable
+fn iter_is_sorted_by_key<'a, T: 'a, F, K>(iter: impl Iterator<Item = &'a T>, f: F) -> bool
+where
+ F: FnMut(&'a T) -> K,
+ K: PartialOrd<K>,
+{
+ let mut last: Option<K> = None;
+ for current in iter.map(f) {
+ if let Some(l) = last {
+ if l > current {
+ return false;
+ }
+ }
+ last = Some(current);
}
+ true
+}
+impl Cache {
pub fn read_from_reader(reader: impl Read) -> Result<Cache> {
- serde_json::from_reader(reader).map_err(|e| e.into())
+ let cache: Cache = serde_json::from_reader(reader)?;
+ ensure!(
+ iter_is_sorted_by_key(cache.iter(), |item| &item.name),
+ "internal error: flags in cache file not sorted"
+ );
+ Ok(cache)
}
pub fn write_to_writer(&self, writer: impl Write) -> Result<()> {
+ ensure!(
+ iter_is_sorted_by_key(self.iter(), |item| &item.name),
+ "internal error: flags in cache file not sorted"
+ );
serde_json::to_writer(writer, self).map_err(|e| e.into())
}
- pub fn add_flag(&mut self, source: Source, flag: Flag) -> Result<()> {
- if self.items.iter().any(|item| item.id == flag.id) {
- return Err(anyhow!(
- "failed to add flag {} from {}: flag already defined",
- flag.id,
- source,
- ));
- }
- let (state, permission) = flag.resolve(self.build_id);
- self.items.push(Item {
- id: flag.id.clone(),
- description: flag.description,
- state,
- permission,
- trace: vec![TracePoint { source, state, permission }],
- });
- Ok(())
- }
-
- pub fn add_override(&mut self, source: Source, override_: Override) -> Result<()> {
- let Some(existing_item) = self.items.iter_mut().find(|item| item.id == override_.id) else {
- return Err(anyhow!("failed to override flag {}: unknown flag", override_.id));
- };
- existing_item.state = override_.state;
- existing_item.permission = override_.permission;
- existing_item.trace.push(TracePoint {
- source,
- state: override_.state,
- permission: override_.permission,
- });
- Ok(())
- }
-
pub fn iter(&self) -> impl Iterator<Item = &Item> {
self.items.iter()
}
+
+ pub fn into_iter(self) -> impl Iterator<Item = Item> {
+ self.items.into_iter()
+ }
+
+ pub fn namespace(&self) -> &str {
+ debug_assert!(!self.namespace.is_empty());
+ &self.namespace
+ }
}
-impl Item {}
+#[derive(Debug)]
+pub struct CacheBuilder {
+ cache: Cache,
+}
+
+impl CacheBuilder {
+ pub fn new(namespace: String) -> Result<CacheBuilder> {
+ ensure!(!namespace.is_empty(), "empty namespace");
+ let cache = Cache { namespace, items: vec![] };
+ Ok(CacheBuilder { cache })
+ }
+
+ pub fn add_flag_declaration(
+ &mut self,
+ source: Source,
+ declaration: FlagDeclaration,
+ ) -> Result<&mut CacheBuilder> {
+ ensure!(!declaration.name.is_empty(), "empty flag name");
+ ensure!(!declaration.description.is_empty(), "empty flag description");
+ ensure!(
+ self.cache.items.iter().all(|item| item.name != declaration.name),
+ "failed to declare flag {} from {}: flag already declared",
+ declaration.name,
+ source
+ );
+ self.cache.items.push(Item {
+ namespace: self.cache.namespace.clone(),
+ name: declaration.name.clone(),
+ description: declaration.description,
+ state: DEFAULT_FLAG_STATE,
+ permission: DEFAULT_FLAG_PERMISSION,
+ trace: vec![Tracepoint {
+ source,
+ state: DEFAULT_FLAG_STATE,
+ permission: DEFAULT_FLAG_PERMISSION,
+ }],
+ });
+ Ok(self)
+ }
+
+ pub fn add_flag_value(
+ &mut self,
+ source: Source,
+ value: FlagValue,
+ ) -> Result<&mut CacheBuilder> {
+ ensure!(!value.namespace.is_empty(), "empty flag namespace");
+ ensure!(!value.name.is_empty(), "empty flag name");
+ ensure!(
+ value.namespace == self.cache.namespace,
+ "failed to set values for flag {}/{} from {}: expected namespace {}",
+ value.namespace,
+ value.name,
+ source,
+ self.cache.namespace
+ );
+ let Some(existing_item) = self.cache.items.iter_mut().find(|item| item.name == value.name) else {
+ bail!("failed to set values for flag {}/{} from {}: flag not declared", value.namespace, value.name, source);
+ };
+ existing_item.state = value.state;
+ existing_item.permission = value.permission;
+ existing_item.trace.push(Tracepoint {
+ source,
+ state: value.state,
+ permission: value.permission,
+ });
+ Ok(self)
+ }
+
+ pub fn build(mut self) -> Cache {
+ self.cache.items.sort_by_cached_key(|item| item.name.clone());
+ self.cache
+ }
+}
#[cfg(test)]
mod tests {
use super::*;
- use crate::aconfig::{FlagState, Permission, Value};
+ use crate::aconfig::{FlagState, Permission};
#[test]
- fn test_add_flag() {
- let mut cache = Cache::new(1);
- cache
- .add_flag(
+ fn test_add_flag_declaration() {
+ let mut builder = CacheBuilder::new("ns".to_string()).unwrap();
+ builder
+ .add_flag_declaration(
Source::File("first.txt".to_string()),
- Flag {
- id: "foo".to_string(),
- description: "desc".to_string(),
- values: vec![Value::default(FlagState::Enabled, Permission::ReadOnly)],
- },
+ FlagDeclaration { name: "foo".to_string(), description: "desc".to_string() },
)
.unwrap();
- let error = cache
- .add_flag(
+ let error = builder
+ .add_flag_declaration(
Source::File("second.txt".to_string()),
- Flag {
- id: "foo".to_string(),
- description: "desc".to_string(),
- values: vec![Value::default(FlagState::Disabled, Permission::ReadOnly)],
- },
+ FlagDeclaration { name: "foo".to_string(), description: "desc".to_string() },
)
.unwrap_err();
assert_eq!(
&format!("{:?}", error),
- "failed to add flag foo from second.txt: flag already defined"
+ "failed to declare flag foo from second.txt: flag already declared"
+ );
+ builder
+ .add_flag_declaration(
+ Source::File("first.txt".to_string()),
+ FlagDeclaration { name: "bar".to_string(), description: "desc".to_string() },
+ )
+ .unwrap();
+
+ let cache = builder.build();
+
+ // check flags are sorted by name
+ assert_eq!(
+ cache.into_iter().map(|item| item.name).collect::<Vec<_>>(),
+ vec!["bar".to_string(), "foo".to_string()]
);
}
#[test]
- fn test_add_override() {
- fn check(cache: &Cache, id: &str, expected: (FlagState, Permission)) -> bool {
- let item = cache.iter().find(|&item| item.id == id).unwrap();
- item.state == expected.0 && item.permission == expected.1
- }
-
- let mut cache = Cache::new(1);
- let error = cache
- .add_override(
+ fn test_add_flag_value() {
+ let mut builder = CacheBuilder::new("ns".to_string()).unwrap();
+ let error = builder
+ .add_flag_value(
Source::Memory,
- Override {
- id: "foo".to_string(),
+ FlagValue {
+ namespace: "ns".to_string(),
+ name: "foo".to_string(),
state: FlagState::Enabled,
permission: Permission::ReadOnly,
},
)
.unwrap_err();
- assert_eq!(&format!("{:?}", error), "failed to override flag foo: unknown flag");
+ assert_eq!(
+ &format!("{:?}", error),
+ "failed to set values for flag ns/foo from <memory>: flag not declared"
+ );
- cache
- .add_flag(
+ builder
+ .add_flag_declaration(
Source::File("first.txt".to_string()),
- Flag {
- id: "foo".to_string(),
- description: "desc".to_string(),
- values: vec![Value::default(FlagState::Enabled, Permission::ReadOnly)],
- },
+ FlagDeclaration { name: "foo".to_string(), description: "desc".to_string() },
)
.unwrap();
- dbg!(&cache);
- assert!(check(&cache, "foo", (FlagState::Enabled, Permission::ReadOnly)));
- cache
- .add_override(
+ builder
+ .add_flag_value(
Source::Memory,
- Override {
- id: "foo".to_string(),
+ FlagValue {
+ namespace: "ns".to_string(),
+ name: "foo".to_string(),
state: FlagState::Disabled,
- permission: Permission::ReadWrite,
+ permission: Permission::ReadOnly,
},
)
.unwrap();
- dbg!(&cache);
- assert!(check(&cache, "foo", (FlagState::Disabled, Permission::ReadWrite)));
- cache
- .add_override(
+ builder
+ .add_flag_value(
Source::Memory,
- Override {
- id: "foo".to_string(),
+ FlagValue {
+ namespace: "ns".to_string(),
+ name: "foo".to_string(),
state: FlagState::Enabled,
permission: Permission::ReadWrite,
},
)
.unwrap();
- assert!(check(&cache, "foo", (FlagState::Enabled, Permission::ReadWrite)));
+
+ // different namespace -> no-op
+ let error = builder
+ .add_flag_value(
+ Source::Memory,
+ FlagValue {
+ namespace: "some-other-namespace".to_string(),
+ name: "foo".to_string(),
+ state: FlagState::Enabled,
+ permission: Permission::ReadOnly,
+ },
+ )
+ .unwrap_err();
+ assert_eq!(&format!("{:?}", error), "failed to set values for flag some-other-namespace/foo from <memory>: expected namespace ns");
+
+ let cache = builder.build();
+ let item = cache.iter().find(|&item| item.name == "foo").unwrap();
+ assert_eq!(FlagState::Enabled, item.state);
+ assert_eq!(Permission::ReadWrite, item.permission);
+ }
+
+ #[test]
+ fn test_reject_empty_cache_namespace() {
+ CacheBuilder::new("".to_string()).unwrap_err();
+ }
+
+ #[test]
+ fn test_reject_empty_flag_declaration_fields() {
+ let mut builder = CacheBuilder::new("ns".to_string()).unwrap();
+
+ let error = builder
+ .add_flag_declaration(
+ Source::Memory,
+ FlagDeclaration { name: "".to_string(), description: "Description".to_string() },
+ )
+ .unwrap_err();
+ assert_eq!(&format!("{:?}", error), "empty flag name");
+
+ let error = builder
+ .add_flag_declaration(
+ Source::Memory,
+ FlagDeclaration { name: "foo".to_string(), description: "".to_string() },
+ )
+ .unwrap_err();
+ assert_eq!(&format!("{:?}", error), "empty flag description");
+ }
+
+ #[test]
+ fn test_reject_empty_flag_value_files() {
+ let mut builder = CacheBuilder::new("ns".to_string()).unwrap();
+ builder
+ .add_flag_declaration(
+ Source::Memory,
+ FlagDeclaration { name: "foo".to_string(), description: "desc".to_string() },
+ )
+ .unwrap();
+
+ let error = builder
+ .add_flag_value(
+ Source::Memory,
+ FlagValue {
+ namespace: "".to_string(),
+ name: "foo".to_string(),
+ state: FlagState::Enabled,
+ permission: Permission::ReadOnly,
+ },
+ )
+ .unwrap_err();
+ assert_eq!(&format!("{:?}", error), "empty flag namespace");
+
+ let error = builder
+ .add_flag_value(
+ Source::Memory,
+ FlagValue {
+ namespace: "ns".to_string(),
+ name: "".to_string(),
+ state: FlagState::Enabled,
+ permission: Permission::ReadOnly,
+ },
+ )
+ .unwrap_err();
+ assert_eq!(&format!("{:?}", error), "empty flag name");
+ }
+
+ #[test]
+ fn test_iter_is_sorted_by_key() {
+ assert!(iter_is_sorted_by_key(["a", "b", "c"].iter(), |s| s));
+ assert!(iter_is_sorted_by_key(Vec::<&str>::new().iter(), |s| s));
+ assert!(!iter_is_sorted_by_key(["a", "c", "b"].iter(), |s| s));
}
}
diff --git a/tools/aconfig/src/codegen_cpp.rs b/tools/aconfig/src/codegen_cpp.rs
new file mode 100644
index 0000000..2aeea6a
--- /dev/null
+++ b/tools/aconfig/src/codegen_cpp.rs
@@ -0,0 +1,214 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use anyhow::Result;
+use serde::Serialize;
+use tinytemplate::TinyTemplate;
+
+use crate::aconfig::{FlagState, Permission};
+use crate::cache::{Cache, Item};
+use crate::commands::OutputFile;
+
+pub fn generate_cpp_code(cache: &Cache) -> Result<OutputFile> {
+ let class_elements: Vec<ClassElement> = cache.iter().map(create_class_element).collect();
+ let readwrite = class_elements.iter().any(|item| item.readwrite);
+ let namespace = cache.namespace().to_lowercase();
+ let context = Context { namespace: namespace.clone(), readwrite, class_elements };
+ let mut template = TinyTemplate::new();
+ template.add_template("cpp_code_gen", include_str!("../templates/cpp.template"))?;
+ let contents = template.render("cpp_code_gen", &context)?;
+ let path = ["aconfig", &(namespace + ".h")].iter().collect();
+ Ok(OutputFile { contents: contents.into(), path })
+}
+
+#[derive(Serialize)]
+struct Context {
+ pub namespace: String,
+ pub readwrite: bool,
+ pub class_elements: Vec<ClassElement>,
+}
+
+#[derive(Serialize)]
+struct ClassElement {
+ pub readwrite: bool,
+ pub default_value: String,
+ pub flag_name: String,
+}
+
+fn create_class_element(item: &Item) -> ClassElement {
+ ClassElement {
+ readwrite: item.permission == Permission::ReadWrite,
+ default_value: if item.state == FlagState::Enabled {
+ "true".to_string()
+ } else {
+ "false".to_string()
+ },
+ flag_name: item.name.clone(),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::aconfig::{FlagDeclaration, FlagState, FlagValue, Permission};
+ use crate::cache::CacheBuilder;
+ use crate::commands::Source;
+
+ #[test]
+ fn test_cpp_codegen_build_time_flag_only() {
+ let namespace = "my_namespace";
+ let mut builder = CacheBuilder::new(namespace.to_string()).unwrap();
+ builder
+ .add_flag_declaration(
+ Source::File("aconfig_one.txt".to_string()),
+ FlagDeclaration {
+ name: "my_flag_one".to_string(),
+ description: "buildtime disable".to_string(),
+ },
+ )
+ .unwrap()
+ .add_flag_value(
+ Source::Memory,
+ FlagValue {
+ namespace: namespace.to_string(),
+ name: "my_flag_one".to_string(),
+ state: FlagState::Disabled,
+ permission: Permission::ReadOnly,
+ },
+ )
+ .unwrap()
+ .add_flag_declaration(
+ Source::File("aconfig_two.txt".to_string()),
+ FlagDeclaration {
+ name: "my_flag_two".to_string(),
+ description: "buildtime enable".to_string(),
+ },
+ )
+ .unwrap()
+ .add_flag_value(
+ Source::Memory,
+ FlagValue {
+ namespace: namespace.to_string(),
+ name: "my_flag_two".to_string(),
+ state: FlagState::Enabled,
+ permission: Permission::ReadOnly,
+ },
+ )
+ .unwrap();
+ let cache = builder.build();
+ let expect_content = r#"#ifndef my_namespace_HEADER_H
+ #define my_namespace_HEADER_H
+ #include "my_namespace.h"
+
+ namespace my_namespace {
+
+ class my_flag_one {
+ public:
+ virtual const bool value() {
+ return false;
+ }
+ }
+
+ class my_flag_two {
+ public:
+ virtual const bool value() {
+ return true;
+ }
+ }
+
+ }
+ #endif
+ "#;
+ let file = generate_cpp_code(&cache).unwrap();
+ assert_eq!("aconfig/my_namespace.h", file.path.to_str().unwrap());
+ assert_eq!(
+ expect_content.replace(' ', ""),
+ String::from_utf8(file.contents).unwrap().replace(' ', "")
+ );
+ }
+
+ #[test]
+ fn test_cpp_codegen_runtime_flag() {
+ let namespace = "my_namespace";
+ let mut builder = CacheBuilder::new(namespace.to_string()).unwrap();
+ builder
+ .add_flag_declaration(
+ Source::File("aconfig_one.txt".to_string()),
+ FlagDeclaration {
+ name: "my_flag_one".to_string(),
+ description: "buildtime disable".to_string(),
+ },
+ )
+ .unwrap()
+ .add_flag_declaration(
+ Source::File("aconfig_two.txt".to_string()),
+ FlagDeclaration {
+ name: "my_flag_two".to_string(),
+ description: "runtime enable".to_string(),
+ },
+ )
+ .unwrap()
+ .add_flag_value(
+ Source::Memory,
+ FlagValue {
+ namespace: namespace.to_string(),
+ name: "my_flag_two".to_string(),
+ state: FlagState::Enabled,
+ permission: Permission::ReadWrite,
+ },
+ )
+ .unwrap();
+ let cache = builder.build();
+ let expect_content = r#"#ifndef my_namespace_HEADER_H
+ #define my_namespace_HEADER_H
+ #include "my_namespace.h"
+
+ #include <server_configurable_flags/get_flags.h>
+ using namespace server_configurable_flags;
+
+ namespace my_namespace {
+
+ class my_flag_one {
+ public:
+ virtual const bool value() {
+ return GetServerConfigurableFlag(
+ "my_namespace",
+ "my_flag_one",
+ "false") == "true";
+ }
+ }
+
+ class my_flag_two {
+ public:
+ virtual const bool value() {
+ return GetServerConfigurableFlag(
+ "my_namespace",
+ "my_flag_two",
+ "true") == "true";
+ }
+ }
+
+ }
+ #endif
+ "#;
+ let file = generate_cpp_code(&cache).unwrap();
+ assert_eq!("aconfig/my_namespace.h", file.path.to_str().unwrap());
+ assert_eq!(
+ expect_content.replace(' ', ""),
+ String::from_utf8(file.contents).unwrap().replace(' ', "")
+ );
+ }
+}
diff --git a/tools/aconfig/src/codegen_java.rs b/tools/aconfig/src/codegen_java.rs
new file mode 100644
index 0000000..733b1c5
--- /dev/null
+++ b/tools/aconfig/src/codegen_java.rs
@@ -0,0 +1,136 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use anyhow::Result;
+use serde::Serialize;
+use std::path::PathBuf;
+use tinytemplate::TinyTemplate;
+
+use crate::aconfig::{FlagState, Permission};
+use crate::cache::{Cache, Item};
+use crate::commands::OutputFile;
+
+pub fn generate_java_code(cache: &Cache) -> Result<OutputFile> {
+ let class_elements: Vec<ClassElement> = cache.iter().map(create_class_element).collect();
+ let readwrite = class_elements.iter().any(|item| item.readwrite);
+ let namespace = cache.namespace();
+ let context = Context { namespace: namespace.to_string(), readwrite, class_elements };
+ let mut template = TinyTemplate::new();
+ template.add_template("java_code_gen", include_str!("../templates/java.template"))?;
+ let contents = template.render("java_code_gen", &context)?;
+ let mut path: PathBuf = namespace.split('.').collect();
+ // TODO: Allow customization of the java class name
+ path.push("Flags.java");
+ Ok(OutputFile { contents: contents.into(), path })
+}
+
+#[derive(Serialize)]
+struct Context {
+ pub namespace: String,
+ pub readwrite: bool,
+ pub class_elements: Vec<ClassElement>,
+}
+
+#[derive(Serialize)]
+struct ClassElement {
+ pub method_name: String,
+ pub readwrite: bool,
+ pub default_value: String,
+ pub feature_name: String,
+ pub flag_name: String,
+}
+
+fn create_class_element(item: &Item) -> ClassElement {
+ ClassElement {
+ method_name: item.name.clone(),
+ readwrite: item.permission == Permission::ReadWrite,
+ default_value: if item.state == FlagState::Enabled {
+ "true".to_string()
+ } else {
+ "false".to_string()
+ },
+ feature_name: item.name.clone(),
+ flag_name: item.name.clone(),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::aconfig::{FlagDeclaration, FlagValue};
+ use crate::cache::CacheBuilder;
+ use crate::commands::Source;
+
+ #[test]
+ fn test_generate_java_code() {
+ let namespace = "com.example";
+ let mut builder = CacheBuilder::new(namespace.to_string()).unwrap();
+ builder
+ .add_flag_declaration(
+ Source::File("test.txt".to_string()),
+ FlagDeclaration {
+ name: "test".to_string(),
+ description: "buildtime enable".to_string(),
+ },
+ )
+ .unwrap()
+ .add_flag_declaration(
+ Source::File("test2.txt".to_string()),
+ FlagDeclaration {
+ name: "test2".to_string(),
+ description: "runtime disable".to_string(),
+ },
+ )
+ .unwrap()
+ .add_flag_value(
+ Source::Memory,
+ FlagValue {
+ namespace: namespace.to_string(),
+ name: "test".to_string(),
+ state: FlagState::Disabled,
+ permission: Permission::ReadOnly,
+ },
+ )
+ .unwrap();
+ let cache = builder.build();
+ let expect_content = r#"package com.example;
+
+ import android.provider.DeviceConfig;
+
+ public final class Flags {
+
+ public static boolean test() {
+ return false;
+ }
+
+ public static boolean test2() {
+ return DeviceConfig.getBoolean(
+ "com.example",
+ "test2__test2",
+ false
+ );
+ }
+
+ }
+ "#;
+ let file = generate_java_code(&cache).unwrap();
+ assert_eq!("com/example/Flags.java", file.path.to_str().unwrap());
+ assert_eq!(
+ expect_content.replace(' ', ""),
+ String::from_utf8(file.contents).unwrap().replace(' ', "")
+ );
+ }
+}
diff --git a/tools/aconfig/src/commands.rs b/tools/aconfig/src/commands.rs
index 73d3357..22de331 100644
--- a/tools/aconfig/src/commands.rs
+++ b/tools/aconfig/src/commands.rs
@@ -14,14 +14,19 @@
* limitations under the License.
*/
-use anyhow::{Context, Result};
+use anyhow::{ensure, Context, Result};
use clap::ValueEnum;
+use protobuf::Message;
use serde::{Deserialize, Serialize};
use std::fmt;
use std::io::Read;
+use std::path::PathBuf;
-use crate::aconfig::{Flag, Override};
-use crate::cache::Cache;
+use crate::aconfig::{FlagDeclarations, FlagValue};
+use crate::cache::{Cache, CacheBuilder};
+use crate::codegen_cpp::generate_cpp_code;
+use crate::codegen_java::generate_java_code;
+use crate::protos::ProtoParsedFlags;
#[derive(Serialize, Deserialize, Clone, Debug)]
pub enum Source {
@@ -44,52 +49,93 @@
pub reader: Box<dyn Read>,
}
-pub fn create_cache(build_id: u32, aconfigs: Vec<Input>, overrides: Vec<Input>) -> Result<Cache> {
- let mut cache = Cache::new(build_id);
+pub struct OutputFile {
+ pub path: PathBuf, // relative to some root directory only main knows about
+ pub contents: Vec<u8>,
+}
- for mut input in aconfigs {
+pub fn create_cache(
+ namespace: &str,
+ declarations: Vec<Input>,
+ values: Vec<Input>,
+) -> Result<Cache> {
+ let mut builder = CacheBuilder::new(namespace.to_owned())?;
+
+ for mut input in declarations {
let mut contents = String::new();
input.reader.read_to_string(&mut contents)?;
- let flags = Flag::try_from_text_proto_list(&contents)
+ let dec_list = FlagDeclarations::try_from_text_proto(&contents)
.with_context(|| format!("Failed to parse {}", input.source))?;
- for flag in flags {
- cache.add_flag(input.source.clone(), flag)?;
+ ensure!(
+ namespace == dec_list.namespace,
+ "Failed to parse {}: expected namespace {}, got {}",
+ input.source,
+ namespace,
+ dec_list.namespace
+ );
+ for d in dec_list.flags.into_iter() {
+ builder.add_flag_declaration(input.source.clone(), d)?;
}
}
- for mut input in overrides {
+ for mut input in values {
let mut contents = String::new();
input.reader.read_to_string(&mut contents)?;
- let overrides = Override::try_from_text_proto_list(&contents)
+ let values_list = FlagValue::try_from_text_proto_list(&contents)
.with_context(|| format!("Failed to parse {}", input.source))?;
- for override_ in overrides {
- cache.add_override(input.source.clone(), override_)?;
+ for v in values_list {
+ // TODO: warn about flag values that do not take effect?
+ let _ = builder.add_flag_value(input.source.clone(), v);
}
}
- Ok(cache)
+ Ok(builder.build())
+}
+
+pub fn create_java_lib(cache: &Cache) -> Result<OutputFile> {
+ generate_java_code(cache)
+}
+
+pub fn create_cpp_lib(cache: &Cache) -> Result<OutputFile> {
+ generate_cpp_code(cache)
}
#[derive(Copy, Clone, Debug, PartialEq, Eq, ValueEnum)]
-pub enum Format {
+pub enum DumpFormat {
Text,
Debug,
+ Protobuf,
}
-pub fn dump_cache(cache: Cache, format: Format) -> Result<()> {
- match format {
- Format::Text => {
- for item in cache.iter() {
- println!("{}: {:?}", item.id, item.state);
+pub fn dump_cache(mut caches: Vec<Cache>, format: DumpFormat) -> Result<Vec<u8>> {
+ let mut output = Vec::new();
+ caches.sort_by_cached_key(|cache| cache.namespace().to_string());
+ for cache in caches.into_iter() {
+ match format {
+ DumpFormat::Text => {
+ let mut lines = vec![];
+ for item in cache.iter() {
+ lines.push(format!(
+ "{}/{}: {:?} {:?}\n",
+ item.namespace, item.name, item.state, item.permission
+ ));
+ }
+ output.append(&mut lines.concat().into());
}
- }
- Format::Debug => {
- for item in cache.iter() {
- println!("{:?}", item);
+ DumpFormat::Debug => {
+ let mut lines = vec![];
+ for item in cache.iter() {
+ lines.push(format!("{:?}\n", item));
+ }
+ output.append(&mut lines.concat().into());
+ }
+ DumpFormat::Protobuf => {
+ let parsed_flags: ProtoParsedFlags = cache.into();
+ parsed_flags.write_to_vec(&mut output)?;
}
}
}
- Ok(())
+ Ok(output)
}
#[cfg(test)]
@@ -97,30 +143,112 @@
use super::*;
use crate::aconfig::{FlagState, Permission};
- #[test]
- fn test_create_cache() {
+ fn create_test_cache_ns1() -> Cache {
let s = r#"
+ namespace: "ns1"
flag {
- id: "a"
+ name: "a"
description: "Description of a"
- value {
- state: ENABLED
- permission: READ_WRITE
- }
+ }
+ flag {
+ name: "b"
+ description: "Description of b"
}
"#;
- let aconfigs = vec![Input { source: Source::Memory, reader: Box::new(s.as_bytes()) }];
+ let declarations = vec![Input { source: Source::Memory, reader: Box::new(s.as_bytes()) }];
let o = r#"
- override {
- id: "a"
+ flag_value {
+ namespace: "ns1"
+ name: "a"
state: DISABLED
permission: READ_ONLY
}
"#;
- let overrides = vec![Input { source: Source::Memory, reader: Box::new(o.as_bytes()) }];
- let cache = create_cache(1, aconfigs, overrides).unwrap();
- let item = cache.iter().find(|&item| item.id == "a").unwrap();
+ let values = vec![Input { source: Source::Memory, reader: Box::new(o.as_bytes()) }];
+ create_cache("ns1", declarations, values).unwrap()
+ }
+
+ fn create_test_cache_ns2() -> Cache {
+ let s = r#"
+ namespace: "ns2"
+ flag {
+ name: "c"
+ description: "Description of c"
+ }
+ "#;
+ let declarations = vec![Input { source: Source::Memory, reader: Box::new(s.as_bytes()) }];
+ let o = r#"
+ flag_value {
+ namespace: "ns2"
+ name: "c"
+ state: DISABLED
+ permission: READ_ONLY
+ }
+ "#;
+ let values = vec![Input { source: Source::Memory, reader: Box::new(o.as_bytes()) }];
+ create_cache("ns2", declarations, values).unwrap()
+ }
+
+ #[test]
+ fn test_create_cache() {
+ let caches = create_test_cache_ns1(); // calls create_cache
+ let item = caches.iter().find(|&item| item.name == "a").unwrap();
assert_eq!(FlagState::Disabled, item.state);
assert_eq!(Permission::ReadOnly, item.permission);
}
+
+ #[test]
+ fn test_dump_text_format() {
+ let caches = vec![create_test_cache_ns1()];
+ let bytes = dump_cache(caches, DumpFormat::Text).unwrap();
+ let text = std::str::from_utf8(&bytes).unwrap();
+ assert!(text.contains("a: Disabled"));
+ }
+
+ #[test]
+ fn test_dump_protobuf_format() {
+ use crate::protos::{ProtoFlagPermission, ProtoFlagState, ProtoTracepoint};
+ use protobuf::Message;
+
+ let caches = vec![create_test_cache_ns1()];
+ let bytes = dump_cache(caches, DumpFormat::Protobuf).unwrap();
+ let actual = ProtoParsedFlags::parse_from_bytes(&bytes).unwrap();
+
+ assert_eq!(
+ vec!["a".to_string(), "b".to_string()],
+ actual.parsed_flag.iter().map(|item| item.name.clone().unwrap()).collect::<Vec<_>>()
+ );
+
+ let item =
+ actual.parsed_flag.iter().find(|item| item.name == Some("b".to_string())).unwrap();
+ assert_eq!(item.namespace(), "ns1");
+ assert_eq!(item.name(), "b");
+ assert_eq!(item.description(), "Description of b");
+ assert_eq!(item.state(), ProtoFlagState::DISABLED);
+ assert_eq!(item.permission(), ProtoFlagPermission::READ_WRITE);
+ let mut tp = ProtoTracepoint::new();
+ tp.set_source("<memory>".to_string());
+ tp.set_state(ProtoFlagState::DISABLED);
+ tp.set_permission(ProtoFlagPermission::READ_WRITE);
+ assert_eq!(item.trace, vec![tp]);
+ }
+
+ #[test]
+ fn test_dump_multiple_caches() {
+ let caches = vec![create_test_cache_ns1(), create_test_cache_ns2()];
+ let bytes = dump_cache(caches, DumpFormat::Protobuf).unwrap();
+ let dump = ProtoParsedFlags::parse_from_bytes(&bytes).unwrap();
+ assert_eq!(
+ dump.parsed_flag
+ .iter()
+ .map(|parsed_flag| format!("{}/{}", parsed_flag.namespace(), parsed_flag.name()))
+ .collect::<Vec<_>>(),
+ vec!["ns1/a".to_string(), "ns1/b".to_string(), "ns2/c".to_string()]
+ );
+
+ let caches = vec![create_test_cache_ns2(), create_test_cache_ns1()];
+ let bytes = dump_cache(caches, DumpFormat::Protobuf).unwrap();
+ let dump_reversed_input = ProtoParsedFlags::parse_from_bytes(&bytes).unwrap();
+ assert_eq!(dump, dump_reversed_input);
+ }
}
diff --git a/tools/aconfig/src/main.rs b/tools/aconfig/src/main.rs
index 62750ae..d02307d 100644
--- a/tools/aconfig/src/main.rs
+++ b/tools/aconfig/src/main.rs
@@ -16,41 +16,64 @@
//! `aconfig` is a build time tool to manage build time configurations, such as feature flags.
-use anyhow::Result;
+use anyhow::{anyhow, ensure, Result};
use clap::{builder::ArgAction, builder::EnumValueParser, Arg, ArgMatches, Command};
+use core::any::Any;
use std::fs;
+use std::io;
+use std::io::Write;
+use std::path::{Path, PathBuf};
mod aconfig;
mod cache;
+mod codegen_cpp;
+mod codegen_java;
mod commands;
mod protos;
use crate::cache::Cache;
-use commands::{Input, Source};
+use commands::{DumpFormat, Input, OutputFile, Source};
fn cli() -> Command {
Command::new("aconfig")
.subcommand_required(true)
.subcommand(
Command::new("create-cache")
- .arg(
- Arg::new("build-id")
- .long("build-id")
- .value_parser(clap::value_parser!(u32))
- .required(true),
- )
- .arg(Arg::new("aconfig").long("aconfig").action(ArgAction::Append))
- .arg(Arg::new("override").long("override").action(ArgAction::Append))
+ .arg(Arg::new("namespace").long("namespace").required(true))
+ .arg(Arg::new("declarations").long("declarations").action(ArgAction::Append))
+ .arg(Arg::new("values").long("values").action(ArgAction::Append))
.arg(Arg::new("cache").long("cache").required(true)),
)
.subcommand(
- Command::new("dump").arg(Arg::new("cache").long("cache").required(true)).arg(
- Arg::new("format")
- .long("format")
- .value_parser(EnumValueParser::<commands::Format>::new())
- .default_value("text"),
- ),
+ Command::new("create-java-lib")
+ .arg(Arg::new("cache").long("cache").required(true))
+ .arg(Arg::new("out").long("out").required(true)),
)
+ .subcommand(
+ Command::new("create-cpp-lib")
+ .arg(Arg::new("cache").long("cache").required(true))
+ .arg(Arg::new("out").long("out").required(true)),
+ )
+ .subcommand(
+ Command::new("dump")
+ .arg(Arg::new("cache").long("cache").action(ArgAction::Append).required(true))
+ .arg(
+ Arg::new("format")
+ .long("format")
+ .value_parser(EnumValueParser::<commands::DumpFormat>::new())
+ .default_value("text"),
+ )
+ .arg(Arg::new("out").long("out").default_value("-")),
+ )
+}
+
+fn get_required_arg<'a, T>(matches: &'a ArgMatches, arg_name: &str) -> Result<&'a T>
+where
+ T: Any + Clone + Send + Sync + 'static,
+{
+ matches
+ .get_one::<T>(arg_name)
+ .ok_or(anyhow!("internal error: required argument '{}' not found", arg_name))
}
fn open_zero_or_more_files(matches: &ArgMatches, arg_name: &str) -> Result<Vec<Input>> {
@@ -62,24 +85,66 @@
Ok(opened_files)
}
+fn write_output_file_realtive_to_dir(root: &Path, output_file: &OutputFile) -> Result<()> {
+ ensure!(
+ root.is_dir(),
+ "output directory {} does not exist or is not a directory",
+ root.display()
+ );
+ let path = root.join(output_file.path.clone());
+ let parent = path
+ .parent()
+ .ok_or(anyhow!("unable to locate parent of output file {}", path.display()))?;
+ fs::create_dir_all(parent)?;
+ let mut file = fs::File::create(path)?;
+ file.write_all(&output_file.contents)?;
+ Ok(())
+}
+
fn main() -> Result<()> {
let matches = cli().get_matches();
match matches.subcommand() {
Some(("create-cache", sub_matches)) => {
- let build_id = *sub_matches.get_one::<u32>("build-id").unwrap();
- let aconfigs = open_zero_or_more_files(sub_matches, "aconfig")?;
- let overrides = open_zero_or_more_files(sub_matches, "override")?;
- let cache = commands::create_cache(build_id, aconfigs, overrides)?;
- let path = sub_matches.get_one::<String>("cache").unwrap();
+ let namespace = get_required_arg::<String>(sub_matches, "namespace")?;
+ let declarations = open_zero_or_more_files(sub_matches, "declarations")?;
+ let values = open_zero_or_more_files(sub_matches, "values")?;
+ let cache = commands::create_cache(namespace, declarations, values)?;
+ let path = get_required_arg::<String>(sub_matches, "cache")?;
let file = fs::File::create(path)?;
cache.write_to_writer(file)?;
}
- Some(("dump", sub_matches)) => {
- let path = sub_matches.get_one::<String>("cache").unwrap();
+ Some(("create-java-lib", sub_matches)) => {
+ let path = get_required_arg::<String>(sub_matches, "cache")?;
let file = fs::File::open(path)?;
let cache = Cache::read_from_reader(file)?;
- let format = sub_matches.get_one("format").unwrap();
- commands::dump_cache(cache, *format)?;
+ let dir = PathBuf::from(get_required_arg::<String>(sub_matches, "out")?);
+ let generated_file = commands::create_java_lib(&cache)?;
+ write_output_file_realtive_to_dir(&dir, &generated_file)?;
+ }
+ Some(("create-cpp-lib", sub_matches)) => {
+ let path = get_required_arg::<String>(sub_matches, "cache")?;
+ let file = fs::File::open(path)?;
+ let cache = Cache::read_from_reader(file)?;
+ let dir = PathBuf::from(get_required_arg::<String>(sub_matches, "out")?);
+ let generated_file = commands::create_cpp_lib(&cache)?;
+ write_output_file_realtive_to_dir(&dir, &generated_file)?;
+ }
+ Some(("dump", sub_matches)) => {
+ let mut caches = Vec::new();
+ for path in sub_matches.get_many::<String>("cache").unwrap_or_default() {
+ let file = fs::File::open(path)?;
+ let cache = Cache::read_from_reader(file)?;
+ caches.push(cache);
+ }
+ let format = get_required_arg::<DumpFormat>(sub_matches, "format")?;
+ let output = commands::dump_cache(caches, *format)?;
+ let path = get_required_arg::<String>(sub_matches, "out")?;
+ let mut file: Box<dyn Write> = if *path == "-" {
+ Box::new(io::stdout())
+ } else {
+ Box::new(fs::File::create(path)?)
+ };
+ file.write_all(&output)?;
}
_ => unreachable!(),
}
diff --git a/tools/aconfig/src/protos.rs b/tools/aconfig/src/protos.rs
index 604eca4..cb75692 100644
--- a/tools/aconfig/src/protos.rs
+++ b/tools/aconfig/src/protos.rs
@@ -28,51 +28,63 @@
// ---- When building with the Android tool-chain ----
#[cfg(not(feature = "cargo"))]
-pub use aconfig_protos::aconfig::Android_config as ProtoAndroidConfig;
+pub use aconfig_protos::aconfig::Flag_declaration as ProtoFlagDeclaration;
#[cfg(not(feature = "cargo"))]
-pub use aconfig_protos::aconfig::Value as ProtoValue;
+pub use aconfig_protos::aconfig::Flag_declarations as ProtoFlagDeclarations;
#[cfg(not(feature = "cargo"))]
-pub use aconfig_protos::aconfig::Flag as ProtoFlag;
+pub use aconfig_protos::aconfig::Flag_value as ProtoFlagValue;
#[cfg(not(feature = "cargo"))]
-pub use aconfig_protos::aconfig::Override_config as ProtoOverrideConfig;
+pub use aconfig_protos::aconfig::Flag_values as ProtoFlagValues;
#[cfg(not(feature = "cargo"))]
-pub use aconfig_protos::aconfig::Override as ProtoOverride;
-
-#[cfg(not(feature = "cargo"))]
-pub use aconfig_protos::aconfig::Permission as ProtoPermission;
+pub use aconfig_protos::aconfig::Flag_permission as ProtoFlagPermission;
#[cfg(not(feature = "cargo"))]
pub use aconfig_protos::aconfig::Flag_state as ProtoFlagState;
+#[cfg(not(feature = "cargo"))]
+pub use aconfig_protos::aconfig::Parsed_flags as ProtoParsedFlags;
+
+#[cfg(not(feature = "cargo"))]
+pub use aconfig_protos::aconfig::Parsed_flag as ProtoParsedFlag;
+
+#[cfg(not(feature = "cargo"))]
+pub use aconfig_protos::aconfig::Tracepoint as ProtoTracepoint;
+
// ---- When building with cargo ----
#[cfg(feature = "cargo")]
include!(concat!(env!("OUT_DIR"), "/aconfig_proto/mod.rs"));
#[cfg(feature = "cargo")]
-pub use aconfig::Android_config as ProtoAndroidConfig;
+pub use aconfig::Flag_declaration as ProtoFlagDeclaration;
#[cfg(feature = "cargo")]
-pub use aconfig::Value as ProtoValue;
+pub use aconfig::Flag_declarations as ProtoFlagDeclarations;
#[cfg(feature = "cargo")]
-pub use aconfig::Flag as ProtoFlag;
+pub use aconfig::Flag_value as ProtoFlagValue;
#[cfg(feature = "cargo")]
-pub use aconfig::Override_config as ProtoOverrideConfig;
+pub use aconfig::Flag_values as ProtoFlagValues;
#[cfg(feature = "cargo")]
-pub use aconfig::Override as ProtoOverride;
-
-#[cfg(feature = "cargo")]
-pub use aconfig::Permission as ProtoPermission;
+pub use aconfig::Flag_permission as ProtoFlagPermission;
#[cfg(feature = "cargo")]
pub use aconfig::Flag_state as ProtoFlagState;
+#[cfg(feature = "cargo")]
+pub use aconfig::Parsed_flags as ProtoParsedFlags;
+
+#[cfg(feature = "cargo")]
+pub use aconfig::Parsed_flag as ProtoParsedFlag;
+
+#[cfg(feature = "cargo")]
+pub use aconfig::Tracepoint as ProtoTracepoint;
+
// ---- Common for both the Android tool-chain and cargo ----
use anyhow::Result;
diff --git a/tools/aconfig/templates/cpp.template b/tools/aconfig/templates/cpp.template
new file mode 100644
index 0000000..ae8b59f
--- /dev/null
+++ b/tools/aconfig/templates/cpp.template
@@ -0,0 +1,25 @@
+#ifndef {namespace}_HEADER_H
+#define {namespace}_HEADER_H
+#include "{namespace}.h"
+{{ if readwrite }}
+#include <server_configurable_flags/get_flags.h>
+using namespace server_configurable_flags;
+{{ endif }}
+namespace {namespace} \{
+ {{ for item in class_elements}}
+ class {item.flag_name} \{
+ public:
+ virtual const bool value() \{
+ {{ if item.readwrite- }}
+ return GetServerConfigurableFlag(
+ "{namespace}",
+ "{item.flag_name}",
+ "{item.default_value}") == "true";
+ {{ -else- }}
+ return {item.default_value};
+ {{ -endif }}
+ }
+ }
+ {{ endfor }}
+}
+#endif
diff --git a/tools/aconfig/templates/java.template b/tools/aconfig/templates/java.template
new file mode 100644
index 0000000..89da18b
--- /dev/null
+++ b/tools/aconfig/templates/java.template
@@ -0,0 +1,19 @@
+package {namespace};
+{{ if readwrite }}
+import android.provider.DeviceConfig;
+{{ endif }}
+public final class Flags \{
+ {{ for item in class_elements}}
+ public static boolean {item.method_name}() \{
+ {{ if item.readwrite- }}
+ return DeviceConfig.getBoolean(
+ "{namespace}",
+ "{item.feature_name}__{item.flag_name}",
+ {item.default_value}
+ );
+ {{ -else- }}
+ return {item.default_value};
+ {{ -endif }}
+ }
+ {{ endfor }}
+}
diff --git a/tools/finalization/localonly-steps.sh b/tools/finalization/localonly-steps.sh
index 6107b3e..7318ca1 100755
--- a/tools/finalization/localonly-steps.sh
+++ b/tools/finalization/localonly-steps.sh
@@ -17,7 +17,7 @@
$top/build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=sdk TARGET_BUILD_VARIANT=userdebug sdk dist sdk_repo DIST_DIR=out/dist
# Build Modules SDKs.
- TARGET_BUILD_VARIANT=userdebug UNBUNDLED_BUILD_SDKS_FROM_SOURCE=true DIST_DIR=out/dist "$top/vendor/google/build/mainline_modules_sdks.sh"
+ TARGET_BUILD_VARIANT=userdebug UNBUNDLED_BUILD_SDKS_FROM_SOURCE=true DIST_DIR=out/dist "$top/vendor/google/build/mainline_modules_sdks.sh" --build-release=latest
# Update prebuilts.
"$top/prebuilts/build-tools/path/linux-x86/python3" -W ignore::DeprecationWarning "$top/prebuilts/sdk/update_prebuilts.py" --local_mode -f ${FINAL_PLATFORM_SDK_VERSION} -e ${FINAL_MAINLINE_EXTENSION} --bug 1 1
diff --git a/tools/rbcrun/Android.bp b/tools/rbcrun/Android.bp
index fcc33ef..4fab858 100644
--- a/tools/rbcrun/Android.bp
+++ b/tools/rbcrun/Android.bp
@@ -34,6 +34,7 @@
pkgPath: "rbcrun",
deps: [
"go-starlark-starlark",
+ "go-starlark-starlarkjson",
"go-starlark-starlarkstruct",
"go-starlark-starlarktest",
],
diff --git a/tools/rbcrun/host.go b/tools/rbcrun/host.go
index f2fda4e..a0fb9e1 100644
--- a/tools/rbcrun/host.go
+++ b/tools/rbcrun/host.go
@@ -24,13 +24,19 @@
"strings"
"go.starlark.net/starlark"
+ "go.starlark.net/starlarkjson"
"go.starlark.net/starlarkstruct"
)
-const callerDirKey = "callerDir"
+type ExecutionMode int
+const (
+ ExecutionModeRbc ExecutionMode = iota
+ ExecutionModeMake ExecutionMode = iota
+)
-var LoadPathRoot = "."
-var shellPath string
+const callerDirKey = "callerDir"
+const shellKey = "shell"
+const executionModeKey = "executionMode"
type modentry struct {
globals starlark.StringDict
@@ -39,20 +45,66 @@
var moduleCache = make(map[string]*modentry)
-var builtins starlark.StringDict
+var rbcBuiltins starlark.StringDict = starlark.StringDict{
+ "struct": starlark.NewBuiltin("struct", starlarkstruct.Make),
+ // To convert find-copy-subdir and product-copy-files-by pattern
+ "rblf_find_files": starlark.NewBuiltin("rblf_find_files", find),
+ // To convert makefile's $(shell cmd)
+ "rblf_shell": starlark.NewBuiltin("rblf_shell", shell),
+ // Output to stderr
+ "rblf_log": starlark.NewBuiltin("rblf_log", log),
+ // To convert makefile's $(wildcard foo*)
+ "rblf_wildcard": starlark.NewBuiltin("rblf_wildcard", wildcard),
+}
-func moduleName2AbsPath(moduleName string, callerDir string) (string, error) {
- path := moduleName
- if ix := strings.LastIndex(path, ":"); ix >= 0 {
- path = path[0:ix] + string(os.PathSeparator) + path[ix+1:]
+var makeBuiltins starlark.StringDict = starlark.StringDict{
+ "struct": starlark.NewBuiltin("struct", starlarkstruct.Make),
+ "json": starlarkjson.Module,
+}
+
+// Takes a module name (the first argument to the load() function) and returns the path
+// it's trying to load, stripping out leading //, and handling leading :s.
+func cleanModuleName(moduleName string, callerDir string) (string, error) {
+ if strings.Count(moduleName, ":") > 1 {
+ return "", fmt.Errorf("at most 1 colon must be present in starlark path: %s", moduleName)
}
- if strings.HasPrefix(path, "//") {
- return filepath.Abs(filepath.Join(LoadPathRoot, path[2:]))
+
+ // We don't have full support for external repositories, but at least support skylib's dicts.
+ if moduleName == "@bazel_skylib//lib:dicts.bzl" {
+ return "external/bazel-skylib/lib/dicts.bzl", nil
+ }
+
+ localLoad := false
+ if strings.HasPrefix(moduleName, "@//") {
+ moduleName = moduleName[3:]
+ } else if strings.HasPrefix(moduleName, "//") {
+ moduleName = moduleName[2:]
} else if strings.HasPrefix(moduleName, ":") {
- return filepath.Abs(filepath.Join(callerDir, path[1:]))
+ moduleName = moduleName[1:]
+ localLoad = true
} else {
- return filepath.Abs(path)
+ return "", fmt.Errorf("load path must start with // or :")
}
+
+ if ix := strings.LastIndex(moduleName, ":"); ix >= 0 {
+ moduleName = moduleName[:ix] + string(os.PathSeparator) + moduleName[ix+1:]
+ }
+
+ if filepath.Clean(moduleName) != moduleName {
+ return "", fmt.Errorf("load path must be clean, found: %s, expected: %s", moduleName, filepath.Clean(moduleName))
+ }
+ if strings.HasPrefix(moduleName, "../") {
+ return "", fmt.Errorf("load path must not start with ../: %s", moduleName)
+ }
+ if strings.HasPrefix(moduleName, "/") {
+ return "", fmt.Errorf("load path starts with /, use // for a absolute path: %s", moduleName)
+ }
+
+ if localLoad {
+ return filepath.Join(callerDir, moduleName), nil
+ }
+
+ return moduleName, nil
}
// loader implements load statement. The format of the loaded module URI is
@@ -61,14 +113,18 @@
// The presence of `|symbol` indicates that the loader should return a single 'symbol'
// bound to None if file is missing.
func loader(thread *starlark.Thread, module string) (starlark.StringDict, error) {
- pipePos := strings.LastIndex(module, "|")
- mustLoad := pipePos < 0
+ mode := thread.Local(executionModeKey).(ExecutionMode)
var defaultSymbol string
- if !mustLoad {
- defaultSymbol = module[pipePos+1:]
- module = module[:pipePos]
+ mustLoad := true
+ if mode == ExecutionModeRbc {
+ pipePos := strings.LastIndex(module, "|")
+ mustLoad = pipePos < 0
+ if !mustLoad {
+ defaultSymbol = module[pipePos+1:]
+ module = module[:pipePos]
+ }
}
- modulePath, err := moduleName2AbsPath(module, thread.Local(callerDirKey).(string))
+ modulePath, err := cleanModuleName(module, thread.Local(callerDirKey).(string))
if err != nil {
return nil, err
}
@@ -100,8 +156,17 @@
}
childThread.SetLocal(callerDirKey, filepath.Dir(modulePath))
- globals, err := starlark.ExecFile(childThread, modulePath, nil, builtins)
- e = &modentry{globals, err}
+ childThread.SetLocal(shellKey, thread.Local(shellKey))
+ childThread.SetLocal(executionModeKey, mode)
+ if mode == ExecutionModeRbc {
+ globals, err := starlark.ExecFile(childThread, modulePath, nil, rbcBuiltins)
+ e = &modentry{globals, err}
+ } else if mode == ExecutionModeMake {
+ globals, err := starlark.ExecFile(childThread, modulePath, nil, makeBuiltins)
+ e = &modentry{globals, err}
+ } else {
+ return nil, fmt.Errorf("unknown executionMode %d", mode)
+ }
} else {
e = &modentry{starlark.StringDict{defaultSymbol: starlark.None}, nil}
}
@@ -189,12 +254,13 @@
// its output the same way as Make's $(shell ) function. The end-of-lines
// ("\n" or "\r\n") are replaced with " " in the result, and the trailing
// end-of-line is removed.
-func shell(_ *starlark.Thread, b *starlark.Builtin, args starlark.Tuple,
+func shell(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple,
kwargs []starlark.Tuple) (starlark.Value, error) {
var command string
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &command); err != nil {
return starlark.None, err
}
+ shellPath := thread.Local(shellKey).(string)
if shellPath == "" {
return starlark.None,
fmt.Errorf("cannot run shell, /bin/sh is missing (running on Windows?)")
@@ -245,45 +311,68 @@
return starlark.None, nil
}
-func setup() {
- // Create the symbols that aid makefile conversion. See README.md
- builtins = starlark.StringDict{
- "struct": starlark.NewBuiltin("struct", starlarkstruct.Make),
- // To convert find-copy-subdir and product-copy-files-by pattern
- "rblf_find_files": starlark.NewBuiltin("rblf_find_files", find),
- // To convert makefile's $(shell cmd)
- "rblf_shell": starlark.NewBuiltin("rblf_shell", shell),
- // Output to stderr
- "rblf_log": starlark.NewBuiltin("rblf_log", log),
- // To convert makefile's $(wildcard foo*)
- "rblf_wildcard": starlark.NewBuiltin("rblf_wildcard", wildcard),
- }
-
- // NOTE(asmundak): OS-specific. Behave similar to Linux `system` call,
- // which always uses /bin/sh to run the command
- shellPath = "/bin/sh"
- if _, err := os.Stat(shellPath); err != nil {
- shellPath = ""
- }
-}
-
// Parses, resolves, and executes a Starlark file.
// filename and src parameters are as for starlark.ExecFile:
// * filename is the name of the file to execute,
// and the name that appears in error messages;
// * src is an optional source of bytes to use instead of filename
// (it can be a string, or a byte array, or an io.Reader instance)
-func Run(filename string, src interface{}) error {
- setup()
+// Returns the top-level starlark variables, the list of starlark files loaded, and an error
+func Run(filename string, src interface{}, mode ExecutionMode) (starlark.StringDict, []string, error) {
+ // NOTE(asmundak): OS-specific. Behave similar to Linux `system` call,
+ // which always uses /bin/sh to run the command
+ shellPath := "/bin/sh"
+ if _, err := os.Stat(shellPath); err != nil {
+ shellPath = ""
+ }
+
mainThread := &starlark.Thread{
Name: "main",
- Print: func(_ *starlark.Thread, msg string) { fmt.Println(msg) },
+ Print: func(_ *starlark.Thread, msg string) {
+ if mode == ExecutionModeRbc {
+ // In rbc mode, rblf_log is used to print to stderr
+ fmt.Println(msg)
+ } else if mode == ExecutionModeMake {
+ fmt.Fprintln(os.Stderr, msg)
+ }
+ },
Load: loader,
}
- absPath, err := filepath.Abs(filename)
- if err == nil {
- mainThread.SetLocal(callerDirKey, filepath.Dir(absPath))
- _, err = starlark.ExecFile(mainThread, absPath, src, builtins)
+ filename, err := filepath.Abs(filename)
+ if err != nil {
+ return nil, nil, err
}
- return err
+ if wd, err := os.Getwd(); err == nil {
+ filename, err = filepath.Rel(wd, filename)
+ if err != nil {
+ return nil, nil, err
+ }
+ if strings.HasPrefix(filename, "../") {
+ return nil, nil, fmt.Errorf("path could not be made relative to workspace root: %s", filename)
+ }
+ } else {
+ return nil, nil, err
+ }
+
+ // Add top-level file to cache for cycle detection purposes
+ moduleCache[filename] = nil
+
+ var results starlark.StringDict
+ mainThread.SetLocal(callerDirKey, filepath.Dir(filename))
+ mainThread.SetLocal(shellKey, shellPath)
+ mainThread.SetLocal(executionModeKey, mode)
+ if mode == ExecutionModeRbc {
+ results, err = starlark.ExecFile(mainThread, filename, src, rbcBuiltins)
+ } else if mode == ExecutionModeMake {
+ results, err = starlark.ExecFile(mainThread, filename, src, makeBuiltins)
+ } else {
+ return results, nil, fmt.Errorf("unknown executionMode %d", mode)
+ }
+ loadedStarlarkFiles := make([]string, 0, len(moduleCache))
+ for file := range moduleCache {
+ loadedStarlarkFiles = append(loadedStarlarkFiles, file)
+ }
+ sort.Strings(loadedStarlarkFiles)
+
+ return results, loadedStarlarkFiles, err
}
diff --git a/tools/rbcrun/host_test.go b/tools/rbcrun/host_test.go
index e109c02..10cac62 100644
--- a/tools/rbcrun/host_test.go
+++ b/tools/rbcrun/host_test.go
@@ -54,7 +54,6 @@
// Common setup for the tests: create thread, change to the test directory
func testSetup(t *testing.T) *starlark.Thread {
- setup()
thread := &starlark.Thread{
Load: func(thread *starlark.Thread, module string) (starlark.StringDict, error) {
if module == "assert.star" {
@@ -78,7 +77,6 @@
// In order to use "assert.star" from go/starlark.net/starlarktest in the tests, provide:
// * load function that handles "assert.star"
// * starlarktest.DataFile function that finds its location
- setup()
if err := os.Chdir(dataDir()); err != nil {
t.Fatal(err)
}
@@ -92,7 +90,9 @@
starlarktest.SetReporter(thread, t)
_, thisSrcFile, _, _ := runtime.Caller(0)
filename := filepath.Join(filepath.Dir(thisSrcFile), starFile)
- if _, err := starlark.ExecFile(thread, filename, nil, builtins); err != nil {
+ thread.SetLocal(executionModeKey, ExecutionModeRbc)
+ thread.SetLocal(shellKey, "/bin/sh")
+ if _, err := starlark.ExecFile(thread, filename, nil, rbcBuiltins); err != nil {
if err, ok := err.(*starlark.EvalError); ok {
t.Fatal(err.Backtrace())
}
@@ -103,7 +103,7 @@
func TestFileOps(t *testing.T) {
// TODO(asmundak): convert this to use exerciseStarlarkTestFile
thread := testSetup(t)
- if _, err := starlark.ExecFile(thread, "file_ops.star", nil, builtins); err != nil {
+ if _, err := starlark.ExecFile(thread, "file_ops.star", nil, rbcBuiltins); err != nil {
if err, ok := err.(*starlark.EvalError); ok {
t.Fatal(err.Backtrace())
}
@@ -122,9 +122,12 @@
}
}
dir := dataDir()
+ if err := os.Chdir(filepath.Dir(dir)); err != nil {
+ t.Fatal(err)
+ }
thread.SetLocal(callerDirKey, dir)
- LoadPathRoot = filepath.Dir(dir)
- if _, err := starlark.ExecFile(thread, "load.star", nil, builtins); err != nil {
+ thread.SetLocal(executionModeKey, ExecutionModeRbc)
+ if _, err := starlark.ExecFile(thread, "testdata/load.star", nil, rbcBuiltins); err != nil {
if err, ok := err.(*starlark.EvalError); ok {
t.Fatal(err.Backtrace())
}
diff --git a/tools/rbcrun/rbcrun/rbcrun.go b/tools/rbcrun/rbcrun/rbcrun.go
index 8dd0f46..b5182f0 100644
--- a/tools/rbcrun/rbcrun/rbcrun.go
+++ b/tools/rbcrun/rbcrun/rbcrun.go
@@ -17,18 +17,22 @@
import (
"flag"
"fmt"
- "go.starlark.net/starlark"
"os"
"rbcrun"
+ "regexp"
+ "strings"
+
+ "go.starlark.net/starlark"
)
var (
+ modeFlag = flag.String("mode", "", "the general behavior of rbcrun. Can be \"rbc\" or \"make\". Required.")
rootdir = flag.String("d", ".", "the value of // for load paths")
perfFile = flag.String("perf", "", "save performance data")
+ identifierRe = regexp.MustCompile("[a-zA-Z_][a-zA-Z0-9_]*")
)
-func main() {
- flag.Parse()
+func getEntrypointStarlarkFile() string {
filename := ""
for _, arg := range flag.Args() {
@@ -42,8 +46,108 @@
flag.Usage()
os.Exit(1)
}
- if stat, err := os.Stat(*rootdir); os.IsNotExist(err) || !stat.IsDir() {
- quit("%s is not a directory\n", *rootdir)
+ return filename
+}
+
+func getMode() rbcrun.ExecutionMode {
+ switch *modeFlag {
+ case "rbc":
+ return rbcrun.ExecutionModeRbc
+ case "make":
+ return rbcrun.ExecutionModeMake
+ case "":
+ quit("-mode flag is required.")
+ default:
+ quit("Unknown -mode value %q, expected 1 of \"rbc\", \"make\"", *modeFlag)
+ }
+ return rbcrun.ExecutionModeMake
+}
+
+var makeStringReplacer = strings.NewReplacer("#", "\\#", "$", "$$")
+
+func cleanStringForMake(s string) (string, error) {
+ if strings.ContainsAny(s, "\\\n") {
+ // \\ in make is literally \\, not a single \, so we can't allow them.
+ // \<newline> in make will produce a space, not a newline.
+ return "", fmt.Errorf("starlark strings exported to make cannot contain backslashes or newlines")
+ }
+ return makeStringReplacer.Replace(s), nil
+}
+
+func getValueInMakeFormat(value starlark.Value, allowLists bool) (string, error) {
+ switch v := value.(type) {
+ case starlark.String:
+ if cleanedValue, err := cleanStringForMake(v.GoString()); err == nil {
+ return cleanedValue, nil
+ } else {
+ return "", err
+ }
+ case starlark.Int:
+ return v.String(), nil
+ case *starlark.List:
+ if !allowLists {
+ return "", fmt.Errorf("nested lists are not allowed to be exported from starlark to make, flatten the list in starlark first")
+ }
+ result := ""
+ for i := 0; i < v.Len(); i++ {
+ value, err := getValueInMakeFormat(v.Index(i), false)
+ if err != nil {
+ return "", err
+ }
+ if i > 0 {
+ result += " "
+ }
+ result += value
+ }
+ return result, nil
+ default:
+ return "", fmt.Errorf("only starlark strings, ints, and lists of strings/ints can be exported to make. Please convert all other types in starlark first. Found type: %s", value.Type())
+ }
+}
+
+func printVarsInMakeFormat(globals starlark.StringDict) error {
+ // We could just directly export top level variables by name instead of going through
+ // a variables_to_export_to_make dictionary, but that wouldn't allow for exporting a
+ // runtime-defined number of variables to make. This can be important because dictionaries
+ // in make are often represented by a unique variable for every key in the dictionary.
+ variablesValue, ok := globals["variables_to_export_to_make"]
+ if !ok {
+ return fmt.Errorf("expected top-level starlark file to have a \"variables_to_export_to_make\" variable")
+ }
+ variables, ok := variablesValue.(*starlark.Dict)
+ if !ok {
+ return fmt.Errorf("expected variables_to_export_to_make to be a dict, got %s", variablesValue.Type())
+ }
+
+ for _, varTuple := range variables.Items() {
+ varNameStarlark, ok := varTuple.Index(0).(starlark.String)
+ if !ok {
+ return fmt.Errorf("all keys in variables_to_export_to_make must be strings, but got %q", varTuple.Index(0).Type())
+ }
+ varName := varNameStarlark.GoString()
+ if !identifierRe.MatchString(varName) {
+ return fmt.Errorf("all variables at the top level starlark file must be valid c identifiers, but got %q", varName)
+ }
+ if varName == "LOADED_STARLARK_FILES" {
+ return fmt.Errorf("the name LOADED_STARLARK_FILES is reserved for use by the starlark interpreter")
+ }
+ valueMake, err := getValueInMakeFormat(varTuple.Index(1), true)
+ if err != nil {
+ return err
+ }
+ // The :=$= is special Kati syntax that means "set and make readonly"
+ fmt.Printf("%s :=$= %s\n", varName, valueMake)
+ }
+ return nil
+}
+
+func main() {
+ flag.Parse()
+ filename := getEntrypointStarlarkFile()
+ mode := getMode()
+
+ if os.Chdir(*rootdir) != nil {
+ quit("could not chdir to %s\n", *rootdir)
}
if *perfFile != "" {
pprof, err := os.Create(*perfFile)
@@ -55,8 +159,7 @@
quit("%s\n", err)
}
}
- rbcrun.LoadPathRoot = *rootdir
- err := rbcrun.Run(filename, nil)
+ variables, loadedStarlarkFiles, err := rbcrun.Run(filename, nil, mode)
rc := 0
if *perfFile != "" {
if err2 := starlark.StopProfile(); err2 != nil {
@@ -71,6 +174,12 @@
quit("%s\n", err)
}
}
+ if mode == rbcrun.ExecutionModeMake {
+ if err := printVarsInMakeFormat(variables); err != nil {
+ quit("%s\n", err)
+ }
+ fmt.Printf("LOADED_STARLARK_FILES := %s\n", strings.Join(loadedStarlarkFiles, " "))
+ }
os.Exit(rc)
}
diff --git a/tools/rbcrun/testdata/module1.star b/tools/rbcrun/testdata/module1.star
index be04f75..02919a0 100644
--- a/tools/rbcrun/testdata/module1.star
+++ b/tools/rbcrun/testdata/module1.star
@@ -2,6 +2,6 @@
load("assert.star", "assert")
# Make sure that builtins are defined for the loaded module, too
-assert.true(rblf_wildcard("module1.star"))
-assert.true(not rblf_wildcard("no_such file"))
+assert.true(rblf_wildcard("testdata/module1.star"))
+assert.true(not rblf_wildcard("testdata/no_such file"))
test = "module1"
diff --git a/tools/releasetools/add_img_to_target_files.py b/tools/releasetools/add_img_to_target_files.py
index ac3271b..8d660f8 100644
--- a/tools/releasetools/add_img_to_target_files.py
+++ b/tools/releasetools/add_img_to_target_files.py
@@ -65,7 +65,7 @@
import ota_metadata_pb2
import rangelib
import sparse_img
-
+from concurrent.futures import ThreadPoolExecutor
from apex_utils import GetApexInfoFromTargetFiles
from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, IsSparseImage, MakeTempFile, ZipWrite
@@ -1083,8 +1083,15 @@
("system_dlkm", has_system_dlkm, AddSystemDlkm, []),
("system_other", has_system_other, AddSystemOther, []),
)
- for call in add_partition_calls:
- add_partition(*call)
+ # If output_zip exists, each add_partition_calls writes bytes to the same output_zip,
+ # which is not thread-safe. So, run them in serial if output_zip exists.
+ if output_zip:
+ for call in add_partition_calls:
+ add_partition(*call)
+ else:
+ with ThreadPoolExecutor(max_workers=len(add_partition_calls)) as executor:
+ for future in [executor.submit(add_partition, *call) for call in add_partition_calls]:
+ future.result()
AddApexInfo(output_zip)
diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py
index 06de622..7adc9fa 100644
--- a/tools/releasetools/common.py
+++ b/tools/releasetools/common.py
@@ -2782,6 +2782,8 @@
def Cleanup():
for i in OPTIONS.tempfiles:
+ if not os.path.exists(i):
+ continue
if os.path.isdir(i):
shutil.rmtree(i, ignore_errors=True)
else:
@@ -4117,6 +4119,17 @@
return fp.read(4) == b'\x3A\xFF\x26\xED'
+def UnsparseImage(filepath, target_path=None):
+ if not IsSparseImage(filepath):
+ return
+ if target_path is None:
+ tmp_img = MakeTempFile(suffix=".img")
+ RunAndCheckOutput(["simg2img", filepath, tmp_img])
+ os.rename(tmp_img, filepath)
+ else:
+ RunAndCheckOutput(["simg2img", filepath, target_path])
+
+
def ParseUpdateEngineConfig(path: str):
"""Parse the update_engine config stored in file `path`
Args
diff --git a/tools/releasetools/img_from_target_files.py b/tools/releasetools/img_from_target_files.py
index f8bdd81..17f4cc5 100755
--- a/tools/releasetools/img_from_target_files.py
+++ b/tools/releasetools/img_from_target_files.py
@@ -118,13 +118,18 @@
entries = [
'OTA/android-info.txt:android-info.txt',
+ 'META/fastboot-info.txt:fastboot-info.txt',
]
with zipfile.ZipFile(input_file) as input_zip:
namelist = input_zip.namelist()
+ if 'PREBUILT_IMAGES/kernel_16k' in namelist:
+ entries.append('PREBUILT_IMAGES/kernel_16k:kernel_16k')
+ if 'PREBUILT_IMAGES/ramdisk_16k.img' in namelist:
+ entries.append('PREBUILT_IMAGES/ramdisk_16k.img:ramdisk_16k.img')
for image_path in [name for name in namelist if name.startswith('IMAGES/')]:
image = os.path.basename(image_path)
- if OPTIONS.bootable_only and image not in('boot.img', 'recovery.img', 'bootloader', 'init_boot.img'):
+ if OPTIONS.bootable_only and image not in ('boot.img', 'recovery.img', 'bootloader', 'init_boot.img'):
continue
if not image.endswith('.img') and image != 'bootloader':
continue
@@ -172,8 +177,8 @@
logger.info('Writing super.img to archive...')
with zipfile.ZipFile(
- output_file, 'a', compression=zipfile.ZIP_DEFLATED,
- allowZip64=True) as output_zip:
+ output_file, 'a', compression=zipfile.ZIP_DEFLATED,
+ allowZip64=True) as output_zip:
common.ZipWrite(output_zip, super_file, 'super.img')
diff --git a/tools/releasetools/merge_ota.py b/tools/releasetools/merge_ota.py
index 7d3d3a3..441312c 100644
--- a/tools/releasetools/merge_ota.py
+++ b/tools/releasetools/merge_ota.py
@@ -14,6 +14,7 @@
import argparse
import logging
+import shlex
import struct
import sys
import update_payload
@@ -34,6 +35,7 @@
logger = logging.getLogger(__name__)
CARE_MAP_ENTRY = "care_map.pb"
+APEX_INFO_ENTRY = "apex_info.pb"
def WriteDataBlob(payload: Payload, outfp: BinaryIO, read_size=1024*64):
@@ -188,6 +190,22 @@
f"OTA {partition_to_ota[part].name} and {payload.name} have duplicating partition {part}")
partition_to_ota[part] = payload
+def ApexInfo(file_paths):
+ if len(file_paths) > 1:
+ logger.info("More than one target file specified, will ignore "
+ "apex_info.pb (if any)")
+ return None
+ with zipfile.ZipFile(file_paths[0], "r", allowZip64=True) as zfp:
+ if APEX_INFO_ENTRY in zfp.namelist():
+ apex_info_bytes = zfp.read(APEX_INFO_ENTRY)
+ return apex_info_bytes
+ return None
+
+def ParseSignerArgs(args):
+ if args is None:
+ return None
+ return shlex.split(args)
+
def main(argv):
parser = argparse.ArgumentParser(description='Merge multiple partial OTAs')
parser.add_argument('packages', type=str, nargs='+',
@@ -196,6 +214,13 @@
help='Paths to private key for signing payload')
parser.add_argument('--search_path', type=str,
help='Search path for framework/signapk.jar')
+ parser.add_argument('--payload_signer', type=str,
+ help='Path to custom payload signer')
+ parser.add_argument('--payload_signer_args', type=ParseSignerArgs,
+ help='Arguments for payload signer if necessary')
+ parser.add_argument('--payload_signer_maximum_signature_size', type=str,
+ help='Maximum signature size (in bytes) that would be '
+ 'generated by the given payload signer')
parser.add_argument('--output', type=str,
help='Paths to output merged ota', required=True)
parser.add_argument('--metadata_ota', type=str,
@@ -203,6 +228,9 @@
parser.add_argument('--private_key_suffix', type=str,
help='Suffix to be appended to package_key path', default=".pk8")
parser.add_argument('-v', action="store_true", help="Enable verbose logging", dest="verbose")
+ parser.epilog = ('This tool can also be used to resign a regular OTA. For a single regular OTA, '
+ 'apex_info.pb will be written to output. When merging multiple OTAs, '
+ 'apex_info.pb will not be written.')
args = parser.parse_args(argv[1:])
file_paths = args.packages
@@ -225,6 +253,13 @@
merged_manifest = MergeManifests(payloads)
+ # Get signing keys
+ key_passwords = common.GetKeyPasswords([args.package_key])
+
+ generator = PayloadGenerator()
+
+ apex_info_bytes = ApexInfo(file_paths)
+
with tempfile.NamedTemporaryFile() as unsigned_payload:
WriteHeaderAndManifest(merged_manifest, unsigned_payload)
ConcatBlobs(payloads, unsigned_payload)
@@ -236,20 +271,31 @@
if args.package_key:
logger.info("Signing payload...")
- signer = PayloadSigner(args.package_key, args.private_key_suffix)
+ # TODO: remove OPTIONS when no longer used as fallback in payload_signer
+ common.OPTIONS.payload_signer_args = None
+ common.OPTIONS.payload_signer_maximum_signature_size = None
+ signer = PayloadSigner(args.package_key, args.private_key_suffix,
+ key_passwords[args.package_key],
+ payload_signer=args.payload_signer,
+ payload_signer_args=args.payload_signer_args,
+ payload_signer_maximum_signature_size=args.payload_signer_maximum_signature_size)
generator.payload_file = unsigned_payload.name
generator.Sign(signer)
logger.info("Payload size: %d", os.path.getsize(generator.payload_file))
logger.info("Writing to %s", args.output)
+
key_passwords = common.GetKeyPasswords([args.package_key])
with tempfile.NamedTemporaryFile(prefix="signed_ota", suffix=".zip") as signed_ota:
with zipfile.ZipFile(signed_ota, "w") as zfp:
generator.WriteToZip(zfp)
care_map_bytes = MergeCareMap(args.packages)
if care_map_bytes:
- zfp.writestr(CARE_MAP_ENTRY, care_map_bytes)
+ common.ZipWriteStr(zfp, CARE_MAP_ENTRY, care_map_bytes)
+ if apex_info_bytes:
+ logger.info("Writing %s", APEX_INFO_ENTRY)
+ common.ZipWriteStr(zfp, APEX_INFO_ENTRY, apex_info_bytes)
AddOtaMetadata(signed_ota.name, metadata_ota,
args.output, args.package_key, key_passwords[args.package_key])
return 0
diff --git a/tools/releasetools/non_ab_ota.py b/tools/releasetools/non_ab_ota.py
index c4fd809..667891c 100644
--- a/tools/releasetools/non_ab_ota.py
+++ b/tools/releasetools/non_ab_ota.py
@@ -23,6 +23,7 @@
from check_target_files_vintf import CheckVintfIfTrebleEnabled, HasPartition
from common import OPTIONS
from ota_utils import UNZIP_PATTERN, FinalizeMetadata, GetPackageMetadata, PropertyFiles
+import subprocess
logger = logging.getLogger(__name__)
@@ -277,7 +278,8 @@
needed_property_files = (
NonAbOtaPropertyFiles(),
)
- FinalizeMetadata(metadata, staging_file, output_file, needed_property_files, package_key=OPTIONS.package_key)
+ FinalizeMetadata(metadata, staging_file, output_file,
+ needed_property_files, package_key=OPTIONS.package_key)
def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
@@ -532,7 +534,8 @@
needed_property_files = (
NonAbOtaPropertyFiles(),
)
- FinalizeMetadata(metadata, staging_file, output_file, needed_property_files, package_key=OPTIONS.package_key)
+ FinalizeMetadata(metadata, staging_file, output_file,
+ needed_property_files, package_key=OPTIONS.package_key)
def GenerateNonAbOtaPackage(target_file, output_file, source_file=None):
@@ -555,8 +558,18 @@
if OPTIONS.extracted_input is not None:
OPTIONS.input_tmp = OPTIONS.extracted_input
else:
- logger.info("unzipping target target-files...")
- OPTIONS.input_tmp = common.UnzipTemp(target_file, UNZIP_PATTERN)
+ if not os.path.isdir(target_file):
+ logger.info("unzipping target target-files...")
+ OPTIONS.input_tmp = common.UnzipTemp(target_file, UNZIP_PATTERN)
+ else:
+ OPTIONS.input_tmp = target_file
+ tmpfile = common.MakeTempFile(suffix=".zip")
+ os.unlink(tmpfile)
+ common.RunAndCheckOutput(
+ ["zip", tmpfile, "-r", ".", "-0"], cwd=target_file)
+ assert zipfile.is_zipfile(tmpfile)
+ target_file = tmpfile
+
OPTIONS.target_tmp = OPTIONS.input_tmp
# If the caller explicitly specified the device-specific extensions path via
diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py
index 1a4a895..afbe81a 100755
--- a/tools/releasetools/ota_from_target_files.py
+++ b/tools/releasetools/ota_from_target_files.py
@@ -273,7 +273,6 @@
PayloadGenerator, SECURITY_PATCH_LEVEL_PROP_NAME, ExtractTargetFiles, CopyTargetFilesDir)
from common import DoesInputFileContain, IsSparseImage
import target_files_diff
-from check_target_files_vintf import CheckVintfIfTrebleEnabled
from non_ab_ota import GenerateNonAbOtaPackage
from payload_signer import PayloadSigner
@@ -526,8 +525,7 @@
def ParseInfoDict(target_file_path):
- with zipfile.ZipFile(target_file_path, 'r', allowZip64=True) as zfp:
- return common.LoadInfoDict(zfp)
+ return common.LoadInfoDict(target_file_path)
def GetTargetFilesZipForCustomVABCCompression(input_file, vabc_compression_param):
@@ -954,6 +952,7 @@
target_info.info_dict['ab_partitions'] = common.ReadFromInputFile(target_file,
AB_PARTITIONS).strip().split("\n")
+ from check_target_files_vintf import CheckVintfIfTrebleEnabled
CheckVintfIfTrebleEnabled(target_file, target_info)
# Metadata to comply with Android OTA package format.
diff --git a/tools/releasetools/ota_utils.py b/tools/releasetools/ota_utils.py
index 9067e78..fa9516e 100644
--- a/tools/releasetools/ota_utils.py
+++ b/tools/releasetools/ota_utils.py
@@ -48,6 +48,7 @@
UNZIP_PATTERN = ['IMAGES/*', 'META/*', 'OTA/*',
'RADIO/*', '*/build.prop', '*/default.prop', '*/build.default', "*/etc/vintf/*"]
SECURITY_PATCH_LEVEL_PROP_NAME = "ro.build.version.security_patch"
+TARGET_FILES_IMAGES_SUBDIR = ["IMAGES", "PREBUILT_IMAGES", "RADIO"]
def FinalizeMetadata(metadata, input_file, output_file, needed_property_files=None, package_key=None, pw=None):
@@ -727,6 +728,15 @@
return path
extracted_dir = common.MakeTempDir("target_files")
common.UnzipToDir(path, extracted_dir, UNZIP_PATTERN + [""])
+ for subdir in TARGET_FILES_IMAGES_SUBDIR:
+ image_dir = os.path.join(extracted_dir, subdir)
+ if not os.path.exists(image_dir):
+ continue
+ for filename in os.listdir(image_dir):
+ if not filename.endswith(".img"):
+ continue
+ common.UnsparseImage(os.path.join(image_dir, filename))
+
return extracted_dir
@@ -1047,12 +1057,18 @@
def CopyTargetFilesDir(input_dir):
output_dir = common.MakeTempDir("target_files")
- IMAGES_DIR = ["IMAGES", "PREBUILT_IMAGES", "RADIO"]
- for subdir in IMAGES_DIR:
+
+ def SymlinkIfNotSparse(src, dst):
+ if common.IsSparseImage(src):
+ return common.UnsparseImage(src, dst)
+ else:
+ return os.link(src, dst)
+
+ for subdir in TARGET_FILES_IMAGES_SUBDIR:
if not os.path.exists(os.path.join(input_dir, subdir)):
continue
shutil.copytree(os.path.join(input_dir, subdir), os.path.join(
- output_dir, subdir), dirs_exist_ok=True, copy_function=os.link)
+ output_dir, subdir), dirs_exist_ok=True, copy_function=SymlinkIfNotSparse)
shutil.copytree(os.path.join(input_dir, "META"), os.path.join(
output_dir, "META"), dirs_exist_ok=True)
diff --git a/tools/releasetools/payload_signer.py b/tools/releasetools/payload_signer.py
index 4f342ac..9933aef 100644
--- a/tools/releasetools/payload_signer.py
+++ b/tools/releasetools/payload_signer.py
@@ -36,11 +36,16 @@
(OPTIONS.package_key) and calls openssl for the signing works.
"""
- def __init__(self, package_key=None, private_key_suffix=None, pw=None, payload_signer=None):
+ def __init__(self, package_key=None, private_key_suffix=None, pw=None, payload_signer=None,
+ payload_signer_args=None, payload_signer_maximum_signature_size=None):
if package_key is None:
package_key = OPTIONS.package_key
if private_key_suffix is None:
private_key_suffix = OPTIONS.private_key_suffix
+ if payload_signer_args is None:
+ payload_signer_args = OPTIONS.payload_signer_args
+ if payload_signer_maximum_signature_size is None:
+ payload_signer_maximum_signature_size = OPTIONS.payload_signer_maximum_signature_size
if payload_signer is None:
# Prepare the payload signing key.
@@ -59,10 +64,10 @@
signing_key)
else:
self.signer = payload_signer
- self.signer_args = OPTIONS.payload_signer_args
- if OPTIONS.payload_signer_maximum_signature_size:
+ self.signer_args = payload_signer_args
+ if payload_signer_maximum_signature_size:
self.maximum_signature_size = int(
- OPTIONS.payload_signer_maximum_signature_size)
+ payload_signer_maximum_signature_size)
else:
# The legacy config uses RSA2048 keys.
logger.warning("The maximum signature size for payload signer is not"
diff --git a/tools/sbom/generate-sbom.py b/tools/sbom/generate-sbom.py
index 56509c9..2415f7e 100755
--- a/tools/sbom/generate-sbom.py
+++ b/tools/sbom/generate-sbom.py
@@ -19,7 +19,6 @@
Usage example:
generate-sbom.py --output_file out/target/product/vsoc_x86_64/sbom.spdx \
--metadata out/target/product/vsoc_x86_64/sbom-metadata.csv \
- --product_out_dir=out/target/product/vsoc_x86_64 \
--build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \
--product_mfr=Google
"""
@@ -89,11 +88,11 @@
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.')
parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.')
parser.add_argument('--metadata', required=True, help='The SBOM metadata file path.')
- parser.add_argument('--product_out_dir', required=True, help='The parent directory of all the installed files.')
parser.add_argument('--build_version', required=True, help='The build version.')
parser.add_argument('--product_mfr', required=True, help='The product manufacturer.')
parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format')
- parser.add_argument('--unbundled', action='store_true', default=False, help='Generate SBOM file for unbundled module')
+ parser.add_argument('--unbundled_apk', action='store_true', default=False, help='Generate SBOM for unbundled APKs')
+ parser.add_argument('--unbundled_apex', action='store_true', default=False, help='Generate SBOM for unbundled APEXs')
return parser.parse_args()
@@ -127,7 +126,6 @@
def checksum(file_path):
- file_path = args.product_out_dir + '/' + file_path
h = hashlib.sha1()
if os.path.islink(file_path):
h.update(os.readlink(file_path).encode('utf-8'))
@@ -265,8 +263,8 @@
def get_sbom_fragments(installed_file_metadata, metadata_file_path):
"""Return SPDX fragment of source/prebuilt packages, which usually contains a SOURCE/PREBUILT
- package, a UPSTREAM package if it's a source package and a external SBOM document reference if
- it's a prebuilt package with sbom_ref defined in its METADATA file.
+ package, a UPSTREAM package and an external SBOM document reference if sbom_ref defined in its
+ METADATA file.
See go/android-spdx and go/android-sbom-gen for more details.
"""
@@ -303,25 +301,33 @@
prebuilt_package = sbom_data.Package(id=prebuilt_package_id,
name=name,
download_location=sbom_data.VALUE_NONE,
- version=args.build_version,
+ version=version if version else args.build_version,
supplier='Organization: ' + args.product_mfr)
- packages.append(prebuilt_package)
- if metadata_file_path:
- metadata_proto = metadata_file_protos[metadata_file_path]
- if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref':
- sbom_url = metadata_proto.third_party.sbom_ref.url
- sbom_checksum = metadata_proto.third_party.sbom_ref.checksum
- upstream_element_id = metadata_proto.third_party.sbom_ref.element_id
- if sbom_url and sbom_checksum and upstream_element_id:
- doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{encode_for_spdxid(name)}'
- external_doc_ref = sbom_data.DocumentExternalReference(id=doc_ref_id,
- uri=sbom_url,
- checksum=sbom_checksum)
- relationships.append(
- sbom_data.Relationship(id1=prebuilt_package_id,
- relationship=sbom_data.RelationshipType.VARIANT_OF,
- id2=doc_ref_id + ':' + upstream_element_id))
+ upstream_package_id = new_package_id(name, PKG_UPSTREAM)
+ upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version = version,
+ supplier=('Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION,
+ download_location=download_location)
+ packages += [prebuilt_package, upstream_package]
+ relationships.append(sbom_data.Relationship(id1=prebuilt_package_id,
+ relationship=sbom_data.RelationshipType.VARIANT_OF,
+ id2=upstream_package_id))
+
+ if metadata_file_path:
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref':
+ sbom_url = metadata_proto.third_party.sbom_ref.url
+ sbom_checksum = metadata_proto.third_party.sbom_ref.checksum
+ upstream_element_id = metadata_proto.third_party.sbom_ref.element_id
+ if sbom_url and sbom_checksum and upstream_element_id:
+ doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{encode_for_spdxid(name)}'
+ external_doc_ref = sbom_data.DocumentExternalReference(id=doc_ref_id,
+ uri=sbom_url,
+ checksum=sbom_checksum)
+ relationships.append(
+ sbom_data.Relationship(id1=upstream_package_id,
+ relationship=sbom_data.RelationshipType.VARIANT_OF,
+ id2=doc_ref_id + ':' + upstream_element_id))
return external_doc_ref, packages, relationships
@@ -334,9 +340,8 @@
return h.hexdigest()
-def save_report(report):
- prefix, _ = os.path.splitext(args.output_file)
- with open(prefix + '-gen-report.txt', 'w', encoding='utf-8') as report_file:
+def save_report(report_file_path, report):
+ with open(report_file_path, 'w', encoding='utf-8') as report_file:
for type, issues in report.items():
report_file.write(type + '\n')
for issue in issues:
@@ -394,7 +399,7 @@
installed_file_metadata['installed_file'], installed_file_metadata['module_path']))
-def generate_sbom_for_unbundled():
+def generate_sbom_for_unbundled_apk():
with open(args.metadata, newline='') as sbom_metadata_file:
reader = csv.DictReader(sbom_metadata_file)
doc = sbom_data.Document(name=args.build_version,
@@ -402,7 +407,7 @@
creators=['Organization: ' + args.product_mfr])
for installed_file_metadata in reader:
installed_file = installed_file_metadata['installed_file']
- if args.output_file != args.product_out_dir + installed_file + '.spdx.json':
+ if args.output_file != installed_file_metadata['build_output_path'] + '.spdx.json':
continue
module_path = installed_file_metadata['module_path']
@@ -412,7 +417,9 @@
version=args.build_version,
supplier='Organization: ' + args.product_mfr)
file_id = new_file_id(installed_file)
- file = sbom_data.File(id=file_id, name=installed_file, checksum=checksum(installed_file))
+ file = sbom_data.File(id=file_id,
+ name=installed_file,
+ checksum=checksum(installed_file_metadata['build_output_path']))
relationship = sbom_data.Relationship(id1=file_id,
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=package_id)
@@ -435,24 +442,25 @@
args = get_args()
log('Args:', vars(args))
- if args.unbundled:
- generate_sbom_for_unbundled()
+ if args.unbundled_apk:
+ generate_sbom_for_unbundled_apk()
return
global metadata_file_protos
metadata_file_protos = {}
- doc = sbom_data.Document(name=args.build_version,
- namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}',
- creators=['Organization: ' + args.product_mfr])
-
product_package = sbom_data.Package(id=sbom_data.SPDXID_PRODUCT,
name=sbom_data.PACKAGE_NAME_PRODUCT,
download_location=sbom_data.VALUE_NONE,
version=args.build_version,
supplier='Organization: ' + args.product_mfr,
files_analyzed=True)
- doc.packages.append(product_package)
+
+ doc = sbom_data.Document(name=args.build_version,
+ namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}',
+ creators=['Organization: ' + args.product_mfr])
+ if not args.unbundled_apex:
+ doc.packages.append(product_package)
doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,
name=sbom_data.PACKAGE_NAME_PLATFORM,
@@ -478,18 +486,21 @@
module_path = installed_file_metadata['module_path']
product_copy_files = installed_file_metadata['product_copy_files']
kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
+ build_output_path = installed_file_metadata['build_output_path']
if not installed_file_has_metadata(installed_file_metadata, report):
continue
- file_path = args.product_out_dir + '/' + installed_file
- if not (os.path.islink(file_path) or os.path.isfile(file_path)):
+ if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)
continue
file_id = new_file_id(installed_file)
doc.files.append(
- sbom_data.File(id=file_id, name=installed_file, checksum=checksum(installed_file)))
- product_package.file_ids.append(file_id)
+ sbom_data.File(id=file_id, name=installed_file, checksum=checksum(build_output_path)))
+ if not args.unbundled_apex:
+ product_package.file_ids.append(file_id)
+ elif len(doc.files) > 1:
+ doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id))
if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):
metadata_file_path = get_metadata_file_path(installed_file_metadata)
@@ -533,16 +544,31 @@
relationship=sbom_data.RelationshipType.GENERATED_FROM,
id2=sbom_data.SPDXID_PLATFORM))
- product_package.verification_code = generate_package_verification_code(doc.files)
+ if not args.unbundled_apex:
+ product_package.verification_code = generate_package_verification_code(doc.files)
+
+ if args.unbundled_apex:
+ doc.describes = doc.files[0].id
# Save SBOM records to output file
doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
- with open(args.output_file, 'w', encoding="utf-8") as file:
- sbom_writers.TagValueWriter.write(doc, file)
+ prefix = args.output_file
+ if prefix.endswith('.spdx'):
+ prefix = prefix.removesuffix('.spdx')
+ elif prefix.endswith('.spdx.json'):
+ prefix = prefix.removesuffix('.spdx.json')
+
+ output_file = prefix + '.spdx'
+ if args.unbundled_apex:
+ output_file = prefix + '-fragment.spdx'
+ with open(output_file, 'w', encoding="utf-8") as file:
+ sbom_writers.TagValueWriter.write(doc, file, fragment=args.unbundled_apex)
if args.json:
- with open(args.output_file+'.json', 'w', encoding="utf-8") as file:
+ with open(prefix + '.spdx.json', 'w', encoding="utf-8") as file:
sbom_writers.JSONWriter.write(doc, file)
+ save_report(prefix + '-gen-report.txt', report)
+
if __name__ == '__main__':
main()
diff --git a/tools/sbom/sbom_data.py b/tools/sbom/sbom_data.py
index d2ef48d..14c4eb2 100644
--- a/tools/sbom/sbom_data.py
+++ b/tools/sbom/sbom_data.py
@@ -80,6 +80,7 @@
DESCRIBES = 'DESCRIBES'
VARIANT_OF = 'VARIANT_OF'
GENERATED_FROM = 'GENERATED_FROM'
+ CONTAINS = 'CONTAINS'
@dataclass
diff --git a/tools/sbom/sbom_writers.py b/tools/sbom/sbom_writers.py
index b1c66c5..85dee9d 100644
--- a/tools/sbom/sbom_writers.py
+++ b/tools/sbom/sbom_writers.py
@@ -110,24 +110,26 @@
return tagvalues
@staticmethod
- def marshal_described_element(sbom_doc):
+ def marshal_described_element(sbom_doc, fragment):
if not sbom_doc.describes:
return None
product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes]
if product_package:
tagvalues = TagValueWriter.marshal_package(product_package[0])
- tagvalues.append(
- f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
+ if not fragment:
+ tagvalues.append(
+ f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
tagvalues.append('')
return tagvalues
file = [f for f in sbom_doc.files if f.id == sbom_doc.describes]
if file:
- tagvalues = [
- f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}'
- ]
+ tagvalues = TagValueWriter.marshal_file(file[0])
+ if not fragment:
+ tagvalues.append(
+ f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
return tagvalues
@@ -180,6 +182,8 @@
def marshal_files(sbom_doc):
tagvalues = []
for file in sbom_doc.files:
+ if file.id == sbom_doc.describes:
+ continue
tagvalues += TagValueWriter.marshal_file(file)
return tagvalues
@@ -204,9 +208,9 @@
content = []
if not fragment:
content += TagValueWriter.marshal_doc_headers(sbom_doc)
- described_element = TagValueWriter.marshal_described_element(sbom_doc)
- if described_element:
- content += described_element
+ described_element = TagValueWriter.marshal_described_element(sbom_doc, fragment)
+ if described_element:
+ content += described_element
content += TagValueWriter.marshal_files(sbom_doc)
tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc)
content += tagvalues