Merge "Revert "Switch NDK libc++ from prebuilts/ndk to prebuilts/clang"" into main
diff --git a/core/Makefile b/core/Makefile
index b3e84c9..5ec5a94 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -3582,8 +3582,19 @@
ifeq ($(BOARD_AVB_ENABLE),true)
$(BUILT_SYSTEMIMAGE): $(BOARD_AVB_SYSTEM_KEY_PATH)
endif
+
+ifeq ($(USE_SOONG_DEFINED_SYSTEM_IMAGE),true)
+ifeq ($(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE),)
+$(error PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE must be set if USE_SOONG_DEFINED_SYSTEM_IMAGE is true)
+endif
+soong_defined_system_image := $(call intermediates-dir-for,ETC,$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE))/$(PRODUCT_SOONG_DEFINED_SYSTEM_IMAGE)
+$(BUILT_SYSTEMIMAGE): $(INSTALLED_FILES_FILE) $(systemimage_intermediates)/file_list.txt $(soong_defined_system_image)
+$(eval $(call copy-one-file, $(soong_defined_system_image), $(BUILT_SYSTEMIMAGE)))
+soong_defined_system_image :=
+else
$(BUILT_SYSTEMIMAGE): $(FULL_SYSTEMIMAGE_DEPS) $(INSTALLED_FILES_FILE) $(systemimage_intermediates)/file_list.txt
$(call build-systemimage-target,$@)
+endif
$(call declare-1p-container,$(BUILT_SYSTEMIMAGE),system/extras)
$(call declare-container-license-deps,$(BUILT_SYSTEMIMAGE),$(FULL_SYSTEMIMAGE_DEPS),$(PRODUCT_OUT)/:/)
@@ -7873,6 +7884,8 @@
droidcore-unbundled: $(PACKED_IMAGE_ARCHIVE_TARGET)
+$(call dist-for-goals,dist_files,$(PACKED_IMAGE_ARCHIVE_TARGET))
+
endif # PACK_DESKTOP_FILESYSTEM_IMAGES
# -----------------------------------------------------------------
diff --git a/core/android_soong_config_vars.mk b/core/android_soong_config_vars.mk
index f2ff286..461b284 100644
--- a/core/android_soong_config_vars.mk
+++ b/core/android_soong_config_vars.mk
@@ -28,6 +28,7 @@
$(call add_soong_config_var,ANDROID,BOARD_USES_ODMIMAGE)
$(call soong_config_set_bool,ANDROID,BOARD_USES_RECOVERY_AS_BOOT,$(BOARD_USES_RECOVERY_AS_BOOT))
+$(call soong_config_set_bool,ANDROID,BOARD_MOVE_GSI_AVB_KEYS_TO_VENDOR_BOOT,$(BOARD_MOVE_GSI_AVB_KEYS_TO_VENDOR_BOOT))
$(call add_soong_config_var,ANDROID,CHECK_DEV_TYPE_VIOLATIONS)
$(call add_soong_config_var,ANDROID,PLATFORM_SEPOLICY_VERSION)
$(call add_soong_config_var,ANDROID,PLATFORM_SEPOLICY_COMPAT_VERSIONS)
@@ -36,9 +37,15 @@
$(call add_soong_config_var,ANDROID,TARGET_ENABLE_MEDIADRM_64)
$(call add_soong_config_var,ANDROID,TARGET_DYNAMIC_64_32_MEDIASERVER)
+# For Sanitizers
+$(call soong_config_set_bool,ANDROID,ASAN_ENABLED,$(if $(filter address,$(SANITIZE_TARGET)),true,false))
+$(call soong_config_set_bool,ANDROID,SANITIZE_TARGET_SYSTEM_ENABLED,$(if $(filter true,$(SANITIZE_TARGET_SYSTEM)),true,false))
+
# PRODUCT_PRECOMPILED_SEPOLICY defaults to true. Explicitly check if it's "false" or not.
$(call soong_config_set_bool,ANDROID,PRODUCT_PRECOMPILED_SEPOLICY,$(if $(filter false,$(PRODUCT_PRECOMPILED_SEPOLICY)),false,true))
+# For art modules
+$(call soong_config_set_bool,art_module,host_prefer_32_bit,$(if $(filter true,$(HOST_PREFER_32_BIT)),true,false))
ifdef ART_DEBUG_OPT_FLAG
$(call soong_config_set,art_module,art_debug_opt_flag,$(ART_DEBUG_OPT_FLAG))
endif
@@ -90,6 +97,7 @@
$(call add_soong_config_var_value,ANDROID,release_avf_allow_preinstalled_apps,$(RELEASE_AVF_ALLOW_PREINSTALLED_APPS))
$(call add_soong_config_var_value,ANDROID,release_avf_enable_device_assignment,$(RELEASE_AVF_ENABLE_DEVICE_ASSIGNMENT))
$(call add_soong_config_var_value,ANDROID,release_avf_enable_dice_changes,$(RELEASE_AVF_ENABLE_DICE_CHANGES))
+$(call add_soong_config_var_value,ANDROID,release_avf_enable_early_vm,$(RELEASE_AVF_ENABLE_EARLY_VM))
$(call add_soong_config_var_value,ANDROID,release_avf_enable_llpvm_changes,$(RELEASE_AVF_ENABLE_LLPVM_CHANGES))
$(call add_soong_config_var_value,ANDROID,release_avf_enable_multi_tenant_microdroid_vm,$(RELEASE_AVF_ENABLE_MULTI_TENANT_MICRODROID_VM))
$(call add_soong_config_var_value,ANDROID,release_avf_enable_network,$(RELEASE_AVF_ENABLE_NETWORK))
@@ -162,3 +170,9 @@
# Enable Profiling module. Also used by platform_bootclasspath.
$(call soong_config_set,ANDROID,release_package_profiling_module,$(RELEASE_PACKAGE_PROFILING_MODULE))
$(call soong_config_set,bootclasspath,release_package_profiling_module,$(RELEASE_PACKAGE_PROFILING_MODULE))
+
+# Add perf-setup build flag to soong
+# Note: BOARD_PERFSETUP_SCRIPT location must be under platform_testing/scripts/perf-setup/.
+ifdef BOARD_PERFSETUP_SCRIPT
+ $(call soong_config_set,perf,board_perfsetup_script,$(notdir $(BOARD_PERFSETUP_SCRIPT)))
+endif
diff --git a/core/base_rules.mk b/core/base_rules.mk
index a9d6494..ca553f6 100644
--- a/core/base_rules.mk
+++ b/core/base_rules.mk
@@ -728,6 +728,14 @@
ifneq (true,$(LOCAL_UNINSTALLABLE_MODULE))
+ifeq ($(EXCLUDE_MCTS),true)
+ ifneq (,$(test_config))
+ ifneq (,$(filter mcts-%,$(LOCAL_COMPATIBILITY_SUITE)))
+ LOCAL_COMPATIBILITY_SUITE := $(filter-out cts,$(LOCAL_COMPATIBILITY_SUITE))
+ endif
+ endif
+endif
+
# If we are building a native test or benchmark and its stem variants are not defined,
# separate the multiple architectures into subdirectories of the testcase folder.
arch_dir :=
diff --git a/core/ravenwood_test_config_template.xml b/core/ravenwood_test_config_template.xml
index 16a22c0..088a55a 100644
--- a/core/ravenwood_test_config_template.xml
+++ b/core/ravenwood_test_config_template.xml
@@ -18,7 +18,7 @@
<option name="test-suite-tag" value="ravenwood" />
<option name="test-suite-tag" value="ravenwood-tests" />
- <option name="java-folder" value="prebuilts/jdk/jdk17/linux-x86/" />
+ <option name="java-folder" value="prebuilts/jdk/jdk21/linux-x86/" />
<option name="use-ravenwood-resources" value="true" />
<option name="exclude-paths" value="java" />
<option name="socket-timeout" value="10000" />
diff --git a/core/release_config.mk b/core/release_config.mk
index 2898868..fe2170e 100644
--- a/core/release_config.mk
+++ b/core/release_config.mk
@@ -131,6 +131,9 @@
_args += --guard=false
endif
_args += --allow-missing=true
+ ifneq (,$(TARGET_PRODUCT))
+ _args += --product $(TARGET_PRODUCT)
+ endif
_flags_dir:=$(OUT_DIR)/soong/release-config
_flags_file:=$(_flags_dir)/release_config-$(TARGET_PRODUCT)-$(TARGET_RELEASE).vars
# release-config generates $(_flags_varmk)
diff --git a/core/robolectric_test_config_template.xml b/core/robolectric_test_config_template.xml
index 56d2312..b1d0c2f 100644
--- a/core/robolectric_test_config_template.xml
+++ b/core/robolectric_test_config_template.xml
@@ -18,7 +18,7 @@
<option name="test-suite-tag" value="robolectric" />
<option name="test-suite-tag" value="robolectric-tests" />
- <option name="java-folder" value="prebuilts/jdk/jdk17/linux-x86/" />
+ <option name="java-folder" value="prebuilts/jdk/jdk21/linux-x86/" />
<option name="exclude-paths" value="java" />
<option name="use-robolectric-resources" value="true" />
diff --git a/core/rust_device_test_config_template.xml b/core/rust_device_test_config_template.xml
index bfd2f47..aacabcb 100644
--- a/core/rust_device_test_config_template.xml
+++ b/core/rust_device_test_config_template.xml
@@ -20,11 +20,11 @@
<target_preparer class="com.android.tradefed.targetprep.PushFilePreparer">
<option name="cleanup" value="true" />
- <option name="push" value="{MODULE}->/data/local/tmp/{MODULE}" />
+ <option name="push" value="{MODULE}->{TEST_INSTALL_BASE}/{MODULE}" />
</target_preparer>
<test class="com.android.tradefed.testtype.rust.RustBinaryTest" >
- <option name="test-device-path" value="/data/local/tmp" />
+ <option name="test-device-path" value="{TEST_INSTALL_BASE}" />
<option name="module-name" value="{MODULE}" />
</test>
</configuration>
diff --git a/core/soong_config.mk b/core/soong_config.mk
index 12b4135..72ee2e4 100644
--- a/core/soong_config.mk
+++ b/core/soong_config.mk
@@ -152,7 +152,6 @@
$(call add_json_str, VendorApiLevel, $(BOARD_API_LEVEL))
$(call add_json_list, ExtraVndkVersions, $(PRODUCT_EXTRA_VNDK_VERSIONS))
$(call add_json_list, DeviceSystemSdkVersions, $(BOARD_SYSTEMSDK_VERSIONS))
-$(call add_json_str, RecoverySnapshotVersion, $(RECOVERY_SNAPSHOT_VERSION))
$(call add_json_list, Platform_systemsdk_versions, $(PLATFORM_SYSTEMSDK_VERSIONS))
$(call add_json_bool, Malloc_low_memory, $(findstring true,$(MALLOC_SVELTE) $(MALLOC_LOW_MEMORY)))
$(call add_json_bool, Malloc_zero_contents, $(call invert_bool,$(filter false,$(MALLOC_ZERO_CONTENTS))))
@@ -167,8 +166,6 @@
$(call add_json_list, BootJars, $(PRODUCT_BOOT_JARS))
$(call add_json_list, ApexBootJars, $(filter-out $(APEX_BOOT_JARS_EXCLUDED), $(PRODUCT_APEX_BOOT_JARS)))
-$(call add_json_bool, VndkSnapshotBuildArtifacts, $(VNDK_SNAPSHOT_BUILD_ARTIFACTS))
-
$(call add_json_map, BuildFlags)
$(foreach flag,$(_ALL_RELEASE_FLAGS),\
$(call add_json_str,$(flag),$(_ALL_RELEASE_FLAGS.$(flag).VALUE)))
@@ -178,24 +175,6 @@
$(call add_json_str,$(flag),$(_ALL_RELEASE_FLAGS.$(flag).TYPE)))
$(call end_json_map)
-$(call add_json_bool, DirectedVendorSnapshot, $(DIRECTED_VENDOR_SNAPSHOT))
-$(call add_json_map, VendorSnapshotModules)
-$(foreach module,$(VENDOR_SNAPSHOT_MODULES),\
- $(call add_json_bool,$(module),true))
-$(call end_json_map)
-
-$(call add_json_bool, DirectedRecoverySnapshot, $(DIRECTED_RECOVERY_SNAPSHOT))
-$(call add_json_map, RecoverySnapshotModules)
-$(foreach module,$(RECOVERY_SNAPSHOT_MODULES),\
- $(call add_json_bool,$(module),true))
-$(call end_json_map)
-
-$(call add_json_list, VendorSnapshotDirsIncluded, $(VENDOR_SNAPSHOT_DIRS_INCLUDED))
-$(call add_json_list, VendorSnapshotDirsExcluded, $(VENDOR_SNAPSHOT_DIRS_EXCLUDED))
-$(call add_json_list, RecoverySnapshotDirsIncluded, $(RECOVERY_SNAPSHOT_DIRS_INCLUDED))
-$(call add_json_list, RecoverySnapshotDirsExcluded, $(RECOVERY_SNAPSHOT_DIRS_EXCLUDED))
-$(call add_json_bool, HostFakeSnapshotEnabled, $(HOST_FAKE_SNAPSHOT_ENABLE))
-
$(call add_json_bool, MultitreeUpdateMeta, $(filter true,$(TARGET_MULTITREE_UPDATE_META)))
$(call add_json_bool, Treble_linker_namespaces, $(filter true,$(PRODUCT_TREBLE_LINKER_NAMESPACES)))
@@ -361,6 +340,12 @@
$(call add_json_list, OemProperties, $(PRODUCT_OEM_PROPERTIES))
+# Do not set ArtTargetIncludeDebugBuild into any value if PRODUCT_ART_TARGET_INCLUDE_DEBUG_BUILD is not set,
+# to have the same behavior from runtime_libart.mk.
+ifneq ($(PRODUCT_ART_TARGET_INCLUDE_DEBUG_BUILD),)
+$(call add_json_bool, ArtTargetIncludeDebugBuild, $(PRODUCT_ART_TARGET_INCLUDE_DEBUG_BUILD))
+endif
+
$(call json_end)
$(file >$(SOONG_VARIABLES).tmp,$(json_contents))
diff --git a/core/tasks/recovery_snapshot.mk b/core/tasks/recovery_snapshot.mk
deleted file mode 100644
index 525273b..0000000
--- a/core/tasks/recovery_snapshot.mk
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright (C) 2020 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-current_makefile := $(lastword $(MAKEFILE_LIST))
-
-# RECOVERY_SNAPSHOT_VERSION must be set to 'current' in order to generate a recovery snapshot.
-ifeq ($(RECOVERY_SNAPSHOT_VERSION),current)
-
-.PHONY: recovery-snapshot
-recovery-snapshot: $(SOONG_RECOVERY_SNAPSHOT_ZIP)
-
-$(call dist-for-goals, recovery-snapshot, $(SOONG_RECOVERY_SNAPSHOT_ZIP))
-
-else # RECOVERY_SNAPSHOT_VERSION is NOT set to 'current'
-
-.PHONY: recovery-snapshot
-recovery-snapshot: PRIVATE_MAKEFILE := $(current_makefile)
-recovery-snapshot:
- $(call echo-error,$(PRIVATE_MAKEFILE),\
- "CANNOT generate Recovery snapshot. RECOVERY_SNAPSHOT_VERSION must be set to 'current'.")
- exit 1
-
-endif # RECOVERY_SNAPSHOT_VERSION
diff --git a/core/tasks/vendor_snapshot.mk b/core/tasks/vendor_snapshot.mk
deleted file mode 100644
index 83c1379..0000000
--- a/core/tasks/vendor_snapshot.mk
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright (C) 2020 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-current_makefile := $(lastword $(MAKEFILE_LIST))
-
-# BOARD_VNDK_VERSION must be set to 'current' in order to generate a vendor snapshot.
-ifeq ($(BOARD_VNDK_VERSION),current)
-
-.PHONY: vendor-snapshot
-vendor-snapshot: $(SOONG_VENDOR_SNAPSHOT_ZIP)
-
-$(call dist-for-goals, vendor-snapshot, $(SOONG_VENDOR_SNAPSHOT_ZIP))
-
-.PHONY: vendor-fake-snapshot
-vendor-fake-snapshot: $(SOONG_VENDOR_FAKE_SNAPSHOT_ZIP)
-
-$(call dist-for-goals, vendor-fake-snapshot, $(SOONG_VENDOR_FAKE_SNAPSHOT_ZIP):fake/$(notdir $(SOONG_VENDOR_FAKE_SNAPSHOT_ZIP)))
-
-else # BOARD_VNDK_VERSION is NOT set to 'current'
-
-.PHONY: vendor-snapshot
-vendor-snapshot: PRIVATE_MAKEFILE := $(current_makefile)
-vendor-snapshot:
- $(call echo-error,$(PRIVATE_MAKEFILE),\
- "CANNOT generate Vendor snapshot. BOARD_VNDK_VERSION must be set to 'current'.")
- exit 1
-
-.PHONY: vendor-fake-snapshot
-vendor-fake-snapshot: PRIVATE_MAKEFILE := $(current_makefile)
-vendor-fake-snapshot:
- $(call echo-error,$(PRIVATE_MAKEFILE),\
- "CANNOT generate Vendor snapshot. BOARD_VNDK_VERSION must be set to 'current'.")
- exit 1
-
-endif # BOARD_VNDK_VERSION
diff --git a/target/board/Android.mk b/target/board/Android.mk
index decc345..8133af9 100644
--- a/target/board/Android.mk
+++ b/target/board/Android.mk
@@ -67,7 +67,6 @@
$(GEN): $(DEVICE_MANIFEST_FILE) $(HOST_OUT_EXECUTABLES)/assemble_vintf
BOARD_SEPOLICY_VERS=$(BOARD_SEPOLICY_VERS) \
PRODUCT_ENFORCE_VINTF_MANIFEST=$(PRODUCT_ENFORCE_VINTF_MANIFEST) \
- PRODUCT_SHIPPING_API_LEVEL=$(PRODUCT_SHIPPING_API_LEVEL) \
$(HOST_OUT_EXECUTABLES)/assemble_vintf -o $@ \
-i $(call normalize-path-list,$(PRIVATE_DEVICE_MANIFEST_FILE))
@@ -99,7 +98,6 @@
$$(GEN): $$(my_fragment_files) $$(HOST_OUT_EXECUTABLES)/assemble_vintf
BOARD_SEPOLICY_VERS=$$(BOARD_SEPOLICY_VERS) \
PRODUCT_ENFORCE_VINTF_MANIFEST=$$(PRODUCT_ENFORCE_VINTF_MANIFEST) \
- PRODUCT_SHIPPING_API_LEVEL=$$(PRODUCT_SHIPPING_API_LEVEL) \
$$(HOST_OUT_EXECUTABLES)/assemble_vintf -o $$@ \
-i $$(call normalize-path-list,$$(PRIVATE_SRC_FILES))
diff --git a/target/product/base_system.mk b/target/product/base_system.mk
index 795d3f8..2b726c1 100644
--- a/target/product/base_system.mk
+++ b/target/product/base_system.mk
@@ -403,7 +403,6 @@
BugReport \
adb \
adevice \
- art-tools \
atest \
bcc \
bit \
@@ -434,6 +433,21 @@
tz_version_host \
tz_version_host_tzdata_apex \
+# For art-tools, if the dependencies have changed, please sync them to art/Android.bp as well.
+PRODUCT_HOST_PACKAGES += \
+ ahat \
+ dexdump \
+ hprof-conv
+# A subset of the tools are disabled when HOST_PREFER_32_BIT is defined as make reports that
+# they are not supported on host (b/129323791). This is likely due to art_apex disabling host
+# APEX builds when HOST_PREFER_32_BIT is set (b/120617876).
+ifneq ($(HOST_PREFER_32_BIT),true)
+PRODUCT_HOST_PACKAGES += \
+ dexlist \
+ oatdump
+endif
+
+
PRODUCT_PACKAGES += init.usb.rc init.usb.configfs.rc
PRODUCT_PACKAGES += etc_hosts
@@ -444,6 +458,11 @@
PRODUCT_SYSTEM_PROPERTIES += debug.atrace.tags.enableflags=0
PRODUCT_SYSTEM_PROPERTIES += persist.traced.enable=1
+# Include kernel configs.
+PRODUCT_PACKAGES += \
+ approved-ogki-builds.xml \
+ kernel-lifetimes.xml
+
# Packages included only for eng or userdebug builds, previously debug tagged
PRODUCT_PACKAGES_DEBUG := \
adb_keys \
diff --git a/target/product/base_vendor.mk b/target/product/base_vendor.mk
index 1854f97..52e2583 100644
--- a/target/product/base_vendor.mk
+++ b/target/product/base_vendor.mk
@@ -105,3 +105,9 @@
PRODUCT_PACKAGES += \
adb_debug.prop \
userdebug_plat_sepolicy.cil
+
+# On eng or userdebug builds, build in perf-setup-sh by default.
+ifneq (,$(filter userdebug eng,$(TARGET_BUILD_VARIANT)))
+PRODUCT_PACKAGES += \
+ perf-setup-sh
+endif
diff --git a/target/product/build_variables.mk b/target/product/build_variables.mk
index 5fe5333..9fc9ff9 100644
--- a/target/product/build_variables.mk
+++ b/target/product/build_variables.mk
@@ -17,5 +17,8 @@
# This file contains the trunk-stable flags that should be exported to all
# Android targets.
+# Control libbinder client caching
+$(call soong_config_set, libbinder, release_libbinder_client_cache, $(RELEASE_LIBBINDER_CLIENT_CACHE))
+
# Use the configured release of sqlite
$(call soong_config_set, libsqlite3, release_package_libsqlite3, $(RELEASE_PACKAGE_LIBSQLITE3))
diff --git a/target/product/go_defaults_common.mk b/target/product/go_defaults_common.mk
index 5218f29..fd4047a 100644
--- a/target/product/go_defaults_common.mk
+++ b/target/product/go_defaults_common.mk
@@ -37,9 +37,9 @@
# leave less information available via JDWP.
PRODUCT_MINIMIZE_JAVA_DEBUG_INFO := true
-# Disable Scudo outside of eng builds to save RAM.
+# Use the low memory allocator outside of eng builds to save RSS.
ifneq (,$(filter eng, $(TARGET_BUILD_VARIANT)))
- PRODUCT_DISABLE_SCUDO := true
+ MALLOC_LOW_MEMORY := true
endif
# Add the system properties.
diff --git a/target/product/runtime_libart.mk b/target/product/runtime_libart.mk
index dc78368..58234a8 100644
--- a/target/product/runtime_libart.mk
+++ b/target/product/runtime_libart.mk
@@ -178,3 +178,6 @@
PRODUCT_SYSTEM_PROPERTIES += \
dalvik.vm.useartservice=true \
dalvik.vm.enable_pr_dexopt=true
+
+# Copy preopted files from system_b on first boot.
+PRODUCT_SYSTEM_PROPERTIES += ro.cp_system_other_odex=1
diff --git a/tools/aconfig/aconfig/src/codegen/java.rs b/tools/aconfig/aconfig/src/codegen/java.rs
index ec22ebc..7238590 100644
--- a/tools/aconfig/aconfig/src/codegen/java.rs
+++ b/tools/aconfig/aconfig/src/codegen/java.rs
@@ -1,18 +1,18 @@
/*
- * Copyright (C) 2023 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+* Copyright (C) 2023 The Android Open Source Project
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
use anyhow::Result;
use serde::Serialize;
@@ -46,7 +46,6 @@
let runtime_lookup_required =
flag_elements.iter().any(|elem| elem.is_read_write) || library_exported;
let container = (flag_elements.first().expect("zero template flags").container).to_string();
-
let context = Context {
flag_elements,
namespace_flags,
@@ -508,12 +507,15 @@
private static FeatureFlags FEATURE_FLAGS = new FeatureFlagsImpl();
}"#;
- let expect_featureflagsimpl_content = r#"
+ let expected_featureflagsmpl_content_0 = r#"
package com.android.aconfig.test;
// TODO(b/303773055): Remove the annotation after access issue is resolved.
import android.compat.annotation.UnsupportedAppUsage;
import android.provider.DeviceConfig;
import android.provider.DeviceConfig.Properties;
+ "#;
+
+ let expected_featureflagsmpl_content_1 = r#"
/** @hide */
public final class FeatureFlagsImpl implements FeatureFlags {
private static boolean aconfig_test_is_cached = false;
@@ -522,48 +524,8 @@
private static boolean disabledRwExported = false;
private static boolean disabledRwInOtherNamespace = false;
private static boolean enabledRw = true;
-
-
- private void load_overrides_aconfig_test() {
- try {
- Properties properties = DeviceConfig.getProperties("aconfig_test");
- disabledRw =
- properties.getBoolean(Flags.FLAG_DISABLED_RW, false);
- disabledRwExported =
- properties.getBoolean(Flags.FLAG_DISABLED_RW_EXPORTED, false);
- enabledRw =
- properties.getBoolean(Flags.FLAG_ENABLED_RW, true);
- } catch (NullPointerException e) {
- throw new RuntimeException(
- "Cannot read value from namespace aconfig_test "
- + "from DeviceConfig. It could be that the code using flag "
- + "executed before SettingsProvider initialization. Please use "
- + "fixed read-only flag by adding is_fixed_read_only: true in "
- + "flag declaration.",
- e
- );
- }
- aconfig_test_is_cached = true;
- }
-
- private void load_overrides_other_namespace() {
- try {
- Properties properties = DeviceConfig.getProperties("other_namespace");
- disabledRwInOtherNamespace =
- properties.getBoolean(Flags.FLAG_DISABLED_RW_IN_OTHER_NAMESPACE, false);
- } catch (NullPointerException e) {
- throw new RuntimeException(
- "Cannot read value from namespace other_namespace "
- + "from DeviceConfig. It could be that the code using flag "
- + "executed before SettingsProvider initialization. Please use "
- + "fixed read-only flag by adding is_fixed_read_only: true in "
- + "flag declaration.",
- e
- );
- }
- other_namespace_is_cached = true;
- }
-
+ "#;
+ let expected_featureflagsmpl_content_2 = r#"
@Override
@com.android.aconfig.annotations.AconfigFlagAccessor
@UnsupportedAppUsage
@@ -632,9 +594,213 @@
}
}
"#;
+
+ let expect_featureflagsimpl_content_old = expected_featureflagsmpl_content_0.to_owned()
+ + expected_featureflagsmpl_content_1
+ + r#"
+ private void load_overrides_aconfig_test() {
+ try {
+ Properties properties = DeviceConfig.getProperties("aconfig_test");
+ disabledRw =
+ properties.getBoolean(Flags.FLAG_DISABLED_RW, false);
+ disabledRwExported =
+ properties.getBoolean(Flags.FLAG_DISABLED_RW_EXPORTED, false);
+ enabledRw =
+ properties.getBoolean(Flags.FLAG_ENABLED_RW, true);
+ } catch (NullPointerException e) {
+ throw new RuntimeException(
+ "Cannot read value from namespace aconfig_test "
+ + "from DeviceConfig. It could be that the code using flag "
+ + "executed before SettingsProvider initialization. Please use "
+ + "fixed read-only flag by adding is_fixed_read_only: true in "
+ + "flag declaration.",
+ e
+ );
+ }
+ aconfig_test_is_cached = true;
+ }
+
+ private void load_overrides_other_namespace() {
+ try {
+ Properties properties = DeviceConfig.getProperties("other_namespace");
+ disabledRwInOtherNamespace =
+ properties.getBoolean(Flags.FLAG_DISABLED_RW_IN_OTHER_NAMESPACE, false);
+ } catch (NullPointerException e) {
+ throw new RuntimeException(
+ "Cannot read value from namespace other_namespace "
+ + "from DeviceConfig. It could be that the code using flag "
+ + "executed before SettingsProvider initialization. Please use "
+ + "fixed read-only flag by adding is_fixed_read_only: true in "
+ + "flag declaration.",
+ e
+ );
+ }
+ other_namespace_is_cached = true;
+ }"#
+ + expected_featureflagsmpl_content_2;
+
let mut file_set = HashMap::from([
("com/android/aconfig/test/Flags.java", expect_flags_content.as_str()),
- ("com/android/aconfig/test/FeatureFlagsImpl.java", expect_featureflagsimpl_content),
+ (
+ "com/android/aconfig/test/FeatureFlagsImpl.java",
+ &expect_featureflagsimpl_content_old,
+ ),
+ ("com/android/aconfig/test/FeatureFlags.java", EXPECTED_FEATUREFLAGS_COMMON_CONTENT),
+ (
+ "com/android/aconfig/test/CustomFeatureFlags.java",
+ EXPECTED_CUSTOMFEATUREFLAGS_CONTENT,
+ ),
+ (
+ "com/android/aconfig/test/FakeFeatureFlagsImpl.java",
+ EXPECTED_FAKEFEATUREFLAGSIMPL_CONTENT,
+ ),
+ ]);
+
+ for file in generated_files {
+ let file_path = file.path.to_str().unwrap();
+ assert!(file_set.contains_key(file_path), "Cannot find {}", file_path);
+ assert_eq!(
+ None,
+ crate::test::first_significant_code_diff(
+ file_set.get(file_path).unwrap(),
+ &String::from_utf8(file.contents).unwrap()
+ ),
+ "File {} content is not correct",
+ file_path
+ );
+ file_set.remove(file_path);
+ }
+
+ assert!(file_set.is_empty());
+
+ let parsed_flags = crate::test::parse_test_flags();
+ let mode = CodegenMode::Production;
+ let modified_parsed_flags =
+ crate::commands::modify_parsed_flags_based_on_mode(parsed_flags, mode).unwrap();
+ let flag_ids =
+ assign_flag_ids(crate::test::TEST_PACKAGE, modified_parsed_flags.iter()).unwrap();
+ let generated_files = generate_java_code(
+ crate::test::TEST_PACKAGE,
+ modified_parsed_flags.into_iter(),
+ mode,
+ flag_ids,
+ true,
+ )
+ .unwrap();
+
+ let expect_featureflagsimpl_content_new = expected_featureflagsmpl_content_0.to_owned()
+ + r#"
+ import android.aconfig.storage.StorageInternalReader;
+ import android.util.Log;
+ "#
+ + expected_featureflagsmpl_content_1
+ + r#"
+ StorageInternalReader reader;
+ boolean readFromNewStorage;
+
+ private final static String TAG = "AconfigJavaCodegen";
+ private final static String SUCCESS_LOG = "success: %s value matches";
+ private final static String MISMATCH_LOG = "error: %s value mismatch, new storage value is %s, old storage value is %s";
+ private final static String ERROR_LOG = "error: failed to read flag value";
+
+ private void init() {
+ if (reader != null) return;
+ if (DeviceConfig.getBoolean("core_experiments_team_internal", "com.android.providers.settings.storage_test_mission_1", false)) {
+ readFromNewStorage = true;
+ try {
+ reader = new StorageInternalReader("system", "com.android.aconfig.test");
+ } catch (Exception e) {
+ reader = null;
+ }
+ }
+ }
+
+ private void load_overrides_aconfig_test() {
+ try {
+ Properties properties = DeviceConfig.getProperties("aconfig_test");
+ disabledRw =
+ properties.getBoolean(Flags.FLAG_DISABLED_RW, false);
+ disabledRwExported =
+ properties.getBoolean(Flags.FLAG_DISABLED_RW_EXPORTED, false);
+ enabledRw =
+ properties.getBoolean(Flags.FLAG_ENABLED_RW, true);
+ } catch (NullPointerException e) {
+ throw new RuntimeException(
+ "Cannot read value from namespace aconfig_test "
+ + "from DeviceConfig. It could be that the code using flag "
+ + "executed before SettingsProvider initialization. Please use "
+ + "fixed read-only flag by adding is_fixed_read_only: true in "
+ + "flag declaration.",
+ e
+ );
+ }
+ aconfig_test_is_cached = true;
+ init();
+ if (readFromNewStorage && reader != null) {
+ boolean val;
+ try {
+ val = reader.getBooleanFlagValue(1);
+ if (val == disabledRw) {
+ Log.i(TAG, String.format(SUCCESS_LOG, "disabledRw"));
+ } else {
+ Log.i(TAG, String.format(MISMATCH_LOG, "disabledRw", val, disabledRw));
+ }
+ val = reader.getBooleanFlagValue(2);
+ if (val == disabledRwExported) {
+ Log.i(TAG, String.format(SUCCESS_LOG, "disabledRwExported"));
+ } else {
+ Log.i(TAG, String.format(MISMATCH_LOG, "disabledRwExported", val, disabledRwExported));
+ }
+ val = reader.getBooleanFlagValue(8);
+ if (val == enabledRw) {
+ Log.i(TAG, String.format(SUCCESS_LOG, "enabledRw"));
+ } else {
+ Log.i(TAG, String.format(MISMATCH_LOG, "enabledRw", val, enabledRw));
+ }
+ } catch (Exception e) {
+ Log.e(TAG, ERROR_LOG, e);
+ }
+ }
+ }
+
+ private void load_overrides_other_namespace() {
+ try {
+ Properties properties = DeviceConfig.getProperties("other_namespace");
+ disabledRwInOtherNamespace =
+ properties.getBoolean(Flags.FLAG_DISABLED_RW_IN_OTHER_NAMESPACE, false);
+ } catch (NullPointerException e) {
+ throw new RuntimeException(
+ "Cannot read value from namespace other_namespace "
+ + "from DeviceConfig. It could be that the code using flag "
+ + "executed before SettingsProvider initialization. Please use "
+ + "fixed read-only flag by adding is_fixed_read_only: true in "
+ + "flag declaration.",
+ e
+ );
+ }
+ other_namespace_is_cached = true;
+ init();
+ if (readFromNewStorage && reader != null) {
+ boolean val;
+ try {
+ val = reader.getBooleanFlagValue(3);
+ if (val == disabledRwInOtherNamespace) {
+ Log.i(TAG, String.format(SUCCESS_LOG, "disabledRwInOtherNamespace"));
+ } else {
+ Log.i(TAG, String.format(MISMATCH_LOG, "disabledRwInOtherNamespace", val, disabledRwInOtherNamespace));
+ }
+ } catch (Exception e) {
+ Log.e(TAG, ERROR_LOG, e);
+ }
+ }
+ }"# + expected_featureflagsmpl_content_2;
+
+ let mut file_set = HashMap::from([
+ ("com/android/aconfig/test/Flags.java", expect_flags_content.as_str()),
+ (
+ "com/android/aconfig/test/FeatureFlagsImpl.java",
+ &expect_featureflagsimpl_content_new,
+ ),
("com/android/aconfig/test/FeatureFlags.java", EXPECTED_FEATUREFLAGS_COMMON_CONTENT),
(
"com/android/aconfig/test/CustomFeatureFlags.java",
@@ -677,7 +843,7 @@
modified_parsed_flags.into_iter(),
mode,
flag_ids,
- false,
+ true,
)
.unwrap();
@@ -870,7 +1036,7 @@
modified_parsed_flags.into_iter(),
mode,
flag_ids,
- false,
+ true,
)
.unwrap();
@@ -991,7 +1157,7 @@
modified_parsed_flags.into_iter(),
mode,
flag_ids,
- false,
+ true,
)
.unwrap();
let expect_featureflags_content = r#"
diff --git a/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template
index cd2e3db..d93d302 100644
--- a/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template
+++ b/tools/aconfig/aconfig/templates/FeatureFlagsImpl.java.template
@@ -10,11 +10,11 @@
import android.provider.DeviceConfig.Properties;
+{{ -if not library_exported }}
{{ -if allow_instrumentation }}
import android.aconfig.storage.StorageInternalReader;
import android.util.Log;
-
-import java.io.File;
+{{ -endif }}
{{ -endif }}
{{ -endif }}
@@ -30,47 +30,38 @@
private static boolean {flag.method_name} = {flag.default_value};
{{ -endif }}
{{ -endfor }}
+{{ -if not library_exported }}
{{ -if allow_instrumentation }}
StorageInternalReader reader;
boolean readFromNewStorage;
private final static String TAG = "AconfigJavaCodegen";
+ private final static String SUCCESS_LOG = "success: %s value matches";
+ private final static String MISMATCH_LOG = "error: %s value mismatch, new storage value is %s, old storage value is %s";
+ private final static String ERROR_LOG = "error: failed to read flag value";
- public FeatureFlagsImpl() \{
- File file = new File("/metadata/aconfig_test_missions/mission_1");
- if (file.exists()) \{
+ private void init() \{
+ if (reader != null) return;
+ if (DeviceConfig.getBoolean("core_experiments_team_internal", "com.android.providers.settings.storage_test_mission_1", false)) \{
readFromNewStorage = true;
- reader = new StorageInternalReader("{container}", "{package_name}");
+ try \{
+ reader = new StorageInternalReader("{container}", "{package_name}");
+ } catch (Exception e) \{
+ reader = null;
+ }
}
}
+
+{{ -endif }}
{{ -endif }}
{{ for namespace_with_flags in namespace_flags }}
private void load_overrides_{namespace_with_flags.namespace}() \{
try \{
-{{ -if allow_instrumentation }}
- boolean val;
-{{ -endif }}
Properties properties = DeviceConfig.getProperties("{namespace_with_flags.namespace}");
{{ -for flag in namespace_with_flags.flags }}
{{ -if flag.is_read_write }}
{flag.method_name} =
properties.getBoolean(Flags.FLAG_{flag.flag_name_constant_suffix}, {flag.default_value});
-{{ -if allow_instrumentation }}
- if (readFromNewStorage) \{
- try \{
- val = reader.getBooleanFlagValue({flag.flag_offset});
- if (val == {flag.method_name}) \{
- Log.i(TAG, "success: {flag.method_name} value matches");
- } else \{
- Log.i(TAG, String.format(
- "error: {flag.method_name} value mismatch, new storage value is %s, old storage value is %s",
- val, {flag.method_name}));
- }
- } catch (Exception e) \{
- Log.e(TAG,"error: failed to read flag value of {flag.method_name}");
- }
- }
-{{ -endif }}
{{ -endif }}
{{ -endfor }}
} catch (NullPointerException e) \{
@@ -84,6 +75,29 @@
);
}
{namespace_with_flags.namespace}_is_cached = true;
+{{ -if not library_exported }}
+{{ -if allow_instrumentation }}
+ init();
+ if (readFromNewStorage && reader != null) \{
+ boolean val;
+ try \{
+{{ -for flag in namespace_with_flags.flags }}
+{{ -if flag.is_read_write }}
+
+ val = reader.getBooleanFlagValue({flag.flag_offset});
+ if (val == {flag.method_name}) \{
+ Log.i(TAG, String.format(SUCCESS_LOG, "{flag.method_name}"));
+ } else \{
+ Log.i(TAG, String.format(MISMATCH_LOG, "{flag.method_name}", val, {flag.method_name}));
+ }
+{{ -endif }}
+{{ -endfor }}
+ } catch (Exception e) \{
+ Log.e(TAG, ERROR_LOG, e);
+ }
+ }
+{{ -endif }}
+{{ -endif }}
}
{{ endfor- }}
{{ -endif }}{#- end of runtime_lookup_required #}
diff --git a/tools/aconfig/aconfig_storage_file/Android.bp b/tools/aconfig/aconfig_storage_file/Android.bp
index 3859194..40b4464 100644
--- a/tools/aconfig/aconfig_storage_file/Android.bp
+++ b/tools/aconfig/aconfig_storage_file/Android.bp
@@ -138,11 +138,28 @@
double_loadable: true,
}
-// storage file parse api java cc_library
+// storage file parse api java library
java_library {
name: "aconfig_storage_file_java",
srcs: [
"srcs/**/*.java",
],
sdk_version: "core_current",
-}
\ No newline at end of file
+ min_sdk_version: "29",
+ host_supported: true,
+ apex_available: [
+ "//apex_available:platform",
+ "//apex_available:anyapex",
+ ],
+}
+
+// storage file parse api java library for core library
+java_library {
+ name: "aconfig_storage_file_java_none",
+ srcs: [
+ "srcs/**/*.java",
+ ],
+ sdk_version: "none",
+ system_modules: "core-all-system-modules",
+ host_supported: true,
+}
diff --git a/tools/aconfig/aconfig_storage_file/tests/Android.bp b/tools/aconfig/aconfig_storage_file/tests/Android.bp
index c33127f..e2e225d 100644
--- a/tools/aconfig/aconfig_storage_file/tests/Android.bp
+++ b/tools/aconfig/aconfig_storage_file/tests/Android.bp
@@ -28,12 +28,11 @@
"srcs/**/*.java",
],
static_libs: [
- "aconfig_storage_file_java",
"androidx.test.runner",
"junit",
],
- sdk_version: "test_current",
test_config: "AndroidStorageJaveTest.xml",
+ certificate: "platform",
data: [
"package.map",
"flag.map",
diff --git a/tools/aconfig/aconfig_storage_read_api/Android.bp b/tools/aconfig/aconfig_storage_read_api/Android.bp
index c2f4c18..619b488 100644
--- a/tools/aconfig/aconfig_storage_read_api/Android.bp
+++ b/tools/aconfig/aconfig_storage_read_api/Android.bp
@@ -87,6 +87,9 @@
generated_sources: ["libcxx_aconfig_storage_read_api_bridge_code"],
whole_static_libs: ["libaconfig_storage_read_api_cxx_bridge"],
export_include_dirs: ["include"],
+ static_libs: [
+ "libbase",
+ ],
host_supported: true,
vendor_available: true,
product_available: true,
@@ -171,8 +174,33 @@
srcs: [
"srcs/android/aconfig/storage/StorageInternalReader.java",
],
+ libs: [
+ "unsupportedappusage",
+ ],
static_libs: [
"aconfig_storage_file_java",
],
sdk_version: "core_current",
+ host_supported: true,
+ min_sdk_version: "29",
+ apex_available: [
+ "//apex_available:platform",
+ "//apex_available:anyapex",
+ ],
+}
+
+java_library {
+ name: "aconfig_storage_reader_java_none",
+ srcs: [
+ "srcs/android/aconfig/storage/StorageInternalReader.java",
+ ],
+ libs: [
+ "unsupportedappusage-sdk-none",
+ ],
+ static_libs: [
+ "aconfig_storage_file_java_none",
+ ],
+ sdk_version: "none",
+ system_modules: "core-all-system-modules",
+ host_supported: true,
}
diff --git a/tools/aconfig/aconfig_storage_read_api/aconfig_storage_read_api.cpp b/tools/aconfig/aconfig_storage_read_api/aconfig_storage_read_api.cpp
index 97ada3a..8e0c4e1 100644
--- a/tools/aconfig/aconfig_storage_read_api/aconfig_storage_read_api.cpp
+++ b/tools/aconfig/aconfig_storage_read_api/aconfig_storage_read_api.cpp
@@ -1,3 +1,4 @@
+#include <android-base/unique_fd.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
@@ -59,22 +60,22 @@
/// Map a storage file
Result<MappedStorageFile*> map_storage_file(std::string const& file) {
- int fd = open(file.c_str(), O_CLOEXEC | O_NOFOLLOW | O_RDONLY);
- if (fd == -1) {
+ android::base::unique_fd ufd(open(file.c_str(), O_CLOEXEC | O_NOFOLLOW | O_RDONLY));
+ if (ufd.get() == -1) {
auto result = Result<MappedStorageFile*>();
result.errmsg = std::string("failed to open ") + file + ": " + strerror(errno);
return result;
};
struct stat fd_stat;
- if (fstat(fd, &fd_stat) < 0) {
+ if (fstat(ufd.get(), &fd_stat) < 0) {
auto result = Result<MappedStorageFile*>();
result.errmsg = std::string("fstat failed: ") + strerror(errno);
return result;
}
size_t file_size = fd_stat.st_size;
- void* const map_result = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0);
+ void* const map_result = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, ufd.get(), 0);
if (map_result == MAP_FAILED) {
auto result = Result<MappedStorageFile*>();
result.errmsg = std::string("mmap failed: ") + strerror(errno);
diff --git a/tools/aconfig/aconfig_storage_read_api/srcs/android/aconfig/storage/StorageInternalReader.java b/tools/aconfig/aconfig_storage_read_api/srcs/android/aconfig/storage/StorageInternalReader.java
index 5f31017..bbb6813 100644
--- a/tools/aconfig/aconfig_storage_read_api/srcs/android/aconfig/storage/StorageInternalReader.java
+++ b/tools/aconfig/aconfig_storage_read_api/srcs/android/aconfig/storage/StorageInternalReader.java
@@ -16,10 +16,13 @@
package android.aconfig.storage;
+import android.compat.annotation.UnsupportedAppUsage;
+
import java.io.FileInputStream;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
+/** @hide */
public class StorageInternalReader {
private static final String MAP_PATH = "/metadata/aconfig/maps/";
@@ -30,16 +33,19 @@
private int mPackageBooleanStartOffset;
+ @UnsupportedAppUsage
public StorageInternalReader(String container, String packageName) {
this(packageName, MAP_PATH + container + ".package.map", BOOT_PATH + container + ".val");
}
+ @UnsupportedAppUsage
public StorageInternalReader(String packageName, String packageMapFile, String flagValueFile) {
mPackageTable = PackageTable.fromBytes(mapStorageFile(packageMapFile));
mFlagValueList = FlagValueList.fromBytes(mapStorageFile(flagValueFile));
mPackageBooleanStartOffset = getPackageBooleanStartOffset(packageName);
}
+ @UnsupportedAppUsage
public boolean getBooleanFlagValue(int index) {
index += mPackageBooleanStartOffset;
if (index >= mFlagValueList.size()) {
diff --git a/tools/aconfig/aflags/src/aconfig_storage_source.rs b/tools/aconfig/aflags/src/aconfig_storage_source.rs
index 04140c7..0dfb956 100644
--- a/tools/aconfig/aflags/src/aconfig_storage_source.rs
+++ b/tools/aconfig/aflags/src/aconfig_storage_source.rs
@@ -1,14 +1,93 @@
use crate::{Flag, FlagPermission, FlagSource, FlagValue, ValuePickedFrom};
use anyhow::{anyhow, Result};
+use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
pub struct AconfigStorageSource {}
+use aconfig_storage_file::protos::ProtoStorageFileInfo;
use aconfig_storage_file::protos::ProtoStorageFiles;
+use aconfig_storage_file::FlagValueAndInfoSummary;
-static STORAGE_INFO_FILE_PATH: &str = "/metadata/aconfig/persistent_storage_file_records.pb";
+static STORAGE_INFO_FILE_PATH: &str = "/metadata/aconfig/storage_records.pb";
+
+fn read_default_values(file_info: ProtoStorageFileInfo) -> Result<HashMap<String, FlagValue>> {
+ let package_map =
+ file_info.package_map.ok_or(anyhow!("storage file is missing package map"))?;
+ let flag_map = file_info.flag_map.ok_or(anyhow!("storage file is missing flag map"))?;
+ let flag_val = file_info.flag_val.ok_or(anyhow!("storage file is missing flag val"))?;
+
+ let mut result = HashMap::new();
+ for listed_flag in aconfig_storage_file::list_flags(&package_map, &flag_map, &flag_val)? {
+ let value = FlagValue::try_from(listed_flag.flag_value.as_str())?;
+ result.insert(listed_flag.package_name + &listed_flag.flag_name, value);
+ }
+ Ok(result)
+}
+
+fn read_next_boot_values(
+ listed_flags: &[FlagValueAndInfoSummary],
+) -> Result<HashMap<String, FlagValue>> {
+ let mut result = HashMap::new();
+ for flag in listed_flags {
+ result.insert(
+ flag.package_name.clone() + &flag.flag_name,
+ FlagValue::try_from(flag.flag_value.as_str())?,
+ );
+ }
+ Ok(result)
+}
+
+fn reconcile(
+ default_values: HashMap<String, FlagValue>,
+ next_boot_values: HashMap<String, FlagValue>,
+ flags_current_boot: &[FlagValueAndInfoSummary],
+ container: &str,
+) -> Result<Vec<Flag>> {
+ let mut result = Vec::new();
+ for listed_flag in flags_current_boot {
+ let default_value = default_values
+ .get(&(listed_flag.package_name.clone() + &listed_flag.flag_name))
+ .copied();
+
+ let name = listed_flag.flag_name.clone();
+ let package = listed_flag.package_name.clone();
+ let value = FlagValue::try_from(listed_flag.flag_value.as_str())?;
+ let container = container.to_string();
+ let staged_value = next_boot_values
+ .get(&(listed_flag.package_name.clone() + &listed_flag.flag_name))
+ .filter(|&v| value != *v)
+ .copied();
+ let permission = if listed_flag.is_readwrite {
+ FlagPermission::ReadWrite
+ } else {
+ FlagPermission::ReadOnly
+ };
+ let value_picked_from = if listed_flag.has_local_override {
+ ValuePickedFrom::Local
+ } else if Some(value) == default_value {
+ ValuePickedFrom::Default
+ } else {
+ ValuePickedFrom::Server
+ };
+
+ result.push(Flag {
+ name,
+ package,
+ value,
+ container,
+ staged_value,
+ permission,
+ value_picked_from,
+
+ // TODO(b/324436145): delete namespace field once DeviceConfig isn't in CLI.
+ namespace: "-".to_string(),
+ });
+ }
+ Ok(result)
+}
impl FlagSource for AconfigStorageSource {
fn list_flags() -> Result<Vec<Flag>> {
@@ -20,30 +99,35 @@
let storage_file_info: ProtoStorageFiles = protobuf::Message::parse_from_bytes(&bytes)?;
for file_info in storage_file_info.files {
- let package_map =
- file_info.package_map.ok_or(anyhow!("storage file is missing package map"))?;
- let flag_map = file_info.flag_map.ok_or(anyhow!("storage file is missing flag map"))?;
- let flag_val = file_info.flag_val.ok_or(anyhow!("storage file is missing flag val"))?;
+ let default_values = read_default_values(file_info.clone())?;
+
let container =
file_info.container.ok_or(anyhow!("storage file is missing container"))?;
+ let package_map = format!("/metadata/aconfig/maps/{container}.package.map");
+ let flag_map = format!("/metadata/aconfig/maps/{container}.flag.map");
+ let flag_info = format!("/metadata/aconfig/boot/{container}.info");
- for listed_flag in aconfig_storage_file::list_flags(&package_map, &flag_map, &flag_val)?
- {
- result.push(Flag {
- name: listed_flag.flag_name,
- package: listed_flag.package_name,
- value: FlagValue::try_from(listed_flag.flag_value.as_str())?,
- container: container.to_string(),
+ let flag_val_current_boot = format!("/metadata/aconfig/boot/{container}.val");
+ let flag_val_next_boot = format!("/metadata/aconfig/flags/{container}.val");
- // TODO(b/324436145): delete namespace field once DeviceConfig isn't in CLI.
- namespace: "-".to_string(),
+ let flags_next_boot = aconfig_storage_file::list_flags_with_info(
+ &package_map,
+ &flag_map,
+ &flag_val_next_boot,
+ &flag_info,
+ )?;
+ let flags_current_boot = aconfig_storage_file::list_flags_with_info(
+ &package_map,
+ &flag_map,
+ &flag_val_current_boot,
+ &flag_info,
+ )?;
- // TODO(b/324436145): Populate with real values once API is available.
- staged_value: None,
- permission: FlagPermission::ReadOnly,
- value_picked_from: ValuePickedFrom::Default,
- });
- }
+ let next_boot_values = read_next_boot_values(&flags_next_boot)?;
+ let processed_flags =
+ reconcile(default_values, next_boot_values, &flags_current_boot, &container)?;
+
+ result.extend(processed_flags);
}
Ok(result)
diff --git a/tools/aconfig/aflags/src/main.rs b/tools/aconfig/aflags/src/main.rs
index 810f2e3..0a5c989 100644
--- a/tools/aconfig/aflags/src/main.rs
+++ b/tools/aconfig/aflags/src/main.rs
@@ -50,6 +50,7 @@
enum ValuePickedFrom {
Default,
Server,
+ Local,
}
impl std::fmt::Display for ValuePickedFrom {
@@ -60,6 +61,7 @@
match &self {
Self::Default => "default",
Self::Server => "server",
+ Self::Local => "local",
}
)
}
@@ -286,7 +288,9 @@
let cli = Cli::parse();
let output = match cli.command {
Command::List { use_new_storage: true, container } => {
- list(FlagSourceType::AconfigStorage, container).map(Some)
+ list(FlagSourceType::AconfigStorage, container)
+ .map_err(|_| anyhow!("storage may not be enabled"))
+ .map(Some)
}
Command::List { use_new_storage: false, container } => {
list(FlagSourceType::DeviceConfig, container).map(Some)
diff --git a/tools/aconfig/fake_device_config/src/android/provider/Log.java b/tools/aconfig/fake_device_config/src/android/provider/Log.java
new file mode 100644
index 0000000..3e7fd0f
--- /dev/null
+++ b/tools/aconfig/fake_device_config/src/android/provider/Log.java
@@ -0,0 +1,15 @@
+package android.util;
+
+public final class Log {
+ public static int i(String tag, String msg) {
+ return 0;
+ }
+
+ public static int e(String tag, String msg) {
+ return 0;
+ }
+
+ public static int e(String tag, String msg, Throwable tr) {
+ return 0;
+ }
+}
diff --git a/tools/check-flagged-apis/src/com/android/checkflaggedapis/Main.kt b/tools/check-flagged-apis/src/com/android/checkflaggedapis/Main.kt
index 1125d39..d323c20 100644
--- a/tools/check-flagged-apis/src/com/android/checkflaggedapis/Main.kt
+++ b/tools/check-flagged-apis/src/com/android/checkflaggedapis/Main.kt
@@ -19,10 +19,10 @@
import android.aconfig.Aconfig
import com.android.tools.metalava.model.BaseItemVisitor
+import com.android.tools.metalava.model.CallableItem
import com.android.tools.metalava.model.ClassItem
import com.android.tools.metalava.model.FieldItem
import com.android.tools.metalava.model.Item
-import com.android.tools.metalava.model.MethodItem
import com.android.tools.metalava.model.text.ApiFile
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.core.ProgramResult
@@ -274,15 +274,15 @@
}
}
- override fun visitMethod(method: MethodItem) {
- getFlagOrNull(method)?.let { flag ->
- val methodName = buildString {
- append(method.name())
+ override fun visitCallable(callable: CallableItem) {
+ getFlagOrNull(callable)?.let { flag ->
+ val callableSignature = buildString {
+ append(callable.name())
append("(")
- method.parameters().joinTo(this, separator = "") { it.type().internalName() }
+ callable.parameters().joinTo(this, separator = "") { it.type().internalName() }
append(")")
}
- val symbol = Symbol.createMethod(method.containingClass().qualifiedName(), methodName)
+ val symbol = Symbol.createMethod(callable.containingClass().qualifiedName(), callableSignature)
output.add(Pair(symbol, flag))
}
}
diff --git a/tools/filelistdiff/allowlist b/tools/filelistdiff/allowlist
index 0a51d0e..943f955 100644
--- a/tools/filelistdiff/allowlist
+++ b/tools/filelistdiff/allowlist
@@ -78,6 +78,10 @@
lib/libwfds.so
lib/libyuv.so
+# b/351258461
+adb_keys
+init.environ.rc
+
# Known diffs only in the Soong system image
lib/libhidcommand_jni.so
lib/libuinputcommand_jni.so
\ No newline at end of file
diff --git a/tools/sbom/Android.bp b/tools/sbom/Android.bp
index 2b2b573..6901b06 100644
--- a/tools/sbom/Android.bp
+++ b/tools/sbom/Android.bp
@@ -33,6 +33,23 @@
],
}
+python_binary_host {
+ name: "gen_sbom",
+ srcs: [
+ "gen_sbom.py",
+ ],
+ version: {
+ py3: {
+ embedded_launcher: true,
+ },
+ },
+ libs: [
+ "metadata_file_proto_py",
+ "libprotobuf-python",
+ "sbom_lib",
+ ],
+}
+
python_library_host {
name: "sbom_lib",
srcs: [
@@ -91,4 +108,4 @@
libs: [
"sbom_lib",
],
-}
\ No newline at end of file
+}
diff --git a/tools/sbom/gen_sbom.py b/tools/sbom/gen_sbom.py
new file mode 100644
index 0000000..a203258
--- /dev/null
+++ b/tools/sbom/gen_sbom.py
@@ -0,0 +1,926 @@
+# !/usr/bin/env python3
+#
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Generate the SBOM of the current target product in SPDX format.
+Usage example:
+ gen_sbom.py --output_file out/soong/sbom/aosp_cf_x86_64_phone/sbom.spdx \
+ --metadata out/soong/metadata/aosp_cf_x86_64_phone/metadata.db \
+ --product_out out/target/vsoc_x86_64
+ --soong_out out/soong
+ --build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \
+ --product_mfr=Google
+"""
+
+import argparse
+import datetime
+import google.protobuf.text_format as text_format
+import hashlib
+import os
+import pathlib
+import queue
+import metadata_file_pb2
+import sbom_data
+import sbom_writers
+import sqlite3
+
+# Package type
+PKG_SOURCE = 'SOURCE'
+PKG_UPSTREAM = 'UPSTREAM'
+PKG_PREBUILT = 'PREBUILT'
+
+# Security tag
+NVD_CPE23 = 'NVD-CPE2.3:'
+
+# Report
+ISSUE_NO_METADATA = 'No metadata generated in Make for installed files:'
+ISSUE_NO_METADATA_FILE = 'No METADATA file found for installed file:'
+ISSUE_METADATA_FILE_INCOMPLETE = 'METADATA file incomplete:'
+ISSUE_UNKNOWN_SECURITY_TAG_TYPE = 'Unknown security tag type:'
+ISSUE_INSTALLED_FILE_NOT_EXIST = 'Non-existent installed files:'
+ISSUE_NO_MODULE_FOUND_FOR_STATIC_DEP = 'No module found for static dependency files:'
+INFO_METADATA_FOUND_FOR_PACKAGE = 'METADATA file found for packages:'
+
+SOONG_PREBUILT_MODULE_TYPES = [
+ 'android_app_import',
+ 'android_library_import',
+ 'cc_prebuilt_binary',
+ 'cc_prebuilt_library',
+ 'cc_prebuilt_library_headers',
+ 'cc_prebuilt_library_shared',
+ 'cc_prebuilt_library_static',
+ 'cc_prebuilt_object',
+ 'dex_import',
+ 'java_import',
+ 'java_sdk_library_import',
+ 'java_system_modules_import',
+ 'libclang_rt_prebuilt_library_static',
+ 'libclang_rt_prebuilt_library_shared',
+ 'llvm_prebuilt_library_static',
+ 'ndk_prebuilt_object',
+ 'ndk_prebuilt_shared_stl',
+ 'nkd_prebuilt_static_stl',
+ 'prebuilt_apex',
+ 'prebuilt_bootclasspath_fragment',
+ 'prebuilt_dsp',
+ 'prebuilt_firmware',
+ 'prebuilt_kernel_modules',
+ 'prebuilt_rfsa',
+ 'prebuilt_root',
+ 'rust_prebuilt_dylib',
+ 'rust_prebuilt_library',
+ 'rust_prebuilt_rlib',
+ 'vndk_prebuilt_shared',
+]
+
+THIRD_PARTY_IDENTIFIER_TYPES = [
+ # Types defined in metadata_file.proto
+ 'Git',
+ 'SVN',
+ 'Hg',
+ 'Darcs',
+ 'VCS',
+ 'Archive',
+ 'PrebuiltByAlphabet',
+ 'LocalSource',
+ 'Other',
+ # OSV ecosystems defined at https://ossf.github.io/osv-schema/#affectedpackage-field.
+ 'Go',
+ 'npm',
+ 'OSS-Fuzz',
+ 'PyPI',
+ 'RubyGems',
+ 'crates.io',
+ 'Hackage',
+ 'GHC',
+ 'Packagist',
+ 'Maven',
+ 'NuGet',
+ 'Linux',
+ 'Debian',
+ 'Alpine',
+ 'Hex',
+ 'Android',
+ 'GitHub Actions',
+ 'Pub',
+ 'ConanCenter',
+ 'Rocky Linux',
+ 'AlmaLinux',
+ 'Bitnami',
+ 'Photon OS',
+ 'CRAN',
+ 'Bioconductor',
+ 'SwiftURL'
+]
+
+
+def get_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.')
+ parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug mode')
+ parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.')
+ parser.add_argument('--metadata', required=True, help='The metadata DB file path.')
+ parser.add_argument('--product_out', required=True, help='The path of PRODUCT_OUT, e.g. out/target/product/vsoc_x86_64.')
+ parser.add_argument('--soong_out', required=True, help='The path of Soong output directory, e.g. out/soong')
+ parser.add_argument('--build_version', required=True, help='The build version.')
+ parser.add_argument('--product_mfr', required=True, help='The product manufacturer.')
+ parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format')
+
+ return parser.parse_args()
+
+
+def log(*info):
+ if args.verbose:
+ for i in info:
+ print(i)
+
+
+def new_package_id(package_name, type):
+ return f'SPDXRef-{type}-{sbom_data.encode_for_spdxid(package_name)}'
+
+
+def new_file_id(file_path):
+ return f'SPDXRef-{sbom_data.encode_for_spdxid(file_path)}'
+
+
+def new_license_id(license_name):
+ return f'LicenseRef-{sbom_data.encode_for_spdxid(license_name)}'
+
+
+def checksum(file_path):
+ h = hashlib.sha1()
+ if os.path.islink(file_path):
+ h.update(os.readlink(file_path).encode('utf-8'))
+ else:
+ with open(file_path, 'rb') as f:
+ h.update(f.read())
+ return f'SHA1: {h.hexdigest()}'
+
+
+def is_soong_prebuilt_module(file_metadata):
+ return (file_metadata['soong_module_type'] and
+ file_metadata['soong_module_type'] in SOONG_PREBUILT_MODULE_TYPES)
+
+
+def is_source_package(file_metadata):
+ module_path = file_metadata['module_path']
+ return module_path.startswith('external/') and not is_prebuilt_package(file_metadata)
+
+
+def is_prebuilt_package(file_metadata):
+ module_path = file_metadata['module_path']
+ if module_path:
+ return (module_path.startswith('prebuilts/') or
+ is_soong_prebuilt_module(file_metadata) or
+ file_metadata['is_prebuilt_make_module'])
+
+ kernel_module_copy_files = file_metadata['kernel_module_copy_files']
+ if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'):
+ return True
+
+ return False
+
+
+def get_source_package_info(file_metadata, metadata_file_path):
+ """Return source package info exists in its METADATA file, currently including name, security tag
+ and external SBOM reference.
+
+ See go/android-spdx and go/android-sbom-gen for more details.
+ """
+ if not metadata_file_path:
+ return file_metadata['module_path'], []
+
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ external_refs = []
+ for tag in metadata_proto.third_party.security.tag:
+ if tag.lower().startswith((NVD_CPE23 + 'cpe:2.3:').lower()):
+ external_refs.append(
+ sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY,
+ type=sbom_data.PackageExternalRefType.cpe23Type,
+ locator=tag.removeprefix(NVD_CPE23)))
+ elif tag.lower().startswith((NVD_CPE23 + 'cpe:/').lower()):
+ external_refs.append(
+ sbom_data.PackageExternalRef(category=sbom_data.PackageExternalRefCategory.SECURITY,
+ type=sbom_data.PackageExternalRefType.cpe22Type,
+ locator=tag.removeprefix(NVD_CPE23)))
+
+ if metadata_proto.name:
+ return metadata_proto.name, external_refs
+ else:
+ return os.path.basename(metadata_file_path), external_refs # return the directory name only as package name
+
+
+def get_prebuilt_package_name(file_metadata, metadata_file_path):
+ """Return name of a prebuilt package, which can be from the METADATA file, metadata file path,
+ module path or kernel module's source path if the installed file is a kernel module.
+
+ See go/android-spdx and go/android-sbom-gen for more details.
+ """
+ name = None
+ if metadata_file_path:
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ if metadata_proto.name:
+ name = metadata_proto.name
+ else:
+ name = metadata_file_path
+ elif file_metadata['module_path']:
+ name = file_metadata['module_path']
+ elif file_metadata['kernel_module_copy_files']:
+ src_path = file_metadata['kernel_module_copy_files'].split(':')[0]
+ name = os.path.dirname(src_path)
+
+ return name.removeprefix('prebuilts/').replace('/', '-')
+
+
+def get_metadata_file_path(file_metadata):
+ """Search for METADATA file of a package and return its path."""
+ metadata_path = ''
+ if file_metadata['module_path']:
+ metadata_path = file_metadata['module_path']
+ elif file_metadata['kernel_module_copy_files']:
+ metadata_path = os.path.dirname(file_metadata['kernel_module_copy_files'].split(':')[0])
+
+ while metadata_path and not os.path.exists(metadata_path + '/METADATA'):
+ metadata_path = os.path.dirname(metadata_path)
+
+ return metadata_path
+
+
+def get_package_version(metadata_file_path):
+ """Return a package's version in its METADATA file."""
+ if not metadata_file_path:
+ return None
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ return metadata_proto.third_party.version
+
+
+def get_package_homepage(metadata_file_path):
+ """Return a package's homepage URL in its METADATA file."""
+ if not metadata_file_path:
+ return None
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ if metadata_proto.third_party.homepage:
+ return metadata_proto.third_party.homepage
+ for url in metadata_proto.third_party.url:
+ if url.type == metadata_file_pb2.URL.Type.HOMEPAGE:
+ return url.value
+
+ return None
+
+
+def get_package_download_location(metadata_file_path):
+ """Return a package's code repository URL in its METADATA file."""
+ if not metadata_file_path:
+ return None
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ if metadata_proto.third_party.url:
+ urls = sorted(metadata_proto.third_party.url, key=lambda url: url.type)
+ if urls[0].type != metadata_file_pb2.URL.Type.HOMEPAGE:
+ return urls[0].value
+ elif len(urls) > 1:
+ return urls[1].value
+
+ return None
+
+
+def get_license_text(license_files):
+ license_text = ''
+ for license_file in license_files:
+ if args.debug:
+ license_text += '#### Content from ' + license_file + '\n'
+ else:
+ license_text += pathlib.Path(license_file).read_text(errors='replace') + '\n\n'
+ return license_text
+
+
+def get_sbom_fragments(installed_file_metadata, metadata_file_path):
+ """Return SPDX fragment of source/prebuilt packages, which usually contains a SOURCE/PREBUILT
+ package, a UPSTREAM package and an external SBOM document reference if sbom_ref defined in its
+ METADATA file.
+
+ See go/android-spdx and go/android-sbom-gen for more details.
+ """
+ external_doc_ref = None
+ packages = []
+ relationships = []
+ licenses = []
+
+ # Info from METADATA file
+ homepage = get_package_homepage(metadata_file_path)
+ version = get_package_version(metadata_file_path)
+ download_location = get_package_download_location(metadata_file_path)
+
+ lics = db.get_package_licenses(installed_file_metadata['module_path'])
+ if not lics:
+ lics = db.get_package_licenses(metadata_file_path)
+
+ if lics:
+ for license_name, license_files in lics.items():
+ if not license_files:
+ continue
+ license_id = new_license_id(license_name)
+ if license_name not in licenses_text:
+ licenses_text[license_name] = get_license_text(license_files.split(' '))
+ licenses.append(sbom_data.License(id=license_id, name=license_name, text=licenses_text[license_name]))
+
+ if is_source_package(installed_file_metadata):
+ # Source fork packages
+ name, external_refs = get_source_package_info(installed_file_metadata, metadata_file_path)
+ source_package_id = new_package_id(name, PKG_SOURCE)
+ source_package = sbom_data.Package(id=source_package_id, name=name, version=args.build_version,
+ download_location=sbom_data.VALUE_NONE,
+ supplier='Organization: ' + args.product_mfr,
+ external_refs=external_refs)
+
+ upstream_package_id = new_package_id(name, PKG_UPSTREAM)
+ upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version=version,
+ supplier=(
+ 'Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION,
+ download_location=download_location)
+ packages += [source_package, upstream_package]
+ relationships.append(sbom_data.Relationship(id1=source_package_id,
+ relationship=sbom_data.RelationshipType.VARIANT_OF,
+ id2=upstream_package_id))
+
+ for license in licenses:
+ source_package.declared_license_ids.append(license.id)
+ upstream_package.declared_license_ids.append(license.id)
+
+ elif is_prebuilt_package(installed_file_metadata):
+ # Prebuilt fork packages
+ name = get_prebuilt_package_name(installed_file_metadata, metadata_file_path)
+ prebuilt_package_id = new_package_id(name, PKG_PREBUILT)
+ prebuilt_package = sbom_data.Package(id=prebuilt_package_id,
+ name=name,
+ download_location=sbom_data.VALUE_NONE,
+ version=version if version else args.build_version,
+ supplier='Organization: ' + args.product_mfr)
+
+ upstream_package_id = new_package_id(name, PKG_UPSTREAM)
+ upstream_package = sbom_data.Package(id=upstream_package_id, name=name, version=version,
+ supplier=(
+ 'Organization: ' + homepage) if homepage else sbom_data.VALUE_NOASSERTION,
+ download_location=download_location)
+ packages += [prebuilt_package, upstream_package]
+ relationships.append(sbom_data.Relationship(id1=prebuilt_package_id,
+ relationship=sbom_data.RelationshipType.VARIANT_OF,
+ id2=upstream_package_id))
+ for license in licenses:
+ prebuilt_package.declared_license_ids.append(license.id)
+ upstream_package.declared_license_ids.append(license.id)
+
+ if metadata_file_path:
+ metadata_proto = metadata_file_protos[metadata_file_path]
+ if metadata_proto.third_party.WhichOneof('sbom') == 'sbom_ref':
+ sbom_url = metadata_proto.third_party.sbom_ref.url
+ sbom_checksum = metadata_proto.third_party.sbom_ref.checksum
+ upstream_element_id = metadata_proto.third_party.sbom_ref.element_id
+ if sbom_url and sbom_checksum and upstream_element_id:
+ doc_ref_id = f'DocumentRef-{PKG_UPSTREAM}-{sbom_data.encode_for_spdxid(name)}'
+ external_doc_ref = sbom_data.DocumentExternalReference(id=doc_ref_id,
+ uri=sbom_url,
+ checksum=sbom_checksum)
+ relationships.append(
+ sbom_data.Relationship(id1=upstream_package_id,
+ relationship=sbom_data.RelationshipType.VARIANT_OF,
+ id2=doc_ref_id + ':' + upstream_element_id))
+
+ return external_doc_ref, packages, relationships, licenses
+
+
+def save_report(report_file_path, report):
+ with open(report_file_path, 'w', encoding='utf-8') as report_file:
+ for type, issues in report.items():
+ report_file.write(type + '\n')
+ for issue in issues:
+ report_file.write('\t' + issue + '\n')
+ report_file.write('\n')
+
+
+# Validate the metadata generated by Make for installed files and report if there is no metadata.
+def installed_file_has_metadata(installed_file_metadata, report):
+ installed_file = installed_file_metadata['installed_file']
+ module_path = installed_file_metadata['module_path']
+ product_copy_files = installed_file_metadata['product_copy_files']
+ kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
+ is_platform_generated = installed_file_metadata['is_platform_generated']
+
+ if (not module_path and
+ not product_copy_files and
+ not kernel_module_copy_files and
+ not is_platform_generated and
+ not installed_file.endswith('.fsv_meta')):
+ report[ISSUE_NO_METADATA].append(installed_file)
+ return False
+
+ return True
+
+
+# Validate identifiers in a package's METADATA.
+# 1) Only known identifier type is allowed
+# 2) Only one identifier's primary_source can be true
+def validate_package_metadata(metadata_file_path, package_metadata):
+ primary_source_found = False
+ for identifier in package_metadata.third_party.identifier:
+ if identifier.type not in THIRD_PARTY_IDENTIFIER_TYPES:
+ sys.exit(f'Unknown value of third_party.identifier.type in {metadata_file_path}/METADATA: {identifier.type}.')
+ if primary_source_found and identifier.primary_source:
+ sys.exit(
+ f'Field "primary_source" is set to true in multiple third_party.identifier in {metadata_file_path}/METADATA.')
+ primary_source_found = identifier.primary_source
+
+
+def report_metadata_file(metadata_file_path, installed_file_metadata, report):
+ if metadata_file_path:
+ report[INFO_METADATA_FOUND_FOR_PACKAGE].append(
+ 'installed_file: {}, module_path: {}, METADATA file: {}'.format(
+ installed_file_metadata['installed_file'],
+ installed_file_metadata['module_path'],
+ metadata_file_path + '/METADATA'))
+
+ package_metadata = metadata_file_pb2.Metadata()
+ with open(metadata_file_path + '/METADATA', 'rt') as f:
+ text_format.Parse(f.read(), package_metadata)
+
+ validate_package_metadata(metadata_file_path, package_metadata)
+
+ if not metadata_file_path in metadata_file_protos:
+ metadata_file_protos[metadata_file_path] = package_metadata
+ if not package_metadata.name:
+ report[ISSUE_METADATA_FILE_INCOMPLETE].append(f'{metadata_file_path}/METADATA does not has "name"')
+
+ if not package_metadata.third_party.version:
+ report[ISSUE_METADATA_FILE_INCOMPLETE].append(
+ f'{metadata_file_path}/METADATA does not has "third_party.version"')
+
+ for tag in package_metadata.third_party.security.tag:
+ if not tag.startswith(NVD_CPE23):
+ report[ISSUE_UNKNOWN_SECURITY_TAG_TYPE].append(
+ f'Unknown security tag type: {tag} in {metadata_file_path}/METADATA')
+ else:
+ report[ISSUE_NO_METADATA_FILE].append(
+ "installed_file: {}, module_path: {}".format(
+ installed_file_metadata['installed_file'], installed_file_metadata['module_path']))
+
+
+# If a file is from a source fork or prebuilt fork package, add its package information to SBOM
+def add_package_of_file(file_id, file_metadata, doc, report):
+ metadata_file_path = get_metadata_file_path(file_metadata)
+ report_metadata_file(metadata_file_path, file_metadata, report)
+
+ external_doc_ref, pkgs, rels, licenses = get_sbom_fragments(file_metadata, metadata_file_path)
+ if len(pkgs) > 0:
+ if external_doc_ref:
+ doc.add_external_ref(external_doc_ref)
+ for p in pkgs:
+ doc.add_package(p)
+ for rel in rels:
+ doc.add_relationship(rel)
+ fork_package_id = pkgs[0].id # The first package should be the source/prebuilt fork package
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=fork_package_id))
+ for license in licenses:
+ doc.add_license(license)
+
+
+# Add STATIC_LINK relationship for static dependencies of a file
+def add_static_deps_of_file(file_id, file_metadata, doc):
+ if not file_metadata['static_dep_files'] and not file_metadata['whole_static_dep_files']:
+ return
+ static_dep_files = []
+ if file_metadata['static_dep_files']:
+ static_dep_files += file_metadata['static_dep_files'].split(' ')
+ if file_metadata['whole_static_dep_files']:
+ static_dep_files += file_metadata['whole_static_dep_files'].split(' ')
+
+ for dep_file in static_dep_files:
+ # Static libs are not shipped on devices, so names are derived from .intermediates paths.
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.STATIC_LINK,
+ id2=new_file_id(
+ dep_file.removeprefix(args.soong_out + '/.intermediates/'))))
+
+
+def add_licenses_of_file(file_id, file_metadata, doc):
+ lics = db.get_module_licenses(file_metadata.get('name', ''), file_metadata['module_path'])
+ if lics:
+ file = next(f for f in doc.files if file_id == f.id)
+ for license_name, license_files in lics.items():
+ if not license_files:
+ continue
+ license_id = new_license_id(license_name)
+ file.concluded_license_ids.append(license_id)
+ if license_name not in licenses_text:
+ license_text = get_license_text(license_files.split(' '))
+ licenses_text[license_name] = license_text
+
+ doc.add_license(sbom_data.License(id=license_id, name=license_name, text=licenses_text[license_name]))
+
+
+def get_all_transitive_static_dep_files_of_installed_files(installed_files_metadata, db, report):
+ # Find all transitive static dep files of all installed files
+ q = queue.Queue()
+ for installed_file_metadata in installed_files_metadata:
+ if installed_file_metadata['static_dep_files']:
+ for f in installed_file_metadata['static_dep_files'].split(' '):
+ q.put(f)
+ if installed_file_metadata['whole_static_dep_files']:
+ for f in installed_file_metadata['whole_static_dep_files'].split(' '):
+ q.put(f)
+
+ all_static_dep_files = {}
+ while not q.empty():
+ dep_file = q.get()
+ if dep_file in all_static_dep_files:
+ # It has been processed
+ continue
+
+ all_static_dep_files[dep_file] = True
+ soong_module = db.get_soong_module_of_built_file(dep_file)
+ if not soong_module:
+ # This should not happen, add to report[ISSUE_NO_MODULE_FOUND_FOR_STATIC_DEP]
+ report[ISSUE_NO_MODULE_FOUND_FOR_STATIC_DEP].append(f)
+ continue
+
+ if soong_module['static_dep_files']:
+ for f in soong_module['static_dep_files'].split(' '):
+ if f not in all_static_dep_files:
+ q.put(f)
+ if soong_module['whole_static_dep_files']:
+ for f in soong_module['whole_static_dep_files'].split(' '):
+ if f not in all_static_dep_files:
+ q.put(f)
+
+ return sorted(all_static_dep_files.keys())
+
+
+class MetadataDb:
+ def __init__(self, db):
+ self.conn = sqlite3.connect(':memory')
+ self.conn.row_factory = sqlite3.Row
+ with sqlite3.connect(db) as c:
+ c.backup(self.conn)
+ self.reorg()
+
+ def reorg(self):
+ # package_license table
+ self.conn.execute("create table package_license as "
+ "select name as package, pkg_default_applicable_licenses as license "
+ "from modules "
+ "where module_type = 'package' ")
+ cursor = self.conn.execute("select package,license from package_license where license like '% %'")
+ multi_licenses_packages = cursor.fetchall()
+ cursor.close()
+ rows = []
+ for p in multi_licenses_packages:
+ licenses = p['license'].strip().split(' ')
+ for lic in licenses:
+ rows.append((p['package'], lic))
+ self.conn.executemany('insert into package_license values (?, ?)', rows)
+ self.conn.commit()
+
+ self.conn.execute("delete from package_license where license like '% %'")
+ self.conn.commit()
+
+ # module_license table
+ self.conn.execute("create table module_license as "
+ "select distinct name as module, package, licenses as license "
+ "from modules "
+ "where licenses != '' ")
+ cursor = self.conn.execute("select module,package,license from module_license where license like '% %'")
+ multi_licenses_modules = cursor.fetchall()
+ cursor.close()
+ rows = []
+ for m in multi_licenses_modules:
+ licenses = m['license'].strip().split(' ')
+ for lic in licenses:
+ rows.append((m['module'], m['package'],lic))
+ self.conn.executemany('insert into module_license values (?, ?, ?)', rows)
+ self.conn.commit()
+
+ self.conn.execute("delete from module_license where license like '% %'")
+ self.conn.commit()
+
+ # module_installed_file table
+ self.conn.execute("create table module_installed_file as "
+ "select id as module_id, name as module_name, package, installed_files as installed_file "
+ "from modules "
+ "where installed_files != '' ")
+ cursor = self.conn.execute("select module_id, module_name, package, installed_file "
+ "from module_installed_file where installed_file like '% %'")
+ multi_installed_file_modules = cursor.fetchall()
+ cursor.close()
+ rows = []
+ for m in multi_installed_file_modules:
+ installed_files = m['installed_file'].strip().split(' ')
+ for f in installed_files:
+ rows.append((m['module_id'], m['module_name'], m['package'], f))
+ self.conn.executemany('insert into module_installed_file values (?, ?, ?, ?)', rows)
+ self.conn.commit()
+
+ self.conn.execute("delete from module_installed_file where installed_file like '% %'")
+ self.conn.commit()
+
+ # module_built_file table
+ self.conn.execute("create table module_built_file as "
+ "select id as module_id, name as module_name, package, built_files as built_file "
+ "from modules "
+ "where built_files != '' ")
+ cursor = self.conn.execute("select module_id, module_name, package, built_file "
+ "from module_built_file where built_file like '% %'")
+ multi_built_file_modules = cursor.fetchall()
+ cursor.close()
+ rows = []
+ for m in multi_built_file_modules:
+ built_files = m['installed_file'].strip().split(' ')
+ for f in built_files:
+ rows.append((m['module_id'], m['module_name'], m['package'], f))
+ self.conn.executemany('insert into module_built_file values (?, ?, ?, ?)', rows)
+ self.conn.commit()
+
+ self.conn.execute("delete from module_built_file where built_file like '% %'")
+ self.conn.commit()
+
+
+ # Indexes
+ self.conn.execute('create index idx_modules_id on modules (id)')
+ self.conn.execute('create index idx_modules_name on modules (name)')
+ self.conn.execute('create index idx_package_licnese_package on package_license (package)')
+ self.conn.execute('create index idx_package_licnese_license on package_license (license)')
+ self.conn.execute('create index idx_module_licnese_module on module_license (module)')
+ self.conn.execute('create index idx_module_licnese_license on module_license (license)')
+ self.conn.execute('create index idx_module_installed_file_module_id on module_installed_file (module_id)')
+ self.conn.execute('create index idx_module_installed_file_installed_file on module_installed_file (installed_file)')
+ self.conn.execute('create index idx_module_built_file_module_id on module_built_file (module_id)')
+ self.conn.execute('create index idx_module_built_file_built_file on module_built_file (built_file)')
+ self.conn.commit()
+
+ if args.debug:
+ with sqlite3.connect(os.path.dirname(args.metadata) + '/compliance-metadata-debug.db') as c:
+ self.conn.backup(c)
+
+
+ def get_installed_files(self):
+ # Get all records from table make_metadata, which contains all installed files and corresponding make modules' metadata
+ cursor = self.conn.execute('select installed_file, module_path, is_prebuilt_make_module, product_copy_files, kernel_module_copy_files, is_platform_generated, license_text from make_metadata')
+ rows = cursor.fetchall()
+ cursor.close()
+ installed_files_metadata = []
+ for row in rows:
+ metadata = dict(zip(row.keys(), row))
+ installed_files_metadata.append(metadata)
+ return installed_files_metadata
+
+ def get_soong_modules(self):
+ # Get all records from table modules, which contains metadata of all soong modules
+ cursor = self.conn.execute('select name, package, package as module_path, module_type as soong_module_type, built_files, installed_files, static_dep_files, whole_static_dep_files from modules')
+ rows = cursor.fetchall()
+ cursor.close()
+ soong_modules = []
+ for row in rows:
+ soong_module = dict(zip(row.keys(), row))
+ soong_modules.append(soong_module)
+ return soong_modules
+
+ def get_package_licenses(self, package):
+ cursor = self.conn.execute('select m.name, m.package, m.lic_license_text as license_text '
+ 'from package_license pl join modules m on pl.license = m.name '
+ 'where pl.package = ?',
+ ('//' + package,))
+ rows = cursor.fetchall()
+ licenses = {}
+ for r in rows:
+ licenses[r['name']] = r['license_text']
+ return licenses
+
+ def get_module_licenses(self, module_name, package):
+ licenses = {}
+ # If property "licenses" is defined on module
+ cursor = self.conn.execute('select m.name, m.package, m.lic_license_text as license_text '
+ 'from module_license ml join modules m on ml.license = m.name '
+ 'where ml.module = ? and ml.package = ?',
+ (module_name, package))
+ rows = cursor.fetchall()
+ for r in rows:
+ licenses[r['name']] = r['license_text']
+ if len(licenses) > 0:
+ return licenses
+
+ # Use default package license
+ cursor = self.conn.execute('select m.name, m.package, m.lic_license_text as license_text '
+ 'from package_license pl join modules m on pl.license = m.name '
+ 'where pl.package = ?',
+ ('//' + package,))
+ rows = cursor.fetchall()
+ for r in rows:
+ licenses[r['name']] = r['license_text']
+ return licenses
+
+ def get_soong_module_of_installed_file(self, installed_file):
+ cursor = self.conn.execute('select name, m.package, m.package as module_path, module_type as soong_module_type, built_files, installed_files, static_dep_files, whole_static_dep_files '
+ 'from modules m join module_installed_file mif on m.id = mif.module_id '
+ 'where mif.installed_file = ?',
+ (installed_file,))
+ rows = cursor.fetchall()
+ cursor.close()
+ if rows:
+ soong_module = dict(zip(rows[0].keys(), rows[0]))
+ return soong_module
+
+ return None
+
+ def get_soong_module_of_built_file(self, built_file):
+ cursor = self.conn.execute('select name, m.package, m.package as module_path, module_type as soong_module_type, built_files, installed_files, static_dep_files, whole_static_dep_files '
+ 'from modules m join module_built_file mbf on m.id = mbf.module_id '
+ 'where mbf.built_file = ?',
+ (built_file,))
+ rows = cursor.fetchall()
+ cursor.close()
+ if rows:
+ soong_module = dict(zip(rows[0].keys(), rows[0]))
+ return soong_module
+
+ return None
+
+
+def main():
+ global args
+ args = get_args()
+ log('Args:', vars(args))
+
+ global db
+ db = MetadataDb(args.metadata)
+ global metadata_file_protos
+ metadata_file_protos = {}
+ global licenses_text
+ licenses_text = {}
+
+ product_package_id = sbom_data.SPDXID_PRODUCT
+ product_package_name = sbom_data.PACKAGE_NAME_PRODUCT
+ product_package = sbom_data.Package(id=product_package_id,
+ name=product_package_name,
+ download_location=sbom_data.VALUE_NONE,
+ version=args.build_version,
+ supplier='Organization: ' + args.product_mfr,
+ files_analyzed=True)
+ doc_name = args.build_version
+ doc = sbom_data.Document(name=doc_name,
+ namespace=f'https://www.google.com/sbom/spdx/android/{doc_name}',
+ creators=['Organization: ' + args.product_mfr],
+ describes=product_package_id)
+
+ doc.packages.append(product_package)
+ doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,
+ name=sbom_data.PACKAGE_NAME_PLATFORM,
+ download_location=sbom_data.VALUE_NONE,
+ version=args.build_version,
+ supplier='Organization: ' + args.product_mfr,
+ declared_license_ids=[sbom_data.SPDXID_LICENSE_APACHE]))
+
+ # Report on some issues and information
+ report = {
+ ISSUE_NO_METADATA: [],
+ ISSUE_NO_METADATA_FILE: [],
+ ISSUE_METADATA_FILE_INCOMPLETE: [],
+ ISSUE_UNKNOWN_SECURITY_TAG_TYPE: [],
+ ISSUE_INSTALLED_FILE_NOT_EXIST: [],
+ ISSUE_NO_MODULE_FOUND_FOR_STATIC_DEP: [],
+ INFO_METADATA_FOUND_FOR_PACKAGE: [],
+ }
+
+ # Get installed files and corresponding make modules' metadata if an installed file is from a make module.
+ installed_files_metadata = db.get_installed_files()
+
+ # Find which Soong module an installed file is from and merge metadata from Make and Soong
+ for installed_file_metadata in installed_files_metadata:
+ soong_module = db.get_soong_module_of_installed_file(installed_file_metadata['installed_file'])
+ if soong_module:
+ # Merge soong metadata to make metadata
+ installed_file_metadata.update(soong_module)
+ else:
+ # For make modules soong_module_type should be empty
+ installed_file_metadata['soong_module_type'] = ''
+ installed_file_metadata['static_dep_files'] = ''
+ installed_file_metadata['whole_static_dep_files'] = ''
+
+ # Scan the metadata and create the corresponding package and file records in SPDX
+ for installed_file_metadata in installed_files_metadata:
+ installed_file = installed_file_metadata['installed_file']
+ module_path = installed_file_metadata['module_path']
+ product_copy_files = installed_file_metadata['product_copy_files']
+ kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
+ build_output_path = installed_file
+ installed_file = installed_file.removeprefix(args.product_out)
+
+ if not installed_file_has_metadata(installed_file_metadata, report):
+ continue
+ if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
+ report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)
+ continue
+
+ file_id = new_file_id(installed_file)
+ sha1 = checksum(build_output_path)
+ f = sbom_data.File(id=file_id, name=installed_file, checksum=sha1)
+ doc.files.append(f)
+ product_package.file_ids.append(file_id)
+
+ if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):
+ add_package_of_file(file_id, installed_file_metadata, doc, report)
+
+ elif module_path or installed_file_metadata['is_platform_generated']:
+ # File from PLATFORM package
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=sbom_data.SPDXID_PLATFORM))
+ if installed_file_metadata['is_platform_generated']:
+ f.concluded_license_ids = [sbom_data.SPDXID_LICENSE_APACHE]
+
+ elif product_copy_files:
+ # Format of product_copy_files: <source path>:<dest path>
+ src_path = product_copy_files.split(':')[0]
+ # So far product_copy_files are copied from directory system, kernel, hardware, frameworks and device,
+ # so process them as files from PLATFORM package
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=sbom_data.SPDXID_PLATFORM))
+ if installed_file_metadata['license_text']:
+ if installed_file_metadata['license_text'] == 'build/soong/licenses/LICENSE':
+ f.concluded_license_ids = [sbom_data.SPDXID_LICENSE_APACHE]
+
+ elif installed_file.endswith('.fsv_meta'):
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=sbom_data.SPDXID_PLATFORM))
+ f.concluded_license_ids = [sbom_data.SPDXID_LICENSE_APACHE]
+
+ elif kernel_module_copy_files.startswith('ANDROID-GEN'):
+ # For the four files generated for _dlkm, _ramdisk partitions
+ doc.add_relationship(sbom_data.Relationship(id1=file_id,
+ relationship=sbom_data.RelationshipType.GENERATED_FROM,
+ id2=sbom_data.SPDXID_PLATFORM))
+
+ # Process static dependencies of the installed file
+ add_static_deps_of_file(file_id, installed_file_metadata, doc)
+
+ # Add licenses of the installed file
+ add_licenses_of_file(file_id, installed_file_metadata, doc)
+
+ # Add all static library files to SBOM
+ for dep_file in get_all_transitive_static_dep_files_of_installed_files(installed_files_metadata, db, report):
+ filepath = dep_file.removeprefix(args.soong_out + '/.intermediates/')
+ file_id = new_file_id(filepath)
+ # SHA1 of empty string. Sometimes .a files might not be built.
+ sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709'
+ if os.path.islink(dep_file) or os.path.isfile(dep_file):
+ sha1 = checksum(dep_file)
+ doc.files.append(sbom_data.File(id=file_id,
+ name=filepath,
+ checksum=sha1))
+ file_metadata = {
+ 'installed_file': dep_file,
+ 'is_prebuilt_make_module': False
+ }
+ file_metadata.update(db.get_soong_module_of_built_file(dep_file))
+ add_package_of_file(file_id, file_metadata, doc, report)
+
+ # Add relationships for static deps of static libraries
+ add_static_deps_of_file(file_id, file_metadata, doc)
+
+ # Add licenses of the static lib
+ add_licenses_of_file(file_id, file_metadata, doc)
+
+ # Save SBOM records to output file
+ doc.generate_packages_verification_code()
+ doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+ prefix = args.output_file
+ if prefix.endswith('.spdx'):
+ prefix = prefix.removesuffix('.spdx')
+ elif prefix.endswith('.spdx.json'):
+ prefix = prefix.removesuffix('.spdx.json')
+
+ output_file = prefix + '.spdx'
+ with open(output_file, 'w', encoding="utf-8") as file:
+ sbom_writers.TagValueWriter.write(doc, file)
+ if args.json:
+ with open(prefix + '.spdx.json', 'w', encoding="utf-8") as file:
+ sbom_writers.JSONWriter.write(doc, file)
+
+ save_report(prefix + '-gen-report.txt', report)
+
+
+if __name__ == '__main__':
+ main()