Merge "Revert "Revert "Support SBOM generation of m build unbundled APEXs.""" into udc-dev am: 56d4b75675
Original change: https://googleplex-android-review.googlesource.com/c/platform/build/+/24032007
Change-Id: I8d252b65fa6c79b7cd5634a2eb5b50005209e892
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
diff --git a/core/config_sanitizers.mk b/core/config_sanitizers.mk
index ebce4c2..b1d797e 100644
--- a/core/config_sanitizers.mk
+++ b/core/config_sanitizers.mk
@@ -140,6 +140,10 @@
$(PRODUCT_MEMTAG_HEAP_ASYNC_INCLUDE_PATHS)
combined_exclude_paths := $(MEMTAG_HEAP_EXCLUDE_PATHS) \
$(PRODUCT_MEMTAG_HEAP_EXCLUDE_PATHS)
+ ifneq ($(PRODUCT_MEMTAG_HEAP_SKIP_DEFAULT_PATHS),true)
+ combined_sync_include_paths += $(PRODUCT_MEMTAG_HEAP_SYNC_DEFAULT_INCLUDE_PATHS)
+ combined_async_include_paths += $(PRODUCT_MEMTAG_HEAP_ASYNC_DEFAULT_INCLUDE_PATHS)
+ endif
ifeq ($(strip $(foreach dir,$(subst $(comma),$(space),$(combined_exclude_paths)),\
$(filter $(dir)%,$(LOCAL_PATH)))),)
diff --git a/core/product.mk b/core/product.mk
index e90e27b..30d7dcd 100644
--- a/core/product.mk
+++ b/core/product.mk
@@ -247,6 +247,16 @@
# Whether any paths should have HWASan enabled for components
_product_list_vars += PRODUCT_HWASAN_INCLUDE_PATHS
+# Whether any paths should have Memtag_heap enabled for components
+_product_list_vars += PRODUCT_MEMTAG_HEAP_ASYNC_INCLUDE_PATHS
+_product_list_vars += PRODUCT_MEMTAG_HEAP_ASYNC_DEFAULT_INCLUDE_PATHS
+_product_list_vars += PRODUCT_MEMTAG_HEAP_SYNC_INCLUDE_PATHS
+_product_list_vars += PRODUCT_MEMTAG_HEAP_SYNC_DEFAULT_INCLUDE_PATHS
+_product_list_vars += PRODUCT_MEMTAG_HEAP_EXCLUDE_PATHS
+
+# Whether this product wants to start with an empty list of default memtag_heap include paths
+_product_single_value_vars += PRODUCT_MEMTAG_HEAP_SKIP_DEFAULT_PATHS
+
# Whether the Scudo hardened allocator is disabled platform-wide
_product_single_value_vars += PRODUCT_DISABLE_SCUDO
diff --git a/core/soong_config.mk b/core/soong_config.mk
index a149e2a..e9a6b64 100644
--- a/core/soong_config.mk
+++ b/core/soong_config.mk
@@ -123,8 +123,8 @@
$(call add_json_list, HWASanIncludePaths, $(HWASAN_INCLUDE_PATHS) $(PRODUCT_HWASAN_INCLUDE_PATHS))
$(call add_json_list, MemtagHeapExcludePaths, $(MEMTAG_HEAP_EXCLUDE_PATHS) $(PRODUCT_MEMTAG_HEAP_EXCLUDE_PATHS))
-$(call add_json_list, MemtagHeapAsyncIncludePaths, $(MEMTAG_HEAP_ASYNC_INCLUDE_PATHS) $(PRODUCT_MEMTAG_HEAP_ASYNC_INCLUDE_PATHS))
-$(call add_json_list, MemtagHeapSyncIncludePaths, $(MEMTAG_HEAP_SYNC_INCLUDE_PATHS) $(PRODUCT_MEMTAG_HEAP_SYNC_INCLUDE_PATHS))
+$(call add_json_list, MemtagHeapAsyncIncludePaths, $(MEMTAG_HEAP_ASYNC_INCLUDE_PATHS) $(PRODUCT_MEMTAG_HEAP_ASYNC_INCLUDE_PATHS) $(if $(filter true,$(PRODUCT_MEMTAG_HEAP_SKIP_DEFAULT_PATHS)),,$(PRODUCT_MEMTAG_HEAP_ASYNC_DEFAULT_INCLUDE_PATHS)))
+$(call add_json_list, MemtagHeapSyncIncludePaths, $(MEMTAG_HEAP_SYNC_INCLUDE_PATHS) $(PRODUCT_MEMTAG_HEAP_SYNC_INCLUDE_PATHS) $(if $(filter true,$(PRODUCT_MEMTAG_HEAP_SKIP_DEFAULT_PATHS)),,$(PRODUCT_MEMTAG_HEAP_SYNC_DEFAULT_INCLUDE_PATHS)))
$(call add_json_bool, DisableScudo, $(filter true,$(PRODUCT_DISABLE_SCUDO)))
diff --git a/target/product/media_system.mk b/target/product/media_system.mk
index 79bd74a..38ba219 100644
--- a/target/product/media_system.mk
+++ b/target/product/media_system.mk
@@ -76,3 +76,7 @@
# Enable CFI for security-sensitive components
$(call inherit-product, $(SRC_TARGET_DIR)/product/cfi-common.mk)
$(call inherit-product-if-exists, vendor/google/products/cfi-vendor.mk)
+
+# Enable MTE for security-sensitive components
+$(call inherit-product, $(SRC_TARGET_DIR)/product/memtag-common.mk)
+$(call inherit-product-if-exists, vendor/google/products/memtag-vendor.mk)
diff --git a/target/product/memtag-common.mk b/target/product/memtag-common.mk
new file mode 100644
index 0000000..829cb41
--- /dev/null
+++ b/target/product/memtag-common.mk
@@ -0,0 +1,30 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is a recommended set of common components to enable MTE for.
+
+PRODUCT_MEMTAG_HEAP_ASYNC_DEFAULT_INCLUDE_PATHS := \
+ external/android-clat \
+ external/iproute2 \
+ external/iptables \
+ external/mtpd \
+ external/ppp \
+ hardware/st/nfc \
+ hardware/st/secure_element \
+ hardware/st/secure_element2 \
+ packages/modules/StatsD \
+ system/bpf \
+ system/netd/netutil_wrappers \
+ system/netd/server
diff --git a/target/product/module_common.mk b/target/product/module_common.mk
index 84bd799..53b2ca6 100644
--- a/target/product/module_common.mk
+++ b/target/product/module_common.mk
@@ -17,6 +17,7 @@
$(call inherit-product, $(SRC_TARGET_DIR)/product/default_art_config.mk)
$(call inherit-product, $(SRC_TARGET_DIR)/product/languages_default.mk)
$(call inherit-product, $(SRC_TARGET_DIR)/product/cfi-common.mk)
+$(call inherit-product, $(SRC_TARGET_DIR)/product/memtag-common.mk)
# Enables treble, which enabled certain -D compilation flags. In particular, libhidlbase
# uses -DENFORCE_VINTF_MANIFEST. See b/185759877
diff --git a/tools/releasetools/check_target_files_signatures.py b/tools/releasetools/check_target_files_signatures.py
index d935607..a7b3523 100755
--- a/tools/releasetools/check_target_files_signatures.py
+++ b/tools/releasetools/check_target_files_signatures.py
@@ -241,7 +241,8 @@
# Signer (minSdkVersion=24, maxSdkVersion=32) certificate SHA-1 digest: 19da94896ce4078c38ca695701f1dec741ec6d67
# ...
certs_info = {}
- certificate_regex = re.compile(r"(Signer (?:#[0-9]+|\(.*\))) (certificate .*):(.*)")
+ certificate_regex = re.compile(
+ r"(Signer (?:#[0-9]+|\(.*\))) (certificate .*):(.*)")
for line in output.splitlines():
m = certificate_regex.match(line)
if not m:
@@ -312,7 +313,7 @@
# This is the list of wildcards of files we extract from |filename|.
apk_extensions = ['*.apk', '*.apex']
- with zipfile.ZipFile(filename) as input_zip:
+ with zipfile.ZipFile(filename, "r") as input_zip:
self.certmap, compressed_extension = common.ReadApkCerts(input_zip)
if compressed_extension:
apk_extensions.append('*.apk' + compressed_extension)
diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py
index b50caaa..abedecf 100644
--- a/tools/releasetools/common.py
+++ b/tools/releasetools/common.py
@@ -35,6 +35,7 @@
import shutil
import subprocess
import sys
+import stat
import tempfile
import threading
import time
@@ -2102,6 +2103,26 @@
shutil.copyfileobj(in_file, out_file)
+def UnzipSingleFile(input_zip: zipfile.ZipFile, info: zipfile.ZipInfo, dirname: str):
+ # According to https://stackoverflow.com/questions/434641/how-do-i-set-permissions-attributes-on-a-file-in-a-zip-file-using-pythons-zip/6297838#6297838
+ # higher bits of |external_attr| are unix file permission and types
+ unix_filetype = info.external_attr >> 16
+
+ def CheckMask(a, mask):
+ return (a & mask) == mask
+
+ def IsSymlink(a):
+ return CheckMask(a, stat.S_IFLNK)
+ # python3.11 zipfile implementation doesn't handle symlink correctly
+ if not IsSymlink(unix_filetype):
+ return input_zip.extract(info, dirname)
+ if dirname is None:
+ dirname = os.getcwd()
+ target = os.path.join(dirname, info.filename)
+ os.makedirs(os.path.dirname(target), exist_ok=True)
+ os.symlink(input_zip.read(info).decode(), target)
+
+
def UnzipToDir(filename, dirname, patterns=None):
"""Unzips the archive to the given directory.
@@ -2112,20 +2133,46 @@
archvie. Non-matching patterns will be filtered out. If there's no match
after the filtering, no file will be unzipped.
"""
- cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
- if patterns is not None:
+ with zipfile.ZipFile(filename, allowZip64=True, mode="r") as input_zip:
# Filter out non-matching patterns. unzip will complain otherwise.
- with zipfile.ZipFile(filename, allowZip64=True) as input_zip:
- names = input_zip.namelist()
- filtered = [
- pattern for pattern in patterns if fnmatch.filter(names, pattern)]
+ entries = input_zip.infolist()
+ # b/283033491
+ # Per https://en.wikipedia.org/wiki/ZIP_(file_format)#Central_directory_file_header
+ # In zip64 mode, central directory record's header_offset field might be
+ # set to 0xFFFFFFFF if header offset is > 2^32. In this case, the extra
+ # fields will contain an 8 byte little endian integer at offset 20
+ # to indicate the actual local header offset.
+ # As of python3.11, python does not handle zip64 central directories
+ # correctly, so we will manually do the parsing here.
- # There isn't any matching files. Don't unzip anything.
- if not filtered:
- return
- cmd.extend(filtered)
+ # ZIP64 central directory extra field has two required fields:
+ # 2 bytes header ID and 2 bytes size field. Thes two require fields have
+ # a total size of 4 bytes. Then it has three other 8 bytes field, followed
+ # by a 4 byte disk number field. The last disk number field is not required
+ # to be present, but if it is present, the total size of extra field will be
+ # divisible by 8(because 2+2+4+8*n is always going to be multiple of 8)
+ # Most extra fields are optional, but when they appear, their must appear
+ # in the order defined by zip64 spec. Since file header offset is the 2nd
+ # to last field in zip64 spec, it will only be at last 8 bytes or last 12-4
+ # bytes, depending on whether disk number is present.
+ for entry in entries:
+ if entry.header_offset == 0xFFFFFFFF:
+ if len(entry.extra) % 8 == 0:
+ entry.header_offset = int.from_bytes(entry.extra[-12:-4], "little")
+ else:
+ entry.header_offset = int.from_bytes(entry.extra[-8:], "little")
+ if patterns is not None:
+ filtered = [info for info in entries if any(
+ [fnmatch.fnmatch(info.filename, p) for p in patterns])]
- RunAndCheckOutput(cmd)
+ # There isn't any matching files. Don't unzip anything.
+ if not filtered:
+ return
+ for info in filtered:
+ UnzipSingleFile(input_zip, info, dirname)
+ else:
+ for info in entries:
+ UnzipSingleFile(input_zip, info, dirname)
def UnzipTemp(filename, patterns=None):