Merge changes from topic "cherrypicker-L80200000962011019:N58900001388333251" into udc-dev
* changes:
Handle symlinks when extracting zipfiles
Handle zip64 extra fields better
Fix python3.11's support for zip64
Use python based unzip function for portability
diff --git a/tools/releasetools/check_target_files_signatures.py b/tools/releasetools/check_target_files_signatures.py
index d935607..a7b3523 100755
--- a/tools/releasetools/check_target_files_signatures.py
+++ b/tools/releasetools/check_target_files_signatures.py
@@ -241,7 +241,8 @@
# Signer (minSdkVersion=24, maxSdkVersion=32) certificate SHA-1 digest: 19da94896ce4078c38ca695701f1dec741ec6d67
# ...
certs_info = {}
- certificate_regex = re.compile(r"(Signer (?:#[0-9]+|\(.*\))) (certificate .*):(.*)")
+ certificate_regex = re.compile(
+ r"(Signer (?:#[0-9]+|\(.*\))) (certificate .*):(.*)")
for line in output.splitlines():
m = certificate_regex.match(line)
if not m:
@@ -312,7 +313,7 @@
# This is the list of wildcards of files we extract from |filename|.
apk_extensions = ['*.apk', '*.apex']
- with zipfile.ZipFile(filename) as input_zip:
+ with zipfile.ZipFile(filename, "r") as input_zip:
self.certmap, compressed_extension = common.ReadApkCerts(input_zip)
if compressed_extension:
apk_extensions.append('*.apk' + compressed_extension)
diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py
index b50caaa..abedecf 100644
--- a/tools/releasetools/common.py
+++ b/tools/releasetools/common.py
@@ -35,6 +35,7 @@
import shutil
import subprocess
import sys
+import stat
import tempfile
import threading
import time
@@ -2102,6 +2103,26 @@
shutil.copyfileobj(in_file, out_file)
+def UnzipSingleFile(input_zip: zipfile.ZipFile, info: zipfile.ZipInfo, dirname: str):
+ # According to https://stackoverflow.com/questions/434641/how-do-i-set-permissions-attributes-on-a-file-in-a-zip-file-using-pythons-zip/6297838#6297838
+ # higher bits of |external_attr| are unix file permission and types
+ unix_filetype = info.external_attr >> 16
+
+ def CheckMask(a, mask):
+ return (a & mask) == mask
+
+ def IsSymlink(a):
+ return CheckMask(a, stat.S_IFLNK)
+ # python3.11 zipfile implementation doesn't handle symlink correctly
+ if not IsSymlink(unix_filetype):
+ return input_zip.extract(info, dirname)
+ if dirname is None:
+ dirname = os.getcwd()
+ target = os.path.join(dirname, info.filename)
+ os.makedirs(os.path.dirname(target), exist_ok=True)
+ os.symlink(input_zip.read(info).decode(), target)
+
+
def UnzipToDir(filename, dirname, patterns=None):
"""Unzips the archive to the given directory.
@@ -2112,20 +2133,46 @@
archvie. Non-matching patterns will be filtered out. If there's no match
after the filtering, no file will be unzipped.
"""
- cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
- if patterns is not None:
+ with zipfile.ZipFile(filename, allowZip64=True, mode="r") as input_zip:
# Filter out non-matching patterns. unzip will complain otherwise.
- with zipfile.ZipFile(filename, allowZip64=True) as input_zip:
- names = input_zip.namelist()
- filtered = [
- pattern for pattern in patterns if fnmatch.filter(names, pattern)]
+ entries = input_zip.infolist()
+ # b/283033491
+ # Per https://en.wikipedia.org/wiki/ZIP_(file_format)#Central_directory_file_header
+ # In zip64 mode, central directory record's header_offset field might be
+ # set to 0xFFFFFFFF if header offset is > 2^32. In this case, the extra
+ # fields will contain an 8 byte little endian integer at offset 20
+ # to indicate the actual local header offset.
+ # As of python3.11, python does not handle zip64 central directories
+ # correctly, so we will manually do the parsing here.
- # There isn't any matching files. Don't unzip anything.
- if not filtered:
- return
- cmd.extend(filtered)
+ # ZIP64 central directory extra field has two required fields:
+ # 2 bytes header ID and 2 bytes size field. Thes two require fields have
+ # a total size of 4 bytes. Then it has three other 8 bytes field, followed
+ # by a 4 byte disk number field. The last disk number field is not required
+ # to be present, but if it is present, the total size of extra field will be
+ # divisible by 8(because 2+2+4+8*n is always going to be multiple of 8)
+ # Most extra fields are optional, but when they appear, their must appear
+ # in the order defined by zip64 spec. Since file header offset is the 2nd
+ # to last field in zip64 spec, it will only be at last 8 bytes or last 12-4
+ # bytes, depending on whether disk number is present.
+ for entry in entries:
+ if entry.header_offset == 0xFFFFFFFF:
+ if len(entry.extra) % 8 == 0:
+ entry.header_offset = int.from_bytes(entry.extra[-12:-4], "little")
+ else:
+ entry.header_offset = int.from_bytes(entry.extra[-8:], "little")
+ if patterns is not None:
+ filtered = [info for info in entries if any(
+ [fnmatch.fnmatch(info.filename, p) for p in patterns])]
- RunAndCheckOutput(cmd)
+ # There isn't any matching files. Don't unzip anything.
+ if not filtered:
+ return
+ for info in filtered:
+ UnzipSingleFile(input_zip, info, dirname)
+ else:
+ for info in entries:
+ UnzipSingleFile(input_zip, info, dirname)
def UnzipTemp(filename, patterns=None):