Merge "Replace extract_srcjars.sh with zipsync"
diff --git a/core/config.mk b/core/config.mk
index ec3c847..909b8b2 100644
--- a/core/config.mk
+++ b/core/config.mk
@@ -553,7 +553,7 @@
# Default R8 behavior when USE_R8 is not specified.
ifndef USE_R8
- USE_R8 := false
+ USE_R8 := true
endif
#
diff --git a/target/board/generic/BoardConfig.mk b/target/board/generic/BoardConfig.mk
index 67d019f..70c78a8 100644
--- a/target/board/generic/BoardConfig.mk
+++ b/target/board/generic/BoardConfig.mk
@@ -68,3 +68,5 @@
# Enable A/B update
TARGET_NO_RECOVERY := true
BOARD_BUILD_SYSTEM_ROOT_IMAGE := true
+
+BOARD_VNDK_VERSION := current
\ No newline at end of file
diff --git a/target/board/generic_arm64/BoardConfig.mk b/target/board/generic_arm64/BoardConfig.mk
index e066e3a..be8ea39 100644
--- a/target/board/generic_arm64/BoardConfig.mk
+++ b/target/board/generic_arm64/BoardConfig.mk
@@ -99,3 +99,5 @@
# Enable A/B update
TARGET_NO_RECOVERY := true
BOARD_BUILD_SYSTEM_ROOT_IMAGE := true
+
+BOARD_VNDK_VERSION := current
diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py
index 7d266f6..aba43e2 100755
--- a/tools/releasetools/ota_from_target_files.py
+++ b/tools/releasetools/ota_from_target_files.py
@@ -159,6 +159,7 @@
import os.path
import shlex
import shutil
+import struct
import subprocess
import sys
import tempfile
@@ -704,7 +705,7 @@
AddCompatibilityArchive(system_updated, vendor_updated)
-def WriteFullOTAPackage(input_zip, output_zip):
+def WriteFullOTAPackage(input_zip, output_file):
target_info = BuildInfo(OPTIONS.info_dict, OPTIONS.oem_dicts)
# We don't know what version it will be installed on top of. We expect the API
@@ -718,6 +719,14 @@
metadata = GetPackageMetadata(target_info)
+ if not OPTIONS.no_signing:
+ staging_file = common.MakeTempFile(suffix='.zip')
+ else:
+ staging_file = output_file
+
+ output_zip = zipfile.ZipFile(
+ staging_file, "w", compression=zipfile.ZIP_DEFLATED)
+
device_specific = common.DeviceSpecificParams(
input_zip=input_zip,
input_version=target_api_version,
@@ -862,7 +871,15 @@
script.SetProgress(1)
script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary)
metadata["ota-required-cache"] = str(script.required_cache)
- WriteMetadata(metadata, output_zip)
+
+ # We haven't written the metadata entry, which will be done in
+ # FinalizeMetadata.
+ common.ZipClose(output_zip)
+
+ needed_property_files = (
+ NonAbOtaPropertyFiles(),
+ )
+ FinalizeMetadata(metadata, staging_file, output_file, needed_property_files)
def WriteMetadata(metadata, output_zip):
@@ -1063,6 +1080,7 @@
return '%s:%d:%d' % (os.path.basename(name), offset, size)
tokens = []
+ tokens.extend(self._GetPrecomputed(zip_file))
for entry in self.required:
tokens.append(ComputeEntryOffsetSize(entry))
for entry in self.optional:
@@ -1081,6 +1099,23 @@
return ','.join(tokens)
+ def _GetPrecomputed(self, input_zip):
+ """Computes the additional tokens to be included into the property-files.
+
+ This applies to tokens without actual ZIP entries, such as
+ payload_metadadata.bin. We want to expose the offset/size to updaters, so
+ that they can download the payload metadata directly with the info.
+
+ Args:
+ input_zip: The input zip file.
+
+ Returns:
+ A list of strings (tokens) to be added to the property-files string.
+ """
+ # pylint: disable=no-self-use
+ # pylint: disable=unused-argument
+ return []
+
class StreamingPropertyFiles(PropertyFiles):
"""A subclass for computing the property-files for streaming A/B OTAs."""
@@ -1101,6 +1136,102 @@
)
+class AbOtaPropertyFiles(StreamingPropertyFiles):
+ """The property-files for A/B OTA that includes payload_metadata.bin info.
+
+ Since P, we expose one more token (aka property-file), in addition to the ones
+ for streaming A/B OTA, for a virtual entry of 'payload_metadata.bin'.
+ 'payload_metadata.bin' is the header part of a payload ('payload.bin'), which
+ doesn't exist as a separate ZIP entry, but can be used to verify if the
+ payload can be applied on the given device.
+
+ For backward compatibility, we keep both of the 'ota-streaming-property-files'
+ and the newly added 'ota-property-files' in P. The new token will only be
+ available in 'ota-property-files'.
+ """
+
+ def __init__(self):
+ super(AbOtaPropertyFiles, self).__init__()
+ self.name = 'ota-property-files'
+
+ def _GetPrecomputed(self, input_zip):
+ offset, size = self._GetPayloadMetadataOffsetAndSize(input_zip)
+ return ['payload_metadata.bin:{}:{}'.format(offset, size)]
+
+ @staticmethod
+ def _GetPayloadMetadataOffsetAndSize(input_zip):
+ """Computes the offset and size of the payload metadata for a given package.
+
+ (From system/update_engine/update_metadata.proto)
+ A delta update file contains all the deltas needed to update a system from
+ one specific version to another specific version. The update format is
+ represented by this struct pseudocode:
+
+ struct delta_update_file {
+ char magic[4] = "CrAU";
+ uint64 file_format_version;
+ uint64 manifest_size; // Size of protobuf DeltaArchiveManifest
+
+ // Only present if format_version > 1:
+ uint32 metadata_signature_size;
+
+ // The Bzip2 compressed DeltaArchiveManifest
+ char manifest[metadata_signature_size];
+
+ // The signature of the metadata (from the beginning of the payload up to
+ // this location, not including the signature itself). This is a
+ // serialized Signatures message.
+ char medatada_signature_message[metadata_signature_size];
+
+ // Data blobs for files, no specific format. The specific offset
+ // and length of each data blob is recorded in the DeltaArchiveManifest.
+ struct {
+ char data[];
+ } blobs[];
+
+ // These two are not signed:
+ uint64 payload_signatures_message_size;
+ char payload_signatures_message[];
+ };
+
+ 'payload-metadata.bin' contains all the bytes from the beginning of the
+ payload, till the end of 'medatada_signature_message'.
+ """
+ payload_info = input_zip.getinfo('payload.bin')
+ payload_offset = payload_info.header_offset + len(payload_info.FileHeader())
+ payload_size = payload_info.file_size
+
+ with input_zip.open('payload.bin', 'r') as payload_fp:
+ header_bin = payload_fp.read(24)
+
+ # network byte order (big-endian)
+ header = struct.unpack("!IQQL", header_bin)
+
+ # 'CrAU'
+ magic = header[0]
+ assert magic == 0x43724155, "Invalid magic: {:x}".format(magic)
+
+ manifest_size = header[2]
+ metadata_signature_size = header[3]
+ metadata_total = 24 + manifest_size + metadata_signature_size
+ assert metadata_total < payload_size
+
+ return (payload_offset, metadata_total)
+
+
+class NonAbOtaPropertyFiles(PropertyFiles):
+ """The property-files for non-A/B OTA.
+
+ For non-A/B OTA, the property-files string contains the info for METADATA
+ entry, with which a system updater can be fetched the package metadata prior
+ to downloading the entire package.
+ """
+
+ def __init__(self):
+ super(NonAbOtaPropertyFiles, self).__init__()
+ self.name = 'ota-property-files'
+
+
def FinalizeMetadata(metadata, input_file, output_file, needed_property_files):
"""Finalizes the metadata and signs an A/B OTA package.
@@ -1135,8 +1266,11 @@
# signing (with an incomplete metadata entry) to allow that to happen. Then
# compute the ZIP entry offsets, write back the final metadata and do the
# final signing.
- prelim_signing = common.MakeTempFile(suffix='.zip')
- SignOutput(input_file, prelim_signing)
+ if OPTIONS.no_signing:
+ prelim_signing = input_file
+ else:
+ prelim_signing = common.MakeTempFile(suffix='.zip')
+ SignOutput(input_file, prelim_signing)
# Open the signed zip. Compute the final metadata that's needed for streaming.
with zipfile.ZipFile(prelim_signing, 'r') as prelim_signing_zip:
@@ -1152,7 +1286,10 @@
common.ZipClose(output_zip)
# Re-sign the package after updating the metadata entry.
- SignOutput(prelim_signing, output_file)
+ if OPTIONS.no_signing:
+ output_file = prelim_signing
+ else:
+ SignOutput(prelim_signing, output_file)
# Reopen the final signed zip to double check the streaming metadata.
with zipfile.ZipFile(output_file, 'r') as output_zip:
@@ -1160,7 +1297,7 @@
property_files.Verify(output_zip, metadata[property_files.name].strip())
-def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip):
+def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
target_info = BuildInfo(OPTIONS.target_info_dict, OPTIONS.oem_dicts)
source_info = BuildInfo(OPTIONS.source_info_dict, OPTIONS.oem_dicts)
@@ -1179,6 +1316,14 @@
metadata = GetPackageMetadata(target_info, source_info)
+ if not OPTIONS.no_signing:
+ staging_file = common.MakeTempFile(suffix='.zip')
+ else:
+ staging_file = output_file
+
+ output_zip = zipfile.ZipFile(
+ staging_file, "w", compression=zipfile.ZIP_DEFLATED)
+
device_specific = common.DeviceSpecificParams(
source_zip=source_zip,
source_version=source_api_version,
@@ -1428,7 +1573,16 @@
else:
script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary)
metadata["ota-required-cache"] = str(script.required_cache)
- WriteMetadata(metadata, output_zip)
+
+ # We haven't written the metadata entry yet, which will be handled in
+ # FinalizeMetadata().
+ common.ZipClose(output_zip)
+
+ # Sign the generated zip package unless no_signing is specified.
+ needed_property_files = (
+ NonAbOtaPropertyFiles(),
+ )
+ FinalizeMetadata(metadata, staging_file, output_file, needed_property_files)
def GetTargetFilesZipForSecondaryImages(input_file, skip_postinstall=False):
@@ -1508,7 +1662,10 @@
source_file=None):
"""Generates an Android OTA package that has A/B update payload."""
# Stage the output zip package for package signing.
- staging_file = common.MakeTempFile(suffix='.zip')
+ if not OPTIONS.no_signing:
+ staging_file = common.MakeTempFile(suffix='.zip')
+ else:
+ staging_file = output_file
output_zip = zipfile.ZipFile(staging_file, "w",
compression=zipfile.ZIP_DEFLATED)
@@ -1573,7 +1730,12 @@
# FinalizeMetadata().
common.ZipClose(output_zip)
+ # AbOtaPropertyFiles intends to replace StreamingPropertyFiles, as it covers
+ # all the info of the latter. However, system updaters and OTA servers need to
+ # take time to switch to the new flag. We keep both of the flags for
+ # P-timeframe, and will remove StreamingPropertyFiles in later release.
needed_property_files = (
+ AbOtaPropertyFiles(),
StreamingPropertyFiles(),
)
FinalizeMetadata(metadata, staging_file, output_file, needed_property_files)
@@ -1776,21 +1938,12 @@
if OPTIONS.device_specific is not None:
OPTIONS.device_specific = os.path.abspath(OPTIONS.device_specific)
- # Set up the output zip. Create a temporary zip file if signing is needed.
- if OPTIONS.no_signing:
- if os.path.exists(args[1]):
- os.unlink(args[1])
- output_zip = zipfile.ZipFile(args[1], "w",
- compression=zipfile.ZIP_DEFLATED)
- else:
- temp_zip_file = tempfile.NamedTemporaryFile()
- output_zip = zipfile.ZipFile(temp_zip_file, "w",
- compression=zipfile.ZIP_DEFLATED)
-
# Generate a full OTA.
if OPTIONS.incremental_source is None:
with zipfile.ZipFile(args[0], 'r') as input_zip:
- WriteFullOTAPackage(input_zip, output_zip)
+ WriteFullOTAPackage(
+ input_zip,
+ output_file=args[1])
# Generate an incremental OTA.
else:
@@ -1799,7 +1952,10 @@
OPTIONS.incremental_source, UNZIP_PATTERN)
with zipfile.ZipFile(args[0], 'r') as input_zip, \
zipfile.ZipFile(OPTIONS.incremental_source, 'r') as source_zip:
- WriteBlockIncrementalOTAPackage(input_zip, source_zip, output_zip)
+ WriteBlockIncrementalOTAPackage(
+ input_zip,
+ source_zip,
+ output_file=args[1])
if OPTIONS.log_diff:
with open(OPTIONS.log_diff, 'w') as out_file:
@@ -1807,13 +1963,6 @@
target_files_diff.recursiveDiff(
'', OPTIONS.source_tmp, OPTIONS.input_tmp, out_file)
- common.ZipClose(output_zip)
-
- # Sign the generated zip package unless no_signing is specified.
- if not OPTIONS.no_signing:
- SignOutput(temp_zip_file.name, args[1])
- temp_zip_file.close()
-
print("done.")
diff --git a/tools/releasetools/test_ota_from_target_files.py b/tools/releasetools/test_ota_from_target_files.py
index e2e85fd..97687e7 100644
--- a/tools/releasetools/test_ota_from_target_files.py
+++ b/tools/releasetools/test_ota_from_target_files.py
@@ -17,15 +17,16 @@
import copy
import os
import os.path
+import subprocess
import unittest
import zipfile
import common
import test_utils
from ota_from_target_files import (
- _LoadOemDicts, BuildInfo, GetPackageMetadata,
+ _LoadOemDicts, AbOtaPropertyFiles, BuildInfo, GetPackageMetadata,
GetTargetFilesZipForSecondaryImages,
- GetTargetFilesZipWithoutPostinstallConfig,
+ GetTargetFilesZipWithoutPostinstallConfig, NonAbOtaPropertyFiles,
Payload, PayloadSigner, POSTINSTALL_CONFIG, PropertyFiles,
StreamingPropertyFiles, WriteFingerprintAssertion)
@@ -842,6 +843,205 @@
AssertionError, property_files.Verify, zip_fp, raw_metadata + 'x')
+class AbOtaPropertyFilesTest(PropertyFilesTest):
+ """Additional sanity checks specialized for AbOtaPropertyFiles."""
+
+ # The size for payload and metadata signature size.
+ SIGNATURE_SIZE = 256
+
+ def setUp(self):
+ self.testdata_dir = test_utils.get_testdata_dir()
+ self.assertTrue(os.path.exists(self.testdata_dir))
+
+ common.OPTIONS.wipe_user_data = False
+ common.OPTIONS.payload_signer = None
+ common.OPTIONS.payload_signer_args = None
+ common.OPTIONS.package_key = os.path.join(self.testdata_dir, 'testkey')
+ common.OPTIONS.key_passwords = {
+ common.OPTIONS.package_key : None,
+ }
+
+ def test_init(self):
+ property_files = AbOtaPropertyFiles()
+ self.assertEqual('ota-property-files', property_files.name)
+ self.assertEqual(
+ (
+ 'payload.bin',
+ 'payload_properties.txt',
+ ),
+ property_files.required)
+ self.assertEqual(
+ (
+ 'care_map.txt',
+ 'compatibility.zip',
+ ),
+ property_files.optional)
+
+ def test_GetPayloadMetadataOffsetAndSize(self):
+ target_file = construct_target_files()
+ payload = Payload()
+ payload.Generate(target_file)
+
+ payload_signer = PayloadSigner()
+ payload.Sign(payload_signer)
+
+ output_file = common.MakeTempFile(suffix='.zip')
+ with zipfile.ZipFile(output_file, 'w') as output_zip:
+ payload.WriteToZip(output_zip)
+
+ # Find out the payload metadata offset and size.
+ property_files = AbOtaPropertyFiles()
+ with zipfile.ZipFile(output_file) as input_zip:
+ # pylint: disable=protected-access
+ payload_offset, metadata_total = (
+ property_files._GetPayloadMetadataOffsetAndSize(input_zip))
+
+ # Read in the metadata signature directly.
+ with open(output_file, 'rb') as verify_fp:
+ verify_fp.seek(payload_offset + metadata_total - self.SIGNATURE_SIZE)
+ metadata_signature = verify_fp.read(self.SIGNATURE_SIZE)
+
+ # Now we extract the metadata hash via brillo_update_payload script, which
+ # will serve as the oracle result.
+ payload_sig_file = common.MakeTempFile(prefix="sig-", suffix=".bin")
+ metadata_sig_file = common.MakeTempFile(prefix="sig-", suffix=".bin")
+ cmd = ['brillo_update_payload', 'hash',
+ '--unsigned_payload', payload.payload_file,
+ '--signature_size', str(self.SIGNATURE_SIZE),
+ '--metadata_hash_file', metadata_sig_file,
+ '--payload_hash_file', payload_sig_file]
+ proc = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ stdoutdata, _ = proc.communicate()
+ self.assertEqual(
+ 0, proc.returncode,
+ 'Failed to run brillo_update_payload: {}'.format(stdoutdata))
+
+ signed_metadata_sig_file = payload_signer.Sign(metadata_sig_file)
+
+ # Finally we can compare the two signatures.
+ with open(signed_metadata_sig_file, 'rb') as verify_fp:
+ self.assertEqual(verify_fp.read(), metadata_signature)
+
+ @staticmethod
+ def _construct_zip_package_withValidPayload(with_metadata=False):
+ # Cannot use _construct_zip_package() since we need a "valid" payload.bin.
+ target_file = construct_target_files()
+ payload = Payload()
+ payload.Generate(target_file)
+
+ payload_signer = PayloadSigner()
+ payload.Sign(payload_signer)
+
+ zip_file = common.MakeTempFile(suffix='.zip')
+ with zipfile.ZipFile(zip_file, 'w') as zip_fp:
+ # 'payload.bin',
+ payload.WriteToZip(zip_fp)
+
+ # Other entries.
+ entries = ['care_map.txt', 'compatibility.zip']
+
+ # Put META-INF/com/android/metadata if needed.
+ if with_metadata:
+ entries.append('META-INF/com/android/metadata')
+
+ for entry in entries:
+ zip_fp.writestr(
+ entry, entry.replace('.', '-').upper(), zipfile.ZIP_STORED)
+
+ return zip_file
+
+ def test_Compute(self):
+ zip_file = self._construct_zip_package_withValidPayload()
+ property_files = AbOtaPropertyFiles()
+ with zipfile.ZipFile(zip_file, 'r') as zip_fp:
+ property_files_string = property_files.Compute(zip_fp)
+
+ tokens = self._parse_property_files_string(property_files_string)
+ # "6" indcludes the four entries above, one metadata entry, and one entry
+ # for payload-metadata.bin.
+ self.assertEqual(6, len(tokens))
+ self._verify_entries(
+ zip_file, tokens, ('care_map.txt', 'compatibility.zip'))
+
+ def test_Finalize(self):
+ zip_file = self._construct_zip_package_withValidPayload(with_metadata=True)
+ property_files = AbOtaPropertyFiles()
+ with zipfile.ZipFile(zip_file, 'r') as zip_fp:
+ # pylint: disable=protected-access
+ raw_metadata = property_files._GetPropertyFilesString(
+ zip_fp, reserve_space=False)
+ property_files_string = property_files.Finalize(zip_fp, len(raw_metadata))
+
+ tokens = self._parse_property_files_string(property_files_string)
+ # "6" indcludes the four entries above, one metadata entry, and one entry
+ # for payload-metadata.bin.
+ self.assertEqual(6, len(tokens))
+ self._verify_entries(
+ zip_file, tokens, ('care_map.txt', 'compatibility.zip'))
+
+ def test_Verify(self):
+ zip_file = self._construct_zip_package_withValidPayload(with_metadata=True)
+ property_files = AbOtaPropertyFiles()
+ with zipfile.ZipFile(zip_file, 'r') as zip_fp:
+ # pylint: disable=protected-access
+ raw_metadata = property_files._GetPropertyFilesString(
+ zip_fp, reserve_space=False)
+
+ property_files.Verify(zip_fp, raw_metadata)
+
+
+class NonAbOtaPropertyFilesTest(PropertyFilesTest):
+ """Additional sanity checks specialized for NonAbOtaPropertyFiles."""
+
+ def test_init(self):
+ property_files = NonAbOtaPropertyFiles()
+ self.assertEqual('ota-property-files', property_files.name)
+ self.assertEqual((), property_files.required)
+ self.assertEqual((), property_files.optional)
+
+ def test_Compute(self):
+ entries = ()
+ zip_file = self._construct_zip_package(entries)
+ property_files = NonAbOtaPropertyFiles()
+ with zipfile.ZipFile(zip_file) as zip_fp:
+ property_files_string = property_files.Compute(zip_fp)
+
+ tokens = self._parse_property_files_string(property_files_string)
+ self.assertEqual(1, len(tokens))
+ self._verify_entries(zip_file, tokens, entries)
+
+ def test_Finalize(self):
+ entries = [
+ 'META-INF/com/android/metadata',
+ ]
+ zip_file = self._construct_zip_package(entries)
+ property_files = NonAbOtaPropertyFiles()
+ with zipfile.ZipFile(zip_file) as zip_fp:
+ # pylint: disable=protected-access
+ raw_metadata = property_files._GetPropertyFilesString(
+ zip_fp, reserve_space=False)
+ property_files_string = property_files.Finalize(zip_fp, len(raw_metadata))
+ tokens = self._parse_property_files_string(property_files_string)
+
+ self.assertEqual(1, len(tokens))
+ # 'META-INF/com/android/metadata' will be key'd as 'metadata'.
+ entries[0] = 'metadata'
+ self._verify_entries(zip_file, tokens, entries)
+
+ def test_Verify(self):
+ entries = (
+ 'META-INF/com/android/metadata',
+ )
+ zip_file = self._construct_zip_package(entries)
+ property_files = NonAbOtaPropertyFiles()
+ with zipfile.ZipFile(zip_file) as zip_fp:
+ # pylint: disable=protected-access
+ raw_metadata = property_files._GetPropertyFilesString(
+ zip_fp, reserve_space=False)
+
+ property_files.Verify(zip_fp, raw_metadata)
+
+
class PayloadSignerTest(unittest.TestCase):
SIGFILE = 'sigfile.bin'