blob: 7ae221aa366bba6aeb0dbfed92ea18e492237797 [file] [log] [blame]
Kelvin Zhangcff4d762020-07-29 16:37:51 -04001# Copyright (C) 2020 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import copy
16import itertools
Yifan Hong125d0b62020-09-24 17:07:03 -070017import logging
Kelvin Zhangcff4d762020-07-29 16:37:51 -040018import os
Kelvin Zhang25ab9982021-06-22 09:51:34 -040019import struct
Kelvin Zhangcff4d762020-07-29 16:37:51 -040020import zipfile
21
Tianjiea2076132020-08-19 17:25:32 -070022import ota_metadata_pb2
Kelvin Zhangcff4d762020-07-29 16:37:51 -040023from common import (ZipDelete, ZipClose, OPTIONS, MakeTempFile,
24 ZipWriteStr, BuildInfo, LoadDictionaryFromFile,
TJ Rhoades6f488e92022-05-01 22:16:22 -070025 SignFile, PARTITIONS_WITH_BUILD_PROP, PartitionBuildProps,
26 GetRamdiskFormat)
Kelvin Zhangcff4d762020-07-29 16:37:51 -040027
Yifan Hong125d0b62020-09-24 17:07:03 -070028logger = logging.getLogger(__name__)
Kelvin Zhang2e417382020-08-20 11:33:11 -040029
30OPTIONS.no_signing = False
31OPTIONS.force_non_ab = False
32OPTIONS.wipe_user_data = False
33OPTIONS.downgrade = False
34OPTIONS.key_passwords = {}
35OPTIONS.package_key = None
36OPTIONS.incremental_source = None
37OPTIONS.retrofit_dynamic_partitions = False
38OPTIONS.output_metadata_path = None
39OPTIONS.boot_variable_file = None
40
Kelvin Zhangcff4d762020-07-29 16:37:51 -040041METADATA_NAME = 'META-INF/com/android/metadata'
Tianjiea2076132020-08-19 17:25:32 -070042METADATA_PROTO_NAME = 'META-INF/com/android/metadata.pb'
Kelvin Zhangcff4d762020-07-29 16:37:51 -040043UNZIP_PATTERN = ['IMAGES/*', 'META/*', 'OTA/*', 'RADIO/*']
Kelvin Zhang05ff7052021-02-10 09:13:26 -050044SECURITY_PATCH_LEVEL_PROP_NAME = "ro.build.version.security_patch"
45
Kelvin Zhangcff4d762020-07-29 16:37:51 -040046
Kelvin Zhangcff4d762020-07-29 16:37:51 -040047def FinalizeMetadata(metadata, input_file, output_file, needed_property_files):
48 """Finalizes the metadata and signs an A/B OTA package.
49
50 In order to stream an A/B OTA package, we need 'ota-streaming-property-files'
51 that contains the offsets and sizes for the ZIP entries. An example
52 property-files string is as follows.
53
54 "payload.bin:679:343,payload_properties.txt:378:45,metadata:69:379"
55
56 OTA server can pass down this string, in addition to the package URL, to the
57 system update client. System update client can then fetch individual ZIP
58 entries (ZIP_STORED) directly at the given offset of the URL.
59
60 Args:
61 metadata: The metadata dict for the package.
62 input_file: The input ZIP filename that doesn't contain the package METADATA
63 entry yet.
64 output_file: The final output ZIP filename.
65 needed_property_files: The list of PropertyFiles' to be generated.
66 """
67
68 def ComputeAllPropertyFiles(input_file, needed_property_files):
69 # Write the current metadata entry with placeholders.
Kelvin Zhang928c2342020-09-22 16:15:57 -040070 with zipfile.ZipFile(input_file, allowZip64=True) as input_zip:
Kelvin Zhangcff4d762020-07-29 16:37:51 -040071 for property_files in needed_property_files:
Tianjiea2076132020-08-19 17:25:32 -070072 metadata.property_files[property_files.name] = property_files.Compute(
73 input_zip)
Kelvin Zhangcff4d762020-07-29 16:37:51 -040074 namelist = input_zip.namelist()
75
Tianjiea2076132020-08-19 17:25:32 -070076 if METADATA_NAME in namelist or METADATA_PROTO_NAME in namelist:
77 ZipDelete(input_file, [METADATA_NAME, METADATA_PROTO_NAME])
Kelvin Zhang928c2342020-09-22 16:15:57 -040078 output_zip = zipfile.ZipFile(input_file, 'a', allowZip64=True)
Kelvin Zhangcff4d762020-07-29 16:37:51 -040079 WriteMetadata(metadata, output_zip)
80 ZipClose(output_zip)
81
82 if OPTIONS.no_signing:
83 return input_file
84
85 prelim_signing = MakeTempFile(suffix='.zip')
86 SignOutput(input_file, prelim_signing)
87 return prelim_signing
88
89 def FinalizeAllPropertyFiles(prelim_signing, needed_property_files):
Kelvin Zhang928c2342020-09-22 16:15:57 -040090 with zipfile.ZipFile(prelim_signing, allowZip64=True) as prelim_signing_zip:
Kelvin Zhangcff4d762020-07-29 16:37:51 -040091 for property_files in needed_property_files:
Tianjiea2076132020-08-19 17:25:32 -070092 metadata.property_files[property_files.name] = property_files.Finalize(
93 prelim_signing_zip,
94 len(metadata.property_files[property_files.name]))
Kelvin Zhangcff4d762020-07-29 16:37:51 -040095
96 # SignOutput(), which in turn calls signapk.jar, will possibly reorder the ZIP
97 # entries, as well as padding the entry headers. We do a preliminary signing
98 # (with an incomplete metadata entry) to allow that to happen. Then compute
99 # the ZIP entry offsets, write back the final metadata and do the final
100 # signing.
101 prelim_signing = ComputeAllPropertyFiles(input_file, needed_property_files)
102 try:
103 FinalizeAllPropertyFiles(prelim_signing, needed_property_files)
104 except PropertyFiles.InsufficientSpaceException:
105 # Even with the preliminary signing, the entry orders may change
106 # dramatically, which leads to insufficiently reserved space during the
107 # first call to ComputeAllPropertyFiles(). In that case, we redo all the
108 # preliminary signing works, based on the already ordered ZIP entries, to
109 # address the issue.
110 prelim_signing = ComputeAllPropertyFiles(
111 prelim_signing, needed_property_files)
112 FinalizeAllPropertyFiles(prelim_signing, needed_property_files)
113
114 # Replace the METADATA entry.
Tianjiea2076132020-08-19 17:25:32 -0700115 ZipDelete(prelim_signing, [METADATA_NAME, METADATA_PROTO_NAME])
Kelvin Zhang928c2342020-09-22 16:15:57 -0400116 output_zip = zipfile.ZipFile(prelim_signing, 'a', allowZip64=True)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400117 WriteMetadata(metadata, output_zip)
118 ZipClose(output_zip)
119
120 # Re-sign the package after updating the metadata entry.
121 if OPTIONS.no_signing:
122 output_file = prelim_signing
123 else:
124 SignOutput(prelim_signing, output_file)
125
126 # Reopen the final signed zip to double check the streaming metadata.
Kelvin Zhang928c2342020-09-22 16:15:57 -0400127 with zipfile.ZipFile(output_file, allowZip64=True) as output_zip:
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400128 for property_files in needed_property_files:
Tianjiea2076132020-08-19 17:25:32 -0700129 property_files.Verify(
130 output_zip, metadata.property_files[property_files.name].strip())
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400131
132 # If requested, dump the metadata to a separate file.
133 output_metadata_path = OPTIONS.output_metadata_path
134 if output_metadata_path:
135 WriteMetadata(metadata, output_metadata_path)
136
137
Tianjiea2076132020-08-19 17:25:32 -0700138def WriteMetadata(metadata_proto, output):
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400139 """Writes the metadata to the zip archive or a file.
140
141 Args:
Tianjiea2076132020-08-19 17:25:32 -0700142 metadata_proto: The metadata protobuf for the package.
143 output: A ZipFile object or a string of the output file path. If a string
144 path is given, the metadata in the protobuf format will be written to
145 {output}.pb, e.g. ota_metadata.pb
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400146 """
147
Tianjiea2076132020-08-19 17:25:32 -0700148 metadata_dict = BuildLegacyOtaMetadata(metadata_proto)
149 legacy_metadata = "".join(["%s=%s\n" % kv for kv in
150 sorted(metadata_dict.items())])
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400151 if isinstance(output, zipfile.ZipFile):
Tianjiea2076132020-08-19 17:25:32 -0700152 ZipWriteStr(output, METADATA_PROTO_NAME, metadata_proto.SerializeToString(),
153 compress_type=zipfile.ZIP_STORED)
154 ZipWriteStr(output, METADATA_NAME, legacy_metadata,
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400155 compress_type=zipfile.ZIP_STORED)
156 return
157
Cole Faustb820bcd2021-10-28 13:59:48 -0700158 with open('{}.pb'.format(output), 'wb') as f:
Tianjiea2076132020-08-19 17:25:32 -0700159 f.write(metadata_proto.SerializeToString())
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400160 with open(output, 'w') as f:
Tianjiea2076132020-08-19 17:25:32 -0700161 f.write(legacy_metadata)
162
163
164def UpdateDeviceState(device_state, build_info, boot_variable_values,
165 is_post_build):
166 """Update the fields of the DeviceState proto with build info."""
167
Tianjie2bb14862020-08-28 16:24:34 -0700168 def UpdatePartitionStates(partition_states):
169 """Update the per-partition state according to its build.prop"""
Kelvin Zhang39aea442020-08-17 11:04:25 -0400170 if not build_info.is_ab:
171 return
Tianjie2bb14862020-08-28 16:24:34 -0700172 build_info_set = ComputeRuntimeBuildInfos(build_info,
173 boot_variable_values)
Kelvin Zhang39aea442020-08-17 11:04:25 -0400174 assert "ab_partitions" in build_info.info_dict,\
Kelvin Zhang05ff7052021-02-10 09:13:26 -0500175 "ab_partitions property required for ab update."
Kelvin Zhang39aea442020-08-17 11:04:25 -0400176 ab_partitions = set(build_info.info_dict.get("ab_partitions"))
177
178 # delta_generator will error out on unused timestamps,
179 # so only generate timestamps for dynamic partitions
180 # used in OTA update.
Yifan Hong5057b952021-01-07 14:09:57 -0800181 for partition in sorted(set(PARTITIONS_WITH_BUILD_PROP) & ab_partitions):
Tianjie2bb14862020-08-28 16:24:34 -0700182 partition_prop = build_info.info_dict.get(
183 '{}.build.prop'.format(partition))
184 # Skip if the partition is missing, or it doesn't have a build.prop
185 if not partition_prop or not partition_prop.build_props:
186 continue
187
188 partition_state = partition_states.add()
189 partition_state.partition_name = partition
190 # Update the partition's runtime device names and fingerprints
191 partition_devices = set()
192 partition_fingerprints = set()
193 for runtime_build_info in build_info_set:
194 partition_devices.add(
195 runtime_build_info.GetPartitionBuildProp('ro.product.device',
196 partition))
197 partition_fingerprints.add(
198 runtime_build_info.GetPartitionFingerprint(partition))
199
200 partition_state.device.extend(sorted(partition_devices))
201 partition_state.build.extend(sorted(partition_fingerprints))
202
203 # TODO(xunchang) set the boot image's version with kmi. Note the boot
204 # image doesn't have a file map.
205 partition_state.version = build_info.GetPartitionBuildProp(
206 'ro.build.date.utc', partition)
207
208 # TODO(xunchang), we can save a call to ComputeRuntimeBuildInfos.
Tianjiea2076132020-08-19 17:25:32 -0700209 build_devices, build_fingerprints = \
210 CalculateRuntimeDevicesAndFingerprints(build_info, boot_variable_values)
211 device_state.device.extend(sorted(build_devices))
212 device_state.build.extend(sorted(build_fingerprints))
213 device_state.build_incremental = build_info.GetBuildProp(
214 'ro.build.version.incremental')
215
Tianjie2bb14862020-08-28 16:24:34 -0700216 UpdatePartitionStates(device_state.partition_state)
Tianjiea2076132020-08-19 17:25:32 -0700217
218 if is_post_build:
219 device_state.sdk_level = build_info.GetBuildProp(
220 'ro.build.version.sdk')
221 device_state.security_patch_level = build_info.GetBuildProp(
222 'ro.build.version.security_patch')
223 # Use the actual post-timestamp, even for a downgrade case.
224 device_state.timestamp = int(build_info.GetBuildProp('ro.build.date.utc'))
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400225
226
227def GetPackageMetadata(target_info, source_info=None):
Tianjiea2076132020-08-19 17:25:32 -0700228 """Generates and returns the metadata proto.
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400229
Tianjiea2076132020-08-19 17:25:32 -0700230 It generates a ota_metadata protobuf that contains the info to be written
231 into an OTA package (META-INF/com/android/metadata.pb). It also handles the
232 detection of downgrade / data wipe based on the global options.
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400233
234 Args:
235 target_info: The BuildInfo instance that holds the target build info.
236 source_info: The BuildInfo instance that holds the source build info, or
237 None if generating full OTA.
238
239 Returns:
Tianjiea2076132020-08-19 17:25:32 -0700240 A protobuf to be written into package metadata entry.
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400241 """
242 assert isinstance(target_info, BuildInfo)
243 assert source_info is None or isinstance(source_info, BuildInfo)
244
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400245 boot_variable_values = {}
246 if OPTIONS.boot_variable_file:
247 d = LoadDictionaryFromFile(OPTIONS.boot_variable_file)
248 for key, values in d.items():
249 boot_variable_values[key] = [val.strip() for val in values.split(',')]
250
Tianjiea2076132020-08-19 17:25:32 -0700251 metadata_proto = ota_metadata_pb2.OtaMetadata()
252 # TODO(xunchang) some fields, e.g. post-device isn't necessary. We can
253 # consider skipping them if they aren't used by clients.
254 UpdateDeviceState(metadata_proto.postcondition, target_info,
255 boot_variable_values, True)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400256
257 if target_info.is_ab and not OPTIONS.force_non_ab:
Tianjiea2076132020-08-19 17:25:32 -0700258 metadata_proto.type = ota_metadata_pb2.OtaMetadata.AB
259 metadata_proto.required_cache = 0
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400260 else:
Tianjiea2076132020-08-19 17:25:32 -0700261 metadata_proto.type = ota_metadata_pb2.OtaMetadata.BLOCK
262 # cache requirement will be updated by the non-A/B codes.
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400263
264 if OPTIONS.wipe_user_data:
Tianjiea2076132020-08-19 17:25:32 -0700265 metadata_proto.wipe = True
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400266
267 if OPTIONS.retrofit_dynamic_partitions:
Tianjiea2076132020-08-19 17:25:32 -0700268 metadata_proto.retrofit_dynamic_partitions = True
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400269
270 is_incremental = source_info is not None
271 if is_incremental:
Tianjiea2076132020-08-19 17:25:32 -0700272 UpdateDeviceState(metadata_proto.precondition, source_info,
273 boot_variable_values, False)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400274 else:
Tianjiea2076132020-08-19 17:25:32 -0700275 metadata_proto.precondition.device.extend(
276 metadata_proto.postcondition.device)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400277
278 # Detect downgrades and set up downgrade flags accordingly.
279 if is_incremental:
Tianjiea2076132020-08-19 17:25:32 -0700280 HandleDowngradeMetadata(metadata_proto, target_info, source_info)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400281
Tianjiea2076132020-08-19 17:25:32 -0700282 return metadata_proto
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400283
284
Tianjiea2076132020-08-19 17:25:32 -0700285def BuildLegacyOtaMetadata(metadata_proto):
286 """Converts the metadata proto to a legacy metadata dict.
287
288 This metadata dict is used to build the legacy metadata text file for
289 backward compatibility. We won't add new keys to the legacy metadata format.
290 If new information is needed, we should add it as a new field in OtaMetadata
291 proto definition.
292 """
293
294 separator = '|'
295
296 metadata_dict = {}
297 if metadata_proto.type == ota_metadata_pb2.OtaMetadata.AB:
298 metadata_dict['ota-type'] = 'AB'
299 elif metadata_proto.type == ota_metadata_pb2.OtaMetadata.BLOCK:
300 metadata_dict['ota-type'] = 'BLOCK'
301 if metadata_proto.wipe:
302 metadata_dict['ota-wipe'] = 'yes'
303 if metadata_proto.retrofit_dynamic_partitions:
304 metadata_dict['ota-retrofit-dynamic-partitions'] = 'yes'
305 if metadata_proto.downgrade:
306 metadata_dict['ota-downgrade'] = 'yes'
307
308 metadata_dict['ota-required-cache'] = str(metadata_proto.required_cache)
309
310 post_build = metadata_proto.postcondition
311 metadata_dict['post-build'] = separator.join(post_build.build)
312 metadata_dict['post-build-incremental'] = post_build.build_incremental
313 metadata_dict['post-sdk-level'] = post_build.sdk_level
314 metadata_dict['post-security-patch-level'] = post_build.security_patch_level
315 metadata_dict['post-timestamp'] = str(post_build.timestamp)
316
317 pre_build = metadata_proto.precondition
318 metadata_dict['pre-device'] = separator.join(pre_build.device)
319 # incremental updates
320 if len(pre_build.build) != 0:
321 metadata_dict['pre-build'] = separator.join(pre_build.build)
322 metadata_dict['pre-build-incremental'] = pre_build.build_incremental
323
Kelvin Zhang05ff7052021-02-10 09:13:26 -0500324 if metadata_proto.spl_downgrade:
325 metadata_dict['spl-downgrade'] = 'yes'
Tianjiea2076132020-08-19 17:25:32 -0700326 metadata_dict.update(metadata_proto.property_files)
327
328 return metadata_dict
329
330
331def HandleDowngradeMetadata(metadata_proto, target_info, source_info):
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400332 # Only incremental OTAs are allowed to reach here.
333 assert OPTIONS.incremental_source is not None
334
335 post_timestamp = target_info.GetBuildProp("ro.build.date.utc")
336 pre_timestamp = source_info.GetBuildProp("ro.build.date.utc")
337 is_downgrade = int(post_timestamp) < int(pre_timestamp)
338
Kelvin Zhang05ff7052021-02-10 09:13:26 -0500339 if OPTIONS.spl_downgrade:
340 metadata_proto.spl_downgrade = True
341
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400342 if OPTIONS.downgrade:
343 if not is_downgrade:
344 raise RuntimeError(
345 "--downgrade or --override_timestamp specified but no downgrade "
346 "detected: pre: %s, post: %s" % (pre_timestamp, post_timestamp))
Tianjiea2076132020-08-19 17:25:32 -0700347 metadata_proto.downgrade = True
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400348 else:
349 if is_downgrade:
350 raise RuntimeError(
351 "Downgrade detected based on timestamp check: pre: %s, post: %s. "
352 "Need to specify --override_timestamp OR --downgrade to allow "
353 "building the incremental." % (pre_timestamp, post_timestamp))
354
355
Tianjie2bb14862020-08-28 16:24:34 -0700356def ComputeRuntimeBuildInfos(default_build_info, boot_variable_values):
357 """Returns a set of build info objects that may exist during runtime."""
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400358
Tianjie2bb14862020-08-28 16:24:34 -0700359 build_info_set = {default_build_info}
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400360 if not boot_variable_values:
Tianjie2bb14862020-08-28 16:24:34 -0700361 return build_info_set
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400362
363 # Calculate all possible combinations of the values for the boot variables.
364 keys = boot_variable_values.keys()
365 value_list = boot_variable_values.values()
366 combinations = [dict(zip(keys, values))
367 for values in itertools.product(*value_list)]
368 for placeholder_values in combinations:
369 # Reload the info_dict as some build properties may change their values
370 # based on the value of ro.boot* properties.
Tianjie2bb14862020-08-28 16:24:34 -0700371 info_dict = copy.deepcopy(default_build_info.info_dict)
Yifan Hong5057b952021-01-07 14:09:57 -0800372 for partition in PARTITIONS_WITH_BUILD_PROP:
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400373 partition_prop_key = "{}.build.prop".format(partition)
374 input_file = info_dict[partition_prop_key].input_file
TJ Rhoades6f488e92022-05-01 22:16:22 -0700375 ramdisk = GetRamdiskFormat(info_dict)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400376 if isinstance(input_file, zipfile.ZipFile):
Kelvin Zhang928c2342020-09-22 16:15:57 -0400377 with zipfile.ZipFile(input_file.filename, allowZip64=True) as input_zip:
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400378 info_dict[partition_prop_key] = \
379 PartitionBuildProps.FromInputFile(input_zip, partition,
TJ Rhoades6f488e92022-05-01 22:16:22 -0700380 placeholder_values,
381 ramdisk)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400382 else:
383 info_dict[partition_prop_key] = \
384 PartitionBuildProps.FromInputFile(input_file, partition,
TJ Rhoades6f488e92022-05-01 22:16:22 -0700385 placeholder_values,
386 ramdisk)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400387 info_dict["build.prop"] = info_dict["system.build.prop"]
Tianjie2bb14862020-08-28 16:24:34 -0700388 build_info_set.add(BuildInfo(info_dict, default_build_info.oem_dicts))
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400389
Tianjie2bb14862020-08-28 16:24:34 -0700390 return build_info_set
391
392
393def CalculateRuntimeDevicesAndFingerprints(default_build_info,
394 boot_variable_values):
395 """Returns a tuple of sets for runtime devices and fingerprints"""
396
397 device_names = set()
398 fingerprints = set()
399 build_info_set = ComputeRuntimeBuildInfos(default_build_info,
400 boot_variable_values)
401 for runtime_build_info in build_info_set:
402 device_names.add(runtime_build_info.device)
403 fingerprints.add(runtime_build_info.fingerprint)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400404 return device_names, fingerprints
405
406
Kelvin Zhang25ab9982021-06-22 09:51:34 -0400407def GetZipEntryOffset(zfp, entry_info):
408 """Get offset to a beginning of a particular zip entry
409 Args:
410 fp: zipfile.ZipFile
411 entry_info: zipfile.ZipInfo
412
413 Returns:
414 (offset, size) tuple
415 """
416 # Don't use len(entry_info.extra). Because that returns size of extra
417 # fields in central directory. We need to look at local file directory,
418 # as these two might have different sizes.
419
420 # We cannot work with zipfile.ZipFile instances, we need a |fp| for the underlying file.
421 zfp = zfp.fp
422 zfp.seek(entry_info.header_offset)
423 data = zfp.read(zipfile.sizeFileHeader)
424 fheader = struct.unpack(zipfile.structFileHeader, data)
425 # Last two fields of local file header are filename length and
426 # extra length
427 filename_len = fheader[-2]
428 extra_len = fheader[-1]
429 offset = entry_info.header_offset
430 offset += zipfile.sizeFileHeader
431 offset += filename_len + extra_len
432 size = entry_info.file_size
433 return (offset, size)
434
435
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400436class PropertyFiles(object):
437 """A class that computes the property-files string for an OTA package.
438
439 A property-files string is a comma-separated string that contains the
440 offset/size info for an OTA package. The entries, which must be ZIP_STORED,
441 can be fetched directly with the package URL along with the offset/size info.
442 These strings can be used for streaming A/B OTAs, or allowing an updater to
443 download package metadata entry directly, without paying the cost of
444 downloading entire package.
445
446 Computing the final property-files string requires two passes. Because doing
447 the whole package signing (with signapk.jar) will possibly reorder the ZIP
448 entries, which may in turn invalidate earlier computed ZIP entry offset/size
449 values.
450
451 This class provides functions to be called for each pass. The general flow is
452 as follows.
453
454 property_files = PropertyFiles()
455 # The first pass, which writes placeholders before doing initial signing.
456 property_files.Compute()
457 SignOutput()
458
459 # The second pass, by replacing the placeholders with actual data.
460 property_files.Finalize()
461 SignOutput()
462
463 And the caller can additionally verify the final result.
464
465 property_files.Verify()
466 """
467
468 def __init__(self):
469 self.name = None
470 self.required = ()
471 self.optional = ()
472
473 def Compute(self, input_zip):
474 """Computes and returns a property-files string with placeholders.
475
476 We reserve extra space for the offset and size of the metadata entry itself,
477 although we don't know the final values until the package gets signed.
478
479 Args:
480 input_zip: The input ZIP file.
481
482 Returns:
483 A string with placeholders for the metadata offset/size info, e.g.
484 "payload.bin:679:343,payload_properties.txt:378:45,metadata: ".
485 """
486 return self.GetPropertyFilesString(input_zip, reserve_space=True)
487
488 class InsufficientSpaceException(Exception):
489 pass
490
491 def Finalize(self, input_zip, reserved_length):
492 """Finalizes a property-files string with actual METADATA offset/size info.
493
494 The input ZIP file has been signed, with the ZIP entries in the desired
495 place (signapk.jar will possibly reorder the ZIP entries). Now we compute
496 the ZIP entry offsets and construct the property-files string with actual
497 data. Note that during this process, we must pad the property-files string
498 to the reserved length, so that the METADATA entry size remains the same.
499 Otherwise the entries' offsets and sizes may change again.
500
501 Args:
502 input_zip: The input ZIP file.
503 reserved_length: The reserved length of the property-files string during
504 the call to Compute(). The final string must be no more than this
505 size.
506
507 Returns:
508 A property-files string including the metadata offset/size info, e.g.
509 "payload.bin:679:343,payload_properties.txt:378:45,metadata:69:379 ".
510
511 Raises:
512 InsufficientSpaceException: If the reserved length is insufficient to hold
513 the final string.
514 """
515 result = self.GetPropertyFilesString(input_zip, reserve_space=False)
516 if len(result) > reserved_length:
517 raise self.InsufficientSpaceException(
518 'Insufficient reserved space: reserved={}, actual={}'.format(
519 reserved_length, len(result)))
520
521 result += ' ' * (reserved_length - len(result))
522 return result
523
524 def Verify(self, input_zip, expected):
525 """Verifies the input ZIP file contains the expected property-files string.
526
527 Args:
528 input_zip: The input ZIP file.
529 expected: The property-files string that's computed from Finalize().
530
531 Raises:
532 AssertionError: On finding a mismatch.
533 """
534 actual = self.GetPropertyFilesString(input_zip)
535 assert actual == expected, \
536 "Mismatching streaming metadata: {} vs {}.".format(actual, expected)
537
538 def GetPropertyFilesString(self, zip_file, reserve_space=False):
539 """
540 Constructs the property-files string per request.
541
542 Args:
543 zip_file: The input ZIP file.
544 reserved_length: The reserved length of the property-files string.
545
546 Returns:
547 A property-files string including the metadata offset/size info, e.g.
548 "payload.bin:679:343,payload_properties.txt:378:45,metadata: ".
549 """
550
551 def ComputeEntryOffsetSize(name):
552 """Computes the zip entry offset and size."""
553 info = zip_file.getinfo(name)
Kelvin Zhang25ab9982021-06-22 09:51:34 -0400554 (offset, size) = GetZipEntryOffset(zip_file, info)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400555 return '%s:%d:%d' % (os.path.basename(name), offset, size)
556
557 tokens = []
558 tokens.extend(self._GetPrecomputed(zip_file))
559 for entry in self.required:
560 tokens.append(ComputeEntryOffsetSize(entry))
561 for entry in self.optional:
562 if entry in zip_file.namelist():
563 tokens.append(ComputeEntryOffsetSize(entry))
564
565 # 'META-INF/com/android/metadata' is required. We don't know its actual
566 # offset and length (as well as the values for other entries). So we reserve
567 # 15-byte as a placeholder ('offset:length'), which is sufficient to cover
568 # the space for metadata entry. Because 'offset' allows a max of 10-digit
569 # (i.e. ~9 GiB), with a max of 4-digit for the length. Note that all the
570 # reserved space serves the metadata entry only.
571 if reserve_space:
572 tokens.append('metadata:' + ' ' * 15)
Tianjiea2076132020-08-19 17:25:32 -0700573 tokens.append('metadata.pb:' + ' ' * 15)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400574 else:
575 tokens.append(ComputeEntryOffsetSize(METADATA_NAME))
Luca Stefanib6075c52021-11-03 17:10:54 +0100576 if METADATA_PROTO_NAME in zip_file.namelist():
577 tokens.append(ComputeEntryOffsetSize(METADATA_PROTO_NAME))
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400578
579 return ','.join(tokens)
580
581 def _GetPrecomputed(self, input_zip):
582 """Computes the additional tokens to be included into the property-files.
583
584 This applies to tokens without actual ZIP entries, such as
585 payload_metadata.bin. We want to expose the offset/size to updaters, so
586 that they can download the payload metadata directly with the info.
587
588 Args:
589 input_zip: The input zip file.
590
591 Returns:
592 A list of strings (tokens) to be added to the property-files string.
593 """
594 # pylint: disable=no-self-use
595 # pylint: disable=unused-argument
596 return []
597
598
599def SignOutput(temp_zip_name, output_zip_name):
600 pw = OPTIONS.key_passwords[OPTIONS.package_key]
601
602 SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw,
603 whole_file=True)
Tianjiea5fca032021-06-01 22:06:28 -0700604
605
606def ConstructOtaApexInfo(target_zip, source_file=None):
607 """If applicable, add the source version to the apex info."""
608
609 def _ReadApexInfo(input_zip):
610 if "META/apex_info.pb" not in input_zip.namelist():
611 logger.warning("target_file doesn't contain apex_info.pb %s", input_zip)
612 return None
613
614 with input_zip.open("META/apex_info.pb", "r") as zfp:
615 return zfp.read()
616
617 target_apex_string = _ReadApexInfo(target_zip)
618 # Return early if the target apex info doesn't exist or is empty.
619 if not target_apex_string:
620 return target_apex_string
621
622 # If the source apex info isn't available, just return the target info
623 if not source_file:
624 return target_apex_string
625
626 with zipfile.ZipFile(source_file, "r", allowZip64=True) as source_zip:
627 source_apex_string = _ReadApexInfo(source_zip)
628 if not source_apex_string:
629 return target_apex_string
630
631 source_apex_proto = ota_metadata_pb2.ApexMetadata()
632 source_apex_proto.ParseFromString(source_apex_string)
633 source_apex_versions = {apex.package_name: apex.version for apex in
634 source_apex_proto.apex_info}
635
636 # If the apex package is available in the source build, initialize the source
637 # apex version.
638 target_apex_proto = ota_metadata_pb2.ApexMetadata()
639 target_apex_proto.ParseFromString(target_apex_string)
640 for target_apex in target_apex_proto.apex_info:
641 name = target_apex.package_name
642 if name in source_apex_versions:
643 target_apex.source_version = source_apex_versions[name]
644
645 return target_apex_proto.SerializeToString()
Kelvin Zhang410bb382022-01-06 09:15:54 -0800646
647
Kelvin Zhangf2728d62022-01-10 11:42:36 -0800648def IsLz4diffCompatible(source_file: str, target_file: str):
649 """Check whether lz4diff versions in two builds are compatible
650
651 Args:
652 source_file: Path to source build's target_file.zip
653 target_file: Path to target build's target_file.zip
654
655 Returns:
656 bool true if and only if lz4diff versions are compatible
657 """
658 if source_file is None or target_file is None:
659 return False
660 # Right now we enable lz4diff as long as source build has liblz4.so.
661 # In the future we might introduce version system to lz4diff as well.
662 if zipfile.is_zipfile(source_file):
663 with zipfile.ZipFile(source_file, "r") as zfp:
664 return "META/liblz4.so" in zfp.namelist()
665 else:
666 assert os.path.isdir(source_file)
667 return os.path.exists(os.path.join(source_file, "META", "liblz4.so"))
668
669
Kelvin Zhang410bb382022-01-06 09:15:54 -0800670def IsZucchiniCompatible(source_file: str, target_file: str):
671 """Check whether zucchini versions in two builds are compatible
672
673 Args:
674 source_file: Path to source build's target_file.zip
675 target_file: Path to target build's target_file.zip
676
677 Returns:
678 bool true if and only if zucchini versions are compatible
679 """
680 if source_file is None or target_file is None:
681 return False
682 assert os.path.exists(source_file)
683 assert os.path.exists(target_file)
684
685 assert zipfile.is_zipfile(source_file) or os.path.isdir(source_file)
686 assert zipfile.is_zipfile(target_file) or os.path.isdir(target_file)
687 _ZUCCHINI_CONFIG_ENTRY_NAME = "META/zucchini_config.txt"
688
689 def ReadEntry(path, entry):
690 # Read an entry inside a .zip file or extracted dir of .zip file
691 if zipfile.is_zipfile(path):
692 with zipfile.ZipFile(path, "r", allowZip64=True) as zfp:
693 if entry in zfp.namelist():
694 return zfp.read(entry).decode()
695 else:
696 entry_path = os.path.join(entry, path)
697 if os.path.exists(entry_path):
698 with open(entry_path, "r") as fp:
699 return fp.read()
700 else:
701 return ""
702 return ReadEntry(source_file, _ZUCCHINI_CONFIG_ENTRY_NAME) == ReadEntry(target_file, _ZUCCHINI_CONFIG_ENTRY_NAME)