blob: 12acc138c7120d60711aa5befe2f7d8570ee3369 [file] [log] [blame]
Kelvin Zhangcff4d762020-07-29 16:37:51 -04001# Copyright (C) 2020 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import copy
16import itertools
Yifan Hong125d0b62020-09-24 17:07:03 -070017import logging
Kelvin Zhangcff4d762020-07-29 16:37:51 -040018import os
Kelvin Zhangb9fdf2d2022-08-12 14:07:31 -070019import shutil
Kelvin Zhang25ab9982021-06-22 09:51:34 -040020import struct
Kelvin Zhangcff4d762020-07-29 16:37:51 -040021import zipfile
22
Tianjiea2076132020-08-19 17:25:32 -070023import ota_metadata_pb2
Kelvin Zhangcff4d762020-07-29 16:37:51 -040024from common import (ZipDelete, ZipClose, OPTIONS, MakeTempFile,
25 ZipWriteStr, BuildInfo, LoadDictionaryFromFile,
TJ Rhoades6f488e92022-05-01 22:16:22 -070026 SignFile, PARTITIONS_WITH_BUILD_PROP, PartitionBuildProps,
27 GetRamdiskFormat)
Kelvin Zhangcff4d762020-07-29 16:37:51 -040028
Yifan Hong125d0b62020-09-24 17:07:03 -070029logger = logging.getLogger(__name__)
Kelvin Zhang2e417382020-08-20 11:33:11 -040030
31OPTIONS.no_signing = False
32OPTIONS.force_non_ab = False
33OPTIONS.wipe_user_data = False
34OPTIONS.downgrade = False
35OPTIONS.key_passwords = {}
36OPTIONS.package_key = None
37OPTIONS.incremental_source = None
38OPTIONS.retrofit_dynamic_partitions = False
39OPTIONS.output_metadata_path = None
40OPTIONS.boot_variable_file = None
41
Kelvin Zhangcff4d762020-07-29 16:37:51 -040042METADATA_NAME = 'META-INF/com/android/metadata'
Tianjiea2076132020-08-19 17:25:32 -070043METADATA_PROTO_NAME = 'META-INF/com/android/metadata.pb'
Kelvin Zhangcff4d762020-07-29 16:37:51 -040044UNZIP_PATTERN = ['IMAGES/*', 'META/*', 'OTA/*', 'RADIO/*']
Kelvin Zhang05ff7052021-02-10 09:13:26 -050045SECURITY_PATCH_LEVEL_PROP_NAME = "ro.build.version.security_patch"
46
Kelvin Zhangcff4d762020-07-29 16:37:51 -040047
Kelvin Zhangcff4d762020-07-29 16:37:51 -040048def FinalizeMetadata(metadata, input_file, output_file, needed_property_files):
49 """Finalizes the metadata and signs an A/B OTA package.
50
51 In order to stream an A/B OTA package, we need 'ota-streaming-property-files'
52 that contains the offsets and sizes for the ZIP entries. An example
53 property-files string is as follows.
54
55 "payload.bin:679:343,payload_properties.txt:378:45,metadata:69:379"
56
57 OTA server can pass down this string, in addition to the package URL, to the
58 system update client. System update client can then fetch individual ZIP
59 entries (ZIP_STORED) directly at the given offset of the URL.
60
61 Args:
62 metadata: The metadata dict for the package.
63 input_file: The input ZIP filename that doesn't contain the package METADATA
64 entry yet.
65 output_file: The final output ZIP filename.
66 needed_property_files: The list of PropertyFiles' to be generated.
67 """
68
69 def ComputeAllPropertyFiles(input_file, needed_property_files):
70 # Write the current metadata entry with placeholders.
Kelvin Zhang928c2342020-09-22 16:15:57 -040071 with zipfile.ZipFile(input_file, allowZip64=True) as input_zip:
Kelvin Zhangcff4d762020-07-29 16:37:51 -040072 for property_files in needed_property_files:
Tianjiea2076132020-08-19 17:25:32 -070073 metadata.property_files[property_files.name] = property_files.Compute(
74 input_zip)
Kelvin Zhangcff4d762020-07-29 16:37:51 -040075 namelist = input_zip.namelist()
76
Tianjiea2076132020-08-19 17:25:32 -070077 if METADATA_NAME in namelist or METADATA_PROTO_NAME in namelist:
78 ZipDelete(input_file, [METADATA_NAME, METADATA_PROTO_NAME])
Kelvin Zhang928c2342020-09-22 16:15:57 -040079 output_zip = zipfile.ZipFile(input_file, 'a', allowZip64=True)
Kelvin Zhangcff4d762020-07-29 16:37:51 -040080 WriteMetadata(metadata, output_zip)
81 ZipClose(output_zip)
82
83 if OPTIONS.no_signing:
84 return input_file
85
86 prelim_signing = MakeTempFile(suffix='.zip')
87 SignOutput(input_file, prelim_signing)
88 return prelim_signing
89
90 def FinalizeAllPropertyFiles(prelim_signing, needed_property_files):
Kelvin Zhang928c2342020-09-22 16:15:57 -040091 with zipfile.ZipFile(prelim_signing, allowZip64=True) as prelim_signing_zip:
Kelvin Zhangcff4d762020-07-29 16:37:51 -040092 for property_files in needed_property_files:
Tianjiea2076132020-08-19 17:25:32 -070093 metadata.property_files[property_files.name] = property_files.Finalize(
94 prelim_signing_zip,
95 len(metadata.property_files[property_files.name]))
Kelvin Zhangcff4d762020-07-29 16:37:51 -040096
97 # SignOutput(), which in turn calls signapk.jar, will possibly reorder the ZIP
98 # entries, as well as padding the entry headers. We do a preliminary signing
99 # (with an incomplete metadata entry) to allow that to happen. Then compute
100 # the ZIP entry offsets, write back the final metadata and do the final
101 # signing.
102 prelim_signing = ComputeAllPropertyFiles(input_file, needed_property_files)
103 try:
104 FinalizeAllPropertyFiles(prelim_signing, needed_property_files)
105 except PropertyFiles.InsufficientSpaceException:
106 # Even with the preliminary signing, the entry orders may change
107 # dramatically, which leads to insufficiently reserved space during the
108 # first call to ComputeAllPropertyFiles(). In that case, we redo all the
109 # preliminary signing works, based on the already ordered ZIP entries, to
110 # address the issue.
111 prelim_signing = ComputeAllPropertyFiles(
112 prelim_signing, needed_property_files)
113 FinalizeAllPropertyFiles(prelim_signing, needed_property_files)
114
115 # Replace the METADATA entry.
Tianjiea2076132020-08-19 17:25:32 -0700116 ZipDelete(prelim_signing, [METADATA_NAME, METADATA_PROTO_NAME])
Kelvin Zhang928c2342020-09-22 16:15:57 -0400117 output_zip = zipfile.ZipFile(prelim_signing, 'a', allowZip64=True)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400118 WriteMetadata(metadata, output_zip)
119 ZipClose(output_zip)
120
121 # Re-sign the package after updating the metadata entry.
122 if OPTIONS.no_signing:
Kelvin Zhangb9fdf2d2022-08-12 14:07:31 -0700123 shutil.copy(prelim_signing, output_file)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400124 else:
125 SignOutput(prelim_signing, output_file)
126
127 # Reopen the final signed zip to double check the streaming metadata.
Kelvin Zhang928c2342020-09-22 16:15:57 -0400128 with zipfile.ZipFile(output_file, allowZip64=True) as output_zip:
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400129 for property_files in needed_property_files:
Tianjiea2076132020-08-19 17:25:32 -0700130 property_files.Verify(
131 output_zip, metadata.property_files[property_files.name].strip())
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400132
133 # If requested, dump the metadata to a separate file.
134 output_metadata_path = OPTIONS.output_metadata_path
135 if output_metadata_path:
136 WriteMetadata(metadata, output_metadata_path)
137
138
Tianjiea2076132020-08-19 17:25:32 -0700139def WriteMetadata(metadata_proto, output):
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400140 """Writes the metadata to the zip archive or a file.
141
142 Args:
Tianjiea2076132020-08-19 17:25:32 -0700143 metadata_proto: The metadata protobuf for the package.
144 output: A ZipFile object or a string of the output file path. If a string
145 path is given, the metadata in the protobuf format will be written to
146 {output}.pb, e.g. ota_metadata.pb
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400147 """
148
Tianjiea2076132020-08-19 17:25:32 -0700149 metadata_dict = BuildLegacyOtaMetadata(metadata_proto)
150 legacy_metadata = "".join(["%s=%s\n" % kv for kv in
151 sorted(metadata_dict.items())])
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400152 if isinstance(output, zipfile.ZipFile):
Tianjiea2076132020-08-19 17:25:32 -0700153 ZipWriteStr(output, METADATA_PROTO_NAME, metadata_proto.SerializeToString(),
154 compress_type=zipfile.ZIP_STORED)
155 ZipWriteStr(output, METADATA_NAME, legacy_metadata,
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400156 compress_type=zipfile.ZIP_STORED)
157 return
158
Cole Faustb820bcd2021-10-28 13:59:48 -0700159 with open('{}.pb'.format(output), 'wb') as f:
Tianjiea2076132020-08-19 17:25:32 -0700160 f.write(metadata_proto.SerializeToString())
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400161 with open(output, 'w') as f:
Tianjiea2076132020-08-19 17:25:32 -0700162 f.write(legacy_metadata)
163
164
165def UpdateDeviceState(device_state, build_info, boot_variable_values,
166 is_post_build):
167 """Update the fields of the DeviceState proto with build info."""
168
Tianjie2bb14862020-08-28 16:24:34 -0700169 def UpdatePartitionStates(partition_states):
170 """Update the per-partition state according to its build.prop"""
Kelvin Zhang39aea442020-08-17 11:04:25 -0400171 if not build_info.is_ab:
172 return
Tianjie2bb14862020-08-28 16:24:34 -0700173 build_info_set = ComputeRuntimeBuildInfos(build_info,
174 boot_variable_values)
Kelvin Zhang39aea442020-08-17 11:04:25 -0400175 assert "ab_partitions" in build_info.info_dict,\
Kelvin Zhang05ff7052021-02-10 09:13:26 -0500176 "ab_partitions property required for ab update."
Kelvin Zhang39aea442020-08-17 11:04:25 -0400177 ab_partitions = set(build_info.info_dict.get("ab_partitions"))
178
179 # delta_generator will error out on unused timestamps,
180 # so only generate timestamps for dynamic partitions
181 # used in OTA update.
Yifan Hong5057b952021-01-07 14:09:57 -0800182 for partition in sorted(set(PARTITIONS_WITH_BUILD_PROP) & ab_partitions):
Tianjie2bb14862020-08-28 16:24:34 -0700183 partition_prop = build_info.info_dict.get(
184 '{}.build.prop'.format(partition))
185 # Skip if the partition is missing, or it doesn't have a build.prop
186 if not partition_prop or not partition_prop.build_props:
187 continue
188
189 partition_state = partition_states.add()
190 partition_state.partition_name = partition
191 # Update the partition's runtime device names and fingerprints
192 partition_devices = set()
193 partition_fingerprints = set()
194 for runtime_build_info in build_info_set:
195 partition_devices.add(
196 runtime_build_info.GetPartitionBuildProp('ro.product.device',
197 partition))
198 partition_fingerprints.add(
199 runtime_build_info.GetPartitionFingerprint(partition))
200
201 partition_state.device.extend(sorted(partition_devices))
202 partition_state.build.extend(sorted(partition_fingerprints))
203
204 # TODO(xunchang) set the boot image's version with kmi. Note the boot
205 # image doesn't have a file map.
206 partition_state.version = build_info.GetPartitionBuildProp(
207 'ro.build.date.utc', partition)
208
209 # TODO(xunchang), we can save a call to ComputeRuntimeBuildInfos.
Tianjiea2076132020-08-19 17:25:32 -0700210 build_devices, build_fingerprints = \
211 CalculateRuntimeDevicesAndFingerprints(build_info, boot_variable_values)
212 device_state.device.extend(sorted(build_devices))
213 device_state.build.extend(sorted(build_fingerprints))
214 device_state.build_incremental = build_info.GetBuildProp(
215 'ro.build.version.incremental')
216
Tianjie2bb14862020-08-28 16:24:34 -0700217 UpdatePartitionStates(device_state.partition_state)
Tianjiea2076132020-08-19 17:25:32 -0700218
219 if is_post_build:
220 device_state.sdk_level = build_info.GetBuildProp(
221 'ro.build.version.sdk')
222 device_state.security_patch_level = build_info.GetBuildProp(
223 'ro.build.version.security_patch')
224 # Use the actual post-timestamp, even for a downgrade case.
225 device_state.timestamp = int(build_info.GetBuildProp('ro.build.date.utc'))
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400226
227
228def GetPackageMetadata(target_info, source_info=None):
Tianjiea2076132020-08-19 17:25:32 -0700229 """Generates and returns the metadata proto.
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400230
Tianjiea2076132020-08-19 17:25:32 -0700231 It generates a ota_metadata protobuf that contains the info to be written
232 into an OTA package (META-INF/com/android/metadata.pb). It also handles the
233 detection of downgrade / data wipe based on the global options.
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400234
235 Args:
236 target_info: The BuildInfo instance that holds the target build info.
237 source_info: The BuildInfo instance that holds the source build info, or
238 None if generating full OTA.
239
240 Returns:
Tianjiea2076132020-08-19 17:25:32 -0700241 A protobuf to be written into package metadata entry.
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400242 """
243 assert isinstance(target_info, BuildInfo)
244 assert source_info is None or isinstance(source_info, BuildInfo)
245
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400246 boot_variable_values = {}
247 if OPTIONS.boot_variable_file:
248 d = LoadDictionaryFromFile(OPTIONS.boot_variable_file)
249 for key, values in d.items():
250 boot_variable_values[key] = [val.strip() for val in values.split(',')]
251
Tianjiea2076132020-08-19 17:25:32 -0700252 metadata_proto = ota_metadata_pb2.OtaMetadata()
253 # TODO(xunchang) some fields, e.g. post-device isn't necessary. We can
254 # consider skipping them if they aren't used by clients.
255 UpdateDeviceState(metadata_proto.postcondition, target_info,
256 boot_variable_values, True)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400257
258 if target_info.is_ab and not OPTIONS.force_non_ab:
Tianjiea2076132020-08-19 17:25:32 -0700259 metadata_proto.type = ota_metadata_pb2.OtaMetadata.AB
260 metadata_proto.required_cache = 0
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400261 else:
Tianjiea2076132020-08-19 17:25:32 -0700262 metadata_proto.type = ota_metadata_pb2.OtaMetadata.BLOCK
263 # cache requirement will be updated by the non-A/B codes.
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400264
265 if OPTIONS.wipe_user_data:
Tianjiea2076132020-08-19 17:25:32 -0700266 metadata_proto.wipe = True
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400267
268 if OPTIONS.retrofit_dynamic_partitions:
Tianjiea2076132020-08-19 17:25:32 -0700269 metadata_proto.retrofit_dynamic_partitions = True
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400270
271 is_incremental = source_info is not None
272 if is_incremental:
Tianjiea2076132020-08-19 17:25:32 -0700273 UpdateDeviceState(metadata_proto.precondition, source_info,
274 boot_variable_values, False)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400275 else:
Tianjiea2076132020-08-19 17:25:32 -0700276 metadata_proto.precondition.device.extend(
277 metadata_proto.postcondition.device)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400278
279 # Detect downgrades and set up downgrade flags accordingly.
280 if is_incremental:
Tianjiea2076132020-08-19 17:25:32 -0700281 HandleDowngradeMetadata(metadata_proto, target_info, source_info)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400282
Tianjiea2076132020-08-19 17:25:32 -0700283 return metadata_proto
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400284
285
Tianjiea2076132020-08-19 17:25:32 -0700286def BuildLegacyOtaMetadata(metadata_proto):
287 """Converts the metadata proto to a legacy metadata dict.
288
289 This metadata dict is used to build the legacy metadata text file for
290 backward compatibility. We won't add new keys to the legacy metadata format.
291 If new information is needed, we should add it as a new field in OtaMetadata
292 proto definition.
293 """
294
295 separator = '|'
296
297 metadata_dict = {}
298 if metadata_proto.type == ota_metadata_pb2.OtaMetadata.AB:
299 metadata_dict['ota-type'] = 'AB'
300 elif metadata_proto.type == ota_metadata_pb2.OtaMetadata.BLOCK:
301 metadata_dict['ota-type'] = 'BLOCK'
302 if metadata_proto.wipe:
303 metadata_dict['ota-wipe'] = 'yes'
304 if metadata_proto.retrofit_dynamic_partitions:
305 metadata_dict['ota-retrofit-dynamic-partitions'] = 'yes'
306 if metadata_proto.downgrade:
307 metadata_dict['ota-downgrade'] = 'yes'
308
309 metadata_dict['ota-required-cache'] = str(metadata_proto.required_cache)
310
311 post_build = metadata_proto.postcondition
312 metadata_dict['post-build'] = separator.join(post_build.build)
313 metadata_dict['post-build-incremental'] = post_build.build_incremental
314 metadata_dict['post-sdk-level'] = post_build.sdk_level
315 metadata_dict['post-security-patch-level'] = post_build.security_patch_level
316 metadata_dict['post-timestamp'] = str(post_build.timestamp)
317
318 pre_build = metadata_proto.precondition
319 metadata_dict['pre-device'] = separator.join(pre_build.device)
320 # incremental updates
321 if len(pre_build.build) != 0:
322 metadata_dict['pre-build'] = separator.join(pre_build.build)
323 metadata_dict['pre-build-incremental'] = pre_build.build_incremental
324
Kelvin Zhang05ff7052021-02-10 09:13:26 -0500325 if metadata_proto.spl_downgrade:
326 metadata_dict['spl-downgrade'] = 'yes'
Tianjiea2076132020-08-19 17:25:32 -0700327 metadata_dict.update(metadata_proto.property_files)
328
329 return metadata_dict
330
331
332def HandleDowngradeMetadata(metadata_proto, target_info, source_info):
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400333 # Only incremental OTAs are allowed to reach here.
334 assert OPTIONS.incremental_source is not None
335
336 post_timestamp = target_info.GetBuildProp("ro.build.date.utc")
337 pre_timestamp = source_info.GetBuildProp("ro.build.date.utc")
338 is_downgrade = int(post_timestamp) < int(pre_timestamp)
339
Kelvin Zhang05ff7052021-02-10 09:13:26 -0500340 if OPTIONS.spl_downgrade:
341 metadata_proto.spl_downgrade = True
342
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400343 if OPTIONS.downgrade:
344 if not is_downgrade:
345 raise RuntimeError(
346 "--downgrade or --override_timestamp specified but no downgrade "
347 "detected: pre: %s, post: %s" % (pre_timestamp, post_timestamp))
Tianjiea2076132020-08-19 17:25:32 -0700348 metadata_proto.downgrade = True
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400349 else:
350 if is_downgrade:
351 raise RuntimeError(
352 "Downgrade detected based on timestamp check: pre: %s, post: %s. "
353 "Need to specify --override_timestamp OR --downgrade to allow "
354 "building the incremental." % (pre_timestamp, post_timestamp))
355
356
Tianjie2bb14862020-08-28 16:24:34 -0700357def ComputeRuntimeBuildInfos(default_build_info, boot_variable_values):
358 """Returns a set of build info objects that may exist during runtime."""
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400359
Tianjie2bb14862020-08-28 16:24:34 -0700360 build_info_set = {default_build_info}
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400361 if not boot_variable_values:
Tianjie2bb14862020-08-28 16:24:34 -0700362 return build_info_set
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400363
364 # Calculate all possible combinations of the values for the boot variables.
365 keys = boot_variable_values.keys()
366 value_list = boot_variable_values.values()
367 combinations = [dict(zip(keys, values))
368 for values in itertools.product(*value_list)]
369 for placeholder_values in combinations:
370 # Reload the info_dict as some build properties may change their values
371 # based on the value of ro.boot* properties.
Tianjie2bb14862020-08-28 16:24:34 -0700372 info_dict = copy.deepcopy(default_build_info.info_dict)
Yifan Hong5057b952021-01-07 14:09:57 -0800373 for partition in PARTITIONS_WITH_BUILD_PROP:
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400374 partition_prop_key = "{}.build.prop".format(partition)
375 input_file = info_dict[partition_prop_key].input_file
TJ Rhoades6f488e92022-05-01 22:16:22 -0700376 ramdisk = GetRamdiskFormat(info_dict)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400377 if isinstance(input_file, zipfile.ZipFile):
Kelvin Zhang928c2342020-09-22 16:15:57 -0400378 with zipfile.ZipFile(input_file.filename, allowZip64=True) as input_zip:
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400379 info_dict[partition_prop_key] = \
380 PartitionBuildProps.FromInputFile(input_zip, partition,
TJ Rhoades6f488e92022-05-01 22:16:22 -0700381 placeholder_values,
382 ramdisk)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400383 else:
384 info_dict[partition_prop_key] = \
385 PartitionBuildProps.FromInputFile(input_file, partition,
TJ Rhoades6f488e92022-05-01 22:16:22 -0700386 placeholder_values,
387 ramdisk)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400388 info_dict["build.prop"] = info_dict["system.build.prop"]
Tianjie2bb14862020-08-28 16:24:34 -0700389 build_info_set.add(BuildInfo(info_dict, default_build_info.oem_dicts))
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400390
Tianjie2bb14862020-08-28 16:24:34 -0700391 return build_info_set
392
393
394def CalculateRuntimeDevicesAndFingerprints(default_build_info,
395 boot_variable_values):
396 """Returns a tuple of sets for runtime devices and fingerprints"""
397
398 device_names = set()
399 fingerprints = set()
400 build_info_set = ComputeRuntimeBuildInfos(default_build_info,
401 boot_variable_values)
402 for runtime_build_info in build_info_set:
403 device_names.add(runtime_build_info.device)
404 fingerprints.add(runtime_build_info.fingerprint)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400405 return device_names, fingerprints
406
407
Kelvin Zhang25ab9982021-06-22 09:51:34 -0400408def GetZipEntryOffset(zfp, entry_info):
409 """Get offset to a beginning of a particular zip entry
410 Args:
411 fp: zipfile.ZipFile
412 entry_info: zipfile.ZipInfo
413
414 Returns:
415 (offset, size) tuple
416 """
417 # Don't use len(entry_info.extra). Because that returns size of extra
418 # fields in central directory. We need to look at local file directory,
419 # as these two might have different sizes.
420
421 # We cannot work with zipfile.ZipFile instances, we need a |fp| for the underlying file.
422 zfp = zfp.fp
423 zfp.seek(entry_info.header_offset)
424 data = zfp.read(zipfile.sizeFileHeader)
425 fheader = struct.unpack(zipfile.structFileHeader, data)
426 # Last two fields of local file header are filename length and
427 # extra length
428 filename_len = fheader[-2]
429 extra_len = fheader[-1]
430 offset = entry_info.header_offset
431 offset += zipfile.sizeFileHeader
432 offset += filename_len + extra_len
433 size = entry_info.file_size
434 return (offset, size)
435
436
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400437class PropertyFiles(object):
438 """A class that computes the property-files string for an OTA package.
439
440 A property-files string is a comma-separated string that contains the
441 offset/size info for an OTA package. The entries, which must be ZIP_STORED,
442 can be fetched directly with the package URL along with the offset/size info.
443 These strings can be used for streaming A/B OTAs, or allowing an updater to
444 download package metadata entry directly, without paying the cost of
445 downloading entire package.
446
447 Computing the final property-files string requires two passes. Because doing
448 the whole package signing (with signapk.jar) will possibly reorder the ZIP
449 entries, which may in turn invalidate earlier computed ZIP entry offset/size
450 values.
451
452 This class provides functions to be called for each pass. The general flow is
453 as follows.
454
455 property_files = PropertyFiles()
456 # The first pass, which writes placeholders before doing initial signing.
457 property_files.Compute()
458 SignOutput()
459
460 # The second pass, by replacing the placeholders with actual data.
461 property_files.Finalize()
462 SignOutput()
463
464 And the caller can additionally verify the final result.
465
466 property_files.Verify()
467 """
468
469 def __init__(self):
470 self.name = None
471 self.required = ()
472 self.optional = ()
473
474 def Compute(self, input_zip):
475 """Computes and returns a property-files string with placeholders.
476
477 We reserve extra space for the offset and size of the metadata entry itself,
478 although we don't know the final values until the package gets signed.
479
480 Args:
481 input_zip: The input ZIP file.
482
483 Returns:
484 A string with placeholders for the metadata offset/size info, e.g.
485 "payload.bin:679:343,payload_properties.txt:378:45,metadata: ".
486 """
487 return self.GetPropertyFilesString(input_zip, reserve_space=True)
488
489 class InsufficientSpaceException(Exception):
490 pass
491
492 def Finalize(self, input_zip, reserved_length):
493 """Finalizes a property-files string with actual METADATA offset/size info.
494
495 The input ZIP file has been signed, with the ZIP entries in the desired
496 place (signapk.jar will possibly reorder the ZIP entries). Now we compute
497 the ZIP entry offsets and construct the property-files string with actual
498 data. Note that during this process, we must pad the property-files string
499 to the reserved length, so that the METADATA entry size remains the same.
500 Otherwise the entries' offsets and sizes may change again.
501
502 Args:
503 input_zip: The input ZIP file.
504 reserved_length: The reserved length of the property-files string during
505 the call to Compute(). The final string must be no more than this
506 size.
507
508 Returns:
509 A property-files string including the metadata offset/size info, e.g.
510 "payload.bin:679:343,payload_properties.txt:378:45,metadata:69:379 ".
511
512 Raises:
513 InsufficientSpaceException: If the reserved length is insufficient to hold
514 the final string.
515 """
516 result = self.GetPropertyFilesString(input_zip, reserve_space=False)
517 if len(result) > reserved_length:
518 raise self.InsufficientSpaceException(
519 'Insufficient reserved space: reserved={}, actual={}'.format(
520 reserved_length, len(result)))
521
522 result += ' ' * (reserved_length - len(result))
523 return result
524
525 def Verify(self, input_zip, expected):
526 """Verifies the input ZIP file contains the expected property-files string.
527
528 Args:
529 input_zip: The input ZIP file.
530 expected: The property-files string that's computed from Finalize().
531
532 Raises:
533 AssertionError: On finding a mismatch.
534 """
535 actual = self.GetPropertyFilesString(input_zip)
536 assert actual == expected, \
537 "Mismatching streaming metadata: {} vs {}.".format(actual, expected)
538
539 def GetPropertyFilesString(self, zip_file, reserve_space=False):
540 """
541 Constructs the property-files string per request.
542
543 Args:
544 zip_file: The input ZIP file.
545 reserved_length: The reserved length of the property-files string.
546
547 Returns:
548 A property-files string including the metadata offset/size info, e.g.
549 "payload.bin:679:343,payload_properties.txt:378:45,metadata: ".
550 """
551
552 def ComputeEntryOffsetSize(name):
553 """Computes the zip entry offset and size."""
554 info = zip_file.getinfo(name)
Kelvin Zhang25ab9982021-06-22 09:51:34 -0400555 (offset, size) = GetZipEntryOffset(zip_file, info)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400556 return '%s:%d:%d' % (os.path.basename(name), offset, size)
557
558 tokens = []
559 tokens.extend(self._GetPrecomputed(zip_file))
560 for entry in self.required:
561 tokens.append(ComputeEntryOffsetSize(entry))
562 for entry in self.optional:
563 if entry in zip_file.namelist():
564 tokens.append(ComputeEntryOffsetSize(entry))
565
566 # 'META-INF/com/android/metadata' is required. We don't know its actual
567 # offset and length (as well as the values for other entries). So we reserve
568 # 15-byte as a placeholder ('offset:length'), which is sufficient to cover
569 # the space for metadata entry. Because 'offset' allows a max of 10-digit
570 # (i.e. ~9 GiB), with a max of 4-digit for the length. Note that all the
571 # reserved space serves the metadata entry only.
572 if reserve_space:
573 tokens.append('metadata:' + ' ' * 15)
Tianjiea2076132020-08-19 17:25:32 -0700574 tokens.append('metadata.pb:' + ' ' * 15)
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400575 else:
576 tokens.append(ComputeEntryOffsetSize(METADATA_NAME))
Luca Stefanib6075c52021-11-03 17:10:54 +0100577 if METADATA_PROTO_NAME in zip_file.namelist():
578 tokens.append(ComputeEntryOffsetSize(METADATA_PROTO_NAME))
Kelvin Zhangcff4d762020-07-29 16:37:51 -0400579
580 return ','.join(tokens)
581
582 def _GetPrecomputed(self, input_zip):
583 """Computes the additional tokens to be included into the property-files.
584
585 This applies to tokens without actual ZIP entries, such as
586 payload_metadata.bin. We want to expose the offset/size to updaters, so
587 that they can download the payload metadata directly with the info.
588
589 Args:
590 input_zip: The input zip file.
591
592 Returns:
593 A list of strings (tokens) to be added to the property-files string.
594 """
595 # pylint: disable=no-self-use
596 # pylint: disable=unused-argument
597 return []
598
599
600def SignOutput(temp_zip_name, output_zip_name):
601 pw = OPTIONS.key_passwords[OPTIONS.package_key]
602
603 SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw,
604 whole_file=True)
Tianjiea5fca032021-06-01 22:06:28 -0700605
606
607def ConstructOtaApexInfo(target_zip, source_file=None):
608 """If applicable, add the source version to the apex info."""
609
610 def _ReadApexInfo(input_zip):
611 if "META/apex_info.pb" not in input_zip.namelist():
612 logger.warning("target_file doesn't contain apex_info.pb %s", input_zip)
613 return None
614
615 with input_zip.open("META/apex_info.pb", "r") as zfp:
616 return zfp.read()
617
618 target_apex_string = _ReadApexInfo(target_zip)
619 # Return early if the target apex info doesn't exist or is empty.
620 if not target_apex_string:
621 return target_apex_string
622
623 # If the source apex info isn't available, just return the target info
624 if not source_file:
625 return target_apex_string
626
627 with zipfile.ZipFile(source_file, "r", allowZip64=True) as source_zip:
628 source_apex_string = _ReadApexInfo(source_zip)
629 if not source_apex_string:
630 return target_apex_string
631
632 source_apex_proto = ota_metadata_pb2.ApexMetadata()
633 source_apex_proto.ParseFromString(source_apex_string)
634 source_apex_versions = {apex.package_name: apex.version for apex in
635 source_apex_proto.apex_info}
636
637 # If the apex package is available in the source build, initialize the source
638 # apex version.
639 target_apex_proto = ota_metadata_pb2.ApexMetadata()
640 target_apex_proto.ParseFromString(target_apex_string)
641 for target_apex in target_apex_proto.apex_info:
642 name = target_apex.package_name
643 if name in source_apex_versions:
644 target_apex.source_version = source_apex_versions[name]
645
646 return target_apex_proto.SerializeToString()
Kelvin Zhang410bb382022-01-06 09:15:54 -0800647
648
Kelvin Zhangf2728d62022-01-10 11:42:36 -0800649def IsLz4diffCompatible(source_file: str, target_file: str):
650 """Check whether lz4diff versions in two builds are compatible
651
652 Args:
653 source_file: Path to source build's target_file.zip
654 target_file: Path to target build's target_file.zip
655
656 Returns:
657 bool true if and only if lz4diff versions are compatible
658 """
659 if source_file is None or target_file is None:
660 return False
661 # Right now we enable lz4diff as long as source build has liblz4.so.
662 # In the future we might introduce version system to lz4diff as well.
663 if zipfile.is_zipfile(source_file):
664 with zipfile.ZipFile(source_file, "r") as zfp:
665 return "META/liblz4.so" in zfp.namelist()
666 else:
667 assert os.path.isdir(source_file)
668 return os.path.exists(os.path.join(source_file, "META", "liblz4.so"))
669
670
Kelvin Zhang410bb382022-01-06 09:15:54 -0800671def IsZucchiniCompatible(source_file: str, target_file: str):
672 """Check whether zucchini versions in two builds are compatible
673
674 Args:
675 source_file: Path to source build's target_file.zip
676 target_file: Path to target build's target_file.zip
677
678 Returns:
679 bool true if and only if zucchini versions are compatible
680 """
681 if source_file is None or target_file is None:
682 return False
683 assert os.path.exists(source_file)
684 assert os.path.exists(target_file)
685
686 assert zipfile.is_zipfile(source_file) or os.path.isdir(source_file)
687 assert zipfile.is_zipfile(target_file) or os.path.isdir(target_file)
688 _ZUCCHINI_CONFIG_ENTRY_NAME = "META/zucchini_config.txt"
689
690 def ReadEntry(path, entry):
691 # Read an entry inside a .zip file or extracted dir of .zip file
692 if zipfile.is_zipfile(path):
693 with zipfile.ZipFile(path, "r", allowZip64=True) as zfp:
694 if entry in zfp.namelist():
695 return zfp.read(entry).decode()
696 else:
697 entry_path = os.path.join(entry, path)
698 if os.path.exists(entry_path):
699 with open(entry_path, "r") as fp:
700 return fp.read()
HÃ¥kan Kvist3db1ef62022-05-03 10:19:41 +0200701 return False
702 sourceEntry = ReadEntry(source_file, _ZUCCHINI_CONFIG_ENTRY_NAME)
703 targetEntry = ReadEntry(target_file, _ZUCCHINI_CONFIG_ENTRY_NAME)
704 return sourceEntry and targetEntry and sourceEntry == targetEntry