blob: 712e3f7cdec3bb1f75e10870b65de378ed2b4019 [file] [log] [blame]
Bill Peckhame9eb5f92019-02-01 15:52:10 -08001#!/usr/bin/env python
2#
3# Copyright (C) 2019 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License"); you may not
6# use this file except in compliance with the License. You may obtain a copy of
7# the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14# License for the specific language governing permissions and limitations under
15# the License.
16
17"""
18This script merges two partial target files packages (one of which contains
19system files, and the other contains non-system files) together, producing a
20complete target files package that can be used to generate an OTA package.
21
22Usage: merge_target_files.py [args]
23
24 --system-target-files system-target-files-zip-archive
25 The input target files package containing system bits. This is a zip
26 archive.
27
Daniel Norman2c99c5b2019-03-07 13:01:48 -080028 --system-item-list system-item-list-file
29 The optional path to a newline-separated config file that replaces the
30 contents of default_system_item_list if provided.
31
32 --system-misc-info-keys system-misc-info-keys-file
33 The optional path to a newline-separated config file that replaces the
34 contents of default_system_misc_info_keys if provided.
35
Bill Peckhame9eb5f92019-02-01 15:52:10 -080036 --other-target-files other-target-files-zip-archive
37 The input target files package containing other bits. This is a zip
38 archive.
39
Daniel Norman2c99c5b2019-03-07 13:01:48 -080040 --other-item-list other-item-list-file
41 The optional path to a newline-separated config file that replaces the
42 contents of default_other_item_list if provided.
43
Bill Peckhame9eb5f92019-02-01 15:52:10 -080044 --output-target-files output-target-files-package
45 The output merged target files package. Also a zip archive.
Daniel Normana4911da2019-03-15 14:36:21 -070046
47 --rebuild_recovery
48 Rebuild the recovery patch used by non-A/B devices and write it to the
49 system image.
Bill Peckham364c1cc2019-03-29 18:27:23 -070050
51 --keep-tmp
52 Keep tempoary files for debugging purposes.
Bill Peckhame9eb5f92019-02-01 15:52:10 -080053"""
54
55from __future__ import print_function
56
Bill Peckhame9eb5f92019-02-01 15:52:10 -080057import fnmatch
58import logging
59import os
60import sys
61import zipfile
62
63import common
64import add_img_to_target_files
65
66logger = logging.getLogger(__name__)
67OPTIONS = common.OPTIONS
68OPTIONS.verbose = True
Bill Peckhamf753e152019-02-19 18:02:46 -080069OPTIONS.system_target_files = None
Daniel Norman2c99c5b2019-03-07 13:01:48 -080070OPTIONS.system_item_list = None
71OPTIONS.system_misc_info_keys = None
Bill Peckhamf753e152019-02-19 18:02:46 -080072OPTIONS.other_target_files = None
Daniel Norman2c99c5b2019-03-07 13:01:48 -080073OPTIONS.other_item_list = None
Bill Peckhamf753e152019-02-19 18:02:46 -080074OPTIONS.output_target_files = None
Daniel Normana4911da2019-03-15 14:36:21 -070075OPTIONS.rebuild_recovery = False
Bill Peckhamf753e152019-02-19 18:02:46 -080076OPTIONS.keep_tmp = False
Bill Peckhame9eb5f92019-02-01 15:52:10 -080077
Daniel Norman2c99c5b2019-03-07 13:01:48 -080078# default_system_item_list is a list of items to extract from the partial
Bill Peckhame9eb5f92019-02-01 15:52:10 -080079# system target files package as is, meaning these items will land in the
80# output target files package exactly as they appear in the input partial
81# system target files package.
82
Daniel Norman2c99c5b2019-03-07 13:01:48 -080083default_system_item_list = [
Bill Peckhame9eb5f92019-02-01 15:52:10 -080084 'META/apkcerts.txt',
85 'META/filesystem_config.txt',
86 'META/root_filesystem_config.txt',
87 'META/system_manifest.xml',
88 'META/system_matrix.xml',
89 'META/update_engine_config.txt',
90 'PRODUCT/*',
91 'ROOT/*',
92 'SYSTEM/*',
93]
94
95# system_extract_special_item_list is a list of items to extract from the
96# partial system target files package that need some special processing, such
97# as some sort of combination with items from the partial other target files
98# package.
99
100system_extract_special_item_list = [
101 'META/*',
102]
103
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800104# default_system_misc_info_keys is a list of keys to obtain from the system instance of
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800105# META/misc_info.txt. The remaining keys from the other instance.
106
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800107default_system_misc_info_keys = [
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800108 'avb_system_hashtree_enable',
109 'avb_system_add_hashtree_footer_args',
110 'avb_system_key_path',
111 'avb_system_algorithm',
112 'avb_system_rollback_index_location',
113 'avb_product_hashtree_enable',
114 'avb_product_add_hashtree_footer_args',
115 'avb_product_services_hashtree_enable',
116 'avb_product_services_add_hashtree_footer_args',
117 'system_root_image',
118 'root_dir',
119 'ab_update',
120 'default_system_dev_certificate',
121 'system_size',
122]
123
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800124# default_other_item_list is a list of items to extract from the partial
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800125# other target files package as is, meaning these items will land in the output
126# target files package exactly as they appear in the input partial other target
127# files package.
128
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800129default_other_item_list = [
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800130 'META/boot_filesystem_config.txt',
131 'META/otakeys.txt',
132 'META/releasetools.py',
133 'META/vendor_filesystem_config.txt',
134 'META/vendor_manifest.xml',
135 'META/vendor_matrix.xml',
136 'BOOT/*',
137 'DATA/*',
138 'ODM/*',
139 'OTA/android-info.txt',
140 'PREBUILT_IMAGES/*',
141 'RADIO/*',
142 'VENDOR/*',
143]
144
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800145# other_extract_special_item_list is a list of items to extract from the
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800146# partial other target files package that need some special processing, such as
147# some sort of combination with items from the partial system target files
148# package.
149
150other_extract_special_item_list = [
151 'META/*',
152]
153
154
155def extract_items(target_files, target_files_temp_dir, extract_item_list):
156 """Extract items from target files to temporary directory.
157
158 This function extracts from the specified target files zip archive into the
159 specified temporary directory, the items specified in the extract item list.
160
161 Args:
162 target_files: The target files zip archive from which to extract items.
163
164 target_files_temp_dir: The temporary directory where the extracted items
165 will land.
166
167 extract_item_list: A list of items to extract.
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800168 """
169
170 logger.info('extracting from %s', target_files)
171
172 # Filter the extract_item_list to remove any items that do not exist in the
173 # zip file. Otherwise, the extraction step will fail.
174
175 with zipfile.ZipFile(
176 target_files,
177 'r',
178 allowZip64=True) as target_files_zipfile:
179 target_files_namelist = target_files_zipfile.namelist()
180
181 filtered_extract_item_list = []
182 for pattern in extract_item_list:
183 matching_namelist = fnmatch.filter(target_files_namelist, pattern)
184 if not matching_namelist:
185 logger.warning('no match for %s', pattern)
186 else:
187 filtered_extract_item_list.append(pattern)
188
Bill Peckham8ff3fbd2019-02-22 10:57:43 -0800189 # Extract from target_files into target_files_temp_dir the
190 # filtered_extract_item_list.
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800191
Bill Peckham8ff3fbd2019-02-22 10:57:43 -0800192 common.UnzipToDir(
193 target_files,
194 target_files_temp_dir,
195 filtered_extract_item_list)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800196
197
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800198def read_config_list(config_file_path):
199 """Reads a config file into a list of strings.
200
201 Expects the file to be newline-separated.
202
203 Args:
204 config_file_path: The path to the config file to open and read.
205 """
206 with open(config_file_path) as config_file:
207 return config_file.read().splitlines()
208
209
Daniel Norman19b9fe92019-03-19 14:48:02 -0700210def validate_config_lists(
211 system_item_list,
212 system_misc_info_keys,
213 other_item_list):
Daniel Normane5964522019-03-19 10:32:03 -0700214 """Performs validations on the merge config lists.
215
216 Args:
217 system_item_list: The list of items to extract from the partial
218 system target files package as is.
219
Daniel Norman19b9fe92019-03-19 14:48:02 -0700220 system_misc_info_keys: A list of keys to obtain from the system instance
221 of META/misc_info.txt. The remaining keys from the other instance.
222
Daniel Normane5964522019-03-19 10:32:03 -0700223 other_item_list: The list of items to extract from the partial
224 other target files package as is.
225
226 Returns:
227 False if a validation fails, otherwise true.
228 """
229 default_combined_item_set = set(default_system_item_list)
230 default_combined_item_set.update(default_other_item_list)
231
232 combined_item_set = set(system_item_list)
233 combined_item_set.update(other_item_list)
234
235 # Check that the merge config lists are not missing any item specified
236 # by the default config lists.
237 difference = default_combined_item_set.difference(combined_item_set)
238 if difference:
239 logger.error('Missing merge config items: %s' % list(difference))
240 logger.error('Please ensure missing items are in either the '
241 'system-item-list or other-item-list files provided to '
242 'this script.')
243 return False
244
Daniel Norman19b9fe92019-03-19 14:48:02 -0700245 if ('dynamic_partition_list' in system_misc_info_keys) or (
246 'super_partition_groups' in system_misc_info_keys):
247 logger.error('Dynamic partition misc info keys should come from '
248 'the other instance of META/misc_info.txt.')
249 return False
250
Daniel Normane5964522019-03-19 10:32:03 -0700251 return True
252
253
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800254def process_ab_partitions_txt(
255 system_target_files_temp_dir,
256 other_target_files_temp_dir,
257 output_target_files_temp_dir):
258 """Perform special processing for META/ab_partitions.txt
259
260 This function merges the contents of the META/ab_partitions.txt files from
261 the system directory and the other directory, placing the merged result in
262 the output directory. The precondition in that the files are already
263 extracted. The post condition is that the output META/ab_partitions.txt
264 contains the merged content. The format for each ab_partitions.txt a one
265 partition name per line. The output file contains the union of the parition
266 names.
267
268 Args:
269 system_target_files_temp_dir: The name of a directory containing the
270 special items extracted from the system target files package.
271
272 other_target_files_temp_dir: The name of a directory containing the
273 special items extracted from the other target files package.
274
275 output_target_files_temp_dir: The name of a directory that will be used
276 to create the output target files package after all the special cases
277 are processed.
278 """
279
280 system_ab_partitions_txt = os.path.join(
281 system_target_files_temp_dir, 'META', 'ab_partitions.txt')
282
283 other_ab_partitions_txt = os.path.join(
284 other_target_files_temp_dir, 'META', 'ab_partitions.txt')
285
286 with open(system_ab_partitions_txt) as f:
287 system_ab_partitions = f.read().splitlines()
288
289 with open(other_ab_partitions_txt) as f:
290 other_ab_partitions = f.read().splitlines()
291
292 output_ab_partitions = set(system_ab_partitions + other_ab_partitions)
293
294 output_ab_partitions_txt = os.path.join(
295 output_target_files_temp_dir, 'META', 'ab_partitions.txt')
296
297 with open(output_ab_partitions_txt, 'w') as output:
298 for partition in sorted(output_ab_partitions):
299 output.write('%s\n' % partition)
300
301
Bill Peckham364c1cc2019-03-29 18:27:23 -0700302def append_recovery_to_filesystem_config(output_target_files_temp_dir):
303 """Perform special processing for META/filesystem_config.txt
304
305 This function appends recovery information to META/filesystem_config.txt
306 so that recovery patch regeneration will succeed.
307
308 Args:
309 output_target_files_temp_dir: The name of a directory that will be used
310 to create the output target files package after all the special cases
311 are processed. We find filesystem_config.txt here.
312 """
313
314 filesystem_config_txt = os.path.join(
315 output_target_files_temp_dir,
316 'META',
317 'filesystem_config.txt')
318
319 with open(filesystem_config_txt, 'a') as f:
320 # TODO(bpeckham) this data is hard coded. It should be generated
321 # programmatically.
322 f.write(
323 'system/bin/install-recovery.sh 0 0 750 '
324 'selabel=u:object_r:install_recovery_exec:s0 capabilities=0x0\n')
325 f.write(
326 'system/recovery-from-boot.p 0 0 644 '
327 'selabel=u:object_r:system_file:s0 capabilities=0x0\n')
328 f.write(
329 'system/etc/recovery.img 0 0 440 '
330 'selabel=u:object_r:install_recovery_exec:s0 capabilities=0x0\n')
331
332
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800333def process_misc_info_txt(
334 system_target_files_temp_dir,
335 other_target_files_temp_dir,
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800336 output_target_files_temp_dir,
337 system_misc_info_keys):
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800338 """Perform special processing for META/misc_info.txt
339
340 This function merges the contents of the META/misc_info.txt files from the
341 system directory and the other directory, placing the merged result in the
342 output directory. The precondition in that the files are already extracted.
343 The post condition is that the output META/misc_info.txt contains the merged
344 content.
345
346 Args:
347 system_target_files_temp_dir: The name of a directory containing the
348 special items extracted from the system target files package.
349
350 other_target_files_temp_dir: The name of a directory containing the
351 special items extracted from the other target files package.
352
353 output_target_files_temp_dir: The name of a directory that will be used
354 to create the output target files package after all the special cases
355 are processed.
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800356
357 system_misc_info_keys: A list of keys to obtain from the system instance
358 of META/misc_info.txt. The remaining keys from the other instance.
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800359 """
360
361 def read_helper(d):
362 misc_info_txt = os.path.join(d, 'META', 'misc_info.txt')
363 with open(misc_info_txt) as f:
364 return list(f.read().splitlines())
365
366 system_info_dict = common.LoadDictionaryFromLines(
367 read_helper(system_target_files_temp_dir))
368
369 # We take most of the misc info from the other target files.
370
371 merged_info_dict = common.LoadDictionaryFromLines(
372 read_helper(other_target_files_temp_dir))
373
374 # Replace certain values in merged_info_dict with values from
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800375 # system_info_dict.
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800376
377 for key in system_misc_info_keys:
378 merged_info_dict[key] = system_info_dict[key]
379
Daniel Norman19b9fe92019-03-19 14:48:02 -0700380 # Merge misc info keys used for Dynamic Partitions.
381 if (merged_info_dict.get('use_dynamic_partitions') == 'true') and (
382 system_info_dict.get('use_dynamic_partitions') == 'true'):
383 merged_info_dict['dynamic_partition_list'] = '%s %s' % (
384 system_info_dict.get('dynamic_partition_list', ''),
385 merged_info_dict.get('dynamic_partition_list', ''))
386 # Partition groups and group sizes are defined by the other (non-system)
387 # misc info file because these values may vary for each board that uses
388 # a shared system image.
389 for partition_group in merged_info_dict['super_partition_groups'].split(' '):
390 if ('super_%s_group_size' % partition_group) not in merged_info_dict:
391 raise common.ExternalError(
392 'Other META/misc_info.txt does not contain required key '
393 'super_%s_group_size.' % partition_group)
394 key = 'super_%s_partition_list' % partition_group
395 merged_info_dict[key] = '%s %s' % (
396 system_info_dict.get(key, ''),
397 merged_info_dict.get(key, ''))
Tao Bao06429d72019-06-28 11:00:05 -0700398 # Ensure that add_img_to_target_files rebuilds super split images for
399 # devices that retrofit dynamic partitions. This flag may have been set to
400 # false in the partial builds to prevent duplicate building of super.img.
Daniel Normanef212272019-06-10 12:50:19 -0700401 merged_dict['build_super_partition'] = 'true'
Daniel Norman19b9fe92019-03-19 14:48:02 -0700402
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800403 output_misc_info_txt = os.path.join(
404 output_target_files_temp_dir,
405 'META', 'misc_info.txt')
406
407 sorted_keys = sorted(merged_info_dict.keys())
408
409 with open(output_misc_info_txt, 'w') as output:
410 for key in sorted_keys:
411 output.write('{}={}\n'.format(key, merged_info_dict[key]))
412
413
414def process_file_contexts_bin(temp_dir, output_target_files_temp_dir):
415 """Perform special processing for META/file_contexts.bin.
416
417 This function combines plat_file_contexts and vendor_file_contexts, which are
418 expected to already be extracted in temp_dir, to produce a merged
419 file_contexts.bin that will land in temp_dir at META/file_contexts.bin.
420
421 Args:
422 temp_dir: The name of a scratch directory that this function can use for
423 intermediate files generated during processing.
424
425 output_target_files_temp_dir: The name of the working directory that must
426 already contain plat_file_contexts and vendor_file_contexts (in the
427 appropriate sub directories), and to which META/file_contexts.bin will be
428 written.
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800429 """
430
431 # To create a merged file_contexts.bin file, we use the system and vendor
432 # file contexts files as input, the m4 tool to combine them, the sorting tool
433 # to sort, and finally the sefcontext_compile tool to generate the final
434 # output. We currently omit a checkfc step since the files had been checked
435 # as part of the build.
436
437 # The m4 step concatenates the two input files contexts files. Since m4
438 # writes to stdout, we receive that into an array of bytes, and then write it
439 # to a file.
440
441 # Collect the file contexts that we're going to combine from SYSTEM, VENDOR,
442 # PRODUCT, and ODM. We require SYSTEM and VENDOR, but others are optional.
443
444 file_contexts_list = []
445
446 for partition in ['SYSTEM', 'VENDOR', 'PRODUCT', 'ODM']:
447 prefix = 'plat' if partition == 'SYSTEM' else partition.lower()
448
449 file_contexts = os.path.join(
450 output_target_files_temp_dir,
451 partition, 'etc', 'selinux', prefix + '_file_contexts')
452
453 mandatory = partition in ['SYSTEM', 'VENDOR']
454
455 if mandatory or os.path.isfile(file_contexts):
456 file_contexts_list.append(file_contexts)
457 else:
458 logger.warning('file not found: %s', file_contexts)
459
460 command = ['m4', '--fatal-warnings', '-s'] + file_contexts_list
461
462 merged_content = common.RunAndCheckOutput(command, verbose=False)
463
464 merged_file_contexts_txt = os.path.join(temp_dir, 'merged_file_contexts.txt')
465
466 with open(merged_file_contexts_txt, 'wb') as f:
467 f.write(merged_content)
468
469 # The sort step sorts the concatenated file.
470
471 sorted_file_contexts_txt = os.path.join(temp_dir, 'sorted_file_contexts.txt')
472 command = ['fc_sort', merged_file_contexts_txt, sorted_file_contexts_txt]
Bill Peckham889b0c62019-02-21 18:53:37 -0800473 common.RunAndWait(command, verbose=True)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800474
475 # Finally, the compile step creates the final META/file_contexts.bin.
476
477 file_contexts_bin = os.path.join(
478 output_target_files_temp_dir,
479 'META', 'file_contexts.bin')
480
481 command = [
482 'sefcontext_compile',
483 '-o', file_contexts_bin,
484 sorted_file_contexts_txt,
485 ]
486
Bill Peckham889b0c62019-02-21 18:53:37 -0800487 common.RunAndWait(command, verbose=True)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800488
489
490def process_special_cases(
491 temp_dir,
492 system_target_files_temp_dir,
493 other_target_files_temp_dir,
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800494 output_target_files_temp_dir,
Bill Peckham364c1cc2019-03-29 18:27:23 -0700495 system_misc_info_keys,
496 rebuild_recovery
497):
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800498 """Perform special-case processing for certain target files items.
499
500 Certain files in the output target files package require special-case
501 processing. This function performs all that special-case processing.
502
503 Args:
504 temp_dir: The name of a scratch directory that this function can use for
505 intermediate files generated during processing.
506
507 system_target_files_temp_dir: The name of a directory containing the
508 special items extracted from the system target files package.
509
510 other_target_files_temp_dir: The name of a directory containing the
511 special items extracted from the other target files package.
512
513 output_target_files_temp_dir: The name of a directory that will be used
514 to create the output target files package after all the special cases
515 are processed.
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800516
517 system_misc_info_keys: A list of keys to obtain from the system instance
518 of META/misc_info.txt. The remaining keys from the other instance.
Bill Peckham364c1cc2019-03-29 18:27:23 -0700519
520 rebuild_recovery: If true, rebuild the recovery patch used by non-A/B
521 devices and write it to the system image.
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800522 """
523
Bill Peckham364c1cc2019-03-29 18:27:23 -0700524 if 'ab_update' in system_misc_info_keys:
525 process_ab_partitions_txt(
526 system_target_files_temp_dir=system_target_files_temp_dir,
527 other_target_files_temp_dir=other_target_files_temp_dir,
528 output_target_files_temp_dir=output_target_files_temp_dir)
529
530 if rebuild_recovery:
531 append_recovery_to_filesystem_config(
532 output_target_files_temp_dir=output_target_files_temp_dir)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800533
534 process_misc_info_txt(
535 system_target_files_temp_dir=system_target_files_temp_dir,
536 other_target_files_temp_dir=other_target_files_temp_dir,
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800537 output_target_files_temp_dir=output_target_files_temp_dir,
538 system_misc_info_keys=system_misc_info_keys)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800539
Bill Peckham889b0c62019-02-21 18:53:37 -0800540 process_file_contexts_bin(
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800541 temp_dir=temp_dir,
542 output_target_files_temp_dir=output_target_files_temp_dir)
543
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800544
545def merge_target_files(
546 temp_dir,
547 system_target_files,
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800548 system_item_list,
549 system_misc_info_keys,
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800550 other_target_files,
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800551 other_item_list,
Daniel Normana4911da2019-03-15 14:36:21 -0700552 output_target_files,
553 rebuild_recovery):
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800554 """Merge two target files packages together.
555
556 This function takes system and other target files packages as input, performs
557 various file extractions, special case processing, and finally creates a
558 merged zip archive as output.
559
560 Args:
561 temp_dir: The name of a directory we use when we extract items from the
562 input target files packages, and also a scratch directory that we use for
563 temporary files.
564
565 system_target_files: The name of the zip archive containing the system
566 partial target files package.
567
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800568 system_item_list: The list of items to extract from the partial system
569 target files package as is, meaning these items will land in the output
570 target files package exactly as they appear in the input partial system
571 target files package.
572
573 system_misc_info_keys: The list of keys to obtain from the system instance
574 of META/misc_info.txt. The remaining keys from the other instance.
575
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800576 other_target_files: The name of the zip archive containing the other
577 partial target files package.
578
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800579 other_item_list: The list of items to extract from the partial other
580 target files package as is, meaning these items will land in the output
581 target files package exactly as they appear in the input partial other
582 target files package.
583
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800584 output_target_files: The name of the output zip archive target files
585 package created by merging system and other.
Daniel Normana4911da2019-03-15 14:36:21 -0700586
587 rebuild_recovery: If true, rebuild the recovery patch used by non-A/B
588 devices and write it to the system image.
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800589 """
590
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800591 logger.info(
592 'starting: merge system %s and other %s into output %s',
593 system_target_files,
594 other_target_files,
595 output_target_files)
596
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800597 # Create directory names that we'll use when we extract files from system,
598 # and other, and for zipping the final output.
599
600 system_target_files_temp_dir = os.path.join(temp_dir, 'system')
601 other_target_files_temp_dir = os.path.join(temp_dir, 'other')
602 output_target_files_temp_dir = os.path.join(temp_dir, 'output')
603
604 # Extract "as is" items from the input system partial target files package.
605 # We extract them directly into the output temporary directory since the
606 # items do not need special case processing.
607
Bill Peckham889b0c62019-02-21 18:53:37 -0800608 extract_items(
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800609 target_files=system_target_files,
610 target_files_temp_dir=output_target_files_temp_dir,
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800611 extract_item_list=system_item_list)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800612
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800613 # Extract "as is" items from the input other partial target files package. We
614 # extract them directly into the output temporary directory since the items
615 # do not need special case processing.
616
Bill Peckham889b0c62019-02-21 18:53:37 -0800617 extract_items(
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800618 target_files=other_target_files,
619 target_files_temp_dir=output_target_files_temp_dir,
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800620 extract_item_list=other_item_list)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800621
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800622 # Extract "special" items from the input system partial target files package.
623 # We extract these items to different directory since they require special
624 # processing before they will end up in the output directory.
625
Bill Peckham889b0c62019-02-21 18:53:37 -0800626 extract_items(
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800627 target_files=system_target_files,
628 target_files_temp_dir=system_target_files_temp_dir,
629 extract_item_list=system_extract_special_item_list)
630
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800631 # Extract "special" items from the input other partial target files package.
632 # We extract these items to different directory since they require special
633 # processing before they will end up in the output directory.
634
Bill Peckham889b0c62019-02-21 18:53:37 -0800635 extract_items(
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800636 target_files=other_target_files,
637 target_files_temp_dir=other_target_files_temp_dir,
638 extract_item_list=other_extract_special_item_list)
639
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800640 # Now that the temporary directories contain all the extracted files, perform
641 # special case processing on any items that need it. After this function
642 # completes successfully, all the files we need to create the output target
643 # files package are in place.
644
Bill Peckham889b0c62019-02-21 18:53:37 -0800645 process_special_cases(
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800646 temp_dir=temp_dir,
647 system_target_files_temp_dir=system_target_files_temp_dir,
648 other_target_files_temp_dir=other_target_files_temp_dir,
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800649 output_target_files_temp_dir=output_target_files_temp_dir,
Bill Peckham364c1cc2019-03-29 18:27:23 -0700650 system_misc_info_keys=system_misc_info_keys,
651 rebuild_recovery=rebuild_recovery)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800652
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800653 # Regenerate IMAGES in the temporary directory.
654
Daniel Normana4911da2019-03-15 14:36:21 -0700655 add_img_args = ['--verbose']
656 if rebuild_recovery:
657 add_img_args.append('--rebuild_recovery')
658 add_img_args.append(output_target_files_temp_dir)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800659
660 add_img_to_target_files.main(add_img_args)
661
662 # Finally, create the output target files zip archive.
663
664 output_zip = os.path.abspath(output_target_files)
665 output_target_files_list = os.path.join(temp_dir, 'output.list')
666 output_target_files_meta_dir = os.path.join(
667 output_target_files_temp_dir, 'META')
668
669 command = [
670 'find',
671 output_target_files_meta_dir,
672 ]
673 # TODO(bpeckham): sort this to be more like build.
674 meta_content = common.RunAndCheckOutput(command, verbose=False)
675 command = [
676 'find',
677 output_target_files_temp_dir,
678 '-path',
679 output_target_files_meta_dir,
680 '-prune',
681 '-o',
682 '-print'
683 ]
684 # TODO(bpeckham): sort this to be more like build.
685 other_content = common.RunAndCheckOutput(command, verbose=False)
686
687 with open(output_target_files_list, 'wb') as f:
688 f.write(meta_content)
689 f.write(other_content)
690
691 command = [
Bill Peckhamf753e152019-02-19 18:02:46 -0800692 'soong_zip',
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800693 '-d',
694 '-o', output_zip,
695 '-C', output_target_files_temp_dir,
696 '-l', output_target_files_list,
697 ]
698 logger.info('creating %s', output_target_files)
Bill Peckham889b0c62019-02-21 18:53:37 -0800699 common.RunAndWait(command, verbose=True)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800700
701
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800702def call_func_with_temp_dir(func, keep_tmp):
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800703 """Manage the creation and cleanup of the temporary directory.
704
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800705 This function calls the given function after first creating a temporary
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800706 directory. It also cleans up the temporary directory.
707
708 Args:
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800709 func: The function to call. Should accept one parameter, the path to
710 the temporary directory.
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800711
712 keep_tmp: Keep the temporary directory after processing is complete.
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800713 """
714
715 # Create a temporary directory. This will serve as the parent of directories
716 # we use when we extract items from the input target files packages, and also
717 # a scratch directory that we use for temporary files.
718
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800719 temp_dir = common.MakeTempDir(prefix='merge_target_files_')
720
721 try:
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800722 func(temp_dir)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800723 except:
724 raise
725 finally:
726 if keep_tmp:
727 logger.info('keeping %s', temp_dir)
728 else:
729 common.Cleanup()
730
731
732def main():
733 """The main function.
734
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800735 Process command line arguments, then call merge_target_files to
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800736 perform the heavy lifting.
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800737 """
738
739 common.InitLogging()
740
Bill Peckhamf753e152019-02-19 18:02:46 -0800741 def option_handler(o, a):
742 if o == '--system-target-files':
743 OPTIONS.system_target_files = a
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800744 elif o == '--system-item-list':
745 OPTIONS.system_item_list = a
746 elif o == '--system-misc-info-keys':
747 OPTIONS.system_misc_info_keys = a
Bill Peckhamf753e152019-02-19 18:02:46 -0800748 elif o == '--other-target-files':
749 OPTIONS.other_target_files = a
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800750 elif o == '--other-item-list':
751 OPTIONS.other_item_list = a
Bill Peckhamf753e152019-02-19 18:02:46 -0800752 elif o == '--output-target-files':
753 OPTIONS.output_target_files = a
Daniel Normana4911da2019-03-15 14:36:21 -0700754 elif o == '--rebuild_recovery':
755 OPTIONS.rebuild_recovery = True
Bill Peckham364c1cc2019-03-29 18:27:23 -0700756 elif o == '--keep-tmp':
Bill Peckhamf753e152019-02-19 18:02:46 -0800757 OPTIONS.keep_tmp = True
758 else:
759 return False
760 return True
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800761
Bill Peckhamf753e152019-02-19 18:02:46 -0800762 args = common.ParseOptions(
763 sys.argv[1:], __doc__,
764 extra_long_opts=[
765 'system-target-files=',
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800766 'system-item-list=',
767 'system-misc-info-keys=',
Bill Peckhamf753e152019-02-19 18:02:46 -0800768 'other-target-files=',
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800769 'other-item-list=',
Bill Peckhamf753e152019-02-19 18:02:46 -0800770 'output-target-files=',
Daniel Normana4911da2019-03-15 14:36:21 -0700771 'rebuild_recovery',
Bill Peckham364c1cc2019-03-29 18:27:23 -0700772 'keep-tmp',
Bill Peckhamf753e152019-02-19 18:02:46 -0800773 ],
774 extra_option_handler=option_handler)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800775
Bill Peckham889b0c62019-02-21 18:53:37 -0800776 if (len(args) != 0 or
Bill Peckhamf753e152019-02-19 18:02:46 -0800777 OPTIONS.system_target_files is None or
778 OPTIONS.other_target_files is None or
779 OPTIONS.output_target_files is None):
780 common.Usage(__doc__)
Bill Peckham889b0c62019-02-21 18:53:37 -0800781 sys.exit(1)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800782
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800783 if OPTIONS.system_item_list:
784 system_item_list = read_config_list(OPTIONS.system_item_list)
785 else:
786 system_item_list = default_system_item_list
787
788 if OPTIONS.system_misc_info_keys:
789 system_misc_info_keys = read_config_list(OPTIONS.system_misc_info_keys)
790 else:
791 system_misc_info_keys = default_system_misc_info_keys
792
793 if OPTIONS.other_item_list:
794 other_item_list = read_config_list(OPTIONS.other_item_list)
795 else:
796 other_item_list = default_other_item_list
797
Daniel Normane5964522019-03-19 10:32:03 -0700798 if not validate_config_lists(
799 system_item_list=system_item_list,
Daniel Norman19b9fe92019-03-19 14:48:02 -0700800 system_misc_info_keys=system_misc_info_keys,
Daniel Normane5964522019-03-19 10:32:03 -0700801 other_item_list=other_item_list):
802 sys.exit(1)
803
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800804 call_func_with_temp_dir(
805 lambda temp_dir: merge_target_files(
806 temp_dir=temp_dir,
807 system_target_files=OPTIONS.system_target_files,
808 system_item_list=system_item_list,
809 system_misc_info_keys=system_misc_info_keys,
810 other_target_files=OPTIONS.other_target_files,
811 other_item_list=other_item_list,
Daniel Normana4911da2019-03-15 14:36:21 -0700812 output_target_files=OPTIONS.output_target_files,
813 rebuild_recovery=OPTIONS.rebuild_recovery),
Daniel Norman2c99c5b2019-03-07 13:01:48 -0800814 OPTIONS.keep_tmp)
Bill Peckhame9eb5f92019-02-01 15:52:10 -0800815
816
817if __name__ == '__main__':
Bill Peckham889b0c62019-02-21 18:53:37 -0800818 main()