blob: 1aeb0566df98f024e9ae2c04b2ad21c41e49c1bb [file] [log] [blame]
Doug Zongkereef39442009-04-02 12:14:19 -07001#!/usr/bin/env python
2#
3# Copyright (C) 2008 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Given a target-files zipfile, produces an OTA package that installs
19that build. An incremental OTA is produced if -i is given, otherwise
20a full OTA is produced.
21
22Usage: ota_from_target_files [flags] input_target_files output_ota_package
23
24 -b (--board_config) <file>
Doug Zongkerfdd8e692009-08-03 17:27:48 -070025 Deprecated.
Doug Zongkereef39442009-04-02 12:14:19 -070026
27 -k (--package_key) <key>
28 Key to use to sign the package (default is
29 "build/target/product/security/testkey").
30
31 -i (--incremental_from) <file>
32 Generate an incremental OTA using the given target-files zip as
33 the starting build.
34
Doug Zongkerdbfaae52009-04-21 17:12:54 -070035 -w (--wipe_user_data)
36 Generate an OTA package that will wipe the user data partition
37 when installed.
38
Doug Zongker962069c2009-04-23 11:41:58 -070039 -n (--no_prereq)
40 Omit the timestamp prereq check normally included at the top of
41 the build scripts (used for developer OTA packages which
42 legitimately need to go back and forth).
43
Doug Zongker1c390a22009-05-14 19:06:36 -070044 -e (--extra_script) <file>
45 Insert the contents of file at the end of the update script.
46
Doug Zongkerc494d7c2009-06-18 08:43:44 -070047 -m (--script_mode) <mode>
48 Specify 'amend' or 'edify' scripts, or 'auto' to pick
49 automatically (this is the default).
50
Doug Zongkereef39442009-04-02 12:14:19 -070051"""
52
53import sys
54
55if sys.hexversion < 0x02040000:
56 print >> sys.stderr, "Python 2.4 or newer is required."
57 sys.exit(1)
58
59import copy
60import os
61import re
62import sha
63import subprocess
64import tempfile
65import time
66import zipfile
67
68import common
Doug Zongkerc494d7c2009-06-18 08:43:44 -070069import amend_generator
70import edify_generator
Doug Zongker03061472009-07-13 18:36:37 -070071import both_generator
Doug Zongkereef39442009-04-02 12:14:19 -070072
73OPTIONS = common.OPTIONS
74OPTIONS.package_key = "build/target/product/security/testkey"
75OPTIONS.incremental_source = None
76OPTIONS.require_verbatim = set()
77OPTIONS.prohibit_verbatim = set(("system/build.prop",))
78OPTIONS.patch_threshold = 0.95
Doug Zongkerdbfaae52009-04-21 17:12:54 -070079OPTIONS.wipe_user_data = False
Doug Zongker962069c2009-04-23 11:41:58 -070080OPTIONS.omit_prereq = False
Doug Zongker1c390a22009-05-14 19:06:36 -070081OPTIONS.extra_script = None
Doug Zongkerc494d7c2009-06-18 08:43:44 -070082OPTIONS.script_mode = 'auto'
Doug Zongkereef39442009-04-02 12:14:19 -070083
84def MostPopularKey(d, default):
85 """Given a dict, return the key corresponding to the largest
86 value. Returns 'default' if the dict is empty."""
87 x = [(v, k) for (k, v) in d.iteritems()]
88 if not x: return default
89 x.sort()
90 return x[-1][1]
91
92
93def IsSymlink(info):
94 """Return true if the zipfile.ZipInfo object passed in represents a
95 symlink."""
96 return (info.external_attr >> 16) == 0120777
97
98
99
100class Item:
101 """Items represent the metadata (user, group, mode) of files and
102 directories in the system image."""
103 ITEMS = {}
104 def __init__(self, name, dir=False):
105 self.name = name
106 self.uid = None
107 self.gid = None
108 self.mode = None
109 self.dir = dir
110
111 if name:
112 self.parent = Item.Get(os.path.dirname(name), dir=True)
113 self.parent.children.append(self)
114 else:
115 self.parent = None
116 if dir:
117 self.children = []
118
119 def Dump(self, indent=0):
120 if self.uid is not None:
121 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode)
122 else:
123 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode)
124 if self.dir:
125 print "%s%s" % (" "*indent, self.descendants)
126 print "%s%s" % (" "*indent, self.best_subtree)
127 for i in self.children:
128 i.Dump(indent=indent+1)
129
130 @classmethod
131 def Get(cls, name, dir=False):
132 if name not in cls.ITEMS:
133 cls.ITEMS[name] = Item(name, dir=dir)
134 return cls.ITEMS[name]
135
136 @classmethod
137 def GetMetadata(cls):
138 """Run the external 'fs_config' program to determine the desired
139 uid, gid, and mode for every Item object."""
140 p = common.Run(["fs_config"], stdin=subprocess.PIPE,
141 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
142 suffix = { False: "", True: "/" }
143 input = "".join(["%s%s\n" % (i.name, suffix[i.dir])
144 for i in cls.ITEMS.itervalues() if i.name])
145 output, error = p.communicate(input)
146 assert not error
147
148 for line in output.split("\n"):
149 if not line: continue
150 name, uid, gid, mode = line.split()
151 i = cls.ITEMS[name]
152 i.uid = int(uid)
153 i.gid = int(gid)
154 i.mode = int(mode, 8)
155 if i.dir:
156 i.children.sort(key=lambda i: i.name)
157
158 def CountChildMetadata(self):
159 """Count up the (uid, gid, mode) tuples for all children and
160 determine the best strategy for using set_perm_recursive and
161 set_perm to correctly chown/chmod all the files to their desired
162 values. Recursively calls itself for all descendants.
163
164 Returns a dict of {(uid, gid, dmode, fmode): count} counting up
165 all descendants of this node. (dmode or fmode may be None.) Also
166 sets the best_subtree of each directory Item to the (uid, gid,
167 dmode, fmode) tuple that will match the most descendants of that
168 Item.
169 """
170
171 assert self.dir
172 d = self.descendants = {(self.uid, self.gid, self.mode, None): 1}
173 for i in self.children:
174 if i.dir:
175 for k, v in i.CountChildMetadata().iteritems():
176 d[k] = d.get(k, 0) + v
177 else:
178 k = (i.uid, i.gid, None, i.mode)
179 d[k] = d.get(k, 0) + 1
180
181 # Find the (uid, gid, dmode, fmode) tuple that matches the most
182 # descendants.
183
184 # First, find the (uid, gid) pair that matches the most
185 # descendants.
186 ug = {}
187 for (uid, gid, _, _), count in d.iteritems():
188 ug[(uid, gid)] = ug.get((uid, gid), 0) + count
189 ug = MostPopularKey(ug, (0, 0))
190
191 # Now find the dmode and fmode that match the most descendants
192 # with that (uid, gid), and choose those.
193 best_dmode = (0, 0755)
194 best_fmode = (0, 0644)
195 for k, count in d.iteritems():
196 if k[:2] != ug: continue
197 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2])
198 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3])
199 self.best_subtree = ug + (best_dmode[1], best_fmode[1])
200
201 return d
202
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700203 def SetPermissions(self, script):
Doug Zongkereef39442009-04-02 12:14:19 -0700204 """Append set_perm/set_perm_recursive commands to 'script' to
205 set all permissions, users, and groups for the tree of files
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700206 rooted at 'self'."""
Doug Zongkereef39442009-04-02 12:14:19 -0700207
208 self.CountChildMetadata()
209
210 def recurse(item, current):
211 # current is the (uid, gid, dmode, fmode) tuple that the current
212 # item (and all its children) have already been set to. We only
213 # need to issue set_perm/set_perm_recursive commands if we're
214 # supposed to be something different.
215 if item.dir:
216 if current != item.best_subtree:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700217 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree)
Doug Zongkereef39442009-04-02 12:14:19 -0700218 current = item.best_subtree
219
220 if item.uid != current[0] or item.gid != current[1] or \
221 item.mode != current[2]:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700222 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
Doug Zongkereef39442009-04-02 12:14:19 -0700223
224 for i in item.children:
225 recurse(i, current)
226 else:
227 if item.uid != current[0] or item.gid != current[1] or \
228 item.mode != current[3]:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700229 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
Doug Zongkereef39442009-04-02 12:14:19 -0700230
231 recurse(self, (-1, -1, -1, -1))
232
233
234def CopySystemFiles(input_zip, output_zip=None,
235 substitute=None):
236 """Copies files underneath system/ in the input zip to the output
237 zip. Populates the Item class with their metadata, and returns a
238 list of symlinks. output_zip may be None, in which case the copy is
239 skipped (but the other side effects still happen). substitute is an
240 optional dict of {output filename: contents} to be output instead of
241 certain input files.
242 """
243
244 symlinks = []
245
246 for info in input_zip.infolist():
247 if info.filename.startswith("SYSTEM/"):
248 basefilename = info.filename[7:]
249 if IsSymlink(info):
250 symlinks.append((input_zip.read(info.filename),
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700251 "/system/" + basefilename))
Doug Zongkereef39442009-04-02 12:14:19 -0700252 else:
253 info2 = copy.copy(info)
254 fn = info2.filename = "system/" + basefilename
255 if substitute and fn in substitute and substitute[fn] is None:
256 continue
257 if output_zip is not None:
258 if substitute and fn in substitute:
259 data = substitute[fn]
260 else:
261 data = input_zip.read(info.filename)
262 output_zip.writestr(info2, data)
263 if fn.endswith("/"):
264 Item.Get(fn[:-1], dir=True)
265 else:
266 Item.Get(fn, dir=False)
267
268 symlinks.sort()
269 return symlinks
270
271
Doug Zongkereef39442009-04-02 12:14:19 -0700272def SignOutput(temp_zip_name, output_zip_name):
273 key_passwords = common.GetKeyPasswords([OPTIONS.package_key])
274 pw = key_passwords[OPTIONS.package_key]
275
276 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw)
277
278
Doug Zongkereef39442009-04-02 12:14:19 -0700279def AppendAssertions(script, input_zip):
Doug Zongkereef39442009-04-02 12:14:19 -0700280 device = GetBuildProp("ro.product.device", input_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700281 script.AssertDevice(device)
Doug Zongkereef39442009-04-02 12:14:19 -0700282
283 info = input_zip.read("OTA/android-info.txt")
284 m = re.search(r"require\s+version-bootloader\s*=\s*(\S+)", info)
Doug Zongker9fc74c72009-06-23 16:27:38 -0700285 if m:
286 bootloaders = m.group(1).split("|")
287 script.AssertSomeBootloader(*bootloaders)
Doug Zongkereef39442009-04-02 12:14:19 -0700288
289
Doug Zongker73ef8252009-07-23 15:12:53 -0700290def MakeRecoveryPatch(output_zip, recovery_img, boot_img):
291 """Generate a binary patch that creates the recovery image starting
292 with the boot image. (Most of the space in these images is just the
293 kernel, which is identical for the two, so the resulting patch
294 should be efficient.) Add it to the output zip, along with a shell
295 script that is run from init.rc on first boot to actually do the
296 patching and install the new recovery image.
297
298 recovery_img and boot_img should be File objects for the
299 corresponding images.
300
301 Returns an Item for the shell script, which must be made
302 executable.
303 """
304
305 patch = Difference(recovery_img, boot_img, "imgdiff")
Doug Zongkercfd7db62009-10-07 11:35:53 -0700306 common.ZipWriteStr(output_zip, "recovery/recovery-from-boot.p", patch)
Doug Zongker73ef8252009-07-23 15:12:53 -0700307 Item.Get("system/recovery-from-boot.p", dir=False)
308
309 # Images with different content will have a different first page, so
310 # we check to see if this recovery has already been installed by
311 # testing just the first 2k.
312 HEADER_SIZE = 2048
313 header_sha1 = sha.sha(recovery_img.data[:HEADER_SIZE]).hexdigest()
314 sh = """#!/system/bin/sh
315if ! applypatch -c MTD:recovery:%(header_size)d:%(header_sha1)s; then
316 log -t recovery "Installing new recovery image"
317 applypatch MTD:boot:%(boot_size)d:%(boot_sha1)s MTD:recovery %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p
318else
319 log -t recovery "Recovery image already installed"
320fi
321""" % { 'boot_size': boot_img.size,
322 'boot_sha1': boot_img.sha1,
323 'header_size': HEADER_SIZE,
324 'header_sha1': header_sha1,
325 'recovery_size': recovery_img.size,
326 'recovery_sha1': recovery_img.sha1 }
Doug Zongkercfd7db62009-10-07 11:35:53 -0700327 common.ZipWriteStr(output_zip, "recovery/etc/install-recovery.sh", sh)
Doug Zongker73ef8252009-07-23 15:12:53 -0700328 return Item.Get("system/etc/install-recovery.sh", dir=False)
329
330
Doug Zongkereef39442009-04-02 12:14:19 -0700331def WriteFullOTAPackage(input_zip, output_zip):
Doug Zongker03061472009-07-13 18:36:37 -0700332 if OPTIONS.script_mode == "auto":
333 script = both_generator.BothGenerator(2)
334 elif OPTIONS.script_mode == "amend":
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700335 script = amend_generator.AmendGenerator()
336 else:
337 # TODO: how to determine this? We don't know what version it will
338 # be installed on top of. For now, we expect the API just won't
339 # change very often.
Doug Zongker03061472009-07-13 18:36:37 -0700340 script = edify_generator.EdifyGenerator(2)
Doug Zongkereef39442009-04-02 12:14:19 -0700341
Doug Zongker962069c2009-04-23 11:41:58 -0700342 if not OPTIONS.omit_prereq:
343 ts = GetBuildProp("ro.build.date.utc", input_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700344 script.AssertOlderBuild(ts)
Doug Zongkereef39442009-04-02 12:14:19 -0700345
346 AppendAssertions(script, input_zip)
347
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700348 script.ShowProgress(0.1, 0)
Doug Zongkereef39442009-04-02 12:14:19 -0700349
Doug Zongkere1c31ba2009-06-23 17:40:35 -0700350 try:
351 common.ZipWriteStr(output_zip, "radio.img", input_zip.read("RADIO/image"))
352 script.WriteFirmwareImage("radio", "radio.img")
353 except KeyError:
354 print "warning: no radio image in input target_files; not flashing radio"
355
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700356 script.ShowProgress(0.5, 0)
Doug Zongkereef39442009-04-02 12:14:19 -0700357
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700358 if OPTIONS.wipe_user_data:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700359 script.FormatPartition("userdata")
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700360
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700361 script.FormatPartition("system")
362 script.Mount("MTD", "system", "/system")
Doug Zongkercfd7db62009-10-07 11:35:53 -0700363 script.UnpackPackageDir("recovery", "/system")
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700364 script.UnpackPackageDir("system", "/system")
Doug Zongkereef39442009-04-02 12:14:19 -0700365
366 symlinks = CopySystemFiles(input_zip, output_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700367 script.MakeSymlinks(symlinks)
Doug Zongkereef39442009-04-02 12:14:19 -0700368
Doug Zongker73ef8252009-07-23 15:12:53 -0700369 boot_img = File("boot.img", common.BuildBootableImage(
370 os.path.join(OPTIONS.input_tmp, "BOOT")))
371 recovery_img = File("recovery.img", common.BuildBootableImage(
372 os.path.join(OPTIONS.input_tmp, "RECOVERY")))
373 i = MakeRecoveryPatch(output_zip, recovery_img, boot_img)
Doug Zongkereef39442009-04-02 12:14:19 -0700374
Doug Zongker73ef8252009-07-23 15:12:53 -0700375 Item.GetMetadata()
Doug Zongkereef39442009-04-02 12:14:19 -0700376
Doug Zongker73ef8252009-07-23 15:12:53 -0700377 # GetMetadata uses the data in android_filesystem_config.h to assign
378 # the uid/gid/mode of all files. We want to override that for the
379 # recovery patching shell script to make it executable.
380 i.uid = 0
381 i.gid = 0
382 i.mode = 0544
383 Item.Get("system").SetPermissions(script)
384
385 common.CheckSize(boot_img.data, "boot.img")
386 common.ZipWriteStr(output_zip, "boot.img", boot_img.data)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700387 script.ShowProgress(0.2, 0)
388
389 script.WriteRawImage("boot", "boot.img")
390 script.ShowProgress(0.2, 10)
Doug Zongkereef39442009-04-02 12:14:19 -0700391
Doug Zongker1c390a22009-05-14 19:06:36 -0700392 if OPTIONS.extra_script is not None:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700393 script.AppendExtra(OPTIONS.extra_script)
Doug Zongker1c390a22009-05-14 19:06:36 -0700394
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700395 script.AddToZip(input_zip, output_zip)
Doug Zongkereef39442009-04-02 12:14:19 -0700396
397
398class File(object):
399 def __init__(self, name, data):
400 self.name = name
401 self.data = data
402 self.size = len(data)
403 self.sha1 = sha.sha(data).hexdigest()
404
405 def WriteToTemp(self):
406 t = tempfile.NamedTemporaryFile()
407 t.write(self.data)
408 t.flush()
409 return t
410
411 def AddToZip(self, z):
Doug Zongker048e7ca2009-06-15 14:31:53 -0700412 common.ZipWriteStr(z, self.name, self.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700413
414
415def LoadSystemFiles(z):
416 """Load all the files from SYSTEM/... in a given target-files
417 ZipFile, and return a dict of {filename: File object}."""
418 out = {}
419 for info in z.infolist():
420 if info.filename.startswith("SYSTEM/") and not IsSymlink(info):
421 fn = "system/" + info.filename[7:]
422 data = z.read(info.filename)
423 out[fn] = File(fn, data)
424 return out
425
426
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700427def Difference(tf, sf, diff_program):
428 """Return the patch (as a string of data) needed to turn sf into tf.
429 diff_program is the name of an external program (or list, if
430 additional arguments are desired) to run to generate the diff.
431 """
Doug Zongkereef39442009-04-02 12:14:19 -0700432
433 ttemp = tf.WriteToTemp()
434 stemp = sf.WriteToTemp()
435
436 ext = os.path.splitext(tf.name)[1]
437
438 try:
439 ptemp = tempfile.NamedTemporaryFile()
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700440 if isinstance(diff_program, list):
441 cmd = copy.copy(diff_program)
442 else:
443 cmd = [diff_program]
444 cmd.append(stemp.name)
445 cmd.append(ttemp.name)
446 cmd.append(ptemp.name)
447 p = common.Run(cmd)
Doug Zongkereef39442009-04-02 12:14:19 -0700448 _, err = p.communicate()
Doug Zongker5da317e2009-06-02 13:38:17 -0700449 if err or p.returncode != 0:
450 print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
451 return None
Doug Zongkereef39442009-04-02 12:14:19 -0700452 diff = ptemp.read()
Doug Zongkereef39442009-04-02 12:14:19 -0700453 finally:
Doug Zongker5da317e2009-06-02 13:38:17 -0700454 ptemp.close()
Doug Zongkereef39442009-04-02 12:14:19 -0700455 stemp.close()
456 ttemp.close()
457
458 return diff
459
460
461def GetBuildProp(property, z):
462 """Return the fingerprint of the build of a given target-files
463 ZipFile object."""
464 bp = z.read("SYSTEM/build.prop")
465 if not property:
466 return bp
467 m = re.search(re.escape(property) + r"=(.*)\n", bp)
468 if not m:
Doug Zongker9fc74c72009-06-23 16:27:38 -0700469 raise common.ExternalError("couldn't find %s in build.prop" % (property,))
Doug Zongkereef39442009-04-02 12:14:19 -0700470 return m.group(1).strip()
471
472
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700473def GetRecoveryAPIVersion(zip):
474 """Returns the version of the recovery API. Version 0 is the older
475 amend code (no separate binary)."""
476 try:
477 version = zip.read("META/recovery-api-version.txt")
478 return int(version)
479 except KeyError:
480 try:
481 # version one didn't have the recovery-api-version.txt file, but
482 # it did include an updater binary.
483 zip.getinfo("OTA/bin/updater")
484 return 1
485 except KeyError:
486 return 0
487
Doug Zongkereef39442009-04-02 12:14:19 -0700488def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700489 source_version = GetRecoveryAPIVersion(source_zip)
490
491 if OPTIONS.script_mode == 'amend':
492 script = amend_generator.AmendGenerator()
493 elif OPTIONS.script_mode == 'edify':
494 if source_version == 0:
495 print ("WARNING: generating edify script for a source that "
496 "can't install it.")
497 script = edify_generator.EdifyGenerator(source_version)
498 elif OPTIONS.script_mode == 'auto':
499 if source_version > 0:
500 script = edify_generator.EdifyGenerator(source_version)
501 else:
502 script = amend_generator.AmendGenerator()
503 else:
504 raise ValueError('unknown script mode "%s"' % (OPTIONS.script_mode,))
Doug Zongkereef39442009-04-02 12:14:19 -0700505
506 print "Loading target..."
507 target_data = LoadSystemFiles(target_zip)
508 print "Loading source..."
509 source_data = LoadSystemFiles(source_zip)
510
511 verbatim_targets = []
512 patch_list = []
513 largest_source_size = 0
514 for fn in sorted(target_data.keys()):
515 tf = target_data[fn]
516 sf = source_data.get(fn, None)
517
518 if sf is None or fn in OPTIONS.require_verbatim:
519 # This file should be included verbatim
520 if fn in OPTIONS.prohibit_verbatim:
Doug Zongker9fc74c72009-06-23 16:27:38 -0700521 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,))
Doug Zongkereef39442009-04-02 12:14:19 -0700522 print "send", fn, "verbatim"
523 tf.AddToZip(output_zip)
524 verbatim_targets.append((fn, tf.size))
525 elif tf.sha1 != sf.sha1:
526 # File is different; consider sending as a patch
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700527 diff_method = "bsdiff"
528 if tf.name.endswith(".gz"):
529 diff_method = "imgdiff"
530 d = Difference(tf, sf, diff_method)
Doug Zongker5da317e2009-06-02 13:38:17 -0700531 if d is not None:
532 print fn, tf.size, len(d), (float(len(d)) / tf.size)
533 if d is None or len(d) > tf.size * OPTIONS.patch_threshold:
Doug Zongkereef39442009-04-02 12:14:19 -0700534 # patch is almost as big as the file; don't bother patching
535 tf.AddToZip(output_zip)
536 verbatim_targets.append((fn, tf.size))
537 else:
Doug Zongker048e7ca2009-06-15 14:31:53 -0700538 common.ZipWriteStr(output_zip, "patch/" + fn + ".p", d)
Doug Zongkereef39442009-04-02 12:14:19 -0700539 patch_list.append((fn, tf, sf, tf.size))
540 largest_source_size = max(largest_source_size, sf.size)
541 else:
542 # Target file identical to source.
543 pass
544
545 total_verbatim_size = sum([i[1] for i in verbatim_targets])
546 total_patched_size = sum([i[3] for i in patch_list])
547
548 source_fp = GetBuildProp("ro.build.fingerprint", source_zip)
549 target_fp = GetBuildProp("ro.build.fingerprint", target_zip)
550
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700551 script.Mount("MTD", "system", "/system")
552 script.AssertSomeFingerprint(source_fp, target_fp)
Doug Zongkereef39442009-04-02 12:14:19 -0700553
Doug Zongker5da317e2009-06-02 13:38:17 -0700554 source_boot = File("/tmp/boot.img",
555 common.BuildBootableImage(
556 os.path.join(OPTIONS.source_tmp, "BOOT")))
557 target_boot = File("/tmp/boot.img",
558 common.BuildBootableImage(
559 os.path.join(OPTIONS.target_tmp, "BOOT")))
560 updating_boot = (source_boot.data != target_boot.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700561
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700562 source_recovery = File("system/recovery.img",
563 common.BuildBootableImage(
564 os.path.join(OPTIONS.source_tmp, "RECOVERY")))
565 target_recovery = File("system/recovery.img",
566 common.BuildBootableImage(
567 os.path.join(OPTIONS.target_tmp, "RECOVERY")))
568 updating_recovery = (source_recovery.data != target_recovery.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700569
570 source_radio = source_zip.read("RADIO/image")
571 target_radio = target_zip.read("RADIO/image")
572 updating_radio = (source_radio != target_radio)
573
574 # The last 0.1 is reserved for creating symlinks, fixing
575 # permissions, and writing the boot image (if necessary).
576 progress_bar_total = 1.0
577 if updating_boot:
578 progress_bar_total -= 0.1
579 if updating_radio:
580 progress_bar_total -= 0.3
581
582 AppendAssertions(script, target_zip)
583
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700584 script.Print("Verifying current system...")
585
Doug Zongkereef39442009-04-02 12:14:19 -0700586 pb_verify = progress_bar_total * 0.3 * \
587 (total_patched_size /
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700588 float(total_patched_size+total_verbatim_size+1))
Doug Zongkereef39442009-04-02 12:14:19 -0700589
590 for i, (fn, tf, sf, size) in enumerate(patch_list):
591 if i % 5 == 0:
592 next_sizes = sum([i[3] for i in patch_list[i:i+5]])
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700593 script.ShowProgress(next_sizes * pb_verify / (total_patched_size+1), 1)
594
595 script.PatchCheck("/"+fn, tf.sha1, sf.sha1)
Doug Zongkereef39442009-04-02 12:14:19 -0700596
Doug Zongker5da317e2009-06-02 13:38:17 -0700597 if updating_boot:
598 d = Difference(target_boot, source_boot, "imgdiff")
599 print "boot target: %d source: %d diff: %d" % (
600 target_boot.size, source_boot.size, len(d))
601
Doug Zongker048e7ca2009-06-15 14:31:53 -0700602 common.ZipWriteStr(output_zip, "patch/boot.img.p", d)
Doug Zongker5da317e2009-06-02 13:38:17 -0700603
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700604 script.PatchCheck("MTD:boot:%d:%s:%d:%s" %
605 (source_boot.size, source_boot.sha1,
606 target_boot.size, target_boot.sha1))
Doug Zongker5da317e2009-06-02 13:38:17 -0700607
608 if patch_list or updating_recovery or updating_boot:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700609 script.CacheFreeSpaceCheck(largest_source_size)
610 script.Print("Unpacking patches...")
611 script.UnpackPackageDir("patch", "/tmp/patchtmp")
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700612
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700613 script.Comment("---- start making changes here ----")
Doug Zongkereef39442009-04-02 12:14:19 -0700614
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700615 if OPTIONS.wipe_user_data:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700616 script.Print("Erasing user data...")
617 script.FormatPartition("userdata")
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700618
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700619 script.Print("Removing unneeded files...")
Doug Zongker0f3298a2009-06-30 08:16:58 -0700620 script.DeleteFiles(["/"+i[0] for i in verbatim_targets] +
621 ["/"+i for i in sorted(source_data)
Doug Zongker3b949f02009-08-24 10:24:32 -0700622 if i not in target_data] +
623 ["/system/recovery.img"])
Doug Zongkereef39442009-04-02 12:14:19 -0700624
625 if updating_boot:
Doug Zongker5da317e2009-06-02 13:38:17 -0700626 # Produce the boot image by applying a patch to the current
627 # contents of the boot partition, and write it back to the
628 # partition.
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700629 script.Print("Patching boot image...")
630 script.ApplyPatch("MTD:boot:%d:%s:%d:%s"
631 % (source_boot.size, source_boot.sha1,
632 target_boot.size, target_boot.sha1),
633 "-",
634 target_boot.size, target_boot.sha1,
635 source_boot.sha1, "/tmp/patchtmp/boot.img.p")
Doug Zongkereef39442009-04-02 12:14:19 -0700636 print "boot image changed; including."
637 else:
638 print "boot image unchanged; skipping."
639
640 if updating_recovery:
Doug Zongker73ef8252009-07-23 15:12:53 -0700641 # Is it better to generate recovery as a patch from the current
642 # boot image, or from the previous recovery image? For large
643 # updates with significant kernel changes, probably the former.
644 # For small updates where the kernel hasn't changed, almost
645 # certainly the latter. We pick the first option. Future
646 # complicated schemes may let us effectively use both.
647 #
648 # A wacky possibility: as long as there is room in the boot
649 # partition, include the binaries and image files from recovery in
650 # the boot image (though not in the ramdisk) so they can be used
651 # as fodder for constructing the recovery image.
652 recovery_sh_item = MakeRecoveryPatch(output_zip,
653 target_recovery, target_boot)
Doug Zongkercfd7db62009-10-07 11:35:53 -0700654 script.UnpackPackageDir("recovery", "/system")
Doug Zongker73ef8252009-07-23 15:12:53 -0700655 print "recovery image changed; including as patch from boot."
Doug Zongkereef39442009-04-02 12:14:19 -0700656 else:
657 print "recovery image unchanged; skipping."
658
659 if updating_radio:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700660 script.ShowProgress(0.3, 10)
661 script.Print("Writing radio image...")
662 script.WriteFirmwareImage("radio", "radio.img")
Doug Zongker048e7ca2009-06-15 14:31:53 -0700663 common.ZipWriteStr(output_zip, "radio.img", target_radio)
Doug Zongkereef39442009-04-02 12:14:19 -0700664 print "radio image changed; including."
665 else:
666 print "radio image unchanged; skipping."
667
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700668 script.Print("Patching system files...")
Doug Zongkereef39442009-04-02 12:14:19 -0700669 pb_apply = progress_bar_total * 0.7 * \
670 (total_patched_size /
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700671 float(total_patched_size+total_verbatim_size+1))
Doug Zongkereef39442009-04-02 12:14:19 -0700672 for i, (fn, tf, sf, size) in enumerate(patch_list):
673 if i % 5 == 0:
674 next_sizes = sum([i[3] for i in patch_list[i:i+5]])
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700675 script.ShowProgress(next_sizes * pb_apply / (total_patched_size+1), 1)
676 script.ApplyPatch("/"+fn, "-", tf.size, tf.sha1,
677 sf.sha1, "/tmp/patchtmp/"+fn+".p")
Doug Zongkereef39442009-04-02 12:14:19 -0700678
679 target_symlinks = CopySystemFiles(target_zip, None)
680
681 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks])
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700682 temp_script = script.MakeTemporary()
Doug Zongker73ef8252009-07-23 15:12:53 -0700683 Item.GetMetadata()
684 if updating_recovery:
685 recovery_sh_item.uid = 0
686 recovery_sh_item.gid = 0
687 recovery_sh_item.mode = 0544
688 Item.Get("system").SetPermissions(temp_script)
Doug Zongkereef39442009-04-02 12:14:19 -0700689
690 # Note that this call will mess up the tree of Items, so make sure
691 # we're done with it.
692 source_symlinks = CopySystemFiles(source_zip, None)
693 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks])
694
695 # Delete all the symlinks in source that aren't in target. This
696 # needs to happen before verbatim files are unpacked, in case a
697 # symlink in the source is replaced by a real file in the target.
698 to_delete = []
699 for dest, link in source_symlinks:
700 if link not in target_symlinks_d:
701 to_delete.append(link)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700702 script.DeleteFiles(to_delete)
Doug Zongkereef39442009-04-02 12:14:19 -0700703
704 if verbatim_targets:
705 pb_verbatim = progress_bar_total * \
706 (total_verbatim_size /
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700707 float(total_patched_size+total_verbatim_size+1))
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700708 script.ShowProgress(pb_verbatim, 5)
709 script.Print("Unpacking new files...")
710 script.UnpackPackageDir("system", "/system")
711
712 script.Print("Finishing up...")
Doug Zongkereef39442009-04-02 12:14:19 -0700713
714 # Create all the symlinks that don't already exist, or point to
715 # somewhere different than what we want. Delete each symlink before
716 # creating it, since the 'symlink' command won't overwrite.
717 to_create = []
718 for dest, link in target_symlinks:
719 if link in source_symlinks_d:
720 if dest != source_symlinks_d[link]:
721 to_create.append((dest, link))
722 else:
723 to_create.append((dest, link))
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700724 script.DeleteFiles([i[1] for i in to_create])
725 script.MakeSymlinks(to_create)
Doug Zongkereef39442009-04-02 12:14:19 -0700726
727 # Now that the symlinks are created, we can set all the
728 # permissions.
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700729 script.AppendScript(temp_script)
Doug Zongkereef39442009-04-02 12:14:19 -0700730
Doug Zongker1c390a22009-05-14 19:06:36 -0700731 if OPTIONS.extra_script is not None:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700732 scirpt.AppendExtra(OPTIONS.extra_script)
Doug Zongker1c390a22009-05-14 19:06:36 -0700733
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700734 script.AddToZip(target_zip, output_zip)
Doug Zongkereef39442009-04-02 12:14:19 -0700735
736
737def main(argv):
738
739 def option_handler(o, a):
740 if o in ("-b", "--board_config"):
Doug Zongkerfdd8e692009-08-03 17:27:48 -0700741 pass # deprecated
Doug Zongkereef39442009-04-02 12:14:19 -0700742 elif o in ("-k", "--package_key"):
743 OPTIONS.package_key = a
Doug Zongkereef39442009-04-02 12:14:19 -0700744 elif o in ("-i", "--incremental_from"):
745 OPTIONS.incremental_source = a
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700746 elif o in ("-w", "--wipe_user_data"):
747 OPTIONS.wipe_user_data = True
Doug Zongker962069c2009-04-23 11:41:58 -0700748 elif o in ("-n", "--no_prereq"):
749 OPTIONS.omit_prereq = True
Doug Zongker1c390a22009-05-14 19:06:36 -0700750 elif o in ("-e", "--extra_script"):
751 OPTIONS.extra_script = a
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700752 elif o in ("-m", "--script_mode"):
753 OPTIONS.script_mode = a
Doug Zongkereef39442009-04-02 12:14:19 -0700754 else:
755 return False
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700756 return True
Doug Zongkereef39442009-04-02 12:14:19 -0700757
758 args = common.ParseOptions(argv, __doc__,
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700759 extra_opts="b:k:i:d:wne:m:",
Doug Zongkereef39442009-04-02 12:14:19 -0700760 extra_long_opts=["board_config=",
761 "package_key=",
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700762 "incremental_from=",
Doug Zongker962069c2009-04-23 11:41:58 -0700763 "wipe_user_data",
Doug Zongker1c390a22009-05-14 19:06:36 -0700764 "no_prereq",
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700765 "extra_script=",
766 "script_mode="],
Doug Zongkereef39442009-04-02 12:14:19 -0700767 extra_option_handler=option_handler)
768
769 if len(args) != 2:
770 common.Usage(__doc__)
771 sys.exit(1)
772
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700773 if OPTIONS.script_mode not in ("amend", "edify", "auto"):
774 raise ValueError('unknown script mode "%s"' % (OPTIONS.script_mode,))
775
Doug Zongker1c390a22009-05-14 19:06:36 -0700776 if OPTIONS.extra_script is not None:
777 OPTIONS.extra_script = open(OPTIONS.extra_script).read()
778
Doug Zongkereef39442009-04-02 12:14:19 -0700779 print "unzipping target target-files..."
780 OPTIONS.input_tmp = common.UnzipTemp(args[0])
Doug Zongkerfdd8e692009-08-03 17:27:48 -0700781
782 common.LoadMaxSizes()
783 if not OPTIONS.max_image_size:
784 print
785 print " WARNING: Failed to load max image sizes; will not enforce"
786 print " image size limits."
787 print
788
Doug Zongkereef39442009-04-02 12:14:19 -0700789 OPTIONS.target_tmp = OPTIONS.input_tmp
790 input_zip = zipfile.ZipFile(args[0], "r")
791 if OPTIONS.package_key:
792 temp_zip_file = tempfile.NamedTemporaryFile()
793 output_zip = zipfile.ZipFile(temp_zip_file, "w",
794 compression=zipfile.ZIP_DEFLATED)
795 else:
796 output_zip = zipfile.ZipFile(args[1], "w",
797 compression=zipfile.ZIP_DEFLATED)
798
799 if OPTIONS.incremental_source is None:
800 WriteFullOTAPackage(input_zip, output_zip)
801 else:
802 print "unzipping source target-files..."
803 OPTIONS.source_tmp = common.UnzipTemp(OPTIONS.incremental_source)
804 source_zip = zipfile.ZipFile(OPTIONS.incremental_source, "r")
805 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip)
806
807 output_zip.close()
808 if OPTIONS.package_key:
809 SignOutput(temp_zip_file.name, args[1])
810 temp_zip_file.close()
811
812 common.Cleanup()
813
814 print "done."
815
816
817if __name__ == '__main__':
818 try:
819 main(sys.argv[1:])
820 except common.ExternalError, e:
821 print
822 print " ERROR: %s" % (e,)
823 print
824 sys.exit(1)