blob: 864c35bbadec7c7591aec0f88e0480cefc259716 [file] [log] [blame]
Doug Zongkereef39442009-04-02 12:14:19 -07001#!/usr/bin/env python
2#
3# Copyright (C) 2008 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Given a target-files zipfile, produces an OTA package that installs
19that build. An incremental OTA is produced if -i is given, otherwise
20a full OTA is produced.
21
22Usage: ota_from_target_files [flags] input_target_files output_ota_package
23
24 -b (--board_config) <file>
Doug Zongkerfdd8e692009-08-03 17:27:48 -070025 Deprecated.
Doug Zongkereef39442009-04-02 12:14:19 -070026
27 -k (--package_key) <key>
28 Key to use to sign the package (default is
29 "build/target/product/security/testkey").
30
31 -i (--incremental_from) <file>
32 Generate an incremental OTA using the given target-files zip as
33 the starting build.
34
Doug Zongkerdbfaae52009-04-21 17:12:54 -070035 -w (--wipe_user_data)
36 Generate an OTA package that will wipe the user data partition
37 when installed.
38
Doug Zongker962069c2009-04-23 11:41:58 -070039 -n (--no_prereq)
40 Omit the timestamp prereq check normally included at the top of
41 the build scripts (used for developer OTA packages which
42 legitimately need to go back and forth).
43
Doug Zongker1c390a22009-05-14 19:06:36 -070044 -e (--extra_script) <file>
45 Insert the contents of file at the end of the update script.
46
Doug Zongkerc494d7c2009-06-18 08:43:44 -070047 -m (--script_mode) <mode>
48 Specify 'amend' or 'edify' scripts, or 'auto' to pick
49 automatically (this is the default).
50
Doug Zongkereef39442009-04-02 12:14:19 -070051"""
52
53import sys
54
55if sys.hexversion < 0x02040000:
56 print >> sys.stderr, "Python 2.4 or newer is required."
57 sys.exit(1)
58
59import copy
60import os
61import re
62import sha
63import subprocess
64import tempfile
65import time
66import zipfile
67
68import common
Doug Zongkerc494d7c2009-06-18 08:43:44 -070069import amend_generator
70import edify_generator
Doug Zongker03061472009-07-13 18:36:37 -070071import both_generator
Doug Zongkereef39442009-04-02 12:14:19 -070072
73OPTIONS = common.OPTIONS
74OPTIONS.package_key = "build/target/product/security/testkey"
75OPTIONS.incremental_source = None
76OPTIONS.require_verbatim = set()
77OPTIONS.prohibit_verbatim = set(("system/build.prop",))
78OPTIONS.patch_threshold = 0.95
Doug Zongkerdbfaae52009-04-21 17:12:54 -070079OPTIONS.wipe_user_data = False
Doug Zongker962069c2009-04-23 11:41:58 -070080OPTIONS.omit_prereq = False
Doug Zongker1c390a22009-05-14 19:06:36 -070081OPTIONS.extra_script = None
Doug Zongkerc494d7c2009-06-18 08:43:44 -070082OPTIONS.script_mode = 'auto'
Doug Zongkereef39442009-04-02 12:14:19 -070083
84def MostPopularKey(d, default):
85 """Given a dict, return the key corresponding to the largest
86 value. Returns 'default' if the dict is empty."""
87 x = [(v, k) for (k, v) in d.iteritems()]
88 if not x: return default
89 x.sort()
90 return x[-1][1]
91
92
93def IsSymlink(info):
94 """Return true if the zipfile.ZipInfo object passed in represents a
95 symlink."""
96 return (info.external_attr >> 16) == 0120777
97
98
99
100class Item:
101 """Items represent the metadata (user, group, mode) of files and
102 directories in the system image."""
103 ITEMS = {}
104 def __init__(self, name, dir=False):
105 self.name = name
106 self.uid = None
107 self.gid = None
108 self.mode = None
109 self.dir = dir
110
111 if name:
112 self.parent = Item.Get(os.path.dirname(name), dir=True)
113 self.parent.children.append(self)
114 else:
115 self.parent = None
116 if dir:
117 self.children = []
118
119 def Dump(self, indent=0):
120 if self.uid is not None:
121 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode)
122 else:
123 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode)
124 if self.dir:
125 print "%s%s" % (" "*indent, self.descendants)
126 print "%s%s" % (" "*indent, self.best_subtree)
127 for i in self.children:
128 i.Dump(indent=indent+1)
129
130 @classmethod
131 def Get(cls, name, dir=False):
132 if name not in cls.ITEMS:
133 cls.ITEMS[name] = Item(name, dir=dir)
134 return cls.ITEMS[name]
135
136 @classmethod
137 def GetMetadata(cls):
138 """Run the external 'fs_config' program to determine the desired
139 uid, gid, and mode for every Item object."""
140 p = common.Run(["fs_config"], stdin=subprocess.PIPE,
141 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
142 suffix = { False: "", True: "/" }
143 input = "".join(["%s%s\n" % (i.name, suffix[i.dir])
144 for i in cls.ITEMS.itervalues() if i.name])
145 output, error = p.communicate(input)
146 assert not error
147
148 for line in output.split("\n"):
149 if not line: continue
150 name, uid, gid, mode = line.split()
151 i = cls.ITEMS[name]
152 i.uid = int(uid)
153 i.gid = int(gid)
154 i.mode = int(mode, 8)
155 if i.dir:
156 i.children.sort(key=lambda i: i.name)
157
158 def CountChildMetadata(self):
159 """Count up the (uid, gid, mode) tuples for all children and
160 determine the best strategy for using set_perm_recursive and
161 set_perm to correctly chown/chmod all the files to their desired
162 values. Recursively calls itself for all descendants.
163
164 Returns a dict of {(uid, gid, dmode, fmode): count} counting up
165 all descendants of this node. (dmode or fmode may be None.) Also
166 sets the best_subtree of each directory Item to the (uid, gid,
167 dmode, fmode) tuple that will match the most descendants of that
168 Item.
169 """
170
171 assert self.dir
172 d = self.descendants = {(self.uid, self.gid, self.mode, None): 1}
173 for i in self.children:
174 if i.dir:
175 for k, v in i.CountChildMetadata().iteritems():
176 d[k] = d.get(k, 0) + v
177 else:
178 k = (i.uid, i.gid, None, i.mode)
179 d[k] = d.get(k, 0) + 1
180
181 # Find the (uid, gid, dmode, fmode) tuple that matches the most
182 # descendants.
183
184 # First, find the (uid, gid) pair that matches the most
185 # descendants.
186 ug = {}
187 for (uid, gid, _, _), count in d.iteritems():
188 ug[(uid, gid)] = ug.get((uid, gid), 0) + count
189 ug = MostPopularKey(ug, (0, 0))
190
191 # Now find the dmode and fmode that match the most descendants
192 # with that (uid, gid), and choose those.
193 best_dmode = (0, 0755)
194 best_fmode = (0, 0644)
195 for k, count in d.iteritems():
196 if k[:2] != ug: continue
197 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2])
198 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3])
199 self.best_subtree = ug + (best_dmode[1], best_fmode[1])
200
201 return d
202
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700203 def SetPermissions(self, script):
Doug Zongkereef39442009-04-02 12:14:19 -0700204 """Append set_perm/set_perm_recursive commands to 'script' to
205 set all permissions, users, and groups for the tree of files
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700206 rooted at 'self'."""
Doug Zongkereef39442009-04-02 12:14:19 -0700207
208 self.CountChildMetadata()
209
210 def recurse(item, current):
211 # current is the (uid, gid, dmode, fmode) tuple that the current
212 # item (and all its children) have already been set to. We only
213 # need to issue set_perm/set_perm_recursive commands if we're
214 # supposed to be something different.
215 if item.dir:
216 if current != item.best_subtree:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700217 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree)
Doug Zongkereef39442009-04-02 12:14:19 -0700218 current = item.best_subtree
219
220 if item.uid != current[0] or item.gid != current[1] or \
221 item.mode != current[2]:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700222 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
Doug Zongkereef39442009-04-02 12:14:19 -0700223
224 for i in item.children:
225 recurse(i, current)
226 else:
227 if item.uid != current[0] or item.gid != current[1] or \
228 item.mode != current[3]:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700229 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
Doug Zongkereef39442009-04-02 12:14:19 -0700230
231 recurse(self, (-1, -1, -1, -1))
232
233
234def CopySystemFiles(input_zip, output_zip=None,
235 substitute=None):
236 """Copies files underneath system/ in the input zip to the output
237 zip. Populates the Item class with their metadata, and returns a
238 list of symlinks. output_zip may be None, in which case the copy is
239 skipped (but the other side effects still happen). substitute is an
240 optional dict of {output filename: contents} to be output instead of
241 certain input files.
242 """
243
244 symlinks = []
245
246 for info in input_zip.infolist():
247 if info.filename.startswith("SYSTEM/"):
248 basefilename = info.filename[7:]
249 if IsSymlink(info):
250 symlinks.append((input_zip.read(info.filename),
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700251 "/system/" + basefilename))
Doug Zongkereef39442009-04-02 12:14:19 -0700252 else:
253 info2 = copy.copy(info)
254 fn = info2.filename = "system/" + basefilename
255 if substitute and fn in substitute and substitute[fn] is None:
256 continue
257 if output_zip is not None:
258 if substitute and fn in substitute:
259 data = substitute[fn]
260 else:
261 data = input_zip.read(info.filename)
262 output_zip.writestr(info2, data)
263 if fn.endswith("/"):
264 Item.Get(fn[:-1], dir=True)
265 else:
266 Item.Get(fn, dir=False)
267
268 symlinks.sort()
269 return symlinks
270
271
Doug Zongkereef39442009-04-02 12:14:19 -0700272def SignOutput(temp_zip_name, output_zip_name):
273 key_passwords = common.GetKeyPasswords([OPTIONS.package_key])
274 pw = key_passwords[OPTIONS.package_key]
275
276 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw)
277
278
Doug Zongkereef39442009-04-02 12:14:19 -0700279def AppendAssertions(script, input_zip):
Doug Zongkereef39442009-04-02 12:14:19 -0700280 device = GetBuildProp("ro.product.device", input_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700281 script.AssertDevice(device)
Doug Zongkereef39442009-04-02 12:14:19 -0700282
283 info = input_zip.read("OTA/android-info.txt")
284 m = re.search(r"require\s+version-bootloader\s*=\s*(\S+)", info)
Doug Zongker171f1cd2009-06-15 22:36:37 -0700285 if m:
286 bootloaders = m.group(1).split("|")
Doug Zongker0493e242009-07-22 09:28:25 -0700287 if "*" not in bootloaders:
288 script.AssertSomeBootloader(*bootloaders)
Doug Zongkereef39442009-04-02 12:14:19 -0700289
290
Doug Zongker73ef8252009-07-23 15:12:53 -0700291def MakeRecoveryPatch(output_zip, recovery_img, boot_img):
292 """Generate a binary patch that creates the recovery image starting
293 with the boot image. (Most of the space in these images is just the
294 kernel, which is identical for the two, so the resulting patch
295 should be efficient.) Add it to the output zip, along with a shell
296 script that is run from init.rc on first boot to actually do the
297 patching and install the new recovery image.
298
299 recovery_img and boot_img should be File objects for the
300 corresponding images.
301
302 Returns an Item for the shell script, which must be made
303 executable.
304 """
305
306 patch = Difference(recovery_img, boot_img, "imgdiff")
307 common.ZipWriteStr(output_zip, "system/recovery-from-boot.p", patch)
308 Item.Get("system/recovery-from-boot.p", dir=False)
309
310 # Images with different content will have a different first page, so
311 # we check to see if this recovery has already been installed by
312 # testing just the first 2k.
313 HEADER_SIZE = 2048
314 header_sha1 = sha.sha(recovery_img.data[:HEADER_SIZE]).hexdigest()
315 sh = """#!/system/bin/sh
316if ! applypatch -c MTD:recovery:%(header_size)d:%(header_sha1)s; then
317 log -t recovery "Installing new recovery image"
318 applypatch MTD:boot:%(boot_size)d:%(boot_sha1)s MTD:recovery %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p
319else
320 log -t recovery "Recovery image already installed"
321fi
322""" % { 'boot_size': boot_img.size,
323 'boot_sha1': boot_img.sha1,
324 'header_size': HEADER_SIZE,
325 'header_sha1': header_sha1,
326 'recovery_size': recovery_img.size,
327 'recovery_sha1': recovery_img.sha1 }
328 common.ZipWriteStr(output_zip, "system/etc/install-recovery.sh", sh)
329 return Item.Get("system/etc/install-recovery.sh", dir=False)
330
331
Doug Zongkereef39442009-04-02 12:14:19 -0700332def WriteFullOTAPackage(input_zip, output_zip):
Doug Zongker03061472009-07-13 18:36:37 -0700333 if OPTIONS.script_mode == "auto":
334 script = both_generator.BothGenerator(2)
335 elif OPTIONS.script_mode == "amend":
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700336 script = amend_generator.AmendGenerator()
337 else:
338 # TODO: how to determine this? We don't know what version it will
339 # be installed on top of. For now, we expect the API just won't
340 # change very often.
Doug Zongker03061472009-07-13 18:36:37 -0700341 script = edify_generator.EdifyGenerator(2)
Doug Zongkereef39442009-04-02 12:14:19 -0700342
Doug Zongker05d3dea2009-06-22 11:32:31 -0700343 device_specific = common.DeviceSpecificParams(
344 input_zip=input_zip,
345 output_zip=output_zip,
346 script=script,
347 input_tmp=OPTIONS.input_tmp)
348
Doug Zongker962069c2009-04-23 11:41:58 -0700349 if not OPTIONS.omit_prereq:
350 ts = GetBuildProp("ro.build.date.utc", input_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700351 script.AssertOlderBuild(ts)
Doug Zongkereef39442009-04-02 12:14:19 -0700352
353 AppendAssertions(script, input_zip)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700354 device_specific.FullOTA_Assertions()
Doug Zongker171f1cd2009-06-15 22:36:37 -0700355
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700356 script.ShowProgress(0.5, 0)
Doug Zongkereef39442009-04-02 12:14:19 -0700357
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700358 if OPTIONS.wipe_user_data:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700359 script.FormatPartition("userdata")
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700360
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700361 script.FormatPartition("system")
362 script.Mount("MTD", "system", "/system")
363 script.UnpackPackageDir("system", "/system")
Doug Zongkereef39442009-04-02 12:14:19 -0700364
365 symlinks = CopySystemFiles(input_zip, output_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700366 script.MakeSymlinks(symlinks)
Doug Zongkereef39442009-04-02 12:14:19 -0700367
Doug Zongker73ef8252009-07-23 15:12:53 -0700368 boot_img = File("boot.img", common.BuildBootableImage(
369 os.path.join(OPTIONS.input_tmp, "BOOT")))
370 recovery_img = File("recovery.img", common.BuildBootableImage(
371 os.path.join(OPTIONS.input_tmp, "RECOVERY")))
372 i = MakeRecoveryPatch(output_zip, recovery_img, boot_img)
Doug Zongkereef39442009-04-02 12:14:19 -0700373
Doug Zongker73ef8252009-07-23 15:12:53 -0700374 Item.GetMetadata()
Doug Zongkereef39442009-04-02 12:14:19 -0700375
Doug Zongker73ef8252009-07-23 15:12:53 -0700376 # GetMetadata uses the data in android_filesystem_config.h to assign
377 # the uid/gid/mode of all files. We want to override that for the
378 # recovery patching shell script to make it executable.
379 i.uid = 0
380 i.gid = 0
381 i.mode = 0544
382 Item.Get("system").SetPermissions(script)
383
384 common.CheckSize(boot_img.data, "boot.img")
385 common.ZipWriteStr(output_zip, "boot.img", boot_img.data)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700386 script.ShowProgress(0.2, 0)
387
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700388 script.ShowProgress(0.2, 10)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700389 script.WriteRawImage("boot", "boot.img")
390
391 script.ShowProgress(0.1, 0)
392 device_specific.FullOTA_InstallEnd()
Doug Zongkereef39442009-04-02 12:14:19 -0700393
Doug Zongker1c390a22009-05-14 19:06:36 -0700394 if OPTIONS.extra_script is not None:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700395 script.AppendExtra(OPTIONS.extra_script)
Doug Zongker1c390a22009-05-14 19:06:36 -0700396
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700397 script.AddToZip(input_zip, output_zip)
Doug Zongkereef39442009-04-02 12:14:19 -0700398
399
400class File(object):
401 def __init__(self, name, data):
402 self.name = name
403 self.data = data
404 self.size = len(data)
405 self.sha1 = sha.sha(data).hexdigest()
406
407 def WriteToTemp(self):
408 t = tempfile.NamedTemporaryFile()
409 t.write(self.data)
410 t.flush()
411 return t
412
413 def AddToZip(self, z):
Doug Zongker048e7ca2009-06-15 14:31:53 -0700414 common.ZipWriteStr(z, self.name, self.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700415
416
417def LoadSystemFiles(z):
418 """Load all the files from SYSTEM/... in a given target-files
419 ZipFile, and return a dict of {filename: File object}."""
420 out = {}
421 for info in z.infolist():
422 if info.filename.startswith("SYSTEM/") and not IsSymlink(info):
423 fn = "system/" + info.filename[7:]
424 data = z.read(info.filename)
425 out[fn] = File(fn, data)
426 return out
427
428
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700429def Difference(tf, sf, diff_program):
430 """Return the patch (as a string of data) needed to turn sf into tf.
431 diff_program is the name of an external program (or list, if
432 additional arguments are desired) to run to generate the diff.
433 """
Doug Zongkereef39442009-04-02 12:14:19 -0700434
435 ttemp = tf.WriteToTemp()
436 stemp = sf.WriteToTemp()
437
438 ext = os.path.splitext(tf.name)[1]
439
440 try:
441 ptemp = tempfile.NamedTemporaryFile()
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700442 if isinstance(diff_program, list):
443 cmd = copy.copy(diff_program)
444 else:
445 cmd = [diff_program]
446 cmd.append(stemp.name)
447 cmd.append(ttemp.name)
448 cmd.append(ptemp.name)
449 p = common.Run(cmd)
Doug Zongkereef39442009-04-02 12:14:19 -0700450 _, err = p.communicate()
Doug Zongker5da317e2009-06-02 13:38:17 -0700451 if err or p.returncode != 0:
452 print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
453 return None
Doug Zongkereef39442009-04-02 12:14:19 -0700454 diff = ptemp.read()
Doug Zongkereef39442009-04-02 12:14:19 -0700455 finally:
Doug Zongker5da317e2009-06-02 13:38:17 -0700456 ptemp.close()
Doug Zongkereef39442009-04-02 12:14:19 -0700457 stemp.close()
458 ttemp.close()
459
460 return diff
461
462
463def GetBuildProp(property, z):
464 """Return the fingerprint of the build of a given target-files
465 ZipFile object."""
466 bp = z.read("SYSTEM/build.prop")
467 if not property:
468 return bp
469 m = re.search(re.escape(property) + r"=(.*)\n", bp)
470 if not m:
Doug Zongker9fc74c72009-06-23 16:27:38 -0700471 raise common.ExternalError("couldn't find %s in build.prop" % (property,))
Doug Zongkereef39442009-04-02 12:14:19 -0700472 return m.group(1).strip()
473
474
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700475def GetRecoveryAPIVersion(zip):
476 """Returns the version of the recovery API. Version 0 is the older
477 amend code (no separate binary)."""
478 try:
479 version = zip.read("META/recovery-api-version.txt")
480 return int(version)
481 except KeyError:
482 try:
483 # version one didn't have the recovery-api-version.txt file, but
484 # it did include an updater binary.
485 zip.getinfo("OTA/bin/updater")
486 return 1
487 except KeyError:
488 return 0
489
Doug Zongkereef39442009-04-02 12:14:19 -0700490def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700491 source_version = GetRecoveryAPIVersion(source_zip)
492
493 if OPTIONS.script_mode == 'amend':
494 script = amend_generator.AmendGenerator()
495 elif OPTIONS.script_mode == 'edify':
496 if source_version == 0:
497 print ("WARNING: generating edify script for a source that "
498 "can't install it.")
499 script = edify_generator.EdifyGenerator(source_version)
500 elif OPTIONS.script_mode == 'auto':
501 if source_version > 0:
502 script = edify_generator.EdifyGenerator(source_version)
503 else:
504 script = amend_generator.AmendGenerator()
505 else:
506 raise ValueError('unknown script mode "%s"' % (OPTIONS.script_mode,))
Doug Zongkereef39442009-04-02 12:14:19 -0700507
Doug Zongker05d3dea2009-06-22 11:32:31 -0700508 device_specific = common.DeviceSpecificParams(
509 source_zip=source_zip,
510 target_zip=target_zip,
511 output_zip=output_zip,
512 script=script)
513
Doug Zongkereef39442009-04-02 12:14:19 -0700514 print "Loading target..."
515 target_data = LoadSystemFiles(target_zip)
516 print "Loading source..."
517 source_data = LoadSystemFiles(source_zip)
518
519 verbatim_targets = []
520 patch_list = []
521 largest_source_size = 0
522 for fn in sorted(target_data.keys()):
523 tf = target_data[fn]
524 sf = source_data.get(fn, None)
525
526 if sf is None or fn in OPTIONS.require_verbatim:
527 # This file should be included verbatim
528 if fn in OPTIONS.prohibit_verbatim:
Doug Zongker9fc74c72009-06-23 16:27:38 -0700529 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,))
Doug Zongkereef39442009-04-02 12:14:19 -0700530 print "send", fn, "verbatim"
531 tf.AddToZip(output_zip)
532 verbatim_targets.append((fn, tf.size))
533 elif tf.sha1 != sf.sha1:
534 # File is different; consider sending as a patch
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700535 diff_method = "bsdiff"
536 if tf.name.endswith(".gz"):
537 diff_method = "imgdiff"
538 d = Difference(tf, sf, diff_method)
Doug Zongker5da317e2009-06-02 13:38:17 -0700539 if d is not None:
540 print fn, tf.size, len(d), (float(len(d)) / tf.size)
541 if d is None or len(d) > tf.size * OPTIONS.patch_threshold:
Doug Zongkereef39442009-04-02 12:14:19 -0700542 # patch is almost as big as the file; don't bother patching
543 tf.AddToZip(output_zip)
544 verbatim_targets.append((fn, tf.size))
545 else:
Doug Zongker048e7ca2009-06-15 14:31:53 -0700546 common.ZipWriteStr(output_zip, "patch/" + fn + ".p", d)
Doug Zongkereef39442009-04-02 12:14:19 -0700547 patch_list.append((fn, tf, sf, tf.size))
548 largest_source_size = max(largest_source_size, sf.size)
549 else:
550 # Target file identical to source.
551 pass
552
553 total_verbatim_size = sum([i[1] for i in verbatim_targets])
554 total_patched_size = sum([i[3] for i in patch_list])
555
556 source_fp = GetBuildProp("ro.build.fingerprint", source_zip)
557 target_fp = GetBuildProp("ro.build.fingerprint", target_zip)
558
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700559 script.Mount("MTD", "system", "/system")
560 script.AssertSomeFingerprint(source_fp, target_fp)
Doug Zongkereef39442009-04-02 12:14:19 -0700561
Doug Zongker5da317e2009-06-02 13:38:17 -0700562 source_boot = File("/tmp/boot.img",
563 common.BuildBootableImage(
564 os.path.join(OPTIONS.source_tmp, "BOOT")))
565 target_boot = File("/tmp/boot.img",
566 common.BuildBootableImage(
567 os.path.join(OPTIONS.target_tmp, "BOOT")))
568 updating_boot = (source_boot.data != target_boot.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700569
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700570 source_recovery = File("system/recovery.img",
571 common.BuildBootableImage(
572 os.path.join(OPTIONS.source_tmp, "RECOVERY")))
573 target_recovery = File("system/recovery.img",
574 common.BuildBootableImage(
575 os.path.join(OPTIONS.target_tmp, "RECOVERY")))
576 updating_recovery = (source_recovery.data != target_recovery.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700577
Doug Zongker05d3dea2009-06-22 11:32:31 -0700578 # We reserve the last 0.3 of the progress bar for the
579 # device-specific IncrementalOTA_InstallEnd() call at the end, which
580 # will typically install a radio image.
581 progress_bar_total = 0.7
Doug Zongkereef39442009-04-02 12:14:19 -0700582 if updating_boot:
583 progress_bar_total -= 0.1
Doug Zongkereef39442009-04-02 12:14:19 -0700584
585 AppendAssertions(script, target_zip)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700586 device_specific.IncrementalOTA_Assertions()
Doug Zongkereef39442009-04-02 12:14:19 -0700587
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700588 script.Print("Verifying current system...")
589
Doug Zongkereef39442009-04-02 12:14:19 -0700590 pb_verify = progress_bar_total * 0.3 * \
591 (total_patched_size /
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700592 float(total_patched_size+total_verbatim_size+1))
Doug Zongkereef39442009-04-02 12:14:19 -0700593
594 for i, (fn, tf, sf, size) in enumerate(patch_list):
595 if i % 5 == 0:
596 next_sizes = sum([i[3] for i in patch_list[i:i+5]])
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700597 script.ShowProgress(next_sizes * pb_verify / (total_patched_size+1), 1)
598
599 script.PatchCheck("/"+fn, tf.sha1, sf.sha1)
Doug Zongkereef39442009-04-02 12:14:19 -0700600
Doug Zongker5da317e2009-06-02 13:38:17 -0700601 if updating_boot:
602 d = Difference(target_boot, source_boot, "imgdiff")
603 print "boot target: %d source: %d diff: %d" % (
604 target_boot.size, source_boot.size, len(d))
605
Doug Zongker048e7ca2009-06-15 14:31:53 -0700606 common.ZipWriteStr(output_zip, "patch/boot.img.p", d)
Doug Zongker5da317e2009-06-02 13:38:17 -0700607
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700608 script.PatchCheck("MTD:boot:%d:%s:%d:%s" %
609 (source_boot.size, source_boot.sha1,
610 target_boot.size, target_boot.sha1))
Doug Zongker5da317e2009-06-02 13:38:17 -0700611
612 if patch_list or updating_recovery or updating_boot:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700613 script.CacheFreeSpaceCheck(largest_source_size)
614 script.Print("Unpacking patches...")
615 script.UnpackPackageDir("patch", "/tmp/patchtmp")
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700616
Doug Zongker05d3dea2009-06-22 11:32:31 -0700617 device_specific.IncrementalOTA_VerifyEnd()
618
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700619 script.Comment("---- start making changes here ----")
Doug Zongkereef39442009-04-02 12:14:19 -0700620
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700621 if OPTIONS.wipe_user_data:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700622 script.Print("Erasing user data...")
623 script.FormatPartition("userdata")
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700624
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700625 script.Print("Removing unneeded files...")
Doug Zongker0f3298a2009-06-30 08:16:58 -0700626 script.DeleteFiles(["/"+i[0] for i in verbatim_targets] +
627 ["/"+i for i in sorted(source_data)
628 if i not in target_data])
Doug Zongkereef39442009-04-02 12:14:19 -0700629
630 if updating_boot:
Doug Zongker5da317e2009-06-02 13:38:17 -0700631 # Produce the boot image by applying a patch to the current
632 # contents of the boot partition, and write it back to the
633 # partition.
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700634 script.Print("Patching boot image...")
635 script.ApplyPatch("MTD:boot:%d:%s:%d:%s"
636 % (source_boot.size, source_boot.sha1,
637 target_boot.size, target_boot.sha1),
638 "-",
639 target_boot.size, target_boot.sha1,
640 source_boot.sha1, "/tmp/patchtmp/boot.img.p")
Doug Zongkereef39442009-04-02 12:14:19 -0700641 print "boot image changed; including."
642 else:
643 print "boot image unchanged; skipping."
644
645 if updating_recovery:
Doug Zongker73ef8252009-07-23 15:12:53 -0700646 # Is it better to generate recovery as a patch from the current
647 # boot image, or from the previous recovery image? For large
648 # updates with significant kernel changes, probably the former.
649 # For small updates where the kernel hasn't changed, almost
650 # certainly the latter. We pick the first option. Future
651 # complicated schemes may let us effectively use both.
652 #
653 # A wacky possibility: as long as there is room in the boot
654 # partition, include the binaries and image files from recovery in
655 # the boot image (though not in the ramdisk) so they can be used
656 # as fodder for constructing the recovery image.
657 recovery_sh_item = MakeRecoveryPatch(output_zip,
658 target_recovery, target_boot)
659 print "recovery image changed; including as patch from boot."
Doug Zongkereef39442009-04-02 12:14:19 -0700660 else:
661 print "recovery image unchanged; skipping."
662
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700663 script.Print("Patching system files...")
Doug Zongkereef39442009-04-02 12:14:19 -0700664 pb_apply = progress_bar_total * 0.7 * \
665 (total_patched_size /
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700666 float(total_patched_size+total_verbatim_size+1))
Doug Zongkereef39442009-04-02 12:14:19 -0700667 for i, (fn, tf, sf, size) in enumerate(patch_list):
668 if i % 5 == 0:
669 next_sizes = sum([i[3] for i in patch_list[i:i+5]])
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700670 script.ShowProgress(next_sizes * pb_apply / (total_patched_size+1), 1)
671 script.ApplyPatch("/"+fn, "-", tf.size, tf.sha1,
672 sf.sha1, "/tmp/patchtmp/"+fn+".p")
Doug Zongkereef39442009-04-02 12:14:19 -0700673
674 target_symlinks = CopySystemFiles(target_zip, None)
675
676 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks])
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700677 temp_script = script.MakeTemporary()
Doug Zongker73ef8252009-07-23 15:12:53 -0700678 Item.GetMetadata()
679 if updating_recovery:
680 recovery_sh_item.uid = 0
681 recovery_sh_item.gid = 0
682 recovery_sh_item.mode = 0544
683 Item.Get("system").SetPermissions(temp_script)
Doug Zongkereef39442009-04-02 12:14:19 -0700684
685 # Note that this call will mess up the tree of Items, so make sure
686 # we're done with it.
687 source_symlinks = CopySystemFiles(source_zip, None)
688 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks])
689
690 # Delete all the symlinks in source that aren't in target. This
691 # needs to happen before verbatim files are unpacked, in case a
692 # symlink in the source is replaced by a real file in the target.
693 to_delete = []
694 for dest, link in source_symlinks:
695 if link not in target_symlinks_d:
696 to_delete.append(link)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700697 script.DeleteFiles(to_delete)
Doug Zongkereef39442009-04-02 12:14:19 -0700698
699 if verbatim_targets:
700 pb_verbatim = progress_bar_total * \
701 (total_verbatim_size /
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700702 float(total_patched_size+total_verbatim_size+1))
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700703 script.ShowProgress(pb_verbatim, 5)
704 script.Print("Unpacking new files...")
705 script.UnpackPackageDir("system", "/system")
706
Doug Zongker05d3dea2009-06-22 11:32:31 -0700707 script.Print("Symlinks and permissions...")
Doug Zongkereef39442009-04-02 12:14:19 -0700708
709 # Create all the symlinks that don't already exist, or point to
710 # somewhere different than what we want. Delete each symlink before
711 # creating it, since the 'symlink' command won't overwrite.
712 to_create = []
713 for dest, link in target_symlinks:
714 if link in source_symlinks_d:
715 if dest != source_symlinks_d[link]:
716 to_create.append((dest, link))
717 else:
718 to_create.append((dest, link))
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700719 script.DeleteFiles([i[1] for i in to_create])
720 script.MakeSymlinks(to_create)
Doug Zongkereef39442009-04-02 12:14:19 -0700721
722 # Now that the symlinks are created, we can set all the
723 # permissions.
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700724 script.AppendScript(temp_script)
Doug Zongkereef39442009-04-02 12:14:19 -0700725
Doug Zongker05d3dea2009-06-22 11:32:31 -0700726 # Write the radio image, if necessary.
727 script.ShowProgress(0.3, 10)
728 device_specific.IncrementalOTA_InstallEnd()
729
Doug Zongker1c390a22009-05-14 19:06:36 -0700730 if OPTIONS.extra_script is not None:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700731 scirpt.AppendExtra(OPTIONS.extra_script)
Doug Zongker1c390a22009-05-14 19:06:36 -0700732
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700733 script.AddToZip(target_zip, output_zip)
Doug Zongkereef39442009-04-02 12:14:19 -0700734
735
736def main(argv):
737
738 def option_handler(o, a):
739 if o in ("-b", "--board_config"):
Doug Zongkerfdd8e692009-08-03 17:27:48 -0700740 pass # deprecated
Doug Zongkereef39442009-04-02 12:14:19 -0700741 elif o in ("-k", "--package_key"):
742 OPTIONS.package_key = a
Doug Zongkereef39442009-04-02 12:14:19 -0700743 elif o in ("-i", "--incremental_from"):
744 OPTIONS.incremental_source = a
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700745 elif o in ("-w", "--wipe_user_data"):
746 OPTIONS.wipe_user_data = True
Doug Zongker962069c2009-04-23 11:41:58 -0700747 elif o in ("-n", "--no_prereq"):
748 OPTIONS.omit_prereq = True
Doug Zongker1c390a22009-05-14 19:06:36 -0700749 elif o in ("-e", "--extra_script"):
750 OPTIONS.extra_script = a
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700751 elif o in ("-m", "--script_mode"):
752 OPTIONS.script_mode = a
Doug Zongkereef39442009-04-02 12:14:19 -0700753 else:
754 return False
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700755 return True
Doug Zongkereef39442009-04-02 12:14:19 -0700756
757 args = common.ParseOptions(argv, __doc__,
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700758 extra_opts="b:k:i:d:wne:m:",
Doug Zongkereef39442009-04-02 12:14:19 -0700759 extra_long_opts=["board_config=",
760 "package_key=",
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700761 "incremental_from=",
Doug Zongker962069c2009-04-23 11:41:58 -0700762 "wipe_user_data",
Doug Zongker1c390a22009-05-14 19:06:36 -0700763 "no_prereq",
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700764 "extra_script=",
765 "script_mode="],
Doug Zongkereef39442009-04-02 12:14:19 -0700766 extra_option_handler=option_handler)
767
768 if len(args) != 2:
769 common.Usage(__doc__)
770 sys.exit(1)
771
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700772 if OPTIONS.script_mode not in ("amend", "edify", "auto"):
773 raise ValueError('unknown script mode "%s"' % (OPTIONS.script_mode,))
774
Doug Zongker1c390a22009-05-14 19:06:36 -0700775 if OPTIONS.extra_script is not None:
776 OPTIONS.extra_script = open(OPTIONS.extra_script).read()
777
Doug Zongkereef39442009-04-02 12:14:19 -0700778 print "unzipping target target-files..."
779 OPTIONS.input_tmp = common.UnzipTemp(args[0])
Doug Zongkerfdd8e692009-08-03 17:27:48 -0700780
781 common.LoadMaxSizes()
782 if not OPTIONS.max_image_size:
783 print
784 print " WARNING: Failed to load max image sizes; will not enforce"
785 print " image size limits."
786 print
787
Doug Zongkereef39442009-04-02 12:14:19 -0700788 OPTIONS.target_tmp = OPTIONS.input_tmp
789 input_zip = zipfile.ZipFile(args[0], "r")
790 if OPTIONS.package_key:
791 temp_zip_file = tempfile.NamedTemporaryFile()
792 output_zip = zipfile.ZipFile(temp_zip_file, "w",
793 compression=zipfile.ZIP_DEFLATED)
794 else:
795 output_zip = zipfile.ZipFile(args[1], "w",
796 compression=zipfile.ZIP_DEFLATED)
797
798 if OPTIONS.incremental_source is None:
799 WriteFullOTAPackage(input_zip, output_zip)
800 else:
801 print "unzipping source target-files..."
802 OPTIONS.source_tmp = common.UnzipTemp(OPTIONS.incremental_source)
803 source_zip = zipfile.ZipFile(OPTIONS.incremental_source, "r")
804 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip)
805
806 output_zip.close()
807 if OPTIONS.package_key:
808 SignOutput(temp_zip_file.name, args[1])
809 temp_zip_file.close()
810
811 common.Cleanup()
812
813 print "done."
814
815
816if __name__ == '__main__':
817 try:
818 main(sys.argv[1:])
819 except common.ExternalError, e:
820 print
821 print " ERROR: %s" % (e,)
822 print
823 sys.exit(1)