blob: a50f452f35d6131296320ab9018be2de1a3e295e [file] [log] [blame]
Doug Zongkereef39442009-04-02 12:14:19 -07001#!/usr/bin/env python
2#
3# Copyright (C) 2008 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Given a target-files zipfile, produces an OTA package that installs
19that build. An incremental OTA is produced if -i is given, otherwise
20a full OTA is produced.
21
22Usage: ota_from_target_files [flags] input_target_files output_ota_package
23
24 -b (--board_config) <file>
Doug Zongkerfdd8e692009-08-03 17:27:48 -070025 Deprecated.
Doug Zongkereef39442009-04-02 12:14:19 -070026
27 -k (--package_key) <key>
28 Key to use to sign the package (default is
29 "build/target/product/security/testkey").
30
31 -i (--incremental_from) <file>
32 Generate an incremental OTA using the given target-files zip as
33 the starting build.
34
Doug Zongkerdbfaae52009-04-21 17:12:54 -070035 -w (--wipe_user_data)
36 Generate an OTA package that will wipe the user data partition
37 when installed.
38
Doug Zongker962069c2009-04-23 11:41:58 -070039 -n (--no_prereq)
40 Omit the timestamp prereq check normally included at the top of
41 the build scripts (used for developer OTA packages which
42 legitimately need to go back and forth).
43
Doug Zongker1c390a22009-05-14 19:06:36 -070044 -e (--extra_script) <file>
45 Insert the contents of file at the end of the update script.
46
Doug Zongkereef39442009-04-02 12:14:19 -070047"""
48
49import sys
50
51if sys.hexversion < 0x02040000:
52 print >> sys.stderr, "Python 2.4 or newer is required."
53 sys.exit(1)
54
55import copy
Doug Zongkerc18736b2009-09-30 09:20:32 -070056import errno
Doug Zongkereef39442009-04-02 12:14:19 -070057import os
58import re
59import sha
60import subprocess
61import tempfile
Doug Zongker761e6422009-09-25 10:45:39 -070062import threading
Doug Zongkereef39442009-04-02 12:14:19 -070063import time
64import zipfile
65
66import common
Doug Zongkerc494d7c2009-06-18 08:43:44 -070067import edify_generator
Doug Zongkereef39442009-04-02 12:14:19 -070068
69OPTIONS = common.OPTIONS
70OPTIONS.package_key = "build/target/product/security/testkey"
71OPTIONS.incremental_source = None
72OPTIONS.require_verbatim = set()
73OPTIONS.prohibit_verbatim = set(("system/build.prop",))
74OPTIONS.patch_threshold = 0.95
Doug Zongkerdbfaae52009-04-21 17:12:54 -070075OPTIONS.wipe_user_data = False
Doug Zongker962069c2009-04-23 11:41:58 -070076OPTIONS.omit_prereq = False
Doug Zongker1c390a22009-05-14 19:06:36 -070077OPTIONS.extra_script = None
Doug Zongker761e6422009-09-25 10:45:39 -070078OPTIONS.worker_threads = 3
Doug Zongkereef39442009-04-02 12:14:19 -070079
80def MostPopularKey(d, default):
81 """Given a dict, return the key corresponding to the largest
82 value. Returns 'default' if the dict is empty."""
83 x = [(v, k) for (k, v) in d.iteritems()]
84 if not x: return default
85 x.sort()
86 return x[-1][1]
87
88
89def IsSymlink(info):
90 """Return true if the zipfile.ZipInfo object passed in represents a
91 symlink."""
92 return (info.external_attr >> 16) == 0120777
93
94
95
96class Item:
97 """Items represent the metadata (user, group, mode) of files and
98 directories in the system image."""
99 ITEMS = {}
100 def __init__(self, name, dir=False):
101 self.name = name
102 self.uid = None
103 self.gid = None
104 self.mode = None
105 self.dir = dir
106
107 if name:
108 self.parent = Item.Get(os.path.dirname(name), dir=True)
109 self.parent.children.append(self)
110 else:
111 self.parent = None
112 if dir:
113 self.children = []
114
115 def Dump(self, indent=0):
116 if self.uid is not None:
117 print "%s%s %d %d %o" % (" "*indent, self.name, self.uid, self.gid, self.mode)
118 else:
119 print "%s%s %s %s %s" % (" "*indent, self.name, self.uid, self.gid, self.mode)
120 if self.dir:
121 print "%s%s" % (" "*indent, self.descendants)
122 print "%s%s" % (" "*indent, self.best_subtree)
123 for i in self.children:
124 i.Dump(indent=indent+1)
125
126 @classmethod
127 def Get(cls, name, dir=False):
128 if name not in cls.ITEMS:
129 cls.ITEMS[name] = Item(name, dir=dir)
130 return cls.ITEMS[name]
131
132 @classmethod
Doug Zongker283e2a12010-03-15 17:52:32 -0700133 def GetMetadata(cls, input_zip):
134
135 try:
136 # See if the target_files contains a record of what the uid,
137 # gid, and mode is supposed to be.
138 output = input_zip.read("META/filesystem_config.txt")
139 except KeyError:
140 # Run the external 'fs_config' program to determine the desired
141 # uid, gid, and mode for every Item object. Note this uses the
142 # one in the client now, which might not be the same as the one
143 # used when this target_files was built.
144 p = common.Run(["fs_config"], stdin=subprocess.PIPE,
145 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
146 suffix = { False: "", True: "/" }
147 input = "".join(["%s%s\n" % (i.name, suffix[i.dir])
148 for i in cls.ITEMS.itervalues() if i.name])
Doug Zongker3475d362010-03-17 16:39:30 -0700149 output, error = p.communicate(input)
Doug Zongker283e2a12010-03-15 17:52:32 -0700150 assert not error
Doug Zongkereef39442009-04-02 12:14:19 -0700151
152 for line in output.split("\n"):
153 if not line: continue
154 name, uid, gid, mode = line.split()
Doug Zongker283e2a12010-03-15 17:52:32 -0700155 i = cls.ITEMS.get(name, None)
156 if i is not None:
157 i.uid = int(uid)
158 i.gid = int(gid)
159 i.mode = int(mode, 8)
160 if i.dir:
161 i.children.sort(key=lambda i: i.name)
162
163 # set metadata for the files generated by this script.
164 i = cls.ITEMS.get("system/recovery-from-boot.p", None)
165 if i: i.uid, i.gid, i.mode = 0, 0, 0644
166 i = cls.ITEMS.get("system/etc/install-recovery.sh", None)
167 if i: i.uid, i.gid, i.mode = 0, 0, 0544
Doug Zongkereef39442009-04-02 12:14:19 -0700168
169 def CountChildMetadata(self):
170 """Count up the (uid, gid, mode) tuples for all children and
171 determine the best strategy for using set_perm_recursive and
172 set_perm to correctly chown/chmod all the files to their desired
173 values. Recursively calls itself for all descendants.
174
175 Returns a dict of {(uid, gid, dmode, fmode): count} counting up
176 all descendants of this node. (dmode or fmode may be None.) Also
177 sets the best_subtree of each directory Item to the (uid, gid,
178 dmode, fmode) tuple that will match the most descendants of that
179 Item.
180 """
181
182 assert self.dir
183 d = self.descendants = {(self.uid, self.gid, self.mode, None): 1}
184 for i in self.children:
185 if i.dir:
186 for k, v in i.CountChildMetadata().iteritems():
187 d[k] = d.get(k, 0) + v
188 else:
189 k = (i.uid, i.gid, None, i.mode)
190 d[k] = d.get(k, 0) + 1
191
192 # Find the (uid, gid, dmode, fmode) tuple that matches the most
193 # descendants.
194
195 # First, find the (uid, gid) pair that matches the most
196 # descendants.
197 ug = {}
198 for (uid, gid, _, _), count in d.iteritems():
199 ug[(uid, gid)] = ug.get((uid, gid), 0) + count
200 ug = MostPopularKey(ug, (0, 0))
201
202 # Now find the dmode and fmode that match the most descendants
203 # with that (uid, gid), and choose those.
204 best_dmode = (0, 0755)
205 best_fmode = (0, 0644)
206 for k, count in d.iteritems():
207 if k[:2] != ug: continue
208 if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2])
209 if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3])
210 self.best_subtree = ug + (best_dmode[1], best_fmode[1])
211
212 return d
213
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700214 def SetPermissions(self, script):
Doug Zongkereef39442009-04-02 12:14:19 -0700215 """Append set_perm/set_perm_recursive commands to 'script' to
216 set all permissions, users, and groups for the tree of files
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700217 rooted at 'self'."""
Doug Zongkereef39442009-04-02 12:14:19 -0700218
219 self.CountChildMetadata()
220
221 def recurse(item, current):
222 # current is the (uid, gid, dmode, fmode) tuple that the current
223 # item (and all its children) have already been set to. We only
224 # need to issue set_perm/set_perm_recursive commands if we're
225 # supposed to be something different.
226 if item.dir:
227 if current != item.best_subtree:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700228 script.SetPermissionsRecursive("/"+item.name, *item.best_subtree)
Doug Zongkereef39442009-04-02 12:14:19 -0700229 current = item.best_subtree
230
231 if item.uid != current[0] or item.gid != current[1] or \
232 item.mode != current[2]:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700233 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
Doug Zongkereef39442009-04-02 12:14:19 -0700234
235 for i in item.children:
236 recurse(i, current)
237 else:
238 if item.uid != current[0] or item.gid != current[1] or \
239 item.mode != current[3]:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700240 script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
Doug Zongkereef39442009-04-02 12:14:19 -0700241
242 recurse(self, (-1, -1, -1, -1))
243
244
245def CopySystemFiles(input_zip, output_zip=None,
246 substitute=None):
247 """Copies files underneath system/ in the input zip to the output
248 zip. Populates the Item class with their metadata, and returns a
249 list of symlinks. output_zip may be None, in which case the copy is
250 skipped (but the other side effects still happen). substitute is an
251 optional dict of {output filename: contents} to be output instead of
252 certain input files.
253 """
254
255 symlinks = []
256
257 for info in input_zip.infolist():
258 if info.filename.startswith("SYSTEM/"):
259 basefilename = info.filename[7:]
260 if IsSymlink(info):
261 symlinks.append((input_zip.read(info.filename),
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700262 "/system/" + basefilename))
Doug Zongkereef39442009-04-02 12:14:19 -0700263 else:
264 info2 = copy.copy(info)
265 fn = info2.filename = "system/" + basefilename
266 if substitute and fn in substitute and substitute[fn] is None:
267 continue
268 if output_zip is not None:
269 if substitute and fn in substitute:
270 data = substitute[fn]
271 else:
272 data = input_zip.read(info.filename)
273 output_zip.writestr(info2, data)
274 if fn.endswith("/"):
275 Item.Get(fn[:-1], dir=True)
276 else:
277 Item.Get(fn, dir=False)
278
279 symlinks.sort()
280 return symlinks
281
282
Doug Zongkereef39442009-04-02 12:14:19 -0700283def SignOutput(temp_zip_name, output_zip_name):
284 key_passwords = common.GetKeyPasswords([OPTIONS.package_key])
285 pw = key_passwords[OPTIONS.package_key]
286
Doug Zongker951495f2009-08-14 12:44:19 -0700287 common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw,
288 whole_file=True)
Doug Zongkereef39442009-04-02 12:14:19 -0700289
290
Doug Zongkereef39442009-04-02 12:14:19 -0700291def AppendAssertions(script, input_zip):
Doug Zongkereef39442009-04-02 12:14:19 -0700292 device = GetBuildProp("ro.product.device", input_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700293 script.AssertDevice(device)
Doug Zongkereef39442009-04-02 12:14:19 -0700294
Doug Zongkereef39442009-04-02 12:14:19 -0700295
Doug Zongker73ef8252009-07-23 15:12:53 -0700296def MakeRecoveryPatch(output_zip, recovery_img, boot_img):
297 """Generate a binary patch that creates the recovery image starting
298 with the boot image. (Most of the space in these images is just the
299 kernel, which is identical for the two, so the resulting patch
300 should be efficient.) Add it to the output zip, along with a shell
301 script that is run from init.rc on first boot to actually do the
302 patching and install the new recovery image.
303
304 recovery_img and boot_img should be File objects for the
305 corresponding images.
306
307 Returns an Item for the shell script, which must be made
308 executable.
309 """
310
Doug Zongker761e6422009-09-25 10:45:39 -0700311 d = Difference(recovery_img, boot_img)
312 _, _, patch = d.ComputePatch()
Doug Zongkercfd7db62009-10-07 11:35:53 -0700313 common.ZipWriteStr(output_zip, "recovery/recovery-from-boot.p", patch)
Doug Zongker73ef8252009-07-23 15:12:53 -0700314 Item.Get("system/recovery-from-boot.p", dir=False)
315
316 # Images with different content will have a different first page, so
317 # we check to see if this recovery has already been installed by
318 # testing just the first 2k.
319 HEADER_SIZE = 2048
320 header_sha1 = sha.sha(recovery_img.data[:HEADER_SIZE]).hexdigest()
321 sh = """#!/system/bin/sh
322if ! applypatch -c MTD:recovery:%(header_size)d:%(header_sha1)s; then
323 log -t recovery "Installing new recovery image"
324 applypatch MTD:boot:%(boot_size)d:%(boot_sha1)s MTD:recovery %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p
325else
326 log -t recovery "Recovery image already installed"
327fi
328""" % { 'boot_size': boot_img.size,
329 'boot_sha1': boot_img.sha1,
330 'header_size': HEADER_SIZE,
331 'header_sha1': header_sha1,
332 'recovery_size': recovery_img.size,
333 'recovery_sha1': recovery_img.sha1 }
Doug Zongkercfd7db62009-10-07 11:35:53 -0700334 common.ZipWriteStr(output_zip, "recovery/etc/install-recovery.sh", sh)
Doug Zongker73ef8252009-07-23 15:12:53 -0700335 return Item.Get("system/etc/install-recovery.sh", dir=False)
336
337
Doug Zongkereef39442009-04-02 12:14:19 -0700338def WriteFullOTAPackage(input_zip, output_zip):
Doug Zongker9ce2ebf2010-04-21 14:08:44 -0700339 # TODO: how to determine this? We don't know what version it will
340 # be installed on top of. For now, we expect the API just won't
341 # change very often.
342 script = edify_generator.EdifyGenerator(3)
Doug Zongkereef39442009-04-02 12:14:19 -0700343
Doug Zongker05d3dea2009-06-22 11:32:31 -0700344 device_specific = common.DeviceSpecificParams(
345 input_zip=input_zip,
Doug Zongker14833602010-02-02 13:12:04 -0800346 input_version=GetRecoveryAPIVersion(input_zip),
Doug Zongker05d3dea2009-06-22 11:32:31 -0700347 output_zip=output_zip,
348 script=script,
349 input_tmp=OPTIONS.input_tmp)
350
Doug Zongker962069c2009-04-23 11:41:58 -0700351 if not OPTIONS.omit_prereq:
352 ts = GetBuildProp("ro.build.date.utc", input_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700353 script.AssertOlderBuild(ts)
Doug Zongkereef39442009-04-02 12:14:19 -0700354
355 AppendAssertions(script, input_zip)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700356 device_specific.FullOTA_Assertions()
Doug Zongker171f1cd2009-06-15 22:36:37 -0700357
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700358 script.ShowProgress(0.5, 0)
Doug Zongkereef39442009-04-02 12:14:19 -0700359
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700360 if OPTIONS.wipe_user_data:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700361 script.FormatPartition("userdata")
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700362
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700363 script.FormatPartition("system")
364 script.Mount("MTD", "system", "/system")
Doug Zongkercfd7db62009-10-07 11:35:53 -0700365 script.UnpackPackageDir("recovery", "/system")
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700366 script.UnpackPackageDir("system", "/system")
Doug Zongkereef39442009-04-02 12:14:19 -0700367
368 symlinks = CopySystemFiles(input_zip, output_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700369 script.MakeSymlinks(symlinks)
Doug Zongkereef39442009-04-02 12:14:19 -0700370
Doug Zongker73ef8252009-07-23 15:12:53 -0700371 boot_img = File("boot.img", common.BuildBootableImage(
372 os.path.join(OPTIONS.input_tmp, "BOOT")))
373 recovery_img = File("recovery.img", common.BuildBootableImage(
374 os.path.join(OPTIONS.input_tmp, "RECOVERY")))
Doug Zongker283e2a12010-03-15 17:52:32 -0700375 MakeRecoveryPatch(output_zip, recovery_img, boot_img)
Doug Zongkereef39442009-04-02 12:14:19 -0700376
Doug Zongker283e2a12010-03-15 17:52:32 -0700377 Item.GetMetadata(input_zip)
Doug Zongker73ef8252009-07-23 15:12:53 -0700378 Item.Get("system").SetPermissions(script)
379
380 common.CheckSize(boot_img.data, "boot.img")
381 common.ZipWriteStr(output_zip, "boot.img", boot_img.data)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700382 script.ShowProgress(0.2, 0)
383
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700384 script.ShowProgress(0.2, 10)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700385 script.WriteRawImage("boot", "boot.img")
386
387 script.ShowProgress(0.1, 0)
388 device_specific.FullOTA_InstallEnd()
Doug Zongkereef39442009-04-02 12:14:19 -0700389
Doug Zongker1c390a22009-05-14 19:06:36 -0700390 if OPTIONS.extra_script is not None:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700391 script.AppendExtra(OPTIONS.extra_script)
Doug Zongker1c390a22009-05-14 19:06:36 -0700392
Doug Zongker14833602010-02-02 13:12:04 -0800393 script.UnmountAll()
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700394 script.AddToZip(input_zip, output_zip)
Doug Zongkereef39442009-04-02 12:14:19 -0700395
396
397class File(object):
398 def __init__(self, name, data):
399 self.name = name
400 self.data = data
401 self.size = len(data)
402 self.sha1 = sha.sha(data).hexdigest()
403
404 def WriteToTemp(self):
405 t = tempfile.NamedTemporaryFile()
406 t.write(self.data)
407 t.flush()
408 return t
409
410 def AddToZip(self, z):
Doug Zongker048e7ca2009-06-15 14:31:53 -0700411 common.ZipWriteStr(z, self.name, self.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700412
413
414def LoadSystemFiles(z):
415 """Load all the files from SYSTEM/... in a given target-files
416 ZipFile, and return a dict of {filename: File object}."""
417 out = {}
418 for info in z.infolist():
419 if info.filename.startswith("SYSTEM/") and not IsSymlink(info):
420 fn = "system/" + info.filename[7:]
421 data = z.read(info.filename)
422 out[fn] = File(fn, data)
423 return out
424
425
Doug Zongker761e6422009-09-25 10:45:39 -0700426DIFF_PROGRAM_BY_EXT = {
427 ".gz" : "imgdiff",
428 ".zip" : ["imgdiff", "-z"],
429 ".jar" : ["imgdiff", "-z"],
430 ".apk" : ["imgdiff", "-z"],
431 ".img" : "imgdiff",
432 }
Doug Zongkereef39442009-04-02 12:14:19 -0700433
Doug Zongkereef39442009-04-02 12:14:19 -0700434
Doug Zongker761e6422009-09-25 10:45:39 -0700435class Difference(object):
436 def __init__(self, tf, sf):
437 self.tf = tf
438 self.sf = sf
439 self.patch = None
Doug Zongkereef39442009-04-02 12:14:19 -0700440
Doug Zongker761e6422009-09-25 10:45:39 -0700441 def ComputePatch(self):
442 """Compute the patch (as a string of data) needed to turn sf into
443 tf. Returns the same tuple as GetPatch()."""
Doug Zongkereef39442009-04-02 12:14:19 -0700444
Doug Zongker761e6422009-09-25 10:45:39 -0700445 tf = self.tf
446 sf = self.sf
447
448 ext = os.path.splitext(tf.name)[1]
449 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
450
451 ttemp = tf.WriteToTemp()
452 stemp = sf.WriteToTemp()
453
454 ext = os.path.splitext(tf.name)[1]
455
456 try:
457 ptemp = tempfile.NamedTemporaryFile()
458 if isinstance(diff_program, list):
459 cmd = copy.copy(diff_program)
460 else:
461 cmd = [diff_program]
462 cmd.append(stemp.name)
463 cmd.append(ttemp.name)
464 cmd.append(ptemp.name)
465 p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
466 _, err = p.communicate()
467 if err or p.returncode != 0:
468 print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
469 return None
470 diff = ptemp.read()
471 finally:
472 ptemp.close()
473 stemp.close()
474 ttemp.close()
475
476 self.patch = diff
477 return self.tf, self.sf, self.patch
478
479
480 def GetPatch(self):
481 """Return a tuple (target_file, source_file, patch_data).
482 patch_data may be None if ComputePatch hasn't been called, or if
483 computing the patch failed."""
484 return self.tf, self.sf, self.patch
485
486
487def ComputeDifferences(diffs):
488 """Call ComputePatch on all the Difference objects in 'diffs'."""
489 print len(diffs), "diffs to compute"
490
491 # Do the largest files first, to try and reduce the long-pole effect.
492 by_size = [(i.tf.size, i) for i in diffs]
493 by_size.sort(reverse=True)
494 by_size = [i[1] for i in by_size]
495
496 lock = threading.Lock()
497 diff_iter = iter(by_size) # accessed under lock
498
499 def worker():
500 try:
501 lock.acquire()
502 for d in diff_iter:
503 lock.release()
504 start = time.time()
505 d.ComputePatch()
506 dur = time.time() - start
507 lock.acquire()
508
509 tf, sf, patch = d.GetPatch()
510 if sf.name == tf.name:
511 name = tf.name
512 else:
513 name = "%s (%s)" % (tf.name, sf.name)
514 if patch is None:
515 print "patching failed! %s" % (name,)
516 else:
517 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
518 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
519 lock.release()
Doug Zongker481c4e62009-09-28 10:07:13 -0700520 except Exception, e:
Doug Zongker761e6422009-09-25 10:45:39 -0700521 print e
522 raise
523
524 # start worker threads; wait for them all to finish.
525 threads = [threading.Thread(target=worker)
526 for i in range(OPTIONS.worker_threads)]
527 for th in threads:
528 th.start()
529 while threads:
530 threads.pop().join()
Doug Zongkereef39442009-04-02 12:14:19 -0700531
532
533def GetBuildProp(property, z):
534 """Return the fingerprint of the build of a given target-files
535 ZipFile object."""
536 bp = z.read("SYSTEM/build.prop")
537 if not property:
538 return bp
539 m = re.search(re.escape(property) + r"=(.*)\n", bp)
540 if not m:
Doug Zongker9fc74c72009-06-23 16:27:38 -0700541 raise common.ExternalError("couldn't find %s in build.prop" % (property,))
Doug Zongkereef39442009-04-02 12:14:19 -0700542 return m.group(1).strip()
543
544
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700545def GetRecoveryAPIVersion(zip):
546 """Returns the version of the recovery API. Version 0 is the older
547 amend code (no separate binary)."""
548 try:
549 version = zip.read("META/recovery-api-version.txt")
550 return int(version)
551 except KeyError:
552 try:
553 # version one didn't have the recovery-api-version.txt file, but
554 # it did include an updater binary.
555 zip.getinfo("OTA/bin/updater")
556 return 1
557 except KeyError:
558 return 0
559
Doug Zongker15604b82009-09-01 17:53:34 -0700560
Doug Zongkereef39442009-04-02 12:14:19 -0700561def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700562 source_version = GetRecoveryAPIVersion(source_zip)
Doug Zongker14833602010-02-02 13:12:04 -0800563 target_version = GetRecoveryAPIVersion(target_zip)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700564
Doug Zongker9ce2ebf2010-04-21 14:08:44 -0700565 if source_version == 0:
566 print ("WARNING: generating edify script for a source that "
567 "can't install it.")
568 script = edify_generator.EdifyGenerator(source_version)
Doug Zongkereef39442009-04-02 12:14:19 -0700569
Doug Zongker05d3dea2009-06-22 11:32:31 -0700570 device_specific = common.DeviceSpecificParams(
571 source_zip=source_zip,
Doug Zongker14833602010-02-02 13:12:04 -0800572 source_version=source_version,
Doug Zongker05d3dea2009-06-22 11:32:31 -0700573 target_zip=target_zip,
Doug Zongker14833602010-02-02 13:12:04 -0800574 target_version=target_version,
Doug Zongker05d3dea2009-06-22 11:32:31 -0700575 output_zip=output_zip,
576 script=script)
577
Doug Zongkereef39442009-04-02 12:14:19 -0700578 print "Loading target..."
579 target_data = LoadSystemFiles(target_zip)
580 print "Loading source..."
581 source_data = LoadSystemFiles(source_zip)
582
583 verbatim_targets = []
584 patch_list = []
Doug Zongker761e6422009-09-25 10:45:39 -0700585 diffs = []
Doug Zongkereef39442009-04-02 12:14:19 -0700586 largest_source_size = 0
587 for fn in sorted(target_data.keys()):
588 tf = target_data[fn]
Doug Zongker761e6422009-09-25 10:45:39 -0700589 assert fn == tf.name
Doug Zongkereef39442009-04-02 12:14:19 -0700590 sf = source_data.get(fn, None)
591
592 if sf is None or fn in OPTIONS.require_verbatim:
593 # This file should be included verbatim
594 if fn in OPTIONS.prohibit_verbatim:
Doug Zongker9fc74c72009-06-23 16:27:38 -0700595 raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,))
Doug Zongkereef39442009-04-02 12:14:19 -0700596 print "send", fn, "verbatim"
597 tf.AddToZip(output_zip)
598 verbatim_targets.append((fn, tf.size))
599 elif tf.sha1 != sf.sha1:
600 # File is different; consider sending as a patch
Doug Zongker761e6422009-09-25 10:45:39 -0700601 diffs.append(Difference(tf, sf))
Doug Zongkereef39442009-04-02 12:14:19 -0700602 else:
603 # Target file identical to source.
604 pass
605
Doug Zongker761e6422009-09-25 10:45:39 -0700606 ComputeDifferences(diffs)
607
608 for diff in diffs:
609 tf, sf, d = diff.GetPatch()
610 if d is None or len(d) > tf.size * OPTIONS.patch_threshold:
611 # patch is almost as big as the file; don't bother patching
612 tf.AddToZip(output_zip)
613 verbatim_targets.append((tf.name, tf.size))
614 else:
615 common.ZipWriteStr(output_zip, "patch/" + tf.name + ".p", d)
Doug Zongker5a482092010-02-17 16:09:18 -0800616 patch_list.append((tf.name, tf, sf, tf.size, sha.sha(d).hexdigest()))
Doug Zongker761e6422009-09-25 10:45:39 -0700617 largest_source_size = max(largest_source_size, sf.size)
Doug Zongkereef39442009-04-02 12:14:19 -0700618
619 source_fp = GetBuildProp("ro.build.fingerprint", source_zip)
620 target_fp = GetBuildProp("ro.build.fingerprint", target_zip)
621
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700622 script.Mount("MTD", "system", "/system")
623 script.AssertSomeFingerprint(source_fp, target_fp)
Doug Zongkereef39442009-04-02 12:14:19 -0700624
Doug Zongker5da317e2009-06-02 13:38:17 -0700625 source_boot = File("/tmp/boot.img",
626 common.BuildBootableImage(
627 os.path.join(OPTIONS.source_tmp, "BOOT")))
628 target_boot = File("/tmp/boot.img",
629 common.BuildBootableImage(
630 os.path.join(OPTIONS.target_tmp, "BOOT")))
631 updating_boot = (source_boot.data != target_boot.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700632
Doug Zongkerf6a8bad2009-05-29 11:41:21 -0700633 source_recovery = File("system/recovery.img",
634 common.BuildBootableImage(
635 os.path.join(OPTIONS.source_tmp, "RECOVERY")))
636 target_recovery = File("system/recovery.img",
637 common.BuildBootableImage(
638 os.path.join(OPTIONS.target_tmp, "RECOVERY")))
639 updating_recovery = (source_recovery.data != target_recovery.data)
Doug Zongkereef39442009-04-02 12:14:19 -0700640
Doug Zongker881dd402009-09-20 14:03:55 -0700641 # Here's how we divide up the progress bar:
642 # 0.1 for verifying the start state (PatchCheck calls)
643 # 0.8 for applying patches (ApplyPatch calls)
644 # 0.1 for unpacking verbatim files, symlinking, and doing the
645 # device-specific commands.
Doug Zongkereef39442009-04-02 12:14:19 -0700646
647 AppendAssertions(script, target_zip)
Doug Zongker05d3dea2009-06-22 11:32:31 -0700648 device_specific.IncrementalOTA_Assertions()
Doug Zongkereef39442009-04-02 12:14:19 -0700649
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700650 script.Print("Verifying current system...")
651
Doug Zongker881dd402009-09-20 14:03:55 -0700652 script.ShowProgress(0.1, 0)
653 total_verify_size = float(sum([i[2].size for i in patch_list]) + 1)
654 if updating_boot:
655 total_verify_size += source_boot.size
656 so_far = 0
Doug Zongkereef39442009-04-02 12:14:19 -0700657
Doug Zongker5a482092010-02-17 16:09:18 -0800658 for fn, tf, sf, size, patch_sha in patch_list:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700659 script.PatchCheck("/"+fn, tf.sha1, sf.sha1)
Doug Zongker881dd402009-09-20 14:03:55 -0700660 so_far += sf.size
661 script.SetProgress(so_far / total_verify_size)
Doug Zongkereef39442009-04-02 12:14:19 -0700662
Doug Zongker5da317e2009-06-02 13:38:17 -0700663 if updating_boot:
Doug Zongker761e6422009-09-25 10:45:39 -0700664 d = Difference(target_boot, source_boot)
665 _, _, d = d.ComputePatch()
Doug Zongker5da317e2009-06-02 13:38:17 -0700666 print "boot target: %d source: %d diff: %d" % (
667 target_boot.size, source_boot.size, len(d))
668
Doug Zongker048e7ca2009-06-15 14:31:53 -0700669 common.ZipWriteStr(output_zip, "patch/boot.img.p", d)
Doug Zongker5da317e2009-06-02 13:38:17 -0700670
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700671 script.PatchCheck("MTD:boot:%d:%s:%d:%s" %
672 (source_boot.size, source_boot.sha1,
673 target_boot.size, target_boot.sha1))
Doug Zongker881dd402009-09-20 14:03:55 -0700674 so_far += source_boot.size
675 script.SetProgress(so_far / total_verify_size)
Doug Zongker5da317e2009-06-02 13:38:17 -0700676
677 if patch_list or updating_recovery or updating_boot:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700678 script.CacheFreeSpaceCheck(largest_source_size)
Doug Zongker5a482092010-02-17 16:09:18 -0800679
Doug Zongker05d3dea2009-06-22 11:32:31 -0700680 device_specific.IncrementalOTA_VerifyEnd()
681
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700682 script.Comment("---- start making changes here ----")
Doug Zongkereef39442009-04-02 12:14:19 -0700683
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700684 if OPTIONS.wipe_user_data:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700685 script.Print("Erasing user data...")
686 script.FormatPartition("userdata")
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700687
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700688 script.Print("Removing unneeded files...")
Doug Zongker0f3298a2009-06-30 08:16:58 -0700689 script.DeleteFiles(["/"+i[0] for i in verbatim_targets] +
690 ["/"+i for i in sorted(source_data)
Doug Zongker3b949f02009-08-24 10:24:32 -0700691 if i not in target_data] +
692 ["/system/recovery.img"])
Doug Zongkereef39442009-04-02 12:14:19 -0700693
Doug Zongker881dd402009-09-20 14:03:55 -0700694 script.ShowProgress(0.8, 0)
695 total_patch_size = float(sum([i[1].size for i in patch_list]) + 1)
696 if updating_boot:
697 total_patch_size += target_boot.size
698 so_far = 0
699
700 script.Print("Patching system files...")
Doug Zongker5a482092010-02-17 16:09:18 -0800701 for fn, tf, sf, size, _ in patch_list:
Doug Zongkerc8d446b2010-02-22 15:41:53 -0800702 script.ApplyPatch("/"+fn, "-", tf.size, tf.sha1, sf.sha1, "patch/"+fn+".p")
Doug Zongker881dd402009-09-20 14:03:55 -0700703 so_far += tf.size
704 script.SetProgress(so_far / total_patch_size)
705
Doug Zongkereef39442009-04-02 12:14:19 -0700706 if updating_boot:
Doug Zongker5da317e2009-06-02 13:38:17 -0700707 # Produce the boot image by applying a patch to the current
708 # contents of the boot partition, and write it back to the
709 # partition.
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700710 script.Print("Patching boot image...")
711 script.ApplyPatch("MTD:boot:%d:%s:%d:%s"
712 % (source_boot.size, source_boot.sha1,
713 target_boot.size, target_boot.sha1),
714 "-",
715 target_boot.size, target_boot.sha1,
Doug Zongkerc8d446b2010-02-22 15:41:53 -0800716 source_boot.sha1, "patch/boot.img.p")
Doug Zongker881dd402009-09-20 14:03:55 -0700717 so_far += target_boot.size
718 script.SetProgress(so_far / total_patch_size)
Doug Zongkereef39442009-04-02 12:14:19 -0700719 print "boot image changed; including."
720 else:
721 print "boot image unchanged; skipping."
722
723 if updating_recovery:
Doug Zongker73ef8252009-07-23 15:12:53 -0700724 # Is it better to generate recovery as a patch from the current
725 # boot image, or from the previous recovery image? For large
726 # updates with significant kernel changes, probably the former.
727 # For small updates where the kernel hasn't changed, almost
728 # certainly the latter. We pick the first option. Future
729 # complicated schemes may let us effectively use both.
730 #
731 # A wacky possibility: as long as there is room in the boot
732 # partition, include the binaries and image files from recovery in
733 # the boot image (though not in the ramdisk) so they can be used
734 # as fodder for constructing the recovery image.
Doug Zongker283e2a12010-03-15 17:52:32 -0700735 MakeRecoveryPatch(output_zip, target_recovery, target_boot)
Doug Zongker42265392010-02-12 10:21:00 -0800736 script.DeleteFiles(["/system/recovery-from-boot.p",
737 "/system/etc/install-recovery.sh"])
Doug Zongker73ef8252009-07-23 15:12:53 -0700738 print "recovery image changed; including as patch from boot."
Doug Zongkereef39442009-04-02 12:14:19 -0700739 else:
740 print "recovery image unchanged; skipping."
741
Doug Zongker881dd402009-09-20 14:03:55 -0700742 script.ShowProgress(0.1, 10)
Doug Zongkereef39442009-04-02 12:14:19 -0700743
744 target_symlinks = CopySystemFiles(target_zip, None)
745
746 target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks])
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700747 temp_script = script.MakeTemporary()
Doug Zongker283e2a12010-03-15 17:52:32 -0700748 Item.GetMetadata(target_zip)
Doug Zongker73ef8252009-07-23 15:12:53 -0700749 Item.Get("system").SetPermissions(temp_script)
Doug Zongkereef39442009-04-02 12:14:19 -0700750
751 # Note that this call will mess up the tree of Items, so make sure
752 # we're done with it.
753 source_symlinks = CopySystemFiles(source_zip, None)
754 source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks])
755
756 # Delete all the symlinks in source that aren't in target. This
757 # needs to happen before verbatim files are unpacked, in case a
758 # symlink in the source is replaced by a real file in the target.
759 to_delete = []
760 for dest, link in source_symlinks:
761 if link not in target_symlinks_d:
762 to_delete.append(link)
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700763 script.DeleteFiles(to_delete)
Doug Zongkereef39442009-04-02 12:14:19 -0700764
765 if verbatim_targets:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700766 script.Print("Unpacking new files...")
767 script.UnpackPackageDir("system", "/system")
768
Doug Zongker42265392010-02-12 10:21:00 -0800769 if updating_recovery:
770 script.Print("Unpacking new recovery...")
771 script.UnpackPackageDir("recovery", "/system")
772
Doug Zongker05d3dea2009-06-22 11:32:31 -0700773 script.Print("Symlinks and permissions...")
Doug Zongkereef39442009-04-02 12:14:19 -0700774
775 # Create all the symlinks that don't already exist, or point to
776 # somewhere different than what we want. Delete each symlink before
777 # creating it, since the 'symlink' command won't overwrite.
778 to_create = []
779 for dest, link in target_symlinks:
780 if link in source_symlinks_d:
781 if dest != source_symlinks_d[link]:
782 to_create.append((dest, link))
783 else:
784 to_create.append((dest, link))
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700785 script.DeleteFiles([i[1] for i in to_create])
786 script.MakeSymlinks(to_create)
Doug Zongkereef39442009-04-02 12:14:19 -0700787
788 # Now that the symlinks are created, we can set all the
789 # permissions.
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700790 script.AppendScript(temp_script)
Doug Zongkereef39442009-04-02 12:14:19 -0700791
Doug Zongker881dd402009-09-20 14:03:55 -0700792 # Do device-specific installation (eg, write radio image).
Doug Zongker05d3dea2009-06-22 11:32:31 -0700793 device_specific.IncrementalOTA_InstallEnd()
794
Doug Zongker1c390a22009-05-14 19:06:36 -0700795 if OPTIONS.extra_script is not None:
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700796 scirpt.AppendExtra(OPTIONS.extra_script)
Doug Zongker1c390a22009-05-14 19:06:36 -0700797
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700798 script.AddToZip(target_zip, output_zip)
Doug Zongkereef39442009-04-02 12:14:19 -0700799
800
801def main(argv):
802
803 def option_handler(o, a):
804 if o in ("-b", "--board_config"):
Doug Zongkerfdd8e692009-08-03 17:27:48 -0700805 pass # deprecated
Doug Zongkereef39442009-04-02 12:14:19 -0700806 elif o in ("-k", "--package_key"):
807 OPTIONS.package_key = a
Doug Zongkereef39442009-04-02 12:14:19 -0700808 elif o in ("-i", "--incremental_from"):
809 OPTIONS.incremental_source = a
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700810 elif o in ("-w", "--wipe_user_data"):
811 OPTIONS.wipe_user_data = True
Doug Zongker962069c2009-04-23 11:41:58 -0700812 elif o in ("-n", "--no_prereq"):
813 OPTIONS.omit_prereq = True
Doug Zongker1c390a22009-05-14 19:06:36 -0700814 elif o in ("-e", "--extra_script"):
815 OPTIONS.extra_script = a
Doug Zongker761e6422009-09-25 10:45:39 -0700816 elif o in ("--worker_threads"):
817 OPTIONS.worker_threads = int(a)
Doug Zongkereef39442009-04-02 12:14:19 -0700818 else:
819 return False
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700820 return True
Doug Zongkereef39442009-04-02 12:14:19 -0700821
822 args = common.ParseOptions(argv, __doc__,
Doug Zongker9ce2ebf2010-04-21 14:08:44 -0700823 extra_opts="b:k:i:d:wne:",
Doug Zongkereef39442009-04-02 12:14:19 -0700824 extra_long_opts=["board_config=",
825 "package_key=",
Doug Zongkerdbfaae52009-04-21 17:12:54 -0700826 "incremental_from=",
Doug Zongker962069c2009-04-23 11:41:58 -0700827 "wipe_user_data",
Doug Zongker1c390a22009-05-14 19:06:36 -0700828 "no_prereq",
Doug Zongkerc494d7c2009-06-18 08:43:44 -0700829 "extra_script=",
Doug Zongker761e6422009-09-25 10:45:39 -0700830 "worker_threads="],
Doug Zongkereef39442009-04-02 12:14:19 -0700831 extra_option_handler=option_handler)
832
833 if len(args) != 2:
834 common.Usage(__doc__)
835 sys.exit(1)
836
Doug Zongker1c390a22009-05-14 19:06:36 -0700837 if OPTIONS.extra_script is not None:
838 OPTIONS.extra_script = open(OPTIONS.extra_script).read()
839
Doug Zongkereef39442009-04-02 12:14:19 -0700840 print "unzipping target target-files..."
841 OPTIONS.input_tmp = common.UnzipTemp(args[0])
Doug Zongkerfdd8e692009-08-03 17:27:48 -0700842
Doug Zongkerc18736b2009-09-30 09:20:32 -0700843 if OPTIONS.device_specific is None:
844 # look for the device-specific tools extension location in the input
845 try:
846 f = open(os.path.join(OPTIONS.input_tmp, "META", "tool-extensions.txt"))
847 ds = f.read().strip()
848 f.close()
849 if ds:
850 ds = os.path.normpath(ds)
851 print "using device-specific extensions in", ds
852 OPTIONS.device_specific = ds
853 except IOError, e:
854 if e.errno == errno.ENOENT:
855 # nothing specified in the file
856 pass
857 else:
858 raise
859
Doug Zongkerfdd8e692009-08-03 17:27:48 -0700860 common.LoadMaxSizes()
861 if not OPTIONS.max_image_size:
862 print
863 print " WARNING: Failed to load max image sizes; will not enforce"
864 print " image size limits."
865 print
866
Doug Zongkereef39442009-04-02 12:14:19 -0700867 OPTIONS.target_tmp = OPTIONS.input_tmp
868 input_zip = zipfile.ZipFile(args[0], "r")
869 if OPTIONS.package_key:
870 temp_zip_file = tempfile.NamedTemporaryFile()
871 output_zip = zipfile.ZipFile(temp_zip_file, "w",
872 compression=zipfile.ZIP_DEFLATED)
873 else:
874 output_zip = zipfile.ZipFile(args[1], "w",
875 compression=zipfile.ZIP_DEFLATED)
876
877 if OPTIONS.incremental_source is None:
878 WriteFullOTAPackage(input_zip, output_zip)
879 else:
880 print "unzipping source target-files..."
881 OPTIONS.source_tmp = common.UnzipTemp(OPTIONS.incremental_source)
882 source_zip = zipfile.ZipFile(OPTIONS.incremental_source, "r")
883 WriteIncrementalOTAPackage(input_zip, source_zip, output_zip)
884
885 output_zip.close()
886 if OPTIONS.package_key:
887 SignOutput(temp_zip_file.name, args[1])
888 temp_zip_file.close()
889
890 common.Cleanup()
891
892 print "done."
893
894
895if __name__ == '__main__':
896 try:
897 main(sys.argv[1:])
898 except common.ExternalError, e:
899 print
900 print " ERROR: %s" % (e,)
901 print
902 sys.exit(1)