Merge "releasetools: Drop the support for file-based OTA generation."
diff --git a/core/Makefile b/core/Makefile
index a98e400..e6b5878 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -1701,8 +1701,8 @@
$(INSTALLED_VENDORIMAGE_TARGET): $(INTERNAL_USERIMAGES_DEPS) $(INTERNAL_VENDORIMAGE_FILES) $(INSTALLED_FILES_FILE_VENDOR) $(BUILD_IMAGE_SRCS) $(DEPMOD) $(BOARD_VENDOR_KERNEL_MODULES)
$(build-vendorimage-target)
-.PHONY: vendorimage-nodeps
-vendorimage-nodeps: | $(INTERNAL_USERIMAGES_DEPS) $(DEPMOD)
+.PHONY: vendorimage-nodeps vnod
+vendorimage-nodeps vnod: | $(INTERNAL_USERIMAGES_DEPS) $(DEPMOD)
$(build-vendorimage-target)
else ifdef BOARD_PREBUILT_VENDORIMAGE
@@ -1905,6 +1905,13 @@
$(BUILT_TARGET_FILES_PACKAGE): PRIVATE_RECOVERY_OUT := RECOVERY
endif
+# Run fs_config while creating the target files package
+# $1: root directory
+# $2: add prefix
+define fs_config
+(cd $(1); find . -type d | sed 's,$$,/,'; find . \! -type d) | cut -c 3- | sort | sed 's,^,$(2),' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC)
+endef
+
# Depending on the various images guarantees that the underlying
# directories are up-to-date.
$(BUILT_TARGET_FILES_PACKAGE): \
@@ -2117,6 +2124,7 @@
ifdef OSRELEASED_DIRECTORY
$(hide) $(ACP) $(TARGET_OUT_OEM)/$(OSRELEASED_DIRECTORY)/product_id $(zip_root)/META/product_id.txt
$(hide) $(ACP) $(TARGET_OUT_OEM)/$(OSRELEASED_DIRECTORY)/product_version $(zip_root)/META/product_version.txt
+ $(hide) $(ACP) $(TARGET_OUT_ETC)/$(OSRELEASED_DIRECTORY)/system_version $(zip_root)/META/system_version.txt
endif
endif
ifeq ($(BREAKPAD_GENERATE_SYMBOLS),true)
@@ -2127,28 +2135,29 @@
$(hide) mkdir -p $(zip_root)/IMAGES
$(hide) cp $(INSTALLED_VENDORIMAGE_TARGET) $(zip_root)/IMAGES/
endif
+ @# Run fs_config on all the system, vendor, boot ramdisk,
+ @# and recovery ramdisk files in the zip, and save the output
+ $(hide) $(call fs_config,$(zip_root)/SYSTEM,system/) > $(zip_root)/META/filesystem_config.txt
+ifdef BOARD_VENDORIMAGE_FILE_SYSTEM_TYPE
+ $(hide) $(call fs_config,$(zip_root)/VENDOR,vendor/) > $(zip_root)/META/vendor_filesystem_config.txt
+endif
+ifeq ($(BOARD_BUILD_SYSTEM_ROOT_IMAGE),true)
+ $(hide) $(call fs_config,$(zip_root)/ROOT,) > $(zip_root)/META/root_filesystem_config.txt
+endif
+ $(hide) $(call fs_config,$(zip_root)/BOOT/RAMDISK,) > $(zip_root)/META/boot_filesystem_config.txt
+ifneq ($(INSTALLED_RECOVERYIMAGE_TARGET),)
+ $(hide) $(call fs_config,$(zip_root)/RECOVERY/RAMDISK,) > $(zip_root)/META/recovery_filesystem_config.txt
+endif
+ifdef INSTALLED_SYSTEMOTHERIMAGE_TARGET
+ $(hide) $(call fs_config,$(zip_root)/SYSTEM_OTHER,system/) > $(zip_root)/META/system_other_filesystem_config.txt
+endif
+ $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH MKBOOTIMG=$(MKBOOTIMG) \
+ ./build/tools/releasetools/add_img_to_target_files -a -v -p $(HOST_OUT) $(zip_root)
@# Zip everything up, preserving symlinks and placing META/ files first to
@# help early validation of the .zip file while uploading it.
$(hide) find $(zip_root)/META | sort >$@.list
$(hide) find $(zip_root) | grep -v "^$(zip_root)/META/" | sort >>$@.list
$(hide) $(SOONG_ZIP) -d -o $@ -C $(zip_root) -l $@.list
- @# Run fs_config on all the system, vendor, boot ramdisk,
- @# and recovery ramdisk files in the zip, and save the output
- $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="SYSTEM/" } /^SYSTEM\// {print "system/" $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/filesystem_config.txt
- $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="VENDOR/" } /^VENDOR\// {print "vendor/" $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/vendor_filesystem_config.txt
-ifeq ($(BOARD_BUILD_SYSTEM_ROOT_IMAGE),true)
- $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="ROOT/" } /^ROOT\// {print $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/root_filesystem_config.txt
-endif
- $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="BOOT/RAMDISK/" } /^BOOT\/RAMDISK\// {print $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/boot_filesystem_config.txt
-ifneq ($(INSTALLED_RECOVERYIMAGE_TARGET),)
- $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="RECOVERY/RAMDISK/" } /^RECOVERY\/RAMDISK\// {print $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/recovery_filesystem_config.txt
-endif
-ifdef INSTALLED_SYSTEMOTHERIMAGE_TARGET
- $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="SYSTEM_OTHER/" } /^SYSTEM_OTHER\// { print "system/" $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/system_other_filesystem_config.txt
-endif
- $(hide) (cd $(zip_root) && zip -qX ../$(notdir $@) META/*filesystem_config.txt)
- $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH MKBOOTIMG=$(MKBOOTIMG) \
- ./build/tools/releasetools/add_img_to_target_files -a -v -p $(HOST_OUT) $@
.PHONY: target-files-package
target-files-package: $(BUILT_TARGET_FILES_PACKAGE)
diff --git a/core/config.mk b/core/config.mk
index 7709d3c..8645a4c 100644
--- a/core/config.mk
+++ b/core/config.mk
@@ -851,7 +851,7 @@
userdataimage-nodeps userdatatarball-nodeps \
cacheimage-nodeps \
bptimage-nodeps \
- vendorimage-nodeps \
+ vnod vendorimage-nodeps \
systemotherimage-nodeps \
ramdisk-nodeps \
bootimage-nodeps \
diff --git a/core/help.mk b/core/help.mk
index 6e0b2c0..c034e79 100644
--- a/core/help.mk
+++ b/core/help.mk
@@ -22,6 +22,7 @@
@echo "droid Default target"
@echo "clean (aka clobber) equivalent to rm -rf out/"
@echo "snod Quickly rebuild the system image from built packages"
+ @echo "vnod Quickly rebuild the vendor image from built packages"
@echo "offline-sdk-docs Generate the HTML for the developer SDK docs"
@echo "doc-comment-check-docs Check HTML doc links & validity, without generating HTML"
@echo "libandroid_runtime All the JNI framework stuff"
diff --git a/tools/releasetools/add_img_to_target_files.py b/tools/releasetools/add_img_to_target_files.py
index 9403a77..0b8f3a0 100755
--- a/tools/releasetools/add_img_to_target_files.py
+++ b/tools/releasetools/add_img_to_target_files.py
@@ -73,6 +73,25 @@
OPTIONS.replace_verity_private_key = False
OPTIONS.is_signing = False
+
+class OutputFile(object):
+ def __init__(self, output_zip, input_dir, prefix, name):
+ self._output_zip = output_zip
+ self.input_name = os.path.join(input_dir, prefix, name)
+
+ if self._output_zip:
+ self._zip_name = os.path.join(prefix, name)
+
+ root, suffix = os.path.splitext(name)
+ self.name = common.MakeTempFile(prefix=root + '-', suffix=suffix)
+ else:
+ self.name = self.input_name
+
+ def Write(self):
+ if self._output_zip:
+ common.ZipWrite(self._output_zip, self.name, self._zip_name)
+
+
def GetCareMap(which, imgname):
"""Generate care_map of system (or vendor) partition"""
@@ -98,10 +117,10 @@
"""Turn the contents of SYSTEM into a system image and store it in
output_zip. Returns the name of the system image file."""
- prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system.img")
- if os.path.exists(prebuilt_path):
+ img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.img")
+ if os.path.exists(img.input_name):
print("system.img already exists in %s, no need to rebuild..." % (prefix,))
- return prebuilt_path
+ return img.input_name
def output_sink(fn, data):
ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w")
@@ -113,74 +132,52 @@
common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img,
boot_img, info_dict=OPTIONS.info_dict)
- block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map")
- imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict,
- block_list=block_list)
+ block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.map")
+ CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system", img,
+ block_list=block_list)
- common.ZipWrite(output_zip, imgname, prefix + "system.img")
- common.ZipWrite(output_zip, block_list, prefix + "system.map")
- return imgname
-
-
-def BuildSystem(input_dir, info_dict, block_list=None):
- """Build the (sparse) system image and return the name of a temp
- file containing it."""
- return CreateImage(input_dir, info_dict, "system", block_list=block_list)
+ return img.name
def AddSystemOther(output_zip, prefix="IMAGES/"):
"""Turn the contents of SYSTEM_OTHER into a system_other image
and store it in output_zip."""
- prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system_other.img")
- if os.path.exists(prebuilt_path):
+ img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system_other.img")
+ if os.path.exists(img.input_name):
print("system_other.img already exists in %s, no need to rebuild..." % (
prefix,))
return
- imgname = BuildSystemOther(OPTIONS.input_tmp, OPTIONS.info_dict)
- common.ZipWrite(output_zip, imgname, prefix + "system_other.img")
-
-def BuildSystemOther(input_dir, info_dict):
- """Build the (sparse) system_other image and return the name of a temp
- file containing it."""
- return CreateImage(input_dir, info_dict, "system_other", block_list=None)
+ CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system_other", img)
def AddVendor(output_zip, prefix="IMAGES/"):
"""Turn the contents of VENDOR into a vendor image and store in it
output_zip."""
- prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "vendor.img")
- if os.path.exists(prebuilt_path):
+ img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.img")
+ if os.path.exists(img.input_name):
print("vendor.img already exists in %s, no need to rebuild..." % (prefix,))
- return prebuilt_path
+ return img.input_name
- block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map")
- imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict,
- block_list=block_list)
- common.ZipWrite(output_zip, imgname, prefix + "vendor.img")
- common.ZipWrite(output_zip, block_list, prefix + "vendor.map")
- return imgname
+ block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.map")
+ CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "vendor", img,
+ block_list=block_list)
+ return img.name
-def BuildVendor(input_dir, info_dict, block_list=None):
- """Build the (sparse) vendor image and return the name of a temp
- file containing it."""
- return CreateImage(input_dir, info_dict, "vendor", block_list=block_list)
-
-
-def CreateImage(input_dir, info_dict, what, block_list=None):
+def CreateImage(input_dir, info_dict, what, output_file, block_list=None):
print("creating " + what + ".img...")
- img = common.MakeTempFile(prefix=what + "-", suffix=".img")
-
# The name of the directory it is making an image out of matters to
# mkyaffs2image. It wants "system" but we have a directory named
# "SYSTEM", so create a symlink.
+ temp_dir = tempfile.mkdtemp()
+ OPTIONS.tempfiles.append(temp_dir)
try:
os.symlink(os.path.join(input_dir, what.upper()),
- os.path.join(input_dir, what))
+ os.path.join(temp_dir, what))
except OSError as e:
# bogus error on my mac version?
# File "./build/tools/releasetools/img_from_target_files"
@@ -215,12 +212,16 @@
if fs_config:
image_props["fs_config"] = fs_config
if block_list:
- image_props["block_list"] = block_list
+ image_props["block_list"] = block_list.name
- succ = build_image.BuildImage(os.path.join(input_dir, what),
- image_props, img)
+ succ = build_image.BuildImage(os.path.join(temp_dir, what),
+ image_props, output_file.name)
assert succ, "build " + what + ".img image failed"
+ output_file.Write()
+ if block_list:
+ block_list.Write()
+
is_verity_partition = "verity_block_device" in image_props
verity_supported = image_props.get("verity") == "true"
if is_verity_partition and verity_supported:
@@ -229,8 +230,6 @@
adjusted_blocks_key = what + "_adjusted_partition_size"
info_dict[adjusted_blocks_key] = int(adjusted_blocks_value)/4096 - 1
- return img
-
def AddUserdata(output_zip, prefix="IMAGES/"):
"""Create a userdata image and store it in output_zip.
@@ -241,8 +240,8 @@
in OPTIONS.info_dict.
"""
- prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "userdata.img")
- if os.path.exists(prebuilt_path):
+ img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "userdata.img")
+ if os.path.exists(img.input_name):
print("userdata.img already exists in %s, no need to rebuild..." % (
prefix,))
return
@@ -265,6 +264,7 @@
# empty dir named "data", or a symlink to the DATA dir,
# and build the image from that.
temp_dir = tempfile.mkdtemp()
+ OPTIONS.tempfiles.append(temp_dir)
user_dir = os.path.join(temp_dir, "data")
empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true")
if empty:
@@ -275,8 +275,6 @@
os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"),
user_dir)
- img = tempfile.NamedTemporaryFile()
-
fstab = OPTIONS.info_dict["fstab"]
if fstab:
image_props["fs_type"] = fstab["/data"].fs_type
@@ -284,17 +282,15 @@
assert succ, "build userdata.img image failed"
common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict)
- common.ZipWrite(output_zip, img.name, prefix + "userdata.img")
- img.close()
- shutil.rmtree(temp_dir)
+ img.Write()
def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"):
"""Create a VBMeta image and store it in output_zip."""
- _, img_file_name = tempfile.mkstemp()
+ img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vbmeta.img")
avbtool = os.getenv('AVBTOOL') or "avbtool"
cmd = [avbtool, "make_vbmeta_image",
- "--output", img_file_name,
+ "--output", img.name,
"--include_descriptors_from_image", boot_img_path,
"--include_descriptors_from_image", system_img_path,
"--generate_dm_verity_cmdline_from_hashtree", system_img_path]
@@ -305,19 +301,19 @@
p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.communicate()
assert p.returncode == 0, "avbtool make_vbmeta_image failed"
- common.ZipWrite(output_zip, img_file_name, prefix + "vbmeta.img")
+ img.Write()
def AddPartitionTable(output_zip, prefix="IMAGES/"):
"""Create a partition table image and store it in output_zip."""
- _, img_file_name = tempfile.mkstemp()
- _, bpt_file_name = tempfile.mkstemp()
+ img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.img")
+ bpt = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.bpt")
# use BPTTOOL from environ, or "bpttool" if empty or not set.
bpttool = os.getenv("BPTTOOL") or "bpttool"
- cmd = [bpttool, "make_table", "--output_json", bpt_file_name,
- "--output_gpt", img_file_name]
+ cmd = [bpttool, "make_table", "--output_json", bpt.name,
+ "--output_gpt", img.name]
input_files_str = OPTIONS.info_dict["board_bpt_input_files"]
input_files = input_files_str.split(" ")
for i in input_files:
@@ -333,15 +329,15 @@
p.communicate()
assert p.returncode == 0, "bpttool make_table failed"
- common.ZipWrite(output_zip, img_file_name, prefix + "partition-table.img")
- common.ZipWrite(output_zip, bpt_file_name, prefix + "partition-table.bpt")
+ img.Write()
+ bpt.Write()
def AddCache(output_zip, prefix="IMAGES/"):
"""Create an empty cache image and store it in output_zip."""
- prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "cache.img")
- if os.path.exists(prebuilt_path):
+ img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "cache.img")
+ if os.path.exists(img.input_name):
print("cache.img already exists in %s, no need to rebuild..." % (prefix,))
return
@@ -362,9 +358,9 @@
# mkyaffs2image. So we create a temp dir, and within it we create an
# empty dir named "cache", and build the image from that.
temp_dir = tempfile.mkdtemp()
+ OPTIONS.tempfiles.append(temp_dir)
user_dir = os.path.join(temp_dir, "cache")
os.mkdir(user_dir)
- img = tempfile.NamedTemporaryFile()
fstab = OPTIONS.info_dict["fstab"]
if fstab:
@@ -373,35 +369,39 @@
assert succ, "build cache.img image failed"
common.CheckSize(img.name, "cache.img", OPTIONS.info_dict)
- common.ZipWrite(output_zip, img.name, prefix + "cache.img")
- img.close()
- os.rmdir(user_dir)
- os.rmdir(temp_dir)
+ img.Write()
def AddImagesToTargetFiles(filename):
- OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename)
+ if os.path.isdir(filename):
+ OPTIONS.input_tmp = os.path.abspath(filename)
+ input_zip = None
+ else:
+ OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename)
if not OPTIONS.add_missing:
- for n in input_zip.namelist():
- if n.startswith("IMAGES/"):
- print("target_files appears to already contain images.")
- sys.exit(1)
+ if os.path.isdir(os.path.join(OPTIONS.input_tmp, "IMAGES")):
+ print("target_files appears to already contain images.")
+ sys.exit(1)
- try:
- input_zip.getinfo("VENDOR/")
- has_vendor = True
- except KeyError:
- has_vendor = False
+ has_vendor = os.path.isdir(os.path.join(OPTIONS.input_tmp, "VENDOR"))
+ has_system_other = os.path.isdir(os.path.join(OPTIONS.input_tmp,
+ "SYSTEM_OTHER"))
- has_system_other = "SYSTEM_OTHER/" in input_zip.namelist()
+ if input_zip:
+ OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp)
- OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp)
-
- common.ZipClose(input_zip)
- output_zip = zipfile.ZipFile(filename, "a",
- compression=zipfile.ZIP_DEFLATED,
- allowZip64=True)
+ common.ZipClose(input_zip)
+ output_zip = zipfile.ZipFile(filename, "a",
+ compression=zipfile.ZIP_DEFLATED,
+ allowZip64=True)
+ else:
+ OPTIONS.info_dict = common.LoadInfoDict(filename, filename)
+ output_zip = None
+ images_dir = os.path.join(OPTIONS.input_tmp, "IMAGES")
+ if not os.path.isdir(images_dir):
+ os.makedirs(images_dir)
+ images_dir = None
has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true")
system_root_image = (OPTIONS.info_dict.get("system_root_image", None) == "true")
@@ -422,7 +422,10 @@
boot_image = common.GetBootableImage(
"IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT")
if boot_image:
- boot_image.AddToZip(output_zip)
+ if output_zip:
+ boot_image.AddToZip(output_zip)
+ else:
+ boot_image.WriteToDir(OPTIONS.input_tmp)
recovery_image = None
if has_recovery:
@@ -438,7 +441,10 @@
recovery_image = common.GetBootableImage(
"IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY")
if recovery_image:
- recovery_image.AddToZip(output_zip)
+ if output_zip:
+ recovery_image.AddToZip(output_zip)
+ else:
+ recovery_image.WriteToDir(OPTIONS.input_tmp)
banner("recovery (two-step image)")
# The special recovery.img for two-step package use.
@@ -446,7 +452,10 @@
"IMAGES/recovery-two-step.img", "recovery-two-step.img",
OPTIONS.input_tmp, "RECOVERY", two_step_image=True)
if recovery_two_step_image:
- recovery_two_step_image.AddToZip(output_zip)
+ if output_zip:
+ recovery_two_step_image.AddToZip(output_zip)
+ else:
+ recovery_two_step_image.WriteToDir(OPTIONS.input_tmp)
banner("system")
system_img_path = AddSystem(
@@ -502,24 +511,39 @@
img_vendor_dir = os.path.join(
OPTIONS.input_tmp, "VENDOR_IMAGES")
if os.path.exists(img_radio_path):
- common.ZipWrite(output_zip, img_radio_path,
- os.path.join("IMAGES", img_name))
+ if output_zip:
+ common.ZipWrite(output_zip, img_radio_path,
+ os.path.join("IMAGES", img_name))
+ else:
+ shutil.copy(img_radio_path, prebuilt_path)
else:
for root, _, files in os.walk(img_vendor_dir):
if img_name in files:
- common.ZipWrite(output_zip, os.path.join(root, img_name),
- os.path.join("IMAGES", img_name))
+ if output_zip:
+ common.ZipWrite(output_zip, os.path.join(root, img_name),
+ os.path.join("IMAGES", img_name))
+ else:
+ shutil.copy(os.path.join(root, img_name), prebuilt_path)
break
- # Zip spec says: All slashes MUST be forward slashes.
- img_path = 'IMAGES/' + img_name
- assert img_path in output_zip.namelist(), "cannot find " + img_name
+ if output_zip:
+ # Zip spec says: All slashes MUST be forward slashes.
+ img_path = 'IMAGES/' + img_name
+ assert img_path in output_zip.namelist(), "cannot find " + img_name
+ else:
+ img_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name)
+ assert os.path.exists(img_path), "cannot find " + img_name
if care_map_list:
file_path = "META/care_map.txt"
- common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list))
+ if output_zip:
+ common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list))
+ else:
+ with open(os.path.join(OPTIONS.input_tmp, file_path), 'w') as fp:
+ fp.write('\n'.join(care_map_list))
- common.ZipClose(output_zip)
+ if output_zip:
+ common.ZipClose(output_zip)
def main(argv):
def option_handler(o, a):
diff --git a/tools/releasetools/blockimgdiff.py b/tools/releasetools/blockimgdiff.py
index c204c90..6c842dc 100644
--- a/tools/releasetools/blockimgdiff.py
+++ b/tools/releasetools/blockimgdiff.py
@@ -24,8 +24,8 @@
import os.path
import re
import subprocess
+import sys
import threading
-import tempfile
from collections import deque, OrderedDict
from hashlib import sha1
@@ -35,69 +35,67 @@
__all__ = ["EmptyImage", "DataImage", "BlockImageDiff"]
-def compute_patch(src, tgt, imgdiff=False):
- srcfd, srcfile = tempfile.mkstemp(prefix="src-")
- tgtfd, tgtfile = tempfile.mkstemp(prefix="tgt-")
- patchfd, patchfile = tempfile.mkstemp(prefix="patch-")
- os.close(patchfd)
+def compute_patch(srcfile, tgtfile, imgdiff=False):
+ patchfile = common.MakeTempFile(prefix="patch-")
- try:
- with os.fdopen(srcfd, "wb") as f_src:
- for p in src:
- f_src.write(p)
+ if imgdiff:
+ p = subprocess.call(
+ ["imgdiff", "-z", srcfile, tgtfile, patchfile],
+ stdout=open(os.devnull, 'w'),
+ stderr=subprocess.STDOUT)
+ else:
+ p = subprocess.call(
+ ["bsdiff", srcfile, tgtfile, patchfile],
+ stdout=open(os.devnull, 'w'),
+ stderr=subprocess.STDOUT)
- with os.fdopen(tgtfd, "wb") as f_tgt:
- for p in tgt:
- f_tgt.write(p)
- try:
- os.unlink(patchfile)
- except OSError:
- pass
- if imgdiff:
- p = subprocess.call(["imgdiff", "-z", srcfile, tgtfile, patchfile],
- stdout=open("/dev/null", "a"),
- stderr=subprocess.STDOUT)
- else:
- p = subprocess.call(["bsdiff", srcfile, tgtfile, patchfile])
+ if p:
+ raise ValueError("diff failed: " + str(p))
- if p:
- raise ValueError("diff failed: " + str(p))
-
- with open(patchfile, "rb") as f:
- return f.read()
- finally:
- try:
- os.unlink(srcfile)
- os.unlink(tgtfile)
- os.unlink(patchfile)
- except OSError:
- pass
+ with open(patchfile, "rb") as f:
+ return f.read()
class Image(object):
+ def RangeSha1(self, ranges):
+ raise NotImplementedError
+
def ReadRangeSet(self, ranges):
raise NotImplementedError
def TotalSha1(self, include_clobbered_blocks=False):
raise NotImplementedError
+ def WriteRangeDataToFd(self, ranges, fd):
+ raise NotImplementedError
+
class EmptyImage(Image):
"""A zero-length image."""
- blocksize = 4096
- care_map = RangeSet()
- clobbered_blocks = RangeSet()
- extended = RangeSet()
- total_blocks = 0
- file_map = {}
+
+ def __init__(self):
+ self.blocksize = 4096
+ self.care_map = RangeSet()
+ self.clobbered_blocks = RangeSet()
+ self.extended = RangeSet()
+ self.total_blocks = 0
+ self.file_map = {}
+
+ def RangeSha1(self, ranges):
+ return sha1().hexdigest()
+
def ReadRangeSet(self, ranges):
return ()
+
def TotalSha1(self, include_clobbered_blocks=False):
# EmptyImage always carries empty clobbered_blocks, so
# include_clobbered_blocks can be ignored.
assert self.clobbered_blocks.size() == 0
return sha1().hexdigest()
+ def WriteRangeDataToFd(self, ranges, fd):
+ raise ValueError("Can't write data from EmptyImage to file")
+
class DataImage(Image):
"""An image wrapped around a single string of data."""
@@ -160,23 +158,39 @@
if clobbered_blocks:
self.file_map["__COPY"] = RangeSet(data=clobbered_blocks)
+ def _GetRangeData(self, ranges):
+ for s, e in ranges:
+ yield self.data[s*self.blocksize:e*self.blocksize]
+
+ def RangeSha1(self, ranges):
+ h = sha1()
+ for data in self._GetRangeData(ranges):
+ h.update(data)
+ return h.hexdigest()
+
def ReadRangeSet(self, ranges):
- return [self.data[s*self.blocksize:e*self.blocksize] for (s, e) in ranges]
+ return [self._GetRangeData(ranges)]
def TotalSha1(self, include_clobbered_blocks=False):
if not include_clobbered_blocks:
- ranges = self.care_map.subtract(self.clobbered_blocks)
- return sha1(self.ReadRangeSet(ranges)).hexdigest()
+ return self.RangeSha1(self.care_map.subtract(self.clobbered_blocks))
else:
return sha1(self.data).hexdigest()
+ def WriteRangeDataToFd(self, ranges, fd):
+ for data in self._GetRangeData(ranges):
+ fd.write(data)
+
class Transfer(object):
- def __init__(self, tgt_name, src_name, tgt_ranges, src_ranges, style, by_id):
+ def __init__(self, tgt_name, src_name, tgt_ranges, src_ranges, tgt_sha1,
+ src_sha1, style, by_id):
self.tgt_name = tgt_name
self.src_name = src_name
self.tgt_ranges = tgt_ranges
self.src_ranges = src_ranges
+ self.tgt_sha1 = tgt_sha1
+ self.src_sha1 = src_sha1
self.style = style
self.intact = (getattr(tgt_ranges, "monotonic", False) and
getattr(src_ranges, "monotonic", False))
@@ -251,6 +265,9 @@
# Implementations are free to break up the data into list/tuple
# elements in any way that is convenient.
#
+# RangeSha1(): a function that returns (as a hex string) the SHA-1
+# hash of all the data in the specified range.
+#
# TotalSha1(): a function that returns (as a hex string) the SHA-1
# hash of all the data in the image (ie, all the blocks in the
# care_map minus clobbered_blocks, or including the clobbered
@@ -332,15 +349,6 @@
self.ComputePatches(prefix)
self.WriteTransfers(prefix)
- def HashBlocks(self, source, ranges): # pylint: disable=no-self-use
- data = source.ReadRangeSet(ranges)
- ctx = sha1()
-
- for p in data:
- ctx.update(p)
-
- return ctx.hexdigest()
-
def WriteTransfers(self, prefix):
def WriteSplitTransfers(out, style, target_blocks):
"""Limit the size of operand in command 'new' and 'zero' to 1024 blocks.
@@ -397,7 +405,7 @@
stashed_blocks += sr.size()
out.append("stash %d %s\n" % (sid, sr.to_string_raw()))
else:
- sh = self.HashBlocks(self.src, sr)
+ sh = self.src.RangeSha1(sr)
if sh in stashes:
stashes[sh] += 1
else:
@@ -429,7 +437,7 @@
mapped_stashes = []
for stash_raw_id, sr in xf.use_stash:
unstashed_src_ranges = unstashed_src_ranges.subtract(sr)
- sh = self.HashBlocks(self.src, sr)
+ sh = self.src.RangeSha1(sr)
sr = xf.src_ranges.map_within(sr)
mapped_stashes.append(sr)
if self.version == 2:
@@ -515,7 +523,7 @@
out.append("%s %s %s %s\n" % (
xf.style,
- self.HashBlocks(self.tgt, xf.tgt_ranges),
+ xf.tgt_sha1,
xf.tgt_ranges.to_string_raw(), src_str))
total += tgt_size
elif xf.style in ("bsdiff", "imgdiff"):
@@ -542,8 +550,8 @@
out.append("%s %d %d %s %s %s %s\n" % (
xf.style,
xf.patch_start, xf.patch_len,
- self.HashBlocks(self.src, xf.src_ranges),
- self.HashBlocks(self.tgt, xf.tgt_ranges),
+ xf.src_sha1,
+ xf.tgt_sha1,
xf.tgt_ranges.to_string_raw(), src_str))
total += tgt_size
elif xf.style == "zero":
@@ -574,8 +582,7 @@
stash_threshold)
if self.version >= 3:
- self.touched_src_sha1 = self.HashBlocks(
- self.src, self.touched_src_ranges)
+ self.touched_src_sha1 = self.src.RangeSha1(self.touched_src_ranges)
# Zero out extended blocks as a workaround for bug 20881595.
if self.tgt.extended:
@@ -674,7 +681,7 @@
if self.version == 2:
stashed_blocks_after += sr.size()
else:
- sh = self.HashBlocks(self.src, sr)
+ sh = self.src.RangeSha1(sr)
if sh not in stashes:
stashed_blocks_after += sr.size()
@@ -731,7 +738,7 @@
stashed_blocks -= sr.size()
heapq.heappush(free_stash_ids, sid)
else:
- sh = self.HashBlocks(self.src, sr)
+ sh = self.src.RangeSha1(sr)
assert sh in stashes
stashes[sh] -= 1
if stashes[sh] == 0:
@@ -745,10 +752,10 @@
def ComputePatches(self, prefix):
print("Reticulating splines...")
- diff_q = []
+ diff_queue = []
patch_num = 0
with open(prefix + ".new.dat", "wb") as new_f:
- for xf in self.transfers:
+ for index, xf in enumerate(self.transfers):
if xf.style == "zero":
tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize
print("%10d %10d (%6.2f%%) %7s %s %s" % (
@@ -756,17 +763,13 @@
str(xf.tgt_ranges)))
elif xf.style == "new":
- for piece in self.tgt.ReadRangeSet(xf.tgt_ranges):
- new_f.write(piece)
+ self.tgt.WriteRangeDataToFd(xf.tgt_ranges, new_f)
tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize
print("%10d %10d (%6.2f%%) %7s %s %s" % (
tgt_size, tgt_size, 100.0, xf.style,
xf.tgt_name, str(xf.tgt_ranges)))
elif xf.style == "diff":
- src = self.src.ReadRangeSet(xf.src_ranges)
- tgt = self.tgt.ReadRangeSet(xf.tgt_ranges)
-
# We can't compare src and tgt directly because they may have
# the same content but be broken up into blocks differently, eg:
#
@@ -775,20 +778,11 @@
# We want those to compare equal, ideally without having to
# actually concatenate the strings (these may be tens of
# megabytes).
-
- src_sha1 = sha1()
- for p in src:
- src_sha1.update(p)
- tgt_sha1 = sha1()
- tgt_size = 0
- for p in tgt:
- tgt_sha1.update(p)
- tgt_size += len(p)
-
- if src_sha1.digest() == tgt_sha1.digest():
+ if xf.src_sha1 == xf.tgt_sha1:
# These are identical; we don't need to generate a patch,
# just issue copy commands on the device.
xf.style = "move"
+ tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize
if xf.src_ranges != xf.tgt_ranges:
print("%10d %10d (%6.2f%%) %7s %s %s (from %s)" % (
tgt_size, tgt_size, 100.0, xf.style,
@@ -815,38 +809,64 @@
xf.tgt_name.split(".")[-1].lower()
in ("apk", "jar", "zip"))
xf.style = "imgdiff" if imgdiff else "bsdiff"
- diff_q.append((tgt_size, src, tgt, xf, patch_num))
+ diff_queue.append((index, imgdiff, patch_num))
patch_num += 1
else:
assert False, "unknown style " + xf.style
- if diff_q:
+ if diff_queue:
if self.threads > 1:
print("Computing patches (using %d threads)..." % (self.threads,))
else:
print("Computing patches...")
- diff_q.sort()
- patches = [None] * patch_num
+ diff_total = len(diff_queue)
+ patches = [None] * diff_total
- # TODO: Rewrite with multiprocessing.ThreadPool?
+ # Using multiprocessing doesn't give additional benefits, due to the
+ # pattern of the code. The diffing work is done by subprocess.call, which
+ # already runs in a separate process (not affected much by the GIL -
+ # Global Interpreter Lock). Using multiprocess also requires either a)
+ # writing the diff input files in the main process before forking, or b)
+ # reopening the image file (SparseImage) in the worker processes. Doing
+ # neither of them further improves the performance.
lock = threading.Lock()
def diff_worker():
while True:
with lock:
- if not diff_q:
+ if not diff_queue:
return
- tgt_size, src, tgt, xf, patchnum = diff_q.pop()
- patch = compute_patch(src, tgt, imgdiff=(xf.style == "imgdiff"))
- size = len(patch)
+ xf_index, imgdiff, patch_index = diff_queue.pop()
+
+ xf = self.transfers[xf_index]
+ src_ranges = xf.src_ranges
+ tgt_ranges = xf.tgt_ranges
+
+ # Needs lock since WriteRangeDataToFd() is stateful (calling seek).
with lock:
- patches[patchnum] = (patch, xf)
- print("%10d %10d (%6.2f%%) %7s %s %s %s" % (
- size, tgt_size, size * 100.0 / tgt_size, xf.style,
- xf.tgt_name if xf.tgt_name == xf.src_name else (
- xf.tgt_name + " (from " + xf.src_name + ")"),
- str(xf.tgt_ranges), str(xf.src_ranges)))
+ src_file = common.MakeTempFile(prefix="src-")
+ with open(src_file, "wb") as fd:
+ self.src.WriteRangeDataToFd(src_ranges, fd)
+
+ tgt_file = common.MakeTempFile(prefix="tgt-")
+ with open(tgt_file, "wb") as fd:
+ self.tgt.WriteRangeDataToFd(tgt_ranges, fd)
+
+ try:
+ patch = compute_patch(src_file, tgt_file, imgdiff)
+ except ValueError as e:
+ raise ValueError(
+ "Failed to generate diff for %s: src=%s, tgt=%s: %s" % (
+ xf.tgt_name, xf.src_ranges, xf.tgt_ranges, e.message))
+
+ with lock:
+ patches[patch_index] = (xf_index, patch)
+ if sys.stdout.isatty():
+ progress = len(patches) * 100 / diff_total
+ # '\033[K' is to clear to EOL.
+ print(' [%d%%] %s\033[K' % (progress, xf.tgt_name), end='\r')
+ sys.stdout.flush()
threads = [threading.Thread(target=diff_worker)
for _ in range(self.threads)]
@@ -854,16 +874,29 @@
th.start()
while threads:
threads.pop().join()
+
+ if sys.stdout.isatty():
+ print('\n')
else:
patches = []
- p = 0
- with open(prefix + ".patch.dat", "wb") as patch_f:
- for patch, xf in patches:
- xf.patch_start = p
+ offset = 0
+ with open(prefix + ".patch.dat", "wb") as patch_fd:
+ for index, patch in patches:
+ xf = self.transfers[index]
xf.patch_len = len(patch)
- patch_f.write(patch)
- p += len(patch)
+ xf.patch_start = offset
+ offset += xf.patch_len
+ patch_fd.write(patch)
+
+ if common.OPTIONS.verbose:
+ tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize
+ print("%10d %10d (%6.2f%%) %7s %s %s %s" % (
+ xf.patch_len, tgt_size, xf.patch_len * 100.0 / tgt_size,
+ xf.style,
+ xf.tgt_name if xf.tgt_name == xf.src_name else (
+ xf.tgt_name + " (from " + xf.src_name + ")"),
+ xf.tgt_ranges, xf.src_ranges))
def AssertSequenceGood(self):
# Simulate the sequences of transfers we will output, and check that:
@@ -1211,7 +1244,9 @@
# Change nothing for small files.
if (tgt_ranges.size() <= max_blocks_per_transfer and
src_ranges.size() <= max_blocks_per_transfer):
- Transfer(tgt_name, src_name, tgt_ranges, src_ranges, style, by_id)
+ Transfer(tgt_name, src_name, tgt_ranges, src_ranges,
+ self.tgt.RangeSha1(tgt_ranges), self.src.RangeSha1(src_ranges),
+ style, by_id)
return
while (tgt_ranges.size() > max_blocks_per_transfer and
@@ -1221,8 +1256,9 @@
tgt_first = tgt_ranges.first(max_blocks_per_transfer)
src_first = src_ranges.first(max_blocks_per_transfer)
- Transfer(tgt_split_name, src_split_name, tgt_first, src_first, style,
- by_id)
+ Transfer(tgt_split_name, src_split_name, tgt_first, src_first,
+ self.tgt.RangeSha1(tgt_first), self.src.RangeSha1(src_first),
+ style, by_id)
tgt_ranges = tgt_ranges.subtract(tgt_first)
src_ranges = src_ranges.subtract(src_first)
@@ -1234,8 +1270,9 @@
assert tgt_ranges.size() and src_ranges.size()
tgt_split_name = "%s-%d" % (tgt_name, pieces)
src_split_name = "%s-%d" % (src_name, pieces)
- Transfer(tgt_split_name, src_split_name, tgt_ranges, src_ranges, style,
- by_id)
+ Transfer(tgt_split_name, src_split_name, tgt_ranges, src_ranges,
+ self.tgt.RangeSha1(tgt_ranges), self.src.RangeSha1(src_ranges),
+ style, by_id)
def AddTransfer(tgt_name, src_name, tgt_ranges, src_ranges, style, by_id,
split=False):
@@ -1244,7 +1281,9 @@
# We specialize diff transfers only (which covers bsdiff/imgdiff/move);
# otherwise add the Transfer() as is.
if style != "diff" or not split:
- Transfer(tgt_name, src_name, tgt_ranges, src_ranges, style, by_id)
+ Transfer(tgt_name, src_name, tgt_ranges, src_ranges,
+ self.tgt.RangeSha1(tgt_ranges), self.src.RangeSha1(src_ranges),
+ style, by_id)
return
# Handle .odex files specially to analyze the block-wise difference. If
diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py
index 2b97518..3224333 100644
--- a/tools/releasetools/common.py
+++ b/tools/releasetools/common.py
@@ -1191,6 +1191,10 @@
t.flush()
return t
+ def WriteToDir(self, d):
+ with open(os.path.join(d, self.name), "wb") as fp:
+ fp.write(self.data)
+
def AddToZip(self, z, compression=None):
ZipWriteStr(z, self.name, self.data, compress_type=compression)
diff --git a/tools/releasetools/sparse_img.py b/tools/releasetools/sparse_img.py
index 4ba7560..7eb60d9 100644
--- a/tools/releasetools/sparse_img.py
+++ b/tools/releasetools/sparse_img.py
@@ -144,6 +144,12 @@
f.seek(16, os.SEEK_SET)
f.write(struct.pack("<2I", self.total_blocks, self.total_chunks))
+ def RangeSha1(self, ranges):
+ h = sha1()
+ for data in self._GetRangeData(ranges):
+ h.update(data)
+ return h.hexdigest()
+
def ReadRangeSet(self, ranges):
return [d for d in self._GetRangeData(ranges)]
@@ -155,10 +161,11 @@
ranges = self.care_map
if not include_clobbered_blocks:
ranges = ranges.subtract(self.clobbered_blocks)
- h = sha1()
- for d in self._GetRangeData(ranges):
- h.update(d)
- return h.hexdigest()
+ return self.RangeSha1(ranges)
+
+ def WriteRangeDataToFd(self, ranges, fd):
+ for data in self._GetRangeData(ranges):
+ fd.write(data)
def _GetRangeData(self, ranges):
"""Generator that produces all the image data in 'ranges'. The