Merge "releasetools: Specify SWITCH_SLOT_ON_REBOOT for secondary payload."
diff --git a/core/soong_config.mk b/core/soong_config.mk
index 639b019..a084f79 100644
--- a/core/soong_config.mk
+++ b/core/soong_config.mk
@@ -133,6 +133,8 @@
$(call add_json_list, NamespacesToExport, $(PRODUCT_SOONG_NAMESPACES))
+$(call add_json_list, PgoAdditionalProfileDirs, $(PGO_ADDITIONAL_PROFILE_DIRS))
+
_contents := $(subst $(comma)$(newline)__SV_END,$(newline)}$(newline),$(_contents)__SV_END)
$(file >$(SOONG_VARIABLES).tmp,$(_contents))
diff --git a/target/product/base.mk b/target/product/base.mk
index 750d3fa..14ff1c2 100644
--- a/target/product/base.mk
+++ b/target/product/base.mk
@@ -31,6 +31,7 @@
bit \
blkid \
bmgr \
+ bpfloader \
bugreport \
bugreportz \
cameraserver \
diff --git a/tools/fs_config/Android.mk b/tools/fs_config/Android.mk
index 3773d38..1247896 100644
--- a/tools/fs_config/Android.mk
+++ b/tools/fs_config/Android.mk
@@ -261,6 +261,7 @@
LOCAL_MODULE := passwd
LOCAL_MODULE_CLASS := ETC
+LOCAL_VENDOR_MODULE := true
include $(BUILD_SYSTEM)/base_rules.mk
@@ -279,6 +280,7 @@
LOCAL_MODULE := group
LOCAL_MODULE_CLASS := ETC
+LOCAL_VENDOR_MODULE := true
include $(BUILD_SYSTEM)/base_rules.mk
diff --git a/tools/releasetools/blockimgdiff.py b/tools/releasetools/blockimgdiff.py
index f1cb6db..d6ff6a6 100644
--- a/tools/releasetools/blockimgdiff.py
+++ b/tools/releasetools/blockimgdiff.py
@@ -15,7 +15,6 @@
from __future__ import print_function
import array
-import common
import copy
import functools
import heapq
@@ -27,9 +26,10 @@
import subprocess
import sys
import threading
-
from collections import deque, OrderedDict
from hashlib import sha1
+
+import common
from rangelib import RangeSet
@@ -256,6 +256,71 @@
return self.score <= other.score
+class ImgdiffStats(object):
+ """A class that collects imgdiff stats.
+
+ It keeps track of the files that will be applied imgdiff while generating
+ BlockImageDiff. It also logs the ones that cannot use imgdiff, with specific
+ reasons. The stats is only meaningful when imgdiff not being disabled by the
+ caller of BlockImageDiff. In addition, only files with supported types
+ (BlockImageDiff.FileTypeSupportedByImgdiff()) are allowed to be logged.
+
+ TODO: The info could be inaccurate due to the unconditional fallback from
+ imgdiff to bsdiff on errors. The fallbacks will be removed.
+ """
+
+ USED_IMGDIFF = "APK files diff'd with imgdiff"
+ USED_IMGDIFF_LARGE_APK = "Large APK files split and diff'd with imgdiff"
+
+ # Reasons for not applying imgdiff on APKs.
+ SKIPPED_TRIMMED = "Not used imgdiff due to trimmed RangeSet"
+ SKIPPED_NONMONOTONIC = "Not used imgdiff due to having non-monotonic ranges"
+
+ # The list of valid reasons, which will also be the dumped order in a report.
+ REASONS = (
+ USED_IMGDIFF,
+ USED_IMGDIFF_LARGE_APK,
+ SKIPPED_TRIMMED,
+ SKIPPED_NONMONOTONIC,
+ )
+
+ def __init__(self):
+ self.stats = {}
+
+ def Log(self, filename, reason):
+ """Logs why imgdiff can or cannot be applied to the given filename.
+
+ Args:
+ filename: The filename string.
+ reason: One of the reason constants listed in REASONS.
+
+ Raises:
+ AssertionError: On unsupported filetypes or invalid reason.
+ """
+ assert BlockImageDiff.FileTypeSupportedByImgdiff(filename)
+ assert reason in self.REASONS
+
+ if reason not in self.stats:
+ self.stats[reason] = set()
+ self.stats[reason].add(filename)
+
+ def Report(self):
+ """Prints a report of the collected imgdiff stats."""
+
+ def print_header(header, separator):
+ print(header)
+ print(separator * len(header) + '\n')
+
+ print_header(' Imgdiff Stats Report ', '=')
+ for key in self.REASONS:
+ if key not in self.stats:
+ continue
+ values = self.stats[key]
+ section_header = ' {} (count: {}) '.format(key, len(values))
+ print_header(section_header, '-')
+ print(''.join([' {}\n'.format(name) for name in values]))
+
+
# BlockImageDiff works on two image objects. An image object is
# anything that provides the following attributes:
#
@@ -311,6 +376,7 @@
self.touched_src_ranges = RangeSet()
self.touched_src_sha1 = None
self.disable_imgdiff = disable_imgdiff
+ self.imgdiff_stats = ImgdiffStats() if not disable_imgdiff else None
assert version in (3, 4)
@@ -337,7 +403,7 @@
"""Returns whether the file type is supported by imgdiff."""
return filename.lower().endswith(('.apk', '.jar', '.zip'))
- def CanUseImgdiff(self, name, tgt_ranges, src_ranges):
+ def CanUseImgdiff(self, name, tgt_ranges, src_ranges, large_apk=False):
"""Checks whether we can apply imgdiff for the given RangeSets.
For files in ZIP format (e.g., APKs, JARs, etc.) we would like to use
@@ -359,17 +425,26 @@
name: The filename to be diff'd.
tgt_ranges: The target RangeSet.
src_ranges: The source RangeSet.
+ large_apk: Whether this is to split a large APK.
Returns:
A boolean result.
"""
- return (
- not self.disable_imgdiff and
- self.FileTypeSupportedByImgdiff(name) and
- tgt_ranges.monotonic and
- src_ranges.monotonic and
- not tgt_ranges.extra.get('trimmed') and
- not src_ranges.extra.get('trimmed'))
+ if self.disable_imgdiff or not self.FileTypeSupportedByImgdiff(name):
+ return False
+
+ if not tgt_ranges.monotonic or not src_ranges.monotonic:
+ self.imgdiff_stats.Log(name, ImgdiffStats.SKIPPED_NONMONOTONIC)
+ return False
+
+ if tgt_ranges.extra.get('trimmed') or src_ranges.extra.get('trimmed'):
+ self.imgdiff_stats.Log(name, ImgdiffStats.SKIPPED_TRIMMED)
+ return False
+
+ reason = (ImgdiffStats.USED_IMGDIFF_LARGE_APK if large_apk
+ else ImgdiffStats.USED_IMGDIFF)
+ self.imgdiff_stats.Log(name, reason)
+ return True
def Compute(self, prefix):
# When looking for a source file to use as the diff input for a
@@ -404,6 +479,10 @@
self.ComputePatches(prefix)
self.WriteTransfers(prefix)
+ # Report the imgdiff stats.
+ if common.OPTIONS.verbose and not self.disable_imgdiff:
+ self.imgdiff_stats.Report()
+
def WriteTransfers(self, prefix):
def WriteSplitTransfers(out, style, target_blocks):
"""Limit the size of operand in command 'new' and 'zero' to 1024 blocks.
@@ -456,7 +535,7 @@
# <# blocks> - <stash refs...>
size = xf.src_ranges.size()
- src_str = [str(size)]
+ src_str_buffer = [str(size)]
unstashed_src_ranges = xf.src_ranges
mapped_stashes = []
@@ -466,7 +545,7 @@
sr = xf.src_ranges.map_within(sr)
mapped_stashes.append(sr)
assert sh in stashes
- src_str.append("%s:%s" % (sh, sr.to_string_raw()))
+ src_str_buffer.append("%s:%s" % (sh, sr.to_string_raw()))
stashes[sh] -= 1
if stashes[sh] == 0:
free_string.append("free %s\n" % (sh,))
@@ -474,17 +553,17 @@
stashes.pop(sh)
if unstashed_src_ranges:
- src_str.insert(1, unstashed_src_ranges.to_string_raw())
+ src_str_buffer.insert(1, unstashed_src_ranges.to_string_raw())
if xf.use_stash:
mapped_unstashed = xf.src_ranges.map_within(unstashed_src_ranges)
- src_str.insert(2, mapped_unstashed.to_string_raw())
+ src_str_buffer.insert(2, mapped_unstashed.to_string_raw())
mapped_stashes.append(mapped_unstashed)
self.AssertPartition(RangeSet(data=(0, size)), mapped_stashes)
else:
- src_str.insert(1, "-")
+ src_str_buffer.insert(1, "-")
self.AssertPartition(RangeSet(data=(0, size)), mapped_stashes)
- src_str = " ".join(src_str)
+ src_str = " ".join(src_str_buffer)
# version 3+:
# zero <rangeset>
@@ -605,11 +684,11 @@
max_allowed = OPTIONS.cache_size * OPTIONS.stash_threshold
print("max stashed blocks: %d (%d bytes), "
"limit: %d bytes (%.2f%%)\n" % (
- max_stashed_blocks, self._max_stashed_size, max_allowed,
- self._max_stashed_size * 100.0 / max_allowed))
+ max_stashed_blocks, self._max_stashed_size, max_allowed,
+ self._max_stashed_size * 100.0 / max_allowed))
else:
print("max stashed blocks: %d (%d bytes), limit: <unknown>\n" % (
- max_stashed_blocks, self._max_stashed_size))
+ max_stashed_blocks, self._max_stashed_size))
def ReviseStashSize(self):
print("Revising stash size...")
@@ -773,9 +852,6 @@
patches = [None] * diff_total
error_messages = []
warning_messages = []
- if sys.stdout.isatty():
- global diff_done
- diff_done = 0
# Using multiprocessing doesn't give additional benefits, due to the
# pattern of the code. The diffing work is done by subprocess.call, which
@@ -791,8 +867,15 @@
if not diff_queue:
return
xf_index, imgdiff, patch_index = diff_queue.pop()
+ xf = self.transfers[xf_index]
- xf = self.transfers[xf_index]
+ if sys.stdout.isatty():
+ diff_left = len(diff_queue)
+ progress = (diff_total - diff_left) * 100 / diff_total
+ # '\033[K' is to clear to EOL.
+ print(' [%3d%%] %s\033[K' % (progress, xf.tgt_name), end='\r')
+ sys.stdout.flush()
+
patch = xf.patch
if not patch:
src_ranges = xf.src_ranges
@@ -812,10 +895,10 @@
except ValueError as e:
message.append(
"Failed to generate %s for %s: tgt=%s, src=%s:\n%s" % (
- "imgdiff" if imgdiff else "bsdiff",
- xf.tgt_name if xf.tgt_name == xf.src_name else
+ "imgdiff" if imgdiff else "bsdiff",
+ xf.tgt_name if xf.tgt_name == xf.src_name else
xf.tgt_name + " (from " + xf.src_name + ")",
- xf.tgt_ranges, xf.src_ranges, e.message))
+ xf.tgt_ranges, xf.src_ranges, e.message))
# TODO(b/68016761): Better handle the holes in mke2fs created
# images.
if imgdiff:
@@ -823,7 +906,7 @@
patch = compute_patch(src_file, tgt_file, imgdiff=False)
message.append(
"Fell back and generated with bsdiff instead for %s" % (
- xf.tgt_name,))
+ xf.tgt_name,))
xf.style = "bsdiff"
with lock:
warning_messages.extend(message)
@@ -831,7 +914,7 @@
except ValueError as e:
message.append(
"Also failed to generate with bsdiff for %s:\n%s" % (
- xf.tgt_name, e.message))
+ xf.tgt_name, e.message))
if message:
with lock:
@@ -839,13 +922,6 @@
with lock:
patches[patch_index] = (xf_index, patch)
- if sys.stdout.isatty():
- global diff_done
- diff_done += 1
- progress = diff_done * 100 / diff_total
- # '\033[K' is to clear to EOL.
- print(' [%d%%] %s\033[K' % (progress, xf.tgt_name), end='\r')
- sys.stdout.flush()
threads = [threading.Thread(target=diff_worker)
for _ in range(self.threads)]
@@ -882,11 +958,11 @@
if common.OPTIONS.verbose:
tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize
print("%10d %10d (%6.2f%%) %7s %s %s %s" % (
- xf.patch_len, tgt_size, xf.patch_len * 100.0 / tgt_size,
- xf.style,
- xf.tgt_name if xf.tgt_name == xf.src_name else (
- xf.tgt_name + " (from " + xf.src_name + ")"),
- xf.tgt_ranges, xf.src_ranges))
+ xf.patch_len, tgt_size, xf.patch_len * 100.0 / tgt_size,
+ xf.style,
+ xf.tgt_name if xf.tgt_name == xf.src_name else (
+ xf.tgt_name + " (from " + xf.src_name + ")"),
+ xf.tgt_ranges, xf.src_ranges))
def AssertSha1Good(self):
"""Check the SHA-1 of the src & tgt blocks in the transfer list.
@@ -1119,7 +1195,8 @@
while sinks:
new_sinks = OrderedDict()
for u in sinks:
- if u not in G: continue
+ if u not in G:
+ continue
s2.appendleft(u)
del G[u]
for iu in u.incoming:
@@ -1132,7 +1209,8 @@
while sources:
new_sources = OrderedDict()
for u in sources:
- if u not in G: continue
+ if u not in G:
+ continue
s1.append(u)
del G[u]
for iu in u.outgoing:
@@ -1141,7 +1219,8 @@
new_sources[iu] = None
sources = new_sources
- if not G: break
+ if not G:
+ break
# Find the "best" vertex to put next. "Best" is the one that
# maximizes the net difference in source blocks saved we get by
@@ -1198,14 +1277,16 @@
intersections = OrderedDict()
for s, e in a.tgt_ranges:
for i in range(s, e):
- if i >= len(source_ranges): break
+ if i >= len(source_ranges):
+ break
# Add all the Transfers in source_ranges[i] to the (ordered) set.
if source_ranges[i] is not None:
for j in source_ranges[i]:
intersections[j] = None
for b in intersections:
- if a is b: continue
+ if a is b:
+ continue
# If the blocks written by A are read by B, then B needs to go before A.
i = a.tgt_ranges.intersect(b.src_ranges)
@@ -1289,7 +1370,7 @@
# calling the costly RangeSha1()s.
if (self.FileTypeSupportedByImgdiff(tgt_name) and
self.tgt.RangeSha1(tgt_ranges) != self.src.RangeSha1(src_ranges)):
- if self.CanUseImgdiff(tgt_name, tgt_ranges, src_ranges):
+ if self.CanUseImgdiff(tgt_name, tgt_ranges, src_ranges, True):
large_apks.append((tgt_name, src_name, tgt_ranges, src_ranges))
return
@@ -1342,8 +1423,9 @@
if tgt_changed < tgt_size * crop_threshold:
assert tgt_changed + tgt_skipped.size() == tgt_size
- print('%10d %10d (%6.2f%%) %s' % (tgt_skipped.size(), tgt_size,
- tgt_skipped.size() * 100.0 / tgt_size, tgt_name))
+ print('%10d %10d (%6.2f%%) %s' % (
+ tgt_skipped.size(), tgt_size,
+ tgt_skipped.size() * 100.0 / tgt_size, tgt_name))
AddSplitTransfers(
"%s-skipped" % (tgt_name,),
"%s-skipped" % (src_name,),
@@ -1481,7 +1563,7 @@
tgt_ranges, src_ranges,
lines)
for index, (patch_start, patch_length, split_tgt_ranges,
- split_src_ranges) in enumerate(split_info_list):
+ split_src_ranges) in enumerate(split_info_list):
with open(patch_file) as f:
f.seek(patch_start)
patch_content = f.read(patch_length)
diff --git a/tools/releasetools/test_blockimgdiff.py b/tools/releasetools/test_blockimgdiff.py
index 4ed8356..a2552d6 100644
--- a/tools/releasetools/test_blockimgdiff.py
+++ b/tools/releasetools/test_blockimgdiff.py
@@ -19,7 +19,8 @@
import unittest
import common
-from blockimgdiff import BlockImageDiff, EmptyImage, HeapItem, Transfer
+from blockimgdiff import (BlockImageDiff, EmptyImage, HeapItem, ImgdiffStats,
+ Transfer)
from rangelib import RangeSet
@@ -196,6 +197,17 @@
self.assertTrue(
block_image_diff.CanUseImgdiff(
"/system/app/app1.apk", RangeSet("10-15"), RangeSet("0-5")))
+ self.assertTrue(
+ block_image_diff.CanUseImgdiff(
+ "/vendor/app/app2.apk", RangeSet("20 25"), RangeSet("30-31"), True))
+
+ self.assertDictEqual(
+ {
+ ImgdiffStats.USED_IMGDIFF : {"/system/app/app1.apk"},
+ ImgdiffStats.USED_IMGDIFF_LARGE_APK : {"/vendor/app/app2.apk"},
+ },
+ block_image_diff.imgdiff_stats.stats)
+
def test_CanUseImgdiff_ineligible(self):
# Disabled by caller.
@@ -223,3 +235,32 @@
self.assertFalse(
block_image_diff.CanUseImgdiff(
"/vendor/app/app3.apk", RangeSet("10-15"), src_ranges))
+
+ # The stats are correctly logged.
+ self.assertDictEqual(
+ {
+ ImgdiffStats.SKIPPED_NONMONOTONIC : {'/system/app/app2.apk'},
+ ImgdiffStats.SKIPPED_TRIMMED : {'/vendor/app/app3.apk'},
+ },
+ block_image_diff.imgdiff_stats.stats)
+
+
+class ImgdiffStatsTest(unittest.TestCase):
+
+ def test_Log(self):
+ imgdiff_stats = ImgdiffStats()
+ imgdiff_stats.Log("/system/app/app2.apk", ImgdiffStats.USED_IMGDIFF)
+ self.assertDictEqual(
+ {
+ ImgdiffStats.USED_IMGDIFF: {'/system/app/app2.apk'},
+ },
+ imgdiff_stats.stats)
+
+ def test_Log_invalidInputs(self):
+ imgdiff_stats = ImgdiffStats()
+
+ self.assertRaises(AssertionError, imgdiff_stats.Log, "/system/bin/gzip",
+ ImgdiffStats.USED_IMGDIFF)
+
+ self.assertRaises(AssertionError, imgdiff_stats.Log, "/system/app/app1.apk",
+ "invalid reason")