Merge "NetlinkEvent - ignore captive portal and pref64 nd user opts"
diff --git a/adb/Android.bp b/adb/Android.bp
index a557090..12d9a14 100644
--- a/adb/Android.bp
+++ b/adb/Android.bp
@@ -317,8 +317,8 @@
static_libs: [
"libadb_crypto",
"libadb_host",
- "libadb_pairing_auth",
- "libadb_pairing_connection",
+ "libadb_pairing_auth",
+ "libadb_pairing_connection",
"libadb_protos",
"libadb_tls_connection",
"libandroidfw",
diff --git a/adb/client/adb_install.cpp b/adb/client/adb_install.cpp
index e4d010c..092a866 100644
--- a/adb/client/adb_install.cpp
+++ b/adb/client/adb_install.cpp
@@ -325,7 +325,12 @@
}
}
- if (first_apk == -1) error_exit("Need at least one APK file on command line");
+ if (first_apk == -1) {
+ if (!silent) {
+ fprintf(stderr, "error: need at least one APK file on command line\n");
+ }
+ return -1;
+ }
auto files = incremental::Files{argv + first_apk, argv + last_apk + 1};
if (silent) {
diff --git a/adb/client/file_sync_client.cpp b/adb/client/file_sync_client.cpp
index 0844428..2ed58b2 100644
--- a/adb/client/file_sync_client.cpp
+++ b/adb/client/file_sync_client.cpp
@@ -42,7 +42,7 @@
#include "adb_client.h"
#include "adb_io.h"
#include "adb_utils.h"
-#include "brotli_utils.h"
+#include "compression_utils.h"
#include "file_sync_protocol.h"
#include "line_printer.h"
#include "sysdeps/errno.h"
@@ -580,8 +580,8 @@
while (true) {
Block output;
- BrotliEncodeResult result = encoder.Encode(&output);
- if (result == BrotliEncodeResult::Error) {
+ EncodeResult result = encoder.Encode(&output);
+ if (result == EncodeResult::Error) {
Error("compressing '%s' locally failed", lpath.c_str());
return false;
}
@@ -592,12 +592,12 @@
WriteOrDie(lpath, rpath, &sbuf, sizeof(SyncRequest) + output.size());
}
- if (result == BrotliEncodeResult::Done) {
+ if (result == EncodeResult::Done) {
sending = false;
break;
- } else if (result == BrotliEncodeResult::NeedInput) {
+ } else if (result == EncodeResult::NeedInput) {
break;
- } else if (result == BrotliEncodeResult::MoreOutput) {
+ } else if (result == EncodeResult::MoreOutput) {
continue;
}
}
@@ -1076,9 +1076,9 @@
while (true) {
std::span<char> output;
- BrotliDecodeResult result = decoder.Decode(&output);
+ DecodeResult result = decoder.Decode(&output);
- if (result == BrotliDecodeResult::Error) {
+ if (result == DecodeResult::Error) {
sc.Error("decompress failed");
adb_unlink(lpath);
return false;
@@ -1097,15 +1097,15 @@
sc.RecordBytesTransferred(msg.data.size);
sc.ReportProgress(name != nullptr ? name : rpath, bytes_copied, expected_size);
- if (result == BrotliDecodeResult::NeedInput) {
+ if (result == DecodeResult::NeedInput) {
break;
- } else if (result == BrotliDecodeResult::MoreOutput) {
+ } else if (result == DecodeResult::MoreOutput) {
continue;
- } else if (result == BrotliDecodeResult::Done) {
+ } else if (result == DecodeResult::Done) {
reading = false;
break;
} else {
- LOG(FATAL) << "invalid BrotliDecodeResult: " << static_cast<int>(result);
+ LOG(FATAL) << "invalid DecodeResult: " << static_cast<int>(result);
}
}
}
diff --git a/adb/brotli_utils.h b/adb/compression_utils.h
similarity index 88%
rename from adb/brotli_utils.h
rename to adb/compression_utils.h
index c5be73d..c445095 100644
--- a/adb/brotli_utils.h
+++ b/adb/compression_utils.h
@@ -23,7 +23,14 @@
#include "types.h"
-enum class BrotliDecodeResult {
+enum class DecodeResult {
+ Error,
+ Done,
+ NeedInput,
+ MoreOutput,
+};
+
+enum class EncodeResult {
Error,
Done,
NeedInput,
@@ -38,7 +45,7 @@
void Append(Block&& block) { input_buffer_.append(std::move(block)); }
- BrotliDecodeResult Decode(std::span<char>* output) {
+ DecodeResult Decode(std::span<char>* output) {
size_t available_in = input_buffer_.front_size();
const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());
@@ -56,16 +63,16 @@
switch (r) {
case BROTLI_DECODER_RESULT_SUCCESS:
- return BrotliDecodeResult::Done;
+ return DecodeResult::Done;
case BROTLI_DECODER_RESULT_ERROR:
- return BrotliDecodeResult::Error;
+ return DecodeResult::Error;
case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
// Brotli guarantees as one of its invariants that if it returns NEEDS_MORE_INPUT,
// it will consume the entire input buffer passed in, so we don't have to worry
// about bytes left over in the front block with more input remaining.
- return BrotliDecodeResult::NeedInput;
+ return DecodeResult::NeedInput;
case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
- return BrotliDecodeResult::MoreOutput;
+ return DecodeResult::MoreOutput;
}
}
@@ -75,13 +82,6 @@
std::unique_ptr<BrotliDecoderState, void (*)(BrotliDecoderState*)> decoder_;
};
-enum class BrotliEncodeResult {
- Error,
- Done,
- NeedInput,
- MoreOutput,
-};
-
template <size_t OutputBlockSize>
struct BrotliEncoder {
explicit BrotliEncoder()
@@ -95,7 +95,7 @@
void Append(Block input) { input_buffer_.append(std::move(input)); }
void Finish() { finished_ = true; }
- BrotliEncodeResult Encode(Block* output) {
+ EncodeResult Encode(Block* output) {
output->clear();
while (true) {
size_t available_in = input_buffer_.front_size();
@@ -112,7 +112,7 @@
if (!BrotliEncoderCompressStream(encoder_.get(), op, &available_in, &next_in,
&available_out, &next_out, nullptr)) {
- return BrotliEncodeResult::Error;
+ return EncodeResult::Error;
}
size_t bytes_consumed = input_buffer_.front_size() - available_in;
@@ -123,14 +123,14 @@
if (BrotliEncoderIsFinished(encoder_.get())) {
output_block_.resize(OutputBlockSize - output_bytes_left_);
*output = std::move(output_block_);
- return BrotliEncodeResult::Done;
+ return EncodeResult::Done;
} else if (output_bytes_left_ == 0) {
*output = std::move(output_block_);
output_block_.resize(OutputBlockSize);
output_bytes_left_ = OutputBlockSize;
- return BrotliEncodeResult::MoreOutput;
+ return EncodeResult::MoreOutput;
} else if (input_buffer_.empty()) {
- return BrotliEncodeResult::NeedInput;
+ return EncodeResult::NeedInput;
}
}
}
diff --git a/adb/daemon/file_sync_service.cpp b/adb/daemon/file_sync_service.cpp
index 07f6e65..5ccddea 100644
--- a/adb/daemon/file_sync_service.cpp
+++ b/adb/daemon/file_sync_service.cpp
@@ -57,7 +57,7 @@
#include "adb_io.h"
#include "adb_trace.h"
#include "adb_utils.h"
-#include "brotli_utils.h"
+#include "compression_utils.h"
#include "file_sync_protocol.h"
#include "security_log_tags.h"
#include "sysdeps/errno.h"
@@ -288,8 +288,8 @@
while (true) {
std::span<char> output;
- BrotliDecodeResult result = decoder.Decode(&output);
- if (result == BrotliDecodeResult::Error) {
+ DecodeResult result = decoder.Decode(&output);
+ if (result == DecodeResult::Error) {
SendSyncFailErrno(s, "decompress failed");
return false;
}
@@ -299,14 +299,14 @@
return false;
}
- if (result == BrotliDecodeResult::NeedInput) {
+ if (result == DecodeResult::NeedInput) {
break;
- } else if (result == BrotliDecodeResult::MoreOutput) {
+ } else if (result == DecodeResult::MoreOutput) {
continue;
- } else if (result == BrotliDecodeResult::Done) {
+ } else if (result == DecodeResult::Done) {
break;
} else {
- LOG(FATAL) << "invalid BrotliDecodeResult: " << static_cast<int>(result);
+ LOG(FATAL) << "invalid DecodeResult: " << static_cast<int>(result);
}
}
}
@@ -591,7 +591,6 @@
static bool recv_uncompressed(borrowed_fd s, unique_fd fd, std::vector<char>& buffer) {
syncmsg msg;
msg.data.id = ID_DATA;
- std::optional<BrotliEncoder<SYNC_DATA_MAX>> encoder;
while (true) {
int r = adb_read(fd.get(), &buffer[0], buffer.size() - sizeof(msg.data));
if (r <= 0) {
@@ -633,8 +632,8 @@
while (true) {
Block output;
- BrotliEncodeResult result = encoder.Encode(&output);
- if (result == BrotliEncodeResult::Error) {
+ EncodeResult result = encoder.Encode(&output);
+ if (result == EncodeResult::Error) {
SendSyncFailErrno(s, "compress failed");
return false;
}
@@ -647,12 +646,12 @@
}
}
- if (result == BrotliEncodeResult::Done) {
+ if (result == EncodeResult::Done) {
sending = false;
break;
- } else if (result == BrotliEncodeResult::NeedInput) {
+ } else if (result == EncodeResult::NeedInput) {
break;
- } else if (result == BrotliEncodeResult::MoreOutput) {
+ } else if (result == EncodeResult::MoreOutput) {
continue;
}
}
diff --git a/fastboot/device/flashing.cpp b/fastboot/device/flashing.cpp
index 7e7e507..fd6ff8e 100644
--- a/fastboot/device/flashing.cpp
+++ b/fastboot/device/flashing.cpp
@@ -135,7 +135,9 @@
return -EOVERFLOW;
}
WipeOverlayfsForPartition(device, partition_name);
- return FlashBlockDevice(handle.fd(), data);
+ int result = FlashBlockDevice(handle.fd(), data);
+ sync();
+ return result;
}
bool UpdateSuper(FastbootDevice* device, const std::string& super_name, bool wipe) {
@@ -165,6 +167,7 @@
return device->WriteFail("Unable to flash new partition table");
}
fs_mgr_overlayfs_teardown();
+ sync();
return device->WriteOkay("Successfully flashed partition table");
}
@@ -204,5 +207,6 @@
return device->WriteFail("Unable to write new partition table");
}
fs_mgr_overlayfs_teardown();
+ sync();
return device->WriteOkay("Successfully updated partition table");
}
diff --git a/init/property_service.cpp b/init/property_service.cpp
index 1e4e127..655b8de 100644
--- a/init/property_service.cpp
+++ b/init/property_service.cpp
@@ -489,7 +489,9 @@
}
LOG(INFO) << "Received sys.powerctl='" << value << "' from pid: " << cr.pid
<< process_log_string;
- DebugRebootLogging();
+ if (!value.empty()) {
+ DebugRebootLogging();
+ }
}
// If a process other than init is writing a non-empty value, it means that process is
diff --git a/init/reboot.cpp b/init/reboot.cpp
index 081f695..d2dc6d3 100644
--- a/init/reboot.cpp
+++ b/init/reboot.cpp
@@ -102,7 +102,15 @@
if (write_to_property) {
SetProperty(LAST_REBOOT_REASON_PROPERTY, reason);
}
- WriteStringToFile(reason, LAST_REBOOT_REASON_FILE);
+ auto fd = unique_fd(TEMP_FAILURE_RETRY(open(
+ LAST_REBOOT_REASON_FILE, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_BINARY, 0666)));
+ if (!fd.ok()) {
+ PLOG(ERROR) << "Could not open '" << LAST_REBOOT_REASON_FILE
+ << "' to persist reboot reason";
+ return;
+ }
+ WriteStringToFd(reason, fd);
+ fsync(fd.get());
}
// represents umount status during reboot / shutdown.
@@ -317,9 +325,9 @@
bool* reboot_monitor_run) {
unsigned int remaining_shutdown_time = 0;
- // 30 seconds more than the timeout passed to the thread as there is a final Umount pass
+ // 300 seconds more than the timeout passed to the thread as there is a final Umount pass
// after the timeout is reached.
- constexpr unsigned int shutdown_watchdog_timeout_default = 30;
+ constexpr unsigned int shutdown_watchdog_timeout_default = 300;
auto shutdown_watchdog_timeout = android::base::GetUintProperty(
"ro.build.shutdown.watchdog.timeout", shutdown_watchdog_timeout_default);
remaining_shutdown_time = shutdown_watchdog_timeout + shutdown_timeout.count() / 1000;
@@ -539,26 +547,6 @@
Timer t;
LOG(INFO) << "Reboot start, reason: " << reason << ", reboot_target: " << reboot_target;
- // Ensure last reboot reason is reduced to canonical
- // alias reported in bootloader or system boot reason.
- size_t skip = 0;
- std::vector<std::string> reasons = Split(reason, ",");
- if (reasons.size() >= 2 && reasons[0] == "reboot" &&
- (reasons[1] == "recovery" || reasons[1] == "bootloader" || reasons[1] == "cold" ||
- reasons[1] == "hard" || reasons[1] == "warm")) {
- skip = strlen("reboot,");
- }
- PersistRebootReason(reason.c_str() + skip, true);
- sync();
-
- // If /data isn't mounted then we can skip the extra reboot steps below, since we don't need to
- // worry about unmounting it.
- if (!IsDataMounted()) {
- sync();
- RebootSystem(cmd, reboot_target);
- abort();
- }
-
bool is_thermal_shutdown = cmd == ANDROID_RB_THERMOFF;
auto shutdown_timeout = 0ms;
@@ -591,6 +579,25 @@
// Start reboot monitor thread
sem_post(&reboot_semaphore);
+ // Ensure last reboot reason is reduced to canonical
+ // alias reported in bootloader or system boot reason.
+ size_t skip = 0;
+ std::vector<std::string> reasons = Split(reason, ",");
+ if (reasons.size() >= 2 && reasons[0] == "reboot" &&
+ (reasons[1] == "recovery" || reasons[1] == "bootloader" || reasons[1] == "cold" ||
+ reasons[1] == "hard" || reasons[1] == "warm")) {
+ skip = strlen("reboot,");
+ }
+ PersistRebootReason(reason.c_str() + skip, true);
+
+ // If /data isn't mounted then we can skip the extra reboot steps below, since we don't need to
+ // worry about unmounting it.
+ if (!IsDataMounted()) {
+ sync();
+ RebootSystem(cmd, reboot_target);
+ abort();
+ }
+
// watchdogd is a vendor specific component but should be alive to complete shutdown safely.
const std::set<std::string> to_starts{"watchdogd"};
std::vector<Service*> stop_first;
diff --git a/init/sysprop/InitProperties.sysprop b/init/sysprop/InitProperties.sysprop
index b876dc0..24c2434 100644
--- a/init/sysprop/InitProperties.sysprop
+++ b/init/sysprop/InitProperties.sysprop
@@ -31,6 +31,6 @@
type: Boolean
scope: Public
access: Readonly
- prop_name: "ro.init.userspace_reboot.is_supported"
+ prop_name: "init.userspace_reboot.is_supported"
integer_as_bool: true
}
diff --git a/init/sysprop/api/com.android.sysprop.init-current.txt b/init/sysprop/api/com.android.sysprop.init-current.txt
index b8bcef9..01f4e9a 100644
--- a/init/sysprop/api/com.android.sysprop.init-current.txt
+++ b/init/sysprop/api/com.android.sysprop.init-current.txt
@@ -2,7 +2,7 @@
module: "android.sysprop.InitProperties"
prop {
api_name: "is_userspace_reboot_supported"
- prop_name: "ro.init.userspace_reboot.is_supported"
+ prop_name: "init.userspace_reboot.is_supported"
integer_as_bool: true
}
prop {
diff --git a/libunwindstack/ElfInterface.cpp b/libunwindstack/ElfInterface.cpp
index 341275d..821e042 100644
--- a/libunwindstack/ElfInterface.cpp
+++ b/libunwindstack/ElfInterface.cpp
@@ -662,7 +662,7 @@
if (note_size - offset < hdr.n_descsz || hdr.n_descsz == 0) {
return "";
}
- std::string build_id(hdr.n_descsz - 1, '\0');
+ std::string build_id(hdr.n_descsz, '\0');
if (memory->ReadFully(note_offset + offset, &build_id[0], hdr.n_descsz)) {
return build_id;
}
diff --git a/libunwindstack/tests/MapInfoGetBuildIDTest.cpp b/libunwindstack/tests/MapInfoGetBuildIDTest.cpp
index 6953e26..70e136b 100644
--- a/libunwindstack/tests/MapInfoGetBuildIDTest.cpp
+++ b/libunwindstack/tests/MapInfoGetBuildIDTest.cpp
@@ -142,15 +142,14 @@
char note_section[128];
Elf32_Nhdr note_header = {};
- note_header.n_namesz = 4; // "GNU"
- note_header.n_descsz = 12; // "ELF_BUILDID"
+ note_header.n_namesz = sizeof("GNU");
+ note_header.n_descsz = sizeof("ELF_BUILDID") - 1;
note_header.n_type = NT_GNU_BUILD_ID;
memcpy(¬e_section, ¬e_header, sizeof(note_header));
size_t note_offset = sizeof(note_header);
- memcpy(¬e_section[note_offset], "GNU", sizeof("GNU"));
- note_offset += sizeof("GNU");
- memcpy(¬e_section[note_offset], "ELF_BUILDID", sizeof("ELF_BUILDID"));
- note_offset += sizeof("ELF_BUILDID");
+ memcpy(¬e_section[note_offset], "GNU", note_header.n_namesz);
+ note_offset += note_header.n_namesz;
+ memcpy(¬e_section[note_offset], "ELF_BUILDID", note_header.n_descsz);
Elf32_Shdr shdr = {};
shdr.sh_type = SHT_NOTE;
@@ -195,4 +194,10 @@
MultipleThreadTest("ELF_BUILDID");
}
+TEST_F(MapInfoGetBuildIDTest, real_elf) {
+ MapInfo map_info(nullptr, nullptr, 0x1000, 0x20000, 0, PROT_READ | PROT_WRITE,
+ TestGetFileDirectory() + "offline/empty_arm64/libc.so");
+ EXPECT_EQ("6df0590c4920f4c7b9f34fe833f37d54", map_info.GetPrintableBuildID());
+}
+
} // namespace unwindstack
diff --git a/libziparchive/include/ziparchive/zip_archive.h b/libziparchive/include/ziparchive/zip_archive.h
index 435bfb6..4697bb7 100644
--- a/libziparchive/include/ziparchive/zip_archive.h
+++ b/libziparchive/include/ziparchive/zip_archive.h
@@ -77,6 +77,10 @@
// footer.
uint32_t uncompressed_length;
+ // If the value of uncompressed length and compressed length are stored in
+ // the zip64 extended info of the extra field.
+ bool zip64_format_size{false};
+
// The offset to the start of data for this ZipEntry.
off64_t offset;
diff --git a/libziparchive/test_ziparchive_large.py b/libziparchive/test_ziparchive_large.py
index c29c37e..6b82f63 100644
--- a/libziparchive/test_ziparchive_large.py
+++ b/libziparchive/test_ziparchive_large.py
@@ -26,13 +26,17 @@
class Zip64Test(unittest.TestCase):
@staticmethod
+ def _WriteFile(path, size_in_kib):
+ contents = os.path.basename(path)[0] * 1024
+ with open(path, 'w') as f:
+ for it in range(0, size_in_kib):
+ f.write(contents)
+
+ @staticmethod
def _AddEntriesToZip(output_zip, entries_dict=None):
for name, size in entries_dict.items():
- contents = name[0] * 1024
file_path = tempfile.NamedTemporaryFile()
- with open(file_path.name, 'w') as f:
- for it in range(0, size):
- f.write(contents)
+ Zip64Test._WriteFile(file_path.name, size)
output_zip.write(file_path.name, arcname = name)
def _getEntryNames(self, zip_name):
@@ -93,6 +97,22 @@
self._ExtractEntries(zip_path.name)
+ def test_forceDataDescriptor(self):
+ file_path = tempfile.NamedTemporaryFile(suffix='.txt')
+ # TODO create the entry > 4GiB.
+ self._WriteFile(file_path.name, 1024)
+
+ zip_path = tempfile.NamedTemporaryFile(suffix='.zip')
+ with zipfile.ZipFile(zip_path, 'w', allowZip64=True) as output_zip:
+ pass
+ # The fd option force writes a data descriptor
+ cmd = ['zip', '-fd', zip_path.name, file_path.name]
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ proc.communicate()
+ read_names = self._getEntryNames(zip_path.name)
+ self.assertEquals([file_path.name[1:]], read_names)
+ self._ExtractEntries(zip_path.name)
+
if __name__ == '__main__':
testsuite = unittest.TestLoader().discover(
os.path.dirname(os.path.realpath(__file__)))
diff --git a/libziparchive/testdata/zip64.zip b/libziparchive/testdata/zip64.zip
new file mode 100644
index 0000000..3f25a4c
--- /dev/null
+++ b/libziparchive/testdata/zip64.zip
Binary files differ
diff --git a/libziparchive/zip_archive.cc b/libziparchive/zip_archive.cc
index 849b68c..031d43a 100644
--- a/libziparchive/zip_archive.cc
+++ b/libziparchive/zip_archive.cc
@@ -57,8 +57,6 @@
#include "zip_archive_common.h"
#include "zip_archive_private.h"
-using android::base::get_unaligned;
-
// Used to turn on crc checks - verify that the content CRC matches the values
// specified in the local file header and the central directory.
static const bool kCrcChecksEnabled = false;
@@ -221,7 +219,7 @@
for (; i >= 0; i--) {
if (scan_buffer[i] == 0x50) {
uint32_t* sig_addr = reinterpret_cast<uint32_t*>(&scan_buffer[i]);
- if (get_unaligned<uint32_t>(sig_addr) == EocdRecord::kSignature) {
+ if (android::base::get_unaligned<uint32_t>(sig_addr) == EocdRecord::kSignature) {
ALOGV("+++ Found EOCD at buf+%d", i);
break;
}
@@ -360,8 +358,9 @@
// Data Size - 2 bytes
uint16_t offset = 0;
while (offset < extraFieldLength - 4) {
- auto headerId = get_unaligned<uint16_t>(extraFieldStart + offset);
- auto dataSize = get_unaligned<uint16_t>(extraFieldStart + offset + 2);
+ auto readPtr = const_cast<uint8_t*>(extraFieldStart + offset);
+ auto headerId = ConsumeUnaligned<uint16_t>(&readPtr);
+ auto dataSize = ConsumeUnaligned<uint16_t>(&readPtr);
offset += 4;
if (dataSize > extraFieldLength - offset) {
@@ -376,54 +375,44 @@
continue;
}
- uint16_t expectedDataSize = 0;
- // We expect the extended field to include both uncompressed and compressed size.
- if (zip32UncompressedSize == UINT32_MAX || zip32CompressedSize == UINT32_MAX) {
- expectedDataSize += 16;
+ std::optional<uint64_t> uncompressedFileSize;
+ std::optional<uint64_t> compressedFileSize;
+ std::optional<uint64_t> localHeaderOffset;
+ if (zip32UncompressedSize == UINT32_MAX) {
+ uncompressedFileSize = ConsumeUnaligned<uint64_t>(&readPtr);
+ }
+ if (zip32CompressedSize == UINT32_MAX) {
+ compressedFileSize = ConsumeUnaligned<uint64_t>(&readPtr);
}
if (zip32LocalFileHeaderOffset == UINT32_MAX) {
- expectedDataSize += 8;
+ localHeaderOffset = ConsumeUnaligned<uint64_t>(&readPtr);
}
- if (expectedDataSize == 0) {
+ // calculate how many bytes we read after the data size field.
+ size_t bytesRead = readPtr - (extraFieldStart + offset);
+ if (bytesRead == 0) {
ALOGW("Zip: Data size should not be 0 in zip64 extended field");
return kInvalidFile;
}
- if (dataSize != expectedDataSize) {
+ if (dataSize != bytesRead) {
auto localOffsetString = zip32LocalFileHeaderOffset.has_value()
? std::to_string(zip32LocalFileHeaderOffset.value())
: "missing";
- ALOGW("Zip: Invalid data size in zip64 extended field, expect %" PRIu16 ", get %" PRIu16
+ ALOGW("Zip: Invalid data size in zip64 extended field, expect %zu , get %" PRIu16
", uncompressed size %" PRIu32 ", compressed size %" PRIu32 ", local header offset %s",
- expectedDataSize, dataSize, zip32UncompressedSize, zip32CompressedSize,
+ bytesRead, dataSize, zip32UncompressedSize, zip32CompressedSize,
localOffsetString.c_str());
return kInvalidFile;
}
- std::optional<uint64_t> uncompressedFileSize;
- std::optional<uint64_t> compressedFileSize;
- std::optional<uint64_t> localHeaderOffset;
- if (zip32UncompressedSize == UINT32_MAX || zip32CompressedSize == UINT32_MAX) {
- uncompressedFileSize = get_unaligned<uint64_t>(extraFieldStart + offset);
- compressedFileSize = get_unaligned<uint64_t>(extraFieldStart + offset + 8);
- offset += 16;
-
- // TODO(xunchang) Support handling file large than UINT32_MAX. It's theoretically possible
- // for libz to (de)compressing file larger than UINT32_MAX. But we should use our own
- // bytes counter to replace stream.total_out.
- if (uncompressedFileSize.value() >= UINT32_MAX || compressedFileSize.value() >= UINT32_MAX) {
- ALOGW(
- "Zip: File size larger than UINT32_MAX isn't supported yet. uncompressed size %" PRIu64
- ", compressed size %" PRIu64,
- uncompressedFileSize.value(), compressedFileSize.value());
- return kInvalidFile;
- }
- }
-
- if (zip32LocalFileHeaderOffset == UINT32_MAX) {
- localHeaderOffset = get_unaligned<uint64_t>(extraFieldStart + offset);
- offset += 8;
+ // TODO(xunchang) Support handling file large than UINT32_MAX. It's theoretically possible
+ // for libz to (de)compressing file larger than UINT32_MAX. But we should use our own
+ // bytes counter to replace stream.total_out.
+ if ((uncompressedFileSize.has_value() && uncompressedFileSize.value() > UINT32_MAX) ||
+ (compressedFileSize.has_value() && compressedFileSize.value() > UINT32_MAX)) {
+ ALOGW("Zip: File size larger than UINT32_MAX isn't supported yet");
+ return kInvalidFile;
}
zip64Info->uncompressed_file_size = uncompressedFileSize;
@@ -625,7 +614,8 @@
}
static int32_t ValidateDataDescriptor(MappedZipFile& mapped_zip, ZipEntry* entry) {
- uint8_t ddBuf[sizeof(DataDescriptor) + sizeof(DataDescriptor::kOptSignature)];
+ // Maximum possible size for data descriptor: 2 * 4 + 2 * 8 = 24 bytes
+ uint8_t ddBuf[24];
off64_t offset = entry->offset;
if (entry->method != kCompressStored) {
offset += entry->compressed_length;
@@ -638,18 +628,26 @@
}
const uint32_t ddSignature = *(reinterpret_cast<const uint32_t*>(ddBuf));
- const uint16_t ddOffset = (ddSignature == DataDescriptor::kOptSignature) ? 4 : 0;
- const DataDescriptor* descriptor = reinterpret_cast<const DataDescriptor*>(ddBuf + ddOffset);
+ uint8_t* ddReadPtr = (ddSignature == DataDescriptor::kOptSignature) ? ddBuf + 4 : ddBuf;
+ DataDescriptor descriptor{};
+ descriptor.crc32 = ConsumeUnaligned<uint32_t>(&ddReadPtr);
+ if (entry->zip64_format_size) {
+ descriptor.compressed_size = ConsumeUnaligned<uint64_t>(&ddReadPtr);
+ descriptor.uncompressed_size = ConsumeUnaligned<uint64_t>(&ddReadPtr);
+ } else {
+ descriptor.compressed_size = ConsumeUnaligned<uint32_t>(&ddReadPtr);
+ descriptor.uncompressed_size = ConsumeUnaligned<uint32_t>(&ddReadPtr);
+ }
// Validate that the values in the data descriptor match those in the central
// directory.
- if (entry->compressed_length != descriptor->compressed_size ||
- entry->uncompressed_length != descriptor->uncompressed_size ||
- entry->crc32 != descriptor->crc32) {
+ if (entry->compressed_length != descriptor.compressed_size ||
+ entry->uncompressed_length != descriptor.uncompressed_size ||
+ entry->crc32 != descriptor.crc32) {
ALOGW("Zip: size/crc32 mismatch. expected {%" PRIu32 ", %" PRIu32 ", %" PRIx32
- "}, was {%" PRIu32 ", %" PRIu32 ", %" PRIx32 "}",
+ "}, was {%" PRIu64 ", %" PRIu64 ", %" PRIx32 "}",
entry->compressed_length, entry->uncompressed_length, entry->crc32,
- descriptor->compressed_size, descriptor->uncompressed_size, descriptor->crc32);
+ descriptor.compressed_size, descriptor.uncompressed_size, descriptor.crc32);
return kInconsistentInformation;
}
@@ -706,18 +704,14 @@
return status;
}
- if (cdr->uncompressed_size == UINT32_MAX || cdr->compressed_size == UINT32_MAX) {
- CHECK(zip64_info.uncompressed_file_size.has_value());
- CHECK(zip64_info.compressed_file_size.has_value());
- // TODO(xunchang) remove the size limit and support entry length > UINT32_MAX.
- data->uncompressed_length = static_cast<uint32_t>(zip64_info.uncompressed_file_size.value());
- data->compressed_length = static_cast<uint32_t>(zip64_info.compressed_file_size.value());
- }
-
- if (local_header_offset == UINT32_MAX) {
- CHECK(zip64_info.local_header_offset.has_value());
- local_header_offset = zip64_info.local_header_offset.value();
- }
+ // TODO(xunchang) remove the size limit and support entry length > UINT32_MAX.
+ data->uncompressed_length =
+ static_cast<uint32_t>(zip64_info.uncompressed_file_size.value_or(cdr->uncompressed_size));
+ data->compressed_length =
+ static_cast<uint32_t>(zip64_info.compressed_file_size.value_or(cdr->compressed_size));
+ local_header_offset = zip64_info.local_header_offset.value_or(local_header_offset);
+ data->zip64_format_size =
+ cdr->uncompressed_size == UINT32_MAX || cdr->compressed_size == UINT32_MAX;
}
if (local_header_offset + static_cast<off64_t>(sizeof(LocalFileHeader)) >= cd_offset) {
@@ -766,6 +760,13 @@
uint64_t lfh_uncompressed_size = lfh->uncompressed_size;
uint64_t lfh_compressed_size = lfh->compressed_size;
if (lfh_uncompressed_size == UINT32_MAX || lfh_compressed_size == UINT32_MAX) {
+ if (lfh_uncompressed_size != UINT32_MAX || lfh_compressed_size != UINT32_MAX) {
+ ALOGW(
+ "Zip: The zip64 extended field in the local header MUST include BOTH original and "
+ "compressed file size fields.");
+ return kInvalidFile;
+ }
+
const off64_t lfh_extra_field_offset = name_offset + lfh->file_name_length;
const uint16_t lfh_extra_field_size = lfh->extra_field_length;
if (lfh_extra_field_offset > cd_offset - lfh_extra_field_size) {
diff --git a/libziparchive/zip_archive_common.h b/libziparchive/zip_archive_common.h
index a92d4d2..d461856 100644
--- a/libziparchive/zip_archive_common.h
+++ b/libziparchive/zip_archive_common.h
@@ -165,15 +165,24 @@
// CRC-32 checksum of the entry.
uint32_t crc32;
- // Compressed size of the entry.
- uint32_t compressed_size;
- // Uncompressed size of the entry.
- uint32_t uncompressed_size;
+
+ // For ZIP64 format archives, the compressed and uncompressed sizes are 8
+ // bytes each. Also, the ZIP64 format MAY be used regardless of the size
+ // of a file. When extracting, if the zip64 extended information extra field
+ // is present for the file the compressed and uncompressed sizes will be 8
+ // byte values.
+
+ // Compressed size of the entry, the field can be either 4 bytes or 8 bytes
+ // in the zip file.
+ uint64_t compressed_size;
+ // Uncompressed size of the entry, the field can be either 4 bytes or 8 bytes
+ // in the zip file.
+ uint64_t uncompressed_size;
private:
DataDescriptor() = default;
DISALLOW_COPY_AND_ASSIGN(DataDescriptor);
-} __attribute__((packed));
+};
// The zip64 end of central directory locator helps to find the zip64 EOCD.
struct Zip64EocdLocator {
diff --git a/libziparchive/zip_archive_private.h b/libziparchive/zip_archive_private.h
index 5f5232f..4ed07aa 100644
--- a/libziparchive/zip_archive_private.h
+++ b/libziparchive/zip_archive_private.h
@@ -28,6 +28,7 @@
#include "android-base/macros.h"
#include "android-base/mapped_file.h"
+#include "android-base/memory.h"
#include "zip_cd_entry_map.h"
#include "zip_error.h"
@@ -104,3 +105,20 @@
bool InitializeCentralDirectory(off64_t cd_start_offset, size_t cd_size);
};
+
+int32_t ExtractToWriter(ZipArchiveHandle handle, ZipEntry* entry, zip_archive::Writer* writer);
+
+// Reads the unaligned data of type |T| and auto increment the offset.
+template <typename T>
+static T ConsumeUnaligned(uint8_t** address) {
+ auto ret = android::base::get_unaligned<T>(*address);
+ *address += sizeof(T);
+ return ret;
+}
+
+// Writes the unaligned data of type |T| and auto increment the offset.
+template <typename T>
+void EmitUnaligned(uint8_t** address, T data) {
+ android::base::put_unaligned<T>(*address, data);
+ *address += sizeof(T);
+}
diff --git a/libziparchive/zip_archive_test.cc b/libziparchive/zip_archive_test.cc
index 69be3df..3563340 100644
--- a/libziparchive/zip_archive_test.cc
+++ b/libziparchive/zip_archive_test.cc
@@ -979,10 +979,11 @@
std::vector<uint8_t> extended_field;
// Fake data to mimic the compressed bytes in the zipfile.
std::vector<uint8_t> compressed_bytes;
+ std::vector<uint8_t> data_descriptor;
size_t GetSize() const {
return local_file_header.size() + file_name.size() + extended_field.size() +
- compressed_bytes.size();
+ compressed_bytes.size() + data_descriptor.size();
}
void CopyToOutput(std::vector<uint8_t>* output) const {
@@ -990,6 +991,7 @@
std::copy(file_name.begin(), file_name.end(), std::back_inserter(*output));
std::copy(extended_field.begin(), extended_field.end(), std::back_inserter(*output));
std::copy(compressed_bytes.begin(), compressed_bytes.end(), std::back_inserter(*output));
+ std::copy(data_descriptor.begin(), data_descriptor.end(), std::back_inserter(*output));
}
};
@@ -1057,7 +1059,7 @@
// Add an entry to the zipfile, construct the corresponding local header and cd entry.
void AddEntry(const std::string& name, const std::vector<uint8_t>& content,
bool uncompressed_size_in_extended, bool compressed_size_in_extended,
- bool local_offset_in_extended) {
+ bool local_offset_in_extended, bool include_data_descriptor = false) {
auto uncompressed_size = static_cast<uint32_t>(content.size());
auto compressed_size = static_cast<uint32_t>(content.size());
uint32_t local_file_header_offset = 0;
@@ -1084,6 +1086,21 @@
ConstructLocalFileHeader(name, &local_entry.local_file_header, uncompressed_size,
compressed_size);
ConstructExtendedField(zip64_fields, &local_entry.extended_field);
+ if (include_data_descriptor) {
+ size_t descriptor_size = compressed_size_in_extended ? 24 : 16;
+ local_entry.data_descriptor.resize(descriptor_size);
+ uint8_t* write_ptr = local_entry.data_descriptor.data();
+ EmitUnaligned<uint32_t>(&write_ptr, DataDescriptor::kOptSignature);
+ EmitUnaligned<uint32_t>(&write_ptr, 0 /* crc */);
+ if (compressed_size_in_extended) {
+ EmitUnaligned<uint64_t>(&write_ptr, compressed_size_in_extended);
+ EmitUnaligned<uint64_t>(&write_ptr, uncompressed_size_in_extended);
+ } else {
+ EmitUnaligned<uint32_t>(&write_ptr, compressed_size_in_extended);
+ EmitUnaligned<uint32_t>(&write_ptr, uncompressed_size_in_extended);
+ }
+ }
+
file_entries_.push_back(std::move(local_entry));
if (local_offset_in_extended) {
@@ -1270,3 +1287,38 @@
0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle));
CloseArchive(handle);
}
+
+TEST_F(Zip64ParseTest, extract) {
+ std::vector<uint8_t> content(200, 'a');
+ AddEntry("a.txt", content, true, true, true);
+ ConstructEocd();
+ ConstructZipFile();
+
+ ZipArchiveHandle handle;
+ ASSERT_EQ(
+ 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle));
+ ZipEntry entry;
+ ASSERT_EQ(0, FindEntry(handle, "a.txt", &entry));
+
+ VectorWriter writer;
+ ASSERT_EQ(0, ExtractToWriter(handle, &entry, &writer));
+ ASSERT_EQ(content, writer.GetOutput());
+}
+
+TEST_F(Zip64ParseTest, extractWithDataDescriptor) {
+ std::vector<uint8_t> content(300, 'b');
+ AddEntry("a.txt", std::vector<uint8_t>(200, 'a'), true, true, true);
+ AddEntry("b.txt", content, true, true, true, true /* data descriptor */);
+ ConstructEocd();
+ ConstructZipFile();
+
+ ZipArchiveHandle handle;
+ ASSERT_EQ(
+ 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle));
+ ZipEntry entry;
+ ASSERT_EQ(0, FindEntry(handle, "b.txt", &entry));
+
+ VectorWriter writer;
+ ASSERT_EQ(0, ExtractToWriter(handle, &entry, &writer));
+ ASSERT_EQ(content, writer.GetOutput());
+}
diff --git a/libziparchive/zip_writer.cc b/libziparchive/zip_writer.cc
index 67279a6..25b1da4 100644
--- a/libziparchive/zip_writer.cc
+++ b/libziparchive/zip_writer.cc
@@ -475,19 +475,16 @@
if (ShouldUseDataDescriptor()) {
// Some versions of ZIP don't allow STORED data to have a trailing DataDescriptor.
// If this file is not seekable, or if the data is compressed, write a DataDescriptor.
- const uint32_t sig = DataDescriptor::kOptSignature;
- if (fwrite(&sig, sizeof(sig), 1, file_) != 1) {
+ // We haven't supported zip64 format yet. Write both uncompressed size and compressed
+ // size as uint32_t.
+ std::vector<uint32_t> dataDescriptor = {
+ DataDescriptor::kOptSignature, current_file_entry_.crc32,
+ current_file_entry_.compressed_size, current_file_entry_.uncompressed_size};
+ if (fwrite(dataDescriptor.data(), dataDescriptor.size() * sizeof(uint32_t), 1, file_) != 1) {
return HandleError(kIoError);
}
- DataDescriptor dd = {};
- dd.crc32 = current_file_entry_.crc32;
- dd.compressed_size = current_file_entry_.compressed_size;
- dd.uncompressed_size = current_file_entry_.uncompressed_size;
- if (fwrite(&dd, sizeof(dd), 1, file_) != 1) {
- return HandleError(kIoError);
- }
- current_offset_ += sizeof(DataDescriptor::kOptSignature) + sizeof(dd);
+ current_offset_ += sizeof(uint32_t) * dataDescriptor.size();
} else {
// Seek back to the header and rewrite to include the size.
if (fseeko(file_, current_file_entry_.local_file_header_offset, SEEK_SET) != 0) {
diff --git a/rootdir/init.rc b/rootdir/init.rc
index 5ebffab..adfdb7b 100644
--- a/rootdir/init.rc
+++ b/rootdir/init.rc
@@ -825,6 +825,11 @@
start zygote
start zygote_secondary
+on boot && property:ro.config.low_ram=true
+ # Tweak background writeout
+ write /proc/sys/vm/dirty_expire_centisecs 200
+ write /proc/sys/vm/dirty_background_ratio 5
+
on boot
# basic network init
ifup lo
@@ -846,11 +851,7 @@
chown root system /sys/block/zram0/writeback
chmod 0664 /sys/block/zram0/writeback
- # Tweak background writeout
- write /proc/sys/vm/dirty_expire_centisecs 200
- write /proc/sys/vm/dirty_background_ratio 5
-
- # F2FS tuning. Set cp_interval larger than dirty_expire_centisecs
+ # F2FS tuning. Set cp_interval larger than dirty_expire_centisecs, 30 secs,
# to avoid power consumption when system becomes mostly idle. Be careful
# to make it too large, since it may bring userdata loss, if they
# are not aware of using fsync()/sync() to prepare sudden power-cut.