Merge "logd: Drop the LogStatistics dependency on LogBufferElement"
diff --git a/adb/Android.bp b/adb/Android.bp
index 9db151d..8747182 100644
--- a/adb/Android.bp
+++ b/adb/Android.bp
@@ -124,11 +124,12 @@
"libadbd_core",
"libadbconnection_server",
"libasyncio",
+ "libbase",
"libbrotli",
"libcutils_sockets",
"libdiagnose_usb",
"libmdnssd",
- "libbase",
+ "libzstd",
"libadb_protos",
"libapp_processes_protos_lite",
@@ -351,6 +352,7 @@
"liblog",
"libziparchive",
"libz",
+ "libzstd",
],
// Don't add anything here, we don't want additional shared dependencies
@@ -483,6 +485,7 @@
"libbrotli",
"libdiagnose_usb",
"liblz4",
+ "libzstd",
],
shared_libs: [
@@ -586,6 +589,7 @@
"libdiagnose_usb",
"liblz4",
"libmdnssd",
+ "libzstd",
],
visibility: [
diff --git a/adb/client/commandline.cpp b/adb/client/commandline.cpp
index eaa32e5..43772ba 100644
--- a/adb/client/commandline.cpp
+++ b/adb/client/commandline.cpp
@@ -1336,6 +1336,8 @@
return CompressionType::Brotli;
} else if (str == "lz4") {
return CompressionType::LZ4;
+ } else if (str == "zstd") {
+ return CompressionType::Zstd;
}
error_exit("unexpected compression type %s", str.c_str());
diff --git a/adb/client/file_sync_client.cpp b/adb/client/file_sync_client.cpp
index 7185939..8bbe2a8 100644
--- a/adb/client/file_sync_client.cpp
+++ b/adb/client/file_sync_client.cpp
@@ -240,6 +240,7 @@
have_sendrecv_v2_ = CanUseFeature(*features, kFeatureSendRecv2);
have_sendrecv_v2_brotli_ = CanUseFeature(*features, kFeatureSendRecv2Brotli);
have_sendrecv_v2_lz4_ = CanUseFeature(*features, kFeatureSendRecv2LZ4);
+ have_sendrecv_v2_zstd_ = CanUseFeature(*features, kFeatureSendRecv2Zstd);
have_sendrecv_v2_dry_run_send_ = CanUseFeature(*features, kFeatureSendRecv2DryRunSend);
std::string error;
fd.reset(adb_connect("sync:", &error));
@@ -268,13 +269,16 @@
bool HaveSendRecv2() const { return have_sendrecv_v2_; }
bool HaveSendRecv2Brotli() const { return have_sendrecv_v2_brotli_; }
bool HaveSendRecv2LZ4() const { return have_sendrecv_v2_lz4_; }
+ bool HaveSendRecv2Zstd() const { return have_sendrecv_v2_zstd_; }
bool HaveSendRecv2DryRunSend() const { return have_sendrecv_v2_dry_run_send_; }
// Resolve a compression type which might be CompressionType::Any to a specific compression
// algorithm.
CompressionType ResolveCompressionType(CompressionType compression) const {
if (compression == CompressionType::Any) {
- if (HaveSendRecv2LZ4()) {
+ if (HaveSendRecv2Zstd()) {
+ return CompressionType::Zstd;
+ } else if (HaveSendRecv2LZ4()) {
return CompressionType::LZ4;
} else if (HaveSendRecv2Brotli()) {
return CompressionType::Brotli;
@@ -374,6 +378,10 @@
msg.send_v2_setup.flags = kSyncFlagLZ4;
break;
+ case CompressionType::Zstd:
+ msg.send_v2_setup.flags = kSyncFlagZstd;
+ break;
+
case CompressionType::Any:
LOG(FATAL) << "unexpected CompressionType::Any";
}
@@ -421,6 +429,10 @@
msg.recv_v2_setup.flags |= kSyncFlagLZ4;
break;
+ case CompressionType::Zstd:
+ msg.recv_v2_setup.flags |= kSyncFlagZstd;
+ break;
+
case CompressionType::Any:
LOG(FATAL) << "unexpected CompressionType::Any";
}
@@ -631,7 +643,8 @@
syncsendbuf sbuf;
sbuf.id = ID_DATA;
- std::variant<std::monostate, NullEncoder, BrotliEncoder, LZ4Encoder> encoder_storage;
+ std::variant<std::monostate, NullEncoder, BrotliEncoder, LZ4Encoder, ZstdEncoder>
+ encoder_storage;
Encoder* encoder = nullptr;
switch (compression) {
case CompressionType::None:
@@ -646,6 +659,10 @@
encoder = &encoder_storage.emplace<LZ4Encoder>(SYNC_DATA_MAX);
break;
+ case CompressionType::Zstd:
+ encoder = &encoder_storage.emplace<ZstdEncoder>(SYNC_DATA_MAX);
+ break;
+
case CompressionType::Any:
LOG(FATAL) << "unexpected CompressionType::Any";
}
@@ -928,6 +945,7 @@
bool have_sendrecv_v2_;
bool have_sendrecv_v2_brotli_;
bool have_sendrecv_v2_lz4_;
+ bool have_sendrecv_v2_zstd_;
bool have_sendrecv_v2_dry_run_send_;
TransferLedger global_ledger_;
@@ -1133,7 +1151,8 @@
uint64_t bytes_copied = 0;
Block buffer(SYNC_DATA_MAX);
- std::variant<std::monostate, NullDecoder, BrotliDecoder, LZ4Decoder> decoder_storage;
+ std::variant<std::monostate, NullDecoder, BrotliDecoder, LZ4Decoder, ZstdDecoder>
+ decoder_storage;
Decoder* decoder = nullptr;
std::span buffer_span(buffer.data(), buffer.size());
@@ -1150,6 +1169,10 @@
decoder = &decoder_storage.emplace<LZ4Decoder>(buffer_span);
break;
+ case CompressionType::Zstd:
+ decoder = &decoder_storage.emplace<ZstdDecoder>(buffer_span);
+ break;
+
case CompressionType::Any:
LOG(FATAL) << "unexpected CompressionType::Any";
}
diff --git a/adb/client/incremental.cpp b/adb/client/incremental.cpp
index a8b0ab3..1e985ed 100644
--- a/adb/client/incremental.cpp
+++ b/adb/client/incremental.cpp
@@ -42,7 +42,7 @@
struct stat st;
if (stat(signature_file.c_str(), &st)) {
if (!silent) {
- fprintf(stderr, "Failed to stat signature file %s. Abort.\n", signature_file.c_str());
+ fprintf(stderr, "Failed to stat signature file %s.\n", signature_file.c_str());
}
return {};
}
@@ -50,11 +50,21 @@
unique_fd fd(adb_open(signature_file.c_str(), O_RDONLY));
if (fd < 0) {
if (!silent) {
- fprintf(stderr, "Failed to open signature file: %s. Abort.\n", signature_file.c_str());
+ fprintf(stderr, "Failed to open signature file: %s.\n", signature_file.c_str());
}
return {};
}
+ std::vector<char> invalid_signature;
+
+ if (st.st_size > kMaxSignatureSize) {
+ if (!silent) {
+ fprintf(stderr, "Signature is too long. Max allowed is %d. Abort.\n",
+ kMaxSignatureSize);
+ }
+ return {std::move(fd), std::move(invalid_signature)};
+ }
+
auto [signature, tree_size] = read_id_sig_headers(fd);
if (auto expected = verity_tree_size_for_file(file_size); tree_size != expected) {
if (!silent) {
@@ -62,7 +72,7 @@
"Verity tree size mismatch in signature file: %s [was %lld, expected %lld].\n",
signature_file.c_str(), (long long)tree_size, (long long)expected);
}
- return {};
+ return {std::move(fd), std::move(invalid_signature)};
}
return {std::move(fd), std::move(signature)};
@@ -72,9 +82,11 @@
static std::pair<unique_fd, std::string> read_and_encode_signature(Size file_size,
std::string signature_file,
bool silent) {
+ std::string encoded_signature;
+
auto [fd, signature] = read_signature(file_size, std::move(signature_file), silent);
- if (!fd.ok()) {
- return {};
+ if (!fd.ok() || signature.empty()) {
+ return {std::move(fd), std::move(encoded_signature)};
}
size_t base64_len = 0;
@@ -82,9 +94,10 @@
if (!silent) {
fprintf(stderr, "Fail to estimate base64 encoded length. Abort.\n");
}
- return {};
+ return {std::move(fd), std::move(encoded_signature)};
}
- std::string encoded_signature(base64_len, '\0');
+
+ encoded_signature.resize(base64_len, '\0');
encoded_signature.resize(EVP_EncodeBlock((uint8_t*)encoded_signature.data(),
(const uint8_t*)signature.data(), signature.size()));
@@ -109,7 +122,7 @@
}
auto [signature_fd, signature] = read_and_encode_signature(st.st_size, file, silent);
- if (!signature_fd.ok()) {
+ if (signature_fd.ok() && signature.empty()) {
return {};
}
@@ -138,9 +151,12 @@
return false;
}
- auto [fd, _] = read_signature(st.st_size, file, true);
- if (!fd.ok()) {
- return false;
+ if (android::base::EndsWithIgnoreCase(file, ".apk")) {
+ // Signature has to be present for APKs.
+ auto [fd, _] = read_signature(st.st_size, file, /*silent=*/true);
+ if (!fd.ok()) {
+ return false;
+ }
}
}
return true;
diff --git a/adb/client/incremental_server.cpp b/adb/client/incremental_server.cpp
index bfe18c0..0654a11 100644
--- a/adb/client/incremental_server.cpp
+++ b/adb/client/incremental_server.cpp
@@ -171,6 +171,8 @@
const std::vector<BlockIdx>& PriorityBlocks() const { return priority_blocks_; }
+ bool hasTree() const { return tree_fd_.ok(); }
+
std::vector<bool> sentBlocks;
NumBlocks sentBlocksCount = 0;
@@ -349,6 +351,9 @@
bool IncrementalServer::SendTreeBlocksForDataBlock(const FileId fileId, const BlockIdx blockIdx) {
auto& file = files_[fileId];
+ if (!file.hasTree()) {
+ return true;
+ }
const int32_t data_block_count = numBytesToNumBlocks(file.size);
const int32_t total_nodes_count(file.sentTreeBlocks.size());
@@ -670,7 +675,8 @@
unique_fd fd(adb_open(signature_file.c_str(), O_RDONLY));
if (fd < 0) {
- error_exit("inc-server: failed to open file '%s'.", signature_file.c_str());
+ D("No signature file found for '%s'('%s')", filepath, signature_file.c_str());
+ return {};
}
auto [tree_offset, tree_size] = skip_id_sig_headers(fd);
diff --git a/adb/client/incremental_utils.h b/adb/client/incremental_utils.h
index fe2914d..4ad60dd 100644
--- a/adb/client/incremental_utils.h
+++ b/adb/client/incremental_utils.h
@@ -33,6 +33,7 @@
constexpr int kBlockSize = 4096;
constexpr int kSha256DigestSize = 32;
constexpr int kDigestSize = kSha256DigestSize;
+constexpr int kMaxSignatureSize = 8096; // incrementalfs.h
constexpr std::string_view IDSIG = ".idsig";
diff --git a/adb/compression_utils.h b/adb/compression_utils.h
index a0c48a2..a747108 100644
--- a/adb/compression_utils.h
+++ b/adb/compression_utils.h
@@ -25,6 +25,7 @@
#include <brotli/decode.h>
#include <brotli/encode.h>
#include <lz4frame.h>
+#include <zstd.h>
#include "types.h"
@@ -381,3 +382,105 @@
std::unique_ptr<LZ4F_cctx, LZ4F_errorCode_t (*)(LZ4F_cctx*)> encoder_;
IOVector output_buffer_;
};
+
+struct ZstdDecoder final : public Decoder {
+ explicit ZstdDecoder(std::span<char> output_buffer)
+ : Decoder(output_buffer), decoder_(ZSTD_createDStream(), ZSTD_freeDStream) {
+ if (!decoder_) {
+ LOG(FATAL) << "failed to initialize Zstd decompression context";
+ }
+ }
+
+ DecodeResult Decode(std::span<char>* output) final {
+ ZSTD_inBuffer in;
+ in.src = input_buffer_.front_data();
+ in.size = input_buffer_.front_size();
+ in.pos = 0;
+
+ ZSTD_outBuffer out;
+ out.dst = output_buffer_.data();
+ // The standard specifies size() as returning size_t, but our current version of
+ // libc++ returns a signed value instead.
+ out.size = static_cast<size_t>(output_buffer_.size());
+ out.pos = 0;
+
+ size_t rc = ZSTD_decompressStream(decoder_.get(), &out, &in);
+ if (ZSTD_isError(rc)) {
+ LOG(ERROR) << "ZSTD_decompressStream failed: " << ZSTD_getErrorName(rc);
+ return DecodeResult::Error;
+ }
+
+ input_buffer_.drop_front(in.pos);
+ if (rc == 0) {
+ if (!input_buffer_.empty()) {
+ LOG(ERROR) << "Zstd stream hit end before reading all data";
+ return DecodeResult::Error;
+ }
+ zstd_done_ = true;
+ }
+
+ *output = std::span<char>(output_buffer_.data(), out.pos);
+
+ if (finished_) {
+ return input_buffer_.empty() && zstd_done_ ? DecodeResult::Done
+ : DecodeResult::MoreOutput;
+ }
+ return DecodeResult::NeedInput;
+ }
+
+ private:
+ bool zstd_done_ = false;
+ std::unique_ptr<ZSTD_DStream, size_t (*)(ZSTD_DStream*)> decoder_;
+};
+
+struct ZstdEncoder final : public Encoder {
+ explicit ZstdEncoder(size_t output_block_size)
+ : Encoder(output_block_size), encoder_(ZSTD_createCStream(), ZSTD_freeCStream) {
+ if (!encoder_) {
+ LOG(FATAL) << "failed to initialize Zstd compression context";
+ }
+ ZSTD_CCtx_setParameter(encoder_.get(), ZSTD_c_compressionLevel, 1);
+ }
+
+ EncodeResult Encode(Block* output) final {
+ ZSTD_inBuffer in;
+ in.src = input_buffer_.front_data();
+ in.size = input_buffer_.front_size();
+ in.pos = 0;
+
+ output->resize(output_block_size_);
+
+ ZSTD_outBuffer out;
+ out.dst = output->data();
+ out.size = static_cast<size_t>(output->size());
+ out.pos = 0;
+
+ ZSTD_EndDirective end_directive = finished_ ? ZSTD_e_end : ZSTD_e_continue;
+ size_t rc = ZSTD_compressStream2(encoder_.get(), &out, &in, end_directive);
+ if (ZSTD_isError(rc)) {
+ LOG(ERROR) << "ZSTD_compressStream2 failed: " << ZSTD_getErrorName(rc);
+ return EncodeResult::Error;
+ }
+
+ input_buffer_.drop_front(in.pos);
+ output->resize(out.pos);
+
+ if (rc == 0) {
+ // Zstd finished flushing its data.
+ if (finished_) {
+ if (!input_buffer_.empty()) {
+ LOG(ERROR) << "ZSTD_compressStream2 finished early";
+ return EncodeResult::Error;
+ }
+ return EncodeResult::Done;
+ } else {
+ return input_buffer_.empty() ? EncodeResult::NeedInput : EncodeResult::MoreOutput;
+ }
+ } else {
+ return EncodeResult::MoreOutput;
+ }
+ }
+
+ private:
+ std::unique_ptr<ZSTD_CStream, size_t (*)(ZSTD_CStream*)> encoder_;
+};
diff --git a/adb/daemon/file_sync_service.cpp b/adb/daemon/file_sync_service.cpp
index d58131e..513b8dd 100644
--- a/adb/daemon/file_sync_service.cpp
+++ b/adb/daemon/file_sync_service.cpp
@@ -272,7 +272,8 @@
syncmsg msg;
Block buffer(SYNC_DATA_MAX);
std::span<char> buffer_span(buffer.data(), buffer.size());
- std::variant<std::monostate, NullDecoder, BrotliDecoder, LZ4Decoder> decoder_storage;
+ std::variant<std::monostate, NullDecoder, BrotliDecoder, LZ4Decoder, ZstdDecoder>
+ decoder_storage;
Decoder* decoder = nullptr;
switch (compression) {
@@ -288,6 +289,10 @@
decoder = &decoder_storage.emplace<LZ4Decoder>(buffer_span);
break;
+ case CompressionType::Zstd:
+ decoder = &decoder_storage.emplace<ZstdDecoder>(buffer_span);
+ break;
+
case CompressionType::Any:
LOG(FATAL) << "unexpected CompressionType::Any";
}
@@ -590,6 +595,15 @@
}
compression = CompressionType::LZ4;
}
+ if (msg.send_v2_setup.flags & kSyncFlagZstd) {
+ msg.send_v2_setup.flags &= ~kSyncFlagZstd;
+ if (compression) {
+ SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d",
+ orig_flags));
+ return false;
+ }
+ compression = CompressionType::Zstd;
+ }
if (msg.send_v2_setup.flags & kSyncFlagDryRun) {
msg.send_v2_setup.flags &= ~kSyncFlagDryRun;
dry_run = true;
@@ -623,7 +637,8 @@
syncmsg msg;
msg.data.id = ID_DATA;
- std::variant<std::monostate, NullEncoder, BrotliEncoder, LZ4Encoder> encoder_storage;
+ std::variant<std::monostate, NullEncoder, BrotliEncoder, LZ4Encoder, ZstdEncoder>
+ encoder_storage;
Encoder* encoder;
switch (compression) {
@@ -639,6 +654,10 @@
encoder = &encoder_storage.emplace<LZ4Encoder>(SYNC_DATA_MAX);
break;
+ case CompressionType::Zstd:
+ encoder = &encoder_storage.emplace<ZstdEncoder>(SYNC_DATA_MAX);
+ break;
+
case CompressionType::Any:
LOG(FATAL) << "unexpected CompressionType::Any";
}
@@ -726,6 +745,15 @@
}
compression = CompressionType::LZ4;
}
+ if (msg.recv_v2_setup.flags & kSyncFlagZstd) {
+ msg.recv_v2_setup.flags &= ~kSyncFlagZstd;
+ if (compression) {
+ SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d",
+ orig_flags));
+ return false;
+ }
+ compression = CompressionType::Zstd;
+ }
if (msg.recv_v2_setup.flags) {
SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.recv_v2_setup.flags));
diff --git a/adb/file_sync_protocol.h b/adb/file_sync_protocol.h
index 8f8f85f..5234c20 100644
--- a/adb/file_sync_protocol.h
+++ b/adb/file_sync_protocol.h
@@ -93,6 +93,7 @@
kSyncFlagNone = 0,
kSyncFlagBrotli = 1,
kSyncFlagLZ4 = 2,
+ kSyncFlagZstd = 4,
kSyncFlagDryRun = 0x8000'0000U,
};
@@ -101,6 +102,7 @@
Any,
Brotli,
LZ4,
+ Zstd,
};
// send_v1 sent the path in a buffer, followed by a comma and the mode as a string.
diff --git a/adb/test_device.py b/adb/test_device.py
index 9f1f403..c1caafc 100755
--- a/adb/test_device.py
+++ b/adb/test_device.py
@@ -1362,6 +1362,10 @@
compression = "lz4"
+class FileOperationsTestZstd(FileOperationsTest.Base):
+ compression = "zstd"
+
+
class DeviceOfflineTest(DeviceTest):
def _get_device_state(self, serialno):
output = subprocess.check_output(self.device.adb_cmd + ['devices'])
diff --git a/adb/tools/check_ms_os_desc.cpp b/adb/tools/check_ms_os_desc.cpp
index 8743ff7..8e85809 100644
--- a/adb/tools/check_ms_os_desc.cpp
+++ b/adb/tools/check_ms_os_desc.cpp
@@ -131,8 +131,6 @@
errx(1, "failed to retrieve MS OS v1 compat descriptor: %s", libusb_error_name(rc));
}
- memcpy(&hdr, data.data(), data.size());
-
struct __attribute__((packed)) ms_os_desc_v1_function {
uint8_t bFirstInterfaceNumber;
uint8_t reserved1;
diff --git a/adb/transport.cpp b/adb/transport.cpp
index 1667011..b6b6984 100644
--- a/adb/transport.cpp
+++ b/adb/transport.cpp
@@ -85,6 +85,7 @@
const char* const kFeatureSendRecv2 = "sendrecv_v2";
const char* const kFeatureSendRecv2Brotli = "sendrecv_v2_brotli";
const char* const kFeatureSendRecv2LZ4 = "sendrecv_v2_lz4";
+const char* const kFeatureSendRecv2Zstd = "sendrecv_v2_zstd";
const char* const kFeatureSendRecv2DryRunSend = "sendrecv_v2_dry_run_send";
namespace {
@@ -1189,6 +1190,7 @@
kFeatureSendRecv2,
kFeatureSendRecv2Brotli,
kFeatureSendRecv2LZ4,
+ kFeatureSendRecv2Zstd,
kFeatureSendRecv2DryRunSend,
// Increment ADB_SERVER_VERSION when adding a feature that adbd needs
// to know about. Otherwise, the client can be stuck running an old
diff --git a/adb/transport.h b/adb/transport.h
index 2ac21cf..b1f2744 100644
--- a/adb/transport.h
+++ b/adb/transport.h
@@ -93,6 +93,8 @@
extern const char* const kFeatureSendRecv2Brotli;
// adbd supports LZ4 for send/recv v2.
extern const char* const kFeatureSendRecv2LZ4;
+// adbd supports Zstd for send/recv v2.
+extern const char* const kFeatureSendRecv2Zstd;
// adbd supports dry-run send for send/recv v2.
extern const char* const kFeatureSendRecv2DryRunSend;