Merge "libsnapshot: Fix intermittent test failure due to missing null check."
diff --git a/TEST_MAPPING b/TEST_MAPPING
index 44c47f3..9b6213a 100644
--- a/TEST_MAPPING
+++ b/TEST_MAPPING
@@ -67,5 +67,11 @@
{
"name": "ziparchive-tests"
}
+ ],
+
+ "postsubmit": [
+ {
+ "name": "ziparchive_tests_large"
+ }
]
}
diff --git a/adb/Android.bp b/adb/Android.bp
index b879753..81d20c1 100644
--- a/adb/Android.bp
+++ b/adb/Android.bp
@@ -298,6 +298,7 @@
"client/fastdeploycallbacks.cpp",
"client/incremental.cpp",
"client/incremental_server.cpp",
+ "client/incremental_utils.cpp",
"shell_service_protocol.cpp",
],
@@ -316,6 +317,7 @@
"libandroidfw",
"libapp_processes_protos_full",
"libbase",
+ "libbrotli",
"libcutils",
"libcrypto_utils",
"libcrypto",
@@ -468,6 +470,7 @@
static_libs: [
"libadbconnection_server",
"libadbd_core",
+ "libbrotli",
"libdiagnose_usb",
],
@@ -526,6 +529,7 @@
name: "libadbd",
defaults: ["adbd_defaults", "host_adbd_supported"],
recovery_available: true,
+ apex_available: ["com.android.adbd"],
// avoid getting duplicate symbol of android::build::getbuildnumber().
use_version_lib: false,
@@ -565,6 +569,7 @@
},
static_libs: [
+ "libbrotli",
"libcutils_sockets",
"libdiagnose_usb",
"libmdnssd",
@@ -580,6 +585,7 @@
defaults: ["adbd_defaults", "host_adbd_supported"],
stl: "libc++_static",
recovery_available: true,
+ apex_available: ["com.android.adbd"],
srcs: [
"daemon/main.cpp",
@@ -603,6 +609,7 @@
"libapp_processes_protos_lite",
"libasyncio",
"libbase",
+ "libbrotli",
"libcap",
"libcrypto_utils",
"libcutils_sockets",
diff --git a/adb/brotli_utils.h b/adb/brotli_utils.h
new file mode 100644
index 0000000..c5be73d
--- /dev/null
+++ b/adb/brotli_utils.h
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <span>
+
+#include <brotli/decode.h>
+#include <brotli/encode.h>
+
+#include "types.h"
+
+enum class BrotliDecodeResult {
+ Error,
+ Done,
+ NeedInput,
+ MoreOutput,
+};
+
+struct BrotliDecoder {
+ explicit BrotliDecoder(std::span<char> output_buffer)
+ : output_buffer_(output_buffer),
+ decoder_(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr),
+ BrotliDecoderDestroyInstance) {}
+
+ void Append(Block&& block) { input_buffer_.append(std::move(block)); }
+
+ BrotliDecodeResult Decode(std::span<char>* output) {
+ size_t available_in = input_buffer_.front_size();
+ const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());
+
+ size_t available_out = output_buffer_.size();
+ uint8_t* next_out = reinterpret_cast<uint8_t*>(output_buffer_.data());
+
+ BrotliDecoderResult r = BrotliDecoderDecompressStream(
+ decoder_.get(), &available_in, &next_in, &available_out, &next_out, nullptr);
+
+ size_t bytes_consumed = input_buffer_.front_size() - available_in;
+ input_buffer_.drop_front(bytes_consumed);
+
+ size_t bytes_emitted = output_buffer_.size() - available_out;
+ *output = std::span<char>(output_buffer_.data(), bytes_emitted);
+
+ switch (r) {
+ case BROTLI_DECODER_RESULT_SUCCESS:
+ return BrotliDecodeResult::Done;
+ case BROTLI_DECODER_RESULT_ERROR:
+ return BrotliDecodeResult::Error;
+ case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
+ // Brotli guarantees as one of its invariants that if it returns NEEDS_MORE_INPUT,
+ // it will consume the entire input buffer passed in, so we don't have to worry
+ // about bytes left over in the front block with more input remaining.
+ return BrotliDecodeResult::NeedInput;
+ case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
+ return BrotliDecodeResult::MoreOutput;
+ }
+ }
+
+ private:
+ IOVector input_buffer_;
+ std::span<char> output_buffer_;
+ std::unique_ptr<BrotliDecoderState, void (*)(BrotliDecoderState*)> decoder_;
+};
+
+enum class BrotliEncodeResult {
+ Error,
+ Done,
+ NeedInput,
+ MoreOutput,
+};
+
+template <size_t OutputBlockSize>
+struct BrotliEncoder {
+ explicit BrotliEncoder()
+ : output_block_(OutputBlockSize),
+ output_bytes_left_(OutputBlockSize),
+ encoder_(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr),
+ BrotliEncoderDestroyInstance) {
+ BrotliEncoderSetParameter(encoder_.get(), BROTLI_PARAM_QUALITY, 1);
+ }
+
+ void Append(Block input) { input_buffer_.append(std::move(input)); }
+ void Finish() { finished_ = true; }
+
+ BrotliEncodeResult Encode(Block* output) {
+ output->clear();
+ while (true) {
+ size_t available_in = input_buffer_.front_size();
+ const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());
+
+ size_t available_out = output_bytes_left_;
+ uint8_t* next_out = reinterpret_cast<uint8_t*>(output_block_.data() +
+ (OutputBlockSize - output_bytes_left_));
+
+ BrotliEncoderOperation op = BROTLI_OPERATION_PROCESS;
+ if (finished_) {
+ op = BROTLI_OPERATION_FINISH;
+ }
+
+ if (!BrotliEncoderCompressStream(encoder_.get(), op, &available_in, &next_in,
+ &available_out, &next_out, nullptr)) {
+ return BrotliEncodeResult::Error;
+ }
+
+ size_t bytes_consumed = input_buffer_.front_size() - available_in;
+ input_buffer_.drop_front(bytes_consumed);
+
+ output_bytes_left_ = available_out;
+
+ if (BrotliEncoderIsFinished(encoder_.get())) {
+ output_block_.resize(OutputBlockSize - output_bytes_left_);
+ *output = std::move(output_block_);
+ return BrotliEncodeResult::Done;
+ } else if (output_bytes_left_ == 0) {
+ *output = std::move(output_block_);
+ output_block_.resize(OutputBlockSize);
+ output_bytes_left_ = OutputBlockSize;
+ return BrotliEncodeResult::MoreOutput;
+ } else if (input_buffer_.empty()) {
+ return BrotliEncodeResult::NeedInput;
+ }
+ }
+ }
+
+ private:
+ bool finished_ = false;
+ IOVector input_buffer_;
+ Block output_block_;
+ size_t output_bytes_left_;
+ std::unique_ptr<BrotliEncoderState, void (*)(BrotliEncoderState*)> encoder_;
+};
diff --git a/adb/client/adb_install.cpp b/adb/client/adb_install.cpp
index 21b8f49..fe9182e 100644
--- a/adb/client/adb_install.cpp
+++ b/adb/client/adb_install.cpp
@@ -286,7 +286,7 @@
}
}
- if (do_sync_push(apk_file, apk_dest.c_str(), false)) {
+ if (do_sync_push(apk_file, apk_dest.c_str(), false, true)) {
result = pm_command(argc, argv);
delete_device_file(apk_dest);
}
diff --git a/adb/client/commandline.cpp b/adb/client/commandline.cpp
index 7c7da08..04b250d 100644
--- a/adb/client/commandline.cpp
+++ b/adb/client/commandline.cpp
@@ -129,15 +129,21 @@
" reverse --remove-all remove all reverse socket connections from device\n"
"\n"
"file transfer:\n"
- " push [--sync] LOCAL... REMOTE\n"
+ " push [--sync] [-zZ] LOCAL... REMOTE\n"
" copy local files/directories to device\n"
" --sync: only push files that are newer on the host than the device\n"
- " pull [-a] REMOTE... LOCAL\n"
+ " -z: enable compression\n"
+ " -Z: disable compression\n"
+ " pull [-azZ] REMOTE... LOCAL\n"
" copy files/dirs from device\n"
" -a: preserve file timestamp and mode\n"
- " sync [all|data|odm|oem|product|system|system_ext|vendor]\n"
+ " -z: enable compression\n"
+ " -Z: disable compression\n"
+ " sync [-lzZ] [all|data|odm|oem|product|system|system_ext|vendor]\n"
" sync a local build from $ANDROID_PRODUCT_OUT to the device (default all)\n"
" -l: list files that would be copied, but don't copy them\n"
+ " -z: enable compression\n"
+ " -Z: disable compression\n"
"\n"
"shell:\n"
" shell [-e ESCAPE] [-n] [-Tt] [-x] [COMMAND...]\n"
@@ -1309,8 +1315,12 @@
}
static void parse_push_pull_args(const char** arg, int narg, std::vector<const char*>* srcs,
- const char** dst, bool* copy_attrs, bool* sync) {
+ const char** dst, bool* copy_attrs, bool* sync, bool* compressed) {
*copy_attrs = false;
+ const char* adb_compression = getenv("ADB_COMPRESSION");
+ if (adb_compression && strcmp(adb_compression, "0") == 0) {
+ *compressed = false;
+ }
srcs->clear();
bool ignore_flags = false;
@@ -1322,6 +1332,14 @@
// Silently ignore for backwards compatibility.
} else if (!strcmp(*arg, "-a")) {
*copy_attrs = true;
+ } else if (!strcmp(*arg, "-z")) {
+ if (compressed != nullptr) {
+ *compressed = true;
+ }
+ } else if (!strcmp(*arg, "-Z")) {
+ if (compressed != nullptr) {
+ *compressed = false;
+ }
} else if (!strcmp(*arg, "--sync")) {
if (sync != nullptr) {
*sync = true;
@@ -1443,6 +1461,26 @@
#endif
}
+static bool _is_valid_os_fd(int fd) {
+ // Disallow invalid FDs and stdin/out/err as well.
+ if (fd < 3) {
+ return false;
+ }
+#ifdef _WIN32
+ auto handle = (HANDLE)fd;
+ DWORD info = 0;
+ if (GetHandleInformation(handle, &info) == 0) {
+ return false;
+ }
+#else
+ int flags = fcntl(fd, F_GETFD);
+ if (flags == -1) {
+ return false;
+ }
+#endif
+ return true;
+}
+
int adb_commandline(int argc, const char** argv) {
bool no_daemon = false;
bool is_daemon = false;
@@ -1856,20 +1894,22 @@
} else if (!strcmp(argv[0], "push")) {
bool copy_attrs = false;
bool sync = false;
+ bool compressed = true;
std::vector<const char*> srcs;
const char* dst = nullptr;
- parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync);
+ parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync, &compressed);
if (srcs.empty() || !dst) error_exit("push requires an argument");
- return do_sync_push(srcs, dst, sync) ? 0 : 1;
+ return do_sync_push(srcs, dst, sync, compressed) ? 0 : 1;
} else if (!strcmp(argv[0], "pull")) {
bool copy_attrs = false;
+ bool compressed = true;
std::vector<const char*> srcs;
const char* dst = ".";
- parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr);
+ parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr, &compressed);
if (srcs.empty()) error_exit("pull requires an argument");
- return do_sync_pull(srcs, dst, copy_attrs) ? 0 : 1;
+ return do_sync_pull(srcs, dst, copy_attrs, compressed) ? 0 : 1;
} else if (!strcmp(argv[0], "install")) {
if (argc < 2) error_exit("install requires an argument");
return install_app(argc, argv);
@@ -1885,18 +1925,38 @@
} else if (!strcmp(argv[0], "sync")) {
std::string src;
bool list_only = false;
- if (argc < 2) {
- // No partition specified: sync all of them.
- } else if (argc >= 2 && strcmp(argv[1], "-l") == 0) {
- list_only = true;
- if (argc == 3) src = argv[2];
- } else if (argc == 2) {
- src = argv[1];
- } else {
- error_exit("usage: adb sync [-l] [PARTITION]");
+ bool compressed = true;
+
+ const char* adb_compression = getenv("ADB_COMPRESSION");
+ if (adb_compression && strcmp(adb_compression, "0") == 0) {
+ compressed = false;
}
- if (src.empty()) src = "all";
+ int opt;
+ while ((opt = getopt(argc, const_cast<char**>(argv), "lzZ")) != -1) {
+ switch (opt) {
+ case 'l':
+ list_only = true;
+ break;
+ case 'z':
+ compressed = true;
+ break;
+ case 'Z':
+ compressed = false;
+ break;
+ default:
+ error_exit("usage: adb sync [-lzZ] [PARTITION]");
+ }
+ }
+
+ if (optind == argc) {
+ src = "all";
+ } else if (optind + 1 == argc) {
+ src = argv[optind];
+ } else {
+ error_exit("usage: adb sync [-lzZ] [PARTITION]");
+ }
+
std::vector<std::string> partitions{"data", "odm", "oem", "product",
"system", "system_ext", "vendor"};
bool found = false;
@@ -1905,7 +1965,7 @@
std::string src_dir{product_file(partition)};
if (!directory_exists(src_dir)) continue;
found = true;
- if (!do_sync_sync(src_dir, "/" + partition, list_only)) return 1;
+ if (!do_sync_sync(src_dir, "/" + partition, list_only, compressed)) return 1;
}
}
if (!found) error_exit("don't know how to sync %s partition", src.c_str());
@@ -2009,17 +2069,28 @@
}
}
} else if (!strcmp(argv[0], "inc-server")) {
- if (argc < 3) {
- error_exit("usage: adb inc-server FD FILE1 FILE2 ...");
+ if (argc < 4) {
+#ifdef _WIN32
+ error_exit("usage: adb inc-server CONNECTION_HANDLE OUTPUT_HANDLE FILE1 FILE2 ...");
+#else
+ error_exit("usage: adb inc-server CONNECTION_FD OUTPUT_FD FILE1 FILE2 ...");
+#endif
}
- int fd = atoi(argv[1]);
- if (fd < 3) {
- // Disallow invalid FDs and stdin/out/err as well.
- error_exit("Invalid fd number given: %d", fd);
+ int connection_fd = atoi(argv[1]);
+ if (!_is_valid_os_fd(connection_fd)) {
+ error_exit("Invalid connection_fd number given: %d", connection_fd);
}
- fd = adb_register_socket(fd);
- close_on_exec(fd);
- return incremental::serve(fd, argc - 2, argv + 2);
+
+ connection_fd = adb_register_socket(connection_fd);
+ close_on_exec(connection_fd);
+
+ int output_fd = atoi(argv[2]);
+ if (!_is_valid_os_fd(output_fd)) {
+ error_exit("Invalid output_fd number given: %d", output_fd);
+ }
+ output_fd = adb_register_socket(output_fd);
+ close_on_exec(output_fd);
+ return incremental::serve(connection_fd, output_fd, argc - 3, argv + 3);
}
error_exit("unknown command %s", argv[0]);
diff --git a/adb/client/fastdeploy.cpp b/adb/client/fastdeploy.cpp
index c5fc12f..de82e14 100644
--- a/adb/client/fastdeploy.cpp
+++ b/adb/client/fastdeploy.cpp
@@ -112,7 +112,7 @@
// but can't be removed until after the push.
unix_close(tf.release());
- if (!do_sync_push(srcs, dst, sync)) {
+ if (!do_sync_push(srcs, dst, sync, true)) {
error_exit("Failed to push fastdeploy agent to device.");
}
}
diff --git a/adb/client/file_sync_client.cpp b/adb/client/file_sync_client.cpp
index d201239..56568c2 100644
--- a/adb/client/file_sync_client.cpp
+++ b/adb/client/file_sync_client.cpp
@@ -42,6 +42,7 @@
#include "adb_client.h"
#include "adb_io.h"
#include "adb_utils.h"
+#include "brotli_utils.h"
#include "file_sync_protocol.h"
#include "line_printer.h"
#include "sysdeps/errno.h"
@@ -53,6 +54,8 @@
#include <android-base/strings.h>
#include <android-base/stringprintf.h>
+using namespace std::literals;
+
typedef void(sync_ls_cb)(unsigned mode, uint64_t size, uint64_t time, const char* name);
struct syncsendbuf {
@@ -113,6 +116,11 @@
uint64_t bytes_expected;
bool expect_multiple_files;
+ private:
+ std::string last_progress_str;
+ std::chrono::steady_clock::time_point last_progress_time;
+
+ public:
TransferLedger() {
Reset();
}
@@ -132,6 +140,8 @@
files_skipped = 0;
bytes_transferred = 0;
bytes_expected = 0;
+ last_progress_str.clear();
+ last_progress_time = {};
}
std::string TransferRate() {
@@ -151,6 +161,12 @@
void ReportProgress(LinePrinter& lp, const std::string& file, uint64_t file_copied_bytes,
uint64_t file_total_bytes) {
+ static constexpr auto kProgressReportInterval = 100ms;
+
+ auto now = std::chrono::steady_clock::now();
+ if (now < last_progress_time + kProgressReportInterval) {
+ return;
+ }
char overall_percentage_str[5] = "?";
if (bytes_expected != 0 && bytes_transferred <= bytes_expected) {
int overall_percentage = static_cast<int>(bytes_transferred * 100 / bytes_expected);
@@ -181,7 +197,11 @@
android::base::StringPrintf("[%4s] %s", overall_percentage_str, file.c_str());
}
}
- lp.Print(output, LinePrinter::LineType::INFO);
+ if (output != last_progress_str) {
+ lp.Print(output, LinePrinter::LineType::INFO);
+ last_progress_str = std::move(output);
+ last_progress_time = now;
+ }
}
void ReportTransferRate(LinePrinter& lp, const std::string& name, TransferDirection direction) {
@@ -214,6 +234,8 @@
} else {
have_stat_v2_ = CanUseFeature(features_, kFeatureStat2);
have_ls_v2_ = CanUseFeature(features_, kFeatureLs2);
+ have_sendrecv_v2_ = CanUseFeature(features_, kFeatureSendRecv2);
+ have_sendrecv_v2_brotli_ = CanUseFeature(features_, kFeatureSendRecv2Brotli);
fd.reset(adb_connect("sync:", &error));
if (fd < 0) {
Error("connect failed: %s", error.c_str());
@@ -237,6 +259,9 @@
line_printer_.KeepInfoLine();
}
+ bool HaveSendRecv2() const { return have_sendrecv_v2_; }
+ bool HaveSendRecv2Brotli() const { return have_sendrecv_v2_brotli_; }
+
const FeatureSet& Features() const { return features_; }
bool IsValid() { return fd >= 0; }
@@ -295,6 +320,62 @@
req->path_length = path.length();
char* data = reinterpret_cast<char*>(req + 1);
memcpy(data, path.data(), path.length());
+ return WriteFdExactly(fd, buf.data(), buf.size());
+ }
+
+ bool SendSend2(std::string_view path, mode_t mode, bool compressed) {
+ if (path.length() > 1024) {
+ Error("SendRequest failed: path too long: %zu", path.length());
+ errno = ENAMETOOLONG;
+ return false;
+ }
+
+ Block buf;
+
+ SyncRequest req;
+ req.id = ID_SEND_V2;
+ req.path_length = path.length();
+
+ syncmsg msg;
+ msg.send_v2_setup.id = ID_SEND_V2;
+ msg.send_v2_setup.mode = mode;
+ msg.send_v2_setup.flags = compressed ? kSyncFlagBrotli : kSyncFlagNone;
+
+ buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.send_v2_setup));
+
+ void* p = buf.data();
+
+ p = mempcpy(p, &req, sizeof(SyncRequest));
+ p = mempcpy(p, path.data(), path.length());
+ p = mempcpy(p, &msg.send_v2_setup, sizeof(msg.send_v2_setup));
+
+ return WriteFdExactly(fd, buf.data(), buf.size());
+ }
+
+ bool SendRecv2(const std::string& path) {
+ if (path.length() > 1024) {
+ Error("SendRequest failed: path too long: %zu", path.length());
+ errno = ENAMETOOLONG;
+ return false;
+ }
+
+ Block buf;
+
+ SyncRequest req;
+ req.id = ID_RECV_V2;
+ req.path_length = path.length();
+
+ syncmsg msg;
+ msg.recv_v2_setup.id = ID_RECV_V2;
+ msg.recv_v2_setup.flags = kSyncFlagBrotli;
+
+ buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.recv_v2_setup));
+
+ void* p = buf.data();
+
+ p = mempcpy(p, &req, sizeof(SyncRequest));
+ p = mempcpy(p, path.data(), path.length());
+ p = mempcpy(p, &msg.recv_v2_setup, sizeof(msg.recv_v2_setup));
return WriteFdExactly(fd, buf.data(), buf.size());
}
@@ -351,8 +432,8 @@
}
if (msg.stat_v1.id != ID_LSTAT_V1) {
- PLOG(FATAL) << "protocol fault: stat response has wrong message id: "
- << msg.stat_v1.id;
+ LOG(FATAL) << "protocol fault: stat response has wrong message id: "
+ << msg.stat_v1.id;
}
if (msg.stat_v1.mode == 0 && msg.stat_v1.size == 0 && msg.stat_v1.mtime == 0) {
@@ -426,7 +507,7 @@
char* p = &buf[0];
SyncRequest* req_send = reinterpret_cast<SyncRequest*>(p);
- req_send->id = ID_SEND;
+ req_send->id = ID_SEND_V1;
req_send->path_length = path_and_mode.length();
p += sizeof(SyncRequest);
memcpy(p, path_and_mode.data(), path_and_mode.size());
@@ -452,11 +533,92 @@
return true;
}
+ bool SendLargeFileCompressed(const std::string& path, mode_t mode, const std::string& lpath,
+ const std::string& rpath, unsigned mtime) {
+ if (!SendSend2(path, mode, true)) {
+ Error("failed to send ID_SEND_V2 message '%s': %s", path.c_str(), strerror(errno));
+ return false;
+ }
+
+ struct stat st;
+ if (stat(lpath.c_str(), &st) == -1) {
+ Error("cannot stat '%s': %s", lpath.c_str(), strerror(errno));
+ return false;
+ }
+
+ uint64_t total_size = st.st_size;
+ uint64_t bytes_copied = 0;
+
+ unique_fd lfd(adb_open(lpath.c_str(), O_RDONLY | O_CLOEXEC));
+ if (lfd < 0) {
+ Error("opening '%s' locally failed: %s", lpath.c_str(), strerror(errno));
+ return false;
+ }
+
+ syncsendbuf sbuf;
+ sbuf.id = ID_DATA;
+
+ BrotliEncoder<SYNC_DATA_MAX> encoder;
+ bool sending = true;
+ while (sending) {
+ Block input(SYNC_DATA_MAX);
+ int r = adb_read(lfd.get(), input.data(), input.size());
+ if (r < 0) {
+ Error("reading '%s' locally failed: %s", lpath.c_str(), strerror(errno));
+ return false;
+ }
+
+ if (r == 0) {
+ encoder.Finish();
+ } else {
+ input.resize(r);
+ encoder.Append(std::move(input));
+ RecordBytesTransferred(r);
+ bytes_copied += r;
+ ReportProgress(rpath, bytes_copied, total_size);
+ }
+
+ while (true) {
+ Block output;
+ BrotliEncodeResult result = encoder.Encode(&output);
+ if (result == BrotliEncodeResult::Error) {
+ Error("compressing '%s' locally failed", lpath.c_str());
+ return false;
+ }
+
+ if (!output.empty()) {
+ sbuf.size = output.size();
+ memcpy(sbuf.data, output.data(), output.size());
+ WriteOrDie(lpath, rpath, &sbuf, sizeof(SyncRequest) + output.size());
+ }
+
+ if (result == BrotliEncodeResult::Done) {
+ sending = false;
+ break;
+ } else if (result == BrotliEncodeResult::NeedInput) {
+ break;
+ } else if (result == BrotliEncodeResult::MoreOutput) {
+ continue;
+ }
+ }
+ }
+
+ syncmsg msg;
+ msg.data.id = ID_DONE;
+ msg.data.size = mtime;
+ RecordFileSent(lpath, rpath);
+ return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data));
+ }
+
bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath,
- const std::string& rpath, unsigned mtime) {
+ const std::string& rpath, unsigned mtime, bool compressed) {
+ if (compressed && HaveSendRecv2Brotli()) {
+ return SendLargeFileCompressed(path, mode, lpath, rpath, mtime);
+ }
+
std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode);
- if (!SendRequest(ID_SEND, path_and_mode)) {
- Error("failed to send ID_SEND message '%s': %s", path_and_mode.c_str(),
+ if (!SendRequest(ID_SEND_V1, path_and_mode.c_str())) {
+ Error("failed to send ID_SEND_V1 message '%s': %s", path_and_mode.c_str(),
strerror(errno));
return false;
}
@@ -470,7 +632,7 @@
uint64_t total_size = st.st_size;
uint64_t bytes_copied = 0;
- unique_fd lfd(adb_open(lpath.c_str(), O_RDONLY));
+ unique_fd lfd(adb_open(lpath.c_str(), O_RDONLY | O_CLOEXEC));
if (lfd < 0) {
Error("opening '%s' locally failed: %s", lpath.c_str(), strerror(errno));
return false;
@@ -478,8 +640,9 @@
syncsendbuf sbuf;
sbuf.id = ID_DATA;
+
while (true) {
- int bytes_read = adb_read(lfd, sbuf.data, max - sizeof(SyncRequest));
+ int bytes_read = adb_read(lfd, sbuf.data, max);
if (bytes_read == -1) {
Error("reading '%s' locally failed: %s", lpath.c_str(), strerror(errno));
return false;
@@ -492,7 +655,6 @@
RecordBytesTransferred(bytes_read);
bytes_copied += bytes_read;
-
ReportProgress(rpath, bytes_copied, total_size);
}
@@ -676,6 +838,8 @@
FeatureSet features_;
bool have_stat_v2_;
bool have_ls_v2_;
+ bool have_sendrecv_v2_;
+ bool have_sendrecv_v2_brotli_;
TransferLedger global_ledger_;
TransferLedger current_ledger_;
@@ -757,7 +921,7 @@
}
static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::string& rpath,
- unsigned mtime, mode_t mode, bool sync) {
+ unsigned mtime, mode_t mode, bool sync, bool compressed) {
if (sync) {
struct stat st;
if (sync_lstat(sc, rpath, &st)) {
@@ -800,16 +964,16 @@
return false;
}
} else {
- if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime)) {
+ if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compressed)) {
return false;
}
}
return sc.ReadAcknowledgements();
}
-static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath,
- const char* name, uint64_t expected_size) {
- if (!sc.SendRequest(ID_RECV, rpath)) return false;
+static bool sync_recv_v1(SyncConnection& sc, const char* rpath, const char* lpath, const char* name,
+ uint64_t expected_size) {
+ if (!sc.SendRequest(ID_RECV_V1, rpath)) return false;
adb_unlink(lpath);
unique_fd lfd(adb_creat(lpath, 0644));
@@ -862,6 +1026,114 @@
return true;
}
+static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpath, const char* name,
+ uint64_t expected_size) {
+ if (!sc.SendRecv2(rpath)) return false;
+
+ adb_unlink(lpath);
+ unique_fd lfd(adb_creat(lpath, 0644));
+ if (lfd < 0) {
+ sc.Error("cannot create '%s': %s", lpath, strerror(errno));
+ return false;
+ }
+
+ uint64_t bytes_copied = 0;
+
+ Block buffer(SYNC_DATA_MAX);
+ BrotliDecoder decoder(std::span(buffer.data(), buffer.size()));
+ bool reading = true;
+ while (reading) {
+ syncmsg msg;
+ if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) {
+ adb_unlink(lpath);
+ return false;
+ }
+
+ if (msg.data.id == ID_DONE) {
+ adb_unlink(lpath);
+ sc.Error("unexpected ID_DONE");
+ return false;
+ }
+
+ if (msg.data.id != ID_DATA) {
+ adb_unlink(lpath);
+ sc.ReportCopyFailure(rpath, lpath, msg);
+ return false;
+ }
+
+ if (msg.data.size > sc.max) {
+ sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max);
+ adb_unlink(lpath);
+ return false;
+ }
+
+ Block block(msg.data.size);
+ if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) {
+ adb_unlink(lpath);
+ return false;
+ }
+ decoder.Append(std::move(block));
+
+ while (true) {
+ std::span<char> output;
+ BrotliDecodeResult result = decoder.Decode(&output);
+
+ if (result == BrotliDecodeResult::Error) {
+ sc.Error("decompress failed");
+ adb_unlink(lpath);
+ return false;
+ }
+
+ if (!output.empty()) {
+ if (!WriteFdExactly(lfd, output.data(), output.size())) {
+ sc.Error("cannot write '%s': %s", lpath, strerror(errno));
+ adb_unlink(lpath);
+ return false;
+ }
+ }
+
+ bytes_copied += output.size();
+
+ sc.RecordBytesTransferred(msg.data.size);
+ sc.ReportProgress(name != nullptr ? name : rpath, bytes_copied, expected_size);
+
+ if (result == BrotliDecodeResult::NeedInput) {
+ break;
+ } else if (result == BrotliDecodeResult::MoreOutput) {
+ continue;
+ } else if (result == BrotliDecodeResult::Done) {
+ reading = false;
+ break;
+ } else {
+ LOG(FATAL) << "invalid BrotliDecodeResult: " << static_cast<int>(result);
+ }
+ }
+ }
+
+ syncmsg msg;
+ if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) {
+ sc.Error("failed to read ID_DONE");
+ return false;
+ }
+
+ if (msg.data.id != ID_DONE) {
+ sc.Error("unexpected message after transfer: id = %d (expected ID_DONE)", msg.data.id);
+ return false;
+ }
+
+ sc.RecordFilesTransferred(1);
+ return true;
+}
+
+static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, const char* name,
+ uint64_t expected_size, bool compressed) {
+ if (sc.HaveSendRecv2() && compressed) {
+ return sync_recv_v2(sc, rpath, lpath, name, expected_size);
+ } else {
+ return sync_recv_v1(sc, rpath, lpath, name, expected_size);
+ }
+}
+
bool do_sync_ls(const char* path) {
SyncConnection sc;
if (!sc.IsValid()) return false;
@@ -937,9 +1209,8 @@
return true;
}
-static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath,
- std::string rpath, bool check_timestamps,
- bool list_only) {
+static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::string rpath,
+ bool check_timestamps, bool list_only, bool compressed) {
sc.NewTransfer();
// Make sure that both directory paths end in a slash.
@@ -1021,7 +1292,7 @@
if (list_only) {
sc.Println("would push: %s -> %s", ci.lpath.c_str(), ci.rpath.c_str());
} else {
- if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false)) {
+ if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compressed)) {
return false;
}
}
@@ -1036,7 +1307,8 @@
return success;
}
-bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync) {
+bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync,
+ bool compressed) {
SyncConnection sc;
if (!sc.IsValid()) return false;
@@ -1101,7 +1373,7 @@
dst_dir.append(android::base::Basename(src_path));
}
- success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false);
+ success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compressed);
continue;
} else if (!should_push_file(st.st_mode)) {
sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, st.st_mode);
@@ -1122,7 +1394,7 @@
sc.NewTransfer();
sc.SetExpectedTotalBytes(st.st_size);
- success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync);
+ success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compressed);
sc.ReportTransferRate(src_path, TransferDirection::push);
}
@@ -1207,8 +1479,8 @@
return r1 ? r1 : r2;
}
-static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath,
- std::string lpath, bool copy_attrs) {
+static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::string lpath,
+ bool copy_attrs, bool compressed) {
sc.NewTransfer();
// Make sure that both directory paths end in a slash.
@@ -1238,7 +1510,7 @@
continue;
}
- if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size)) {
+ if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compressed)) {
return false;
}
@@ -1255,8 +1527,8 @@
return true;
}
-bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst,
- bool copy_attrs, const char* name) {
+bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs,
+ bool compressed, const char* name) {
SyncConnection sc;
if (!sc.IsValid()) return false;
@@ -1330,7 +1602,7 @@
dst_dir.append(android::base::Basename(src_path));
}
- success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs);
+ success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compressed);
continue;
} else if (!should_pull_file(src_st.st_mode)) {
sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, src_st.st_mode);
@@ -1349,7 +1621,7 @@
sc.NewTransfer();
sc.SetExpectedTotalBytes(src_st.st_size);
- if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size)) {
+ if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compressed)) {
success = false;
continue;
}
@@ -1365,11 +1637,12 @@
return success;
}
-bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only) {
+bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only,
+ bool compressed) {
SyncConnection sc;
if (!sc.IsValid()) return false;
- bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only);
+ bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compressed);
if (!list_only) {
sc.ReportOverallTransferRate(TransferDirection::push);
}
diff --git a/adb/client/file_sync_client.h b/adb/client/file_sync_client.h
index df7f14c..de3f192 100644
--- a/adb/client/file_sync_client.h
+++ b/adb/client/file_sync_client.h
@@ -20,8 +20,10 @@
#include <vector>
bool do_sync_ls(const char* path);
-bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync);
+bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync,
+ bool compressed);
bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs,
- const char* name = nullptr);
+ bool compressed, const char* name = nullptr);
-bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only);
+bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only,
+ bool compressed);
diff --git a/adb/client/incremental.cpp b/adb/client/incremental.cpp
index 6499d46..a9e65dc 100644
--- a/adb/client/incremental.cpp
+++ b/adb/client/incremental.cpp
@@ -41,37 +41,35 @@
static inline int32_t read_int32(borrowed_fd fd) {
int32_t result;
- ReadFully(fd, &result, sizeof(result));
- return result;
-}
-
-static inline int32_t read_be_int32(borrowed_fd fd) {
- return int32_t(be32toh(read_int32(fd)));
+ return ReadFdExactly(fd, &result, sizeof(result)) ? result : -1;
}
static inline void append_int(borrowed_fd fd, std::vector<char>* bytes) {
- int32_t be_val = read_int32(fd);
+ int32_t le_val = read_int32(fd);
auto old_size = bytes->size();
- bytes->resize(old_size + sizeof(be_val));
- memcpy(bytes->data() + old_size, &be_val, sizeof(be_val));
+ bytes->resize(old_size + sizeof(le_val));
+ memcpy(bytes->data() + old_size, &le_val, sizeof(le_val));
}
static inline void append_bytes_with_size(borrowed_fd fd, std::vector<char>* bytes) {
- int32_t be_size = read_int32(fd);
- int32_t size = int32_t(be32toh(be_size));
+ int32_t le_size = read_int32(fd);
+ if (le_size < 0) {
+ return;
+ }
+ int32_t size = int32_t(le32toh(le_size));
auto old_size = bytes->size();
- bytes->resize(old_size + sizeof(be_size) + size);
- memcpy(bytes->data() + old_size, &be_size, sizeof(be_size));
- ReadFully(fd, bytes->data() + old_size + sizeof(be_size), size);
+ bytes->resize(old_size + sizeof(le_size) + size);
+ memcpy(bytes->data() + old_size, &le_size, sizeof(le_size));
+ ReadFdExactly(fd, bytes->data() + old_size + sizeof(le_size), size);
}
static inline std::pair<std::vector<char>, int32_t> read_id_sig_headers(borrowed_fd fd) {
std::vector<char> result;
append_int(fd, &result); // version
- append_bytes_with_size(fd, &result); // verityRootHash
- append_bytes_with_size(fd, &result); // v3Digest
- append_bytes_with_size(fd, &result); // pkcs7SignatureBlock
- auto tree_size = read_be_int32(fd); // size of the verity tree
+ append_bytes_with_size(fd, &result); // hashingInfo
+ append_bytes_with_size(fd, &result); // signingInfo
+ auto le_tree_size = read_int32(fd);
+ auto tree_size = int32_t(le32toh(le_tree_size)); // size of the verity tree
return {std::move(result), tree_size};
}
@@ -197,20 +195,72 @@
auto fd_param = std::to_string(osh);
#endif
+ // pipe for child process to write output
+ int print_fds[2];
+ if (adb_socketpair(print_fds) != 0) {
+ fprintf(stderr, "Failed to create socket pair for child to print to parent\n");
+ return {};
+ }
+ auto [pipe_read_fd, pipe_write_fd] = print_fds;
+ auto pipe_write_fd_param = std::to_string(intptr_t(adb_get_os_handle(pipe_write_fd)));
+ close_on_exec(pipe_read_fd);
+
std::vector<std::string> args(std::move(files));
- args.insert(args.begin(), {"inc-server", fd_param});
- auto child = adb_launch_process(adb_path, std::move(args), {connection_fd.get()});
+ args.insert(args.begin(), {"inc-server", fd_param, pipe_write_fd_param});
+ auto child =
+ adb_launch_process(adb_path, std::move(args), {connection_fd.get(), pipe_write_fd});
if (!child) {
fprintf(stderr, "adb: failed to fork: %s\n", strerror(errno));
return {};
}
+ adb_close(pipe_write_fd);
+
auto killOnExit = [](Process* p) { p->kill(); };
std::unique_ptr<Process, decltype(killOnExit)> serverKiller(&child, killOnExit);
- // TODO: Terminate server process if installation fails.
- serverKiller.release();
+ Result result = wait_for_installation(pipe_read_fd);
+ adb_close(pipe_read_fd);
+
+ if (result == Result::Success) {
+ // adb client exits now but inc-server can continue
+ serverKiller.release();
+ }
return child;
}
+Result wait_for_installation(int read_fd) {
+ static constexpr int maxMessageSize = 256;
+ std::vector<char> child_stdout(CHUNK_SIZE);
+ int bytes_read;
+ int buf_size = 0;
+ // TODO(b/150865433): optimize child's output parsing
+ while ((bytes_read = adb_read(read_fd, child_stdout.data() + buf_size,
+ child_stdout.size() - buf_size)) > 0) {
+ // print to parent's stdout
+ fprintf(stdout, "%.*s", bytes_read, child_stdout.data() + buf_size);
+
+ buf_size += bytes_read;
+ const std::string_view stdout_str(child_stdout.data(), buf_size);
+ // wait till installation either succeeds or fails
+ if (stdout_str.find("Success") != std::string::npos) {
+ return Result::Success;
+ }
+ // on failure, wait for full message
+ static constexpr auto failure_msg_head = "Failure ["sv;
+ if (const auto begin_itr = stdout_str.find(failure_msg_head);
+ begin_itr != std::string::npos) {
+ if (buf_size >= maxMessageSize) {
+ return Result::Failure;
+ }
+ const auto end_itr = stdout_str.rfind("]");
+ if (end_itr != std::string::npos && end_itr >= begin_itr + failure_msg_head.size()) {
+ return Result::Failure;
+ }
+ }
+ child_stdout.resize(buf_size + CHUNK_SIZE);
+ }
+ return Result::None;
+}
+
} // namespace incremental
diff --git a/adb/client/incremental.h b/adb/client/incremental.h
index 4b9f6bd..731e6fb 100644
--- a/adb/client/incremental.h
+++ b/adb/client/incremental.h
@@ -27,4 +27,7 @@
std::optional<Process> install(std::vector<std::string> files);
+enum class Result { Success, Failure, None };
+Result wait_for_installation(int read_fd);
+
} // namespace incremental
diff --git a/adb/client/incremental_server.cpp b/adb/client/incremental_server.cpp
index 2512d05..4b87d0a 100644
--- a/adb/client/incremental_server.cpp
+++ b/adb/client/incremental_server.cpp
@@ -18,13 +18,6 @@
#include "incremental_server.h"
-#include "adb.h"
-#include "adb_io.h"
-#include "adb_trace.h"
-#include "adb_unique_fd.h"
-#include "adb_utils.h"
-#include "sysdeps.h"
-
#include <android-base/endian.h>
#include <android-base/strings.h>
#include <inttypes.h>
@@ -41,29 +34,41 @@
#include <type_traits>
#include <unordered_set>
+#include "adb.h"
+#include "adb_io.h"
+#include "adb_trace.h"
+#include "adb_unique_fd.h"
+#include "adb_utils.h"
+#include "incremental_utils.h"
+#include "sysdeps.h"
+
namespace incremental {
static constexpr int kBlockSize = 4096;
static constexpr int kCompressedSizeMax = kBlockSize * 0.95;
-static constexpr short kCompressionNone = 0;
-static constexpr short kCompressionLZ4 = 1;
+static constexpr int8_t kTypeData = 0;
+static constexpr int8_t kCompressionNone = 0;
+static constexpr int8_t kCompressionLZ4 = 1;
static constexpr int kCompressBound = std::max(kBlockSize, LZ4_COMPRESSBOUND(kBlockSize));
static constexpr auto kReadBufferSize = 128 * 1024;
+static constexpr int kPollTimeoutMillis = 300000; // 5 minutes
using BlockSize = int16_t;
using FileId = int16_t;
using BlockIdx = int32_t;
using NumBlocks = int32_t;
-using CompressionType = int16_t;
+using BlockType = int8_t;
+using CompressionType = int8_t;
using RequestType = int16_t;
using ChunkHeader = int32_t;
using MagicType = uint32_t;
static constexpr MagicType INCR = 0x494e4352; // LE INCR
-static constexpr RequestType EXIT = 0;
+static constexpr RequestType SERVING_COMPLETE = 0;
static constexpr RequestType BLOCK_MISSING = 1;
static constexpr RequestType PREFETCH = 2;
+static constexpr RequestType DESTROY = 3;
static constexpr inline int64_t roundDownToBlockOffset(int64_t val) {
return val & ~(kBlockSize - 1);
@@ -123,7 +128,8 @@
// Placed before actual data bytes of each block
struct ResponseHeader {
FileId file_id; // 2 bytes
- CompressionType compression_type; // 2 bytes
+ BlockType block_type; // 1 byte
+ CompressionType compression_type; // 1 byte
BlockIdx block_idx; // 4 bytes
BlockSize block_size; // 2 bytes
} __attribute__((packed));
@@ -134,6 +140,7 @@
// Plain file
File(const char* filepath, FileId id, int64_t size, unique_fd fd) : File(filepath, id, size) {
this->fd_ = std::move(fd);
+ priority_blocks_ = PriorityBlocksForFile(filepath, fd_.get(), size);
}
int64_t ReadBlock(BlockIdx block_idx, void* buf, bool* is_zip_compressed,
std::string* error) const {
@@ -145,6 +152,7 @@
}
const unique_fd& RawFd() const { return fd_; }
+ const std::vector<BlockIdx>& PriorityBlocks() const { return priority_blocks_; }
std::vector<bool> sentBlocks;
NumBlocks sentBlocksCount = 0;
@@ -158,12 +166,13 @@
sentBlocks.resize(numBytesToNumBlocks(size));
}
unique_fd fd_;
+ std::vector<BlockIdx> priority_blocks_;
};
class IncrementalServer {
public:
- IncrementalServer(unique_fd fd, std::vector<File> files)
- : adb_fd_(std::move(fd)), files_(std::move(files)) {
+ IncrementalServer(unique_fd adb_fd, unique_fd output_fd, std::vector<File> files)
+ : adb_fd_(std::move(adb_fd)), output_fd_(std::move(output_fd)), files_(std::move(files)) {
buffer_.reserve(kReadBufferSize);
}
@@ -174,14 +183,23 @@
const File* file;
BlockIdx overallIndex = 0;
BlockIdx overallEnd = 0;
+ BlockIdx priorityIndex = 0;
- PrefetchState(const File& f) : file(&f), overallEnd((BlockIdx)f.sentBlocks.size()) {}
- PrefetchState(const File& f, BlockIdx start, int count)
+ explicit PrefetchState(const File& f, BlockIdx start, int count)
: file(&f),
overallIndex(start),
overallEnd(std::min<BlockIdx>(start + count, f.sentBlocks.size())) {}
- bool done() const { return overallIndex >= overallEnd; }
+ explicit PrefetchState(const File& f)
+ : PrefetchState(f, 0, (BlockIdx)f.sentBlocks.size()) {}
+
+ bool done() const {
+ const bool overallSent = (overallIndex >= overallEnd);
+ if (file->PriorityBlocks().empty()) {
+ return overallSent;
+ }
+ return overallSent && (priorityIndex >= (BlockIdx)file->PriorityBlocks().size());
+ }
};
bool SkipToRequest(void* buffer, size_t* size, bool blocking);
@@ -197,9 +215,10 @@
void Send(const void* data, size_t size, bool flush);
void Flush();
using TimePoint = decltype(std::chrono::high_resolution_clock::now());
- bool Exit(std::optional<TimePoint> startTime, int missesCount, int missesSent);
+ bool ServingComplete(std::optional<TimePoint> startTime, int missesCount, int missesSent);
unique_fd const adb_fd_;
+ unique_fd const output_fd_;
std::vector<File> files_;
// Incoming data buffer.
@@ -210,6 +229,9 @@
long long sentSize_ = 0;
std::vector<char> pendingBlocks_;
+
+ // True when client notifies that all the data has been received
+ bool servingComplete_;
};
bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking) {
@@ -217,7 +239,8 @@
// Looking for INCR magic.
bool magic_found = false;
int bcur = 0;
- for (int bsize = buffer_.size(); bcur + 4 < bsize; ++bcur) {
+ int bsize = buffer_.size();
+ for (bcur = 0; bcur + 4 < bsize; ++bcur) {
uint32_t magic = be32toh(*(uint32_t*)(buffer_.data() + bcur));
if (magic == INCR) {
magic_found = true;
@@ -226,8 +249,8 @@
}
if (bcur > 0) {
- // Stream the rest to stderr.
- fprintf(stderr, "%.*s", bcur, buffer_.data());
+ // output the rest.
+ WriteFdExactly(output_fd_, buffer_.data(), bcur);
erase_buffer_head(bcur);
}
@@ -239,17 +262,26 @@
}
adb_pollfd pfd = {adb_fd_.get(), POLLIN, 0};
- auto res = adb_poll(&pfd, 1, blocking ? -1 : 0);
+ auto res = adb_poll(&pfd, 1, blocking ? kPollTimeoutMillis : 0);
+
if (res != 1) {
+ WriteFdExactly(output_fd_, buffer_.data(), buffer_.size());
if (res < 0) {
- fprintf(stderr, "Failed to poll: %s\n", strerror(errno));
+ D("Failed to poll: %s\n", strerror(errno));
+ return false;
+ }
+ if (blocking) {
+ fprintf(stderr, "Timed out waiting for data from device.\n");
+ }
+ if (blocking && servingComplete_) {
+ // timeout waiting from client. Serving is complete, so quit.
return false;
}
*size = 0;
return true;
}
- auto bsize = buffer_.size();
+ bsize = buffer_.size();
buffer_.resize(kReadBufferSize);
int r = adb_read(adb_fd_, buffer_.data() + bsize, kReadBufferSize - bsize);
if (r > 0) {
@@ -257,21 +289,19 @@
continue;
}
- if (r == -1) {
- fprintf(stderr, "Failed to read from fd %d: %d. Exit\n", adb_fd_.get(), errno);
- return false;
- }
-
- // socket is closed
- return false;
+ D("Failed to read from fd %d: %d. Exit\n", adb_fd_.get(), errno);
+ break;
}
+ // socket is closed. print remaining messages
+ WriteFdExactly(output_fd_, buffer_.data(), buffer_.size());
+ return false;
}
std::optional<RequestCommand> IncrementalServer::ReadRequest(bool blocking) {
uint8_t commandBuf[sizeof(RequestCommand)];
auto size = sizeof(commandBuf);
if (!SkipToRequest(&commandBuf, &size, blocking)) {
- return {{EXIT}};
+ return {{DESTROY}};
}
if (size < sizeof(RequestCommand)) {
return {};
@@ -316,14 +346,16 @@
++compressed_;
blockSize = compressedSize;
header = reinterpret_cast<ResponseHeader*>(data);
- header->compression_type = toBigEndian(kCompressionLZ4);
+ header->compression_type = kCompressionLZ4;
} else {
++uncompressed_;
blockSize = bytesRead;
header = reinterpret_cast<ResponseHeader*>(raw);
- header->compression_type = toBigEndian(kCompressionNone);
+ header->compression_type = kCompressionNone;
}
+ header->block_type = kTypeData;
+
header->file_id = toBigEndian(fileId);
header->block_size = toBigEndian(blockSize);
header->block_idx = toBigEndian(blockIdx);
@@ -337,6 +369,7 @@
bool IncrementalServer::SendDone() {
ResponseHeader header;
header.file_id = -1;
+ header.block_type = 0;
header.compression_type = 0;
header.block_idx = 0;
header.block_size = 0;
@@ -351,6 +384,17 @@
while (!prefetches_.empty() && blocksToSend > 0) {
auto& prefetch = prefetches_.front();
const auto& file = *prefetch.file;
+ const auto& priority_blocks = file.PriorityBlocks();
+ if (!priority_blocks.empty()) {
+ for (auto& i = prefetch.priorityIndex;
+ blocksToSend > 0 && i < (BlockIdx)priority_blocks.size(); ++i) {
+ if (auto res = SendBlock(file.id, priority_blocks[i]); res == SendResult::Sent) {
+ --blocksToSend;
+ } else if (res == SendResult::Error) {
+ fprintf(stderr, "Failed to send priority block %" PRId32 "\n", i);
+ }
+ }
+ }
for (auto& i = prefetch.overallIndex; blocksToSend > 0 && i < prefetch.overallEnd; ++i) {
if (auto res = SendBlock(file.id, i); res == SendResult::Sent) {
--blocksToSend;
@@ -391,17 +435,17 @@
pendingBlocks_.clear();
}
-bool IncrementalServer::Exit(std::optional<TimePoint> startTime, int missesCount, int missesSent) {
+bool IncrementalServer::ServingComplete(std::optional<TimePoint> startTime, int missesCount,
+ int missesSent) {
+ servingComplete_ = true;
using namespace std::chrono;
auto endTime = high_resolution_clock::now();
- fprintf(stderr,
- "Connection failed or received exit command. Exit.\n"
- "Misses: %d, of those unique: %d; sent compressed: %d, uncompressed: "
- "%d, mb: %.3f\n"
- "Total time taken: %.3fms\n",
- missesCount, missesSent, compressed_, uncompressed_, sentSize_ / 1024.0 / 1024.0,
- duration_cast<microseconds>(endTime - (startTime ? *startTime : endTime)).count() /
- 1000.0);
+ D("Streaming completed.\n"
+ "Misses: %d, of those unique: %d; sent compressed: %d, uncompressed: "
+ "%d, mb: %.3f\n"
+ "Total time taken: %.3fms\n",
+ missesCount, missesSent, compressed_, uncompressed_, sentSize_ / 1024.0 / 1024.0,
+ duration_cast<microseconds>(endTime - (startTime ? *startTime : endTime)).count() / 1000.0);
return true;
}
@@ -425,7 +469,7 @@
std::all_of(files_.begin(), files_.end(), [](const File& f) {
return f.sentBlocksCount == NumBlocks(f.sentBlocks.size());
})) {
- fprintf(stdout, "All files should be loaded. Notifying the device.\n");
+ fprintf(stderr, "All files should be loaded. Notifying the device.\n");
SendDone();
doneSent = true;
}
@@ -446,9 +490,14 @@
BlockIdx blockIdx = request->block_idx;
switch (request->request_type) {
- case EXIT: {
+ case DESTROY: {
// Stop everything.
- return Exit(startTime, missesCount, missesSent);
+ return true;
+ }
+ case SERVING_COMPLETE: {
+ // Not stopping the server here.
+ ServingComplete(startTime, missesCount, missesSent);
+ break;
}
case BLOCK_MISSING: {
++missesCount;
@@ -502,8 +551,9 @@
}
}
-bool serve(int adb_fd, int argc, const char** argv) {
- auto connection_fd = unique_fd(adb_fd);
+bool serve(int connection_fd, int output_fd, int argc, const char** argv) {
+ auto connection_ufd = unique_fd(connection_fd);
+ auto output_ufd = unique_fd(output_fd);
if (argc <= 0) {
error_exit("inc-server: must specify at least one file.");
}
@@ -526,7 +576,7 @@
files.emplace_back(filepath, i, st.st_size, std::move(fd));
}
- IncrementalServer server(std::move(connection_fd), std::move(files));
+ IncrementalServer server(std::move(connection_ufd), std::move(output_ufd), std::move(files));
printf("Serving...\n");
fclose(stdin);
fclose(stdout);
diff --git a/adb/client/incremental_server.h b/adb/client/incremental_server.h
index 53f011e..55b8215 100644
--- a/adb/client/incremental_server.h
+++ b/adb/client/incremental_server.h
@@ -21,6 +21,6 @@
// Expecting arguments like:
// {FILE1 FILE2 ...}
// Where FILE* are files to serve.
-bool serve(int adbFd, int argc, const char** argv);
+bool serve(int connection_fd, int output_fd, int argc, const char** argv);
} // namespace incremental
diff --git a/adb/client/incremental_utils.cpp b/adb/client/incremental_utils.cpp
new file mode 100644
index 0000000..caadb26
--- /dev/null
+++ b/adb/client/incremental_utils.cpp
@@ -0,0 +1,282 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define TRACE_TAG INCREMENTAL
+
+#include "incremental_utils.h"
+
+#include <android-base/mapped_file.h>
+#include <android-base/strings.h>
+#include <ziparchive/zip_archive.h>
+#include <ziparchive/zip_writer.h>
+
+#include <cinttypes>
+#include <numeric>
+#include <unordered_set>
+
+#include "adb_trace.h"
+#include "sysdeps.h"
+
+static constexpr int kBlockSize = 4096;
+
+static constexpr inline int32_t offsetToBlockIndex(int64_t offset) {
+ return (offset & ~(kBlockSize - 1)) >> 12;
+}
+
+template <class T>
+T valueAt(int fd, off64_t offset) {
+ T t;
+ memset(&t, 0, sizeof(T));
+ if (adb_pread(fd, &t, sizeof(T), offset) != sizeof(T)) {
+ memset(&t, -1, sizeof(T));
+ }
+
+ return t;
+}
+
+static void appendBlocks(int32_t start, int count, std::vector<int32_t>* blocks) {
+ if (count == 1) {
+ blocks->push_back(start);
+ } else {
+ auto oldSize = blocks->size();
+ blocks->resize(oldSize + count);
+ std::iota(blocks->begin() + oldSize, blocks->end(), start);
+ }
+}
+
+template <class T>
+static void unduplicate(std::vector<T>& v) {
+ std::unordered_set<T> uniques(v.size());
+ v.erase(std::remove_if(v.begin(), v.end(),
+ [&uniques](T t) { return !uniques.insert(t).second; }),
+ v.end());
+}
+
+static off64_t CentralDirOffset(int fd, int64_t fileSize) {
+ static constexpr int kZipEocdRecMinSize = 22;
+ static constexpr int32_t kZipEocdRecSig = 0x06054b50;
+ static constexpr int kZipEocdCentralDirSizeFieldOffset = 12;
+ static constexpr int kZipEocdCommentLengthFieldOffset = 20;
+
+ int32_t sigBuf = 0;
+ off64_t eocdOffset = -1;
+ off64_t maxEocdOffset = fileSize - kZipEocdRecMinSize;
+ int16_t commentLenBuf = 0;
+
+ // Search from the end of zip, backward to find beginning of EOCD
+ for (int16_t commentLen = 0; commentLen < fileSize; ++commentLen) {
+ sigBuf = valueAt<int32_t>(fd, maxEocdOffset - commentLen);
+ if (sigBuf == kZipEocdRecSig) {
+ commentLenBuf = valueAt<int16_t>(
+ fd, maxEocdOffset - commentLen + kZipEocdCommentLengthFieldOffset);
+ if (commentLenBuf == commentLen) {
+ eocdOffset = maxEocdOffset - commentLen;
+ break;
+ }
+ }
+ }
+
+ if (eocdOffset < 0) {
+ return -1;
+ }
+
+ off64_t cdLen = static_cast<int64_t>(
+ valueAt<int32_t>(fd, eocdOffset + kZipEocdCentralDirSizeFieldOffset));
+
+ return eocdOffset - cdLen;
+}
+
+// Does not support APKs larger than 4GB
+static off64_t SignerBlockOffset(int fd, int64_t fileSize) {
+ static constexpr int kApkSigBlockMinSize = 32;
+ static constexpr int kApkSigBlockFooterSize = 24;
+ static constexpr int64_t APK_SIG_BLOCK_MAGIC_HI = 0x3234206b636f6c42l;
+ static constexpr int64_t APK_SIG_BLOCK_MAGIC_LO = 0x20676953204b5041l;
+
+ off64_t cdOffset = CentralDirOffset(fd, fileSize);
+ if (cdOffset < 0) {
+ return -1;
+ }
+ // CD offset is where original signer block ends. Search backwards for magic and footer.
+ if (cdOffset < kApkSigBlockMinSize ||
+ valueAt<int64_t>(fd, cdOffset - 2 * sizeof(int64_t)) != APK_SIG_BLOCK_MAGIC_LO ||
+ valueAt<int64_t>(fd, cdOffset - sizeof(int64_t)) != APK_SIG_BLOCK_MAGIC_HI) {
+ return -1;
+ }
+ int32_t signerSizeInFooter = valueAt<int32_t>(fd, cdOffset - kApkSigBlockFooterSize);
+ off64_t signerBlockOffset = cdOffset - signerSizeInFooter - sizeof(int64_t);
+ if (signerBlockOffset < 0) {
+ return -1;
+ }
+ int32_t signerSizeInHeader = valueAt<int32_t>(fd, signerBlockOffset);
+ if (signerSizeInFooter != signerSizeInHeader) {
+ return -1;
+ }
+
+ return signerBlockOffset;
+}
+
+static std::vector<int32_t> ZipPriorityBlocks(off64_t signerBlockOffset, int64_t fileSize) {
+ int32_t signerBlockIndex = offsetToBlockIndex(signerBlockOffset);
+ int32_t lastBlockIndex = offsetToBlockIndex(fileSize);
+ const auto numPriorityBlocks = lastBlockIndex - signerBlockIndex + 1;
+
+ std::vector<int32_t> zipPriorityBlocks;
+
+ // Some magic here: most of zip libraries perform a scan for EOCD record starting at the offset
+ // of a maximum comment size from the end of the file. This means the last 65-ish KBs will be
+ // accessed first, followed by the rest of the central directory blocks. Make sure we
+ // send the data in the proper order, as central directory can be quite big by itself.
+ static constexpr auto kMaxZipCommentSize = 64 * 1024;
+ static constexpr auto kNumBlocksInEocdSearch = kMaxZipCommentSize / kBlockSize + 1;
+ if (numPriorityBlocks > kNumBlocksInEocdSearch) {
+ appendBlocks(lastBlockIndex - kNumBlocksInEocdSearch + 1, kNumBlocksInEocdSearch,
+ &zipPriorityBlocks);
+ appendBlocks(signerBlockIndex, numPriorityBlocks - kNumBlocksInEocdSearch,
+ &zipPriorityBlocks);
+ } else {
+ appendBlocks(signerBlockIndex, numPriorityBlocks, &zipPriorityBlocks);
+ }
+
+ // Somehow someone keeps accessing the start of the archive, even if there's nothing really
+ // interesting there...
+ appendBlocks(0, 1, &zipPriorityBlocks);
+ return zipPriorityBlocks;
+}
+
+[[maybe_unused]] static ZipArchiveHandle openZipArchiveFd(int fd) {
+ bool transferFdOwnership = false;
+#ifdef _WIN32
+ //
+ // Need to create a special CRT FD here as the current one is not compatible with
+ // normal read()/write() calls that libziparchive uses.
+ // To make this work we have to create a copy of the file handle, as CRT doesn't care
+ // and closes it together with the new descriptor.
+ //
+ // Note: don't move this into a helper function, it's better to be hard to reuse because
+ // the code is ugly and won't work unless it's a last resort.
+ //
+ auto handle = adb_get_os_handle(fd);
+ HANDLE dupedHandle;
+ if (!::DuplicateHandle(::GetCurrentProcess(), handle, ::GetCurrentProcess(), &dupedHandle, 0,
+ false, DUPLICATE_SAME_ACCESS)) {
+ D("%s failed at DuplicateHandle: %d", __func__, (int)::GetLastError());
+ return {};
+ }
+ fd = _open_osfhandle((intptr_t)dupedHandle, _O_RDONLY | _O_BINARY);
+ if (fd < 0) {
+ D("%s failed at _open_osfhandle: %d", __func__, errno);
+ ::CloseHandle(handle);
+ return {};
+ }
+ transferFdOwnership = true;
+#endif
+ ZipArchiveHandle zip;
+ if (OpenArchiveFd(fd, "apk_fd", &zip, transferFdOwnership) != 0) {
+ D("%s failed at OpenArchiveFd: %d", __func__, errno);
+#ifdef _WIN32
+ // "_close()" is a secret WinCRT name for the regular close() function.
+ _close(fd);
+#endif
+ return {};
+ }
+ return zip;
+}
+
+static std::pair<ZipArchiveHandle, std::unique_ptr<android::base::MappedFile>> openZipArchive(
+ int fd, int64_t fileSize) {
+#ifndef __LP64__
+ if (fileSize >= INT_MAX) {
+ return {openZipArchiveFd(fd), nullptr};
+ }
+#endif
+ auto mapping =
+ android::base::MappedFile::FromOsHandle(adb_get_os_handle(fd), 0, fileSize, PROT_READ);
+ if (!mapping) {
+ D("%s failed at FromOsHandle: %d", __func__, errno);
+ return {};
+ }
+ ZipArchiveHandle zip;
+ if (OpenArchiveFromMemory(mapping->data(), mapping->size(), "apk_mapping", &zip) != 0) {
+ D("%s failed at OpenArchiveFromMemory: %d", __func__, errno);
+ return {};
+ }
+ return {zip, std::move(mapping)};
+}
+
+// TODO(b/151676293): avoid using libziparchive as it reads local file headers
+// which causes additional performance cost. Instead, only read from central directory.
+static std::vector<int32_t> InstallationPriorityBlocks(int fd, int64_t fileSize) {
+ auto [zip, _] = openZipArchive(fd, fileSize);
+ if (!zip) {
+ return {};
+ }
+
+ void* cookie = nullptr;
+ if (StartIteration(zip, &cookie) != 0) {
+ D("%s failed at StartIteration: %d", __func__, errno);
+ return {};
+ }
+
+ std::vector<int32_t> installationPriorityBlocks;
+ ZipEntry entry;
+ std::string_view entryName;
+ while (Next(cookie, &entry, &entryName) == 0) {
+ if (entryName == "resources.arsc" || entryName == "AndroidManifest.xml" ||
+ entryName.starts_with("lib/")) {
+ // Full entries are needed for installation
+ off64_t entryStartOffset = entry.offset;
+ off64_t entryEndOffset =
+ entryStartOffset +
+ (entry.method == kCompressStored ? entry.uncompressed_length
+ : entry.compressed_length) +
+ (entry.has_data_descriptor ? 16 /* sizeof(DataDescriptor) */ : 0);
+ int32_t startBlockIndex = offsetToBlockIndex(entryStartOffset);
+ int32_t endBlockIndex = offsetToBlockIndex(entryEndOffset);
+ int32_t numNewBlocks = endBlockIndex - startBlockIndex + 1;
+ appendBlocks(startBlockIndex, numNewBlocks, &installationPriorityBlocks);
+ D("\tadding to priority blocks: '%.*s'", (int)entryName.size(), entryName.data());
+ } else if (entryName == "classes.dex") {
+ // Only the head is needed for installation
+ int32_t startBlockIndex = offsetToBlockIndex(entry.offset);
+ appendBlocks(startBlockIndex, 1, &installationPriorityBlocks);
+ }
+ }
+
+ EndIteration(cookie);
+ CloseArchive(zip);
+ return installationPriorityBlocks;
+}
+
+namespace incremental {
+std::vector<int32_t> PriorityBlocksForFile(const std::string& filepath, int fd, int64_t fileSize) {
+ if (!android::base::EndsWithIgnoreCase(filepath, ".apk")) {
+ return {};
+ }
+ off64_t signerOffset = SignerBlockOffset(fd, fileSize);
+ if (signerOffset < 0) {
+ // No signer block? not a valid APK
+ return {};
+ }
+ std::vector<int32_t> priorityBlocks = ZipPriorityBlocks(signerOffset, fileSize);
+ std::vector<int32_t> installationPriorityBlocks = InstallationPriorityBlocks(fd, fileSize);
+
+ priorityBlocks.insert(priorityBlocks.end(), installationPriorityBlocks.begin(),
+ installationPriorityBlocks.end());
+ unduplicate(priorityBlocks);
+ return priorityBlocks;
+}
+} // namespace incremental
diff --git a/adb/client/incremental_utils.h b/adb/client/incremental_utils.h
new file mode 100644
index 0000000..8bcf6c0
--- /dev/null
+++ b/adb/client/incremental_utils.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+#include <string>
+#include <vector>
+
+namespace incremental {
+std::vector<int32_t> PriorityBlocksForFile(const std::string& filepath, int fd, int64_t fileSize);
+} // namespace incremental
\ No newline at end of file
diff --git a/adb/client/pairing/pairing_client.cpp b/adb/client/pairing/pairing_client.cpp
index 2f878bf..04bbceb 100644
--- a/adb/client/pairing/pairing_client.cpp
+++ b/adb/client/pairing/pairing_client.cpp
@@ -141,7 +141,12 @@
cert_.size(), priv_key_.data(), priv_key_.size()));
CHECK(connection_);
- if (!pairing_connection_start(connection_.get(), fd.release(), OnPairingResult, this)) {
+#ifdef _WIN32
+ int osh = cast_handle_to_int(adb_get_os_handle(fd.release()));
+#else
+ int osh = adb_get_os_handle(fd.release());
+#endif
+ if (!pairing_connection_start(connection_.get(), osh, OnPairingResult, this)) {
LOG(ERROR) << "PairingClient failed to start the PairingConnection";
state_ = State::Stopped;
return false;
diff --git a/adb/daemon/file_sync_service.cpp b/adb/daemon/file_sync_service.cpp
index edf5683..07f6e65 100644
--- a/adb/daemon/file_sync_service.cpp
+++ b/adb/daemon/file_sync_service.cpp
@@ -32,6 +32,8 @@
#include <utime.h>
#include <memory>
+#include <optional>
+#include <span>
#include <string>
#include <vector>
@@ -55,10 +57,12 @@
#include "adb_io.h"
#include "adb_trace.h"
#include "adb_utils.h"
+#include "brotli_utils.h"
#include "file_sync_protocol.h"
#include "security_log_tags.h"
#include "sysdeps/errno.h"
+using android::base::borrowed_fd;
using android::base::Dirname;
using android::base::StringPrintf;
@@ -249,7 +253,7 @@
// Make sure that SendFail from adb_io.cpp isn't accidentally used in this file.
#pragma GCC poison SendFail
-static bool SendSyncFail(int fd, const std::string& reason) {
+static bool SendSyncFail(borrowed_fd fd, const std::string& reason) {
D("sync: failure: %s", reason.c_str());
syncmsg msg;
@@ -258,13 +262,89 @@
return WriteFdExactly(fd, &msg.data, sizeof(msg.data)) && WriteFdExactly(fd, reason);
}
-static bool SendSyncFailErrno(int fd, const std::string& reason) {
+static bool SendSyncFailErrno(borrowed_fd fd, const std::string& reason) {
return SendSyncFail(fd, StringPrintf("%s: %s", reason.c_str(), strerror(errno)));
}
-static bool handle_send_file(int s, const char* path, uint32_t* timestamp, uid_t uid, gid_t gid,
- uint64_t capabilities, mode_t mode, std::vector<char>& buffer,
- bool do_unlink) {
+static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp) {
+ syncmsg msg;
+ Block decode_buffer(SYNC_DATA_MAX);
+ BrotliDecoder decoder(std::span(decode_buffer.data(), decode_buffer.size()));
+ while (true) {
+ if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false;
+
+ if (msg.data.id != ID_DATA) {
+ if (msg.data.id == ID_DONE) {
+ *timestamp = msg.data.size;
+ return true;
+ }
+ SendSyncFail(s, "invalid data message");
+ return false;
+ }
+
+ Block block(msg.data.size);
+ if (!ReadFdExactly(s, block.data(), msg.data.size)) return false;
+ decoder.Append(std::move(block));
+
+ while (true) {
+ std::span<char> output;
+ BrotliDecodeResult result = decoder.Decode(&output);
+ if (result == BrotliDecodeResult::Error) {
+ SendSyncFailErrno(s, "decompress failed");
+ return false;
+ }
+
+ if (!WriteFdExactly(fd, output.data(), output.size())) {
+ SendSyncFailErrno(s, "write failed");
+ return false;
+ }
+
+ if (result == BrotliDecodeResult::NeedInput) {
+ break;
+ } else if (result == BrotliDecodeResult::MoreOutput) {
+ continue;
+ } else if (result == BrotliDecodeResult::Done) {
+ break;
+ } else {
+ LOG(FATAL) << "invalid BrotliDecodeResult: " << static_cast<int>(result);
+ }
+ }
+ }
+
+ __builtin_unreachable();
+}
+
+static bool handle_send_file_uncompressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp,
+ std::vector<char>& buffer) {
+ syncmsg msg;
+
+ while (true) {
+ if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false;
+
+ if (msg.data.id != ID_DATA) {
+ if (msg.data.id == ID_DONE) {
+ *timestamp = msg.data.size;
+ return true;
+ }
+ SendSyncFail(s, "invalid data message");
+ return false;
+ }
+
+ if (msg.data.size > buffer.size()) { // TODO: resize buffer?
+ SendSyncFail(s, "oversize data message");
+ return false;
+ }
+ if (!ReadFdExactly(s, &buffer[0], msg.data.size)) return false;
+ if (!WriteFdExactly(fd, &buffer[0], msg.data.size)) {
+ SendSyncFailErrno(s, "write failed");
+ return false;
+ }
+ }
+}
+
+static bool handle_send_file(borrowed_fd s, const char* path, uint32_t* timestamp, uid_t uid,
+ gid_t gid, uint64_t capabilities, mode_t mode, bool compressed,
+ std::vector<char>& buffer, bool do_unlink) {
int rc;
syncmsg msg;
@@ -302,45 +382,33 @@
fchmod(fd.get(), mode);
}
- rc = posix_fadvise(fd.get(), 0, 0,
- POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED);
- if (rc != 0) {
- D("[ Failed to fadvise: %s ]", strerror(rc));
- }
-
- while (true) {
- if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) goto fail;
-
- if (msg.data.id != ID_DATA) {
- if (msg.data.id == ID_DONE) {
- *timestamp = msg.data.size;
- break;
- }
- SendSyncFail(s, "invalid data message");
- goto abort;
+ {
+ rc = posix_fadvise(fd.get(), 0, 0,
+ POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED);
+ if (rc != 0) {
+ D("[ Failed to fadvise: %s ]", strerror(rc));
}
- if (msg.data.size > buffer.size()) { // TODO: resize buffer?
- SendSyncFail(s, "oversize data message");
- goto abort;
+ bool result;
+ if (compressed) {
+ result = handle_send_file_compressed(s, std::move(fd), timestamp);
+ } else {
+ result = handle_send_file_uncompressed(s, std::move(fd), timestamp, buffer);
}
- if (!ReadFdExactly(s, &buffer[0], msg.data.size)) goto abort;
-
- if (!WriteFdExactly(fd.get(), &buffer[0], msg.data.size)) {
- SendSyncFailErrno(s, "write failed");
+ if (!result) {
goto fail;
}
- }
- if (!update_capabilities(path, capabilities)) {
- SendSyncFailErrno(s, "update_capabilities failed");
- goto fail;
- }
+ if (!update_capabilities(path, capabilities)) {
+ SendSyncFailErrno(s, "update_capabilities failed");
+ goto fail;
+ }
- msg.status.id = ID_OKAY;
- msg.status.msglen = 0;
- return WriteFdExactly(s, &msg.status, sizeof(msg.status));
+ msg.status.id = ID_OKAY;
+ msg.status.msglen = 0;
+ return WriteFdExactly(s, &msg.status, sizeof(msg.status));
+ }
fail:
// If there's a problem on the device, we'll send an ID_FAIL message and
@@ -371,7 +439,6 @@
if (!ReadFdExactly(s, &buffer[0], msg.data.size)) break;
}
-abort:
if (do_unlink) adb_unlink(path);
return false;
}
@@ -432,23 +499,8 @@
}
#endif
-static bool do_send(int s, const std::string& spec, std::vector<char>& buffer) {
- // 'spec' is of the form "/some/path,0755". Break it up.
- size_t comma = spec.find_last_of(',');
- if (comma == std::string::npos) {
- SendSyncFail(s, "missing , in ID_SEND");
- return false;
- }
-
- std::string path = spec.substr(0, comma);
-
- errno = 0;
- mode_t mode = strtoul(spec.substr(comma + 1).c_str(), nullptr, 0);
- if (errno != 0) {
- SendSyncFail(s, "bad mode");
- return false;
- }
-
+static bool send_impl(int s, const std::string& path, mode_t mode, bool compressed,
+ std::vector<char>& buffer) {
// Don't delete files before copying if they are not "regular" or symlinks.
struct stat st;
bool do_unlink = (lstat(path.c_str(), &st) == -1) || S_ISREG(st.st_mode) ||
@@ -474,8 +526,8 @@
adbd_fs_config(path.c_str(), 0, nullptr, &uid, &gid, &mode, &capabilities);
}
- result = handle_send_file(s, path.c_str(), ×tamp, uid, gid, capabilities, mode, buffer,
- do_unlink);
+ result = handle_send_file(s, path.c_str(), ×tamp, uid, gid, capabilities, mode,
+ compressed, buffer, do_unlink);
}
if (!result) {
@@ -491,7 +543,125 @@
return true;
}
-static bool do_recv(int s, const char* path, std::vector<char>& buffer) {
+static bool do_send_v1(int s, const std::string& spec, std::vector<char>& buffer) {
+ // 'spec' is of the form "/some/path,0755". Break it up.
+ size_t comma = spec.find_last_of(',');
+ if (comma == std::string::npos) {
+ SendSyncFail(s, "missing , in ID_SEND_V1");
+ return false;
+ }
+
+ std::string path = spec.substr(0, comma);
+
+ errno = 0;
+ mode_t mode = strtoul(spec.substr(comma + 1).c_str(), nullptr, 0);
+ if (errno != 0) {
+ SendSyncFail(s, "bad mode");
+ return false;
+ }
+
+ return send_impl(s, path, mode, false, buffer);
+}
+
+static bool do_send_v2(int s, const std::string& path, std::vector<char>& buffer) {
+ // Read the setup packet.
+ syncmsg msg;
+ int rc = ReadFdExactly(s, &msg.send_v2_setup, sizeof(msg.send_v2_setup));
+ if (rc == 0) {
+ LOG(ERROR) << "failed to read send_v2 setup packet: EOF";
+ return false;
+ } else if (rc < 0) {
+ PLOG(ERROR) << "failed to read send_v2 setup packet";
+ }
+
+ bool compressed = false;
+ if (msg.send_v2_setup.flags & kSyncFlagBrotli) {
+ msg.send_v2_setup.flags &= ~kSyncFlagBrotli;
+ compressed = true;
+ }
+ if (msg.send_v2_setup.flags) {
+ SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.send_v2_setup.flags));
+ return false;
+ }
+
+ errno = 0;
+ return send_impl(s, path, msg.send_v2_setup.mode, compressed, buffer);
+}
+
+static bool recv_uncompressed(borrowed_fd s, unique_fd fd, std::vector<char>& buffer) {
+ syncmsg msg;
+ msg.data.id = ID_DATA;
+ std::optional<BrotliEncoder<SYNC_DATA_MAX>> encoder;
+ while (true) {
+ int r = adb_read(fd.get(), &buffer[0], buffer.size() - sizeof(msg.data));
+ if (r <= 0) {
+ if (r == 0) break;
+ SendSyncFailErrno(s, "read failed");
+ return false;
+ }
+ msg.data.size = r;
+
+ if (!WriteFdExactly(s, &msg.data, sizeof(msg.data)) || !WriteFdExactly(s, &buffer[0], r)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static bool recv_compressed(borrowed_fd s, unique_fd fd) {
+ syncmsg msg;
+ msg.data.id = ID_DATA;
+
+ BrotliEncoder<SYNC_DATA_MAX> encoder;
+
+ bool sending = true;
+ while (sending) {
+ Block input(SYNC_DATA_MAX);
+ int r = adb_read(fd.get(), input.data(), input.size());
+ if (r < 0) {
+ SendSyncFailErrno(s, "read failed");
+ return false;
+ }
+
+ if (r == 0) {
+ encoder.Finish();
+ } else {
+ input.resize(r);
+ encoder.Append(std::move(input));
+ }
+
+ while (true) {
+ Block output;
+ BrotliEncodeResult result = encoder.Encode(&output);
+ if (result == BrotliEncodeResult::Error) {
+ SendSyncFailErrno(s, "compress failed");
+ return false;
+ }
+
+ if (!output.empty()) {
+ msg.data.size = output.size();
+ if (!WriteFdExactly(s, &msg.data, sizeof(msg.data)) ||
+ !WriteFdExactly(s, output.data(), output.size())) {
+ return false;
+ }
+ }
+
+ if (result == BrotliEncodeResult::Done) {
+ sending = false;
+ break;
+ } else if (result == BrotliEncodeResult::NeedInput) {
+ break;
+ } else if (result == BrotliEncodeResult::MoreOutput) {
+ continue;
+ }
+ }
+ }
+
+ return true;
+}
+
+static bool recv_impl(borrowed_fd s, const char* path, bool compressed, std::vector<char>& buffer) {
__android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path);
unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC));
@@ -505,26 +675,51 @@
D("[ Failed to fadvise: %s ]", strerror(rc));
}
- syncmsg msg;
- msg.data.id = ID_DATA;
- while (true) {
- int r = adb_read(fd.get(), &buffer[0], buffer.size() - sizeof(msg.data));
- if (r <= 0) {
- if (r == 0) break;
- SendSyncFailErrno(s, "read failed");
- return false;
- }
- msg.data.size = r;
- if (!WriteFdExactly(s, &msg.data, sizeof(msg.data)) || !WriteFdExactly(s, &buffer[0], r)) {
- return false;
- }
+ bool result;
+ if (compressed) {
+ result = recv_compressed(s, std::move(fd));
+ } else {
+ result = recv_uncompressed(s, std::move(fd), buffer);
}
+ if (!result) {
+ return false;
+ }
+
+ syncmsg msg;
msg.data.id = ID_DONE;
msg.data.size = 0;
return WriteFdExactly(s, &msg.data, sizeof(msg.data));
}
+static bool do_recv_v1(borrowed_fd s, const char* path, std::vector<char>& buffer) {
+ return recv_impl(s, path, false, buffer);
+}
+
+static bool do_recv_v2(borrowed_fd s, const char* path, std::vector<char>& buffer) {
+ syncmsg msg;
+ // Read the setup packet.
+ int rc = ReadFdExactly(s, &msg.recv_v2_setup, sizeof(msg.recv_v2_setup));
+ if (rc == 0) {
+ LOG(ERROR) << "failed to read recv_v2 setup packet: EOF";
+ return false;
+ } else if (rc < 0) {
+ PLOG(ERROR) << "failed to read recv_v2 setup packet";
+ }
+
+ bool compressed = false;
+ if (msg.recv_v2_setup.flags & kSyncFlagBrotli) {
+ msg.recv_v2_setup.flags &= ~kSyncFlagBrotli;
+ compressed = true;
+ }
+ if (msg.recv_v2_setup.flags) {
+ SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.recv_v2_setup.flags));
+ return false;
+ }
+
+ return recv_impl(s, path, compressed, buffer);
+}
+
static const char* sync_id_to_name(uint32_t id) {
switch (id) {
case ID_LSTAT_V1:
@@ -537,10 +732,14 @@
return "list_v1";
case ID_LIST_V2:
return "list_v2";
- case ID_SEND:
- return "send";
- case ID_RECV:
- return "recv";
+ case ID_SEND_V1:
+ return "send_v1";
+ case ID_SEND_V2:
+ return "send_v2";
+ case ID_RECV_V1:
+ return "recv_v1";
+ case ID_RECV_V2:
+ return "recv_v2";
case ID_QUIT:
return "quit";
default:
@@ -585,11 +784,17 @@
case ID_LIST_V2:
if (!do_list_v2(fd, name)) return false;
break;
- case ID_SEND:
- if (!do_send(fd, name, buffer)) return false;
+ case ID_SEND_V1:
+ if (!do_send_v1(fd, name, buffer)) return false;
break;
- case ID_RECV:
- if (!do_recv(fd, name, buffer)) return false;
+ case ID_SEND_V2:
+ if (!do_send_v2(fd, name, buffer)) return false;
+ break;
+ case ID_RECV_V1:
+ if (!do_recv_v1(fd, name, buffer)) return false;
+ break;
+ case ID_RECV_V2:
+ if (!do_recv_v2(fd, name, buffer)) return false;
break;
case ID_QUIT:
return false;
diff --git a/adb/fdevent/fdevent.cpp b/adb/fdevent/fdevent.cpp
index 562f587..fd55020 100644
--- a/adb/fdevent/fdevent.cpp
+++ b/adb/fdevent/fdevent.cpp
@@ -63,7 +63,10 @@
int fd_num = fd.get();
- fdevent* fde = new fdevent();
+ auto [it, inserted] = this->installed_fdevents_.emplace(fd_num, fdevent{});
+ CHECK(inserted);
+
+ fdevent* fde = &it->second;
fde->id = fdevent_id_++;
fde->state = 0;
fde->fd = std::move(fd);
@@ -76,10 +79,6 @@
LOG(ERROR) << "failed to set non-blocking mode for fd " << fde->fd.get();
}
- auto [it, inserted] = this->installed_fdevents_.emplace(fd_num, fde);
- CHECK(inserted);
- UNUSED(it);
-
this->Register(fde);
return fde;
}
@@ -92,12 +91,12 @@
this->Unregister(fde);
- auto erased = this->installed_fdevents_.erase(fde->fd.get());
+ unique_fd fd = std::move(fde->fd);
+
+ auto erased = this->installed_fdevents_.erase(fd.get());
CHECK_EQ(1UL, erased);
- unique_fd result = std::move(fde->fd);
- delete fde;
- return result;
+ return fd;
}
void fdevent_context::Add(fdevent* fde, unsigned events) {
@@ -123,9 +122,9 @@
for (const auto& [fd, fde] : this->installed_fdevents_) {
UNUSED(fd);
- auto timeout_opt = fde->timeout;
+ auto timeout_opt = fde.timeout;
if (timeout_opt) {
- auto deadline = fde->last_active + *timeout_opt;
+ auto deadline = fde.last_active + *timeout_opt;
auto time_left = duration_cast<std::chrono::milliseconds>(deadline - now);
if (time_left < 0ms) {
time_left = 0ms;
@@ -194,11 +193,13 @@
#endif
}
-static auto& g_ambient_fdevent_context =
- *new std::unique_ptr<fdevent_context>(fdevent_create_context());
+static auto& g_ambient_fdevent_context() {
+ static auto context = fdevent_create_context().release();
+ return context;
+}
static fdevent_context* fdevent_get_ambient() {
- return g_ambient_fdevent_context.get();
+ return g_ambient_fdevent_context();
}
fdevent* fdevent_create(int fd, fd_func func, void* arg) {
@@ -256,5 +257,6 @@
}
void fdevent_reset() {
- g_ambient_fdevent_context = fdevent_create_context();
+ auto old = std::exchange(g_ambient_fdevent_context(), fdevent_create_context().release());
+ delete old;
}
diff --git a/adb/fdevent/fdevent.h b/adb/fdevent/fdevent.h
index 86814d7..9fc3b2c 100644
--- a/adb/fdevent/fdevent.h
+++ b/adb/fdevent/fdevent.h
@@ -52,6 +52,20 @@
unsigned events;
};
+struct fdevent final {
+ uint64_t id;
+
+ unique_fd fd;
+ int force_eof = 0;
+
+ uint16_t state = 0;
+ std::optional<std::chrono::milliseconds> timeout;
+ std::chrono::steady_clock::time_point last_active;
+
+ std::variant<fd_func, fd_func2> func;
+ void* arg = nullptr;
+};
+
struct fdevent_context {
public:
virtual ~fdevent_context() = default;
@@ -113,7 +127,7 @@
std::atomic<bool> terminate_loop_ = false;
protected:
- std::unordered_map<int, fdevent*> installed_fdevents_;
+ std::unordered_map<int, fdevent> installed_fdevents_;
private:
uint64_t fdevent_id_ = 0;
@@ -121,20 +135,6 @@
std::deque<std::function<void()>> run_queue_ GUARDED_BY(run_queue_mutex_);
};
-struct fdevent {
- uint64_t id;
-
- unique_fd fd;
- int force_eof = 0;
-
- uint16_t state = 0;
- std::optional<std::chrono::milliseconds> timeout;
- std::chrono::steady_clock::time_point last_active;
-
- std::variant<fd_func, fd_func2> func;
- void* arg = nullptr;
-};
-
// Backwards compatibility shims that forward to the global fdevent_context.
fdevent* fdevent_create(int fd, fd_func func, void* arg);
fdevent* fdevent_create(int fd, fd_func2 func, void* arg);
diff --git a/adb/fdevent/fdevent_epoll.cpp b/adb/fdevent/fdevent_epoll.cpp
index e3d1674..4ef41d1 100644
--- a/adb/fdevent/fdevent_epoll.cpp
+++ b/adb/fdevent/fdevent_epoll.cpp
@@ -155,15 +155,15 @@
event_map[fde] = events;
}
- for (const auto& [fd, fde] : installed_fdevents_) {
+ for (auto& [fd, fde] : installed_fdevents_) {
unsigned events = 0;
- if (auto it = event_map.find(fde); it != event_map.end()) {
+ if (auto it = event_map.find(&fde); it != event_map.end()) {
events = it->second;
}
if (events == 0) {
- if (fde->timeout) {
- auto deadline = fde->last_active + *fde->timeout;
+ if (fde.timeout) {
+ auto deadline = fde.last_active + *fde.timeout;
if (deadline < post_poll) {
events |= FDE_TIMEOUT;
}
@@ -171,13 +171,13 @@
}
if (events != 0) {
- LOG(DEBUG) << dump_fde(fde) << " got events " << std::hex << std::showbase
+ LOG(DEBUG) << dump_fde(&fde) << " got events " << std::hex << std::showbase
<< events;
- fde_events.push_back({fde, events});
- fde->last_active = post_poll;
+ fde_events.push_back({&fde, events});
+ fde.last_active = post_poll;
}
}
- this->HandleEvents(std::move(fde_events));
+ this->HandleEvents(fde_events);
fde_events.clear();
}
diff --git a/adb/fdevent/fdevent_epoll.h b/adb/fdevent/fdevent_epoll.h
index 684fa32..6214d2e 100644
--- a/adb/fdevent/fdevent_epoll.h
+++ b/adb/fdevent/fdevent_epoll.h
@@ -47,12 +47,7 @@
protected:
virtual void Interrupt() final;
- public:
- // All operations to fdevent should happen only in the main thread.
- // That's why we don't need a lock for fdevent.
- std::unordered_map<int, fdevent*> epoll_node_map_;
- std::list<fdevent*> pending_list_;
-
+ private:
unique_fd epoll_fd_;
unique_fd interrupt_fd_;
fdevent* interrupt_fde_ = nullptr;
diff --git a/adb/fdevent/fdevent_poll.cpp b/adb/fdevent/fdevent_poll.cpp
index cc4a7a1..ac86c08 100644
--- a/adb/fdevent/fdevent_poll.cpp
+++ b/adb/fdevent/fdevent_poll.cpp
@@ -103,24 +103,27 @@
void fdevent_context_poll::Loop() {
main_thread_id_ = android::base::GetThreadId();
+ std::vector<adb_pollfd> pollfds;
+ std::vector<fdevent_event> poll_events;
+
while (true) {
if (terminate_loop_) {
break;
}
D("--- --- waiting for events");
- std::vector<adb_pollfd> pollfds;
+ pollfds.clear();
for (const auto& [fd, fde] : this->installed_fdevents_) {
adb_pollfd pfd;
pfd.fd = fd;
pfd.events = 0;
- if (fde->state & FDE_READ) {
+ if (fde.state & FDE_READ) {
pfd.events |= POLLIN;
}
- if (fde->state & FDE_WRITE) {
+ if (fde.state & FDE_WRITE) {
pfd.events |= POLLOUT;
}
- if (fde->state & FDE_ERROR) {
+ if (fde.state & FDE_ERROR) {
pfd.events |= POLLERR;
}
#if defined(__linux__)
@@ -147,7 +150,6 @@
}
auto post_poll = std::chrono::steady_clock::now();
- std::vector<fdevent_event> poll_events;
for (const auto& pollfd : pollfds) {
unsigned events = 0;
@@ -170,7 +172,7 @@
auto it = this->installed_fdevents_.find(pollfd.fd);
CHECK(it != this->installed_fdevents_.end());
- fdevent* fde = it->second;
+ fdevent* fde = &it->second;
if (events == 0) {
if (fde->timeout) {
@@ -187,7 +189,8 @@
fde->last_active = post_poll;
}
}
- this->HandleEvents(std::move(poll_events));
+ this->HandleEvents(poll_events);
+ poll_events.clear();
}
main_thread_id_.reset();
diff --git a/adb/file_sync_protocol.h b/adb/file_sync_protocol.h
index 87ede0c..fd9a516 100644
--- a/adb/file_sync_protocol.h
+++ b/adb/file_sync_protocol.h
@@ -27,8 +27,10 @@
#define ID_DENT_V1 MKID('D', 'E', 'N', 'T')
#define ID_DENT_V2 MKID('D', 'N', 'T', '2')
-#define ID_SEND MKID('S', 'E', 'N', 'D')
-#define ID_RECV MKID('R', 'E', 'C', 'V')
+#define ID_SEND_V1 MKID('S', 'E', 'N', 'D')
+#define ID_SEND_V2 MKID('S', 'N', 'D', '2')
+#define ID_RECV_V1 MKID('R', 'E', 'C', 'V')
+#define ID_RECV_V2 MKID('R', 'C', 'V', '2')
#define ID_DONE MKID('D', 'O', 'N', 'E')
#define ID_DATA MKID('D', 'A', 'T', 'A')
#define ID_OKAY MKID('O', 'K', 'A', 'Y')
@@ -87,6 +89,26 @@
uint32_t namelen;
}; // followed by `namelen` bytes of the name.
+enum SyncFlag : uint32_t {
+ kSyncFlagNone = 0,
+ kSyncFlagBrotli = 1,
+};
+
+// send_v1 sent the path in a buffer, followed by a comma and the mode as a string.
+// send_v2 sends just the path in the first request, and then sends another syncmsg (with the
+// same ID!) with details.
+struct __attribute__((packed)) sync_send_v2 {
+ uint32_t id;
+ uint32_t mode;
+ uint32_t flags;
+};
+
+// Likewise, recv_v1 just sent the path without any accompanying data.
+struct __attribute__((packed)) sync_recv_v2 {
+ uint32_t id;
+ uint32_t flags;
+};
+
struct __attribute__((packed)) sync_data {
uint32_t id;
uint32_t size;
@@ -104,6 +126,8 @@
sync_dent_v2 dent_v2;
sync_data data;
sync_status status;
+ sync_send_v2 send_v2_setup;
+ sync_recv_v2 recv_v2_setup;
};
#define SYNC_DATA_MAX (64 * 1024)
diff --git a/adb/libs/adbconnection/Android.bp b/adb/libs/adbconnection/Android.bp
index f6b0a42..ce2ab51 100644
--- a/adb/libs/adbconnection/Android.bp
+++ b/adb/libs/adbconnection/Android.bp
@@ -18,6 +18,11 @@
use_version_lib: false,
recovery_available: true,
+ apex_available: [
+ "com.android.adbd",
+ // TODO(b/151398197) remove the below
+ "//apex_available:platform",
+ ],
compile_multilib: "both",
}
diff --git a/adb/pairing_auth/aes_128_gcm.cpp b/adb/pairing_auth/aes_128_gcm.cpp
index 2978834..51520d8 100644
--- a/adb/pairing_auth/aes_128_gcm.cpp
+++ b/adb/pairing_auth/aes_128_gcm.cpp
@@ -19,7 +19,6 @@
#include <android-base/endian.h>
#include <android-base/logging.h>
-#include <openssl/crypto.h>
#include <openssl/evp.h>
#include <openssl/hkdf.h>
#include <openssl/rand.h>
@@ -28,155 +27,64 @@
namespace pairing {
namespace {
-static const size_t kHkdfKeyLength = 256;
-
-struct Header {
- uint32_t payload;
- uint8_t iv[AES_128_GCM_IV_SIZE];
- uint8_t tag[AES_128_GCM_TAG_SIZE];
-} __attribute__((packed));
+// Size of AES-128-GCM key, in bytes
+static constexpr size_t kHkdfKeyLength = 16;
} // namespace
-// static
-const EVP_CIPHER* Aes128Gcm::cipher_ = EVP_aes_128_gcm();
-
Aes128Gcm::Aes128Gcm(const uint8_t* key_material, size_t key_material_len) {
CHECK(key_material);
CHECK_NE(key_material_len, 0ul);
- context_.reset(EVP_CIPHER_CTX_new());
- CHECK(context_.get());
- // Start with a random number for our counter
- CHECK_EQ(RAND_bytes(counter_.data(), counter_.size()), 1);
-
- uint8_t key[kHkdfKeyLength] = {};
- uint8_t salt[64] = "this is the salt";
- uint8_t info[64] = "this is the info";
- CHECK_EQ(HKDF(key, sizeof(key), EVP_sha256(), key_material, key_material_len, salt,
- sizeof(salt), info, sizeof(info)),
+ uint8_t key[kHkdfKeyLength];
+ uint8_t info[] = "adb pairing_auth aes-128-gcm key";
+ CHECK_EQ(HKDF(key, sizeof(key), EVP_sha256(), key_material, key_material_len, nullptr, 0, info,
+ sizeof(info) - 1),
1);
- CHECK_EQ(AES_set_encrypt_key(key, sizeof(key), &aes_key_), 0);
+ CHECK(EVP_AEAD_CTX_init(context_.get(), EVP_aead_aes_128_gcm(), key, sizeof(key),
+ EVP_AEAD_DEFAULT_TAG_LENGTH, nullptr));
}
-int Aes128Gcm::Encrypt(const uint8_t* in, size_t in_len, uint8_t* out, size_t out_len) {
- if (out_len < EncryptedSize(in_len)) {
- LOG(ERROR) << "out buffer size (sz=" << out_len
- << ") not big enough (sz=" << EncryptedSize(in_len) << ")";
- return -1;
- }
- auto& header = *reinterpret_cast<Header*>(out);
- // Place the IV in the header
- memcpy(header.iv, counter_.data(), counter_.size());
- int status = EVP_EncryptInit_ex(context_.get(), cipher_, nullptr,
- reinterpret_cast<const uint8_t*>(&aes_key_), counter_.data());
- counter_.Increase();
- if (status != 1) {
- return -1;
+std::optional<size_t> Aes128Gcm::Encrypt(const uint8_t* in, size_t in_len, uint8_t* out,
+ size_t out_len) {
+ std::vector<uint8_t> nonce(EVP_AEAD_nonce_length(EVP_AEAD_CTX_aead(context_.get())), 0);
+ memcpy(nonce.data(), &enc_sequence_, sizeof(enc_sequence_));
+ size_t written_sz;
+ if (!EVP_AEAD_CTX_seal(context_.get(), out, &written_sz, out_len, nonce.data(), nonce.size(),
+ in, in_len, nullptr, 0)) {
+ LOG(ERROR) << "Failed to encrypt (in_len=" << in_len << ", out_len=" << out_len
+ << ", out_len_needed=" << EncryptedSize(in_len) << ")";
+ return std::nullopt;
}
- int cipherLen = 0;
- out += sizeof(header);
- status = EVP_EncryptUpdate(context_.get(), out, &cipherLen, in, in_len);
- if (status != 1 || cipherLen < 0) {
- return -1;
- }
-
- // Padding is enabled by default, so EVP_EncryptFinal_ex will pad any
- // remaining partial data up to the block size.
- int padding = 0;
- status = EVP_EncryptFinal_ex(context_.get(), out + cipherLen, &padding);
- if (status != 1 || padding < 0) {
- return -1;
- }
-
- // Place the tag in the header
- status = EVP_CIPHER_CTX_ctrl(context_.get(), EVP_CTRL_GCM_GET_TAG, sizeof(header.tag),
- header.tag);
- if (status != 1) {
- return -1;
- }
- // Place the payload size in the header
- uint32_t totalLen = sizeof(header) + cipherLen + padding;
- header.payload = htonl(static_cast<uint32_t>(cipherLen) + static_cast<uint32_t>(padding));
- return totalLen;
+ ++enc_sequence_;
+ return written_sz;
}
-int Aes128Gcm::Decrypt(const uint8_t* in, size_t in_len, uint8_t* out, size_t out_len) {
- if (in_len < sizeof(Header)) {
- return 0;
- }
- if (out_len < DecryptedSize(in, in_len)) {
- return 0;
- }
- const auto& header = *reinterpret_cast<const Header*>(in);
- uint32_t payload = ntohl(header.payload);
- uint32_t expected_inlen = sizeof(Header) + payload;
- if (in_len < expected_inlen) {
- // Not enough data available
- return 0;
- }
- // Initialized with expected IV from header
- int status = EVP_DecryptInit_ex(context_.get(), cipher_, nullptr,
- reinterpret_cast<const uint8_t*>(&aes_key_), header.iv);
- if (status != 1) {
- return -1;
+std::optional<size_t> Aes128Gcm::Decrypt(const uint8_t* in, size_t in_len, uint8_t* out,
+ size_t out_len) {
+ std::vector<uint8_t> nonce(EVP_AEAD_nonce_length(EVP_AEAD_CTX_aead(context_.get())), 0);
+ memcpy(nonce.data(), &dec_sequence_, sizeof(dec_sequence_));
+ size_t written_sz;
+ if (!EVP_AEAD_CTX_open(context_.get(), out, &written_sz, out_len, nonce.data(), nonce.size(),
+ in, in_len, nullptr, 0)) {
+ LOG(ERROR) << "Failed to decrypt (in_len=" << in_len << ", out_len=" << out_len
+ << ", out_len_needed=" << DecryptedSize(in_len) << ")";
+ return std::nullopt;
}
- int decrypted_len = 0;
- status = EVP_DecryptUpdate(context_.get(), out, &decrypted_len, in + sizeof(header), payload);
- if (status != 1 || decrypted_len < 0) {
- return -1;
- }
-
- // Set expected tag from header
- status = EVP_CIPHER_CTX_ctrl(context_.get(), EVP_CTRL_GCM_SET_TAG, sizeof(header.tag),
- const_cast<uint8_t*>(header.tag));
- if (status != 1) {
- return -1;
- }
-
- // This is the padding. It can be ignored.
- int len = 0;
- status = EVP_DecryptFinal_ex(context_.get(), out + decrypted_len, &len);
- if (status != 1) {
- LOG(ERROR) << "EVP_DecryptFinal_ex failed. Tag mismatch";
- return -1;
- }
-
- // Return the length without the padding.
- return decrypted_len;
+ ++dec_sequence_;
+ return written_sz;
}
size_t Aes128Gcm::EncryptedSize(size_t size) {
- // We need to account for block alignment of the encrypted data.
- // According to openssl.org/docs/man1.0.2/man3/EVP_EncryptUpdate.html,
- // "The amount of data written depends on the block alignment of the
- // encrypted data ..."
- // ".. the amount of data written may be anything from zero bytes to
- // (inl + cipher_block_size - 1) ..."
- const size_t cipher_block_size = EVP_CIPHER_block_size(cipher_);
- size_t padding = cipher_block_size - (size % cipher_block_size);
- if (padding != cipher_block_size) {
- size += padding;
- }
- return size + sizeof(Header);
+ // https://commondatastorage.googleapis.com/chromium-boringssl-docs/aead.h.html#EVP_AEAD_CTX_seal
+ return size + EVP_AEAD_max_overhead(EVP_AEAD_CTX_aead(context_.get()));
}
-size_t Aes128Gcm::DecryptedSize(const uint8_t* encrypted_data, size_t encrypted_size) {
- if (encrypted_size < sizeof(Header)) {
- // Not enough data yet
- return 0;
- }
- auto header = reinterpret_cast<const Header*>(encrypted_data);
- uint32_t payload = ntohl(header->payload);
- size_t total_size = payload + sizeof(Header);
- if (encrypted_size < total_size) {
- // There's enough data for the header but not enough data for the
- // payload. Indicate that there's not enough data for now.
- return 0;
- }
- return payload;
+size_t Aes128Gcm::DecryptedSize(size_t size) {
+ // https://commondatastorage.googleapis.com/chromium-boringssl-docs/aead.h.html#EVP_AEAD_CTX_open
+ return size;
}
} // namespace pairing
diff --git a/adb/pairing_auth/include/adb/pairing/aes_128_gcm.h b/adb/pairing_auth/include/adb/pairing/aes_128_gcm.h
index 490dd12..6be5856 100644
--- a/adb/pairing_auth/include/adb/pairing/aes_128_gcm.h
+++ b/adb/pairing_auth/include/adb/pairing/aes_128_gcm.h
@@ -16,17 +16,12 @@
#pragma once
-#include <openssl/aes.h>
-#include <openssl/cipher.h>
-
#include <stdint.h>
-#include "adb/pairing/counter.h"
+#include <optional>
+#include <vector>
-// This is the default size of the initialization vector (iv) for AES-128-GCM
-#define AES_128_GCM_IV_SIZE 12
-// This is the full tag size for AES-128-GCM
-#define AES_128_GCM_TAG_SIZE 16
+#include <openssl/aead.h>
namespace adb {
namespace pairing {
@@ -42,7 +37,7 @@
// suitable for decryption with this class.
// The method returns the number of bytes placed in |out| on success and a
// negative value if an error occurs.
- int Encrypt(const uint8_t* in, size_t in_len, uint8_t* out, size_t out_len);
+ std::optional<size_t> Encrypt(const uint8_t* in, size_t in_len, uint8_t* out, size_t out_len);
// Decrypt a block of data in |in| of length |in_len|, this consumes all data
// in |in_len| bytes of data. The decrypted output is placed in the |out|
// buffer of length |out_len|. On successful decryption the number of bytes in
@@ -50,22 +45,18 @@
// The method returns the number of bytes consumed from the |in| buffer. If
// there is not enough data available in |in| the method returns zero. If
// an error occurs the method returns a negative value.
- int Decrypt(const uint8_t* in, size_t in_len, uint8_t* out, size_t out_len);
+ std::optional<size_t> Decrypt(const uint8_t* in, size_t in_len, uint8_t* out, size_t out_len);
// Return a safe amount of buffer storage needed to encrypt |size| bytes.
size_t EncryptedSize(size_t size);
- // Return a safe amount of buffer storage needed to decrypt the encrypted
- // data in |encrypted_data| which is of length |encrypted_size|. Returns 0 if
- // there is not enough data available to determine the required size.
- size_t DecryptedSize(const uint8_t* encrypted_data, size_t encrypted_size);
-
- static const EVP_CIPHER* cipher_;
+ // Return a safe amount of buffer storage needed to decrypt |size| bytes.
+ size_t DecryptedSize(size_t size);
private:
- bssl::UniquePtr<EVP_CIPHER_CTX> context_;
- AES_KEY aes_key_;
- // We're going to use this counter for our iv so that it never repeats
- Counter<AES_128_GCM_IV_SIZE> counter_;
+ bssl::ScopedEVP_AEAD_CTX context_;
+ // Sequence numbers to use as nonces in the encryption scheme
+ uint64_t dec_sequence_ = 0;
+ uint64_t enc_sequence_ = 0;
};
} // namespace pairing
diff --git a/adb/pairing_auth/include/adb/pairing/counter.h b/adb/pairing_auth/include/adb/pairing/counter.h
deleted file mode 100644
index 263ceb7..0000000
--- a/adb/pairing_auth/include/adb/pairing/counter.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright (C) 2020 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include <stddef.h>
-#include <stdint.h>
-
-namespace adb {
-namespace pairing {
-
-template <size_t N>
-class Counter {
- public:
- void Increase() {
- for (size_t i = sizeof(counter_) - 1; i < sizeof(counter_); --i) {
- if (++counter_[i] != 0) {
- break;
- }
- }
- }
-
- uint8_t* data() { return counter_; }
- const uint8_t* data() const { return counter_; }
-
- constexpr size_t size() const { return sizeof(counter_); }
-
- uint8_t& operator[](size_t index) { return counter_[index]; }
- const uint8_t& operator[](size_t index) const { return counter_[index]; }
-
- private:
- uint8_t counter_[N];
-};
-
-} // namespace pairing
-} // namespace adb
diff --git a/adb/pairing_auth/pairing_auth.cpp b/adb/pairing_auth/pairing_auth.cpp
index 96bc110..0ac04e6 100644
--- a/adb/pairing_auth/pairing_auth.cpp
+++ b/adb/pairing_auth/pairing_auth.cpp
@@ -75,8 +75,8 @@
// Returns a safe buffer size for encrypting a buffer of size |len|.
size_t SafeEncryptedSize(size_t len);
- // Returns a safe buffer size for decrypting a buffer |buf|.
- size_t SafeDecryptedSize(const Data& buf);
+ // Returns a safe buffer size for decrypting a buffer of size |len|.
+ size_t SafeDecryptedSize(size_t len);
private:
Data our_msg_;
@@ -167,12 +167,12 @@
// Determine the size for the encrypted data based on the raw data.
Data encrypted(cipher_->EncryptedSize(data.size()));
- int bytes = cipher_->Encrypt(data.data(), data.size(), encrypted.data(), encrypted.size());
- if (bytes < 0) {
+ auto out_size = cipher_->Encrypt(data.data(), data.size(), encrypted.data(), encrypted.size());
+ if (!out_size.has_value() || *out_size == 0) {
LOG(ERROR) << "Unable to encrypt data";
return Data();
}
- encrypted.resize(bytes);
+ encrypted.resize(*out_size);
return encrypted;
}
@@ -182,14 +182,14 @@
CHECK(!data.empty());
// Determine the size for the decrypted data based on the raw data.
- Data decrypted(cipher_->DecryptedSize(data.data(), data.size()));
+ Data decrypted(cipher_->DecryptedSize(data.size()));
size_t decrypted_size = decrypted.size();
- int bytes = cipher_->Decrypt(data.data(), data.size(), decrypted.data(), decrypted_size);
- if (bytes <= 0) {
+ auto out_size = cipher_->Decrypt(data.data(), data.size(), decrypted.data(), decrypted_size);
+ if (!out_size.has_value() || *out_size == 0) {
LOG(ERROR) << "Unable to decrypt data";
return Data();
}
- decrypted.resize(bytes);
+ decrypted.resize(*out_size);
return decrypted;
}
@@ -199,9 +199,9 @@
return cipher_->EncryptedSize(len);
}
-size_t PairingAuthCtx::SafeDecryptedSize(const PairingAuthCtx::Data& buf) {
+size_t PairingAuthCtx::SafeDecryptedSize(size_t len) {
CHECK(cipher_);
- return cipher_->DecryptedSize(buf.data(), buf.size());
+ return cipher_->DecryptedSize(len);
}
PairingAuthCtx* pairing_auth_server_new(const uint8_t* pswd, size_t len) {
@@ -271,8 +271,8 @@
CHECK(ctx);
CHECK(buf);
CHECK_GT(len, 0U);
- std::vector<uint8_t> p(buf, buf + len);
- return ctx->SafeDecryptedSize(p);
+ // We no longer need buf for EVP_AEAD
+ return ctx->SafeDecryptedSize(len);
}
bool pairing_auth_decrypt(PairingAuthCtx* ctx, const uint8_t* inbuf, size_t inlen, uint8_t* outbuf,
diff --git a/adb/pairing_auth/tests/Android.bp b/adb/pairing_auth/tests/Android.bp
index 292fff5..213123d 100644
--- a/adb/pairing_auth/tests/Android.bp
+++ b/adb/pairing_auth/tests/Android.bp
@@ -18,7 +18,6 @@
name: "adb_pairing_auth_test",
srcs: [
"aes_128_gcm_test.cpp",
- "counter_test.cpp",
"pairing_auth_test.cpp",
],
diff --git a/adb/pairing_auth/tests/aes_128_gcm_test.cpp b/adb/pairing_auth/tests/aes_128_gcm_test.cpp
index e1a20e8..55689d6 100644
--- a/adb/pairing_auth/tests/aes_128_gcm_test.cpp
+++ b/adb/pairing_auth/tests/aes_128_gcm_test.cpp
@@ -39,7 +39,7 @@
const uint8_t msg[] = "alice and bob, sitting in a binary tree";
uint8_t material[256];
uint8_t encrypted[1024];
- uint8_t out_buf[1024];
+ uint8_t out_buf[1024] = {};
RAND_bytes(material, sizeof(material));
Aes128Gcm alice(material, sizeof(material));
@@ -47,82 +47,16 @@
;
ASSERT_GE(alice.EncryptedSize(sizeof(msg)), sizeof(msg));
- int encrypted_size = alice.Encrypt(msg, sizeof(msg), encrypted, sizeof(encrypted));
- ASSERT_GT(encrypted_size, 0);
+ auto encrypted_size = alice.Encrypt(msg, sizeof(msg), encrypted, sizeof(encrypted));
+ ASSERT_TRUE(encrypted_size.has_value());
+ ASSERT_GT(*encrypted_size, 0);
size_t out_size = sizeof(out_buf);
- ASSERT_GE(bob.DecryptedSize(encrypted, sizeof(encrypted)), sizeof(msg));
- int decrypted_size = bob.Decrypt(encrypted, sizeof(encrypted), out_buf, out_size);
- ASSERT_EQ(sizeof(msg), decrypted_size);
- memset(out_buf + decrypted_size, 0, sizeof(out_buf) - decrypted_size);
+ ASSERT_GE(bob.DecryptedSize(*encrypted_size), sizeof(msg));
+ auto decrypted_size = bob.Decrypt(encrypted, *encrypted_size, out_buf, out_size);
+ ASSERT_TRUE(decrypted_size.has_value());
+ ASSERT_EQ(sizeof(msg), *decrypted_size);
ASSERT_STREQ(reinterpret_cast<const char*>(msg), reinterpret_cast<const char*>(out_buf));
}
-TEST(Aes128GcmTest, padding) {
- // Test with block-align data as well as unaligned data.
- const size_t cipher_block_size = EVP_CIPHER_block_size(Aes128Gcm::cipher_);
- uint8_t material[256];
- RAND_bytes(material, sizeof(material));
- Aes128Gcm alice(material, sizeof(material));
- Aes128Gcm bob(material, sizeof(material));
- ;
- std::vector<uint8_t> msg;
- std::vector<uint8_t> encrypted;
- std::vector<uint8_t> decrypted;
-
- // Test with aligned data
- {
- msg.resize(cipher_block_size);
- RAND_bytes(msg.data(), msg.size());
-
- // encrypt
- size_t safe_encrypted_sz = alice.EncryptedSize(msg.size());
- ASSERT_GE(safe_encrypted_sz, msg.size());
- encrypted.resize(safe_encrypted_sz);
- int encrypted_size =
- alice.Encrypt(msg.data(), msg.size(), encrypted.data(), encrypted.size());
- ASSERT_GT(encrypted_size, 0);
- ASSERT_LE(encrypted_size, safe_encrypted_sz);
- encrypted.resize(encrypted_size);
-
- // decrypt
- size_t safe_decrypted_size = bob.DecryptedSize(encrypted.data(), encrypted.size());
- ASSERT_GE(safe_decrypted_size, msg.size());
- decrypted.resize(safe_decrypted_size);
- int decrypted_size =
- bob.Decrypt(encrypted.data(), encrypted.size(), decrypted.data(), decrypted.size());
- ASSERT_GT(decrypted_size, 0);
- ASSERT_LE(decrypted_size, safe_decrypted_size);
- ASSERT_EQ(msg.size(), decrypted_size);
- ASSERT_EQ(memcmp(msg.data(), decrypted.data(), decrypted.size()), 0);
- }
-
- // Test with unaligned data
- {
- msg.resize(cipher_block_size + 1);
- RAND_bytes(msg.data(), msg.size());
-
- // encrypt
- size_t safe_encrypted_sz = alice.EncryptedSize(msg.size());
- ASSERT_GE(safe_encrypted_sz, msg.size());
- encrypted.resize(safe_encrypted_sz);
- int encrypted_size =
- alice.Encrypt(msg.data(), msg.size(), encrypted.data(), encrypted.size());
- ASSERT_GT(encrypted_size, 0);
- ASSERT_LE(encrypted_size, safe_encrypted_sz);
- encrypted.resize(encrypted_size);
-
- // decrypt
- size_t safe_decrypted_size = bob.DecryptedSize(encrypted.data(), encrypted.size());
- ASSERT_GE(safe_decrypted_size, msg.size());
- decrypted.resize(safe_decrypted_size);
- int decrypted_size =
- bob.Decrypt(encrypted.data(), encrypted.size(), decrypted.data(), decrypted.size());
- ASSERT_GT(decrypted_size, 0);
- ASSERT_LE(decrypted_size, safe_decrypted_size);
- ASSERT_EQ(msg.size(), decrypted_size);
- ASSERT_EQ(memcmp(msg.data(), decrypted.data(), decrypted.size()), 0);
- }
-}
-
} // namespace pairing
} // namespace adb
diff --git a/adb/pairing_auth/tests/counter_test.cpp b/adb/pairing_auth/tests/counter_test.cpp
deleted file mode 100644
index b338551..0000000
--- a/adb/pairing_auth/tests/counter_test.cpp
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright (C) 2020 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <gtest/gtest.h>
-
-#include <adb/pairing/counter.h>
-
-namespace adb {
-namespace pairing {
-
-static constexpr size_t kTestCounterSize = 13;
-static const uint8_t kZeroes[64] = {0};
-
-TEST(AdbCounterTest, size_match) {
- Counter<kTestCounterSize> counter;
- ASSERT_EQ(kTestCounterSize, counter.size());
-}
-
-TEST(AdbCounterTest, Increase) {
- Counter<kTestCounterSize> counter;
- memset(counter.data(), 0, counter.size());
- counter.Increase();
- EXPECT_EQ(1, counter[counter.size() - 1]);
- EXPECT_EQ(0, memcmp(counter.data(), kZeroes, counter.size() - 1));
-}
-
-TEST(AdbCounterTest, rollover_first_byte) {
- Counter<kTestCounterSize> counter;
- memset(counter.data(), 0, counter.size());
- counter[counter.size() - 1] = 0xFF;
- counter.Increase();
- EXPECT_EQ(0, counter[counter.size() - 1]);
- EXPECT_EQ(1, counter[counter.size() - 2]);
- EXPECT_EQ(0, memcmp(counter.data(), kZeroes, counter.size() - 2));
-}
-
-TEST(AdbCounterTest, multiple_rollover) {
- Counter<kTestCounterSize> counter;
- memset(counter.data(), 0xFF, counter.size());
- memset(counter.data(), 0, counter.size() - 3);
- counter.Increase();
- EXPECT_EQ(0, counter[counter.size() - 5]);
- EXPECT_EQ(1, counter[counter.size() - 4]);
- EXPECT_EQ(0, counter[counter.size() - 3]);
- EXPECT_EQ(0, counter[counter.size() - 2]);
- EXPECT_EQ(0, counter[counter.size() - 1]);
-}
-
-TEST(AdbCounterTest, full_rollover) {
- Counter<kTestCounterSize> counter;
- memset(counter.data(), 0xFF, counter.size());
- counter.Increase();
- EXPECT_EQ(0, memcmp(counter.data(), kZeroes, counter.size()));
-}
-
-} // namespace pairing
-} // namespace adb
diff --git a/adb/pairing_connection/pairing_connection.cpp b/adb/pairing_connection/pairing_connection.cpp
index a26a6b4..ffe49a9 100644
--- a/adb/pairing_connection/pairing_connection.cpp
+++ b/adb/pairing_connection/pairing_connection.cpp
@@ -278,13 +278,13 @@
if (fd < 0) {
return false;
}
+ fd_.reset(fd);
State expected = State::Ready;
if (!state_.compare_exchange_strong(expected, State::ExchangingMsgs)) {
return false;
}
- fd_.reset(fd);
cb_ = cb;
opaque_ = opaque;
diff --git a/adb/sysdeps.h b/adb/sysdeps.h
index 4efbc02..3e781b8 100644
--- a/adb/sysdeps.h
+++ b/adb/sysdeps.h
@@ -276,6 +276,7 @@
class Process {
public:
constexpr explicit Process(HANDLE h = nullptr) : h_(h) {}
+ constexpr Process(Process&& other) : h_(std::exchange(other.h_, nullptr)) {}
~Process() { close(); }
constexpr explicit operator bool() const { return h_ != nullptr; }
@@ -292,6 +293,8 @@
}
private:
+ DISALLOW_COPY_AND_ASSIGN(Process);
+
void close() {
if (*this) {
::CloseHandle(h_);
@@ -666,6 +669,8 @@
class Process {
public:
constexpr explicit Process(pid_t pid) : pid_(pid) {}
+ constexpr Process(Process&& other) : pid_(std::exchange(other.pid_, -1)) {}
+
constexpr explicit operator bool() const { return pid_ >= 0; }
void wait() {
@@ -682,6 +687,8 @@
}
private:
+ DISALLOW_COPY_AND_ASSIGN(Process);
+
pid_t pid_;
};
diff --git a/adb/transport.cpp b/adb/transport.cpp
index 447a8fe..ed4a93b 100644
--- a/adb/transport.cpp
+++ b/adb/transport.cpp
@@ -82,6 +82,8 @@
const char* const kFeatureAbbExec = "abb_exec";
const char* const kFeatureRemountShell = "remount_shell";
const char* const kFeatureTrackApp = "track_app";
+const char* const kFeatureSendRecv2 = "sendrecv_v2";
+const char* const kFeatureSendRecv2Brotli = "sendrecv_v2_brotli";
namespace {
@@ -498,12 +500,18 @@
auto x509 = GenerateX509Certificate(evp_pkey.get());
auto x509_str = X509ToPEMString(x509.get());
auto evp_str = Key::ToPEMString(evp_pkey.get());
+#ifdef _WIN32
+ int osh = cast_handle_to_int(adb_get_os_handle(fd_));
+#else
+ int osh = adb_get_os_handle(fd_);
+#endif
+
#if ADB_HOST
tls_ = TlsConnection::Create(TlsConnection::Role::Client,
#else
tls_ = TlsConnection::Create(TlsConnection::Role::Server,
#endif
- x509_str, evp_str, fd_);
+ x509_str, evp_str, osh);
CHECK(tls_);
#if ADB_HOST
// TLS 1.3 gives the client no message if the server rejected the
@@ -1177,6 +1185,8 @@
kFeatureAbbExec,
kFeatureRemountShell,
kFeatureTrackApp,
+ kFeatureSendRecv2,
+ kFeatureSendRecv2Brotli,
// Increment ADB_SERVER_VERSION when adding a feature that adbd needs
// to know about. Otherwise, the client can be stuck running an old
// version of the server even after upgrading their copy of adb.
diff --git a/adb/transport.h b/adb/transport.h
index a62349e..3a77cf6 100644
--- a/adb/transport.h
+++ b/adb/transport.h
@@ -81,9 +81,14 @@
extern const char* const kFeatureAbbExec;
// adbd properly updates symlink timestamps on push.
extern const char* const kFeatureFixedPushSymlinkTimestamp;
+// Implement `adb remount` via shelling out to /system/bin/remount.
extern const char* const kFeatureRemountShell;
// adbd supports `track-app` service reporting debuggable/profileable apps.
extern const char* const kFeatureTrackApp;
+// adbd supports version 2 of send/recv.
+extern const char* const kFeatureSendRecv2;
+// adbd supports brotli for send/recv v2.
+extern const char* const kFeatureSendRecv2Brotli;
TransportId NextTransportId();
diff --git a/adb/types.h b/adb/types.h
index c619fff..deca7ea 100644
--- a/adb/types.h
+++ b/adb/types.h
@@ -150,6 +150,22 @@
IOVector& operator=(const IOVector& copy) = delete;
IOVector& operator=(IOVector&& move) noexcept;
+ const value_type* front_data() const {
+ if (chain_.empty()) {
+ return nullptr;
+ }
+
+ return chain_.front().data() + begin_offset_;
+ }
+
+ size_type front_size() const {
+ if (chain_.empty()) {
+ return 0;
+ }
+
+ return chain_.front().size() - begin_offset_;
+ }
+
size_type size() const { return chain_length_ - begin_offset_; }
bool empty() const { return size() == 0; }
diff --git a/base/liblog_symbols.cpp b/base/liblog_symbols.cpp
index ba4c161..1f4b69b 100644
--- a/base/liblog_symbols.cpp
+++ b/base/liblog_symbols.cpp
@@ -46,7 +46,7 @@
}
DLSYM(__android_log_set_logger)
- DLSYM(__android_log_write_logger_data)
+ DLSYM(__android_log_write_log_message)
DLSYM(__android_log_logd_logger)
DLSYM(__android_log_stderr_logger)
DLSYM(__android_log_set_aborter)
@@ -69,7 +69,7 @@
static std::optional<LibLogFunctions> liblog_functions = []() -> std::optional<LibLogFunctions> {
return LibLogFunctions{
.__android_log_set_logger = __android_log_set_logger,
- .__android_log_write_logger_data = __android_log_write_logger_data,
+ .__android_log_write_log_message = __android_log_write_log_message,
.__android_log_logd_logger = __android_log_logd_logger,
.__android_log_stderr_logger = __android_log_stderr_logger,
.__android_log_set_aborter = __android_log_set_aborter,
diff --git a/base/liblog_symbols.h b/base/liblog_symbols.h
index b4ab06a..2e6b47f 100644
--- a/base/liblog_symbols.h
+++ b/base/liblog_symbols.h
@@ -25,13 +25,10 @@
struct LibLogFunctions {
void (*__android_log_set_logger)(__android_logger_function logger);
- void (*__android_log_write_logger_data)(struct __android_logger_data* logger_data,
- const char* msg);
+ void (*__android_log_write_log_message)(struct __android_log_message* log_message);
- void (*__android_log_logd_logger)(const struct __android_logger_data* logger_data,
- const char* msg);
- void (*__android_log_stderr_logger)(const struct __android_logger_data* logger_data,
- const char* message);
+ void (*__android_log_logd_logger)(const struct __android_log_message* log_message);
+ void (*__android_log_stderr_logger)(const struct __android_log_message* log_message);
void (*__android_log_set_aborter)(__android_aborter_function aborter);
void (*__android_log_call_aborter)(const char* abort_message);
diff --git a/base/logging.cpp b/base/logging.cpp
index 9a6e0fb..cd460eb 100644
--- a/base/logging.cpp
+++ b/base/logging.cpp
@@ -349,9 +349,9 @@
static auto& liblog_functions = GetLibLogFunctions();
if (liblog_functions) {
- __android_logger_data logger_data = {sizeof(__android_logger_data), lg_id, priority, tag,
- static_cast<const char*>(nullptr), 0};
- liblog_functions->__android_log_logd_logger(&logger_data, message);
+ __android_log_message log_message = {sizeof(__android_log_message), lg_id, priority, tag,
+ static_cast<const char*>(nullptr), 0, message};
+ liblog_functions->__android_log_logd_logger(&log_message);
} else {
__android_log_buf_print(lg_id, priority, tag, "%s", message);
}
@@ -426,13 +426,13 @@
// std::function<>, which is the not-thread-safe alternative.
static std::atomic<LogFunction*> logger_function(nullptr);
auto* old_logger_function = logger_function.exchange(new LogFunction(logger));
- liblog_functions->__android_log_set_logger([](const struct __android_logger_data* logger_data,
- const char* message) {
- auto log_id = log_id_tToLogId(logger_data->buffer_id);
- auto severity = PriorityToLogSeverity(logger_data->priority);
+ liblog_functions->__android_log_set_logger([](const struct __android_log_message* log_message) {
+ auto log_id = log_id_tToLogId(log_message->buffer_id);
+ auto severity = PriorityToLogSeverity(log_message->priority);
auto& function = *logger_function.load(std::memory_order_acquire);
- function(log_id, severity, logger_data->tag, logger_data->file, logger_data->line, message);
+ function(log_id, severity, log_message->tag, log_message->file, log_message->line,
+ log_message->message);
});
delete old_logger_function;
} else {
@@ -576,9 +576,9 @@
static auto& liblog_functions = GetLibLogFunctions();
int32_t priority = LogSeverityToPriority(severity);
if (liblog_functions) {
- __android_logger_data logger_data = {
- sizeof(__android_logger_data), LOG_ID_DEFAULT, priority, tag, file, line};
- liblog_functions->__android_log_write_logger_data(&logger_data, message);
+ __android_log_message log_message = {
+ sizeof(__android_log_message), LOG_ID_DEFAULT, priority, tag, file, line, message};
+ liblog_functions->__android_log_write_log_message(&log_message);
} else {
if (tag == nullptr) {
std::lock_guard<std::recursive_mutex> lock(TagLock());
diff --git a/debuggerd/libdebuggerd/tombstone.cpp b/debuggerd/libdebuggerd/tombstone.cpp
index b3f059c..bb3c260 100644
--- a/debuggerd/libdebuggerd/tombstone.cpp
+++ b/debuggerd/libdebuggerd/tombstone.cpp
@@ -457,8 +457,8 @@
return;
}
- logger_list = android_logger_list_open(
- android_name_to_log_id(filename), ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, tail, pid);
+ logger_list =
+ android_logger_list_open(android_name_to_log_id(filename), ANDROID_LOG_NONBLOCK, tail, pid);
if (!logger_list) {
ALOGE("Unable to open %s: %s\n", filename, strerror(errno));
diff --git a/diagnose_usb/Android.bp b/diagnose_usb/Android.bp
index 6bee28c..93d13bd 100644
--- a/diagnose_usb/Android.bp
+++ b/diagnose_usb/Android.bp
@@ -3,6 +3,11 @@
cflags: ["-Wall", "-Wextra", "-Werror"],
host_supported: true,
recovery_available: true,
+ apex_available: [
+ "com.android.adbd",
+ // TODO(b/151398197) remove the below
+ "//apex_available:platform",
+ ],
target: {
windows: {
enabled: true,
diff --git a/fs_mgr/fs_mgr.cpp b/fs_mgr/fs_mgr.cpp
index c0c0e99..5475cae 100644
--- a/fs_mgr/fs_mgr.cpp
+++ b/fs_mgr/fs_mgr.cpp
@@ -96,6 +96,7 @@
using android::base::Basename;
using android::base::GetBoolProperty;
+using android::base::GetUintProperty;
using android::base::Readlink;
using android::base::Realpath;
using android::base::SetProperty;
@@ -1545,11 +1546,16 @@
return ret;
}
+static std::chrono::milliseconds GetMillisProperty(const std::string& name,
+ std::chrono::milliseconds default_value) {
+ auto value = GetUintProperty(name, static_cast<uint64_t>(default_value.count()));
+ return std::chrono::milliseconds(std::move(value));
+}
+
static bool fs_mgr_unmount_all_data_mounts(const std::string& block_device) {
LINFO << __FUNCTION__ << "(): about to umount everything on top of " << block_device;
Timer t;
- // TODO(b/135984674): should be configured via a read-only property.
- std::chrono::milliseconds timeout = 5s;
+ auto timeout = GetMillisProperty("init.userspace_reboot.userdata_remount.timeoutmillis", 5s);
while (true) {
bool umount_done = true;
Fstab proc_mounts;
diff --git a/fs_mgr/tests/Android.bp b/fs_mgr/tests/Android.bp
index 4f6ec5a..28dee88 100644
--- a/fs_mgr/tests/Android.bp
+++ b/fs_mgr/tests/Android.bp
@@ -18,6 +18,7 @@
"cts",
"device-tests",
"vts",
+ "vts10",
],
compile_multilib: "both",
multilib: {
diff --git a/init/Android.bp b/init/Android.bp
index 9053c39..d512a4e 100644
--- a/init/Android.bp
+++ b/init/Android.bp
@@ -259,6 +259,7 @@
"cts",
"device-tests",
"vts",
+ "vts10",
],
}
diff --git a/init/init.cpp b/init/init.cpp
index 5444a32..a88d127 100644
--- a/init/init.cpp
+++ b/init/init.cpp
@@ -45,6 +45,7 @@
#include <android-base/properties.h>
#include <android-base/stringprintf.h>
#include <android-base/strings.h>
+#include <backtrace/Backtrace.h>
#include <fs_avb/fs_avb.h>
#include <fs_mgr_vendor_overlay.h>
#include <keyutils.h>
@@ -137,7 +138,7 @@
static void WakeEpoll() {
constexpr char value[] = "1";
- write(wake_epoll_fd, value, sizeof(value));
+ TEMP_FAILURE_RETRY(write(wake_epoll_fd, value, sizeof(value)));
}
static class PropWaiterState {
@@ -218,6 +219,16 @@
prop_waiter_state.ResetWaitForProp();
}
+static void UnwindMainThreadStack() {
+ std::unique_ptr<Backtrace> backtrace(Backtrace::Create(BACKTRACE_CURRENT_PROCESS, 1));
+ if (!backtrace->Unwind(0)) {
+ LOG(ERROR) << __FUNCTION__ << ": Failed to unwind callstack.";
+ }
+ for (size_t i = 0; i < backtrace->NumFrames(); i++) {
+ LOG(ERROR) << backtrace->FormatFrameData(i);
+ }
+}
+
static class ShutdownState {
public:
void TriggerShutdown(const std::string& command) {
@@ -227,6 +238,15 @@
// action queue. Instead we set this flag and ensure that shutdown happens before the next
// command is run in the main init loop.
auto lock = std::lock_guard{shutdown_command_lock_};
+ if (do_shutdown_) {
+ LOG(ERROR) << "TriggerShutdown called while a previous shutdown command '"
+ << shutdown_command_ << "' has not been handled";
+ UnwindMainThreadStack();
+ }
+ if (IsShuttingDown()) {
+ LOG(ERROR) << "TriggerShutdown called while init is already shutting down";
+ UnwindMainThreadStack();
+ }
shutdown_command_ = command;
do_shutdown_ = true;
WakeEpoll();
@@ -848,6 +868,8 @@
auto shutdown_command = shutdown_state.CheckShutdown();
if (shutdown_command) {
+ LOG(INFO) << "Got shutdown_command '" << *shutdown_command
+ << "' Calling HandlePowerctlMessage()";
HandlePowerctlMessage(*shutdown_command);
}
diff --git a/init/reboot.cpp b/init/reboot.cpp
index f006df3..081f695 100644
--- a/init/reboot.cpp
+++ b/init/reboot.cpp
@@ -72,6 +72,7 @@
using android::base::boot_clock;
using android::base::GetBoolProperty;
+using android::base::GetUintProperty;
using android::base::SetProperty;
using android::base::Split;
using android::base::Timer;
@@ -732,6 +733,12 @@
return Error() << "'/system/bin/apexd --unmount-all' failed : " << status;
}
+static std::chrono::milliseconds GetMillisProperty(const std::string& name,
+ std::chrono::milliseconds default_value) {
+ auto value = GetUintProperty(name, static_cast<uint64_t>(default_value.count()));
+ return std::chrono::milliseconds(std::move(value));
+}
+
static Result<void> DoUserspaceReboot() {
LOG(INFO) << "Userspace reboot initiated";
auto guard = android::base::make_scope_guard([] {
@@ -769,10 +776,13 @@
sync();
LOG(INFO) << "sync() took " << sync_timer;
}
- // TODO(b/135984674): do we need shutdown animation for userspace reboot?
- // TODO(b/135984674): control userspace timeout via read-only property?
- StopServicesAndLogViolations(stop_first, 10s, true /* SIGTERM */);
- if (int r = StopServicesAndLogViolations(stop_first, 20s, false /* SIGKILL */); r > 0) {
+ auto sigterm_timeout = GetMillisProperty("init.userspace_reboot.sigterm.timeoutmillis", 5s);
+ auto sigkill_timeout = GetMillisProperty("init.userspace_reboot.sigkill.timeoutmillis", 10s);
+ LOG(INFO) << "Timeout to terminate services : " << sigterm_timeout.count() << "ms"
+ << "Timeout to kill services: " << sigkill_timeout.count() << "ms";
+ StopServicesAndLogViolations(stop_first, sigterm_timeout, true /* SIGTERM */);
+ if (int r = StopServicesAndLogViolations(stop_first, sigkill_timeout, false /* SIGKILL */);
+ r > 0) {
// TODO(b/135984674): store information about offending services for debugging.
return Error() << r << " post-data services are still running";
}
@@ -782,8 +792,8 @@
if (auto result = CallVdc("volume", "reset"); !result.ok()) {
return result;
}
- if (int r = StopServicesAndLogViolations(GetDebuggingServices(true /* only_post_data */), 5s,
- false /* SIGKILL */);
+ if (int r = StopServicesAndLogViolations(GetDebuggingServices(true /* only_post_data */),
+ sigkill_timeout, false /* SIGKILL */);
r > 0) {
// TODO(b/135984674): store information about offending services for debugging.
return Error() << r << " debugging services are still running";
@@ -827,8 +837,8 @@
return;
}
LOG(INFO) << "Starting userspace reboot watchdog";
- // TODO(b/135984674): this should be configured via a read-only sysprop.
- std::chrono::milliseconds timeout = 60s;
+ auto timeout = GetMillisProperty("init.userspace_reboot.watchdog.timeoutmillis", 5min);
+ LOG(INFO) << "UserspaceRebootWatchdog timeout: " << timeout.count() << "ms";
if (!WaitForProperty("sys.boot_completed", "1", timeout)) {
LOG(ERROR) << "Failed to boot in " << timeout.count() << "ms. Switching to full reboot";
// In this case device is in a boot loop. Only way to recover is to do dirty reboot.
diff --git a/init/selinux.cpp b/init/selinux.cpp
index 808cb7f..5a0255a 100644
--- a/init/selinux.cpp
+++ b/init/selinux.cpp
@@ -63,12 +63,15 @@
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/parseint.h>
+#include <android-base/strings.h>
#include <android-base/unique_fd.h>
#include <fs_avb/fs_avb.h>
+#include <fs_mgr.h>
#include <libgsi/libgsi.h>
#include <libsnapshot/snapshot.h>
#include <selinux/android.h>
+#include "block_dev_initializer.h"
#include "debug_ramdisk.h"
#include "reboot_utils.h"
#include "util.h"
@@ -598,6 +601,74 @@
return vendor_android_version;
}
+// This is for R system.img/system_ext.img to work on old vendor.img as system_ext.img
+// is introduced in R. We mount system_ext in second stage init because the first-stage
+// init in boot.img won't be updated in the system-only OTA scenario.
+void MountMissingSystemPartitions() {
+ android::fs_mgr::Fstab fstab;
+ if (!ReadDefaultFstab(&fstab)) {
+ LOG(ERROR) << "Could not read default fstab";
+ }
+
+ android::fs_mgr::Fstab mounts;
+ if (!ReadFstabFromFile("/proc/mounts", &mounts)) {
+ LOG(ERROR) << "Could not read /proc/mounts";
+ }
+
+ static const std::vector<std::string> kPartitionNames = {"system_ext", "product"};
+
+ android::fs_mgr::Fstab extra_fstab;
+ for (const auto& name : kPartitionNames) {
+ if (GetEntryForMountPoint(&mounts, "/"s + name)) {
+ // The partition is already mounted.
+ continue;
+ }
+
+ auto system_entry = GetEntryForMountPoint(&fstab, "/system");
+ if (!system_entry) {
+ LOG(ERROR) << "Could not find mount entry for /system";
+ break;
+ }
+ if (!system_entry->fs_mgr_flags.logical) {
+ LOG(INFO) << "Skipping mount of " << name << ", system is not dynamic.";
+ break;
+ }
+
+ auto entry = *system_entry;
+ auto partition_name = name + fs_mgr_get_slot_suffix();
+ auto replace_name = "system"s + fs_mgr_get_slot_suffix();
+
+ entry.mount_point = "/"s + name;
+ entry.blk_device =
+ android::base::StringReplace(entry.blk_device, replace_name, partition_name, false);
+ if (!fs_mgr_update_logical_partition(&entry)) {
+ LOG(ERROR) << "Could not update logical partition";
+ continue;
+ }
+
+ extra_fstab.emplace_back(std::move(entry));
+ }
+
+ SkipMountingPartitions(&extra_fstab);
+ if (extra_fstab.empty()) {
+ return;
+ }
+
+ BlockDevInitializer block_dev_init;
+ for (auto& entry : extra_fstab) {
+ if (access(entry.blk_device.c_str(), F_OK) != 0) {
+ auto block_dev = android::base::Basename(entry.blk_device);
+ if (!block_dev_init.InitDmDevice(block_dev)) {
+ LOG(ERROR) << "Failed to find device-mapper node: " << block_dev;
+ continue;
+ }
+ }
+ if (fs_mgr_do_mount_one(entry)) {
+ LOG(ERROR) << "Could not mount " << entry.mount_point;
+ }
+ }
+}
+
int SetupSelinux(char** argv) {
SetStdioToDevNull(argv);
InitKernelLogging(argv);
@@ -608,6 +679,8 @@
boot_clock::time_point start_time = boot_clock::now();
+ MountMissingSystemPartitions();
+
// Set up SELinux, loading the SELinux policy.
SelinuxSetupKernelLogging();
SelinuxInitialize();
diff --git a/libasyncio/Android.bp b/libasyncio/Android.bp
index 4ab439d..44e7933 100644
--- a/libasyncio/Android.bp
+++ b/libasyncio/Android.bp
@@ -28,6 +28,10 @@
defaults: ["libasyncio_defaults"],
vendor_available: true,
recovery_available: true,
+ apex_available: [
+ "//apex_available:platform",
+ "com.android.adbd",
+ ],
host_supported: true,
srcs: [
"AsyncIO.cpp",
diff --git a/liblog/README.md b/liblog/README.md
index 871399a..f64f376 100644
--- a/liblog/README.md
+++ b/liblog/README.md
@@ -118,10 +118,9 @@
finally a call closing the logs. A single log can be opened with `android_logger_list_open()`; or
multiple logs can be opened with `android_logger_list_alloc()`, calling in turn the
`android_logger_open()` for each log id. Each entry can be retrieved with
-`android_logger_list_read()`. The log(s) can be closed with `android_logger_list_free()`. The logs
-should be opened with an `ANDROID_LOG_RDONLY` mode. `ANDROID_LOG_NONBLOCK` mode will report when
-the log reading is done with an `EAGAIN` error return code, otherwise the
-`android_logger_list_read()` call will block for new entries.
+`android_logger_list_read()`. The log(s) can be closed with `android_logger_list_free()`.
+`ANDROID_LOG_NONBLOCK` mode will report when the log reading is done with an `EAGAIN` error return
+code, otherwise the `android_logger_list_read()` call will block for new entries.
The `ANDROID_LOG_WRAP` mode flag to the `android_logger_list_alloc_time()` signals logd to quiesce
the reader until the buffer is about to prune at the start time then proceed to dumping content.
@@ -130,14 +129,12 @@
logs to the persistent logs from before the last reboot.
The value returned by `android_logger_open()` can be used as a parameter to the
-`android_logger_clear()` function to empty the sub-log. It is recommended to only open log
-`ANDROID_LOG_WRONLY` in that case.
+`android_logger_clear()` function to empty the sub-log.
The value returned by `android_logger_open()` can be used as a parameter to the
`android_logger_get_log_(size|readable_size|version)` to retrieve the sub-log maximum size, readable
size and log buffer format protocol version respectively. `android_logger_get_id()` returns the id
-that was used when opening the sub-log. It is recommended to open the log `ANDROID_LOG_RDONLY` in
-these cases.
+that was used when opening the sub-log.
Errors
------
diff --git a/liblog/include/android/log.h b/liblog/include/android/log.h
index 43a91ab..512c7cd 100644
--- a/liblog/include/android/log.h
+++ b/liblog/include/android/log.h
@@ -184,20 +184,21 @@
* Logger data struct used for writing log messages to liblog via __android_log_write_logger_data()
* and sending log messages to user defined loggers specified in __android_log_set_logger().
*/
-struct __android_logger_data {
- size_t struct_size; /* Must be set to sizeof(__android_logger_data) and is used for versioning. */
- int32_t buffer_id; /* log_id_t or -1 to represent 'default'. */
- int32_t priority; /* android_LogPriority values. */
- const char* tag;
- const char* file; /* Optional file name, may be set to nullptr. */
- uint32_t line; /* Optional line number, ignore if file is nullptr. */
+struct __android_log_message {
+ size_t
+ struct_size; /** Must be set to sizeof(__android_log_message) and is used for versioning. */
+ int32_t buffer_id; /** {@link log_id_t} values. */
+ int32_t priority; /** {@link android_LogPriority} values. */
+ const char* tag; /** The tag for the log message. */
+ const char* file; /** Optional file name, may be set to nullptr. */
+ uint32_t line; /** Optional line number, ignore if file is nullptr. */
+ const char* message; /** The log message itself. */
};
/**
* Prototype for the 'logger' function that is called for every log message.
*/
-typedef void (*__android_logger_function)(const struct __android_logger_data* logger_data,
- const char* message);
+typedef void (*__android_logger_function)(const struct __android_log_message* log_message);
/**
* Prototype for the 'abort' function that is called when liblog will abort due to
* __android_log_assert() failures.
@@ -206,52 +207,85 @@
#if !defined(__ANDROID__) || __ANDROID_API__ >= 30
/**
- * Writes the log message specified with logger_data and msg to the log. logger_data includes
- * additional file name and line number information that a logger may use. logger_data is versioned
- * for backwards compatibility.
+ * Writes the log message specified by log_message. log_message includes additional file name and
+ * line number information that a logger may use. log_message is versioned for backwards
+ * compatibility.
* This assumes that loggability has already been checked through __android_log_is_loggable().
* Higher level logging libraries, such as libbase, first check loggability, then format their
* buffers, then pass the message to liblog via this function, and therefore we do not want to
* duplicate the loggability check here.
+ *
+ * @param log_message the log message itself, see {@link __android_log_message}.
+ *
+ * Available since API level 30.
*/
-void __android_log_write_logger_data(struct __android_logger_data* logger_data, const char* msg)
- __INTRODUCED_IN(30);
+void __android_log_write_log_message(struct __android_log_message* log_message) __INTRODUCED_IN(30);
/**
* Sets a user defined logger function. All log messages sent to liblog will be set to the
- * function pointer specified by logger for processing.
+ * function pointer specified by logger for processing. It is not expected that log messages are
+ * already terminated with a new line. This function should add new lines if required for line
+ * separation.
+ *
+ * @param logger the new function that will handle log messages.
+ *
+ * Available since API level 30.
*/
void __android_log_set_logger(__android_logger_function logger) __INTRODUCED_IN(30);
/**
* Writes the log message to logd. This is an __android_logger_function and can be provided to
* __android_log_set_logger(). It is the default logger when running liblog on a device.
+ *
+ * @param log_message the log message to write, see {@link __android_log_message}.
+ *
+ * Available since API level 30.
*/
-void __android_log_logd_logger(const struct __android_logger_data* logger_data, const char* msg)
- __INTRODUCED_IN(30);
+void __android_log_logd_logger(const struct __android_log_message* log_message) __INTRODUCED_IN(30);
/**
* Writes the log message to stderr. This is an __android_logger_function and can be provided to
* __android_log_set_logger(). It is the default logger when running liblog on host.
+ *
+ * @param log_message the log message to write, see {@link __android_log_message}.
+ *
+ * Available since API level 30.
*/
-void __android_log_stderr_logger(const struct __android_logger_data* logger_data,
- const char* message) __INTRODUCED_IN(30);
+void __android_log_stderr_logger(const struct __android_log_message* log_message)
+ __INTRODUCED_IN(30);
/**
- * Sets a user defined aborter function that is called for __android_log_assert() failures.
+ * Sets a user defined aborter function that is called for __android_log_assert() failures. This
+ * user defined aborter function is highly recommended to abort and be noreturn, but is not strictly
+ * required to.
+ *
+ * @param aborter the new aborter function, see {@link __android_aborter_function}.
+ *
+ * Available since API level 30.
*/
void __android_log_set_aborter(__android_aborter_function aborter) __INTRODUCED_IN(30);
/**
* Calls the stored aborter function. This allows for other logging libraries to use the same
* aborter function by calling this function in liblog.
+ *
+ * @param abort_message an additional message supplied when aborting, for example this is used to
+ * call android_set_abort_message() in __android_log_default_aborter().
+ *
+ * Available since API level 30.
*/
void __android_log_call_aborter(const char* abort_message) __INTRODUCED_IN(30);
/**
* Sets android_set_abort_message() on device then aborts(). This is the default aborter.
+ *
+ * @param abort_message an additional message supplied when aborting. This functions calls
+ * android_set_abort_message() with its contents.
+ *
+ * Available since API level 30.
*/
-void __android_log_default_aborter(const char* abort_message) __INTRODUCED_IN(30);
+void __android_log_default_aborter(const char* abort_message) __attribute__((noreturn))
+__INTRODUCED_IN(30);
/**
* Use the per-tag properties "log.tag.<tagname>" along with the minimum priority from
@@ -263,7 +297,13 @@
* minimum priority needed to log. If only one is set, then that value is used to determine the
* minimum priority needed. If none are set, then default_priority is used.
*
- * prio is ANDROID_LOG_VERBOSE to ANDROID_LOG_FATAL.
+ * @param prio the priority to test, takes {@link android_LogPriority} values.
+ * @param tag the tag to test.
+ * @param len the length of the tag.
+ * @param default_prio the default priority to use if no properties or minimum priority are set.
+ * @return an integer where 1 indicates that the message is loggable and 0 indicates that it is not.
+ *
+ * Available since API level 30.
*/
int __android_log_is_loggable(int prio, const char* tag, int default_prio) __INTRODUCED_IN(30);
int __android_log_is_loggable_len(int prio, const char* tag, size_t len, int default_prio)
@@ -272,13 +312,22 @@
/**
* Sets the minimum priority that will be logged for this process.
*
- * This returns the previous set minimum priority, or ANDROID_LOG_DEFAULT if none was set.
+ * @param priority the new minimum priority to set, takes @{link android_LogPriority} values.
+ * @return the previous set minimum priority as @{link android_LogPriority} values, or
+ * ANDROID_LOG_DEFAULT if none was set.
+ *
+ * Available since API level 30.
*/
int32_t __android_log_set_minimum_priority(int32_t priority) __INTRODUCED_IN(30);
/**
* Gets the minimum priority that will be logged for this process. If none has been set by a
* previous __android_log_set_minimum_priority() call, this returns ANDROID_LOG_DEFAULT.
+ *
+ * @return the current minimum priority as @{link android_LogPriority} values, or
+ * ANDROID_LOG_DEFAULT if none is set.
+ *
+ * Available since API level 30.
*/
int32_t __android_log_get_minimum_priority(void) __INTRODUCED_IN(30);
@@ -286,6 +335,10 @@
* Sets the default tag if no tag is provided when writing a log message. Defaults to
* getprogname(). This truncates tag to the maximum log message size, though appropriate tags
* should be much smaller.
+ *
+ * @param tag the new log tag.
+ *
+ * Available since API level 30.
*/
void __android_log_set_default_tag(const char* tag) __INTRODUCED_IN(30);
#endif
diff --git a/liblog/include/log/log_read.h b/liblog/include/log/log_read.h
index 18c1c33..05ad25f 100644
--- a/liblog/include/log/log_read.h
+++ b/liblog/include/log/log_read.h
@@ -141,10 +141,6 @@
char* buf, size_t len);
int android_logger_set_prune_list(struct logger_list* logger_list, const char* buf, size_t len);
-#define ANDROID_LOG_RDONLY O_RDONLY
-#define ANDROID_LOG_WRONLY O_WRONLY
-#define ANDROID_LOG_RDWR O_RDWR
-#define ANDROID_LOG_ACCMODE O_ACCMODE
#ifndef O_NONBLOCK
#define ANDROID_LOG_NONBLOCK 0x00000800
#else
diff --git a/liblog/liblog.map.txt b/liblog/liblog.map.txt
index 198cdae..6ca1a16 100644
--- a/liblog/liblog.map.txt
+++ b/liblog/liblog.map.txt
@@ -65,7 +65,7 @@
android_log_parser_reset; # llndk
};
-LIGLOG_R { # introduced=30
+LIBLOG_R { # introduced=30
global:
__android_log_call_aborter;
__android_log_default_aborter;
@@ -77,7 +77,7 @@
__android_log_set_logger;
__android_log_set_minimum_priority;
__android_log_stderr_logger;
- __android_log_write_logger_data;
+ __android_log_write_log_message;
};
LIBLOG_PRIVATE {
diff --git a/liblog/logger_write.cpp b/liblog/logger_write.cpp
index a8620a0..c174b85 100644
--- a/liblog/logger_write.cpp
+++ b/liblog/logger_write.cpp
@@ -250,8 +250,7 @@
#endif
}
-void __android_log_stderr_logger(const struct __android_logger_data* logger_data,
- const char* message) {
+void __android_log_stderr_logger(const struct __android_log_message* log_message) {
struct tm now;
time_t t = time(nullptr);
@@ -268,33 +267,32 @@
static_assert(arraysize(log_characters) - 1 == ANDROID_LOG_SILENT,
"Mismatch in size of log_characters and values in android_LogPriority");
int32_t priority =
- logger_data->priority > ANDROID_LOG_SILENT ? ANDROID_LOG_FATAL : logger_data->priority;
+ log_message->priority > ANDROID_LOG_SILENT ? ANDROID_LOG_FATAL : log_message->priority;
char priority_char = log_characters[priority];
uint64_t tid = GetThreadId();
- if (logger_data->file != nullptr) {
+ if (log_message->file != nullptr) {
fprintf(stderr, "%s %c %s %5d %5" PRIu64 " %s:%u] %s\n",
- logger_data->tag ? logger_data->tag : "nullptr", priority_char, timestamp, getpid(),
- tid, logger_data->file, logger_data->line, message);
+ log_message->tag ? log_message->tag : "nullptr", priority_char, timestamp, getpid(),
+ tid, log_message->file, log_message->line, log_message->message);
} else {
fprintf(stderr, "%s %c %s %5d %5" PRIu64 " %s\n",
- logger_data->tag ? logger_data->tag : "nullptr", priority_char, timestamp, getpid(),
- tid, message);
+ log_message->tag ? log_message->tag : "nullptr", priority_char, timestamp, getpid(),
+ tid, log_message->message);
}
}
-void __android_log_logd_logger(const struct __android_logger_data* logger_data,
- const char* message) {
- int buffer_id = logger_data->buffer_id == LOG_ID_DEFAULT ? LOG_ID_MAIN : logger_data->buffer_id;
+void __android_log_logd_logger(const struct __android_log_message* log_message) {
+ int buffer_id = log_message->buffer_id == LOG_ID_DEFAULT ? LOG_ID_MAIN : log_message->buffer_id;
struct iovec vec[3];
vec[0].iov_base =
- const_cast<unsigned char*>(reinterpret_cast<const unsigned char*>(&logger_data->priority));
+ const_cast<unsigned char*>(reinterpret_cast<const unsigned char*>(&log_message->priority));
vec[0].iov_len = 1;
- vec[1].iov_base = const_cast<void*>(static_cast<const void*>(logger_data->tag));
- vec[1].iov_len = strlen(logger_data->tag) + 1;
- vec[2].iov_base = const_cast<void*>(static_cast<const void*>(message));
- vec[2].iov_len = strlen(message) + 1;
+ vec[1].iov_base = const_cast<void*>(static_cast<const void*>(log_message->tag));
+ vec[1].iov_len = strlen(log_message->tag) + 1;
+ vec[2].iov_base = const_cast<void*>(static_cast<const void*>(log_message->message));
+ vec[2].iov_len = strlen(log_message->message) + 1;
write_to_log(static_cast<log_id_t>(buffer_id), vec, 3);
}
@@ -303,29 +301,29 @@
return __android_log_buf_write(LOG_ID_MAIN, prio, tag, msg);
}
-void __android_log_write_logger_data(__android_logger_data* logger_data, const char* msg) {
+void __android_log_write_log_message(__android_log_message* log_message) {
ErrnoRestorer errno_restorer;
- if (logger_data->buffer_id != LOG_ID_DEFAULT && logger_data->buffer_id != LOG_ID_MAIN &&
- logger_data->buffer_id != LOG_ID_SYSTEM && logger_data->buffer_id != LOG_ID_RADIO &&
- logger_data->buffer_id != LOG_ID_CRASH) {
+ if (log_message->buffer_id != LOG_ID_DEFAULT && log_message->buffer_id != LOG_ID_MAIN &&
+ log_message->buffer_id != LOG_ID_SYSTEM && log_message->buffer_id != LOG_ID_RADIO &&
+ log_message->buffer_id != LOG_ID_CRASH) {
return;
}
auto tag_lock = std::shared_lock{default_tag_lock, std::defer_lock};
- if (logger_data->tag == nullptr) {
+ if (log_message->tag == nullptr) {
tag_lock.lock();
- logger_data->tag = GetDefaultTag().c_str();
+ log_message->tag = GetDefaultTag().c_str();
}
#if __BIONIC__
- if (logger_data->priority == ANDROID_LOG_FATAL) {
- android_set_abort_message(msg);
+ if (log_message->priority == ANDROID_LOG_FATAL) {
+ android_set_abort_message(log_message->message);
}
#endif
auto lock = std::shared_lock{logger_function_lock};
- logger_function(logger_data, msg);
+ logger_function(log_message);
}
int __android_log_buf_write(int bufID, int prio, const char* tag, const char* msg) {
@@ -335,8 +333,9 @@
return 0;
}
- __android_logger_data logger_data = {sizeof(__android_logger_data), bufID, prio, tag, nullptr, 0};
- __android_log_write_logger_data(&logger_data, msg);
+ __android_log_message log_message = {
+ sizeof(__android_log_message), bufID, prio, tag, nullptr, 0, msg};
+ __android_log_write_log_message(&log_message);
return 1;
}
@@ -351,9 +350,9 @@
vsnprintf(buf, LOG_BUF_SIZE, fmt, ap);
- __android_logger_data logger_data = {
- sizeof(__android_logger_data), LOG_ID_MAIN, prio, tag, nullptr, 0};
- __android_log_write_logger_data(&logger_data, buf);
+ __android_log_message log_message = {
+ sizeof(__android_log_message), LOG_ID_MAIN, prio, tag, nullptr, 0, buf};
+ __android_log_write_log_message(&log_message);
return 1;
}
@@ -371,9 +370,9 @@
vsnprintf(buf, LOG_BUF_SIZE, fmt, ap);
va_end(ap);
- __android_logger_data logger_data = {
- sizeof(__android_logger_data), LOG_ID_MAIN, prio, tag, nullptr, 0};
- __android_log_write_logger_data(&logger_data, buf);
+ __android_log_message log_message = {
+ sizeof(__android_log_message), LOG_ID_MAIN, prio, tag, nullptr, 0, buf};
+ __android_log_write_log_message(&log_message);
return 1;
}
@@ -391,8 +390,9 @@
vsnprintf(buf, LOG_BUF_SIZE, fmt, ap);
va_end(ap);
- __android_logger_data logger_data = {sizeof(__android_logger_data), bufID, prio, tag, nullptr, 0};
- __android_log_write_logger_data(&logger_data, buf);
+ __android_log_message log_message = {
+ sizeof(__android_log_message), bufID, prio, tag, nullptr, 0, buf};
+ __android_log_write_log_message(&log_message);
return 1;
}
diff --git a/liblog/pmsg_reader.cpp b/liblog/pmsg_reader.cpp
index 64a92b7..129d767 100644
--- a/liblog/pmsg_reader.cpp
+++ b/liblog/pmsg_reader.cpp
@@ -185,7 +185,7 @@
/* Add just enough clues in logger_list and transp to make API function */
memset(&logger_list, 0, sizeof(logger_list));
- logger_list.mode = ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK | ANDROID_LOG_RDONLY;
+ logger_list.mode = ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK;
logger_list.log_mask = (unsigned)-1;
if (logId != LOG_ID_ANY) {
logger_list.log_mask = (1 << logId);
diff --git a/liblog/tests/Android.bp b/liblog/tests/Android.bp
index b4bb77f..fffb809 100644
--- a/liblog/tests/Android.bp
+++ b/liblog/tests/Android.bp
@@ -97,6 +97,7 @@
test_suites: [
"cts",
"vts",
+ "vts10",
],
}
diff --git a/liblog/tests/liblog_benchmark.cpp b/liblog/tests/liblog_benchmark.cpp
index 4366f3d..3a6ed90 100644
--- a/liblog/tests/liblog_benchmark.cpp
+++ b/liblog/tests/liblog_benchmark.cpp
@@ -648,8 +648,7 @@
static void BM_log_latency(benchmark::State& state) {
pid_t pid = getpid();
- struct logger_list* logger_list =
- android_logger_list_open(LOG_ID_EVENTS, ANDROID_LOG_RDONLY, 0, pid);
+ struct logger_list* logger_list = android_logger_list_open(LOG_ID_EVENTS, 0, 0, pid);
if (!logger_list) {
fprintf(stderr, "Unable to open events log: %s\n", strerror(errno));
@@ -723,8 +722,7 @@
static void BM_log_delay(benchmark::State& state) {
pid_t pid = getpid();
- struct logger_list* logger_list =
- android_logger_list_open(LOG_ID_EVENTS, ANDROID_LOG_RDONLY, 0, pid);
+ struct logger_list* logger_list = android_logger_list_open(LOG_ID_EVENTS, 0, 0, pid);
if (!logger_list) {
fprintf(stderr, "Unable to open events log: %s\n", strerror(errno));
diff --git a/liblog/tests/liblog_global_state.cpp b/liblog/tests/liblog_global_state.cpp
index 9a181ef..3508818 100644
--- a/liblog/tests/liblog_global_state.cpp
+++ b/liblog/tests/liblog_global_state.cpp
@@ -59,16 +59,15 @@
static unsigned int expected_line;
static std::string expected_message = "libbase test message";
- auto liblog_logger_function = [](const struct __android_logger_data* logger_data,
- const char* message) {
+ auto liblog_logger_function = [](const struct __android_log_message* log_message) {
message_seen = true;
- EXPECT_EQ(sizeof(__android_logger_data), logger_data->struct_size);
- EXPECT_EQ(LOG_ID_DEFAULT, logger_data->buffer_id);
- EXPECT_EQ(ANDROID_LOG_WARN, logger_data->priority);
- EXPECT_STREQ(LOG_TAG, logger_data->tag);
- EXPECT_EQ(expected_file, logger_data->file);
- EXPECT_EQ(expected_line, logger_data->line);
- EXPECT_EQ(expected_message, message);
+ EXPECT_EQ(sizeof(__android_log_message), log_message->struct_size);
+ EXPECT_EQ(LOG_ID_DEFAULT, log_message->buffer_id);
+ EXPECT_EQ(ANDROID_LOG_WARN, log_message->priority);
+ EXPECT_STREQ(LOG_TAG, log_message->tag);
+ EXPECT_EQ(expected_file, log_message->file);
+ EXPECT_EQ(expected_line, log_message->line);
+ EXPECT_EQ(expected_message, log_message->message);
};
__android_log_set_logger(liblog_logger_function);
@@ -111,16 +110,15 @@
static int expected_priority = ANDROID_LOG_WARN;
static std::string expected_message = "libbase test message";
- auto liblog_logger_function = [](const struct __android_logger_data* logger_data,
- const char* message) {
+ auto liblog_logger_function = [](const struct __android_log_message* log_message) {
message_seen = true;
- EXPECT_EQ(sizeof(__android_logger_data), logger_data->struct_size);
- EXPECT_EQ(expected_buffer_id, logger_data->buffer_id);
- EXPECT_EQ(expected_priority, logger_data->priority);
- EXPECT_STREQ(LOG_TAG, logger_data->tag);
- EXPECT_STREQ(nullptr, logger_data->file);
- EXPECT_EQ(0U, logger_data->line);
- EXPECT_EQ(expected_message, message);
+ EXPECT_EQ(sizeof(__android_log_message), log_message->struct_size);
+ EXPECT_EQ(expected_buffer_id, log_message->buffer_id);
+ EXPECT_EQ(expected_priority, log_message->priority);
+ EXPECT_STREQ(LOG_TAG, log_message->tag);
+ EXPECT_STREQ(nullptr, log_message->file);
+ EXPECT_EQ(0U, log_message->line);
+ EXPECT_EQ(expected_message, log_message->message);
};
__android_log_set_logger(liblog_logger_function);
diff --git a/liblog/tests/liblog_test.cpp b/liblog/tests/liblog_test.cpp
index 773bac2..a031531 100644
--- a/liblog/tests/liblog_test.cpp
+++ b/liblog/tests/liblog_test.cpp
@@ -82,7 +82,7 @@
pid_t pid = getpid();
auto logger_list = std::unique_ptr<struct logger_list, ListCloser>{
- android_logger_list_open(log_buffer, ANDROID_LOG_RDONLY, 1000, pid)};
+ android_logger_list_open(log_buffer, 0, 1000, pid)};
ASSERT_TRUE(logger_list);
write_messages();
@@ -106,7 +106,7 @@
}
auto logger_list_non_block = std::unique_ptr<struct logger_list, ListCloser>{
- android_logger_list_open(log_buffer, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, 1000, pid)};
+ android_logger_list_open(log_buffer, ANDROID_LOG_NONBLOCK, 1000, pid)};
ASSERT_TRUE(logger_list_non_block);
size_t count = 0;
@@ -572,8 +572,7 @@
v += pid & 0xFFFF;
- ASSERT_TRUE(NULL != (logger_list = android_logger_list_open(
- LOG_ID_EVENTS, ANDROID_LOG_RDONLY, 1000, pid)));
+ ASSERT_TRUE(NULL != (logger_list = android_logger_list_open(LOG_ID_EVENTS, 0, 1000, pid)));
int count = 0;
@@ -728,8 +727,7 @@
v += pid & 0xFFFF;
- ASSERT_TRUE(NULL != (logger_list = android_logger_list_open(
- LOG_ID_EVENTS, ANDROID_LOG_RDONLY, 1000, pid)));
+ ASSERT_TRUE(NULL != (logger_list = android_logger_list_open(LOG_ID_EVENTS, 0, 1000, pid)));
int count = 0;
@@ -1093,11 +1091,11 @@
pid_t pid = getpid();
auto logger_list1 = std::unique_ptr<struct logger_list, ListCloser>{
- android_logger_list_open(LOG_ID_MAIN, ANDROID_LOG_RDONLY, expected_count1, pid)};
+ android_logger_list_open(LOG_ID_MAIN, 0, expected_count1, pid)};
ASSERT_TRUE(logger_list1);
auto logger_list2 = std::unique_ptr<struct logger_list, ListCloser>{
- android_logger_list_open(LOG_ID_MAIN, ANDROID_LOG_RDONLY, expected_count2, pid)};
+ android_logger_list_open(LOG_ID_MAIN, 0, expected_count2, pid)};
ASSERT_TRUE(logger_list2);
for (int i = 25; i > 0; --i) {
@@ -1128,14 +1126,12 @@
}
// Test again with the nonblocking reader.
- auto logger_list_non_block1 =
- std::unique_ptr<struct logger_list, ListCloser>{android_logger_list_open(
- LOG_ID_MAIN, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, expected_count1, pid)};
+ auto logger_list_non_block1 = std::unique_ptr<struct logger_list, ListCloser>{
+ android_logger_list_open(LOG_ID_MAIN, ANDROID_LOG_NONBLOCK, expected_count1, pid)};
ASSERT_TRUE(logger_list_non_block1);
- auto logger_list_non_block2 =
- std::unique_ptr<struct logger_list, ListCloser>{android_logger_list_open(
- LOG_ID_MAIN, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, expected_count2, pid)};
+ auto logger_list_non_block2 = std::unique_ptr<struct logger_list, ListCloser>{
+ android_logger_list_open(LOG_ID_MAIN, ANDROID_LOG_NONBLOCK, expected_count2, pid)};
ASSERT_TRUE(logger_list_non_block2);
count1 = 0;
count2 = 0;
@@ -1542,8 +1538,8 @@
pid_t pid = getpid();
- struct logger_list* logger_list = android_logger_list_open(
- LOG_ID_EVENTS, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK, 1000, pid);
+ struct logger_list* logger_list =
+ android_logger_list_open(LOG_ID_EVENTS, ANDROID_LOG_NONBLOCK, 1000, pid);
int count = 0;
if (logger_list == NULL) return count;
@@ -1832,10 +1828,8 @@
gid = getgid();
pid_t pid = getpid();
- ASSERT_TRUE(NULL !=
- (logger_list = android_logger_list_open(
- LOG_ID_SECURITY, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK,
- 1000, pid)));
+ ASSERT_TRUE(NULL != (logger_list = android_logger_list_open(LOG_ID_SECURITY, ANDROID_LOG_NONBLOCK,
+ 1000, pid)));
log_time ts(CLOCK_MONOTONIC);
diff --git a/liblog/tests/log_read_test.cpp b/liblog/tests/log_read_test.cpp
index 1be99aa..3e09617 100644
--- a/liblog/tests/log_read_test.cpp
+++ b/liblog/tests/log_read_test.cpp
@@ -34,8 +34,7 @@
// This test assumes the log buffers are filled with noise from
// normal operations. It will fail if done immediately after a
// logcat -c.
- struct logger_list* logger_list =
- android_logger_list_alloc(ANDROID_LOG_WRONLY, 0, 0);
+ struct logger_list* logger_list = android_logger_list_alloc(0, 0, 0);
for (int i = LOG_ID_MIN; i < LOG_ID_MAX; ++i) {
log_id_t id = static_cast<log_id_t>(i);
diff --git a/liblog/tests/log_wrap_test.cpp b/liblog/tests/log_wrap_test.cpp
index e06964f..755898a 100644
--- a/liblog/tests/log_wrap_test.cpp
+++ b/liblog/tests/log_wrap_test.cpp
@@ -32,7 +32,7 @@
static void read_with_wrap() {
// Read the last line in the log to get a starting timestamp. We're assuming
// the log is not empty.
- const int mode = ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK;
+ const int mode = ANDROID_LOG_NONBLOCK;
struct logger_list* logger_list =
android_logger_list_open(LOG_ID_MAIN, mode, 1000, 0);
diff --git a/libziparchive/Android.bp b/libziparchive/Android.bp
index 4081b21..786e7b3 100644
--- a/libziparchive/Android.bp
+++ b/libziparchive/Android.bp
@@ -212,3 +212,20 @@
data: ["cli-tests/**/*"],
target_required: ["cli-test", "ziptool"],
}
+
+python_test_host {
+ name: "ziparchive_tests_large",
+ srcs: ["test_ziparchive_large.py"],
+ main: "test_ziparchive_large.py",
+ version: {
+ py2: {
+ enabled: true,
+ embedded_launcher: false,
+ },
+ py3: {
+ enabled: false,
+ embedded_launcher: false,
+ },
+ },
+ test_suites: ["general-tests"],
+}
diff --git a/libziparchive/test_ziparchive_large.py b/libziparchive/test_ziparchive_large.py
new file mode 100644
index 0000000..c29c37e
--- /dev/null
+++ b/libziparchive/test_ziparchive_large.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Unittests for parsing files in zip64 format"""
+
+import os
+import subprocess
+import tempfile
+import unittest
+import zipfile
+import time
+
+class Zip64Test(unittest.TestCase):
+ @staticmethod
+ def _AddEntriesToZip(output_zip, entries_dict=None):
+ for name, size in entries_dict.items():
+ contents = name[0] * 1024
+ file_path = tempfile.NamedTemporaryFile()
+ with open(file_path.name, 'w') as f:
+ for it in range(0, size):
+ f.write(contents)
+ output_zip.write(file_path.name, arcname = name)
+
+ def _getEntryNames(self, zip_name):
+ cmd = ['ziptool', 'zipinfo', '-1', zip_name]
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ output, _ = proc.communicate()
+ self.assertEquals(0, proc.returncode)
+ self.assertNotEqual(None, output)
+ return output.split()
+
+ def _ExtractEntries(self, zip_name):
+ temp_dir = tempfile.mkdtemp()
+ cmd = ['ziptool', 'unzip', '-d', temp_dir, zip_name]
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ proc.communicate()
+ self.assertEquals(0, proc.returncode)
+
+ def test_entriesSmallerThan2G(self):
+ zip_path = tempfile.NamedTemporaryFile(suffix='.zip')
+ # Add a few entries with each of them smaller than 2GiB. But the entire zip file is larger
+ # than 4GiB in size.
+ with zipfile.ZipFile(zip_path, 'w', allowZip64=True) as output_zip:
+ entry_dict = {'a.txt': 1025 * 1024, 'b.txt': 1025 * 1024, 'c.txt': 1025 * 1024,
+ 'd.txt': 1025 * 1024, 'e.txt': 1024}
+ self._AddEntriesToZip(output_zip, entry_dict)
+
+ read_names = self._getEntryNames(zip_path.name)
+ self.assertEquals(sorted(entry_dict.keys()), sorted(read_names))
+ self._ExtractEntries(zip_path.name)
+
+
+ def test_largeNumberOfEntries(self):
+ zip_path = tempfile.NamedTemporaryFile(suffix='.zip')
+ entry_dict = {}
+ # Add 100k entries (more than 65535|UINT16_MAX).
+ for num in range(0, 100 * 1024):
+ entry_dict[str(num)] = 50
+
+ with zipfile.ZipFile(zip_path, 'w', allowZip64=True) as output_zip:
+ self._AddEntriesToZip(output_zip, entry_dict)
+
+ read_names = self._getEntryNames(zip_path.name)
+ self.assertEquals(sorted(entry_dict.keys()), sorted(read_names))
+ self._ExtractEntries(zip_path.name)
+
+
+ def test_largeCompressedEntries(self):
+ zip_path = tempfile.NamedTemporaryFile(suffix='.zip')
+ with zipfile.ZipFile(zip_path, 'w', compression=zipfile.ZIP_DEFLATED,
+ allowZip64=True) as output_zip:
+ # Add entries close to 4GiB in size. Somehow the python library will put the (un)compressed
+ # sizes in the extra field. Test if our ziptool should be able to parse it.
+ entry_dict = {'e.txt': 4095 * 1024, 'f.txt': 4095 * 1024}
+ self._AddEntriesToZip(output_zip, entry_dict)
+
+ read_names = self._getEntryNames(zip_path.name)
+ self.assertEquals(sorted(entry_dict.keys()), sorted(read_names))
+ self._ExtractEntries(zip_path.name)
+
+
+if __name__ == '__main__':
+ testsuite = unittest.TestLoader().discover(
+ os.path.dirname(os.path.realpath(__file__)))
+ unittest.TextTestRunner(verbosity=2).run(testsuite)
diff --git a/libziparchive/zip_archive.cc b/libziparchive/zip_archive.cc
index 6b6502b..9812026 100644
--- a/libziparchive/zip_archive.cc
+++ b/libziparchive/zip_archive.cc
@@ -139,9 +139,60 @@
uint64_t cd_start_offset;
};
-static ZipError FindCentralDirectoryInfoForZip64(CentralDirectoryInfo* /* cdInfo */) {
- ALOGW("Zip: Parsing zip64 EOCD isn't supported yet.");
- return kInvalidFile;
+static ZipError FindCentralDirectoryInfoForZip64(const char* debugFileName, ZipArchive* archive,
+ off64_t eocdOffset, CentralDirectoryInfo* cdInfo) {
+ if (eocdOffset <= sizeof(Zip64EocdLocator)) {
+ ALOGW("Zip: %s: Not enough space for zip64 eocd locator", debugFileName);
+ return kInvalidFile;
+ }
+ // We expect to find the zip64 eocd locator immediately before the zip eocd.
+ const int64_t locatorOffset = eocdOffset - sizeof(Zip64EocdLocator);
+ Zip64EocdLocator zip64EocdLocator{};
+ if (!archive->mapped_zip.ReadAtOffset(reinterpret_cast<uint8_t*>((&zip64EocdLocator)),
+ sizeof(Zip64EocdLocator), locatorOffset)) {
+ ALOGW("Zip: %s: Read %zu from offset %" PRId64 " failed %s", debugFileName,
+ sizeof(Zip64EocdLocator), locatorOffset, debugFileName);
+ return kIoError;
+ }
+
+ if (zip64EocdLocator.locator_signature != Zip64EocdLocator::kSignature) {
+ ALOGW("Zip: %s: Zip64 eocd locator signature not found at offset %" PRId64, debugFileName,
+ locatorOffset);
+ return kInvalidFile;
+ }
+
+ const int64_t zip64EocdOffset = zip64EocdLocator.zip64_eocd_offset;
+ if (zip64EocdOffset > locatorOffset - sizeof(Zip64EocdRecord)) {
+ ALOGW("Zip: %s: Bad zip64 eocd offset %" PRIu64, debugFileName, zip64EocdOffset);
+ return kInvalidOffset;
+ }
+
+ Zip64EocdRecord zip64EocdRecord{};
+ if (!archive->mapped_zip.ReadAtOffset(reinterpret_cast<uint8_t*>(&zip64EocdRecord),
+ sizeof(Zip64EocdRecord), zip64EocdOffset)) {
+ ALOGW("Zip: %s: read %zu from offset %" PRId64 " failed %s", debugFileName,
+ sizeof(Zip64EocdLocator), static_cast<int64_t>(zip64EocdOffset), debugFileName);
+ return kIoError;
+ }
+
+ if (zip64EocdRecord.record_signature != Zip64EocdRecord::kSignature) {
+ ALOGW("Zip: %s: Zip64 eocd record signature not found at offset %" PRId64, debugFileName,
+ zip64EocdOffset);
+ return kInvalidFile;
+ }
+
+ if (zip64EocdRecord.cd_start_offset > zip64EocdOffset - zip64EocdRecord.cd_size) {
+ ALOGW("Zip: %s: Bad offset for zip64 central directory. cd offset %" PRIu64 ", cd size %" PRIu64
+ ", zip64 eocd offset %" PRIu64,
+ debugFileName, zip64EocdRecord.cd_start_offset, zip64EocdRecord.cd_size, zip64EocdOffset);
+ return kInvalidOffset;
+ }
+
+ *cdInfo = {.num_records = zip64EocdRecord.num_records,
+ .cd_size = zip64EocdRecord.cd_size,
+ .cd_start_offset = zip64EocdRecord.cd_start_offset};
+
+ return kSuccess;
}
static ZipError FindCentralDirectoryInfo(const char* debug_file_name, ZipArchive* archive,
@@ -195,7 +246,7 @@
if (eocd->cd_size == UINT32_MAX || eocd->cd_start_offset == UINT32_MAX) {
ALOGV("Looking for the zip64 EOCD, cd_size: %" PRIu32 "cd_start_offset: %" PRId32,
eocd->cd_size, eocd->cd_start_offset);
- return FindCentralDirectoryInfoForZip64(cdInfo);
+ return FindCentralDirectoryInfoForZip64(debug_file_name, archive, eocd_offset, cdInfo);
}
/*
@@ -291,13 +342,104 @@
return kSuccess;
}
+static ZipError ParseZip64ExtendedInfoInExtraField(
+ const uint8_t* extraFieldStart, uint16_t extraFieldLength, uint32_t zip32UncompressedSize,
+ uint32_t zip32CompressedSize, std::optional<uint32_t> zip32LocalFileHeaderOffset,
+ Zip64ExtendedInfo* zip64Info) {
+ if (extraFieldLength <= 4) {
+ ALOGW("Zip: Extra field isn't large enough to hold zip64 info, size %" PRIu16,
+ extraFieldLength);
+ return kInvalidFile;
+ }
+
+ // Each header MUST consist of:
+ // Header ID - 2 bytes
+ // Data Size - 2 bytes
+ uint16_t offset = 0;
+ while (offset < extraFieldLength - 4) {
+ auto headerId = get_unaligned<uint16_t>(extraFieldStart + offset);
+ auto dataSize = get_unaligned<uint16_t>(extraFieldStart + offset + 2);
+
+ offset += 4;
+ if (dataSize > extraFieldLength - offset) {
+ ALOGW("Zip: Data size exceeds the boundary of extra field, data size %" PRIu16, dataSize);
+ return kInvalidOffset;
+ }
+
+ // Skip the other types of extensible data fields. Details in
+ // https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT section 4.5
+ if (headerId != Zip64ExtendedInfo::kHeaderId) {
+ offset += dataSize;
+ continue;
+ }
+
+ uint16_t expectedDataSize = 0;
+ // We expect the extended field to include both uncompressed and compressed size.
+ if (zip32UncompressedSize == UINT32_MAX || zip32CompressedSize == UINT32_MAX) {
+ expectedDataSize += 16;
+ }
+ if (zip32LocalFileHeaderOffset == UINT32_MAX) {
+ expectedDataSize += 8;
+ }
+
+ if (expectedDataSize == 0) {
+ ALOGW("Zip: Data size should not be 0 in zip64 extended field");
+ return kInvalidFile;
+ }
+
+ if (dataSize != expectedDataSize) {
+ auto localOffsetString = zip32LocalFileHeaderOffset.has_value()
+ ? std::to_string(zip32LocalFileHeaderOffset.value())
+ : "missing";
+ ALOGW("Zip: Invalid data size in zip64 extended field, expect %" PRIu16 ", get %" PRIu16
+ ", uncompressed size %" PRIu32 ", compressed size %" PRIu32 ", local header offset %s",
+ expectedDataSize, dataSize, zip32UncompressedSize, zip32CompressedSize,
+ localOffsetString.c_str());
+ return kInvalidFile;
+ }
+
+ std::optional<uint64_t> uncompressedFileSize;
+ std::optional<uint64_t> compressedFileSize;
+ std::optional<uint64_t> localHeaderOffset;
+ if (zip32UncompressedSize == UINT32_MAX || zip32CompressedSize == UINT32_MAX) {
+ uncompressedFileSize = get_unaligned<uint64_t>(extraFieldStart + offset);
+ compressedFileSize = get_unaligned<uint64_t>(extraFieldStart + offset + 8);
+ offset += 16;
+
+ // TODO(xunchang) Support handling file large than UINT32_MAX. It's theoretically possible
+ // for libz to (de)compressing file larger than UINT32_MAX. But we should use our own
+ // bytes counter to replace stream.total_out.
+ if (uncompressedFileSize.value() >= UINT32_MAX || compressedFileSize.value() >= UINT32_MAX) {
+ ALOGW(
+ "Zip: File size larger than UINT32_MAX isn't supported yet. uncompressed size %" PRIu64
+ ", compressed size %" PRIu64,
+ uncompressedFileSize.value(), compressedFileSize.value());
+ return kInvalidFile;
+ }
+ }
+
+ if (zip32LocalFileHeaderOffset == UINT32_MAX) {
+ localHeaderOffset = get_unaligned<uint64_t>(extraFieldStart + offset);
+ offset += 8;
+ }
+
+ zip64Info->uncompressed_file_size = uncompressedFileSize;
+ zip64Info->compressed_file_size = compressedFileSize;
+ zip64Info->local_header_offset = localHeaderOffset;
+ return kSuccess;
+ }
+
+ ALOGW("Zip: zip64 extended info isn't found in the extra field.");
+ return kInvalidFile;
+}
+
/*
* Parses the Zip archive's Central Directory. Allocates and populates the
* hash table.
*
* Returns 0 on success.
*/
-static int32_t ParseZipArchive(ZipArchive* archive) {
+static ZipError ParseZipArchive(ZipArchive* archive) {
const uint8_t* const cd_ptr = archive->central_directory.GetBasePtr();
const size_t cd_length = archive->central_directory.GetMapLength();
const uint64_t num_entries = archive->num_entries;
@@ -327,7 +469,7 @@
return kInvalidFile;
}
- const CentralDirectoryRecord* cdr = reinterpret_cast<const CentralDirectoryRecord*>(ptr);
+ auto cdr = reinterpret_cast<const CentralDirectoryRecord*>(ptr);
if (cdr->record_signature != CentralDirectoryRecord::kSignature) {
ALOGW("Zip: missed a central dir sig (at %" PRIu64 ")", i);
return kInvalidFile;
@@ -355,9 +497,15 @@
off64_t local_header_offset = cdr->local_file_header_offset;
if (local_header_offset == UINT32_MAX) {
- // TODO(xunchang) parse the zip64 eocd
- ALOGW("Zip: Parsing zip64 cd entry isn't supported yet");
- return kInvalidFile;
+ Zip64ExtendedInfo zip64_info{};
+ if (auto status = ParseZip64ExtendedInfoInExtraField(
+ extra_field, extra_length, cdr->uncompressed_size, cdr->compressed_size,
+ cdr->local_file_header_offset, &zip64_info);
+ status != kSuccess) {
+ return status;
+ }
+ CHECK(zip64_info.local_header_offset.has_value());
+ local_header_offset = zip64_info.local_header_offset.value();
}
if (local_header_offset >= archive->directory_offset) {
@@ -405,7 +553,7 @@
ALOGV("+++ zip good scan %" PRIu64 " entries", num_entries);
- return 0;
+ return kSuccess;
}
static int32_t OpenArchiveInternal(ZipArchive* archive, const char* debug_file_name) {
@@ -522,7 +670,7 @@
return kInvalidOffset;
}
- const CentralDirectoryRecord* cdr = reinterpret_cast<const CentralDirectoryRecord*>(ptr);
+ auto cdr = reinterpret_cast<const CentralDirectoryRecord*>(ptr);
// The offset of the start of the central directory in the zipfile.
// We keep this lying around so that we can sanity check all our lengths
@@ -546,8 +694,27 @@
// the extra field.
if (cdr->uncompressed_size == UINT32_MAX || cdr->compressed_size == UINT32_MAX ||
cdr->local_file_header_offset == UINT32_MAX) {
- ALOGW("Zip: Parsing zip64 local file header isn't supported yet");
- return kInvalidFile;
+ const uint8_t* extra_field = ptr + sizeof(CentralDirectoryRecord) + cdr->file_name_length;
+ Zip64ExtendedInfo zip64_info{};
+ if (auto status = ParseZip64ExtendedInfoInExtraField(
+ extra_field, cdr->extra_field_length, cdr->uncompressed_size, cdr->compressed_size,
+ cdr->local_file_header_offset, &zip64_info);
+ status != kSuccess) {
+ return status;
+ }
+
+ if (cdr->uncompressed_size == UINT32_MAX || cdr->compressed_size == UINT32_MAX) {
+ CHECK(zip64_info.uncompressed_file_size.has_value());
+ CHECK(zip64_info.compressed_file_size.has_value());
+ // TODO(xunchang) remove the size limit and support entry length > UINT32_MAX.
+ data->uncompressed_length = static_cast<uint32_t>(zip64_info.uncompressed_file_size.value());
+ data->compressed_length = static_cast<uint32_t>(zip64_info.compressed_file_size.value());
+ }
+
+ if (local_header_offset == UINT32_MAX) {
+ CHECK(zip64_info.local_header_offset.has_value());
+ local_header_offset = zip64_info.local_header_offset.value();
+ }
}
if (local_header_offset + static_cast<off64_t>(sizeof(LocalFileHeader)) >= cd_offset) {
@@ -562,14 +729,68 @@
return kIoError;
}
- const LocalFileHeader* lfh = reinterpret_cast<const LocalFileHeader*>(lfh_buf);
-
+ auto lfh = reinterpret_cast<const LocalFileHeader*>(lfh_buf);
if (lfh->lfh_signature != LocalFileHeader::kSignature) {
ALOGW("Zip: didn't find signature at start of lfh, offset=%" PRId64,
static_cast<int64_t>(local_header_offset));
return kInvalidOffset;
}
+ // Check that the local file header name matches the declared name in the central directory.
+ CHECK_LE(entryName.size(), UINT16_MAX);
+ auto nameLen = static_cast<uint16_t>(entryName.size());
+ if (lfh->file_name_length != nameLen) {
+ ALOGW("Zip: lfh name length did not match central directory for %s: %" PRIu16 " %" PRIu16,
+ std::string(entryName).c_str(), lfh->file_name_length, nameLen);
+ return kInconsistentInformation;
+ }
+ const off64_t name_offset = local_header_offset + sizeof(LocalFileHeader);
+ if (name_offset > cd_offset - lfh->file_name_length) {
+ ALOGW("Zip: lfh name has invalid declared length");
+ return kInvalidOffset;
+ }
+
+ std::vector<uint8_t> name_buf(nameLen);
+ if (!archive->mapped_zip.ReadAtOffset(name_buf.data(), nameLen, name_offset)) {
+ ALOGW("Zip: failed reading lfh name from offset %" PRId64, static_cast<int64_t>(name_offset));
+ return kIoError;
+ }
+ if (memcmp(entryName.data(), name_buf.data(), nameLen) != 0) {
+ ALOGW("Zip: lfh name did not match central directory");
+ return kInconsistentInformation;
+ }
+
+ uint64_t lfh_uncompressed_size = lfh->uncompressed_size;
+ uint64_t lfh_compressed_size = lfh->compressed_size;
+ if (lfh_uncompressed_size == UINT32_MAX || lfh_compressed_size == UINT32_MAX) {
+ const off64_t lfh_extra_field_offset = name_offset + lfh->file_name_length;
+ const uint16_t lfh_extra_field_size = lfh->extra_field_length;
+ if (lfh_extra_field_offset > cd_offset - lfh_extra_field_size) {
+ ALOGW("Zip: extra field has a bad size for entry %s", std::string(entryName).c_str());
+ return kInvalidOffset;
+ }
+
+ std::vector<uint8_t> local_extra_field(lfh_extra_field_size);
+ if (!archive->mapped_zip.ReadAtOffset(local_extra_field.data(), lfh_extra_field_size,
+ lfh_extra_field_offset)) {
+ ALOGW("Zip: failed reading lfh extra field from offset %" PRId64, lfh_extra_field_offset);
+ return kIoError;
+ }
+
+ Zip64ExtendedInfo zip64_info{};
+ if (auto status = ParseZip64ExtendedInfoInExtraField(
+ local_extra_field.data(), lfh_extra_field_size, lfh->uncompressed_size,
+ lfh->compressed_size, std::nullopt, &zip64_info);
+ status != kSuccess) {
+ return status;
+ }
+
+ CHECK(zip64_info.uncompressed_file_size.has_value());
+ CHECK(zip64_info.compressed_file_size.has_value());
+ lfh_uncompressed_size = zip64_info.uncompressed_file_size.value();
+ lfh_compressed_size = zip64_info.compressed_file_size.value();
+ }
+
// Paranoia: Match the values specified in the local file header
// to those specified in the central directory.
@@ -595,12 +816,12 @@
// header agree on the crc, compressed, and uncompressed sizes of the entry.
if ((lfh->gpb_flags & kGPBDDFlagMask) == 0) {
data->has_data_descriptor = 0;
- if (data->compressed_length != lfh->compressed_size ||
- data->uncompressed_length != lfh->uncompressed_size || data->crc32 != lfh->crc32) {
+ if (data->compressed_length != lfh_compressed_size ||
+ data->uncompressed_length != lfh_uncompressed_size || data->crc32 != lfh->crc32) {
ALOGW("Zip: size/crc32 mismatch. expected {%" PRIu32 ", %" PRIu32 ", %" PRIx32
- "}, was {%" PRIu32 ", %" PRIu32 ", %" PRIx32 "}",
- data->compressed_length, data->uncompressed_length, data->crc32, lfh->compressed_size,
- lfh->uncompressed_size, lfh->crc32);
+ "}, was {%" PRIu64 ", %" PRIu64 ", %" PRIx32 "}",
+ data->compressed_length, data->uncompressed_length, data->crc32, lfh_compressed_size,
+ lfh_uncompressed_size, lfh->crc32);
return kInconsistentInformation;
}
} else {
@@ -623,30 +844,6 @@
// Currently only needed to implement zipinfo.
data->is_text = (cdr->internal_file_attributes & 1);
- // Check that the local file header name matches the declared
- // name in the central directory.
- CHECK_LE(entryName.size(), UINT16_MAX);
- auto nameLen = static_cast<uint16_t>(entryName.size());
- if (lfh->file_name_length != nameLen) {
- ALOGW("Zip: lfh name length did not match central directory for %s: %" PRIu16 " %" PRIu16,
- std::string(entryName).c_str(), lfh->file_name_length, nameLen);
- return kInconsistentInformation;
- }
- const off64_t name_offset = local_header_offset + sizeof(LocalFileHeader);
- if (name_offset + lfh->file_name_length > cd_offset) {
- ALOGW("Zip: lfh name has invalid declared length");
- return kInvalidOffset;
- }
- std::vector<uint8_t> name_buf(nameLen);
- if (!archive->mapped_zip.ReadAtOffset(name_buf.data(), nameLen, name_offset)) {
- ALOGW("Zip: failed reading lfh name from offset %" PRId64, static_cast<int64_t>(name_offset));
- return kIoError;
- }
- if (memcmp(entryName.data(), name_buf.data(), nameLen) != 0) {
- ALOGW("Zip: lfh name did not match central directory");
- return kInconsistentInformation;
- }
-
const off64_t data_offset = local_header_offset + sizeof(LocalFileHeader) +
lfh->file_name_length + lfh->extra_field_length;
if (data_offset > cd_offset) {
diff --git a/libziparchive/zip_archive_test.cc b/libziparchive/zip_archive_test.cc
index 6c1a9a1..fbabf96 100644
--- a/libziparchive/zip_archive_test.cc
+++ b/libziparchive/zip_archive_test.cc
@@ -14,8 +14,6 @@
* limitations under the License.
*/
-#include "zip_archive_private.h"
-
#include <errno.h>
#include <fcntl.h>
#include <getopt.h>
@@ -23,6 +21,7 @@
#include <string.h>
#include <unistd.h>
+#include <map>
#include <memory>
#include <set>
#include <string_view>
@@ -31,12 +30,16 @@
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/mapped_file.h>
+#include <android-base/memory.h>
#include <android-base/strings.h>
#include <android-base/unique_fd.h>
#include <gtest/gtest.h>
#include <ziparchive/zip_archive.h>
#include <ziparchive/zip_archive_stream_entry.h>
+#include "zip_archive_common.h"
+#include "zip_archive_private.h"
+
static std::string test_data_dir = android::base::GetExecutableDirectory() + "/testdata";
static const std::string kValidZip = "valid.zip";
@@ -966,3 +969,290 @@
ASSERT_EQ(0u, writer.GetOutput().size());
}
}
+
+// The class constructs a zipfile with zip64 format, and test the parsing logic.
+class Zip64ParseTest : public ::testing::Test {
+ protected:
+ struct LocalFileEntry {
+ std::vector<uint8_t> local_file_header;
+ std::string file_name;
+ std::vector<uint8_t> extended_field;
+ // Fake data to mimic the compressed bytes in the zipfile.
+ std::vector<uint8_t> compressed_bytes;
+
+ size_t GetSize() const {
+ return local_file_header.size() + file_name.size() + extended_field.size() +
+ compressed_bytes.size();
+ }
+
+ void CopyToOutput(std::vector<uint8_t>* output) const {
+ std::copy(local_file_header.begin(), local_file_header.end(), std::back_inserter(*output));
+ std::copy(file_name.begin(), file_name.end(), std::back_inserter(*output));
+ std::copy(extended_field.begin(), extended_field.end(), std::back_inserter(*output));
+ std::copy(compressed_bytes.begin(), compressed_bytes.end(), std::back_inserter(*output));
+ }
+ };
+
+ struct CdRecordEntry {
+ std::vector<uint8_t> central_directory_record;
+ std::string file_name;
+ std::vector<uint8_t> extended_field;
+
+ size_t GetSize() const {
+ return central_directory_record.size() + file_name.size() + extended_field.size();
+ }
+
+ void CopyToOutput(std::vector<uint8_t>* output) const {
+ std::copy(central_directory_record.begin(), central_directory_record.end(),
+ std::back_inserter(*output));
+ std::copy(file_name.begin(), file_name.end(), std::back_inserter(*output));
+ std::copy(extended_field.begin(), extended_field.end(), std::back_inserter(*output));
+ }
+ };
+
+ static void ConstructLocalFileHeader(const std::string& name, std::vector<uint8_t>* output,
+ uint32_t uncompressed_size, uint32_t compressed_size) {
+ LocalFileHeader lfh = {};
+ lfh.lfh_signature = LocalFileHeader::kSignature;
+ lfh.compressed_size = compressed_size;
+ lfh.uncompressed_size = uncompressed_size;
+ lfh.file_name_length = static_cast<uint16_t>(name.size());
+ lfh.extra_field_length = 20;
+ *output = std::vector<uint8_t>(reinterpret_cast<uint8_t*>(&lfh),
+ reinterpret_cast<uint8_t*>(&lfh) + sizeof(LocalFileHeader));
+ }
+
+ // Put one zip64 extended info in the extended field.
+ static void ConstructExtendedField(const std::vector<uint64_t>& zip64_fields,
+ std::vector<uint8_t>* output) {
+ ASSERT_FALSE(zip64_fields.empty());
+ uint16_t data_size = 8 * static_cast<uint16_t>(zip64_fields.size());
+ std::vector<uint8_t> extended_field(data_size + 4);
+ android::base::put_unaligned(extended_field.data(), Zip64ExtendedInfo::kHeaderId);
+ android::base::put_unaligned(extended_field.data() + 2, data_size);
+ size_t offset = 4;
+ for (const auto& field : zip64_fields) {
+ android::base::put_unaligned(extended_field.data() + offset, field);
+ offset += 8;
+ }
+
+ *output = std::move(extended_field);
+ }
+
+ static void ConstructCentralDirectoryRecord(const std::string& name, uint32_t uncompressed_size,
+ uint32_t compressed_size, uint32_t local_offset,
+ std::vector<uint8_t>* output) {
+ CentralDirectoryRecord cdr = {};
+ cdr.record_signature = CentralDirectoryRecord::kSignature;
+ cdr.compressed_size = uncompressed_size;
+ cdr.uncompressed_size = compressed_size;
+ cdr.file_name_length = static_cast<uint16_t>(name.size());
+ cdr.extra_field_length = local_offset == UINT32_MAX ? 28 : 20;
+ cdr.local_file_header_offset = local_offset;
+ *output =
+ std::vector<uint8_t>(reinterpret_cast<uint8_t*>(&cdr),
+ reinterpret_cast<uint8_t*>(&cdr) + sizeof(CentralDirectoryRecord));
+ }
+
+ // Add an entry to the zipfile, construct the corresponding local header and cd entry.
+ void AddEntry(const std::string& name, const std::vector<uint8_t>& content,
+ bool uncompressed_size_in_extended, bool compressed_size_in_extended,
+ bool local_offset_in_extended) {
+ auto uncompressed_size = static_cast<uint32_t>(content.size());
+ auto compressed_size = static_cast<uint32_t>(content.size());
+ uint32_t local_file_header_offset = 0;
+ std::for_each(file_entries_.begin(), file_entries_.end(),
+ [&local_file_header_offset](const LocalFileEntry& file_entry) {
+ local_file_header_offset += file_entry.GetSize();
+ });
+
+ std::vector<uint64_t> zip64_fields;
+ if (uncompressed_size_in_extended) {
+ zip64_fields.push_back(uncompressed_size);
+ uncompressed_size = UINT32_MAX;
+ }
+ if (compressed_size_in_extended) {
+ zip64_fields.push_back(compressed_size);
+ compressed_size = UINT32_MAX;
+ }
+ LocalFileEntry local_entry = {
+ .local_file_header = {},
+ .file_name = name,
+ .extended_field = {},
+ .compressed_bytes = content,
+ };
+ ConstructLocalFileHeader(name, &local_entry.local_file_header, uncompressed_size,
+ compressed_size);
+ ConstructExtendedField(zip64_fields, &local_entry.extended_field);
+ file_entries_.push_back(std::move(local_entry));
+
+ if (local_offset_in_extended) {
+ zip64_fields.push_back(local_file_header_offset);
+ local_file_header_offset = UINT32_MAX;
+ }
+ CdRecordEntry cd_entry = {
+ .central_directory_record = {},
+ .file_name = name,
+ .extended_field = {},
+ };
+ ConstructCentralDirectoryRecord(name, uncompressed_size, compressed_size,
+ local_file_header_offset, &cd_entry.central_directory_record);
+ ConstructExtendedField(zip64_fields, &cd_entry.extended_field);
+ cd_entries_.push_back(std::move(cd_entry));
+ }
+
+ void ConstructEocd() {
+ ASSERT_EQ(file_entries_.size(), cd_entries_.size());
+ Zip64EocdRecord zip64_eocd = {};
+ zip64_eocd.record_signature = Zip64EocdRecord::kSignature;
+ zip64_eocd.num_records = file_entries_.size();
+ zip64_eocd.cd_size = 0;
+ std::for_each(
+ cd_entries_.begin(), cd_entries_.end(),
+ [&zip64_eocd](const CdRecordEntry& cd_entry) { zip64_eocd.cd_size += cd_entry.GetSize(); });
+ zip64_eocd.cd_start_offset = 0;
+ std::for_each(file_entries_.begin(), file_entries_.end(),
+ [&zip64_eocd](const LocalFileEntry& file_entry) {
+ zip64_eocd.cd_start_offset += file_entry.GetSize();
+ });
+ zip64_eocd_record_ =
+ std::vector<uint8_t>(reinterpret_cast<uint8_t*>(&zip64_eocd),
+ reinterpret_cast<uint8_t*>(&zip64_eocd) + sizeof(Zip64EocdRecord));
+
+ Zip64EocdLocator zip64_locator = {};
+ zip64_locator.locator_signature = Zip64EocdLocator::kSignature;
+ zip64_locator.zip64_eocd_offset = zip64_eocd.cd_start_offset + zip64_eocd.cd_size;
+ zip64_eocd_locator_ =
+ std::vector<uint8_t>(reinterpret_cast<uint8_t*>(&zip64_locator),
+ reinterpret_cast<uint8_t*>(&zip64_locator) + sizeof(Zip64EocdLocator));
+
+ EocdRecord eocd = {};
+ eocd.eocd_signature = EocdRecord::kSignature,
+ eocd.num_records = file_entries_.size() > UINT16_MAX
+ ? UINT16_MAX
+ : static_cast<uint16_t>(file_entries_.size());
+ eocd.cd_size = UINT32_MAX;
+ eocd.cd_start_offset = UINT32_MAX;
+ eocd_record_ = std::vector<uint8_t>(reinterpret_cast<uint8_t*>(&eocd),
+ reinterpret_cast<uint8_t*>(&eocd) + sizeof(EocdRecord));
+ }
+
+ // Concatenate all the local file entries, cd entries, and eocd metadata.
+ void ConstructZipFile() {
+ for (const auto& file_entry : file_entries_) {
+ file_entry.CopyToOutput(&zip_content_);
+ }
+ for (const auto& cd_entry : cd_entries_) {
+ cd_entry.CopyToOutput(&zip_content_);
+ }
+ std::copy(zip64_eocd_record_.begin(), zip64_eocd_record_.end(),
+ std::back_inserter(zip_content_));
+ std::copy(zip64_eocd_locator_.begin(), zip64_eocd_locator_.end(),
+ std::back_inserter(zip_content_));
+ std::copy(eocd_record_.begin(), eocd_record_.end(), std::back_inserter(zip_content_));
+ }
+
+ std::vector<uint8_t> zip_content_;
+
+ std::vector<LocalFileEntry> file_entries_;
+ std::vector<CdRecordEntry> cd_entries_;
+ std::vector<uint8_t> zip64_eocd_record_;
+ std::vector<uint8_t> zip64_eocd_locator_;
+ std::vector<uint8_t> eocd_record_;
+};
+
+TEST_F(Zip64ParseTest, openFile) {
+ AddEntry("a.txt", std::vector<uint8_t>(100, 'a'), true, true, false);
+ ConstructEocd();
+ ConstructZipFile();
+
+ ZipArchiveHandle handle;
+ ASSERT_EQ(
+ 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle));
+ CloseArchive(handle);
+}
+
+TEST_F(Zip64ParseTest, openFilelocalOffsetInExtendedField) {
+ AddEntry("a.txt", std::vector<uint8_t>(100, 'a'), true, true, true);
+ AddEntry("b.txt", std::vector<uint8_t>(200, 'b'), true, true, true);
+ ConstructEocd();
+ ConstructZipFile();
+
+ ZipArchiveHandle handle;
+ ASSERT_EQ(
+ 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle));
+ CloseArchive(handle);
+}
+
+TEST_F(Zip64ParseTest, openFileCompressedNotInExtendedField) {
+ AddEntry("a.txt", std::vector<uint8_t>(100, 'a'), true, false, false);
+ ConstructEocd();
+ ConstructZipFile();
+
+ ZipArchiveHandle handle;
+ // Zip64 extended fields must include both uncompressed and compressed size.
+ ASSERT_NE(
+ 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle));
+ CloseArchive(handle);
+}
+
+TEST_F(Zip64ParseTest, findEntry) {
+ AddEntry("a.txt", std::vector<uint8_t>(200, 'a'), true, true, true);
+ AddEntry("b.txt", std::vector<uint8_t>(300, 'b'), true, true, false);
+ ConstructEocd();
+ ConstructZipFile();
+
+ ZipArchiveHandle handle;
+ ASSERT_EQ(
+ 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle));
+ ZipEntry entry;
+ ASSERT_EQ(0, FindEntry(handle, "a.txt", &entry));
+ ASSERT_EQ(200, entry.uncompressed_length);
+ ASSERT_EQ(200, entry.compressed_length);
+
+ ASSERT_EQ(0, FindEntry(handle, "b.txt", &entry));
+ ASSERT_EQ(300, entry.uncompressed_length);
+ ASSERT_EQ(300, entry.compressed_length);
+ CloseArchive(handle);
+}
+
+TEST_F(Zip64ParseTest, openFileIncorrectDataSizeInLocalExtendedField) {
+ AddEntry("a.txt", std::vector<uint8_t>(100, 'a'), true, true, false);
+ ASSERT_EQ(1, file_entries_.size());
+ auto& extended_field = file_entries_[0].extended_field;
+ // data size exceeds the extended field size in local header.
+ android::base::put_unaligned<uint16_t>(extended_field.data() + 2, 30);
+ ConstructEocd();
+ ConstructZipFile();
+
+ ZipArchiveHandle handle;
+ ASSERT_EQ(
+ 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle));
+ ZipEntry entry;
+ ASSERT_NE(0, FindEntry(handle, "a.txt", &entry));
+
+ CloseArchive(handle);
+}
+
+TEST_F(Zip64ParseTest, iterates) {
+ std::set<std::string_view> names{"a.txt", "b.txt", "c.txt", "d.txt", "e.txt"};
+ for (const auto& name : names) {
+ AddEntry(std::string(name), std::vector<uint8_t>(100, name[0]), true, true, true);
+ }
+ ConstructEocd();
+ ConstructZipFile();
+
+ ZipArchiveHandle handle;
+ ASSERT_EQ(
+ 0, OpenArchiveFromMemory(zip_content_.data(), zip_content_.size(), "debug_zip64", &handle));
+
+ void* iteration_cookie;
+ ASSERT_EQ(0, StartIteration(handle, &iteration_cookie));
+ std::set<std::string_view> result;
+ std::string_view name;
+ ZipEntry entry;
+ while (Next(iteration_cookie, &entry, &name) == 0) result.emplace(name);
+ ASSERT_EQ(names, result);
+
+ CloseArchive(handle);
+}
diff --git a/logcat/logcat.cpp b/logcat/logcat.cpp
index 76a970f..b065855 100644
--- a/logcat/logcat.cpp
+++ b/logcat/logcat.cpp
@@ -513,7 +513,7 @@
unsigned long setLogSize = 0;
const char* setPruneList = nullptr;
const char* setId = nullptr;
- int mode = ANDROID_LOG_RDONLY;
+ int mode = 0;
std::string forceFilters;
size_t tail_lines = 0;
log_time tail_time(log_time::EPOCH);
@@ -591,8 +591,7 @@
break;
}
if (long_options[option_index].name == wrap_str) {
- mode |= ANDROID_LOG_WRAP | ANDROID_LOG_RDONLY |
- ANDROID_LOG_NONBLOCK;
+ mode |= ANDROID_LOG_WRAP | ANDROID_LOG_NONBLOCK;
// ToDo: implement API that supports setting a wrap timeout
size_t dummy = ANDROID_LOG_WRAP_DEFAULT_TIMEOUT;
if (optarg && (!ParseUint(optarg, &dummy) || dummy < 1)) {
@@ -626,21 +625,19 @@
case 'c':
clearLog = true;
- mode |= ANDROID_LOG_WRONLY;
break;
case 'L':
- mode |= ANDROID_LOG_RDONLY | ANDROID_LOG_PSTORE |
- ANDROID_LOG_NONBLOCK;
+ mode |= ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK;
break;
case 'd':
- mode |= ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK;
+ mode |= ANDROID_LOG_NONBLOCK;
break;
case 't':
got_t = true;
- mode |= ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK;
+ mode |= ANDROID_LOG_NONBLOCK;
FALLTHROUGH_INTENDED;
case 'T':
if (strspn(optarg, "0123456789") != strlen(optarg)) {
diff --git a/logd/LogTags.cpp b/logd/LogTags.cpp
index 0cc7886..8299e66 100644
--- a/logd/LogTags.cpp
+++ b/logd/LogTags.cpp
@@ -289,9 +289,8 @@
// special pmsg event for log tags, and build up our internal
// database with any found.
void LogTags::ReadPersistEventLogTags() {
- struct logger_list* logger_list = android_logger_list_alloc(
- ANDROID_LOG_RDONLY | ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK, 0,
- (pid_t)0);
+ struct logger_list* logger_list =
+ android_logger_list_alloc(ANDROID_LOG_PSTORE | ANDROID_LOG_NONBLOCK, 0, (pid_t)0);
if (!logger_list) return;
struct logger* e = android_logger_open(logger_list, LOG_ID_EVENTS);
diff --git a/logd/tests/Android.bp b/logd/tests/Android.bp
index d39da8a..2519a84 100644
--- a/logd/tests/Android.bp
+++ b/logd/tests/Android.bp
@@ -64,5 +64,6 @@
test_suites: [
"cts",
"vts",
+ "vts10",
],
}
diff --git a/logd/tests/logd_test.cpp b/logd/tests/logd_test.cpp
index f47bee1..c7f3480 100644
--- a/logd/tests/logd_test.cpp
+++ b/logd/tests/logd_test.cpp
@@ -870,10 +870,8 @@
ASSERT_EQ(0, info.si_status);
struct logger_list* logger_list;
- ASSERT_TRUE(nullptr !=
- (logger_list = android_logger_list_open(
- LOG_ID_EVENTS, ANDROID_LOG_RDONLY | ANDROID_LOG_NONBLOCK,
- 0, pid)));
+ ASSERT_TRUE(nullptr != (logger_list = android_logger_list_open(LOG_ID_EVENTS,
+ ANDROID_LOG_NONBLOCK, 0, pid)));
int expected_count = (count < 2) ? count : 2;
int expected_chatty_count = (count <= 2) ? 0 : 1;
diff --git a/qemu_pipe/Android.bp b/qemu_pipe/Android.bp
index c6bda4a..ad86a4e 100644
--- a/qemu_pipe/Android.bp
+++ b/qemu_pipe/Android.bp
@@ -4,6 +4,11 @@
name: "libqemu_pipe",
vendor_available: true,
recovery_available: true,
+ apex_available: [
+ "com.android.adbd",
+ // TODO(b/151398197) remove the below
+ "//apex_available:platform",
+ ],
sanitize: {
misc_undefined: ["integer"],
},