Merge "fs_mgr: Fix executable path in clean_scratch_files."
diff --git a/adb/Android.bp b/adb/Android.bp
index 12d9a14..6386a78 100644
--- a/adb/Android.bp
+++ b/adb/Android.bp
@@ -470,6 +470,7 @@
"libadbd_core",
"libbrotli",
"libdiagnose_usb",
+ "liblz4",
],
shared_libs: [
@@ -571,6 +572,7 @@
"libbrotli",
"libcutils_sockets",
"libdiagnose_usb",
+ "liblz4",
"libmdnssd",
],
@@ -605,6 +607,7 @@
"libadbd_services",
"libasyncio",
"libcap",
+ "liblz4",
"libminijail",
"libssl",
],
diff --git a/adb/adb.cpp b/adb/adb.cpp
index 6c03f74..08d3904 100644
--- a/adb/adb.cpp
+++ b/adb/adb.cpp
@@ -1071,19 +1071,25 @@
return 0;
}
+static bool g_reject_kill_server = false;
+void adb_set_reject_kill_server(bool value) {
+ g_reject_kill_server = value;
+}
+
HostRequestResult handle_host_request(std::string_view service, TransportType type,
const char* serial, TransportId transport_id, int reply_fd,
asocket* s) {
if (service == "kill") {
- fprintf(stderr, "adb server killed by remote request\n");
- fflush(stdout);
+ if (g_reject_kill_server) {
+ LOG(WARNING) << "adb server ignoring kill-server";
+ SendFail(reply_fd, "kill-server rejected by remote server");
+ } else {
+ fprintf(stderr, "adb server killed by remote request\n");
+ SendOkay(reply_fd);
- // Send a reply even though we don't read it anymore, so that old versions
- // of adb that do read it don't spew error messages.
- SendOkay(reply_fd);
-
- // Rely on process exit to close the socket for us.
- exit(0);
+ // Rely on process exit to close the socket for us.
+ exit(0);
+ }
}
LOG(DEBUG) << "handle_host_request(" << service << ")";
@@ -1192,9 +1198,9 @@
FeatureSet features = supported_features();
// Abuse features to report libusb status.
if (should_use_libusb()) {
- features.insert(kFeatureLibusb);
+ features.emplace_back(kFeatureLibusb);
}
- features.insert(kFeaturePushSync);
+ features.emplace_back(kFeaturePushSync);
SendOkay(reply_fd, FeatureSetToString(features));
return HostRequestResult::Handled;
}
diff --git a/adb/adb.h b/adb/adb.h
index 7bc60fc..476ed9b 100644
--- a/adb/adb.h
+++ b/adb/adb.h
@@ -14,8 +14,7 @@
* limitations under the License.
*/
-#ifndef __ADB_H
-#define __ADB_H
+#pragma once
#include <limits.h>
#include <stdint.h>
@@ -237,6 +236,7 @@
void parse_banner(const std::string&, atransport* t);
+#if ADB_HOST
// On startup, the adb server needs to wait until all of the connected devices are ready.
// To do this, we need to know when the scan has identified all of the potential new transports, and
// when each transport becomes ready.
@@ -250,6 +250,12 @@
// Wait until device scan has completed and every transport is ready, or a timeout elapses.
void adb_wait_for_device_initialization();
+#endif // ADB_HOST
+
+#if ADB_HOST
+// When ssh-forwarding to a remote adb server, kill-server is almost never what you actually want,
+// and unfortunately, many other tools issue it. This adds a knob to reject kill-servers.
+void adb_set_reject_kill_server(bool reject);
+#endif
void usb_init();
-#endif
diff --git a/adb/client/adb_client.cpp b/adb/client/adb_client.cpp
index f724cb5..0ad0465 100644
--- a/adb/client/adb_client.cpp
+++ b/adb/client/adb_client.cpp
@@ -37,6 +37,7 @@
#include <vector>
#include <android-base/file.h>
+#include <android-base/no_destructor.h>
#include <android-base/stringprintf.h>
#include <android-base/strings.h>
#include <android-base/thread_annotations.h>
@@ -204,9 +205,25 @@
return false;
}
- // The server might send OKAY, so consume that.
char buf[4];
- ReadFdExactly(fd.get(), buf, 4);
+ if (!ReadFdExactly(fd.get(), buf, 4)) {
+ fprintf(stderr, "error: failed to read response from server\n");
+ return false;
+ }
+
+ if (memcmp(buf, "OKAY", 4) == 0) {
+ // Nothing to do.
+ } else if (memcmp(buf, "FAIL", 4) == 0) {
+ std::string output, error;
+ if (!ReadProtocolString(fd.get(), &output, &error)) {
+ fprintf(stderr, "error: %s\n", error.c_str());
+ return false;
+ }
+
+ fprintf(stderr, "error: %s\n", output.c_str());
+ return false;
+ }
+
// Now that no more data is expected, wait for socket orderly shutdown or error, indicating
// server death.
ReadOrderlyShutdown(fd.get());
@@ -399,18 +416,14 @@
return android::base::StringPrintf("%s:%s", prefix, command);
}
-bool adb_get_feature_set(FeatureSet* feature_set, std::string* error) {
- static FeatureSet* features = nullptr;
- if (!features) {
+const FeatureSet& adb_get_feature_set() {
+ static const android::base::NoDestructor<FeatureSet> features([] {
std::string result;
- if (adb_query(format_host_command("features"), &result, error)) {
- features = new FeatureSet(StringToFeatureSet(result));
+ if (!adb_query(format_host_command("features"), &result, &result)) {
+ fprintf(stderr, "failed to get feature set: %s\n", result.c_str());
+ return FeatureSet{};
}
- }
- if (features) {
- *feature_set = *features;
- return true;
- }
- feature_set->clear();
- return false;
+ return StringToFeatureSet(result);
+ }());
+ return *features;
}
diff --git a/adb/client/adb_client.h b/adb/client/adb_client.h
index 1c6cde7..bf0be40 100644
--- a/adb/client/adb_client.h
+++ b/adb/client/adb_client.h
@@ -76,7 +76,7 @@
std::string format_host_command(const char* _Nonnull command);
// Get the feature set of the current preferred transport.
-bool adb_get_feature_set(FeatureSet* _Nonnull feature_set, std::string* _Nonnull error);
+const FeatureSet& adb_get_feature_set();
#if defined(__linux__)
// Get the path of a file containing the path to the server executable, if the socket spec set via
diff --git a/adb/client/adb_install.cpp b/adb/client/adb_install.cpp
index 092a866..3810cc8 100644
--- a/adb/client/adb_install.cpp
+++ b/adb/client/adb_install.cpp
@@ -57,10 +57,8 @@
}
static bool can_use_feature(const char* feature) {
- FeatureSet features;
- std::string error;
- if (!adb_get_feature_set(&features, &error)) {
- fprintf(stderr, "error: %s\n", error.c_str());
+ auto&& features = adb_get_feature_set();
+ if (features.empty()) {
return false;
}
return CanUseFeature(features, feature);
@@ -290,7 +288,7 @@
}
}
- if (do_sync_push(apk_file, apk_dest.c_str(), false, true)) {
+ if (do_sync_push(apk_file, apk_dest.c_str(), false, CompressionType::Any, false)) {
result = pm_command(argc, argv);
delete_device_file(apk_dest);
}
diff --git a/adb/client/bugreport.cpp b/adb/client/bugreport.cpp
index ab93f7d..e162aaa 100644
--- a/adb/client/bugreport.cpp
+++ b/adb/client/bugreport.cpp
@@ -282,5 +282,5 @@
bool Bugreport::DoSyncPull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs,
const char* name) {
- return do_sync_pull(srcs, dst, copy_attrs, false, name);
+ return do_sync_pull(srcs, dst, copy_attrs, CompressionType::None, name);
}
diff --git a/adb/client/commandline.cpp b/adb/client/commandline.cpp
index 04b250d..6a7493f 100644
--- a/adb/client/commandline.cpp
+++ b/adb/client/commandline.cpp
@@ -129,20 +129,22 @@
" reverse --remove-all remove all reverse socket connections from device\n"
"\n"
"file transfer:\n"
- " push [--sync] [-zZ] LOCAL... REMOTE\n"
+ " push [--sync] [-z ALGORITHM] [-Z] LOCAL... REMOTE\n"
" copy local files/directories to device\n"
" --sync: only push files that are newer on the host than the device\n"
- " -z: enable compression\n"
+ " -n: dry run: push files to device without storing to the filesystem\n"
+ " -z: enable compression with a specified algorithm (any, none, brotli)\n"
" -Z: disable compression\n"
- " pull [-azZ] REMOTE... LOCAL\n"
+ " pull [-a] [-z ALGORITHM] [-Z] REMOTE... LOCAL\n"
" copy files/dirs from device\n"
" -a: preserve file timestamp and mode\n"
- " -z: enable compression\n"
+ " -z: enable compression with a specified algorithm (any, none, brotli)\n"
" -Z: disable compression\n"
- " sync [-lzZ] [all|data|odm|oem|product|system|system_ext|vendor]\n"
+ " sync [-l] [-z ALGORITHM] [-Z] [all|data|odm|oem|product|system|system_ext|vendor]\n"
" sync a local build from $ANDROID_PRODUCT_OUT to the device (default all)\n"
+ " -n: dry run: push files to device without storing to the filesystem\n"
" -l: list files that would be copied, but don't copy them\n"
- " -z: enable compression\n"
+ " -z: enable compression with a specified algorithm (any, none, brotli)\n"
" -Z: disable compression\n"
"\n"
"shell:\n"
@@ -670,10 +672,8 @@
}
static int adb_shell(int argc, const char** argv) {
- FeatureSet features;
- std::string error_message;
- if (!adb_get_feature_set(&features, &error_message)) {
- fprintf(stderr, "error: %s\n", error_message.c_str());
+ auto&& features = adb_get_feature_set();
+ if (features.empty()) {
return 1;
}
@@ -777,13 +777,10 @@
}
static int adb_abb(int argc, const char** argv) {
- FeatureSet features;
- std::string error_message;
- if (!adb_get_feature_set(&features, &error_message)) {
- fprintf(stderr, "error: %s\n", error_message.c_str());
+ auto&& features = adb_get_feature_set();
+ if (features.empty()) {
return 1;
}
-
if (!CanUseFeature(features, kFeatureAbb)) {
error_exit("abb is not supported by the device");
}
@@ -1162,9 +1159,8 @@
// Use shell protocol if it's supported and the caller doesn't explicitly
// disable it.
if (!disable_shell_protocol) {
- FeatureSet features;
- std::string error;
- if (adb_get_feature_set(&features, &error)) {
+ auto&& features = adb_get_feature_set();
+ if (!features.empty()) {
use_shell_protocol = CanUseFeature(features, kFeatureShell2);
} else {
// Device was unreachable.
@@ -1314,12 +1310,36 @@
return 0;
}
+static CompressionType parse_compression_type(const std::string& str, bool allow_numbers) {
+ if (allow_numbers) {
+ if (str == "0") {
+ return CompressionType::None;
+ } else if (str == "1") {
+ return CompressionType::Any;
+ }
+ }
+
+ if (str == "any") {
+ return CompressionType::Any;
+ } else if (str == "none") {
+ return CompressionType::None;
+ }
+
+ if (str == "brotli") {
+ return CompressionType::Brotli;
+ } else if (str == "lz4") {
+ return CompressionType::LZ4;
+ }
+
+ error_exit("unexpected compression type %s", str.c_str());
+}
+
static void parse_push_pull_args(const char** arg, int narg, std::vector<const char*>* srcs,
- const char** dst, bool* copy_attrs, bool* sync, bool* compressed) {
+ const char** dst, bool* copy_attrs, bool* sync,
+ CompressionType* compression, bool* dry_run) {
*copy_attrs = false;
- const char* adb_compression = getenv("ADB_COMPRESSION");
- if (adb_compression && strcmp(adb_compression, "0") == 0) {
- *compressed = false;
+ if (const char* adb_compression = getenv("ADB_COMPRESSION")) {
+ *compression = parse_compression_type(adb_compression, true);
}
srcs->clear();
@@ -1333,13 +1353,15 @@
} else if (!strcmp(*arg, "-a")) {
*copy_attrs = true;
} else if (!strcmp(*arg, "-z")) {
- if (compressed != nullptr) {
- *compressed = true;
+ if (narg < 2) {
+ error_exit("-z requires an argument");
}
+ *compression = parse_compression_type(*++arg, false);
+ --narg;
} else if (!strcmp(*arg, "-Z")) {
- if (compressed != nullptr) {
- *compressed = false;
- }
+ *compression = CompressionType::None;
+ } else if (dry_run && !strcmp(*arg, "-n")) {
+ *dry_run = true;
} else if (!strcmp(*arg, "--sync")) {
if (sync != nullptr) {
*sync = true;
@@ -1788,10 +1810,8 @@
}
return adb_connect_command(android::base::StringPrintf("tcpip:%d", port));
} else if (!strcmp(argv[0], "remount")) {
- FeatureSet features;
- std::string error;
- if (!adb_get_feature_set(&features, &error)) {
- fprintf(stderr, "error: %s\n", error.c_str());
+ auto&& features = adb_get_feature_set();
+ if (features.empty()) {
return 1;
}
@@ -1894,22 +1914,25 @@
} else if (!strcmp(argv[0], "push")) {
bool copy_attrs = false;
bool sync = false;
- bool compressed = true;
+ bool dry_run = false;
+ CompressionType compression = CompressionType::Any;
std::vector<const char*> srcs;
const char* dst = nullptr;
- parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync, &compressed);
+ parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync, &compression,
+ &dry_run);
if (srcs.empty() || !dst) error_exit("push requires an argument");
- return do_sync_push(srcs, dst, sync, compressed) ? 0 : 1;
+ return do_sync_push(srcs, dst, sync, compression, dry_run) ? 0 : 1;
} else if (!strcmp(argv[0], "pull")) {
bool copy_attrs = false;
- bool compressed = true;
+ CompressionType compression = CompressionType::Any;
std::vector<const char*> srcs;
const char* dst = ".";
- parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr, &compressed);
+ parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr, &compression,
+ nullptr);
if (srcs.empty()) error_exit("pull requires an argument");
- return do_sync_pull(srcs, dst, copy_attrs, compressed) ? 0 : 1;
+ return do_sync_pull(srcs, dst, copy_attrs, compression) ? 0 : 1;
} else if (!strcmp(argv[0], "install")) {
if (argc < 2) error_exit("install requires an argument");
return install_app(argc, argv);
@@ -1925,27 +1948,30 @@
} else if (!strcmp(argv[0], "sync")) {
std::string src;
bool list_only = false;
- bool compressed = true;
+ bool dry_run = false;
+ CompressionType compression = CompressionType::Any;
- const char* adb_compression = getenv("ADB_COMPRESSION");
- if (adb_compression && strcmp(adb_compression, "0") == 0) {
- compressed = false;
+ if (const char* adb_compression = getenv("ADB_COMPRESSION"); adb_compression) {
+ compression = parse_compression_type(adb_compression, true);
}
int opt;
- while ((opt = getopt(argc, const_cast<char**>(argv), "lzZ")) != -1) {
+ while ((opt = getopt(argc, const_cast<char**>(argv), "lnz:Z")) != -1) {
switch (opt) {
case 'l':
list_only = true;
break;
+ case 'n':
+ dry_run = true;
+ break;
case 'z':
- compressed = true;
+ compression = parse_compression_type(optarg, false);
break;
case 'Z':
- compressed = false;
+ compression = CompressionType::None;
break;
default:
- error_exit("usage: adb sync [-lzZ] [PARTITION]");
+ error_exit("usage: adb sync [-l] [-n] [-z ALGORITHM] [-Z] [PARTITION]");
}
}
@@ -1954,7 +1980,7 @@
} else if (optind + 1 == argc) {
src = argv[optind];
} else {
- error_exit("usage: adb sync [-lzZ] [PARTITION]");
+ error_exit("usage: adb sync [-l] [-n] [-z ALGORITHM] [-Z] [PARTITION]");
}
std::vector<std::string> partitions{"data", "odm", "oem", "product",
@@ -1965,7 +1991,9 @@
std::string src_dir{product_file(partition)};
if (!directory_exists(src_dir)) continue;
found = true;
- if (!do_sync_sync(src_dir, "/" + partition, list_only, compressed)) return 1;
+ if (!do_sync_sync(src_dir, "/" + partition, list_only, compression, dry_run)) {
+ return 1;
+ }
}
}
if (!found) error_exit("don't know how to sync %s partition", src.c_str());
@@ -2006,10 +2034,8 @@
} else if (!strcmp(argv[0], "track-jdwp")) {
return adb_connect_command("track-jdwp");
} else if (!strcmp(argv[0], "track-app")) {
- FeatureSet features;
- std::string error;
- if (!adb_get_feature_set(&features, &error)) {
- fprintf(stderr, "error: %s\n", error.c_str());
+ auto&& features = adb_get_feature_set();
+ if (features.empty()) {
return 1;
}
if (!CanUseFeature(features, kFeatureTrackApp)) {
@@ -2038,10 +2064,8 @@
return 0;
} else if (!strcmp(argv[0], "features")) {
// Only list the features common to both the adb client and the device.
- FeatureSet features;
- std::string error;
- if (!adb_get_feature_set(&features, &error)) {
- fprintf(stderr, "error: %s\n", error.c_str());
+ auto&& features = adb_get_feature_set();
+ if (features.empty()) {
return 1;
}
diff --git a/adb/client/fastdeploy.cpp b/adb/client/fastdeploy.cpp
index de82e14..bc4b91b 100644
--- a/adb/client/fastdeploy.cpp
+++ b/adb/client/fastdeploy.cpp
@@ -112,7 +112,7 @@
// but can't be removed until after the push.
unix_close(tf.release());
- if (!do_sync_push(srcs, dst, sync, true)) {
+ if (!do_sync_push(srcs, dst, sync, CompressionType::Any, false)) {
error_exit("Failed to push fastdeploy agent to device.");
}
}
diff --git a/adb/client/file_sync_client.cpp b/adb/client/file_sync_client.cpp
index 2ed58b2..f2c673a 100644
--- a/adb/client/file_sync_client.cpp
+++ b/adb/client/file_sync_client.cpp
@@ -34,6 +34,7 @@
#include <memory>
#include <sstream>
#include <string>
+#include <variant>
#include <vector>
#include "sysdeps.h"
@@ -224,18 +225,22 @@
class SyncConnection {
public:
- SyncConnection() : acknowledgement_buffer_(sizeof(sync_status) + SYNC_DATA_MAX) {
+ SyncConnection()
+ : acknowledgement_buffer_(sizeof(sync_status) + SYNC_DATA_MAX),
+ features_(adb_get_feature_set()) {
acknowledgement_buffer_.resize(0);
max = SYNC_DATA_MAX; // TODO: decide at runtime.
- std::string error;
- if (!adb_get_feature_set(&features_, &error)) {
- Error("failed to get feature set: %s", error.c_str());
+ if (features_.empty()) {
+ Error("failed to get feature set");
} else {
have_stat_v2_ = CanUseFeature(features_, kFeatureStat2);
have_ls_v2_ = CanUseFeature(features_, kFeatureLs2);
have_sendrecv_v2_ = CanUseFeature(features_, kFeatureSendRecv2);
have_sendrecv_v2_brotli_ = CanUseFeature(features_, kFeatureSendRecv2Brotli);
+ have_sendrecv_v2_lz4_ = CanUseFeature(features_, kFeatureSendRecv2LZ4);
+ have_sendrecv_v2_dry_run_send_ = CanUseFeature(features_, kFeatureSendRecv2DryRunSend);
+ std::string error;
fd.reset(adb_connect("sync:", &error));
if (fd < 0) {
Error("connect failed: %s", error.c_str());
@@ -261,6 +266,22 @@
bool HaveSendRecv2() const { return have_sendrecv_v2_; }
bool HaveSendRecv2Brotli() const { return have_sendrecv_v2_brotli_; }
+ bool HaveSendRecv2LZ4() const { return have_sendrecv_v2_lz4_; }
+ bool HaveSendRecv2DryRunSend() const { return have_sendrecv_v2_dry_run_send_; }
+
+ // Resolve a compression type which might be CompressionType::Any to a specific compression
+ // algorithm.
+ CompressionType ResolveCompressionType(CompressionType compression) const {
+ if (compression == CompressionType::Any) {
+ if (HaveSendRecv2LZ4()) {
+ return CompressionType::LZ4;
+ } else if (HaveSendRecv2Brotli()) {
+ return CompressionType::Brotli;
+ }
+ return CompressionType::None;
+ }
+ return compression;
+ }
const FeatureSet& Features() const { return features_; }
@@ -323,7 +344,7 @@
return WriteFdExactly(fd, buf.data(), buf.size());
}
- bool SendSend2(std::string_view path, mode_t mode, bool compressed) {
+ bool SendSend2(std::string_view path, mode_t mode, CompressionType compression, bool dry_run) {
if (path.length() > 1024) {
Error("SendRequest failed: path too long: %zu", path.length());
errno = ENAMETOOLONG;
@@ -339,7 +360,26 @@
syncmsg msg;
msg.send_v2_setup.id = ID_SEND_V2;
msg.send_v2_setup.mode = mode;
- msg.send_v2_setup.flags = compressed ? kSyncFlagBrotli : kSyncFlagNone;
+ msg.send_v2_setup.flags = 0;
+ switch (compression) {
+ case CompressionType::None:
+ break;
+
+ case CompressionType::Brotli:
+ msg.send_v2_setup.flags = kSyncFlagBrotli;
+ break;
+
+ case CompressionType::LZ4:
+ msg.send_v2_setup.flags = kSyncFlagLZ4;
+ break;
+
+ case CompressionType::Any:
+ LOG(FATAL) << "unexpected CompressionType::Any";
+ }
+
+ if (dry_run) {
+ msg.send_v2_setup.flags |= kSyncFlagDryRun;
+ }
buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.send_v2_setup));
@@ -352,7 +392,7 @@
return WriteFdExactly(fd, buf.data(), buf.size());
}
- bool SendRecv2(const std::string& path) {
+ bool SendRecv2(const std::string& path, CompressionType compression) {
if (path.length() > 1024) {
Error("SendRequest failed: path too long: %zu", path.length());
errno = ENAMETOOLONG;
@@ -367,7 +407,22 @@
syncmsg msg;
msg.recv_v2_setup.id = ID_RECV_V2;
- msg.recv_v2_setup.flags = kSyncFlagBrotli;
+ msg.recv_v2_setup.flags = 0;
+ switch (compression) {
+ case CompressionType::None:
+ break;
+
+ case CompressionType::Brotli:
+ msg.recv_v2_setup.flags |= kSyncFlagBrotli;
+ break;
+
+ case CompressionType::LZ4:
+ msg.recv_v2_setup.flags |= kSyncFlagLZ4;
+ break;
+
+ case CompressionType::Any:
+ LOG(FATAL) << "unexpected CompressionType::Any";
+ }
buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.recv_v2_setup));
@@ -494,7 +549,12 @@
// difference to "adb sync" performance.
bool SendSmallFile(const std::string& path, mode_t mode, const std::string& lpath,
const std::string& rpath, unsigned mtime, const char* data,
- size_t data_length) {
+ size_t data_length, bool dry_run) {
+ if (dry_run) {
+ // We need to use send v2 for dry run.
+ return SendLargeFile(path, mode, lpath, rpath, mtime, CompressionType::None, dry_run);
+ }
+
std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode);
if (path_and_mode.length() > 1024) {
Error("SendSmallFile failed: path too long: %zu", path_and_mode.length());
@@ -533,9 +593,21 @@
return true;
}
- bool SendLargeFileCompressed(const std::string& path, mode_t mode, const std::string& lpath,
- const std::string& rpath, unsigned mtime) {
- if (!SendSend2(path, mode, true)) {
+ bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath,
+ const std::string& rpath, unsigned mtime, CompressionType compression,
+ bool dry_run) {
+ if (dry_run && !HaveSendRecv2DryRunSend()) {
+ Error("dry-run not supported by the device");
+ return false;
+ }
+
+ if (!HaveSendRecv2()) {
+ return SendLargeFileLegacy(path, mode, lpath, rpath, mtime);
+ }
+
+ compression = ResolveCompressionType(compression);
+
+ if (!SendSend2(path, mode, compression, dry_run)) {
Error("failed to send ID_SEND_V2 message '%s': %s", path.c_str(), strerror(errno));
return false;
}
@@ -558,7 +630,25 @@
syncsendbuf sbuf;
sbuf.id = ID_DATA;
- BrotliEncoder<SYNC_DATA_MAX> encoder;
+ std::variant<std::monostate, NullEncoder, BrotliEncoder, LZ4Encoder> encoder_storage;
+ Encoder* encoder = nullptr;
+ switch (compression) {
+ case CompressionType::None:
+ encoder = &encoder_storage.emplace<NullEncoder>(SYNC_DATA_MAX);
+ break;
+
+ case CompressionType::Brotli:
+ encoder = &encoder_storage.emplace<BrotliEncoder>(SYNC_DATA_MAX);
+ break;
+
+ case CompressionType::LZ4:
+ encoder = &encoder_storage.emplace<LZ4Encoder>(SYNC_DATA_MAX);
+ break;
+
+ case CompressionType::Any:
+ LOG(FATAL) << "unexpected CompressionType::Any";
+ }
+
bool sending = true;
while (sending) {
Block input(SYNC_DATA_MAX);
@@ -569,10 +659,10 @@
}
if (r == 0) {
- encoder.Finish();
+ encoder->Finish();
} else {
input.resize(r);
- encoder.Append(std::move(input));
+ encoder->Append(std::move(input));
RecordBytesTransferred(r);
bytes_copied += r;
ReportProgress(rpath, bytes_copied, total_size);
@@ -580,7 +670,7 @@
while (true) {
Block output;
- EncodeResult result = encoder.Encode(&output);
+ EncodeResult result = encoder->Encode(&output);
if (result == EncodeResult::Error) {
Error("compressing '%s' locally failed", lpath.c_str());
return false;
@@ -610,12 +700,8 @@
return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data));
}
- bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath,
- const std::string& rpath, unsigned mtime, bool compressed) {
- if (compressed && HaveSendRecv2Brotli()) {
- return SendLargeFileCompressed(path, mode, lpath, rpath, mtime);
- }
-
+ bool SendLargeFileLegacy(const std::string& path, mode_t mode, const std::string& lpath,
+ const std::string& rpath, unsigned mtime) {
std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode);
if (!SendRequest(ID_SEND_V1, path_and_mode)) {
Error("failed to send ID_SEND_V1 message '%s': %s", path_and_mode.c_str(),
@@ -835,11 +921,13 @@
private:
std::deque<std::pair<std::string, std::string>> deferred_acknowledgements_;
Block acknowledgement_buffer_;
- FeatureSet features_;
+ const FeatureSet& features_;
bool have_stat_v2_;
bool have_ls_v2_;
bool have_sendrecv_v2_;
bool have_sendrecv_v2_brotli_;
+ bool have_sendrecv_v2_lz4_;
+ bool have_sendrecv_v2_dry_run_send_;
TransferLedger global_ledger_;
TransferLedger current_ledger_;
@@ -921,7 +1009,8 @@
}
static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::string& rpath,
- unsigned mtime, mode_t mode, bool sync, bool compressed) {
+ unsigned mtime, mode_t mode, bool sync, CompressionType compression,
+ bool dry_run) {
if (sync) {
struct stat st;
if (sync_lstat(sc, rpath, &st)) {
@@ -942,7 +1031,7 @@
}
buf[data_length++] = '\0';
- if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, buf, data_length)) {
+ if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, buf, data_length, dry_run)) {
return false;
}
return sc.ReadAcknowledgements();
@@ -960,11 +1049,12 @@
sc.Error("failed to read all of '%s': %s", lpath.c_str(), strerror(errno));
return false;
}
- if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, data.data(), data.size())) {
+ if (!sc.SendSmallFile(rpath, mode, lpath, rpath, mtime, data.data(), data.size(),
+ dry_run)) {
return false;
}
} else {
- if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compressed)) {
+ if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compression, dry_run)) {
return false;
}
}
@@ -1027,8 +1117,10 @@
}
static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpath, const char* name,
- uint64_t expected_size) {
- if (!sc.SendRecv2(rpath)) return false;
+ uint64_t expected_size, CompressionType compression) {
+ compression = sc.ResolveCompressionType(compression);
+
+ if (!sc.SendRecv2(rpath, compression)) return false;
adb_unlink(lpath);
unique_fd lfd(adb_creat(lpath, 0644));
@@ -1040,9 +1132,28 @@
uint64_t bytes_copied = 0;
Block buffer(SYNC_DATA_MAX);
- BrotliDecoder decoder(std::span(buffer.data(), buffer.size()));
- bool reading = true;
- while (reading) {
+ std::variant<std::monostate, NullDecoder, BrotliDecoder, LZ4Decoder> decoder_storage;
+ Decoder* decoder = nullptr;
+
+ std::span buffer_span(buffer.data(), buffer.size());
+ switch (compression) {
+ case CompressionType::None:
+ decoder = &decoder_storage.emplace<NullDecoder>(buffer_span);
+ break;
+
+ case CompressionType::Brotli:
+ decoder = &decoder_storage.emplace<BrotliDecoder>(buffer_span);
+ break;
+
+ case CompressionType::LZ4:
+ decoder = &decoder_storage.emplace<LZ4Decoder>(buffer_span);
+ break;
+
+ case CompressionType::Any:
+ LOG(FATAL) << "unexpected CompressionType::Any";
+ }
+
+ while (true) {
syncmsg msg;
if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) {
adb_unlink(lpath);
@@ -1050,33 +1161,32 @@
}
if (msg.data.id == ID_DONE) {
- adb_unlink(lpath);
- sc.Error("unexpected ID_DONE");
- return false;
- }
-
- if (msg.data.id != ID_DATA) {
+ if (!decoder->Finish()) {
+ sc.Error("unexpected ID_DONE");
+ return false;
+ }
+ } else if (msg.data.id != ID_DATA) {
adb_unlink(lpath);
sc.ReportCopyFailure(rpath, lpath, msg);
return false;
- }
+ } else {
+ if (msg.data.size > sc.max) {
+ sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max);
+ adb_unlink(lpath);
+ return false;
+ }
- if (msg.data.size > sc.max) {
- sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max);
- adb_unlink(lpath);
- return false;
+ Block block(msg.data.size);
+ if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) {
+ adb_unlink(lpath);
+ return false;
+ }
+ decoder->Append(std::move(block));
}
- Block block(msg.data.size);
- if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) {
- adb_unlink(lpath);
- return false;
- }
- decoder.Append(std::move(block));
-
while (true) {
std::span<char> output;
- DecodeResult result = decoder.Decode(&output);
+ DecodeResult result = decoder->Decode(&output);
if (result == DecodeResult::Error) {
sc.Error("decompress failed");
@@ -1093,8 +1203,7 @@
}
bytes_copied += output.size();
-
- sc.RecordBytesTransferred(msg.data.size);
+ sc.RecordBytesTransferred(output.size());
sc.ReportProgress(name != nullptr ? name : rpath, bytes_copied, expected_size);
if (result == DecodeResult::NeedInput) {
@@ -1102,33 +1211,19 @@
} else if (result == DecodeResult::MoreOutput) {
continue;
} else if (result == DecodeResult::Done) {
- reading = false;
- break;
+ sc.RecordFilesTransferred(1);
+ return true;
} else {
LOG(FATAL) << "invalid DecodeResult: " << static_cast<int>(result);
}
}
}
-
- syncmsg msg;
- if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) {
- sc.Error("failed to read ID_DONE");
- return false;
- }
-
- if (msg.data.id != ID_DONE) {
- sc.Error("unexpected message after transfer: id = %d (expected ID_DONE)", msg.data.id);
- return false;
- }
-
- sc.RecordFilesTransferred(1);
- return true;
}
static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, const char* name,
- uint64_t expected_size, bool compressed) {
- if (sc.HaveSendRecv2() && compressed) {
- return sync_recv_v2(sc, rpath, lpath, name, expected_size);
+ uint64_t expected_size, CompressionType compression) {
+ if (sc.HaveSendRecv2()) {
+ return sync_recv_v2(sc, rpath, lpath, name, expected_size, compression);
} else {
return sync_recv_v1(sc, rpath, lpath, name, expected_size);
}
@@ -1210,7 +1305,8 @@
}
static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::string rpath,
- bool check_timestamps, bool list_only, bool compressed) {
+ bool check_timestamps, bool list_only,
+ CompressionType compression, bool dry_run) {
sc.NewTransfer();
// Make sure that both directory paths end in a slash.
@@ -1292,7 +1388,8 @@
if (list_only) {
sc.Println("would push: %s -> %s", ci.lpath.c_str(), ci.rpath.c_str());
} else {
- if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compressed)) {
+ if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compression,
+ dry_run)) {
return false;
}
}
@@ -1308,7 +1405,7 @@
}
bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync,
- bool compressed) {
+ CompressionType compression, bool dry_run) {
SyncConnection sc;
if (!sc.IsValid()) return false;
@@ -1373,7 +1470,8 @@
dst_dir.append(android::base::Basename(src_path));
}
- success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compressed);
+ success &=
+ copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compression, dry_run);
continue;
} else if (!should_push_file(st.st_mode)) {
sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, st.st_mode);
@@ -1394,7 +1492,8 @@
sc.NewTransfer();
sc.SetExpectedTotalBytes(st.st_size);
- success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compressed);
+ success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compression,
+ dry_run);
sc.ReportTransferRate(src_path, TransferDirection::push);
}
@@ -1480,7 +1579,7 @@
}
static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::string lpath,
- bool copy_attrs, bool compressed) {
+ bool copy_attrs, CompressionType compression) {
sc.NewTransfer();
// Make sure that both directory paths end in a slash.
@@ -1510,7 +1609,7 @@
continue;
}
- if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compressed)) {
+ if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compression)) {
return false;
}
@@ -1528,7 +1627,7 @@
}
bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs,
- bool compressed, const char* name) {
+ CompressionType compression, const char* name) {
SyncConnection sc;
if (!sc.IsValid()) return false;
@@ -1602,7 +1701,7 @@
dst_dir.append(android::base::Basename(src_path));
}
- success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compressed);
+ success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compression);
continue;
} else if (!should_pull_file(src_st.st_mode)) {
sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, src_st.st_mode);
@@ -1621,7 +1720,7 @@
sc.NewTransfer();
sc.SetExpectedTotalBytes(src_st.st_size);
- if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compressed)) {
+ if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compression)) {
success = false;
continue;
}
@@ -1638,11 +1737,11 @@
}
bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only,
- bool compressed) {
+ CompressionType compression, bool dry_run) {
SyncConnection sc;
if (!sc.IsValid()) return false;
- bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compressed);
+ bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compression, dry_run);
if (!list_only) {
sc.ReportOverallTransferRate(TransferDirection::push);
}
diff --git a/adb/client/file_sync_client.h b/adb/client/file_sync_client.h
index de3f192..cb8ca93 100644
--- a/adb/client/file_sync_client.h
+++ b/adb/client/file_sync_client.h
@@ -19,11 +19,13 @@
#include <string>
#include <vector>
+#include "file_sync_protocol.h"
+
bool do_sync_ls(const char* path);
bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync,
- bool compressed);
+ CompressionType compression, bool dry_run);
bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs,
- bool compressed, const char* name = nullptr);
+ CompressionType compression, const char* name = nullptr);
bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only,
- bool compressed);
+ CompressionType compression, bool dry_run);
diff --git a/adb/client/incremental.cpp b/adb/client/incremental.cpp
index 9765292..2814932 100644
--- a/adb/client/incremental.cpp
+++ b/adb/client/incremental.cpp
@@ -16,13 +16,13 @@
#include "incremental.h"
-#include <android-base/endian.h>
+#include "incremental_utils.h"
+
#include <android-base/file.h>
#include <android-base/stringprintf.h>
#include <openssl/base64.h>
#include "adb_client.h"
-#include "adb_io.h"
#include "adb_utils.h"
#include "commandline.h"
#include "sysdeps.h"
@@ -31,65 +31,8 @@
namespace incremental {
-namespace {
-
-static constexpr auto IDSIG = ".idsig"sv;
-
using android::base::StringPrintf;
-using Size = int64_t;
-
-static inline int32_t read_int32(borrowed_fd fd) {
- int32_t result;
- return ReadFdExactly(fd, &result, sizeof(result)) ? result : -1;
-}
-
-static inline void append_int(borrowed_fd fd, std::vector<char>* bytes) {
- int32_t le_val = read_int32(fd);
- auto old_size = bytes->size();
- bytes->resize(old_size + sizeof(le_val));
- memcpy(bytes->data() + old_size, &le_val, sizeof(le_val));
-}
-
-static inline void append_bytes_with_size(borrowed_fd fd, std::vector<char>* bytes) {
- int32_t le_size = read_int32(fd);
- if (le_size < 0) {
- return;
- }
- int32_t size = int32_t(le32toh(le_size));
- auto old_size = bytes->size();
- bytes->resize(old_size + sizeof(le_size) + size);
- memcpy(bytes->data() + old_size, &le_size, sizeof(le_size));
- ReadFdExactly(fd, bytes->data() + old_size + sizeof(le_size), size);
-}
-
-static inline std::pair<std::vector<char>, int32_t> read_id_sig_headers(borrowed_fd fd) {
- std::vector<char> result;
- append_int(fd, &result); // version
- append_bytes_with_size(fd, &result); // hashingInfo
- append_bytes_with_size(fd, &result); // signingInfo
- auto le_tree_size = read_int32(fd);
- auto tree_size = int32_t(le32toh(le_tree_size)); // size of the verity tree
- return {std::move(result), tree_size};
-}
-
-static inline Size verity_tree_size_for_file(Size fileSize) {
- constexpr int INCFS_DATA_FILE_BLOCK_SIZE = 4096;
- constexpr int SHA256_DIGEST_SIZE = 32;
- constexpr int digest_size = SHA256_DIGEST_SIZE;
- constexpr int hash_per_block = INCFS_DATA_FILE_BLOCK_SIZE / digest_size;
-
- Size total_tree_block_count = 0;
-
- auto block_count = 1 + (fileSize - 1) / INCFS_DATA_FILE_BLOCK_SIZE;
- auto hash_block_count = block_count;
- for (auto i = 0; hash_block_count > 1; i++) {
- hash_block_count = (hash_block_count + hash_per_block - 1) / hash_per_block;
- total_tree_block_count += hash_block_count;
- }
- return total_tree_block_count * INCFS_DATA_FILE_BLOCK_SIZE;
-}
-
// Read, verify and return the signature bytes. Keeping fd at the position of start of verity tree.
static std::pair<unique_fd, std::vector<char>> read_signature(Size file_size,
std::string signature_file,
@@ -104,7 +47,7 @@
return {};
}
- unique_fd fd(adb_open(signature_file.c_str(), O_RDONLY | O_CLOEXEC));
+ unique_fd fd(adb_open(signature_file.c_str(), O_RDONLY));
if (fd < 0) {
if (!silent) {
fprintf(stderr, "Failed to open signature file: %s. Abort.\n", signature_file.c_str());
@@ -172,9 +115,8 @@
return {};
}
- auto file_desc =
- StringPrintf("%s:%lld:%s:%s", android::base::Basename(file).c_str(),
- (long long)st.st_size, std::to_string(i).c_str(), signature.c_str());
+ auto file_desc = StringPrintf("%s:%lld:%d:%s:1", android::base::Basename(file).c_str(),
+ (long long)st.st_size, i, signature.c_str());
command_args.push_back(std::move(file_desc));
signature_fds.push_back(std::move(signature_fd));
@@ -190,21 +132,9 @@
return {};
}
- // Pushing verity trees for all installation files.
- for (auto&& local_fd : signature_fds) {
- if (!copy_to_file(local_fd.get(), connection_fd.get())) {
- if (!silent) {
- fprintf(stderr, "Failed to stream tree bytes: %s. Abort.\n", strerror(errno));
- }
- return {};
- }
- }
-
return connection_fd;
}
-} // namespace
-
bool can_install(const Files& files) {
for (const auto& file : files) {
struct stat st;
diff --git a/adb/client/incremental_server.cpp b/adb/client/incremental_server.cpp
index 4b87d0a..1a1361c 100644
--- a/adb/client/incremental_server.cpp
+++ b/adb/client/incremental_server.cpp
@@ -1,4 +1,4 @@
-/*
+/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -44,9 +44,10 @@
namespace incremental {
-static constexpr int kBlockSize = 4096;
+static constexpr int kHashesPerBlock = kBlockSize / kDigestSize;
static constexpr int kCompressedSizeMax = kBlockSize * 0.95;
static constexpr int8_t kTypeData = 0;
+static constexpr int8_t kTypeHash = 1;
static constexpr int8_t kCompressionNone = 0;
static constexpr int8_t kCompressionLZ4 = 1;
static constexpr int kCompressBound = std::max(kBlockSize, LZ4_COMPRESSBOUND(kBlockSize));
@@ -132,41 +133,64 @@
CompressionType compression_type; // 1 byte
BlockIdx block_idx; // 4 bytes
BlockSize block_size; // 2 bytes
+
+ static constexpr size_t responseSizeFor(size_t dataSize) {
+ return dataSize + sizeof(ResponseHeader);
+ }
+} __attribute__((packed));
+
+template <size_t Size = kBlockSize>
+struct BlockBuffer {
+ ResponseHeader header;
+ char data[Size];
} __attribute__((packed));
// Holds streaming state for a file
class File {
public:
// Plain file
- File(const char* filepath, FileId id, int64_t size, unique_fd fd) : File(filepath, id, size) {
+ File(const char* filepath, FileId id, int64_t size, unique_fd fd, int64_t tree_offset,
+ unique_fd tree_fd)
+ : File(filepath, id, size, tree_offset) {
this->fd_ = std::move(fd);
+ this->tree_fd_ = std::move(tree_fd);
priority_blocks_ = PriorityBlocksForFile(filepath, fd_.get(), size);
}
- int64_t ReadBlock(BlockIdx block_idx, void* buf, bool* is_zip_compressed,
- std::string* error) const {
- char* buf_ptr = static_cast<char*>(buf);
+ int64_t ReadDataBlock(BlockIdx block_idx, void* buf, bool* is_zip_compressed) const {
int64_t bytes_read = -1;
const off64_t offsetStart = blockIndexToOffset(block_idx);
- bytes_read = adb_pread(fd_, &buf_ptr[sizeof(ResponseHeader)], kBlockSize, offsetStart);
+ bytes_read = adb_pread(fd_, buf, kBlockSize, offsetStart);
+ return bytes_read;
+ }
+ int64_t ReadTreeBlock(BlockIdx block_idx, void* buf) const {
+ int64_t bytes_read = -1;
+ const off64_t offsetStart = tree_offset_ + blockIndexToOffset(block_idx);
+ bytes_read = adb_pread(tree_fd_, buf, kBlockSize, offsetStart);
return bytes_read;
}
- const unique_fd& RawFd() const { return fd_; }
const std::vector<BlockIdx>& PriorityBlocks() const { return priority_blocks_; }
std::vector<bool> sentBlocks;
NumBlocks sentBlocksCount = 0;
+ std::vector<bool> sentTreeBlocks;
+
const char* const filepath;
const FileId id;
const int64_t size;
private:
- File(const char* filepath, FileId id, int64_t size) : filepath(filepath), id(id), size(size) {
+ File(const char* filepath, FileId id, int64_t size, int64_t tree_offset)
+ : filepath(filepath), id(id), size(size), tree_offset_(tree_offset) {
sentBlocks.resize(numBytesToNumBlocks(size));
+ sentTreeBlocks.resize(verity_tree_blocks_for_file(size));
}
unique_fd fd_;
std::vector<BlockIdx> priority_blocks_;
+
+ unique_fd tree_fd_;
+ const int64_t tree_offset_;
};
class IncrementalServer {
@@ -174,6 +198,8 @@
IncrementalServer(unique_fd adb_fd, unique_fd output_fd, std::vector<File> files)
: adb_fd_(std::move(adb_fd)), output_fd_(std::move(output_fd)), files_(std::move(files)) {
buffer_.reserve(kReadBufferSize);
+ pendingBlocksBuffer_.resize(kChunkFlushSize + 2 * kBlockSize);
+ pendingBlocks_ = pendingBlocksBuffer_.data() + sizeof(ChunkHeader);
}
bool Serve();
@@ -208,7 +234,11 @@
void erase_buffer_head(int count) { buffer_.erase(buffer_.begin(), buffer_.begin() + count); }
enum class SendResult { Sent, Skipped, Error };
- SendResult SendBlock(FileId fileId, BlockIdx blockIdx, bool flush = false);
+ SendResult SendDataBlock(FileId fileId, BlockIdx blockIdx, bool flush = false);
+
+ bool SendTreeBlock(FileId fileId, int32_t fileBlockIdx, BlockIdx blockIdx);
+ bool SendTreeBlocksForDataBlock(FileId fileId, BlockIdx blockIdx);
+
bool SendDone();
void RunPrefetching();
@@ -228,7 +258,10 @@
int compressed_ = 0, uncompressed_ = 0;
long long sentSize_ = 0;
- std::vector<char> pendingBlocks_;
+ static constexpr auto kChunkFlushSize = 31 * kBlockSize;
+
+ std::vector<char> pendingBlocksBuffer_;
+ char* pendingBlocks_ = nullptr;
// True when client notifies that all the data has been received
bool servingComplete_;
@@ -250,7 +283,7 @@
if (bcur > 0) {
// output the rest.
- WriteFdExactly(output_fd_, buffer_.data(), bcur);
+ (void)WriteFdExactly(output_fd_, buffer_.data(), bcur);
erase_buffer_head(bcur);
}
@@ -265,9 +298,10 @@
auto res = adb_poll(&pfd, 1, blocking ? kPollTimeoutMillis : 0);
if (res != 1) {
- WriteFdExactly(output_fd_, buffer_.data(), buffer_.size());
+ auto err = errno;
+ (void)WriteFdExactly(output_fd_, buffer_.data(), buffer_.size());
if (res < 0) {
- D("Failed to poll: %s\n", strerror(errno));
+ D("Failed to poll: %s", strerror(err));
return false;
}
if (blocking) {
@@ -289,7 +323,7 @@
continue;
}
- D("Failed to read from fd %d: %d. Exit\n", adb_fd_.get(), errno);
+ D("Failed to read from fd %d: %d. Exit", adb_fd_.get(), errno);
break;
}
// socket is closed. print remaining messages
@@ -313,56 +347,113 @@
return request;
}
-auto IncrementalServer::SendBlock(FileId fileId, BlockIdx blockIdx, bool flush) -> SendResult {
+bool IncrementalServer::SendTreeBlocksForDataBlock(const FileId fileId, const BlockIdx blockIdx) {
+ auto& file = files_[fileId];
+ const int32_t data_block_count = numBytesToNumBlocks(file.size);
+
+ const int32_t total_nodes_count(file.sentTreeBlocks.size());
+ const int32_t leaf_nodes_count = (data_block_count + kHashesPerBlock - 1) / kHashesPerBlock;
+
+ const int32_t leaf_nodes_offset = total_nodes_count - leaf_nodes_count;
+
+ // Leaf level, sending only 1 block.
+ const int32_t leaf_idx = leaf_nodes_offset + blockIdx / kHashesPerBlock;
+ if (file.sentTreeBlocks[leaf_idx]) {
+ return true;
+ }
+ if (!SendTreeBlock(fileId, blockIdx, leaf_idx)) {
+ return false;
+ }
+ file.sentTreeBlocks[leaf_idx] = true;
+
+ // Non-leaf, sending EVERYTHING. This should be done only once.
+ if (leaf_nodes_offset == 0 || file.sentTreeBlocks[0]) {
+ return true;
+ }
+
+ for (int32_t i = 0; i < leaf_nodes_offset; ++i) {
+ if (!SendTreeBlock(fileId, blockIdx, i)) {
+ return false;
+ }
+ file.sentTreeBlocks[i] = true;
+ }
+ return true;
+}
+
+bool IncrementalServer::SendTreeBlock(FileId fileId, int32_t fileBlockIdx, BlockIdx blockIdx) {
+ const auto& file = files_[fileId];
+
+ BlockBuffer buffer;
+ const int64_t bytesRead = file.ReadTreeBlock(blockIdx, buffer.data);
+ if (bytesRead <= 0) {
+ fprintf(stderr, "Failed to get data for %s.idsig at blockIdx=%d.\n", file.filepath,
+ blockIdx);
+ return false;
+ }
+
+ buffer.header.compression_type = kCompressionNone;
+ buffer.header.block_type = kTypeHash;
+ buffer.header.file_id = toBigEndian(fileId);
+ buffer.header.block_size = toBigEndian(int16_t(bytesRead));
+ buffer.header.block_idx = toBigEndian(blockIdx);
+
+ Send(&buffer, ResponseHeader::responseSizeFor(bytesRead), /*flush=*/false);
+
+ return true;
+}
+
+auto IncrementalServer::SendDataBlock(FileId fileId, BlockIdx blockIdx, bool flush) -> SendResult {
auto& file = files_[fileId];
if (blockIdx >= static_cast<long>(file.sentBlocks.size())) {
- fprintf(stderr, "Failed to read file %s at block %" PRId32 " (past end).\n", file.filepath,
- blockIdx);
- return SendResult::Error;
+ // may happen as we schedule some extra blocks for reported page misses
+ D("Skipped reading file %s at block %" PRId32 " (past end).", file.filepath, blockIdx);
+ return SendResult::Skipped;
}
if (file.sentBlocks[blockIdx]) {
return SendResult::Skipped;
}
- std::string error;
- char raw[sizeof(ResponseHeader) + kBlockSize];
- bool isZipCompressed = false;
- const int64_t bytesRead = file.ReadBlock(blockIdx, &raw, &isZipCompressed, &error);
- if (bytesRead < 0) {
- fprintf(stderr, "Failed to get data for %s at blockIdx=%d (%s).\n", file.filepath, blockIdx,
- error.c_str());
+
+ if (!SendTreeBlocksForDataBlock(fileId, blockIdx)) {
return SendResult::Error;
}
- ResponseHeader* header = nullptr;
- char data[sizeof(ResponseHeader) + kCompressBound];
- char* compressed = data + sizeof(*header);
+ BlockBuffer raw;
+ bool isZipCompressed = false;
+ const int64_t bytesRead = file.ReadDataBlock(blockIdx, raw.data, &isZipCompressed);
+ if (bytesRead < 0) {
+ fprintf(stderr, "Failed to get data for %s at blockIdx=%d (%d).\n", file.filepath, blockIdx,
+ errno);
+ return SendResult::Error;
+ }
+
+ BlockBuffer<kCompressBound> compressed;
int16_t compressedSize = 0;
if (!isZipCompressed) {
- compressedSize =
- LZ4_compress_default(raw + sizeof(*header), compressed, bytesRead, kCompressBound);
+ compressedSize = LZ4_compress_default(raw.data, compressed.data, bytesRead, kCompressBound);
}
int16_t blockSize;
+ ResponseHeader* header;
if (compressedSize > 0 && compressedSize < kCompressedSizeMax) {
++compressed_;
blockSize = compressedSize;
- header = reinterpret_cast<ResponseHeader*>(data);
+ header = &compressed.header;
header->compression_type = kCompressionLZ4;
} else {
++uncompressed_;
blockSize = bytesRead;
- header = reinterpret_cast<ResponseHeader*>(raw);
+ header = &raw.header;
header->compression_type = kCompressionNone;
}
header->block_type = kTypeData;
-
header->file_id = toBigEndian(fileId);
header->block_size = toBigEndian(blockSize);
header->block_idx = toBigEndian(blockIdx);
file.sentBlocks[blockIdx] = true;
file.sentBlocksCount += 1;
- Send(header, sizeof(*header) + blockSize, flush);
+ Send(header, ResponseHeader::responseSizeFor(blockSize), flush);
+
return SendResult::Sent;
}
@@ -388,7 +479,8 @@
if (!priority_blocks.empty()) {
for (auto& i = prefetch.priorityIndex;
blocksToSend > 0 && i < (BlockIdx)priority_blocks.size(); ++i) {
- if (auto res = SendBlock(file.id, priority_blocks[i]); res == SendResult::Sent) {
+ if (auto res = SendDataBlock(file.id, priority_blocks[i]);
+ res == SendResult::Sent) {
--blocksToSend;
} else if (res == SendResult::Error) {
fprintf(stderr, "Failed to send priority block %" PRId32 "\n", i);
@@ -396,7 +488,7 @@
}
}
for (auto& i = prefetch.overallIndex; blocksToSend > 0 && i < prefetch.overallEnd; ++i) {
- if (auto res = SendBlock(file.id, i); res == SendResult::Sent) {
+ if (auto res = SendDataBlock(file.id, i); res == SendResult::Sent) {
--blocksToSend;
} else if (res == SendResult::Error) {
fprintf(stderr, "Failed to send block %" PRId32 "\n", i);
@@ -409,30 +501,25 @@
}
void IncrementalServer::Send(const void* data, size_t size, bool flush) {
- constexpr auto kChunkFlushSize = 31 * kBlockSize;
-
- if (pendingBlocks_.empty()) {
- pendingBlocks_.resize(sizeof(ChunkHeader));
- }
- pendingBlocks_.insert(pendingBlocks_.end(), static_cast<const char*>(data),
- static_cast<const char*>(data) + size);
- if (flush || pendingBlocks_.size() > kChunkFlushSize) {
+ pendingBlocks_ = std::copy_n(static_cast<const char*>(data), size, pendingBlocks_);
+ if (flush || pendingBlocks_ - pendingBlocksBuffer_.data() > kChunkFlushSize) {
Flush();
}
}
void IncrementalServer::Flush() {
- if (pendingBlocks_.empty()) {
+ auto dataBytes = pendingBlocks_ - (pendingBlocksBuffer_.data() + sizeof(ChunkHeader));
+ if (dataBytes == 0) {
return;
}
- *(ChunkHeader*)pendingBlocks_.data() =
- toBigEndian<int32_t>(pendingBlocks_.size() - sizeof(ChunkHeader));
- if (!WriteFdExactly(adb_fd_, pendingBlocks_.data(), pendingBlocks_.size())) {
- fprintf(stderr, "Failed to write %d bytes\n", int(pendingBlocks_.size()));
+ *(ChunkHeader*)pendingBlocksBuffer_.data() = toBigEndian<int32_t>(dataBytes);
+ auto totalBytes = sizeof(ChunkHeader) + dataBytes;
+ if (!WriteFdExactly(adb_fd_, pendingBlocksBuffer_.data(), totalBytes)) {
+ fprintf(stderr, "Failed to write %d bytes\n", int(totalBytes));
}
- sentSize_ += pendingBlocks_.size();
- pendingBlocks_.clear();
+ sentSize_ += totalBytes;
+ pendingBlocks_ = pendingBlocksBuffer_.data() + sizeof(ChunkHeader);
}
bool IncrementalServer::ServingComplete(std::optional<TimePoint> startTime, int missesCount,
@@ -443,7 +530,7 @@
D("Streaming completed.\n"
"Misses: %d, of those unique: %d; sent compressed: %d, uncompressed: "
"%d, mb: %.3f\n"
- "Total time taken: %.3fms\n",
+ "Total time taken: %.3fms",
missesCount, missesSent, compressed_, uncompressed_, sentSize_ / 1024.0 / 1024.0,
duration_cast<microseconds>(endTime - (startTime ? *startTime : endTime)).count() / 1000.0);
return true;
@@ -510,9 +597,21 @@
fileId, blockIdx);
break;
}
- // fprintf(stderr, "\treading file %d block %04d\n", (int)fileId,
- // (int)blockIdx);
- if (auto res = SendBlock(fileId, blockIdx, true); res == SendResult::Error) {
+
+ if (VLOG_IS_ON(INCREMENTAL)) {
+ auto& file = files_[fileId];
+ auto posP = std::find(file.PriorityBlocks().begin(),
+ file.PriorityBlocks().end(), blockIdx);
+ D("\tMISSING BLOCK: reading file %d block %04d (in priority: %d of %d)",
+ (int)fileId, (int)blockIdx,
+ posP == file.PriorityBlocks().end()
+ ? -1
+ : int(posP - file.PriorityBlocks().begin()),
+ int(file.PriorityBlocks().size()));
+ }
+
+ if (auto res = SendDataBlock(fileId, blockIdx, true);
+ res == SendResult::Error) {
fprintf(stderr, "Failed to send block %" PRId32 ".\n", blockIdx);
} else if (res == SendResult::Sent) {
++missesSent;
@@ -536,7 +635,7 @@
fileId);
break;
}
- D("Received prefetch request for file_id %" PRId16 ".\n", fileId);
+ D("Received prefetch request for file_id %" PRId16 ".", fileId);
prefetches_.emplace_back(files_[fileId]);
break;
}
@@ -551,6 +650,43 @@
}
}
+static std::pair<unique_fd, int64_t> open_fd(const char* filepath) {
+ struct stat st;
+ if (stat(filepath, &st)) {
+ error_exit("inc-server: failed to stat input file '%s'.", filepath);
+ }
+
+ unique_fd fd(adb_open(filepath, O_RDONLY));
+ if (fd < 0) {
+ error_exit("inc-server: failed to open file '%s'.", filepath);
+ }
+
+ return {std::move(fd), st.st_size};
+}
+
+static std::pair<unique_fd, int64_t> open_signature(int64_t file_size, const char* filepath) {
+ std::string signature_file(filepath);
+ signature_file += IDSIG;
+
+ unique_fd fd(adb_open(signature_file.c_str(), O_RDONLY));
+ if (fd < 0) {
+ error_exit("inc-server: failed to open file '%s'.", signature_file.c_str());
+ }
+
+ auto [tree_offset, tree_size] = skip_id_sig_headers(fd);
+ if (auto expected = verity_tree_size_for_file(file_size); tree_size != expected) {
+ error_exit("Verity tree size mismatch in signature file: %s [was %lld, expected %lld].\n",
+ signature_file.c_str(), (long long)tree_size, (long long)expected);
+ }
+
+ int32_t data_block_count = numBytesToNumBlocks(file_size);
+ int32_t leaf_nodes_count = (data_block_count + kHashesPerBlock - 1) / kHashesPerBlock;
+ D("Verity tree loaded: %s, tree size: %d (%d blocks, %d leafs)", signature_file.c_str(),
+ int(tree_size), int(numBytesToNumBlocks(tree_size)), int(leaf_nodes_count));
+
+ return {std::move(fd), tree_offset};
+}
+
bool serve(int connection_fd, int output_fd, int argc, const char** argv) {
auto connection_ufd = unique_fd(connection_fd);
auto output_ufd = unique_fd(output_fd);
@@ -563,17 +699,11 @@
for (int i = 0; i < argc; ++i) {
auto filepath = argv[i];
- struct stat st;
- if (stat(filepath, &st)) {
- fprintf(stderr, "Failed to stat input file %s. Abort.\n", filepath);
- return {};
- }
+ auto [file_fd, file_size] = open_fd(filepath);
+ auto [sign_fd, sign_offset] = open_signature(file_size, filepath);
- unique_fd fd(adb_open(filepath, O_RDONLY));
- if (fd < 0) {
- error_exit("inc-server: failed to open file '%s'.", filepath);
- }
- files.emplace_back(filepath, i, st.st_size, std::move(fd));
+ files.emplace_back(filepath, i, file_size, std::move(file_fd), sign_offset,
+ std::move(sign_fd));
}
IncrementalServer server(std::move(connection_ufd), std::move(output_ufd), std::move(files));
diff --git a/adb/client/incremental_utils.cpp b/adb/client/incremental_utils.cpp
index fa501e4..dd117d2 100644
--- a/adb/client/incremental_utils.cpp
+++ b/adb/client/incremental_utils.cpp
@@ -18,6 +18,7 @@
#include "incremental_utils.h"
+#include <android-base/endian.h>
#include <android-base/mapped_file.h>
#include <android-base/strings.h>
#include <ziparchive/zip_archive.h>
@@ -28,19 +29,98 @@
#include <numeric>
#include <unordered_set>
+#include "adb_io.h"
#include "adb_trace.h"
#include "sysdeps.h"
using namespace std::literals;
-static constexpr int kBlockSize = 4096;
+namespace incremental {
static constexpr inline int32_t offsetToBlockIndex(int64_t offset) {
return (offset & ~(kBlockSize - 1)) >> 12;
}
+Size verity_tree_blocks_for_file(Size fileSize) {
+ if (fileSize == 0) {
+ return 0;
+ }
+
+ constexpr int hash_per_block = kBlockSize / kDigestSize;
+
+ Size total_tree_block_count = 0;
+
+ auto block_count = 1 + (fileSize - 1) / kBlockSize;
+ auto hash_block_count = block_count;
+ for (auto i = 0; hash_block_count > 1; i++) {
+ hash_block_count = (hash_block_count + hash_per_block - 1) / hash_per_block;
+ total_tree_block_count += hash_block_count;
+ }
+ return total_tree_block_count;
+}
+
+Size verity_tree_size_for_file(Size fileSize) {
+ return verity_tree_blocks_for_file(fileSize) * kBlockSize;
+}
+
+static inline int32_t read_int32(borrowed_fd fd) {
+ int32_t result;
+ return ReadFdExactly(fd, &result, sizeof(result)) ? result : -1;
+}
+
+static inline int32_t skip_int(borrowed_fd fd) {
+ return adb_lseek(fd, 4, SEEK_CUR);
+}
+
+static inline void append_int(borrowed_fd fd, std::vector<char>* bytes) {
+ int32_t le_val = read_int32(fd);
+ auto old_size = bytes->size();
+ bytes->resize(old_size + sizeof(le_val));
+ memcpy(bytes->data() + old_size, &le_val, sizeof(le_val));
+}
+
+static inline void append_bytes_with_size(borrowed_fd fd, std::vector<char>* bytes) {
+ int32_t le_size = read_int32(fd);
+ if (le_size < 0) {
+ return;
+ }
+ int32_t size = int32_t(le32toh(le_size));
+ auto old_size = bytes->size();
+ bytes->resize(old_size + sizeof(le_size) + size);
+ memcpy(bytes->data() + old_size, &le_size, sizeof(le_size));
+ ReadFdExactly(fd, bytes->data() + old_size + sizeof(le_size), size);
+}
+
+static inline int32_t skip_bytes_with_size(borrowed_fd fd) {
+ int32_t le_size = read_int32(fd);
+ if (le_size < 0) {
+ return -1;
+ }
+ int32_t size = int32_t(le32toh(le_size));
+ return (int32_t)adb_lseek(fd, size, SEEK_CUR);
+}
+
+std::pair<std::vector<char>, int32_t> read_id_sig_headers(borrowed_fd fd) {
+ std::vector<char> result;
+ append_int(fd, &result); // version
+ append_bytes_with_size(fd, &result); // hashingInfo
+ append_bytes_with_size(fd, &result); // signingInfo
+ auto le_tree_size = read_int32(fd);
+ auto tree_size = int32_t(le32toh(le_tree_size)); // size of the verity tree
+ return {std::move(result), tree_size};
+}
+
+std::pair<off64_t, ssize_t> skip_id_sig_headers(borrowed_fd fd) {
+ skip_int(fd); // version
+ skip_bytes_with_size(fd); // hashingInfo
+ auto offset = skip_bytes_with_size(fd); // signingInfo
+ auto le_tree_size = read_int32(fd);
+ auto tree_size = int32_t(le32toh(le_tree_size)); // size of the verity tree
+ return {offset + sizeof(le_tree_size), tree_size};
+}
+
template <class T>
-T valueAt(int fd, off64_t offset) {
+static T valueAt(borrowed_fd fd, off64_t offset) {
T t;
memset(&t, 0, sizeof(T));
if (adb_pread(fd, &t, sizeof(T), offset) != sizeof(T)) {
@@ -68,7 +148,7 @@
v.end());
}
-static off64_t CentralDirOffset(int fd, int64_t fileSize) {
+static off64_t CentralDirOffset(borrowed_fd fd, Size fileSize) {
static constexpr int kZipEocdRecMinSize = 22;
static constexpr int32_t kZipEocdRecSig = 0x06054b50;
static constexpr int kZipEocdCentralDirSizeFieldOffset = 12;
@@ -103,7 +183,7 @@
}
// Does not support APKs larger than 4GB
-static off64_t SignerBlockOffset(int fd, int64_t fileSize) {
+static off64_t SignerBlockOffset(borrowed_fd fd, Size fileSize) {
static constexpr int kApkSigBlockMinSize = 32;
static constexpr int kApkSigBlockFooterSize = 24;
static constexpr int64_t APK_SIG_BLOCK_MAGIC_HI = 0x3234206b636f6c42l;
@@ -132,7 +212,7 @@
return signerBlockOffset;
}
-static std::vector<int32_t> ZipPriorityBlocks(off64_t signerBlockOffset, int64_t fileSize) {
+static std::vector<int32_t> ZipPriorityBlocks(off64_t signerBlockOffset, Size fileSize) {
int32_t signerBlockIndex = offsetToBlockIndex(signerBlockOffset);
int32_t lastBlockIndex = offsetToBlockIndex(fileSize);
const auto numPriorityBlocks = lastBlockIndex - signerBlockIndex + 1;
@@ -160,7 +240,7 @@
return zipPriorityBlocks;
}
-[[maybe_unused]] static ZipArchiveHandle openZipArchiveFd(int fd) {
+[[maybe_unused]] static ZipArchiveHandle openZipArchiveFd(borrowed_fd fd) {
bool transferFdOwnership = false;
#ifdef _WIN32
//
@@ -179,20 +259,22 @@
D("%s failed at DuplicateHandle: %d", __func__, (int)::GetLastError());
return {};
}
- fd = _open_osfhandle((intptr_t)dupedHandle, _O_RDONLY | _O_BINARY);
- if (fd < 0) {
+ int osfd = _open_osfhandle((intptr_t)dupedHandle, _O_RDONLY | _O_BINARY);
+ if (osfd < 0) {
D("%s failed at _open_osfhandle: %d", __func__, errno);
::CloseHandle(handle);
return {};
}
transferFdOwnership = true;
+#else
+ int osfd = fd.get();
#endif
ZipArchiveHandle zip;
- if (OpenArchiveFd(fd, "apk_fd", &zip, transferFdOwnership) != 0) {
+ if (OpenArchiveFd(osfd, "apk_fd", &zip, transferFdOwnership) != 0) {
D("%s failed at OpenArchiveFd: %d", __func__, errno);
#ifdef _WIN32
// "_close()" is a secret WinCRT name for the regular close() function.
- _close(fd);
+ _close(osfd);
#endif
return {};
}
@@ -200,7 +282,7 @@
}
static std::pair<ZipArchiveHandle, std::unique_ptr<android::base::MappedFile>> openZipArchive(
- int fd, int64_t fileSize) {
+ borrowed_fd fd, Size fileSize) {
#ifndef __LP64__
if (fileSize >= INT_MAX) {
return {openZipArchiveFd(fd), nullptr};
@@ -220,7 +302,7 @@
return {zip, std::move(mapping)};
}
-static std::vector<int32_t> InstallationPriorityBlocks(int fd, int64_t fileSize) {
+static std::vector<int32_t> InstallationPriorityBlocks(borrowed_fd fd, Size fileSize) {
static constexpr std::array<std::string_view, 3> additional_matches = {
"resources.arsc"sv, "AndroidManifest.xml"sv, "classes.dex"sv};
auto [zip, _] = openZipArchive(fd, fileSize);
@@ -249,8 +331,9 @@
if (entryName == "classes.dex"sv) {
// Only the head is needed for installation
int32_t startBlockIndex = offsetToBlockIndex(entry.offset);
- appendBlocks(startBlockIndex, 1, &installationPriorityBlocks);
- D("\tadding to priority blocks: '%.*s' 1", (int)entryName.size(), entryName.data());
+ appendBlocks(startBlockIndex, 2, &installationPriorityBlocks);
+ D("\tadding to priority blocks: '%.*s' (%d)", (int)entryName.size(), entryName.data(),
+ 2);
} else {
// Full entries are needed for installation
off64_t entryStartOffset = entry.offset;
@@ -273,9 +356,9 @@
return installationPriorityBlocks;
}
-namespace incremental {
-std::vector<int32_t> PriorityBlocksForFile(const std::string& filepath, int fd, int64_t fileSize) {
- if (!android::base::EndsWithIgnoreCase(filepath, ".apk")) {
+std::vector<int32_t> PriorityBlocksForFile(const std::string& filepath, borrowed_fd fd,
+ Size fileSize) {
+ if (!android::base::EndsWithIgnoreCase(filepath, ".apk"sv)) {
return {};
}
off64_t signerOffset = SignerBlockOffset(fd, fileSize);
@@ -291,4 +374,5 @@
unduplicate(priorityBlocks);
return priorityBlocks;
}
+
} // namespace incremental
diff --git a/adb/client/incremental_utils.h b/adb/client/incremental_utils.h
index 8bcf6c0..d969d94 100644
--- a/adb/client/incremental_utils.h
+++ b/adb/client/incremental_utils.h
@@ -16,11 +16,31 @@
#pragma once
-#include <stdint.h>
+#include "adb_unique_fd.h"
#include <string>
+#include <string_view>
+#include <utility>
#include <vector>
+#include <stdint.h>
+
namespace incremental {
-std::vector<int32_t> PriorityBlocksForFile(const std::string& filepath, int fd, int64_t fileSize);
-} // namespace incremental
\ No newline at end of file
+
+using Size = int64_t;
+constexpr int kBlockSize = 4096;
+constexpr int kSha256DigestSize = 32;
+constexpr int kDigestSize = kSha256DigestSize;
+
+constexpr std::string_view IDSIG = ".idsig";
+
+std::vector<int32_t> PriorityBlocksForFile(const std::string& filepath, borrowed_fd fd,
+ Size fileSize);
+
+Size verity_tree_blocks_for_file(Size fileSize);
+Size verity_tree_size_for_file(Size fileSize);
+
+std::pair<std::vector<char>, int32_t> read_id_sig_headers(borrowed_fd fd);
+std::pair<off64_t, ssize_t> skip_id_sig_headers(borrowed_fd fd);
+
+} // namespace incremental
diff --git a/adb/client/main.cpp b/adb/client/main.cpp
index 4a9eadc..05e210f 100644
--- a/adb/client/main.cpp
+++ b/adb/client/main.cpp
@@ -105,7 +105,12 @@
fdevent_run_on_main_thread([]() { exit(0); });
});
- char* leak = getenv("ADB_LEAK");
+ const char* reject_kill_server = getenv("ADB_REJECT_KILL_SERVER");
+ if (reject_kill_server && strcmp(reject_kill_server, "1") == 0) {
+ adb_set_reject_kill_server(true);
+ }
+
+ const char* leak = getenv("ADB_LEAK");
if (leak && strcmp(leak, "1") == 0) {
intentionally_leak();
}
diff --git a/adb/compression_utils.h b/adb/compression_utils.h
index c445095..a0c48a2 100644
--- a/adb/compression_utils.h
+++ b/adb/compression_utils.h
@@ -16,10 +16,15 @@
#pragma once
+#include <algorithm>
+#include <memory>
#include <span>
+#include <android-base/logging.h>
+
#include <brotli/decode.h>
#include <brotli/encode.h>
+#include <lz4frame.h>
#include "types.h"
@@ -37,15 +42,103 @@
MoreOutput,
};
-struct BrotliDecoder {
+struct Decoder {
+ void Append(Block&& block) { input_buffer_.append(std::move(block)); }
+ bool Finish() {
+ bool old = std::exchange(finished_, true);
+ if (old) {
+ LOG(FATAL) << "Decoder::Finish called while already finished?";
+ return false;
+ }
+ return true;
+ }
+
+ virtual DecodeResult Decode(std::span<char>* output) = 0;
+
+ protected:
+ Decoder(std::span<char> output_buffer) : output_buffer_(output_buffer) {}
+ ~Decoder() = default;
+
+ bool finished_ = false;
+ IOVector input_buffer_;
+ std::span<char> output_buffer_;
+};
+
+struct Encoder {
+ void Append(Block input) { input_buffer_.append(std::move(input)); }
+ bool Finish() {
+ bool old = std::exchange(finished_, true);
+ if (old) {
+ LOG(FATAL) << "Decoder::Finish called while already finished?";
+ return false;
+ }
+ return true;
+ }
+
+ virtual EncodeResult Encode(Block* output) = 0;
+
+ protected:
+ explicit Encoder(size_t output_block_size) : output_block_size_(output_block_size) {}
+ ~Encoder() = default;
+
+ const size_t output_block_size_;
+ bool finished_ = false;
+ IOVector input_buffer_;
+};
+
+struct NullDecoder final : public Decoder {
+ explicit NullDecoder(std::span<char> output_buffer) : Decoder(output_buffer) {}
+
+ DecodeResult Decode(std::span<char>* output) final {
+ size_t available_out = output_buffer_.size();
+ void* p = output_buffer_.data();
+ while (available_out > 0 && !input_buffer_.empty()) {
+ size_t len = std::min(available_out, input_buffer_.front_size());
+ p = mempcpy(p, input_buffer_.front_data(), len);
+ available_out -= len;
+ input_buffer_.drop_front(len);
+ }
+ *output = std::span(output_buffer_.data(), static_cast<char*>(p));
+ if (input_buffer_.empty()) {
+ return finished_ ? DecodeResult::Done : DecodeResult::NeedInput;
+ }
+ return DecodeResult::MoreOutput;
+ }
+};
+
+struct NullEncoder final : public Encoder {
+ explicit NullEncoder(size_t output_block_size) : Encoder(output_block_size) {}
+
+ EncodeResult Encode(Block* output) final {
+ output->clear();
+ output->resize(output_block_size_);
+
+ size_t available_out = output->size();
+ void* p = output->data();
+
+ while (available_out > 0 && !input_buffer_.empty()) {
+ size_t len = std::min(available_out, input_buffer_.front_size());
+ p = mempcpy(p, input_buffer_.front_data(), len);
+ available_out -= len;
+ input_buffer_.drop_front(len);
+ }
+
+ output->resize(output->size() - available_out);
+
+ if (input_buffer_.empty()) {
+ return finished_ ? EncodeResult::Done : EncodeResult::NeedInput;
+ }
+ return EncodeResult::MoreOutput;
+ }
+};
+
+struct BrotliDecoder final : public Decoder {
explicit BrotliDecoder(std::span<char> output_buffer)
- : output_buffer_(output_buffer),
+ : Decoder(output_buffer),
decoder_(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr),
BrotliDecoderDestroyInstance) {}
- void Append(Block&& block) { input_buffer_.append(std::move(block)); }
-
- DecodeResult Decode(std::span<char>* output) {
+ DecodeResult Decode(std::span<char>* output) final {
size_t available_in = input_buffer_.front_size();
const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());
@@ -63,7 +156,8 @@
switch (r) {
case BROTLI_DECODER_RESULT_SUCCESS:
- return DecodeResult::Done;
+ // We need to wait for ID_DONE from the other end.
+ return finished_ ? DecodeResult::Done : DecodeResult::NeedInput;
case BROTLI_DECODER_RESULT_ERROR:
return DecodeResult::Error;
case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
@@ -77,33 +171,29 @@
}
private:
- IOVector input_buffer_;
- std::span<char> output_buffer_;
std::unique_ptr<BrotliDecoderState, void (*)(BrotliDecoderState*)> decoder_;
};
-template <size_t OutputBlockSize>
-struct BrotliEncoder {
- explicit BrotliEncoder()
- : output_block_(OutputBlockSize),
- output_bytes_left_(OutputBlockSize),
+struct BrotliEncoder final : public Encoder {
+ explicit BrotliEncoder(size_t output_block_size)
+ : Encoder(output_block_size),
+ output_block_(output_block_size_),
+ output_bytes_left_(output_block_size_),
encoder_(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr),
BrotliEncoderDestroyInstance) {
BrotliEncoderSetParameter(encoder_.get(), BROTLI_PARAM_QUALITY, 1);
}
- void Append(Block input) { input_buffer_.append(std::move(input)); }
- void Finish() { finished_ = true; }
-
- EncodeResult Encode(Block* output) {
+ EncodeResult Encode(Block* output) final {
output->clear();
+
while (true) {
size_t available_in = input_buffer_.front_size();
const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());
size_t available_out = output_bytes_left_;
- uint8_t* next_out = reinterpret_cast<uint8_t*>(output_block_.data() +
- (OutputBlockSize - output_bytes_left_));
+ uint8_t* next_out = reinterpret_cast<uint8_t*>(
+ output_block_.data() + (output_block_size_ - output_bytes_left_));
BrotliEncoderOperation op = BROTLI_OPERATION_PROCESS;
if (finished_) {
@@ -121,13 +211,13 @@
output_bytes_left_ = available_out;
if (BrotliEncoderIsFinished(encoder_.get())) {
- output_block_.resize(OutputBlockSize - output_bytes_left_);
+ output_block_.resize(output_block_size_ - output_bytes_left_);
*output = std::move(output_block_);
return EncodeResult::Done;
} else if (output_bytes_left_ == 0) {
*output = std::move(output_block_);
- output_block_.resize(OutputBlockSize);
- output_bytes_left_ = OutputBlockSize;
+ output_block_.resize(output_block_size_);
+ output_bytes_left_ = output_block_size_;
return EncodeResult::MoreOutput;
} else if (input_buffer_.empty()) {
return EncodeResult::NeedInput;
@@ -136,9 +226,158 @@
}
private:
- bool finished_ = false;
- IOVector input_buffer_;
Block output_block_;
size_t output_bytes_left_;
std::unique_ptr<BrotliEncoderState, void (*)(BrotliEncoderState*)> encoder_;
};
+
+struct LZ4Decoder final : public Decoder {
+ explicit LZ4Decoder(std::span<char> output_buffer)
+ : Decoder(output_buffer), decoder_(nullptr, nullptr) {
+ LZ4F_dctx* dctx;
+ if (LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) != 0) {
+ LOG(FATAL) << "failed to initialize LZ4 decompression context";
+ }
+ decoder_ = std::unique_ptr<LZ4F_dctx, decltype(&LZ4F_freeDecompressionContext)>(
+ dctx, LZ4F_freeDecompressionContext);
+ }
+
+ DecodeResult Decode(std::span<char>* output) final {
+ size_t available_in = input_buffer_.front_size();
+ const char* next_in = input_buffer_.front_data();
+
+ size_t available_out = output_buffer_.size();
+ char* next_out = output_buffer_.data();
+
+ size_t rc = LZ4F_decompress(decoder_.get(), next_out, &available_out, next_in,
+ &available_in, nullptr);
+ if (LZ4F_isError(rc)) {
+ LOG(ERROR) << "LZ4F_decompress failed: " << LZ4F_getErrorName(rc);
+ return DecodeResult::Error;
+ }
+
+ input_buffer_.drop_front(available_in);
+
+ if (rc == 0) {
+ if (!input_buffer_.empty()) {
+ LOG(ERROR) << "LZ4 stream hit end before reading all data";
+ return DecodeResult::Error;
+ }
+ lz4_done_ = true;
+ }
+
+ *output = std::span<char>(output_buffer_.data(), available_out);
+
+ if (finished_) {
+ return input_buffer_.empty() && lz4_done_ ? DecodeResult::Done
+ : DecodeResult::MoreOutput;
+ }
+
+ return DecodeResult::NeedInput;
+ }
+
+ private:
+ bool lz4_done_ = false;
+ std::unique_ptr<LZ4F_dctx, LZ4F_errorCode_t (*)(LZ4F_dctx*)> decoder_;
+};
+
+struct LZ4Encoder final : public Encoder {
+ explicit LZ4Encoder(size_t output_block_size)
+ : Encoder(output_block_size), encoder_(nullptr, nullptr) {
+ LZ4F_cctx* cctx;
+ if (LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) != 0) {
+ LOG(FATAL) << "failed to initialize LZ4 compression context";
+ }
+ encoder_ = std::unique_ptr<LZ4F_cctx, decltype(&LZ4F_freeCompressionContext)>(
+ cctx, LZ4F_freeCompressionContext);
+ Block header(LZ4F_HEADER_SIZE_MAX);
+ size_t rc = LZ4F_compressBegin(encoder_.get(), header.data(), header.size(), nullptr);
+ if (LZ4F_isError(rc)) {
+ LOG(FATAL) << "LZ4F_compressBegin failed: %s", LZ4F_getErrorName(rc);
+ }
+ header.resize(rc);
+ output_buffer_.append(std::move(header));
+ }
+
+ // As an optimization, only emit a block if we have an entire output block ready, or we're done.
+ bool OutputReady() const {
+ return output_buffer_.size() >= output_block_size_ || lz4_finalized_;
+ }
+
+ // TODO: Switch the output type to IOVector to remove a copy?
+ EncodeResult Encode(Block* output) final {
+ size_t available_in = input_buffer_.front_size();
+ const char* next_in = input_buffer_.front_data();
+
+ // LZ4 makes no guarantees about being able to recover from trying to compress with an
+ // insufficiently large output buffer. LZ4F_compressBound tells us how much buffer we
+ // need to compress a given number of bytes, but the smallest value seems to be bigger
+ // than SYNC_DATA_MAX, so we need to buffer ourselves.
+
+ // Input size chosen to be a local maximum for LZ4F_compressBound (i.e. the block size).
+ constexpr size_t max_input_size = 65536;
+ const size_t encode_block_size = LZ4F_compressBound(max_input_size, nullptr);
+
+ if (available_in != 0) {
+ if (lz4_finalized_) {
+ LOG(ERROR) << "LZ4Encoder received data after Finish?";
+ return EncodeResult::Error;
+ }
+
+ available_in = std::min(available_in, max_input_size);
+
+ Block encode_block(encode_block_size);
+ size_t available_out = encode_block.capacity();
+ char* next_out = encode_block.data();
+
+ size_t rc = LZ4F_compressUpdate(encoder_.get(), next_out, available_out, next_in,
+ available_in, nullptr);
+ if (LZ4F_isError(rc)) {
+ LOG(ERROR) << "LZ4F_compressUpdate failed: " << LZ4F_getErrorName(rc);
+ return EncodeResult::Error;
+ }
+
+ input_buffer_.drop_front(available_in);
+
+ available_out -= rc;
+ next_out += rc;
+
+ encode_block.resize(encode_block_size - available_out);
+ output_buffer_.append(std::move(encode_block));
+ }
+
+ if (finished_ && !lz4_finalized_) {
+ lz4_finalized_ = true;
+
+ Block final_block(encode_block_size + 4);
+ size_t rc = LZ4F_compressEnd(encoder_.get(), final_block.data(), final_block.size(),
+ nullptr);
+ if (LZ4F_isError(rc)) {
+ LOG(ERROR) << "LZ4F_compressEnd failed: " << LZ4F_getErrorName(rc);
+ return EncodeResult::Error;
+ }
+
+ final_block.resize(rc);
+ output_buffer_.append(std::move(final_block));
+ }
+
+ if (OutputReady()) {
+ size_t len = std::min(output_block_size_, output_buffer_.size());
+ *output = output_buffer_.take_front(len).coalesce();
+ } else {
+ output->clear();
+ }
+
+ if (lz4_finalized_ && output_buffer_.empty()) {
+ return EncodeResult::Done;
+ } else if (OutputReady()) {
+ return EncodeResult::MoreOutput;
+ }
+ return EncodeResult::NeedInput;
+ }
+
+ private:
+ bool lz4_finalized_ = false;
+ std::unique_ptr<LZ4F_cctx, LZ4F_errorCode_t (*)(LZ4F_cctx*)> encoder_;
+ IOVector output_buffer_;
+};
diff --git a/adb/daemon/file_sync_service.cpp b/adb/daemon/file_sync_service.cpp
index 5ccddea..d58131e 100644
--- a/adb/daemon/file_sync_service.cpp
+++ b/adb/daemon/file_sync_service.cpp
@@ -35,6 +35,7 @@
#include <optional>
#include <span>
#include <string>
+#include <variant>
#include <vector>
#include <android-base/file.h>
@@ -266,37 +267,60 @@
return SendSyncFail(fd, StringPrintf("%s: %s", reason.c_str(), strerror(errno)));
}
-static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp) {
+static bool handle_send_file_data(borrowed_fd s, unique_fd fd, uint32_t* timestamp,
+ CompressionType compression) {
syncmsg msg;
- Block decode_buffer(SYNC_DATA_MAX);
- BrotliDecoder decoder(std::span(decode_buffer.data(), decode_buffer.size()));
+ Block buffer(SYNC_DATA_MAX);
+ std::span<char> buffer_span(buffer.data(), buffer.size());
+ std::variant<std::monostate, NullDecoder, BrotliDecoder, LZ4Decoder> decoder_storage;
+ Decoder* decoder = nullptr;
+
+ switch (compression) {
+ case CompressionType::None:
+ decoder = &decoder_storage.emplace<NullDecoder>(buffer_span);
+ break;
+
+ case CompressionType::Brotli:
+ decoder = &decoder_storage.emplace<BrotliDecoder>(buffer_span);
+ break;
+
+ case CompressionType::LZ4:
+ decoder = &decoder_storage.emplace<LZ4Decoder>(buffer_span);
+ break;
+
+ case CompressionType::Any:
+ LOG(FATAL) << "unexpected CompressionType::Any";
+ }
+
while (true) {
if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false;
- if (msg.data.id != ID_DATA) {
- if (msg.data.id == ID_DONE) {
- *timestamp = msg.data.size;
- return true;
- }
+ if (msg.data.id == ID_DONE) {
+ *timestamp = msg.data.size;
+ decoder->Finish();
+ } else if (msg.data.id == ID_DATA) {
+ Block block(msg.data.size);
+ if (!ReadFdExactly(s, block.data(), msg.data.size)) return false;
+ decoder->Append(std::move(block));
+ } else {
SendSyncFail(s, "invalid data message");
return false;
}
- Block block(msg.data.size);
- if (!ReadFdExactly(s, block.data(), msg.data.size)) return false;
- decoder.Append(std::move(block));
-
while (true) {
std::span<char> output;
- DecodeResult result = decoder.Decode(&output);
+ DecodeResult result = decoder->Decode(&output);
if (result == DecodeResult::Error) {
SendSyncFailErrno(s, "decompress failed");
return false;
}
- if (!WriteFdExactly(fd, output.data(), output.size())) {
- SendSyncFailErrno(s, "write failed");
- return false;
+ // fd is -1 if the client is pushing with --dry-run.
+ if (fd != -1) {
+ if (!WriteFdExactly(fd, output.data(), output.size())) {
+ SendSyncFailErrno(s, "write failed");
+ return false;
+ }
}
if (result == DecodeResult::NeedInput) {
@@ -304,7 +328,7 @@
} else if (result == DecodeResult::MoreOutput) {
continue;
} else if (result == DecodeResult::Done) {
- break;
+ return true;
} else {
LOG(FATAL) << "invalid DecodeResult: " << static_cast<int>(result);
}
@@ -314,102 +338,67 @@
__builtin_unreachable();
}
-static bool handle_send_file_uncompressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp,
- std::vector<char>& buffer) {
- syncmsg msg;
-
- while (true) {
- if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false;
-
- if (msg.data.id != ID_DATA) {
- if (msg.data.id == ID_DONE) {
- *timestamp = msg.data.size;
- return true;
- }
- SendSyncFail(s, "invalid data message");
- return false;
- }
-
- if (msg.data.size > buffer.size()) { // TODO: resize buffer?
- SendSyncFail(s, "oversize data message");
- return false;
- }
- if (!ReadFdExactly(s, &buffer[0], msg.data.size)) return false;
- if (!WriteFdExactly(fd, &buffer[0], msg.data.size)) {
- SendSyncFailErrno(s, "write failed");
- return false;
- }
- }
-}
-
static bool handle_send_file(borrowed_fd s, const char* path, uint32_t* timestamp, uid_t uid,
- gid_t gid, uint64_t capabilities, mode_t mode, bool compressed,
- std::vector<char>& buffer, bool do_unlink) {
- int rc;
+ gid_t gid, uint64_t capabilities, mode_t mode,
+ CompressionType compression, bool dry_run, std::vector<char>& buffer,
+ bool do_unlink) {
syncmsg msg;
+ unique_fd fd;
- __android_log_security_bswrite(SEC_TAG_ADB_SEND_FILE, path);
-
- unique_fd fd(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode));
-
- if (fd < 0 && errno == ENOENT) {
- if (!secure_mkdirs(Dirname(path))) {
- SendSyncFailErrno(s, "secure_mkdirs failed");
- goto fail;
- }
+ if (!dry_run) {
+ __android_log_security_bswrite(SEC_TAG_ADB_SEND_FILE, path);
fd.reset(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode));
- }
- if (fd < 0 && errno == EEXIST) {
- fd.reset(adb_open_mode(path, O_WRONLY | O_CLOEXEC, mode));
- }
- if (fd < 0) {
- SendSyncFailErrno(s, "couldn't create file");
- goto fail;
- } else {
- if (fchown(fd.get(), uid, gid) == -1) {
- SendSyncFailErrno(s, "fchown failed");
- goto fail;
+
+ if (fd < 0 && errno == ENOENT) {
+ if (!secure_mkdirs(Dirname(path))) {
+ SendSyncFailErrno(s, "secure_mkdirs failed");
+ goto fail;
+ }
+ fd.reset(adb_open_mode(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, mode));
}
+ if (fd < 0 && errno == EEXIST) {
+ fd.reset(adb_open_mode(path, O_WRONLY | O_CLOEXEC, mode));
+ }
+ if (fd < 0) {
+ SendSyncFailErrno(s, "couldn't create file");
+ goto fail;
+ } else {
+ if (fchown(fd.get(), uid, gid) == -1) {
+ SendSyncFailErrno(s, "fchown failed");
+ goto fail;
+ }
#if defined(__ANDROID__)
- // Not all filesystems support setting SELinux labels. http://b/23530370.
- selinux_android_restorecon(path, 0);
+ // Not all filesystems support setting SELinux labels. http://b/23530370.
+ selinux_android_restorecon(path, 0);
#endif
- // fchown clears the setuid bit - restore it if present.
- // Ignore the result of calling fchmod. It's not supported
- // by all filesystems, so we don't check for success. b/12441485
- fchmod(fd.get(), mode);
- }
+ // fchown clears the setuid bit - restore it if present.
+ // Ignore the result of calling fchmod. It's not supported
+ // by all filesystems, so we don't check for success. b/12441485
+ fchmod(fd.get(), mode);
+ }
- {
- rc = posix_fadvise(fd.get(), 0, 0,
- POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED);
+ int rc = posix_fadvise(fd.get(), 0, 0,
+ POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE | POSIX_FADV_WILLNEED);
if (rc != 0) {
D("[ Failed to fadvise: %s ]", strerror(rc));
}
-
- bool result;
- if (compressed) {
- result = handle_send_file_compressed(s, std::move(fd), timestamp);
- } else {
- result = handle_send_file_uncompressed(s, std::move(fd), timestamp, buffer);
- }
-
- if (!result) {
- goto fail;
- }
-
- if (!update_capabilities(path, capabilities)) {
- SendSyncFailErrno(s, "update_capabilities failed");
- goto fail;
- }
-
- msg.status.id = ID_OKAY;
- msg.status.msglen = 0;
- return WriteFdExactly(s, &msg.status, sizeof(msg.status));
}
+ if (!handle_send_file_data(s, std::move(fd), timestamp, compression)) {
+ goto fail;
+ }
+
+ if (!update_capabilities(path, capabilities)) {
+ SendSyncFailErrno(s, "update_capabilities failed");
+ goto fail;
+ }
+
+ msg.status.id = ID_OKAY;
+ msg.status.msglen = 0;
+ return WriteFdExactly(s, &msg.status, sizeof(msg.status));
+
fail:
// If there's a problem on the device, we'll send an ID_FAIL message and
// close the socket. Unfortunately the kernel will sometimes throw that
@@ -448,7 +437,7 @@
uint32_t* timestamp, std::vector<char>& buffer)
__attribute__((error("no symlinks on Windows")));
#else
-static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp,
+static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp, bool dry_run,
std::vector<char>& buffer) {
syncmsg msg;
@@ -467,19 +456,21 @@
if (!ReadFdExactly(s, &buffer[0], len)) return false;
std::string buf_link;
- if (!android::base::Readlink(path, &buf_link) || (buf_link != &buffer[0])) {
- adb_unlink(path.c_str());
- auto ret = symlink(&buffer[0], path.c_str());
- if (ret && errno == ENOENT) {
- if (!secure_mkdirs(Dirname(path))) {
- SendSyncFailErrno(s, "secure_mkdirs failed");
+ if (!dry_run) {
+ if (!android::base::Readlink(path, &buf_link) || (buf_link != &buffer[0])) {
+ adb_unlink(path.c_str());
+ auto ret = symlink(&buffer[0], path.c_str());
+ if (ret && errno == ENOENT) {
+ if (!secure_mkdirs(Dirname(path))) {
+ SendSyncFailErrno(s, "secure_mkdirs failed");
+ return false;
+ }
+ ret = symlink(&buffer[0], path.c_str());
+ }
+ if (ret) {
+ SendSyncFailErrno(s, "symlink failed");
return false;
}
- ret = symlink(&buffer[0], path.c_str());
- }
- if (ret) {
- SendSyncFailErrno(s, "symlink failed");
- return false;
}
}
@@ -499,12 +490,15 @@
}
#endif
-static bool send_impl(int s, const std::string& path, mode_t mode, bool compressed,
- std::vector<char>& buffer) {
+static bool send_impl(int s, const std::string& path, mode_t mode, CompressionType compression,
+ bool dry_run, std::vector<char>& buffer) {
// Don't delete files before copying if they are not "regular" or symlinks.
struct stat st;
- bool do_unlink = (lstat(path.c_str(), &st) == -1) || S_ISREG(st.st_mode) ||
- (S_ISLNK(st.st_mode) && !S_ISLNK(mode));
+ bool do_unlink = false;
+ if (!dry_run) {
+ do_unlink = (lstat(path.c_str(), &st) == -1) || S_ISREG(st.st_mode) ||
+ (S_ISLNK(st.st_mode) && !S_ISLNK(mode));
+ }
if (do_unlink) {
adb_unlink(path.c_str());
}
@@ -512,7 +506,7 @@
bool result;
uint32_t timestamp;
if (S_ISLNK(mode)) {
- result = handle_send_link(s, path, ×tamp, buffer);
+ result = handle_send_link(s, path, ×tamp, dry_run, buffer);
} else {
// Copy user permission bits to "group" and "other" permissions.
mode &= 0777;
@@ -522,12 +516,12 @@
uid_t uid = -1;
gid_t gid = -1;
uint64_t capabilities = 0;
- if (should_use_fs_config(path)) {
+ if (should_use_fs_config(path) && !dry_run) {
adbd_fs_config(path.c_str(), 0, nullptr, &uid, &gid, &mode, &capabilities);
}
result = handle_send_file(s, path.c_str(), ×tamp, uid, gid, capabilities, mode,
- compressed, buffer, do_unlink);
+ compression, dry_run, buffer, do_unlink);
}
if (!result) {
@@ -560,7 +554,7 @@
return false;
}
- return send_impl(s, path, mode, false, buffer);
+ return send_impl(s, path, mode, CompressionType::None, false, buffer);
}
static bool do_send_v2(int s, const std::string& path, std::vector<char>& buffer) {
@@ -574,45 +568,80 @@
PLOG(ERROR) << "failed to read send_v2 setup packet";
}
- bool compressed = false;
+ bool dry_run = false;
+ std::optional<CompressionType> compression;
+
+ uint32_t orig_flags = msg.send_v2_setup.flags;
if (msg.send_v2_setup.flags & kSyncFlagBrotli) {
msg.send_v2_setup.flags &= ~kSyncFlagBrotli;
- compressed = true;
+ if (compression) {
+ SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d",
+ orig_flags));
+ return false;
+ }
+ compression = CompressionType::Brotli;
}
+ if (msg.send_v2_setup.flags & kSyncFlagLZ4) {
+ msg.send_v2_setup.flags &= ~kSyncFlagLZ4;
+ if (compression) {
+ SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d",
+ orig_flags));
+ return false;
+ }
+ compression = CompressionType::LZ4;
+ }
+ if (msg.send_v2_setup.flags & kSyncFlagDryRun) {
+ msg.send_v2_setup.flags &= ~kSyncFlagDryRun;
+ dry_run = true;
+ }
+
if (msg.send_v2_setup.flags) {
SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.send_v2_setup.flags));
return false;
}
errno = 0;
- return send_impl(s, path, msg.send_v2_setup.mode, compressed, buffer);
+ return send_impl(s, path, msg.send_v2_setup.mode, compression.value_or(CompressionType::None),
+ dry_run, buffer);
}
-static bool recv_uncompressed(borrowed_fd s, unique_fd fd, std::vector<char>& buffer) {
- syncmsg msg;
- msg.data.id = ID_DATA;
- while (true) {
- int r = adb_read(fd.get(), &buffer[0], buffer.size() - sizeof(msg.data));
- if (r <= 0) {
- if (r == 0) break;
- SendSyncFailErrno(s, "read failed");
- return false;
- }
- msg.data.size = r;
+static bool recv_impl(borrowed_fd s, const char* path, CompressionType compression,
+ std::vector<char>& buffer) {
+ __android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path);
- if (!WriteFdExactly(s, &msg.data, sizeof(msg.data)) || !WriteFdExactly(s, &buffer[0], r)) {
- return false;
- }
+ unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC));
+ if (fd < 0) {
+ SendSyncFailErrno(s, "open failed");
+ return false;
}
- return true;
-}
+ int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE);
+ if (rc != 0) {
+ D("[ Failed to fadvise: %s ]", strerror(rc));
+ }
-static bool recv_compressed(borrowed_fd s, unique_fd fd) {
syncmsg msg;
msg.data.id = ID_DATA;
- BrotliEncoder<SYNC_DATA_MAX> encoder;
+ std::variant<std::monostate, NullEncoder, BrotliEncoder, LZ4Encoder> encoder_storage;
+ Encoder* encoder;
+
+ switch (compression) {
+ case CompressionType::None:
+ encoder = &encoder_storage.emplace<NullEncoder>(SYNC_DATA_MAX);
+ break;
+
+ case CompressionType::Brotli:
+ encoder = &encoder_storage.emplace<BrotliEncoder>(SYNC_DATA_MAX);
+ break;
+
+ case CompressionType::LZ4:
+ encoder = &encoder_storage.emplace<LZ4Encoder>(SYNC_DATA_MAX);
+ break;
+
+ case CompressionType::Any:
+ LOG(FATAL) << "unexpected CompressionType::Any";
+ }
bool sending = true;
while (sending) {
@@ -624,15 +653,15 @@
}
if (r == 0) {
- encoder.Finish();
+ encoder->Finish();
} else {
input.resize(r);
- encoder.Append(std::move(input));
+ encoder->Append(std::move(input));
}
while (true) {
Block output;
- EncodeResult result = encoder.Encode(&output);
+ EncodeResult result = encoder->Encode(&output);
if (result == EncodeResult::Error) {
SendSyncFailErrno(s, "compress failed");
return false;
@@ -657,42 +686,13 @@
}
}
- return true;
-}
-
-static bool recv_impl(borrowed_fd s, const char* path, bool compressed, std::vector<char>& buffer) {
- __android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path);
-
- unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC));
- if (fd < 0) {
- SendSyncFailErrno(s, "open failed");
- return false;
- }
-
- int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE);
- if (rc != 0) {
- D("[ Failed to fadvise: %s ]", strerror(rc));
- }
-
- bool result;
- if (compressed) {
- result = recv_compressed(s, std::move(fd));
- } else {
- result = recv_uncompressed(s, std::move(fd), buffer);
- }
-
- if (!result) {
- return false;
- }
-
- syncmsg msg;
msg.data.id = ID_DONE;
msg.data.size = 0;
return WriteFdExactly(s, &msg.data, sizeof(msg.data));
}
static bool do_recv_v1(borrowed_fd s, const char* path, std::vector<char>& buffer) {
- return recv_impl(s, path, false, buffer);
+ return recv_impl(s, path, CompressionType::None, buffer);
}
static bool do_recv_v2(borrowed_fd s, const char* path, std::vector<char>& buffer) {
@@ -706,17 +706,33 @@
PLOG(ERROR) << "failed to read recv_v2 setup packet";
}
- bool compressed = false;
+ std::optional<CompressionType> compression;
+ uint32_t orig_flags = msg.recv_v2_setup.flags;
if (msg.recv_v2_setup.flags & kSyncFlagBrotli) {
msg.recv_v2_setup.flags &= ~kSyncFlagBrotli;
- compressed = true;
+ if (compression) {
+ SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d",
+ orig_flags));
+ return false;
+ }
+ compression = CompressionType::Brotli;
}
+ if (msg.recv_v2_setup.flags & kSyncFlagLZ4) {
+ msg.recv_v2_setup.flags &= ~kSyncFlagLZ4;
+ if (compression) {
+ SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d",
+ orig_flags));
+ return false;
+ }
+ compression = CompressionType::LZ4;
+ }
+
if (msg.recv_v2_setup.flags) {
SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.recv_v2_setup.flags));
return false;
}
- return recv_impl(s, path, compressed, buffer);
+ return recv_impl(s, path, compression.value_or(CompressionType::None), buffer);
}
static const char* sync_id_to_name(uint32_t id) {
diff --git a/adb/daemon/usb_ffs.cpp b/adb/daemon/usb_ffs.cpp
index 7bd611b..e538ca8 100644
--- a/adb/daemon/usb_ffs.cpp
+++ b/adb/daemon/usb_ffs.cpp
@@ -300,7 +300,6 @@
}
// Signal only when writing the descriptors to ffs
android::base::SetProperty("sys.usb.ffs.ready", "1");
- *out_control = std::move(control);
}
bulk_out.reset(adb_open(USB_FFS_ADB_OUT, O_RDONLY));
@@ -315,6 +314,7 @@
return false;
}
+ *out_control = std::move(control);
*out_bulk_in = std::move(bulk_in);
*out_bulk_out = std::move(bulk_out);
return true;
diff --git a/adb/file_sync_protocol.h b/adb/file_sync_protocol.h
index fd9a516..8f8f85f 100644
--- a/adb/file_sync_protocol.h
+++ b/adb/file_sync_protocol.h
@@ -92,6 +92,15 @@
enum SyncFlag : uint32_t {
kSyncFlagNone = 0,
kSyncFlagBrotli = 1,
+ kSyncFlagLZ4 = 2,
+ kSyncFlagDryRun = 0x8000'0000U,
+};
+
+enum class CompressionType {
+ None,
+ Any,
+ Brotli,
+ LZ4,
};
// send_v1 sent the path in a buffer, followed by a comma and the mode as a string.
diff --git a/adb/libs/adbconnection/Android.bp b/adb/libs/adbconnection/Android.bp
index ce2ab51..f7d2dc1 100644
--- a/adb/libs/adbconnection/Android.bp
+++ b/adb/libs/adbconnection/Android.bp
@@ -53,6 +53,7 @@
linux: {
version_script: "libadbconnection_client.map.txt",
},
+ darwin: { enabled: false },
},
stubs: {
symbol_file: "libadbconnection_client.map.txt",
diff --git a/adb/sysdeps.h b/adb/sysdeps.h
index 7ea30d1..7326ab1 100644
--- a/adb/sysdeps.h
+++ b/adb/sysdeps.h
@@ -14,11 +14,11 @@
* limitations under the License.
*/
+#pragma once
+
/* this file contains system-dependent definitions used by ADB
* they're related to threads, sockets and file descriptors
*/
-#ifndef _ADB_SYSDEPS_H
-#define _ADB_SYSDEPS_H
#ifdef __CYGWIN__
# undef _WIN32
@@ -43,7 +43,7 @@
#include "sysdeps/stat.h"
#if defined(__APPLE__)
-static void* mempcpy(void* dst, const void* src, size_t n) {
+static inline void* mempcpy(void* dst, const void* src, size_t n) {
return static_cast<char*>(memcpy(dst, src, n)) + n;
}
#endif
@@ -76,13 +76,13 @@
#define OS_PATH_SEPARATOR_STR "\\"
#define ENV_PATH_SEPARATOR_STR ";"
-static __inline__ bool adb_is_separator(char c) {
+static inline bool adb_is_separator(char c) {
return c == '\\' || c == '/';
}
extern int adb_thread_setname(const std::string& name);
-static __inline__ void close_on_exec(borrowed_fd fd) {
+static inline void close_on_exec(borrowed_fd fd) {
/* nothing really */
}
@@ -113,7 +113,7 @@
extern int adb_getlogin_r(char* buf, size_t bufsize);
// See the comments for the !defined(_WIN32) version of unix_close().
-static __inline__ int unix_close(int fd) {
+static inline int unix_close(int fd) {
return close(fd);
}
#undef close
@@ -123,7 +123,7 @@
extern int unix_read_interruptible(borrowed_fd fd, void* buf, size_t len);
// See the comments for the !defined(_WIN32) version of unix_read().
-static __inline__ int unix_read(borrowed_fd fd, void* buf, size_t len) {
+static inline int unix_read(borrowed_fd fd, void* buf, size_t len) {
return TEMP_FAILURE_RETRY(unix_read_interruptible(fd, buf, len));
}
@@ -134,7 +134,7 @@
#define pread ___xxx_pread
// See the comments for the !defined(_WIN32) version of unix_write().
-static __inline__ int unix_write(borrowed_fd fd, const void* buf, size_t len) {
+static inline int unix_write(borrowed_fd fd, const void* buf, size_t len) {
return write(fd.get(), buf, len);
}
#undef write
@@ -144,14 +144,14 @@
#define pwrite ___xxx_pwrite
// See the comments for the !defined(_WIN32) version of unix_lseek().
-static __inline__ int unix_lseek(borrowed_fd fd, int pos, int where) {
+static inline int unix_lseek(borrowed_fd fd, int pos, int where) {
return lseek(fd.get(), pos, where);
}
#undef lseek
#define lseek ___xxx_lseek
// See the comments for the !defined(_WIN32) version of adb_open_mode().
-static __inline__ int adb_open_mode(const char* path, int options, int mode) {
+static inline int adb_open_mode(const char* path, int options, int mode) {
return adb_open(path, options);
}
@@ -208,7 +208,7 @@
extern int adb_poll(adb_pollfd* fds, size_t nfds, int timeout);
#define poll ___xxx_poll
-static __inline__ int adb_is_absolute_host_path(const char* path) {
+static inline int adb_is_absolute_host_path(const char* path) {
return isalpha(path[0]) && path[1] == ':' && path[2] == '\\';
}
@@ -392,15 +392,15 @@
#define OS_PATH_SEPARATOR_STR "/"
#define ENV_PATH_SEPARATOR_STR ":"
-static __inline__ bool adb_is_separator(char c) {
+static inline bool adb_is_separator(char c) {
return c == '/';
}
-static __inline__ int get_fd_flags(borrowed_fd fd) {
+static inline int get_fd_flags(borrowed_fd fd) {
return fcntl(fd.get(), F_GETFD);
}
-static __inline__ void close_on_exec(borrowed_fd fd) {
+static inline void close_on_exec(borrowed_fd fd) {
int flags = get_fd_flags(fd);
if (flags >= 0 && (flags & FD_CLOEXEC) == 0) {
fcntl(fd.get(), F_SETFD, flags | FD_CLOEXEC);
@@ -416,7 +416,7 @@
// by unix_read(), unix_write(), unix_close()). Also, the C Runtime has
// configurable CR/LF translation which defaults to text mode, but is settable
// with _setmode().
-static __inline__ int unix_open(std::string_view path, int options, ...) {
+static inline int unix_open(std::string_view path, int options, ...) {
std::string zero_terminated(path.begin(), path.end());
if ((options & O_CREAT) == 0) {
return TEMP_FAILURE_RETRY(open(zero_terminated.c_str(), options));
@@ -432,7 +432,7 @@
// Similar to the two-argument adb_open(), but takes a mode parameter for file
// creation. See adb_open() for more info.
-static __inline__ int adb_open_mode(const char* pathname, int options, int mode) {
+static inline int adb_open_mode(const char* pathname, int options, int mode) {
return TEMP_FAILURE_RETRY(open(pathname, options, mode));
}
@@ -443,7 +443,7 @@
// sysdeps_win32.cpp) uses Windows native file I/O and bypasses the C Runtime
// and its CR/LF translation. The returned file descriptor should be used with
// adb_read(), adb_write(), adb_close(), etc.
-static __inline__ int adb_open(const char* pathname, int options) {
+static inline int adb_open(const char* pathname, int options) {
int fd = TEMP_FAILURE_RETRY(open(pathname, options));
if (fd < 0) return -1;
close_on_exec(fd);
@@ -452,7 +452,7 @@
#undef open
#define open ___xxx_open
-static __inline__ int adb_shutdown(borrowed_fd fd, int direction = SHUT_RDWR) {
+static inline int adb_shutdown(borrowed_fd fd, int direction = SHUT_RDWR) {
return shutdown(fd.get(), direction);
}
@@ -462,7 +462,7 @@
// Closes a file descriptor that came from adb_open() or adb_open_mode(), but
// not designed to take a file descriptor from unix_open(). See the comments
// for adb_open() for more info.
-__inline__ int adb_close(int fd) {
+inline int adb_close(int fd) {
return close(fd);
}
#undef close
@@ -471,23 +471,23 @@
// On Windows, ADB has an indirection layer for file descriptors. If we get a
// Win32 SOCKET object from an external library, we have to map it in to that
// indirection layer, which this does.
-__inline__ int adb_register_socket(int s) {
+inline int adb_register_socket(int s) {
return s;
}
-static __inline__ int adb_gethostname(char* name, size_t len) {
+static inline int adb_gethostname(char* name, size_t len) {
return gethostname(name, len);
}
-static __inline__ int adb_getlogin_r(char* buf, size_t bufsize) {
+static inline int adb_getlogin_r(char* buf, size_t bufsize) {
return getlogin_r(buf, bufsize);
}
-static __inline__ int adb_read(borrowed_fd fd, void* buf, size_t len) {
+static inline int adb_read(borrowed_fd fd, void* buf, size_t len) {
return TEMP_FAILURE_RETRY(read(fd.get(), buf, len));
}
-static __inline__ int adb_pread(borrowed_fd fd, void* buf, size_t len, off64_t offset) {
+static inline int adb_pread(borrowed_fd fd, void* buf, size_t len, off64_t offset) {
#if defined(__APPLE__)
return TEMP_FAILURE_RETRY(pread(fd.get(), buf, len, offset));
#else
@@ -496,7 +496,7 @@
}
// Like unix_read(), but does not handle EINTR.
-static __inline__ int unix_read_interruptible(borrowed_fd fd, void* buf, size_t len) {
+static inline int unix_read_interruptible(borrowed_fd fd, void* buf, size_t len) {
return read(fd.get(), buf, len);
}
@@ -505,11 +505,11 @@
#undef pread
#define pread ___xxx_pread
-static __inline__ int adb_write(borrowed_fd fd, const void* buf, size_t len) {
+static inline int adb_write(borrowed_fd fd, const void* buf, size_t len) {
return TEMP_FAILURE_RETRY(write(fd.get(), buf, len));
}
-static __inline__ int adb_pwrite(int fd, const void* buf, size_t len, off64_t offset) {
+static inline int adb_pwrite(int fd, const void* buf, size_t len, off64_t offset) {
#if defined(__APPLE__)
return TEMP_FAILURE_RETRY(pwrite(fd, buf, len, offset));
#else
@@ -522,7 +522,7 @@
#undef pwrite
#define pwrite ___xxx_pwrite
-static __inline__ int64_t adb_lseek(borrowed_fd fd, int64_t pos, int where) {
+static inline int64_t adb_lseek(borrowed_fd fd, int64_t pos, int where) {
#if defined(__APPLE__)
return lseek(fd.get(), pos, where);
#else
@@ -532,13 +532,13 @@
#undef lseek
#define lseek ___xxx_lseek
-static __inline__ int adb_unlink(const char* path) {
+static inline int adb_unlink(const char* path) {
return unlink(path);
}
#undef unlink
#define unlink ___xxx_unlink
-static __inline__ int adb_creat(const char* path, int mode) {
+static inline int adb_creat(const char* path, int mode) {
int fd = TEMP_FAILURE_RETRY(creat(path, mode));
if (fd < 0) return -1;
@@ -549,7 +549,7 @@
#undef creat
#define creat ___xxx_creat
-static __inline__ int unix_isatty(borrowed_fd fd) {
+static inline int unix_isatty(borrowed_fd fd) {
return isatty(fd.get());
}
#define isatty ___xxx_isatty
@@ -576,8 +576,8 @@
int network_connect(const std::string& host, int port, int type, int timeout, std::string* error);
-static __inline__ int adb_socket_accept(borrowed_fd serverfd, struct sockaddr* addr,
- socklen_t* addrlen) {
+static inline int adb_socket_accept(borrowed_fd serverfd, struct sockaddr* addr,
+ socklen_t* addrlen) {
int fd;
fd = TEMP_FAILURE_RETRY(accept(serverfd.get(), addr, addrlen));
@@ -606,7 +606,7 @@
#define unix_lseek adb_lseek
#define unix_close adb_close
-static __inline__ int adb_thread_setname(const std::string& name) {
+static inline int adb_thread_setname(const std::string& name) {
#ifdef __APPLE__
return pthread_setname_np(name.c_str());
#else
@@ -619,19 +619,19 @@
#endif
}
-static __inline__ int adb_setsockopt(borrowed_fd fd, int level, int optname, const void* optval,
- socklen_t optlen) {
+static inline int adb_setsockopt(borrowed_fd fd, int level, int optname, const void* optval,
+ socklen_t optlen) {
return setsockopt(fd.get(), level, optname, optval, optlen);
}
#undef setsockopt
#define setsockopt ___xxx_setsockopt
-static __inline__ int unix_socketpair(int d, int type, int protocol, int sv[2]) {
+static inline int unix_socketpair(int d, int type, int protocol, int sv[2]) {
return socketpair(d, type, protocol, sv);
}
-static __inline__ int adb_socketpair(int sv[2]) {
+static inline int adb_socketpair(int sv[2]) {
int rc;
rc = unix_socketpair(AF_UNIX, SOCK_STREAM, 0, sv);
@@ -646,32 +646,32 @@
#define socketpair ___xxx_socketpair
typedef struct pollfd adb_pollfd;
-static __inline__ int adb_poll(adb_pollfd* fds, size_t nfds, int timeout) {
+static inline int adb_poll(adb_pollfd* fds, size_t nfds, int timeout) {
return TEMP_FAILURE_RETRY(poll(fds, nfds, timeout));
}
#define poll ___xxx_poll
-static __inline__ int adb_mkdir(const std::string& path, int mode) {
+static inline int adb_mkdir(const std::string& path, int mode) {
return mkdir(path.c_str(), mode);
}
#undef mkdir
#define mkdir ___xxx_mkdir
-static __inline__ int adb_rename(const char* oldpath, const char* newpath) {
+static inline int adb_rename(const char* oldpath, const char* newpath) {
return rename(oldpath, newpath);
}
-static __inline__ int adb_is_absolute_host_path(const char* path) {
+static inline int adb_is_absolute_host_path(const char* path) {
return path[0] == '/';
}
-static __inline__ int adb_get_os_handle(borrowed_fd fd) {
+static inline int adb_get_os_handle(borrowed_fd fd) {
return fd.get();
}
-static __inline__ int cast_handle_to_int(int fd) {
+static inline int cast_handle_to_int(int fd) {
return fd;
}
@@ -721,5 +721,3 @@
// Win32 defines ERROR, which we don't need, but which conflicts with google3 logging.
#undef ERROR
#endif
-
-#endif /* _ADB_SYSDEPS_H */
diff --git a/adb/test_device.py b/adb/test_device.py
index 6a9ff89..f5e4cbb 100755
--- a/adb/test_device.py
+++ b/adb/test_device.py
@@ -77,8 +77,7 @@
class DeviceTest(unittest.TestCase):
- def setUp(self):
- self.device = adb.get_device()
+ device = adb.get_device()
class AbbTest(DeviceTest):
@@ -753,535 +752,611 @@
return files
-class FileOperationsTest(DeviceTest):
- SCRATCH_DIR = '/data/local/tmp'
- DEVICE_TEMP_FILE = SCRATCH_DIR + '/adb_test_file'
- DEVICE_TEMP_DIR = SCRATCH_DIR + '/adb_test_dir'
+class FileOperationsTest:
+ class Base(DeviceTest):
+ SCRATCH_DIR = '/data/local/tmp'
+ DEVICE_TEMP_FILE = SCRATCH_DIR + '/adb_test_file'
+ DEVICE_TEMP_DIR = SCRATCH_DIR + '/adb_test_dir'
- def _verify_remote(self, checksum, remote_path):
- dev_md5, _ = self.device.shell([get_md5_prog(self.device),
- remote_path])[0].split()
- self.assertEqual(checksum, dev_md5)
+ def setUp(self):
+ self.previous_env = os.environ.get("ADB_COMPRESSION")
+ os.environ["ADB_COMPRESSION"] = self.compression
- def _verify_local(self, checksum, local_path):
- with open(local_path, 'rb') as host_file:
- host_md5 = compute_md5(host_file.read())
- self.assertEqual(host_md5, checksum)
+ def tearDown(self):
+ if self.previous_env is None:
+ del os.environ["ADB_COMPRESSION"]
+ else:
+ os.environ["ADB_COMPRESSION"] = self.previous_env
- def test_push(self):
- """Push a randomly generated file to specified device."""
- kbytes = 512
- tmp = tempfile.NamedTemporaryFile(mode='wb', delete=False)
- rand_str = os.urandom(1024 * kbytes)
- tmp.write(rand_str)
- tmp.close()
+ def _verify_remote(self, checksum, remote_path):
+ dev_md5, _ = self.device.shell([get_md5_prog(self.device),
+ remote_path])[0].split()
+ self.assertEqual(checksum, dev_md5)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE])
- self.device.push(local=tmp.name, remote=self.DEVICE_TEMP_FILE)
+ def _verify_local(self, checksum, local_path):
+ with open(local_path, 'rb') as host_file:
+ host_md5 = compute_md5(host_file.read())
+ self.assertEqual(host_md5, checksum)
- self._verify_remote(compute_md5(rand_str), self.DEVICE_TEMP_FILE)
- self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE])
+ def test_push(self):
+ """Push a randomly generated file to specified device."""
+ kbytes = 512
+ tmp = tempfile.NamedTemporaryFile(mode='wb', delete=False)
+ rand_str = os.urandom(1024 * kbytes)
+ tmp.write(rand_str)
+ tmp.close()
- os.remove(tmp.name)
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE])
+ self.device.push(local=tmp.name, remote=self.DEVICE_TEMP_FILE)
- def test_push_dir(self):
- """Push a randomly generated directory of files to the device."""
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
+ self._verify_remote(compute_md5(rand_str), self.DEVICE_TEMP_FILE)
+ self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE])
- try:
- host_dir = tempfile.mkdtemp()
+ os.remove(tmp.name)
- # Make sure the temp directory isn't setuid, or else adb will complain.
- os.chmod(host_dir, 0o700)
-
- # Create 32 random files.
- temp_files = make_random_host_files(in_dir=host_dir, num_files=32)
- self.device.push(host_dir, self.DEVICE_TEMP_DIR)
-
- for temp_file in temp_files:
- remote_path = posixpath.join(self.DEVICE_TEMP_DIR,
- os.path.basename(host_dir),
- temp_file.base_name)
- self._verify_remote(temp_file.checksum, remote_path)
+ def test_push_dir(self):
+ """Push a randomly generated directory of files to the device."""
self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
- def disabled_test_push_empty(self):
- """Push an empty directory to the device."""
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
+ try:
+ host_dir = tempfile.mkdtemp()
- try:
- host_dir = tempfile.mkdtemp()
+ # Make sure the temp directory isn't setuid, or else adb will complain.
+ os.chmod(host_dir, 0o700)
- # Make sure the temp directory isn't setuid, or else adb will complain.
- os.chmod(host_dir, 0o700)
+ # Create 32 random files.
+ temp_files = make_random_host_files(in_dir=host_dir, num_files=32)
+ self.device.push(host_dir, self.DEVICE_TEMP_DIR)
- # Create an empty directory.
- empty_dir_path = os.path.join(host_dir, 'empty')
- os.mkdir(empty_dir_path);
+ for temp_file in temp_files:
+ remote_path = posixpath.join(self.DEVICE_TEMP_DIR,
+ os.path.basename(host_dir),
+ temp_file.base_name)
+ self._verify_remote(temp_file.checksum, remote_path)
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
- self.device.push(empty_dir_path, self.DEVICE_TEMP_DIR)
-
- remote_path = os.path.join(self.DEVICE_TEMP_DIR, "empty")
- test_empty_cmd = ["[", "-d", remote_path, "]"]
- rc, _, _ = self.device.shell_nocheck(test_empty_cmd)
-
- self.assertEqual(rc, 0)
+ def disabled_test_push_empty(self):
+ """Push an empty directory to the device."""
self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
- @unittest.skipIf(sys.platform == "win32", "symlinks require elevated privileges on windows")
- def test_push_symlink(self):
- """Push a symlink.
+ try:
+ host_dir = tempfile.mkdtemp()
- Bug: http://b/31491920
- """
- try:
- host_dir = tempfile.mkdtemp()
+ # Make sure the temp directory isn't setuid, or else adb will complain.
+ os.chmod(host_dir, 0o700)
- # Make sure the temp directory isn't setuid, or else adb will
- # complain.
- os.chmod(host_dir, 0o700)
+ # Create an empty directory.
+ empty_dir_path = os.path.join(host_dir, 'empty')
+ os.mkdir(empty_dir_path);
- with open(os.path.join(host_dir, 'foo'), 'w') as f:
- f.write('foo')
+ self.device.push(empty_dir_path, self.DEVICE_TEMP_DIR)
- symlink_path = os.path.join(host_dir, 'symlink')
- os.symlink('foo', symlink_path)
+ remote_path = os.path.join(self.DEVICE_TEMP_DIR, "empty")
+ test_empty_cmd = ["[", "-d", remote_path, "]"]
+ rc, _, _ = self.device.shell_nocheck(test_empty_cmd)
+
+ self.assertEqual(rc, 0)
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ @unittest.skipIf(sys.platform == "win32", "symlinks require elevated privileges on windows")
+ def test_push_symlink(self):
+ """Push a symlink.
+
+ Bug: http://b/31491920
+ """
+ try:
+ host_dir = tempfile.mkdtemp()
+
+ # Make sure the temp directory isn't setuid, or else adb will
+ # complain.
+ os.chmod(host_dir, 0o700)
+
+ with open(os.path.join(host_dir, 'foo'), 'w') as f:
+ f.write('foo')
+
+ symlink_path = os.path.join(host_dir, 'symlink')
+ os.symlink('foo', symlink_path)
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
+ self.device.push(symlink_path, self.DEVICE_TEMP_DIR)
+ rc, out, _ = self.device.shell_nocheck(
+ ['cat', posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')])
+ self.assertEqual(0, rc)
+ self.assertEqual(out.strip(), 'foo')
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ def test_multiple_push(self):
+ """Push multiple files to the device in one adb push command.
+
+ Bug: http://b/25324823
+ """
self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
- self.device.push(symlink_path, self.DEVICE_TEMP_DIR)
- rc, out, _ = self.device.shell_nocheck(
- ['cat', posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')])
- self.assertEqual(0, rc)
- self.assertEqual(out.strip(), 'foo')
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
- def test_multiple_push(self):
- """Push multiple files to the device in one adb push command.
-
- Bug: http://b/25324823
- """
-
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
-
- try:
- host_dir = tempfile.mkdtemp()
-
- # Create some random files and a subdirectory containing more files.
- temp_files = make_random_host_files(in_dir=host_dir, num_files=4)
-
- subdir = os.path.join(host_dir, 'subdir')
- os.mkdir(subdir)
- subdir_temp_files = make_random_host_files(in_dir=subdir,
- num_files=4)
-
- paths = [x.full_path for x in temp_files]
- paths.append(subdir)
- self.device._simple_call(['push'] + paths + [self.DEVICE_TEMP_DIR])
-
- for temp_file in temp_files:
- remote_path = posixpath.join(self.DEVICE_TEMP_DIR,
- temp_file.base_name)
- self._verify_remote(temp_file.checksum, remote_path)
-
- for subdir_temp_file in subdir_temp_files:
- remote_path = posixpath.join(self.DEVICE_TEMP_DIR,
- # BROKEN: http://b/25394682
- # 'subdir';
- temp_file.base_name)
- self._verify_remote(temp_file.checksum, remote_path)
-
-
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
-
- @requires_non_root
- def test_push_error_reporting(self):
- """Make sure that errors that occur while pushing a file get reported
-
- Bug: http://b/26816782
- """
- with tempfile.NamedTemporaryFile() as tmp_file:
- tmp_file.write(b'\0' * 1024 * 1024)
- tmp_file.flush()
try:
- self.device.push(local=tmp_file.name, remote='/system/')
- self.fail('push should not have succeeded')
+ host_dir = tempfile.mkdtemp()
+
+ # Create some random files and a subdirectory containing more files.
+ temp_files = make_random_host_files(in_dir=host_dir, num_files=4)
+
+ subdir = os.path.join(host_dir, 'subdir')
+ os.mkdir(subdir)
+ subdir_temp_files = make_random_host_files(in_dir=subdir,
+ num_files=4)
+
+ paths = [x.full_path for x in temp_files]
+ paths.append(subdir)
+ self.device._simple_call(['push'] + paths + [self.DEVICE_TEMP_DIR])
+
+ for temp_file in temp_files:
+ remote_path = posixpath.join(self.DEVICE_TEMP_DIR,
+ temp_file.base_name)
+ self._verify_remote(temp_file.checksum, remote_path)
+
+ for subdir_temp_file in subdir_temp_files:
+ remote_path = posixpath.join(self.DEVICE_TEMP_DIR,
+ # BROKEN: http://b/25394682
+ # 'subdir';
+ temp_file.base_name)
+ self._verify_remote(temp_file.checksum, remote_path)
+
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ @requires_non_root
+ def test_push_error_reporting(self):
+ """Make sure that errors that occur while pushing a file get reported
+
+ Bug: http://b/26816782
+ """
+ with tempfile.NamedTemporaryFile() as tmp_file:
+ tmp_file.write(b'\0' * 1024 * 1024)
+ tmp_file.flush()
+ try:
+ self.device.push(local=tmp_file.name, remote='/system/')
+ self.fail('push should not have succeeded')
+ except subprocess.CalledProcessError as e:
+ output = e.output
+
+ self.assertTrue(b'Permission denied' in output or
+ b'Read-only file system' in output)
+
+ @requires_non_root
+ def test_push_directory_creation(self):
+ """Regression test for directory creation.
+
+ Bug: http://b/110953234
+ """
+ with tempfile.NamedTemporaryFile() as tmp_file:
+ tmp_file.write(b'\0' * 1024 * 1024)
+ tmp_file.flush()
+ remote_path = self.DEVICE_TEMP_DIR + '/test_push_directory_creation'
+ self.device.shell(['rm', '-rf', remote_path])
+
+ remote_path += '/filename'
+ self.device.push(local=tmp_file.name, remote=remote_path)
+
+ def disabled_test_push_multiple_slash_root(self):
+ """Regression test for pushing to //data/local/tmp.
+
+ Bug: http://b/141311284
+
+ Disabled because this broken on the adbd side as well: b/141943968
+ """
+ with tempfile.NamedTemporaryFile() as tmp_file:
+ tmp_file.write('\0' * 1024 * 1024)
+ tmp_file.flush()
+ remote_path = '/' + self.DEVICE_TEMP_DIR + '/test_push_multiple_slash_root'
+ self.device.shell(['rm', '-rf', remote_path])
+ self.device.push(local=tmp_file.name, remote=remote_path)
+
+ def _test_pull(self, remote_file, checksum):
+ tmp_write = tempfile.NamedTemporaryFile(mode='wb', delete=False)
+ tmp_write.close()
+ self.device.pull(remote=remote_file, local=tmp_write.name)
+ with open(tmp_write.name, 'rb') as tmp_read:
+ host_contents = tmp_read.read()
+ host_md5 = compute_md5(host_contents)
+ self.assertEqual(checksum, host_md5)
+ os.remove(tmp_write.name)
+
+ @requires_non_root
+ def test_pull_error_reporting(self):
+ self.device.shell(['touch', self.DEVICE_TEMP_FILE])
+ self.device.shell(['chmod', 'a-rwx', self.DEVICE_TEMP_FILE])
+
+ try:
+ output = self.device.pull(remote=self.DEVICE_TEMP_FILE, local='x')
except subprocess.CalledProcessError as e:
output = e.output
- self.assertTrue(b'Permission denied' in output or
- b'Read-only file system' in output)
+ self.assertIn(b'Permission denied', output)
- @requires_non_root
- def test_push_directory_creation(self):
- """Regression test for directory creation.
+ self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE])
- Bug: http://b/110953234
- """
- with tempfile.NamedTemporaryFile() as tmp_file:
- tmp_file.write(b'\0' * 1024 * 1024)
- tmp_file.flush()
- remote_path = self.DEVICE_TEMP_DIR + '/test_push_directory_creation'
- self.device.shell(['rm', '-rf', remote_path])
+ def test_pull(self):
+ """Pull a randomly generated file from specified device."""
+ kbytes = 512
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE])
+ cmd = ['dd', 'if=/dev/urandom',
+ 'of={}'.format(self.DEVICE_TEMP_FILE), 'bs=1024',
+ 'count={}'.format(kbytes)]
+ self.device.shell(cmd)
+ dev_md5, _ = self.device.shell(
+ [get_md5_prog(self.device), self.DEVICE_TEMP_FILE])[0].split()
+ self._test_pull(self.DEVICE_TEMP_FILE, dev_md5)
+ self.device.shell_nocheck(['rm', self.DEVICE_TEMP_FILE])
- remote_path += '/filename'
- self.device.push(local=tmp_file.name, remote=remote_path)
+ def test_pull_dir(self):
+ """Pull a randomly generated directory of files from the device."""
+ try:
+ host_dir = tempfile.mkdtemp()
- def disabled_test_push_multiple_slash_root(self):
- """Regression test for pushing to //data/local/tmp.
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
- Bug: http://b/141311284
+ # Populate device directory with random files.
+ temp_files = make_random_device_files(
+ self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
- Disabled because this broken on the adbd side as well: b/141943968
- """
- with tempfile.NamedTemporaryFile() as tmp_file:
- tmp_file.write('\0' * 1024 * 1024)
- tmp_file.flush()
- remote_path = '/' + self.DEVICE_TEMP_DIR + '/test_push_multiple_slash_root'
- self.device.shell(['rm', '-rf', remote_path])
- self.device.push(local=tmp_file.name, remote=remote_path)
+ self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir)
- def _test_pull(self, remote_file, checksum):
- tmp_write = tempfile.NamedTemporaryFile(mode='wb', delete=False)
- tmp_write.close()
- self.device.pull(remote=remote_file, local=tmp_write.name)
- with open(tmp_write.name, 'rb') as tmp_read:
- host_contents = tmp_read.read()
- host_md5 = compute_md5(host_contents)
- self.assertEqual(checksum, host_md5)
- os.remove(tmp_write.name)
+ for temp_file in temp_files:
+ host_path = os.path.join(
+ host_dir, posixpath.basename(self.DEVICE_TEMP_DIR),
+ temp_file.base_name)
+ self._verify_local(temp_file.checksum, host_path)
- @requires_non_root
- def test_pull_error_reporting(self):
- self.device.shell(['touch', self.DEVICE_TEMP_FILE])
- self.device.shell(['chmod', 'a-rwx', self.DEVICE_TEMP_FILE])
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
- try:
- output = self.device.pull(remote=self.DEVICE_TEMP_FILE, local='x')
- except subprocess.CalledProcessError as e:
- output = e.output
+ def test_pull_dir_symlink(self):
+ """Pull a directory into a symlink to a directory.
- self.assertIn(b'Permission denied', output)
+ Bug: http://b/27362811
+ """
+ if os.name != 'posix':
+ raise unittest.SkipTest('requires POSIX')
- self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE])
+ try:
+ host_dir = tempfile.mkdtemp()
+ real_dir = os.path.join(host_dir, 'dir')
+ symlink = os.path.join(host_dir, 'symlink')
+ os.mkdir(real_dir)
+ os.symlink(real_dir, symlink)
- def test_pull(self):
- """Pull a randomly generated file from specified device."""
- kbytes = 512
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE])
- cmd = ['dd', 'if=/dev/urandom',
- 'of={}'.format(self.DEVICE_TEMP_FILE), 'bs=1024',
- 'count={}'.format(kbytes)]
- self.device.shell(cmd)
- dev_md5, _ = self.device.shell(
- [get_md5_prog(self.device), self.DEVICE_TEMP_FILE])[0].split()
- self._test_pull(self.DEVICE_TEMP_FILE, dev_md5)
- self.device.shell_nocheck(['rm', self.DEVICE_TEMP_FILE])
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
- def test_pull_dir(self):
- """Pull a randomly generated directory of files from the device."""
- try:
- host_dir = tempfile.mkdtemp()
+ # Populate device directory with random files.
+ temp_files = make_random_device_files(
+ self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
+ self.device.pull(remote=self.DEVICE_TEMP_DIR, local=symlink)
- # Populate device directory with random files.
- temp_files = make_random_device_files(
- self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
+ for temp_file in temp_files:
+ host_path = os.path.join(
+ real_dir, posixpath.basename(self.DEVICE_TEMP_DIR),
+ temp_file.base_name)
+ self._verify_local(temp_file.checksum, host_path)
- self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir)
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+ def test_pull_dir_symlink_collision(self):
+ """Pull a directory into a colliding symlink to directory."""
+ if os.name != 'posix':
+ raise unittest.SkipTest('requires POSIX')
+
+ try:
+ host_dir = tempfile.mkdtemp()
+ real_dir = os.path.join(host_dir, 'real')
+ tmp_dirname = os.path.basename(self.DEVICE_TEMP_DIR)
+ symlink = os.path.join(host_dir, tmp_dirname)
+ os.mkdir(real_dir)
+ os.symlink(real_dir, symlink)
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
+
+ # Populate device directory with random files.
+ temp_files = make_random_device_files(
+ self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
+
+ self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir)
+
+ for temp_file in temp_files:
+ host_path = os.path.join(real_dir, temp_file.base_name)
+ self._verify_local(temp_file.checksum, host_path)
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ def test_pull_dir_nonexistent(self):
+ """Pull a directory of files from the device to a nonexistent path."""
+ try:
+ host_dir = tempfile.mkdtemp()
+ dest_dir = os.path.join(host_dir, 'dest')
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
+
+ # Populate device directory with random files.
+ temp_files = make_random_device_files(
+ self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
+
+ self.device.pull(remote=self.DEVICE_TEMP_DIR, local=dest_dir)
+
+ for temp_file in temp_files:
+ host_path = os.path.join(dest_dir, temp_file.base_name)
+ self._verify_local(temp_file.checksum, host_path)
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ # selinux prevents adbd from accessing symlinks on /data/local/tmp.
+ def disabled_test_pull_symlink_dir(self):
+ """Pull a symlink to a directory of symlinks to files."""
+ try:
+ host_dir = tempfile.mkdtemp()
+
+ remote_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'contents')
+ remote_links = posixpath.join(self.DEVICE_TEMP_DIR, 'links')
+ remote_symlink = posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', remote_dir, remote_links])
+ self.device.shell(['ln', '-s', remote_links, remote_symlink])
+
+ # Populate device directory with random files.
+ temp_files = make_random_device_files(
+ self.device, in_dir=remote_dir, num_files=32)
+
+ for temp_file in temp_files:
+ self.device.shell(
+ ['ln', '-s', '../contents/{}'.format(temp_file.base_name),
+ posixpath.join(remote_links, temp_file.base_name)])
+
+ self.device.pull(remote=remote_symlink, local=host_dir)
+
+ for temp_file in temp_files:
+ host_path = os.path.join(
+ host_dir, 'symlink', temp_file.base_name)
+ self._verify_local(temp_file.checksum, host_path)
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ def test_pull_empty(self):
+ """Pull a directory containing an empty directory from the device."""
+ try:
+ host_dir = tempfile.mkdtemp()
+
+ remote_empty_path = posixpath.join(self.DEVICE_TEMP_DIR, 'empty')
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', remote_empty_path])
+
+ self.device.pull(remote=remote_empty_path, local=host_dir)
+ self.assertTrue(os.path.isdir(os.path.join(host_dir, 'empty')))
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ def test_multiple_pull(self):
+ """Pull a randomly generated directory of files from the device."""
+
+ try:
+ host_dir = tempfile.mkdtemp()
+
+ subdir = posixpath.join(self.DEVICE_TEMP_DIR, 'subdir')
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', subdir])
+
+ # Create some random files and a subdirectory containing more files.
+ temp_files = make_random_device_files(
+ self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=4)
+
+ subdir_temp_files = make_random_device_files(
+ self.device, in_dir=subdir, num_files=4, prefix='subdir_')
+
+ paths = [x.full_path for x in temp_files]
+ paths.append(subdir)
+ self.device._simple_call(['pull'] + paths + [host_dir])
+
+ for temp_file in temp_files:
+ local_path = os.path.join(host_dir, temp_file.base_name)
+ self._verify_local(temp_file.checksum, local_path)
+
+ for subdir_temp_file in subdir_temp_files:
+ local_path = os.path.join(host_dir,
+ 'subdir',
+ subdir_temp_file.base_name)
+ self._verify_local(subdir_temp_file.checksum, local_path)
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ def verify_sync(self, device, temp_files, device_dir):
+ """Verifies that a list of temp files was synced to the device."""
+ # Confirm that every file on the device mirrors that on the host.
for temp_file in temp_files:
- host_path = os.path.join(
- host_dir, posixpath.basename(self.DEVICE_TEMP_DIR),
- temp_file.base_name)
- self._verify_local(temp_file.checksum, host_path)
+ device_full_path = posixpath.join(
+ device_dir, temp_file.base_name)
+ dev_md5, _ = device.shell(
+ [get_md5_prog(self.device), device_full_path])[0].split()
+ self.assertEqual(temp_file.checksum, dev_md5)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ def test_sync(self):
+ """Sync a host directory to the data partition."""
- def test_pull_dir_symlink(self):
- """Pull a directory into a symlink to a directory.
+ try:
+ base_dir = tempfile.mkdtemp()
- Bug: http://b/27362811
- """
- if os.name != 'posix':
- raise unittest.SkipTest('requires POSIX')
+ # Create mirror device directory hierarchy within base_dir.
+ full_dir_path = base_dir + self.DEVICE_TEMP_DIR
+ os.makedirs(full_dir_path)
- try:
- host_dir = tempfile.mkdtemp()
- real_dir = os.path.join(host_dir, 'dir')
- symlink = os.path.join(host_dir, 'symlink')
- os.mkdir(real_dir)
- os.symlink(real_dir, symlink)
+ # Create 32 random files within the host mirror.
+ temp_files = make_random_host_files(
+ in_dir=full_dir_path, num_files=32)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
+ # Clean up any stale files on the device.
+ device = adb.get_device() # pylint: disable=no-member
+ device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- # Populate device directory with random files.
- temp_files = make_random_device_files(
- self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
+ old_product_out = os.environ.get('ANDROID_PRODUCT_OUT')
+ os.environ['ANDROID_PRODUCT_OUT'] = base_dir
+ device.sync('data')
+ if old_product_out is None:
+ del os.environ['ANDROID_PRODUCT_OUT']
+ else:
+ os.environ['ANDROID_PRODUCT_OUT'] = old_product_out
- self.device.pull(remote=self.DEVICE_TEMP_DIR, local=symlink)
+ self.verify_sync(device, temp_files, self.DEVICE_TEMP_DIR)
- for temp_file in temp_files:
- host_path = os.path.join(
- real_dir, posixpath.basename(self.DEVICE_TEMP_DIR),
- temp_file.base_name)
- self._verify_local(temp_file.checksum, host_path)
+ #self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if base_dir is not None:
+ shutil.rmtree(base_dir)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ def test_push_sync(self):
+ """Sync a host directory to a specific path."""
- def test_pull_dir_symlink_collision(self):
- """Pull a directory into a colliding symlink to directory."""
- if os.name != 'posix':
- raise unittest.SkipTest('requires POSIX')
+ try:
+ temp_dir = tempfile.mkdtemp()
+ temp_files = make_random_host_files(in_dir=temp_dir, num_files=32)
- try:
- host_dir = tempfile.mkdtemp()
- real_dir = os.path.join(host_dir, 'real')
- tmp_dirname = os.path.basename(self.DEVICE_TEMP_DIR)
- symlink = os.path.join(host_dir, tmp_dirname)
- os.mkdir(real_dir)
- os.symlink(real_dir, symlink)
+ device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'sync_src_dst')
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
+ # Clean up any stale files on the device.
+ device = adb.get_device() # pylint: disable=no-member
+ device.shell(['rm', '-rf', device_dir])
- # Populate device directory with random files.
- temp_files = make_random_device_files(
- self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
+ device.push(temp_dir, device_dir, sync=True)
- self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir)
+ self.verify_sync(device, temp_files, device_dir)
- for temp_file in temp_files:
- host_path = os.path.join(real_dir, temp_file.base_name)
- self._verify_local(temp_file.checksum, host_path)
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if temp_dir is not None:
+ shutil.rmtree(temp_dir)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ def test_push_dry_run_nonexistent_file(self):
+ """Push with dry run."""
- def test_pull_dir_nonexistent(self):
- """Pull a directory of files from the device to a nonexistent path."""
- try:
- host_dir = tempfile.mkdtemp()
- dest_dir = os.path.join(host_dir, 'dest')
+ for file_size in [8, 1024 * 1024]:
+ try:
+ device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'push_dry_run')
+ device_file = posixpath.join(device_dir, 'file')
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', device_dir])
- # Populate device directory with random files.
- temp_files = make_random_device_files(
- self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
+ host_dir = tempfile.mkdtemp()
+ host_file = posixpath.join(host_dir, 'file')
- self.device.pull(remote=self.DEVICE_TEMP_DIR, local=dest_dir)
+ with open(host_file, "w") as f:
+ f.write('x' * file_size)
- for temp_file in temp_files:
- host_path = os.path.join(dest_dir, temp_file.base_name)
- self._verify_local(temp_file.checksum, host_path)
+ self.device._simple_call(['push', '-n', host_file, device_file])
+ rc, _, _ = self.device.shell_nocheck(['[', '-e', device_file, ']'])
+ self.assertNotEqual(0, rc)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
- # selinux prevents adbd from accessing symlinks on /data/local/tmp.
- def disabled_test_pull_symlink_dir(self):
- """Pull a symlink to a directory of symlinks to files."""
- try:
- host_dir = tempfile.mkdtemp()
+ def test_push_dry_run_existent_file(self):
+ """Push with dry run."""
- remote_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'contents')
- remote_links = posixpath.join(self.DEVICE_TEMP_DIR, 'links')
- remote_symlink = posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')
+ for file_size in [8, 1024 * 1024]:
+ try:
+ device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'push_dry_run')
+ device_file = posixpath.join(device_dir, 'file')
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', remote_dir, remote_links])
- self.device.shell(['ln', '-s', remote_links, remote_symlink])
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', device_dir])
+ self.device.shell(['echo', 'foo', '>', device_file])
- # Populate device directory with random files.
- temp_files = make_random_device_files(
- self.device, in_dir=remote_dir, num_files=32)
+ host_dir = tempfile.mkdtemp()
+ host_file = posixpath.join(host_dir, 'file')
- for temp_file in temp_files:
- self.device.shell(
- ['ln', '-s', '../contents/{}'.format(temp_file.base_name),
- posixpath.join(remote_links, temp_file.base_name)])
+ with open(host_file, "w") as f:
+ f.write('x' * file_size)
- self.device.pull(remote=remote_symlink, local=host_dir)
+ self.device._simple_call(['push', '-n', host_file, device_file])
+ stdout, stderr = self.device.shell(['cat', device_file])
+ self.assertEqual(stdout.strip(), "foo")
- for temp_file in temp_files:
- host_path = os.path.join(
- host_dir, 'symlink', temp_file.base_name)
- self._verify_local(temp_file.checksum, host_path)
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ def test_unicode_paths(self):
+ """Ensure that we can support non-ASCII paths, even on Windows."""
+ name = u'로보카 폴리'
- def test_pull_empty(self):
- """Pull a directory containing an empty directory from the device."""
- try:
- host_dir = tempfile.mkdtemp()
+ self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*'])
+ remote_path = u'/data/local/tmp/adb-test-{}'.format(name)
- remote_empty_path = posixpath.join(self.DEVICE_TEMP_DIR, 'empty')
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', remote_empty_path])
+ ## push.
+ tf = tempfile.NamedTemporaryFile('wb', suffix=name, delete=False)
+ tf.close()
+ self.device.push(tf.name, remote_path)
+ os.remove(tf.name)
+ self.assertFalse(os.path.exists(tf.name))
- self.device.pull(remote=remote_empty_path, local=host_dir)
- self.assertTrue(os.path.isdir(os.path.join(host_dir, 'empty')))
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ # Verify that the device ended up with the expected UTF-8 path
+ output = self.device.shell(
+ ['ls', '/data/local/tmp/adb-test-*'])[0].strip()
+ self.assertEqual(remote_path, output)
- def test_multiple_pull(self):
- """Pull a randomly generated directory of files from the device."""
+ # pull.
+ self.device.pull(remote_path, tf.name)
+ self.assertTrue(os.path.exists(tf.name))
+ os.remove(tf.name)
+ self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*'])
- try:
- host_dir = tempfile.mkdtemp()
- subdir = posixpath.join(self.DEVICE_TEMP_DIR, 'subdir')
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', subdir])
+class FileOperationsTestUncompressed(FileOperationsTest.Base):
+ compression = "none"
- # Create some random files and a subdirectory containing more files.
- temp_files = make_random_device_files(
- self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=4)
- subdir_temp_files = make_random_device_files(
- self.device, in_dir=subdir, num_files=4, prefix='subdir_')
+class FileOperationsTestBrotli(FileOperationsTest.Base):
+ compression = "brotli"
- paths = [x.full_path for x in temp_files]
- paths.append(subdir)
- self.device._simple_call(['pull'] + paths + [host_dir])
- for temp_file in temp_files:
- local_path = os.path.join(host_dir, temp_file.base_name)
- self._verify_local(temp_file.checksum, local_path)
-
- for subdir_temp_file in subdir_temp_files:
- local_path = os.path.join(host_dir,
- 'subdir',
- subdir_temp_file.base_name)
- self._verify_local(subdir_temp_file.checksum, local_path)
-
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
-
- def verify_sync(self, device, temp_files, device_dir):
- """Verifies that a list of temp files was synced to the device."""
- # Confirm that every file on the device mirrors that on the host.
- for temp_file in temp_files:
- device_full_path = posixpath.join(
- device_dir, temp_file.base_name)
- dev_md5, _ = device.shell(
- [get_md5_prog(self.device), device_full_path])[0].split()
- self.assertEqual(temp_file.checksum, dev_md5)
-
- def test_sync(self):
- """Sync a host directory to the data partition."""
-
- try:
- base_dir = tempfile.mkdtemp()
-
- # Create mirror device directory hierarchy within base_dir.
- full_dir_path = base_dir + self.DEVICE_TEMP_DIR
- os.makedirs(full_dir_path)
-
- # Create 32 random files within the host mirror.
- temp_files = make_random_host_files(
- in_dir=full_dir_path, num_files=32)
-
- # Clean up any stale files on the device.
- device = adb.get_device() # pylint: disable=no-member
- device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
-
- old_product_out = os.environ.get('ANDROID_PRODUCT_OUT')
- os.environ['ANDROID_PRODUCT_OUT'] = base_dir
- device.sync('data')
- if old_product_out is None:
- del os.environ['ANDROID_PRODUCT_OUT']
- else:
- os.environ['ANDROID_PRODUCT_OUT'] = old_product_out
-
- self.verify_sync(device, temp_files, self.DEVICE_TEMP_DIR)
-
- #self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if base_dir is not None:
- shutil.rmtree(base_dir)
-
- def test_push_sync(self):
- """Sync a host directory to a specific path."""
-
- try:
- temp_dir = tempfile.mkdtemp()
- temp_files = make_random_host_files(in_dir=temp_dir, num_files=32)
-
- device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'sync_src_dst')
-
- # Clean up any stale files on the device.
- device = adb.get_device() # pylint: disable=no-member
- device.shell(['rm', '-rf', device_dir])
-
- device.push(temp_dir, device_dir, sync=True)
-
- self.verify_sync(device, temp_files, device_dir)
-
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if temp_dir is not None:
- shutil.rmtree(temp_dir)
-
- def test_unicode_paths(self):
- """Ensure that we can support non-ASCII paths, even on Windows."""
- name = u'로보카 폴리'
-
- self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*'])
- remote_path = u'/data/local/tmp/adb-test-{}'.format(name)
-
- ## push.
- tf = tempfile.NamedTemporaryFile('wb', suffix=name, delete=False)
- tf.close()
- self.device.push(tf.name, remote_path)
- os.remove(tf.name)
- self.assertFalse(os.path.exists(tf.name))
-
- # Verify that the device ended up with the expected UTF-8 path
- output = self.device.shell(
- ['ls', '/data/local/tmp/adb-test-*'])[0].strip()
- self.assertEqual(remote_path, output)
-
- # pull.
- self.device.pull(remote_path, tf.name)
- self.assertTrue(os.path.exists(tf.name))
- os.remove(tf.name)
- self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*'])
+class FileOperationsTestLZ4(FileOperationsTest.Base):
+ compression = "lz4"
class DeviceOfflineTest(DeviceTest):
diff --git a/adb/transport.cpp b/adb/transport.cpp
index e06dbe3..963c3c1 100644
--- a/adb/transport.cpp
+++ b/adb/transport.cpp
@@ -29,7 +29,6 @@
#include <unistd.h>
#include <algorithm>
-#include <deque>
#include <list>
#include <memory>
#include <mutex>
@@ -40,6 +39,7 @@
#include <adb/crypto/x509_generator.h>
#include <adb/tls/tls_connection.h>
#include <android-base/logging.h>
+#include <android-base/no_destructor.h>
#include <android-base/parsenetaddress.h>
#include <android-base/stringprintf.h>
#include <android-base/strings.h>
@@ -84,6 +84,8 @@
const char* const kFeatureTrackApp = "track_app";
const char* const kFeatureSendRecv2 = "sendrecv_v2";
const char* const kFeatureSendRecv2Brotli = "sendrecv_v2_brotli";
+const char* const kFeatureSendRecv2LZ4 = "sendrecv_v2_lz4";
+const char* const kFeatureSendRecv2DryRunSend = "sendrecv_v2_dry_run_send";
namespace {
@@ -1168,26 +1170,29 @@
}
const FeatureSet& supported_features() {
- // Local static allocation to avoid global non-POD variables.
- static const FeatureSet* features = new FeatureSet{
- kFeatureShell2,
- kFeatureCmd,
- kFeatureStat2,
- kFeatureLs2,
- kFeatureFixedPushMkdir,
- kFeatureApex,
- kFeatureAbb,
- kFeatureFixedPushSymlinkTimestamp,
- kFeatureAbbExec,
- kFeatureRemountShell,
- kFeatureTrackApp,
- kFeatureSendRecv2,
- kFeatureSendRecv2Brotli,
- // Increment ADB_SERVER_VERSION when adding a feature that adbd needs
- // to know about. Otherwise, the client can be stuck running an old
- // version of the server even after upgrading their copy of adb.
- // (http://b/24370690)
- };
+ static const android::base::NoDestructor<FeatureSet> features([] {
+ return FeatureSet{
+ kFeatureShell2,
+ kFeatureCmd,
+ kFeatureStat2,
+ kFeatureLs2,
+ kFeatureFixedPushMkdir,
+ kFeatureApex,
+ kFeatureAbb,
+ kFeatureFixedPushSymlinkTimestamp,
+ kFeatureAbbExec,
+ kFeatureRemountShell,
+ kFeatureTrackApp,
+ kFeatureSendRecv2,
+ kFeatureSendRecv2Brotli,
+ kFeatureSendRecv2LZ4,
+ kFeatureSendRecv2DryRunSend,
+ // Increment ADB_SERVER_VERSION when adding a feature that adbd needs
+ // to know about. Otherwise, the client can be stuck running an old
+ // version of the server even after upgrading their copy of adb.
+ // (http://b/24370690)
+ };
+ }());
return *features;
}
@@ -1201,16 +1206,20 @@
return FeatureSet();
}
- auto names = android::base::Split(features_string, ",");
- return FeatureSet(names.begin(), names.end());
+ return android::base::Split(features_string, ",");
+}
+
+template <class Range, class Value>
+static bool contains(const Range& r, const Value& v) {
+ return std::find(std::begin(r), std::end(r), v) != std::end(r);
}
bool CanUseFeature(const FeatureSet& feature_set, const std::string& feature) {
- return feature_set.count(feature) > 0 && supported_features().count(feature) > 0;
+ return contains(feature_set, feature) && contains(supported_features(), feature);
}
bool atransport::has_feature(const std::string& feature) const {
- return features_.count(feature) > 0;
+ return contains(features_, feature);
}
void atransport::SetFeatures(const std::string& features_string) {
diff --git a/adb/transport.h b/adb/transport.h
index b1984db..e93c31c 100644
--- a/adb/transport.h
+++ b/adb/transport.h
@@ -30,7 +30,7 @@
#include <string>
#include <string_view>
#include <thread>
-#include <unordered_set>
+#include <vector>
#include <android-base/macros.h>
#include <android-base/thread_annotations.h>
@@ -40,7 +40,10 @@
#include "adb_unique_fd.h"
#include "types.h"
-typedef std::unordered_set<std::string> FeatureSet;
+// Even though the feature set is used as a set, we only have a dozen or two
+// of available features at any moment. Vector works much better in terms of
+// both memory usage and performance for these sizes.
+using FeatureSet = std::vector<std::string>;
namespace adb {
namespace tls {
@@ -88,6 +91,10 @@
extern const char* const kFeatureSendRecv2;
// adbd supports brotli for send/recv v2.
extern const char* const kFeatureSendRecv2Brotli;
+// adbd supports LZ4 for send/recv v2.
+extern const char* const kFeatureSendRecv2LZ4;
+// adbd supports dry-run send for send/recv v2.
+extern const char* const kFeatureSendRecv2DryRunSend;
TransportId NextTransportId();
diff --git a/adb/transport_test.cpp b/adb/transport_test.cpp
index 00beb3a..a9ada4a 100644
--- a/adb/transport_test.cpp
+++ b/adb/transport_test.cpp
@@ -66,7 +66,7 @@
ASSERT_TRUE(t.has_feature("bar"));
t.SetFeatures(FeatureSetToString(FeatureSet{"foo", "bar", "foo"}));
- ASSERT_EQ(2U, t.features().size());
+ ASSERT_LE(2U, t.features().size());
ASSERT_TRUE(t.has_feature("foo"));
ASSERT_TRUE(t.has_feature("bar"));
diff --git a/adb/types.cpp b/adb/types.cpp
index 26b77ab..9cdf32b 100644
--- a/adb/types.cpp
+++ b/adb/types.cpp
@@ -51,7 +51,7 @@
auto dropped = 0u;
while (dropped < len) {
const auto next = chain_[start_index_].size() - begin_offset_;
- if (dropped + next < len) {
+ if (dropped + next <= len) {
pop_front_block();
dropped += next;
} else {
diff --git a/adb/types.h b/adb/types.h
index deca7ea..620aa8e 100644
--- a/adb/types.h
+++ b/adb/types.h
@@ -155,7 +155,7 @@
return nullptr;
}
- return chain_.front().data() + begin_offset_;
+ return chain_[start_index_].data() + begin_offset_;
}
size_type front_size() const {
@@ -163,7 +163,7 @@
return 0;
}
- return chain_.front().size() - begin_offset_;
+ return chain_[start_index_].size() - begin_offset_;
}
size_type size() const { return chain_length_ - begin_offset_; }
diff --git a/adb/types_test.cpp b/adb/types_test.cpp
index 2c99f95..41fa1db 100644
--- a/adb/types_test.cpp
+++ b/adb/types_test.cpp
@@ -117,3 +117,20 @@
ASSERT_EQ(1ULL, bc.size());
ASSERT_EQ(create_block("x"), bc.coalesce());
}
+
+TEST(IOVector, drop_front) {
+ IOVector vec;
+
+ vec.append(create_block('x', 2));
+ vec.append(create_block('y', 1000));
+ ASSERT_EQ(2U, vec.front_size());
+ ASSERT_EQ(1002U, vec.size());
+
+ vec.drop_front(1);
+ ASSERT_EQ(1U, vec.front_size());
+ ASSERT_EQ(1001U, vec.size());
+
+ vec.drop_front(1);
+ ASSERT_EQ(1000U, vec.front_size());
+ ASSERT_EQ(1000U, vec.size());
+}
diff --git a/base/properties.cpp b/base/properties.cpp
index 35e41a8..5c9ec7e 100644
--- a/base/properties.cpp
+++ b/base/properties.cpp
@@ -133,7 +133,6 @@
ts.tv_nsec = ns.count();
}
-// TODO: boot_clock?
using AbsTime = std::chrono::time_point<std::chrono::steady_clock>;
static void UpdateTimeSpec(timespec& ts, std::chrono::milliseconds relative_timeout,
diff --git a/debuggerd/handler/debuggerd_handler.cpp b/debuggerd/handler/debuggerd_handler.cpp
index ac28fe9..121a074 100644
--- a/debuggerd/handler/debuggerd_handler.cpp
+++ b/debuggerd/handler/debuggerd_handler.cpp
@@ -417,23 +417,28 @@
// us to fork off a process to read memory from.
char buf[4];
rc = TEMP_FAILURE_RETRY(read(input_read.get(), &buf, sizeof(buf)));
- if (rc == -1) {
- async_safe_format_log(ANDROID_LOG_FATAL, "libc", "read of IPC pipe failed: %s", strerror(errno));
- return 1;
- } else if (rc == 0) {
- async_safe_format_log(ANDROID_LOG_FATAL, "libc", "crash_dump helper failed to exec");
- return 1;
- } else if (rc != 1) {
- async_safe_format_log(ANDROID_LOG_FATAL, "libc",
- "read of IPC pipe returned unexpected value: %zd", rc);
- return 1;
- } else if (buf[0] != '\1') {
- async_safe_format_log(ANDROID_LOG_FATAL, "libc", "crash_dump helper reported failure");
- return 1;
- }
- // crash_dump is ptracing us, fork off a copy of our address space for it to use.
- create_vm_process();
+ bool success = false;
+ if (rc == 1 && buf[0] == '\1') {
+ // crash_dump successfully started, and is ptracing us.
+ // Fork off a copy of our address space for it to use.
+ create_vm_process();
+ success = true;
+ } else {
+ // Something went wrong, log it.
+ if (rc == -1) {
+ async_safe_format_log(ANDROID_LOG_FATAL, "libc", "read of IPC pipe failed: %s",
+ strerror(errno));
+ } else if (rc == 0) {
+ async_safe_format_log(ANDROID_LOG_FATAL, "libc",
+ "crash_dump helper failed to exec, or was killed");
+ } else if (rc != 1) {
+ async_safe_format_log(ANDROID_LOG_FATAL, "libc",
+ "read of IPC pipe returned unexpected value: %zd", rc);
+ } else if (buf[0] != '\1') {
+ async_safe_format_log(ANDROID_LOG_FATAL, "libc", "crash_dump helper reported failure");
+ }
+ }
// Don't leave a zombie child.
int status;
@@ -444,14 +449,16 @@
async_safe_format_log(ANDROID_LOG_FATAL, "libc", "crash_dump helper crashed or stopped");
}
- if (thread_info->siginfo->si_signo != BIONIC_SIGNAL_DEBUGGER) {
- // For crashes, we don't need to minimize pause latency.
- // Wait for the dump to complete before having the process exit, to avoid being murdered by
- // ActivityManager or init.
- TEMP_FAILURE_RETRY(read(input_read, &buf, sizeof(buf)));
+ if (success) {
+ if (thread_info->siginfo->si_signo != BIONIC_SIGNAL_DEBUGGER) {
+ // For crashes, we don't need to minimize pause latency.
+ // Wait for the dump to complete before having the process exit, to avoid being murdered by
+ // ActivityManager or init.
+ TEMP_FAILURE_RETRY(read(input_read, &buf, sizeof(buf)));
+ }
}
- return 0;
+ return success ? 0 : 1;
}
static void resend_signal(siginfo_t* info) {
diff --git a/fastboot/fuzzy_fastboot/Android.bp b/fastboot/fuzzy_fastboot/Android.bp
index bb54fd9..aa449b2 100644
--- a/fastboot/fuzzy_fastboot/Android.bp
+++ b/fastboot/fuzzy_fastboot/Android.bp
@@ -49,6 +49,6 @@
auto_gen_config: false,
test_suites: [
"general-tests",
- "vts-core",
+ "vts",
],
}
diff --git a/fs_mgr/fs_mgr.cpp b/fs_mgr/fs_mgr.cpp
index 5475cae..1486e87 100644
--- a/fs_mgr/fs_mgr.cpp
+++ b/fs_mgr/fs_mgr.cpp
@@ -97,7 +97,6 @@
using android::base::Basename;
using android::base::GetBoolProperty;
using android::base::GetUintProperty;
-using android::base::Readlink;
using android::base::Realpath;
using android::base::SetProperty;
using android::base::StartsWith;
@@ -1552,8 +1551,8 @@
return std::chrono::milliseconds(std::move(value));
}
-static bool fs_mgr_unmount_all_data_mounts(const std::string& block_device) {
- LINFO << __FUNCTION__ << "(): about to umount everything on top of " << block_device;
+static bool fs_mgr_unmount_all_data_mounts(const std::string& data_block_device) {
+ LINFO << __FUNCTION__ << "(): about to umount everything on top of " << data_block_device;
Timer t;
auto timeout = GetMillisProperty("init.userspace_reboot.userdata_remount.timeoutmillis", 5s);
while (true) {
@@ -1565,7 +1564,13 @@
}
// Now proceed with other bind mounts on top of /data.
for (const auto& entry : proc_mounts) {
- if (entry.blk_device == block_device) {
+ std::string block_device;
+ if (StartsWith(entry.blk_device, "/dev/block") &&
+ !Realpath(entry.blk_device, &block_device)) {
+ PWARNING << __FUNCTION__ << "(): failed to realpath " << entry.blk_device;
+ block_device = entry.blk_device;
+ }
+ if (data_block_device == block_device) {
if (umount2(entry.mount_point.c_str(), 0) != 0) {
PERROR << __FUNCTION__ << "(): Failed to umount " << entry.mount_point;
umount_done = false;
@@ -1577,7 +1582,8 @@
return true;
}
if (t.duration() > timeout) {
- LERROR << __FUNCTION__ << "(): Timed out unmounting all mounts on " << block_device;
+ LERROR << __FUNCTION__ << "(): Timed out unmounting all mounts on "
+ << data_block_device;
Fstab remaining_mounts;
if (!ReadFstabFromFile("/proc/mounts", &remaining_mounts)) {
LERROR << __FUNCTION__ << "(): Can't read /proc/mounts";
@@ -1616,14 +1622,11 @@
return true;
}
-FstabEntry* fs_mgr_get_mounted_entry_for_userdata(Fstab* fstab, const FstabEntry& mounted_entry) {
- if (mounted_entry.mount_point != "/data") {
- LERROR << mounted_entry.mount_point << " is not /data";
- return nullptr;
- }
+FstabEntry* fs_mgr_get_mounted_entry_for_userdata(Fstab* fstab,
+ const std::string& data_block_device) {
std::vector<std::string> dm_stack;
- if (!UnwindDmDeviceStack(mounted_entry.blk_device, &dm_stack)) {
- LERROR << "Failed to unwind dm-device stack for " << mounted_entry.blk_device;
+ if (!UnwindDmDeviceStack(data_block_device, &dm_stack)) {
+ LERROR << "Failed to unwind dm-device stack for " << data_block_device;
return nullptr;
}
for (auto& entry : *fstab) {
@@ -1637,15 +1640,15 @@
continue;
}
block_device = entry.blk_device;
- } else if (!Readlink(entry.blk_device, &block_device)) {
- PWARNING << "Failed to read link " << entry.blk_device;
+ } else if (!Realpath(entry.blk_device, &block_device)) {
+ PWARNING << "Failed to realpath " << entry.blk_device;
block_device = entry.blk_device;
}
if (std::find(dm_stack.begin(), dm_stack.end(), block_device) != dm_stack.end()) {
return &entry;
}
}
- LERROR << "Didn't find entry that was used to mount /data onto " << mounted_entry.blk_device;
+ LERROR << "Didn't find entry that was used to mount /data onto " << data_block_device;
return nullptr;
}
@@ -1656,14 +1659,17 @@
LERROR << "Can't read /proc/mounts";
return -1;
}
- std::string block_device;
auto mounted_entry = GetEntryForMountPoint(&proc_mounts, "/data");
if (mounted_entry == nullptr) {
LERROR << "/data is not mounted";
return -1;
}
- block_device = mounted_entry->blk_device;
- auto fstab_entry = fs_mgr_get_mounted_entry_for_userdata(fstab, *mounted_entry);
+ std::string block_device;
+ if (!Realpath(mounted_entry->blk_device, &block_device)) {
+ PERROR << "Failed to realpath " << mounted_entry->blk_device;
+ return -1;
+ }
+ auto fstab_entry = fs_mgr_get_mounted_entry_for_userdata(fstab, block_device);
if (fstab_entry == nullptr) {
LERROR << "Can't find /data in fstab";
return -1;
diff --git a/fs_mgr/fs_mgr_format.cpp b/fs_mgr/fs_mgr_format.cpp
index 7be024f..778c0c1 100644
--- a/fs_mgr/fs_mgr_format.cpp
+++ b/fs_mgr/fs_mgr_format.cpp
@@ -121,7 +121,7 @@
}
static int format_f2fs(const std::string& fs_blkdev, uint64_t dev_sz, bool crypt_footer,
- bool needs_projid, bool needs_casefold) {
+ bool needs_projid, bool needs_casefold, bool fs_compress) {
if (!dev_sz) {
int rc = get_dev_sz(fs_blkdev, &dev_sz);
if (rc) {
@@ -147,6 +147,12 @@
args.push_back("-C");
args.push_back("utf8");
}
+ if (fs_compress) {
+ args.push_back("-O");
+ args.push_back("compression");
+ args.push_back("-O");
+ args.push_back("extra_attr");
+ }
args.push_back(fs_blkdev.c_str());
args.push_back(size_str.c_str());
@@ -166,7 +172,7 @@
if (entry.fs_type == "f2fs") {
return format_f2fs(entry.blk_device, entry.length, crypt_footer, needs_projid,
- needs_casefold);
+ needs_casefold, entry.fs_mgr_flags.fs_compress);
} else if (entry.fs_type == "ext4") {
return format_ext4(entry.blk_device, entry.mount_point, crypt_footer, needs_projid,
entry.fs_mgr_flags.ext_meta_csum);
diff --git a/fs_mgr/fs_mgr_fstab.cpp b/fs_mgr/fs_mgr_fstab.cpp
index b218f21..0825a77 100644
--- a/fs_mgr/fs_mgr_fstab.cpp
+++ b/fs_mgr/fs_mgr_fstab.cpp
@@ -178,6 +178,7 @@
CheckFlag("slotselect_other", slot_select_other);
CheckFlag("fsverity", fs_verity);
CheckFlag("metadata_csum", ext_meta_csum);
+ CheckFlag("fscompress", fs_compress);
#undef CheckFlag
diff --git a/fs_mgr/include/fs_mgr.h b/fs_mgr/include/fs_mgr.h
index 3d556c9..86090c1 100644
--- a/fs_mgr/include/fs_mgr.h
+++ b/fs_mgr/include/fs_mgr.h
@@ -107,10 +107,9 @@
// it destroys verity devices from device mapper after the device is unmounted.
int fs_mgr_umount_all(android::fs_mgr::Fstab* fstab);
-// Finds a entry in |fstab| that was used to mount a /data |mounted_entry| from
-// /proc/mounts.
+// Finds a entry in |fstab| that was used to mount a /data on |data_block_device|.
android::fs_mgr::FstabEntry* fs_mgr_get_mounted_entry_for_userdata(
- android::fs_mgr::Fstab* fstab, const android::fs_mgr::FstabEntry& mounted_entry);
+ android::fs_mgr::Fstab* fstab, const std::string& data_block_device);
int fs_mgr_remount_userdata_into_checkpointing(android::fs_mgr::Fstab* fstab);
// Finds the dm_bow device on which this block device is stacked, or returns
diff --git a/fs_mgr/include_fstab/fstab/fstab.h b/fs_mgr/include_fstab/fstab/fstab.h
index 79d9402..7216402 100644
--- a/fs_mgr/include_fstab/fstab/fstab.h
+++ b/fs_mgr/include_fstab/fstab/fstab.h
@@ -84,6 +84,7 @@
bool slot_select_other : 1;
bool fs_verity : 1;
bool ext_meta_csum : 1;
+ bool fs_compress : 1;
} fs_mgr_flags = {};
bool is_encryptable() const {
diff --git a/fs_mgr/libdm/Android.bp b/fs_mgr/libdm/Android.bp
index 0499e8d..58241b3 100644
--- a/fs_mgr/libdm/Android.bp
+++ b/fs_mgr/libdm/Android.bp
@@ -79,7 +79,7 @@
cc_test {
name: "vts_libdm_test",
defaults: ["libdm_test_defaults"],
- test_suites: ["vts-core"],
+ test_suites: ["vts"],
test_min_api_level: 29,
}
diff --git a/fs_mgr/libfiemap/Android.bp b/fs_mgr/libfiemap/Android.bp
index ac589c7..9d18a44 100644
--- a/fs_mgr/libfiemap/Android.bp
+++ b/fs_mgr/libfiemap/Android.bp
@@ -81,7 +81,7 @@
"fiemap_writer_test.cpp",
],
- test_suites: ["vts-core", "device-tests"],
+ test_suites: ["vts", "device-tests"],
auto_gen_config: true,
test_min_api_level: 29,
require_root: true,
diff --git a/fs_mgr/liblp/Android.bp b/fs_mgr/liblp/Android.bp
index ad19f38..a779a78 100644
--- a/fs_mgr/liblp/Android.bp
+++ b/fs_mgr/liblp/Android.bp
@@ -97,7 +97,7 @@
cc_test {
name: "vts_core_liblp_test",
defaults: ["liblp_test_defaults"],
- test_suites: ["vts-core"],
+ test_suites: ["vts"],
auto_gen_config: true,
test_min_api_level: 29,
require_root: true,
diff --git a/fs_mgr/liblp/liblp_test.xml b/fs_mgr/liblp/liblp_test.xml
index d9ee12e..98414b1 100644
--- a/fs_mgr/liblp/liblp_test.xml
+++ b/fs_mgr/liblp/liblp_test.xml
@@ -19,7 +19,6 @@
<option name="cleanup" value="true" />
<option name="push" value="liblp_test->/data/local/tmp/liblp_test" />
</target_preparer>
- <option name="test-suite-tag" value="vts-core" />
<test class="com.android.tradefed.testtype.GTest" >
<option name="native-test-device-path" value="/data/local/tmp" />
<option name="module-name" value="liblp_test" />
diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp
index 996fbca..a209ea6 100644
--- a/fs_mgr/libsnapshot/Android.bp
+++ b/fs_mgr/libsnapshot/Android.bp
@@ -182,7 +182,7 @@
"libstorage_literals_headers",
],
test_suites: [
- "vts-core",
+ "vts",
"device-tests"
],
test_min_api_level: 29,
diff --git a/fs_mgr/tests/AndroidTest.xml b/fs_mgr/tests/AndroidTest.xml
index 0ff8995..de835b3 100644
--- a/fs_mgr/tests/AndroidTest.xml
+++ b/fs_mgr/tests/AndroidTest.xml
@@ -21,6 +21,9 @@
<option name="push" value="CtsFsMgrTestCases->/data/local/tmp/CtsFsMgrTestCases" />
<option name="append-bitness" value="true" />
</target_preparer>
+ <target_preparer class="com.android.tradefed.targetprep.RootTargetPreparer">
+ <option name="throw-on-error" value="false" />
+ </target_preparer>
<test class="com.android.tradefed.testtype.GTest" >
<option name="native-test-device-path" value="/data/local/tmp" />
<option name="module-name" value="CtsFsMgrTestCases" />
diff --git a/fs_mgr/tests/fs_mgr_test.cpp b/fs_mgr/tests/fs_mgr_test.cpp
index 3fec608..27c8aae 100644
--- a/fs_mgr/tests/fs_mgr_test.cpp
+++ b/fs_mgr/tests/fs_mgr_test.cpp
@@ -1020,6 +1020,8 @@
ASSERT_TRUE(ReadFstabFromFile("/proc/mounts", &proc_mounts)) << "Failed to read /proc/mounts";
auto mounted_entry = GetEntryForMountPoint(&proc_mounts, "/data");
ASSERT_NE(mounted_entry, nullptr) << "/data is not mounted";
- ASSERT_NE(nullptr, fs_mgr_get_mounted_entry_for_userdata(&fstab, *mounted_entry))
+ std::string block_device;
+ ASSERT_TRUE(android::base::Realpath(mounted_entry->blk_device, &block_device));
+ ASSERT_NE(nullptr, fs_mgr_get_mounted_entry_for_userdata(&fstab, block_device))
<< "/data wasn't mounted from default fstab";
}
diff --git a/healthd/BatteryMonitor.cpp b/healthd/BatteryMonitor.cpp
index 8b67e22..599f500 100644
--- a/healthd/BatteryMonitor.cpp
+++ b/healthd/BatteryMonitor.cpp
@@ -258,7 +258,7 @@
props.batteryVoltage = getIntField(mHealthdConfig->batteryVoltagePath) / 1000;
if (!mHealthdConfig->batteryCurrentNowPath.isEmpty())
- props.batteryCurrent = getIntField(mHealthdConfig->batteryCurrentNowPath) / 1000;
+ props.batteryCurrent = getIntField(mHealthdConfig->batteryCurrentNowPath);
if (!mHealthdConfig->batteryFullChargePath.isEmpty())
props.batteryFullCharge = getIntField(mHealthdConfig->batteryFullChargePath);
diff --git a/init/AndroidTest.xml b/init/AndroidTest.xml
index 920dc6c..17f509a 100644
--- a/init/AndroidTest.xml
+++ b/init/AndroidTest.xml
@@ -24,6 +24,9 @@
<option name="push" value="CtsInitTestCases->/data/local/tmp/CtsInitTestCases" />
<option name="append-bitness" value="true" />
</target_preparer>
+ <target_preparer class="com.android.tradefed.targetprep.RootTargetPreparer">
+ <option name="throw-on-error" value="false" />
+ </target_preparer>
<test class="com.android.tradefed.testtype.GTest" >
<option name="native-test-device-path" value="/data/local/tmp" />
<option name="module-name" value="CtsInitTestCases" />
diff --git a/init/first_stage_init.cpp b/init/first_stage_init.cpp
index bd71cb5..ef8ffbe 100644
--- a/init/first_stage_init.cpp
+++ b/init/first_stage_init.cpp
@@ -234,7 +234,16 @@
old_root_dir.reset();
}
- Modprobe m({"/lib/modules"});
+ std::string module_load_file = "modules.load";
+ if (IsRecoveryMode() && !ForceNormalBoot(cmdline)) {
+ struct stat fileStat;
+ std::string recovery_load_path = "/lib/modules/modules.load.recovery";
+ if (!stat(recovery_load_path.c_str(), &fileStat)) {
+ module_load_file = "modules.load.recovery";
+ }
+ }
+
+ Modprobe m({"/lib/modules"}, module_load_file);
auto want_console = ALLOW_FIRST_STAGE_CONSOLE && FirstStageConsole(cmdline);
if (!m.LoadListedModules(!want_console)) {
if (want_console) {
diff --git a/init/parser.cpp b/init/parser.cpp
index 507ee4a..5c18551 100644
--- a/init/parser.cpp
+++ b/init/parser.cpp
@@ -41,7 +41,7 @@
}
void Parser::ParseData(const std::string& filename, std::string* data) {
- data->push_back('\n'); // TODO: fix tokenizer
+ data->push_back('\n');
data->push_back('\0');
parse_state state;
diff --git a/init/reboot.cpp b/init/reboot.cpp
index d2dc6d3..2f91663 100644
--- a/init/reboot.cpp
+++ b/init/reboot.cpp
@@ -838,20 +838,25 @@
}
static void UserspaceRebootWatchdogThread() {
- if (!WaitForProperty("sys.init.userspace_reboot.in_progress", "1", 20s)) {
- // TODO(b/135984674): should we reboot instead?
- LOG(WARNING) << "Userspace reboot didn't start in 20 seconds. Stopping watchdog";
- return;
+ auto started_timeout = GetMillisProperty("init.userspace_reboot.started.timeoutmillis", 10s);
+ if (!WaitForProperty("sys.init.userspace_reboot.in_progress", "1", started_timeout)) {
+ LOG(ERROR) << "Userspace reboot didn't start in " << started_timeout.count()
+ << "ms. Switching to full reboot";
+ // Init might be wedged, don't try to write reboot reason into a persistent property and do
+ // a dirty reboot.
+ PersistRebootReason("userspace_failed,watchdog_triggered,failed_to_start", false);
+ RebootSystem(ANDROID_RB_RESTART2, "userspace_failed,watchdog_triggered,failed_to_start");
}
LOG(INFO) << "Starting userspace reboot watchdog";
- auto timeout = GetMillisProperty("init.userspace_reboot.watchdog.timeoutmillis", 5min);
- LOG(INFO) << "UserspaceRebootWatchdog timeout: " << timeout.count() << "ms";
- if (!WaitForProperty("sys.boot_completed", "1", timeout)) {
- LOG(ERROR) << "Failed to boot in " << timeout.count() << "ms. Switching to full reboot";
+ auto watchdog_timeout = GetMillisProperty("init.userspace_reboot.watchdog.timeoutmillis", 5min);
+ LOG(INFO) << "UserspaceRebootWatchdog timeout: " << watchdog_timeout.count() << "ms";
+ if (!WaitForProperty("sys.boot_completed", "1", watchdog_timeout)) {
+ LOG(ERROR) << "Failed to boot in " << watchdog_timeout.count()
+ << "ms. Switching to full reboot";
// In this case device is in a boot loop. Only way to recover is to do dirty reboot.
// Since init might be wedged, don't try to write reboot reason into a persistent property.
- PersistRebootReason("userspace_failed,watchdog_triggered", false);
- RebootSystem(ANDROID_RB_RESTART2, "userspace_failed,watchdog_triggered");
+ PersistRebootReason("userspace_failed,watchdog_triggered,failed_to_boot", false);
+ RebootSystem(ANDROID_RB_RESTART2, "userspace_failed,watchdog_triggered,failed_to_boot");
}
LOG(INFO) << "Device booted, stopping userspace reboot watchdog";
}
diff --git a/init/service.cpp b/init/service.cpp
index b12d11a..20400a0 100644
--- a/init/service.cpp
+++ b/init/service.cpp
@@ -90,7 +90,9 @@
<< "\") has incorrect label or no domain transition from " << mycon.get()
<< " to another SELinux domain defined. Have you configured your "
"service correctly? https://source.android.com/security/selinux/"
- "device-policy#label_new_services_and_address_denials";
+ "device-policy#label_new_services_and_address_denials. Note: this "
+ "error shows up even in permissive mode in order to make auditing "
+ "denials possible.";
}
if (rc < 0) {
return Error() << "Could not get process context";
diff --git a/init/test_kill_services/Android.bp b/init/test_kill_services/Android.bp
index f6e85e2..d59e548 100644
--- a/init/test_kill_services/Android.bp
+++ b/init/test_kill_services/Android.bp
@@ -3,5 +3,9 @@
srcs: ["init_kill_services_test.cpp"],
shared_libs: ["libbase"],
test_suites: ["general-tests"],
- require_root: true,
+
+ // TODO(b/153565474): switch back to auto-generation
+ // and add back:
+ // require_root: true,
+ auto_gen_config: false,
}
diff --git a/init/test_kill_services/AndroidTest.xml b/init/test_kill_services/AndroidTest.xml
new file mode 100644
index 0000000..c1dcd59
--- /dev/null
+++ b/init/test_kill_services/AndroidTest.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2020 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Runs init_kill_services_test.">
+ <option name="test-suite-tag" value="apct" />
+ <option name="test-suite-tag" value="apct-native" />
+
+ <!-- cannot be autogenerated: b/153565474 -->
+ <target_preparer class="com.android.tradefed.targetprep.RebootTargetPreparer" />
+
+ <target_preparer class="com.android.tradefed.targetprep.RootTargetPreparer"/>
+
+ <target_preparer class="com.android.tradefed.targetprep.PushFilePreparer">
+ <option name="cleanup" value="true" />
+ <option name="push" value="init_kill_services_test->/data/local/tmp/init_kill_services_test" />
+ </target_preparer>
+
+ <test class="com.android.tradefed.testtype.GTest" >
+ <option name="native-test-device-path" value="/data/local/tmp" />
+ <option name="module-name" value="init_kill_services_test" />
+ </test>
+</configuration>
diff --git a/init/tokenizer_test.cpp b/init/tokenizer_test.cpp
index 6b31683..0122884 100644
--- a/init/tokenizer_test.cpp
+++ b/init/tokenizer_test.cpp
@@ -28,7 +28,7 @@
void RunTest(const std::string& data, const std::vector<std::vector<std::string>>& expected_tokens) {
auto data_copy = std::string{data};
- data_copy.push_back('\n'); // TODO: fix tokenizer
+ data_copy.push_back('\n');
data_copy.push_back('\0');
parse_state state;
diff --git a/init/ueventd.cpp b/init/ueventd.cpp
index d2b503b..7514b61 100644
--- a/init/ueventd.cpp
+++ b/init/ueventd.cpp
@@ -285,7 +285,6 @@
// Keep the current product name base configuration so we remain backwards compatible and
// allow it to override everything.
- // TODO: cleanup platform ueventd.rc to remove vendor specific device node entries (b/34968103)
auto hardware = android::base::GetProperty("ro.hardware", "");
auto ueventd_configuration = ParseConfig({"/system/etc/ueventd.rc", "/vendor/ueventd.rc",
diff --git a/init/ueventd_test.cpp b/init/ueventd_test.cpp
index 2d7d2f8..fc3cdfb 100644
--- a/init/ueventd_test.cpp
+++ b/init/ueventd_test.cpp
@@ -53,11 +53,7 @@
};
std::vector<std::thread> threads;
- // TODO(b/63712782): Structured bindings + templated containers are broken in clang :(
- // for (const auto& [file, parameter] : files_and_parameters) {
- for (const auto& pair : files_and_parameters) {
- const auto& file = pair.first;
- const auto& parameter = pair.second;
+ for (const auto& [file, parameter] : files_and_parameters) {
threads.emplace_back(std::thread(make_thread_function(file, parameter)));
}
diff --git a/libcutils/Android.bp b/libcutils/Android.bp
index dccf588..175b2b7 100644
--- a/libcutils/Android.bp
+++ b/libcutils/Android.bp
@@ -346,7 +346,7 @@
cc_test {
name: "KernelLibcutilsTest",
- test_suites: ["general-tests", "vts-core"],
+ test_suites: ["general-tests", "vts"],
defaults: ["libcutils_test_static_defaults"],
test_config: "KernelLibcutilsTest.xml",
}
diff --git a/liblog/include/log/log_read.h b/liblog/include/log/log_read.h
index 05ad25f..f3be0b8 100644
--- a/liblog/include/log/log_read.h
+++ b/liblog/include/log/log_read.h
@@ -48,8 +48,6 @@
* access to raw information, or parsing is an issue.
*/
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wzero-length-array"
struct logger_entry {
uint16_t len; /* length of the payload */
uint16_t hdr_size; /* sizeof(struct logger_entry) */
@@ -59,9 +57,7 @@
uint32_t nsec; /* nanoseconds */
uint32_t lid; /* log id of the payload, bottom 4 bits currently */
uint32_t uid; /* generating process's uid */
- char msg[0]; /* the entry's payload */
};
-#pragma clang diagnostic pop
/*
* The maximum size of the log entry payload that can be
@@ -83,32 +79,9 @@
struct logger_entry entry;
} __attribute__((aligned(4)));
#ifdef __cplusplus
- /* Matching log_time operators */
- bool operator==(const log_msg& T) const {
- return (entry.sec == T.entry.sec) && (entry.nsec == T.entry.nsec);
- }
- bool operator!=(const log_msg& T) const {
- return !(*this == T);
- }
- bool operator<(const log_msg& T) const {
- return (entry.sec < T.entry.sec) ||
- ((entry.sec == T.entry.sec) && (entry.nsec < T.entry.nsec));
- }
- bool operator>=(const log_msg& T) const {
- return !(*this < T);
- }
- bool operator>(const log_msg& T) const {
- return (entry.sec > T.entry.sec) ||
- ((entry.sec == T.entry.sec) && (entry.nsec > T.entry.nsec));
- }
- bool operator<=(const log_msg& T) const {
- return !(*this > T);
- }
uint64_t nsec() const {
return static_cast<uint64_t>(entry.sec) * NS_PER_SEC + entry.nsec;
}
-
- /* packet methods */
log_id_t id() {
return static_cast<log_id_t>(entry.lid);
}
diff --git a/liblog/logprint.cpp b/liblog/logprint.cpp
index e32878a..5c69bf8 100644
--- a/liblog/logprint.cpp
+++ b/liblog/logprint.cpp
@@ -509,12 +509,12 @@
* format: <priority:1><tag:N>\0<message:N>\0
*
* tag str
- * starts at buf->msg+1
+ * starts at buf + buf->hdr_size + 1
* msg
- * starts at buf->msg+1+len(tag)+1
+ * starts at buf + buf->hdr_size + 1 + len(tag) + 1
*
- * The message may have been truncated by the kernel log driver.
- * When that happens, we must null-terminate the message ourselves.
+ * The message may have been truncated. When that happens, we must null-terminate the message
+ * ourselves.
*/
if (buf->len < 3) {
/*
@@ -529,11 +529,11 @@
int msgEnd = -1;
int i;
- char* msg = buf->msg;
- if (buf->hdr_size != sizeof(struct logger_entry)) {
- fprintf(stderr, "+++ LOG: entry illegal hdr_size\n");
+ if (buf->hdr_size < sizeof(logger_entry)) {
+ fprintf(stderr, "+++ LOG: hdr_size must be at least as big as struct logger_entry\n");
return -1;
}
+ char* msg = reinterpret_cast<char*>(buf) + buf->hdr_size;
entry->uid = buf->uid;
for (i = 1; i < buf->len; i++) {
@@ -985,11 +985,11 @@
entry->pid = buf->pid;
entry->tid = buf->tid;
- eventData = (const unsigned char*)buf->msg;
- if (buf->hdr_size != sizeof(struct logger_entry)) {
- fprintf(stderr, "+++ LOG: entry illegal hdr_size\n");
+ if (buf->hdr_size < sizeof(logger_entry)) {
+ fprintf(stderr, "+++ LOG: hdr_size must be at least as big as struct logger_entry\n");
return -1;
}
+ eventData = reinterpret_cast<unsigned char*>(buf) + buf->hdr_size;
if (buf->lid == LOG_ID_SECURITY) {
entry->priority = ANDROID_LOG_WARN;
}
@@ -1048,7 +1048,7 @@
}
if ((result == 1) && fmtStr) {
/* We overflowed :-(, let's repaint the line w/o format dressings */
- eventData = (const unsigned char*)buf->msg;
+ eventData = reinterpret_cast<unsigned char*>(buf) + buf->hdr_size;
eventData += 4;
outBuf = messageBuf;
outRemaining = messageBufLen - 1;
diff --git a/liblog/pmsg_reader.cpp b/liblog/pmsg_reader.cpp
index 129d767..0e39aab 100644
--- a/liblog/pmsg_reader.cpp
+++ b/liblog/pmsg_reader.cpp
@@ -96,7 +96,7 @@
((logger_list->start.tv_sec != buf.l.realtime.tv_sec) ||
(logger_list->start.tv_nsec <= buf.l.realtime.tv_nsec)))) &&
(!logger_list->pid || (logger_list->pid == buf.p.pid))) {
- char* msg = log_msg->entry.msg;
+ char* msg = reinterpret_cast<char*>(&log_msg->entry) + log_msg->entry.hdr_size;
*msg = buf.prio;
fd = atomic_load(&logger_list->fd);
if (fd <= 0) {
diff --git a/liblog/tests/liblog_test.cpp b/liblog/tests/liblog_test.cpp
index a031531..048bf61 100644
--- a/liblog/tests/liblog_test.cpp
+++ b/liblog/tests/liblog_test.cpp
@@ -97,10 +97,7 @@
ASSERT_EQ(log_buffer, log_msg.id());
ASSERT_EQ(pid, log_msg.entry.pid);
- // TODO: Should this be an assert?
- if (log_msg.msg() == nullptr) {
- continue;
- }
+ ASSERT_NE(nullptr, log_msg.msg());
check_message(log_msg, &found);
}
@@ -121,10 +118,7 @@
ASSERT_EQ(log_buffer, log_msg.id());
ASSERT_EQ(pid, log_msg.entry.pid);
- // TODO: Should this be an assert?
- if (log_msg.msg() == nullptr) {
- continue;
- }
+ ASSERT_NE(nullptr, log_msg.msg());
found = false;
check_message(log_msg, &found);
@@ -1031,7 +1025,7 @@
#endif
}
-// TODO: This test is tautological. android_logger_list_read() calls recv() with
+// Note: This test is tautological. android_logger_list_read() calls recv() with
// LOGGER_ENTRY_MAX_PAYLOAD as its size argument, so it's not possible for this test to read a
// payload larger than that size.
TEST(liblog, too_big_payload) {
@@ -1982,7 +1976,6 @@
#endif
}
-// TODO: Do we need to check that we didn't actually write anything if we return a failure here?
TEST(liblog,
android_errorWriteWithInfoLog__android_logger_list_read__null_data) {
#ifdef __ANDROID__
diff --git a/liblog/tests/logprint_test.cpp b/liblog/tests/logprint_test.cpp
index 7ca02ac..72e53f9 100644
--- a/liblog/tests/logprint_test.cpp
+++ b/liblog/tests/logprint_test.cpp
@@ -14,8 +14,14 @@
* limitations under the License.
*/
+#include <log/logprint.h>
+
+#include <string>
+
#include <gtest/gtest.h>
+#include <log/log_read.h>
+
size_t convertPrintable(char* p, const char* message, size_t messageLen);
TEST(liblog, convertPrintable_ascii) {
@@ -85,3 +91,63 @@
EXPECT_EQ(output_size, strlen(expected_output));
EXPECT_STREQ(expected_output, output);
}
+
+TEST(liblog, log_print_different_header_size) {
+ constexpr int32_t kPid = 123;
+ constexpr uint32_t kTid = 456;
+ constexpr uint32_t kSec = 1000;
+ constexpr uint32_t kNsec = 999;
+ constexpr uint32_t kLid = LOG_ID_MAIN;
+ constexpr uint32_t kUid = 987;
+ constexpr char kPriority = ANDROID_LOG_ERROR;
+
+ auto create_buf = [](char* buf, size_t len, uint16_t hdr_size) {
+ memset(buf, 0, len);
+ logger_entry* header = reinterpret_cast<logger_entry*>(buf);
+ header->hdr_size = hdr_size;
+ header->pid = kPid;
+ header->tid = kTid;
+ header->sec = kSec;
+ header->nsec = kNsec;
+ header->lid = kLid;
+ header->uid = kUid;
+ char* message = buf + header->hdr_size;
+ uint16_t message_len = 0;
+ message[message_len++] = kPriority;
+ message[message_len++] = 'T';
+ message[message_len++] = 'a';
+ message[message_len++] = 'g';
+ message[message_len++] = '\0';
+ message[message_len++] = 'm';
+ message[message_len++] = 's';
+ message[message_len++] = 'g';
+ message[message_len++] = '!';
+ message[message_len++] = '\0';
+ header->len = message_len;
+ };
+
+ auto check_entry = [&](const AndroidLogEntry& entry) {
+ EXPECT_EQ(kSec, static_cast<uint32_t>(entry.tv_sec));
+ EXPECT_EQ(kNsec, static_cast<uint32_t>(entry.tv_nsec));
+ EXPECT_EQ(kPriority, entry.priority);
+ EXPECT_EQ(kUid, static_cast<uint32_t>(entry.uid));
+ EXPECT_EQ(kPid, entry.pid);
+ EXPECT_EQ(kTid, static_cast<uint32_t>(entry.tid));
+ EXPECT_STREQ("Tag", entry.tag);
+ EXPECT_EQ(4U, entry.tagLen); // Apparently taglen includes the nullptr?
+ EXPECT_EQ(4U, entry.messageLen);
+ EXPECT_STREQ("msg!", entry.message);
+ };
+ alignas(logger_entry) char buf[LOGGER_ENTRY_MAX_LEN];
+ create_buf(buf, sizeof(buf), sizeof(logger_entry));
+
+ AndroidLogEntry entry_normal_size;
+ ASSERT_EQ(0,
+ android_log_processLogBuffer(reinterpret_cast<logger_entry*>(buf), &entry_normal_size));
+ check_entry(entry_normal_size);
+
+ create_buf(buf, sizeof(buf), sizeof(logger_entry) + 3);
+ AndroidLogEntry entry_odd_size;
+ ASSERT_EQ(0, android_log_processLogBuffer(reinterpret_cast<logger_entry*>(buf), &entry_odd_size));
+ check_entry(entry_odd_size);
+}
\ No newline at end of file
diff --git a/libmodprobe/include/modprobe/modprobe.h b/libmodprobe/include/modprobe/modprobe.h
index ee6ae7a..297036e 100644
--- a/libmodprobe/include/modprobe/modprobe.h
+++ b/libmodprobe/include/modprobe/modprobe.h
@@ -24,7 +24,7 @@
class Modprobe {
public:
- Modprobe(const std::vector<std::string>&);
+ Modprobe(const std::vector<std::string>&, const std::string load_file = "modules.load");
bool LoadListedModules(bool strict = true);
bool LoadWithAliases(const std::string& module_name, bool strict,
diff --git a/libmodprobe/libmodprobe.cpp b/libmodprobe/libmodprobe.cpp
index f22bbf1..d193796 100644
--- a/libmodprobe/libmodprobe.cpp
+++ b/libmodprobe/libmodprobe.cpp
@@ -312,7 +312,7 @@
}
}
-Modprobe::Modprobe(const std::vector<std::string>& base_paths) {
+Modprobe::Modprobe(const std::vector<std::string>& base_paths, const std::string load_file) {
using namespace std::placeholders;
for (const auto& base_path : base_paths) {
@@ -326,7 +326,7 @@
ParseCfg(base_path + "/modules.softdep", softdep_callback);
auto load_callback = std::bind(&Modprobe::ParseLoadCallback, this, _1);
- ParseCfg(base_path + "/modules.load", load_callback);
+ ParseCfg(base_path + "/" + load_file, load_callback);
auto options_callback = std::bind(&Modprobe::ParseOptionsCallback, this, _1);
ParseCfg(base_path + "/modules.options", options_callback);
diff --git a/libprocessgroup/Android.bp b/libprocessgroup/Android.bp
index 618a5c5..2c1b255 100644
--- a/libprocessgroup/Android.bp
+++ b/libprocessgroup/Android.bp
@@ -13,6 +13,10 @@
enabled: true,
},
},
+ apex_available: [
+ "//apex_available:platform",
+ "//apex_available:anyapex",
+ ],
}
cc_library {
@@ -52,4 +56,8 @@
"-Werror",
"-Wexit-time-destructors",
],
+ apex_available: [
+ "//apex_available:platform",
+ "//apex_available:anyapex",
+ ],
}
diff --git a/libstats/push_compat/Android.bp b/libstats/push_compat/Android.bp
index fcd8c83..caea048 100644
--- a/libstats/push_compat/Android.bp
+++ b/libstats/push_compat/Android.bp
@@ -47,6 +47,7 @@
defaults: ["libstatspush_compat_defaults"],
export_include_dirs: ["include"],
static_libs: ["libgtest_prod"],
+ apex_available: ["com.android.resolv"],
}
cc_test {
diff --git a/libstats/socket/Android.bp b/libstats/socket/Android.bp
index 9fd9fbc..690dc94 100644
--- a/libstats/socket/Android.bp
+++ b/libstats/socket/Android.bp
@@ -54,6 +54,7 @@
name: "libstatssocket_headers",
export_include_dirs: ["include"],
host_supported: true,
+ apex_available: ["com.android.resolv"],
}
cc_benchmark {
diff --git a/libunwindstack/DexFile.cpp b/libunwindstack/DexFile.cpp
index bf63abf..8fc3d23 100644
--- a/libunwindstack/DexFile.cpp
+++ b/libunwindstack/DexFile.cpp
@@ -50,6 +50,22 @@
std::unique_ptr<DexFile> DexFile::Create(uint64_t dex_file_offset_in_memory, Memory* memory,
MapInfo* info) {
+ if (UNLIKELY(!HasDexSupport())) {
+ return nullptr;
+ }
+
+ size_t max_size = info->end - dex_file_offset_in_memory;
+ if (memory->IsLocal()) {
+ size_t size = max_size;
+
+ std::string err_msg;
+ std::unique_ptr<art_api::dex::DexFile> art_dex_file = DexFile::OpenFromMemory(
+ reinterpret_cast<void const*>(dex_file_offset_in_memory), &size, info->name, &err_msg);
+ if (art_dex_file != nullptr && size <= max_size) {
+ return std::unique_ptr<DexFile>(new DexFile(art_dex_file));
+ }
+ }
+
if (!info->name.empty()) {
std::unique_ptr<DexFile> dex_file =
DexFileFromFile::Create(dex_file_offset_in_memory - info->start + info->offset, info->name);
@@ -57,7 +73,7 @@
return dex_file;
}
}
- return DexFileFromMemory::Create(dex_file_offset_in_memory, memory, info->name);
+ return DexFileFromMemory::Create(dex_file_offset_in_memory, memory, info->name, max_size);
}
bool DexFile::GetMethodInformation(uint64_t dex_offset, std::string* method_name,
@@ -94,7 +110,8 @@
std::unique_ptr<DexFileFromMemory> DexFileFromMemory::Create(uint64_t dex_file_offset_in_memory,
Memory* memory,
- const std::string& name) {
+ const std::string& name,
+ size_t max_size) {
if (UNLIKELY(!HasDexSupport())) {
return nullptr;
}
@@ -105,6 +122,9 @@
std::string error_msg;
std::unique_ptr<art_api::dex::DexFile> art_dex_file =
OpenFromMemory(backing_memory.data(), &size, name, &error_msg);
+ if (size > max_size) {
+ return nullptr;
+ }
if (art_dex_file != nullptr) {
return std::unique_ptr<DexFileFromMemory>(
diff --git a/libunwindstack/DexFile.h b/libunwindstack/DexFile.h
index 4e8369f..fe185da 100644
--- a/libunwindstack/DexFile.h
+++ b/libunwindstack/DexFile.h
@@ -55,7 +55,8 @@
class DexFileFromMemory : public DexFile {
public:
static std::unique_ptr<DexFileFromMemory> Create(uint64_t dex_file_offset_in_memory,
- Memory* memory, const std::string& name);
+ Memory* memory, const std::string& name,
+ size_t max_size);
private:
DexFileFromMemory(std::unique_ptr<art_api::dex::DexFile>& art_dex_file,
diff --git a/libunwindstack/MemoryLocal.h b/libunwindstack/MemoryLocal.h
index 29aaf12..7e027cf 100644
--- a/libunwindstack/MemoryLocal.h
+++ b/libunwindstack/MemoryLocal.h
@@ -28,6 +28,8 @@
MemoryLocal() = default;
virtual ~MemoryLocal() = default;
+ bool IsLocal() const override { return true; }
+
size_t Read(uint64_t addr, void* dst, size_t size) override;
};
diff --git a/libunwindstack/include/unwindstack/Memory.h b/libunwindstack/include/unwindstack/Memory.h
index 3106564..ecd908a 100644
--- a/libunwindstack/include/unwindstack/Memory.h
+++ b/libunwindstack/include/unwindstack/Memory.h
@@ -41,6 +41,8 @@
virtual void Clear() {}
+ virtual bool IsLocal() const { return false; }
+
virtual size_t Read(uint64_t addr, void* dst, size_t size) = 0;
bool ReadFully(uint64_t addr, void* dst, size_t size);
diff --git a/libunwindstack/tests/DexFileTest.cpp b/libunwindstack/tests/DexFileTest.cpp
index dc935a3..1deba01 100644
--- a/libunwindstack/tests/DexFileTest.cpp
+++ b/libunwindstack/tests/DexFileTest.cpp
@@ -21,6 +21,7 @@
#include <unordered_map>
+#include <MemoryLocal.h>
#include <android-base/file.h>
#include <gtest/gtest.h>
#include <unwindstack/MapInfo.h>
@@ -109,7 +110,7 @@
memory.SetMemory(0x1000, kDexData, 10);
- EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "") == nullptr);
+ EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "", sizeof(kDexData)) == nullptr);
}
TEST(DexFileTest, from_memory_fail_too_small_for_data) {
@@ -117,7 +118,7 @@
memory.SetMemory(0x1000, kDexData, sizeof(kDexData) - 2);
- EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "") == nullptr);
+ EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "", sizeof(kDexData)) == nullptr);
}
TEST(DexFileTest, from_memory_open) {
@@ -125,7 +126,7 @@
memory.SetMemory(0x1000, kDexData, sizeof(kDexData));
- EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "") != nullptr);
+ EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "", sizeof(kDexData)) != nullptr);
}
TEST(DexFileTest, from_memory_no_leak) {
@@ -136,7 +137,7 @@
size_t first_allocated_bytes = 0;
size_t last_allocated_bytes = 0;
for (size_t i = 0; i < kNumLeakLoops; i++) {
- EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "") != nullptr);
+ EXPECT_TRUE(DexFileFromMemory::Create(0x1000, &memory, "", sizeof(kDexData)) != nullptr);
ASSERT_NO_FATAL_FAILURE(CheckForLeak(i, &first_allocated_bytes, &last_allocated_bytes));
}
}
@@ -213,6 +214,43 @@
EXPECT_TRUE(dex_file == nullptr);
}
+TEST(DexFileTest, create_using_memory_size_too_small) {
+ MemoryFake memory;
+ memory.SetMemory(0x4000, kDexData, sizeof(kDexData));
+ MapInfo info(nullptr, nullptr, 0x100, sizeof(kDexData) - 2, 0x200, 0x5, "/does/not/exist");
+ EXPECT_TRUE(DexFile::Create(0x4000, &memory, &info) != nullptr);
+}
+
+class MemoryLocalFake : public MemoryLocal {
+ public:
+ MemoryLocalFake(size_t memory_size) : backing_(memory_size) {}
+ virtual ~MemoryLocalFake() = default;
+
+ void* Data() { return backing_.data(); }
+
+ private:
+ std::vector<void*> backing_;
+};
+
+TEST(DexFileTest, create_using_local_memory) {
+ MemoryLocalFake memory(sizeof(kDexData));
+
+ memcpy(memory.Data(), kDexData, sizeof(kDexData));
+ uint64_t start = reinterpret_cast<uint64_t>(memory.Data());
+ MapInfo info(nullptr, nullptr, start, start + 0x1000, 0x200, 0x5, "/does/not/exist");
+ EXPECT_TRUE(DexFile::Create(start, &memory, &info) != nullptr);
+}
+
+TEST(DexFileTest, create_using_local_memory_size_too_small) {
+ MemoryLocalFake memory(sizeof(kDexData));
+
+ memcpy(memory.Data(), kDexData, sizeof(kDexData));
+ uint64_t start = reinterpret_cast<uint64_t>(memory.Data());
+ MapInfo info(nullptr, nullptr, start, start + sizeof(kDexData) - 2, 0x200, 0x5,
+ "/does/not/exist");
+ EXPECT_TRUE(DexFile::Create(start, &memory, &info) == nullptr);
+}
+
TEST(DexFileTest, get_method) {
MemoryFake memory;
memory.SetMemory(0x4000, kDexData, sizeof(kDexData));
diff --git a/libziparchive/zip_archive.cc b/libziparchive/zip_archive.cc
index 8f9774f..7bf2120 100644
--- a/libziparchive/zip_archive.cc
+++ b/libziparchive/zip_archive.cc
@@ -59,7 +59,7 @@
// Used to turn on crc checks - verify that the content CRC matches the values
// specified in the local file header and the central directory.
-static const bool kCrcChecksEnabled = false;
+static constexpr bool kCrcChecksEnabled = false;
// The maximum number of bytes to scan backwards for the EOCD start.
static const uint32_t kMaxEOCDSearch = kMaxCommentLen + sizeof(EocdRecord);
@@ -137,6 +137,19 @@
uint64_t cd_start_offset;
};
+// Reads |T| at |readPtr| and increments |readPtr|. Returns std::nullopt if the boundary check
+// fails.
+template <typename T>
+static std::optional<T> TryConsumeUnaligned(uint8_t** readPtr, const uint8_t* bufStart,
+ size_t bufSize) {
+ if (bufSize < sizeof(T) || *readPtr - bufStart > bufSize - sizeof(T)) {
+ ALOGW("Zip: %zu byte read exceeds the boundary of allocated buf, offset %zu, bufSize %zu",
+ sizeof(T), *readPtr - bufStart, bufSize);
+ return std::nullopt;
+ }
+ return ConsumeUnaligned<T>(readPtr);
+}
+
static ZipError FindCentralDirectoryInfoForZip64(const char* debugFileName, ZipArchive* archive,
off64_t eocdOffset, CentralDirectoryInfo* cdInfo) {
if (eocdOffset <= sizeof(Zip64EocdLocator)) {
@@ -379,13 +392,19 @@
std::optional<uint64_t> compressedFileSize;
std::optional<uint64_t> localHeaderOffset;
if (zip32UncompressedSize == UINT32_MAX) {
- uncompressedFileSize = ConsumeUnaligned<uint64_t>(&readPtr);
+ uncompressedFileSize =
+ TryConsumeUnaligned<uint64_t>(&readPtr, extraFieldStart, extraFieldLength);
+ if (!uncompressedFileSize.has_value()) return kInvalidOffset;
}
if (zip32CompressedSize == UINT32_MAX) {
- compressedFileSize = ConsumeUnaligned<uint64_t>(&readPtr);
+ compressedFileSize =
+ TryConsumeUnaligned<uint64_t>(&readPtr, extraFieldStart, extraFieldLength);
+ if (!compressedFileSize.has_value()) return kInvalidOffset;
}
if (zip32LocalFileHeaderOffset == UINT32_MAX) {
- localHeaderOffset = ConsumeUnaligned<uint64_t>(&readPtr);
+ localHeaderOffset =
+ TryConsumeUnaligned<uint64_t>(&readPtr, extraFieldStart, extraFieldLength);
+ if (!localHeaderOffset.has_value()) return kInvalidOffset;
}
// calculate how many bytes we read after the data size field.
@@ -606,6 +625,10 @@
static int32_t ValidateDataDescriptor(MappedZipFile& mapped_zip, const ZipEntry64* entry) {
// Maximum possible size for data descriptor: 2 * 4 + 2 * 8 = 24 bytes
+ // The zip format doesn't specify the size of data descriptor. But we won't read OOB here even
+ // if the descriptor isn't present. Because the size cd + eocd in the end of the zipfile is
+ // larger than 24 bytes. And if the descriptor contains invalid data, we'll abort due to
+ // kInconsistentInformation.
uint8_t ddBuf[24];
off64_t offset = entry->offset;
if (entry->method != kCompressStored) {
@@ -1009,19 +1032,17 @@
// the data appended to it.
class MemoryWriter : public zip_archive::Writer {
public:
- static MemoryWriter Create(uint8_t* buf, size_t size, const ZipEntry64* entry) {
+ static std::unique_ptr<MemoryWriter> Create(uint8_t* buf, size_t size, const ZipEntry64* entry) {
const uint64_t declared_length = entry->uncompressed_length;
if (declared_length > size) {
ALOGW("Zip: file size %" PRIu64 " is larger than the buffer size %zu.", declared_length,
size);
- return MemoryWriter{nullptr, 0};
+ return nullptr;
}
- return MemoryWriter(buf, size);
+ return std::unique_ptr<MemoryWriter>(new MemoryWriter(buf, size));
}
- bool IsValid() const { return buf_ != nullptr; }
-
virtual bool Append(uint8_t* buf, size_t buf_size) override {
if (size_ < buf_size || bytes_written_ > size_ - buf_size) {
ALOGW("Zip: Unexpected size %zu (declared) vs %zu (actual)", size_,
@@ -1053,17 +1074,17 @@
// block device).
//
// Returns a valid FileWriter on success, |nullptr| if an error occurred.
- static FileWriter Create(int fd, const ZipEntry64* entry) {
+ static std::unique_ptr<FileWriter> Create(int fd, const ZipEntry64* entry) {
const uint64_t declared_length = entry->uncompressed_length;
const off64_t current_offset = lseek64(fd, 0, SEEK_CUR);
if (current_offset == -1) {
ALOGW("Zip: unable to seek to current location on fd %d: %s", fd, strerror(errno));
- return FileWriter{};
+ return nullptr;
}
if (declared_length > SIZE_MAX || declared_length > INT64_MAX) {
ALOGW("Zip: file size %" PRIu64 " is too large to extract.", declared_length);
- return FileWriter{};
+ return nullptr;
}
#if defined(__linux__)
@@ -1081,7 +1102,7 @@
if (result == -1 && errno == ENOSPC) {
ALOGW("Zip: unable to allocate %" PRIu64 " bytes at offset %" PRId64 ": %s",
declared_length, static_cast<int64_t>(current_offset), strerror(errno));
- return FileWriter{};
+ return nullptr;
}
}
#endif // __linux__
@@ -1089,7 +1110,7 @@
struct stat sb;
if (fstat(fd, &sb) == -1) {
ALOGW("Zip: unable to fstat file: %s", strerror(errno));
- return FileWriter{};
+ return nullptr;
}
// Block device doesn't support ftruncate(2).
@@ -1098,11 +1119,11 @@
if (result == -1) {
ALOGW("Zip: unable to truncate file to %" PRId64 ": %s",
static_cast<int64_t>(declared_length + current_offset), strerror(errno));
- return FileWriter{};
+ return nullptr;
}
}
- return FileWriter(fd, declared_length);
+ return std::unique_ptr<FileWriter>(new FileWriter(fd, declared_length));
}
FileWriter(FileWriter&& other) noexcept
@@ -1112,8 +1133,6 @@
other.fd_ = -1;
}
- bool IsValid() const { return fd_ != -1; }
-
virtual bool Append(uint8_t* buf, size_t buf_size) override {
if (declared_length_ < buf_size || total_bytes_written_ > declared_length_ - buf_size) {
ALOGW("Zip: Unexpected size %zu (declared) vs %zu (actual)", declared_length_,
@@ -1318,11 +1337,15 @@
if (!writer->Append(&buf[0], block_size)) {
return kIoError;
}
- crc = crc32(crc, &buf[0], block_size);
+ if (crc_out) {
+ crc = crc32(crc, &buf[0], block_size);
+ }
count += block_size;
}
- *crc_out = crc;
+ if (crc_out) {
+ *crc_out = crc;
+ }
return 0;
}
@@ -1335,9 +1358,11 @@
int32_t return_value = -1;
uint64_t crc = 0;
if (method == kCompressStored) {
- return_value = CopyEntryToWriter(handle->mapped_zip, entry, writer, &crc);
+ return_value =
+ CopyEntryToWriter(handle->mapped_zip, entry, writer, kCrcChecksEnabled ? &crc : nullptr);
} else if (method == kCompressDeflated) {
- return_value = InflateEntryToWriter(handle->mapped_zip, entry, writer, &crc);
+ return_value =
+ InflateEntryToWriter(handle->mapped_zip, entry, writer, kCrcChecksEnabled ? &crc : nullptr);
}
if (!return_value && entry->has_data_descriptor) {
@@ -1365,11 +1390,11 @@
int32_t ExtractToMemory(ZipArchiveHandle archive, const ZipEntry64* entry, uint8_t* begin,
size_t size) {
auto writer = MemoryWriter::Create(begin, size, entry);
- if (!writer.IsValid()) {
+ if (!writer) {
return kIoError;
}
- return ExtractToWriter(archive, entry, &writer);
+ return ExtractToWriter(archive, entry, writer.get());
}
int32_t ExtractEntryToFile(ZipArchiveHandle archive, const ZipEntry* entry, int fd) {
@@ -1379,11 +1404,11 @@
int32_t ExtractEntryToFile(ZipArchiveHandle archive, const ZipEntry64* entry, int fd) {
auto writer = FileWriter::Create(fd, entry);
- if (!writer.IsValid()) {
+ if (!writer) {
return kIoError;
}
- return ExtractToWriter(archive, entry, &writer);
+ return ExtractToWriter(archive, entry, writer.get());
}
int GetFileDescriptor(const ZipArchiveHandle archive) {
@@ -1497,8 +1522,9 @@
return false;
}
} else {
- if (off < 0 || off > data_length_) {
- ALOGE("Zip: invalid offset: %" PRId64 ", data length: %" PRId64, off, data_length_);
+ if (off < 0 || data_length_ < len || off > data_length_ - len) {
+ ALOGE("Zip: invalid offset: %" PRId64 ", read length: %zu, data length: %" PRId64, off, len,
+ data_length_);
return false;
}
memcpy(buf, static_cast<const uint8_t*>(base_ptr_) + off, len);
diff --git a/libziparchive/zip_archive_benchmark.cpp b/libziparchive/zip_archive_benchmark.cpp
index 09d3b8a..cfa5912 100644
--- a/libziparchive/zip_archive_benchmark.cpp
+++ b/libziparchive/zip_archive_benchmark.cpp
@@ -17,7 +17,6 @@
#include <cstdio>
#include <cstdlib>
#include <cstring>
-#include <iostream>
#include <string>
#include <tuple>
#include <vector>
@@ -28,17 +27,20 @@
#include <ziparchive/zip_archive_stream_entry.h>
#include <ziparchive/zip_writer.h>
-static TemporaryFile* CreateZip() {
- TemporaryFile* result = new TemporaryFile;
+static std::unique_ptr<TemporaryFile> CreateZip(int size = 4, int count = 1000) {
+ auto result = std::make_unique<TemporaryFile>();
FILE* fp = fdopen(result->fd, "w");
ZipWriter writer(fp);
std::string lastName = "file";
- for (size_t i = 0; i < 1000; i++) {
+ for (size_t i = 0; i < count; i++) {
// Make file names longer and longer.
lastName = lastName + std::to_string(i);
writer.StartEntry(lastName.c_str(), ZipWriter::kCompress);
- writer.WriteBytes("helo", 4);
+ while (size > 0) {
+ writer.WriteBytes("helo", 4);
+ size -= 4;
+ }
writer.FinishEntry();
}
writer.Finish();
@@ -106,5 +108,28 @@
}
BENCHMARK(StartAlignedEntry)->Arg(2)->Arg(16)->Arg(1024)->Arg(4096);
+static void ExtractEntry(benchmark::State& state) {
+ std::unique_ptr<TemporaryFile> temp_file(CreateZip(1024 * 1024, 1));
+
+ ZipArchiveHandle handle;
+ ZipEntry data;
+ if (OpenArchive(temp_file->path, &handle)) {
+ state.SkipWithError("Failed to open archive");
+ }
+ if (FindEntry(handle, "file0", &data)) {
+ state.SkipWithError("Failed to find archive entry");
+ }
+
+ std::vector<uint8_t> buffer(1024 * 1024);
+ for (auto _ : state) {
+ if (ExtractToMemory(handle, &data, buffer.data(), uint32_t(buffer.size()))) {
+ state.SkipWithError("Failed to extract archive entry");
+ break;
+ }
+ }
+ CloseArchive(handle);
+}
+
+BENCHMARK(ExtractEntry)->Arg(2)->Arg(16)->Arg(1024);
BENCHMARK_MAIN();
diff --git a/libziparchive/zip_archive_test.cc b/libziparchive/zip_archive_test.cc
index 3d4e580..f5429be 100644
--- a/libziparchive/zip_archive_test.cc
+++ b/libziparchive/zip_archive_test.cc
@@ -344,8 +344,8 @@
// Known facts about a.txt, from zipinfo -v.
ASSERT_EQ(63, data.offset);
ASSERT_EQ(kCompressDeflated, data.method);
- ASSERT_EQ(static_cast<uint32_t>(17), data.uncompressed_length);
- ASSERT_EQ(static_cast<uint32_t>(13), data.compressed_length);
+ ASSERT_EQ(17u, data.uncompressed_length);
+ ASSERT_EQ(13u, data.compressed_length);
ASSERT_EQ(0x950821c5, data.crc32);
ASSERT_EQ(static_cast<uint32_t>(0x438a8005), data.mod_time);
@@ -505,9 +505,12 @@
ZipEntry64 entry;
ASSERT_EQ(0, FindEntry(handle, "empty.txt", &entry));
- ASSERT_EQ(static_cast<uint32_t>(0), entry.uncompressed_length);
+ ASSERT_EQ(0u, entry.uncompressed_length);
+ // Extraction to a 1 byte buffer should succeed.
uint8_t buffer[1];
ASSERT_EQ(0, ExtractToMemory(handle, &entry, buffer, 1));
+ // Extraction to an empty buffer should succeed.
+ ASSERT_EQ(0, ExtractToMemory(handle, &entry, nullptr, 0));
TemporaryFile tmp_output_file;
ASSERT_NE(-1, tmp_output_file.fd);
@@ -782,7 +785,7 @@
// "abdcdefghijk").
ZipEntry64 entry;
ASSERT_EQ(0, FindEntry(handle, "name", &entry));
- ASSERT_EQ(static_cast<uint32_t>(12), entry.uncompressed_length);
+ ASSERT_EQ(12u, entry.uncompressed_length);
entry_out->resize(12);
(*error_code_out) = ExtractToMemory(handle, &entry, &((*entry_out)[0]), 12);
diff --git a/logd/FlushCommand.cpp b/logd/FlushCommand.cpp
index bd17555..0845504 100644
--- a/logd/FlushCommand.cpp
+++ b/logd/FlushCommand.cpp
@@ -49,18 +49,8 @@
return;
}
if (entry->mTimeout.tv_sec || entry->mTimeout.tv_nsec) {
- if (mReader.logbuf().isMonotonic()) {
- LogTimeEntry::unlock();
- return;
- }
- // If the user changes the time in a gross manner that
- // invalidates the timeout, fall through and trigger.
- log_time now(CLOCK_REALTIME);
- if (((entry->mEnd + entry->mTimeout) > now) &&
- (now > entry->mEnd)) {
- LogTimeEntry::unlock();
- return;
- }
+ LogTimeEntry::unlock();
+ return;
}
entry->triggerReader_Locked();
LogTimeEntry::unlock();
diff --git a/logd/FlushCommand.h b/logd/FlushCommand.h
index ceaf393..a69d439 100644
--- a/logd/FlushCommand.h
+++ b/logd/FlushCommand.h
@@ -16,7 +16,7 @@
#ifndef _FLUSH_COMMAND_H
#define _FLUSH_COMMAND_H
-#include <private/android_logger.h>
+#include <android/log.h>
#include <sysutils/SocketClientCommand.h>
class LogBufferElement;
diff --git a/logd/LogBuffer.cpp b/logd/LogBuffer.cpp
index 1cf2061..1f8ad05 100644
--- a/logd/LogBuffer.cpp
+++ b/logd/LogBuffer.cpp
@@ -27,6 +27,7 @@
#include <unistd.h>
#include <unordered_map>
+#include <utility>
#include <cutils/properties.h>
#include <private/android_logger.h>
@@ -43,8 +44,6 @@
// Default
#define log_buffer_size(id) mMaxSize[id]
-const log_time LogBuffer::pruneMargin(3, 0);
-
void LogBuffer::init() {
log_id_for_each(i) {
mLastSet[i] = false;
@@ -390,59 +389,7 @@
// assumes LogBuffer::wrlock() held, owns elem, look after garbage collection
void LogBuffer::log(LogBufferElement* elem) {
- // cap on how far back we will sort in-place, otherwise append
- static uint32_t too_far_back = 5; // five seconds
- // Insert elements in time sorted order if possible
- // NB: if end is region locked, place element at end of list
- LogBufferElementCollection::iterator it = mLogElements.end();
- LogBufferElementCollection::iterator last = it;
- if (__predict_true(it != mLogElements.begin())) --it;
- if (__predict_false(it == mLogElements.begin()) ||
- __predict_true((*it)->getRealTime() <= elem->getRealTime()) ||
- __predict_false((((*it)->getRealTime().tv_sec - too_far_back) >
- elem->getRealTime().tv_sec) &&
- (elem->getLogId() != LOG_ID_KERNEL) &&
- ((*it)->getLogId() != LOG_ID_KERNEL))) {
- mLogElements.push_back(elem);
- } else {
- log_time end(log_time::EPOCH);
- bool end_set = false;
- bool end_always = false;
-
- LogTimeEntry::rdlock();
-
- LastLogTimes::iterator times = mTimes.begin();
- while (times != mTimes.end()) {
- LogTimeEntry* entry = times->get();
- if (!entry->mNonBlock) {
- end_always = true;
- break;
- }
- // it passing mEnd is blocked by the following checks.
- if (!end_set || (end <= entry->mEnd)) {
- end = entry->mEnd;
- end_set = true;
- }
- times++;
- }
-
- if (end_always || (end_set && (end > (*it)->getRealTime()))) {
- mLogElements.push_back(elem);
- } else {
- // should be short as timestamps are localized near end()
- do {
- last = it;
- if (__predict_false(it == mLogElements.begin())) {
- break;
- }
- --it;
- } while (((*it)->getRealTime() > elem->getRealTime()) &&
- (!end_set || (end <= (*it)->getRealTime())));
- mLogElements.insert(last, elem);
- }
- LogTimeEntry::unlock();
- }
-
+ mLogElements.push_back(elem);
stats.add(elem);
maybePrune(elem->getLogId());
}
@@ -614,12 +561,11 @@
}
void clear(LogBufferElement* element) {
- log_time current =
- element->getRealTime() - log_time(EXPIRE_RATELIMIT, 0);
+ uint64_t current = element->getRealTime().nsec() - (EXPIRE_RATELIMIT * NS_PER_SEC);
for (LogBufferElementMap::iterator it = map.begin(); it != map.end();) {
LogBufferElement* mapElement = it->second;
- if ((mapElement->getDropped() >= EXPIRE_THRESHOLD) &&
- (current > mapElement->getRealTime())) {
+ if (mapElement->getDropped() >= EXPIRE_THRESHOLD &&
+ current > mapElement->getRealTime().nsec()) {
it = map.erase(it);
} else {
++it;
@@ -628,16 +574,6 @@
}
};
-// Determine if watermark is within pruneMargin + 1s from the end of the list,
-// the caller will use this result to set an internal busy flag indicating
-// the prune operation could not be completed because a reader is blocking
-// the request.
-bool LogBuffer::isBusy(log_time watermark) {
- LogBufferElementCollection::iterator ei = mLogElements.end();
- --ei;
- return watermark < ((*ei)->getRealTime() - pruneMargin - log_time(1, 0));
-}
-
// If the selected reader is blocking our pruning progress, decide on
// what kind of mitigation is necessary to unblock the situation.
void LogBuffer::kickMe(LogTimeEntry* me, log_id_t id, unsigned long pruneRows) {
@@ -726,8 +662,6 @@
}
times++;
}
- log_time watermark(log_time::tv_sec_max, log_time::tv_nsec_max);
- if (oldest) watermark = oldest->mStart - pruneMargin;
LogBufferElementCollection::iterator it;
@@ -749,9 +683,9 @@
mLastSet[id] = true;
}
- if (oldest && (watermark <= element->getRealTime())) {
- busy = isBusy(watermark);
- if (busy) kickMe(oldest, id, pruneRows);
+ if (oldest && oldest->mStart <= element->getSequence()) {
+ busy = true;
+ kickMe(oldest, id, pruneRows);
break;
}
@@ -837,8 +771,8 @@
while (it != mLogElements.end()) {
LogBufferElement* element = *it;
- if (oldest && (watermark <= element->getRealTime())) {
- busy = isBusy(watermark);
+ if (oldest && oldest->mStart <= element->getSequence()) {
+ busy = true;
// Do not let chatty eliding trigger any reader mitigation
break;
}
@@ -989,9 +923,9 @@
mLastSet[id] = true;
}
- if (oldest && (watermark <= element->getRealTime())) {
- busy = isBusy(watermark);
- if (!whitelist && busy) kickMe(oldest, id, pruneRows);
+ if (oldest && oldest->mStart <= element->getSequence()) {
+ busy = true;
+ if (!whitelist) kickMe(oldest, id, pruneRows);
break;
}
@@ -1022,9 +956,9 @@
mLastSet[id] = true;
}
- if (oldest && (watermark <= element->getRealTime())) {
- busy = isBusy(watermark);
- if (busy) kickMe(oldest, id, pruneRows);
+ if (oldest && oldest->mStart <= element->getSequence()) {
+ busy = true;
+ kickMe(oldest, id, pruneRows);
break;
}
@@ -1111,60 +1045,36 @@
return retval;
}
-log_time LogBuffer::flushTo(SocketClient* reader, const log_time& start,
- pid_t* lastTid, bool privileged, bool security,
- int (*filter)(const LogBufferElement* element,
- void* arg),
- void* arg) {
+uint64_t LogBuffer::flushTo(SocketClient* reader, uint64_t start, pid_t* lastTid, bool privileged,
+ bool security,
+ int (*filter)(const LogBufferElement* element, void* arg), void* arg) {
LogBufferElementCollection::iterator it;
uid_t uid = reader->getUid();
rdlock();
- if (start == log_time::EPOCH) {
+ if (start <= 1) {
// client wants to start from the beginning
it = mLogElements.begin();
} else {
- // Cap to 300 iterations we look back for out-of-order entries.
- size_t count = 300;
-
// Client wants to start from some specified time. Chances are
// we are better off starting from the end of the time sorted list.
- LogBufferElementCollection::iterator last;
- for (last = it = mLogElements.end(); it != mLogElements.begin();
+ for (it = mLogElements.end(); it != mLogElements.begin();
/* do nothing */) {
--it;
LogBufferElement* element = *it;
- if (element->getRealTime() > start) {
- last = it;
- } else if (element->getRealTime() == start) {
- last = ++it;
- break;
- } else if (!--count) {
+ if (element->getSequence() <= start) {
+ it++;
break;
}
}
- it = last;
}
- log_time curr = start;
+ uint64_t curr = start;
- LogBufferElement* lastElement = nullptr; // iterator corruption paranoia
- static const size_t maxSkip = 4194304; // maximum entries to skip
- size_t skip = maxSkip;
for (; it != mLogElements.end(); ++it) {
LogBufferElement* element = *it;
- if (!--skip) {
- android::prdebug("reader.per: too many elements skipped");
- break;
- }
- if (element == lastElement) {
- android::prdebug("reader.per: identical elements");
- break;
- }
- lastElement = element;
-
if (!privileged && (element->getUid() != uid)) {
continue;
}
@@ -1205,7 +1115,6 @@
return curr;
}
- skip = maxSkip;
rdlock();
}
unlock();
diff --git a/logd/LogBuffer.h b/logd/LogBuffer.h
index c2d5b97..16225a5 100644
--- a/logd/LogBuffer.h
+++ b/logd/LogBuffer.h
@@ -118,11 +118,10 @@
// lastTid is an optional context to help detect if the last previous
// valid message was from the same source so we can differentiate chatty
// filter types (identical or expired)
- log_time flushTo(SocketClient* writer, const log_time& start,
+ uint64_t flushTo(SocketClient* writer, uint64_t start,
pid_t* lastTid, // &lastTid[LOG_ID_MAX] or nullptr
bool privileged, bool security,
- int (*filter)(const LogBufferElement* element,
- void* arg) = nullptr,
+ int (*filter)(const LogBufferElement* element, void* arg) = nullptr,
void* arg = nullptr);
bool clear(log_id_t id, uid_t uid = AID_ROOT);
@@ -175,10 +174,8 @@
private:
static constexpr size_t minPrune = 4;
static constexpr size_t maxPrune = 256;
- static const log_time pruneMargin;
void maybePrune(log_id_t id);
- bool isBusy(log_time watermark);
void kickMe(LogTimeEntry* me, log_id_t id, unsigned long pruneRows);
bool prune(log_id_t id, unsigned long pruneRows, uid_t uid = AID_ROOT);
diff --git a/logd/LogBufferElement.cpp b/logd/LogBufferElement.cpp
index ec81933..3714800 100644
--- a/logd/LogBufferElement.cpp
+++ b/logd/LogBufferElement.cpp
@@ -30,15 +30,15 @@
#include "LogReader.h"
#include "LogUtils.h"
-const log_time LogBufferElement::FLUSH_ERROR((uint32_t)-1, (uint32_t)-1);
+const uint64_t LogBufferElement::FLUSH_ERROR(0);
atomic_int_fast64_t LogBufferElement::sequence(1);
-LogBufferElement::LogBufferElement(log_id_t log_id, log_time realtime,
- uid_t uid, pid_t pid, pid_t tid,
- const char* msg, uint16_t len)
+LogBufferElement::LogBufferElement(log_id_t log_id, log_time realtime, uid_t uid, pid_t pid,
+ pid_t tid, const char* msg, uint16_t len)
: mUid(uid),
mPid(pid),
mTid(tid),
+ mSequence(sequence.fetch_add(1, memory_order_relaxed)),
mRealTime(realtime),
mMsgLen(len),
mLogId(log_id),
@@ -51,6 +51,7 @@
: mUid(elem.mUid),
mPid(elem.mPid),
mTid(elem.mTid),
+ mSequence(elem.mSequence),
mRealTime(elem.mRealTime),
mMsgLen(elem.mMsgLen),
mLogId(elem.mLogId),
@@ -244,7 +245,7 @@
return retval;
}
-log_time LogBufferElement::flushTo(SocketClient* reader, LogBuffer* parent, bool lastSame) {
+uint64_t LogBufferElement::flushTo(SocketClient* reader, LogBuffer* parent, bool lastSame) {
struct logger_entry entry = {};
entry.hdr_size = sizeof(struct logger_entry);
@@ -263,7 +264,7 @@
if (mDropped) {
entry.len = populateDroppedMessage(buffer, parent, lastSame);
- if (!entry.len) return mRealTime;
+ if (!entry.len) return mSequence;
iovec[1].iov_base = buffer;
} else {
entry.len = mMsgLen;
@@ -271,9 +272,7 @@
}
iovec[1].iov_len = entry.len;
- log_time retval = reader->sendDatav(iovec, 1 + (entry.len != 0))
- ? FLUSH_ERROR
- : mRealTime;
+ uint64_t retval = reader->sendDatav(iovec, 1 + (entry.len != 0)) ? FLUSH_ERROR : mSequence;
if (buffer) free(buffer);
diff --git a/logd/LogBufferElement.h b/logd/LogBufferElement.h
index fd790e4..434b7db 100644
--- a/logd/LogBufferElement.h
+++ b/logd/LogBufferElement.h
@@ -39,6 +39,7 @@
const uint32_t mUid;
const uint32_t mPid;
const uint32_t mTid;
+ uint64_t mSequence;
log_time mRealTime;
union {
char* mMsg; // mDropped == false
@@ -90,10 +91,12 @@
const char* getMsg() const {
return mDropped ? nullptr : mMsg;
}
+ uint64_t getSequence() const { return mSequence; }
+ static uint64_t getCurrentSequence() { return sequence.load(memory_order_relaxed); }
log_time getRealTime(void) const {
return mRealTime;
}
- static const log_time FLUSH_ERROR;
- log_time flushTo(SocketClient* writer, LogBuffer* parent, bool lastSame);
+ static const uint64_t FLUSH_ERROR;
+ uint64_t flushTo(SocketClient* writer, LogBuffer* parent, bool lastSame);
};
diff --git a/logd/LogReader.cpp b/logd/LogReader.cpp
index 9db8c00..f79d39c 100644
--- a/logd/LogReader.cpp
+++ b/logd/LogReader.cpp
@@ -89,8 +89,7 @@
static const char _timeout[] = " timeout=";
cp = strstr(buffer, _timeout);
if (cp) {
- timeout = atol(cp + sizeof(_timeout) - 1) * NS_PER_SEC +
- log_time(CLOCK_REALTIME).nsec();
+ timeout = atol(cp + sizeof(_timeout) - 1) * NS_PER_SEC + log_time(CLOCK_MONOTONIC).nsec();
}
unsigned int logMask = -1;
@@ -130,70 +129,53 @@
nonBlock = true;
}
- log_time sequence = start;
- //
- // This somewhat expensive data validation operation is required
- // for non-blocking, with timeout. The incoming timestamp must be
- // in range of the list, if not, return immediately. This is
- // used to prevent us from from getting stuck in timeout processing
- // with an invalid time.
- //
- // Find if time is really present in the logs, monotonic or real, implicit
- // conversion from monotonic or real as necessary to perform the check.
- // Exit in the check loop ASAP as you find a transition from older to
- // newer, but use the last entry found to ensure overlap.
- //
- if (nonBlock && (sequence != log_time::EPOCH) && timeout) {
- class LogFindStart { // A lambda by another name
- private:
+ uint64_t sequence = 1;
+ // Convert realtime to sequence number
+ if (start != log_time::EPOCH) {
+ class LogFindStart {
const pid_t mPid;
const unsigned mLogMask;
- bool mStartTimeSet;
- log_time mStart;
- log_time& mSequence;
- log_time mLast;
- bool mIsMonotonic;
+ bool startTimeSet;
+ const log_time start;
+ uint64_t& sequence;
+ uint64_t last;
+ bool isMonotonic;
- public:
- LogFindStart(pid_t pid, unsigned logMask, log_time& sequence,
+ public:
+ LogFindStart(unsigned logMask, pid_t pid, log_time start, uint64_t& sequence,
bool isMonotonic)
: mPid(pid),
mLogMask(logMask),
- mStartTimeSet(false),
- mStart(sequence),
- mSequence(sequence),
- mLast(sequence),
- mIsMonotonic(isMonotonic) {
- }
+ startTimeSet(false),
+ start(start),
+ sequence(sequence),
+ last(sequence),
+ isMonotonic(isMonotonic) {}
static int callback(const LogBufferElement* element, void* obj) {
LogFindStart* me = reinterpret_cast<LogFindStart*>(obj);
if ((!me->mPid || (me->mPid == element->getPid())) &&
(me->mLogMask & (1 << element->getLogId()))) {
- log_time real = element->getRealTime();
- if (me->mStart == real) {
- me->mSequence = real;
- me->mStartTimeSet = true;
+ if (me->start == element->getRealTime()) {
+ me->sequence = element->getSequence();
+ me->startTimeSet = true;
return -1;
- } else if (!me->mIsMonotonic || android::isMonotonic(real)) {
- if (me->mStart < real) {
- me->mSequence = me->mLast;
- me->mStartTimeSet = true;
+ } else if (!me->isMonotonic || android::isMonotonic(element->getRealTime())) {
+ if (me->start < element->getRealTime()) {
+ me->sequence = me->last;
+ me->startTimeSet = true;
return -1;
}
- me->mLast = real;
+ me->last = element->getSequence();
} else {
- me->mLast = real;
+ me->last = element->getSequence();
}
}
return false;
}
- bool found() {
- return mStartTimeSet;
- }
-
- } logFindStart(pid, logMask, sequence,
+ bool found() { return startTimeSet; }
+ } logFindStart(logMask, pid, start, sequence,
logbuf().isMonotonic() && android::isMonotonic(start));
logbuf().flushTo(cli, sequence, nullptr, FlushCommand::hasReadLogs(cli),
@@ -201,24 +183,27 @@
logFindStart.callback, &logFindStart);
if (!logFindStart.found()) {
- doSocketDelete(cli);
- return false;
+ if (nonBlock) {
+ doSocketDelete(cli);
+ return false;
+ }
+ sequence = LogBufferElement::getCurrentSequence();
}
}
android::prdebug(
- "logdr: UID=%d GID=%d PID=%d %c tail=%lu logMask=%x pid=%d "
- "start=%" PRIu64 "ns timeout=%" PRIu64 "ns\n",
- cli->getUid(), cli->getGid(), cli->getPid(), nonBlock ? 'n' : 'b', tail,
- logMask, (int)pid, sequence.nsec(), timeout);
+ "logdr: UID=%d GID=%d PID=%d %c tail=%lu logMask=%x pid=%d "
+ "start=%" PRIu64 "ns timeout=%" PRIu64 "ns\n",
+ cli->getUid(), cli->getGid(), cli->getPid(), nonBlock ? 'n' : 'b', tail, logMask,
+ (int)pid, start.nsec(), timeout);
- if (sequence == log_time::EPOCH) {
+ if (start == log_time::EPOCH) {
timeout = 0;
}
LogTimeEntry::wrlock();
- auto entry = std::make_unique<LogTimeEntry>(
- *this, cli, nonBlock, tail, logMask, pid, sequence, timeout);
+ auto entry = std::make_unique<LogTimeEntry>(*this, cli, nonBlock, tail, logMask, pid, start,
+ sequence, timeout);
if (!entry->startReader_Locked()) {
LogTimeEntry::unlock();
return false;
diff --git a/logd/LogTimes.cpp b/logd/LogTimes.cpp
index 208d67f..ed8d2f5 100644
--- a/logd/LogTimes.cpp
+++ b/logd/LogTimes.cpp
@@ -18,8 +18,6 @@
#include <string.h>
#include <sys/prctl.h>
-#include <private/android_logger.h>
-
#include "FlushCommand.h"
#include "LogBuffer.h"
#include "LogReader.h"
@@ -27,9 +25,9 @@
pthread_mutex_t LogTimeEntry::timesLock = PTHREAD_MUTEX_INITIALIZER;
-LogTimeEntry::LogTimeEntry(LogReader& reader, SocketClient* client,
- bool nonBlock, unsigned long tail, log_mask_t logMask,
- pid_t pid, log_time start, uint64_t timeout)
+LogTimeEntry::LogTimeEntry(LogReader& reader, SocketClient* client, bool nonBlock,
+ unsigned long tail, log_mask_t logMask, pid_t pid, log_time start_time,
+ uint64_t start, uint64_t timeout)
: leadingDropped(false),
mReader(reader),
mLogMask(logMask),
@@ -38,9 +36,9 @@
mTail(tail),
mIndex(0),
mClient(client),
+ mStartTime(start_time),
mStart(start),
- mNonBlock(nonBlock),
- mEnd(log_time(android_log_clockid())) {
+ mNonBlock(nonBlock) {
mTimeout.tv_sec = timeout / NS_PER_SEC;
mTimeout.tv_nsec = timeout % NS_PER_SEC;
memset(mLastTid, 0, sizeof(mLastTid));
@@ -81,12 +79,12 @@
wrlock();
- log_time start = me->mStart;
+ uint64_t start = me->mStart;
while (!me->mRelease) {
if (me->mTimeout.tv_sec || me->mTimeout.tv_nsec) {
- if (pthread_cond_timedwait(&me->threadTriggeredCondition,
- ×Lock, &me->mTimeout) == ETIMEDOUT) {
+ if (pthread_cond_clockwait(&me->threadTriggeredCondition, ×Lock, CLOCK_MONOTONIC,
+ &me->mTimeout) == ETIMEDOUT) {
me->mTimeout.tv_sec = 0;
me->mTimeout.tv_nsec = 0;
}
@@ -105,13 +103,22 @@
start = logbuf.flushTo(client, start, me->mLastTid, privileged,
security, FilterSecondPass, me);
+ // We only ignore entries before the original start time for the first flushTo(), if we
+ // get entries after this first flush before the original start time, then the client
+ // wouldn't have seen them.
+ // Note: this is still racy and may skip out of order events that came in since the last
+ // time the client disconnected and then reconnected with the new start time. The long term
+ // solution here is that clients must request events since a specific sequence number.
+ me->mStartTime.tv_sec = 0;
+ me->mStartTime.tv_nsec = 0;
+
wrlock();
if (start == LogBufferElement::FLUSH_ERROR) {
break;
}
- me->mStart = start + log_time(0, 1);
+ me->mStart = start + 1;
if (me->mNonBlock || me->mRelease) {
break;
@@ -158,11 +165,11 @@
}
if (me->mCount == 0) {
- me->mStart = element->getRealTime();
+ me->mStart = element->getSequence();
}
- if ((!me->mPid || (me->mPid == element->getPid())) &&
- (me->isWatching(element->getLogId()))) {
+ if ((!me->mPid || me->mPid == element->getPid()) && me->isWatching(element->getLogId()) &&
+ (me->mStartTime == log_time::EPOCH || me->mStartTime <= element->getRealTime())) {
++me->mCount;
}
@@ -177,7 +184,7 @@
LogTimeEntry::wrlock();
- me->mStart = element->getRealTime();
+ me->mStart = element->getSequence();
if (me->skipAhead[element->getLogId()]) {
me->skipAhead[element->getLogId()]--;
@@ -204,6 +211,10 @@
goto skip;
}
+ if (me->mStartTime != log_time::EPOCH && element->getRealTime() <= me->mStartTime) {
+ goto skip;
+ }
+
if (me->mRelease) {
goto stop;
}
diff --git a/logd/LogTimes.h b/logd/LogTimes.h
index 9f6f203..a99c73b 100644
--- a/logd/LogTimes.h
+++ b/logd/LogTimes.h
@@ -49,16 +49,16 @@
unsigned long mTail;
unsigned long mIndex;
- public:
- LogTimeEntry(LogReader& reader, SocketClient* client, bool nonBlock,
- unsigned long tail, log_mask_t logMask, pid_t pid,
- log_time start, uint64_t timeout);
+ public:
+ LogTimeEntry(LogReader& reader, SocketClient* client, bool nonBlock, unsigned long tail,
+ log_mask_t logMask, pid_t pid, log_time start_time, uint64_t sequence,
+ uint64_t timeout);
SocketClient* mClient;
- log_time mStart;
- struct timespec mTimeout;
+ log_time mStartTime;
+ uint64_t mStart;
+ struct timespec mTimeout; // CLOCK_MONOTONIC based timeout used for log wrapping.
const bool mNonBlock;
- const log_time mEnd; // only relevant if mNonBlock
// Protect List manipulations
static void wrlock(void) {
diff --git a/logd/tests/logd_test.cpp b/logd/tests/logd_test.cpp
index c7f3480..d57b79e 100644
--- a/logd/tests/logd_test.cpp
+++ b/logd/tests/logd_test.cpp
@@ -246,7 +246,7 @@
std::cerr << std::flush;
fflush(stdout);
fflush(stderr);
- EXPECT_EQ(sizeof(logger_entry), msg->entry.hdr_size);
+ EXPECT_GE(msg->entry.hdr_size, sizeof(logger_entry));
fprintf(stderr, "%s: [%u] ", prefix, msg->len());
fprintf(stderr, "hdr_size=%u ", msg->entry.hdr_size);
diff --git a/rootdir/init.rc b/rootdir/init.rc
index adfdb7b..6564e8f 100644
--- a/rootdir/init.rc
+++ b/rootdir/init.rc
@@ -1063,6 +1063,12 @@
start vold
exec - system system -- /system/bin/vdc checkpoint resetCheckpoint
exec - system system -- /system/bin/vdc checkpoint markBootAttempt
+ # Unmount /data_mirror mounts in the reverse order of corresponding mounts.
+ umount /data_mirror/data_ce/null/0
+ umount /data_mirror/data_ce/null
+ umount /data_mirror/data_de/null
+ umount /data_mirror/cur_profiles
+ umount /data_mirror
remount_userdata
start bootanim
diff --git a/trusty/keymaster/4.0/android.hardware.keymaster@4.0-service.trusty.xml b/trusty/keymaster/4.0/android.hardware.keymaster@4.0-service.trusty.xml
index aa30707..65eb854 100644
--- a/trusty/keymaster/4.0/android.hardware.keymaster@4.0-service.trusty.xml
+++ b/trusty/keymaster/4.0/android.hardware.keymaster@4.0-service.trusty.xml
@@ -2,10 +2,6 @@
<hal format="hidl">
<name>android.hardware.keymaster</name>
<transport>hwbinder</transport>
- <version>4.0</version>
- <interface>
- <name>IKeymasterDevice</name>
- <instance>default</instance>
- </interface>
+ <fqname>@4.0::IKeymasterDevice/default</fqname>
</hal>
</manifest>