adb: add interfaces for Encoder/Decoder.
More groundwork to support more compression algorithms.
Bug: https://issuetracker.google.com/150827486
Test: python3 -m unittest test_device.FileOperationsTest{Uncompressed,Brotli}
Change-Id: I638493083b83e3f6c6854b631471e9d6b50bd79f
diff --git a/adb/client/adb_install.cpp b/adb/client/adb_install.cpp
index 092a866..f022b8b 100644
--- a/adb/client/adb_install.cpp
+++ b/adb/client/adb_install.cpp
@@ -290,7 +290,7 @@
}
}
- if (do_sync_push(apk_file, apk_dest.c_str(), false, true)) {
+ if (do_sync_push(apk_file, apk_dest.c_str(), false, CompressionType::Any)) {
result = pm_command(argc, argv);
delete_device_file(apk_dest);
}
diff --git a/adb/client/bugreport.cpp b/adb/client/bugreport.cpp
index ab93f7d..e162aaa 100644
--- a/adb/client/bugreport.cpp
+++ b/adb/client/bugreport.cpp
@@ -282,5 +282,5 @@
bool Bugreport::DoSyncPull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs,
const char* name) {
- return do_sync_pull(srcs, dst, copy_attrs, false, name);
+ return do_sync_pull(srcs, dst, copy_attrs, CompressionType::None, name);
}
diff --git a/adb/client/commandline.cpp b/adb/client/commandline.cpp
index 04b250d..9078ae9 100644
--- a/adb/client/commandline.cpp
+++ b/adb/client/commandline.cpp
@@ -129,20 +129,20 @@
" reverse --remove-all remove all reverse socket connections from device\n"
"\n"
"file transfer:\n"
- " push [--sync] [-zZ] LOCAL... REMOTE\n"
+ " push [--sync] [-z ALGORITHM] [-Z] LOCAL... REMOTE\n"
" copy local files/directories to device\n"
" --sync: only push files that are newer on the host than the device\n"
- " -z: enable compression\n"
+ " -z: enable compression with a specified algorithm (any, none, brotli)\n"
" -Z: disable compression\n"
- " pull [-azZ] REMOTE... LOCAL\n"
+ " pull [-a] [-z ALGORITHM] [-Z] REMOTE... LOCAL\n"
" copy files/dirs from device\n"
" -a: preserve file timestamp and mode\n"
- " -z: enable compression\n"
+ " -z: enable compression with a specified algorithm (any, none, brotli)\n"
" -Z: disable compression\n"
- " sync [-lzZ] [all|data|odm|oem|product|system|system_ext|vendor]\n"
+ " sync [-l] [-z ALGORITHM] [-Z] [all|data|odm|oem|product|system|system_ext|vendor]\n"
" sync a local build from $ANDROID_PRODUCT_OUT to the device (default all)\n"
" -l: list files that would be copied, but don't copy them\n"
- " -z: enable compression\n"
+ " -z: enable compression with a specified algorithm (any, none, brotli)\n"
" -Z: disable compression\n"
"\n"
"shell:\n"
@@ -1314,12 +1314,34 @@
return 0;
}
+static CompressionType parse_compression_type(const std::string& str, bool allow_numbers) {
+ if (allow_numbers) {
+ if (str == "0") {
+ return CompressionType::None;
+ } else if (str == "1") {
+ return CompressionType::Any;
+ }
+ }
+
+ if (str == "any") {
+ return CompressionType::Any;
+ } else if (str == "none") {
+ return CompressionType::None;
+ }
+
+ if (str == "brotli") {
+ return CompressionType::Brotli;
+ }
+
+ error_exit("unexpected compression type %s", str.c_str());
+}
+
static void parse_push_pull_args(const char** arg, int narg, std::vector<const char*>* srcs,
- const char** dst, bool* copy_attrs, bool* sync, bool* compressed) {
+ const char** dst, bool* copy_attrs, bool* sync,
+ CompressionType* compression) {
*copy_attrs = false;
- const char* adb_compression = getenv("ADB_COMPRESSION");
- if (adb_compression && strcmp(adb_compression, "0") == 0) {
- *compressed = false;
+ if (const char* adb_compression = getenv("ADB_COMPRESSION")) {
+ *compression = parse_compression_type(adb_compression, true);
}
srcs->clear();
@@ -1333,13 +1355,13 @@
} else if (!strcmp(*arg, "-a")) {
*copy_attrs = true;
} else if (!strcmp(*arg, "-z")) {
- if (compressed != nullptr) {
- *compressed = true;
+ if (narg < 2) {
+ error_exit("-z requires an argument");
}
+ *compression = parse_compression_type(*++arg, false);
+ --narg;
} else if (!strcmp(*arg, "-Z")) {
- if (compressed != nullptr) {
- *compressed = false;
- }
+ *compression = CompressionType::None;
} else if (!strcmp(*arg, "--sync")) {
if (sync != nullptr) {
*sync = true;
@@ -1894,22 +1916,22 @@
} else if (!strcmp(argv[0], "push")) {
bool copy_attrs = false;
bool sync = false;
- bool compressed = true;
+ CompressionType compression = CompressionType::Any;
std::vector<const char*> srcs;
const char* dst = nullptr;
- parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync, &compressed);
+ parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync, &compression);
if (srcs.empty() || !dst) error_exit("push requires an argument");
- return do_sync_push(srcs, dst, sync, compressed) ? 0 : 1;
+ return do_sync_push(srcs, dst, sync, compression) ? 0 : 1;
} else if (!strcmp(argv[0], "pull")) {
bool copy_attrs = false;
- bool compressed = true;
+ CompressionType compression = CompressionType::Any;
std::vector<const char*> srcs;
const char* dst = ".";
- parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr, &compressed);
+ parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr, &compression);
if (srcs.empty()) error_exit("pull requires an argument");
- return do_sync_pull(srcs, dst, copy_attrs, compressed) ? 0 : 1;
+ return do_sync_pull(srcs, dst, copy_attrs, compression) ? 0 : 1;
} else if (!strcmp(argv[0], "install")) {
if (argc < 2) error_exit("install requires an argument");
return install_app(argc, argv);
@@ -1925,27 +1947,26 @@
} else if (!strcmp(argv[0], "sync")) {
std::string src;
bool list_only = false;
- bool compressed = true;
+ CompressionType compression = CompressionType::Any;
- const char* adb_compression = getenv("ADB_COMPRESSION");
- if (adb_compression && strcmp(adb_compression, "0") == 0) {
- compressed = false;
+ if (const char* adb_compression = getenv("ADB_COMPRESSION"); adb_compression) {
+ compression = parse_compression_type(adb_compression, true);
}
int opt;
- while ((opt = getopt(argc, const_cast<char**>(argv), "lzZ")) != -1) {
+ while ((opt = getopt(argc, const_cast<char**>(argv), "lz:Z")) != -1) {
switch (opt) {
case 'l':
list_only = true;
break;
case 'z':
- compressed = true;
+ compression = parse_compression_type(optarg, false);
break;
case 'Z':
- compressed = false;
+ compression = CompressionType::None;
break;
default:
- error_exit("usage: adb sync [-lzZ] [PARTITION]");
+ error_exit("usage: adb sync [-l] [-z ALGORITHM] [-Z] [PARTITION]");
}
}
@@ -1954,7 +1975,7 @@
} else if (optind + 1 == argc) {
src = argv[optind];
} else {
- error_exit("usage: adb sync [-lzZ] [PARTITION]");
+ error_exit("usage: adb sync [-l] [-z ALGORITHM] [-Z] [PARTITION]");
}
std::vector<std::string> partitions{"data", "odm", "oem", "product",
@@ -1965,7 +1986,7 @@
std::string src_dir{product_file(partition)};
if (!directory_exists(src_dir)) continue;
found = true;
- if (!do_sync_sync(src_dir, "/" + partition, list_only, compressed)) return 1;
+ if (!do_sync_sync(src_dir, "/" + partition, list_only, compression)) return 1;
}
}
if (!found) error_exit("don't know how to sync %s partition", src.c_str());
diff --git a/adb/client/fastdeploy.cpp b/adb/client/fastdeploy.cpp
index de82e14..37f1a90 100644
--- a/adb/client/fastdeploy.cpp
+++ b/adb/client/fastdeploy.cpp
@@ -112,7 +112,7 @@
// but can't be removed until after the push.
unix_close(tf.release());
- if (!do_sync_push(srcs, dst, sync, true)) {
+ if (!do_sync_push(srcs, dst, sync, CompressionType::Any)) {
error_exit("Failed to push fastdeploy agent to device.");
}
}
diff --git a/adb/client/file_sync_client.cpp b/adb/client/file_sync_client.cpp
index 2ed58b2..c71880c 100644
--- a/adb/client/file_sync_client.cpp
+++ b/adb/client/file_sync_client.cpp
@@ -34,6 +34,7 @@
#include <memory>
#include <sstream>
#include <string>
+#include <variant>
#include <vector>
#include "sysdeps.h"
@@ -262,6 +263,18 @@
bool HaveSendRecv2() const { return have_sendrecv_v2_; }
bool HaveSendRecv2Brotli() const { return have_sendrecv_v2_brotli_; }
+ // Resolve a compression type which might be CompressionType::Any to a specific compression
+ // algorithm.
+ CompressionType ResolveCompressionType(CompressionType compression) const {
+ if (compression == CompressionType::Any) {
+ if (HaveSendRecv2Brotli()) {
+ return CompressionType::Brotli;
+ }
+ return CompressionType::None;
+ }
+ return compression;
+ }
+
const FeatureSet& Features() const { return features_; }
bool IsValid() { return fd >= 0; }
@@ -323,7 +336,7 @@
return WriteFdExactly(fd, buf.data(), buf.size());
}
- bool SendSend2(std::string_view path, mode_t mode, bool compressed) {
+ bool SendSend2(std::string_view path, mode_t mode, CompressionType compression) {
if (path.length() > 1024) {
Error("SendRequest failed: path too long: %zu", path.length());
errno = ENAMETOOLONG;
@@ -339,7 +352,18 @@
syncmsg msg;
msg.send_v2_setup.id = ID_SEND_V2;
msg.send_v2_setup.mode = mode;
- msg.send_v2_setup.flags = compressed ? kSyncFlagBrotli : kSyncFlagNone;
+ msg.send_v2_setup.flags = 0;
+ switch (compression) {
+ case CompressionType::None:
+ break;
+
+ case CompressionType::Brotli:
+ msg.send_v2_setup.flags = kSyncFlagBrotli;
+ break;
+
+ case CompressionType::Any:
+ LOG(FATAL) << "unexpected CompressionType::Any";
+ }
buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.send_v2_setup));
@@ -352,7 +376,7 @@
return WriteFdExactly(fd, buf.data(), buf.size());
}
- bool SendRecv2(const std::string& path) {
+ bool SendRecv2(const std::string& path, CompressionType compression) {
if (path.length() > 1024) {
Error("SendRequest failed: path too long: %zu", path.length());
errno = ENAMETOOLONG;
@@ -367,7 +391,18 @@
syncmsg msg;
msg.recv_v2_setup.id = ID_RECV_V2;
- msg.recv_v2_setup.flags = kSyncFlagBrotli;
+ msg.recv_v2_setup.flags = 0;
+ switch (compression) {
+ case CompressionType::None:
+ break;
+
+ case CompressionType::Brotli:
+ msg.recv_v2_setup.flags |= kSyncFlagBrotli;
+ break;
+
+ case CompressionType::Any:
+ LOG(FATAL) << "unexpected CompressionType::Any";
+ }
buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.recv_v2_setup));
@@ -533,9 +568,15 @@
return true;
}
- bool SendLargeFileCompressed(const std::string& path, mode_t mode, const std::string& lpath,
- const std::string& rpath, unsigned mtime) {
- if (!SendSend2(path, mode, true)) {
+ bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath,
+ const std::string& rpath, unsigned mtime, CompressionType compression) {
+ if (!HaveSendRecv2()) {
+ return SendLargeFileLegacy(path, mode, lpath, rpath, mtime);
+ }
+
+ compression = ResolveCompressionType(compression);
+
+ if (!SendSend2(path, mode, compression)) {
Error("failed to send ID_SEND_V2 message '%s': %s", path.c_str(), strerror(errno));
return false;
}
@@ -558,7 +599,21 @@
syncsendbuf sbuf;
sbuf.id = ID_DATA;
- BrotliEncoder<SYNC_DATA_MAX> encoder;
+ std::variant<std::monostate, NullEncoder, BrotliEncoder> encoder_storage;
+ Encoder* encoder = nullptr;
+ switch (compression) {
+ case CompressionType::None:
+ encoder = &encoder_storage.emplace<NullEncoder>(SYNC_DATA_MAX);
+ break;
+
+ case CompressionType::Brotli:
+ encoder = &encoder_storage.emplace<BrotliEncoder>(SYNC_DATA_MAX);
+ break;
+
+ case CompressionType::Any:
+ LOG(FATAL) << "unexpected CompressionType::Any";
+ }
+
bool sending = true;
while (sending) {
Block input(SYNC_DATA_MAX);
@@ -569,10 +624,10 @@
}
if (r == 0) {
- encoder.Finish();
+ encoder->Finish();
} else {
input.resize(r);
- encoder.Append(std::move(input));
+ encoder->Append(std::move(input));
RecordBytesTransferred(r);
bytes_copied += r;
ReportProgress(rpath, bytes_copied, total_size);
@@ -580,7 +635,7 @@
while (true) {
Block output;
- EncodeResult result = encoder.Encode(&output);
+ EncodeResult result = encoder->Encode(&output);
if (result == EncodeResult::Error) {
Error("compressing '%s' locally failed", lpath.c_str());
return false;
@@ -610,12 +665,8 @@
return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data));
}
- bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath,
- const std::string& rpath, unsigned mtime, bool compressed) {
- if (compressed && HaveSendRecv2Brotli()) {
- return SendLargeFileCompressed(path, mode, lpath, rpath, mtime);
- }
-
+ bool SendLargeFileLegacy(const std::string& path, mode_t mode, const std::string& lpath,
+ const std::string& rpath, unsigned mtime) {
std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode);
if (!SendRequest(ID_SEND_V1, path_and_mode)) {
Error("failed to send ID_SEND_V1 message '%s': %s", path_and_mode.c_str(),
@@ -921,7 +972,7 @@
}
static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::string& rpath,
- unsigned mtime, mode_t mode, bool sync, bool compressed) {
+ unsigned mtime, mode_t mode, bool sync, CompressionType compression) {
if (sync) {
struct stat st;
if (sync_lstat(sc, rpath, &st)) {
@@ -964,7 +1015,7 @@
return false;
}
} else {
- if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compressed)) {
+ if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compression)) {
return false;
}
}
@@ -1027,8 +1078,10 @@
}
static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpath, const char* name,
- uint64_t expected_size) {
- if (!sc.SendRecv2(rpath)) return false;
+ uint64_t expected_size, CompressionType compression) {
+ compression = sc.ResolveCompressionType(compression);
+
+ if (!sc.SendRecv2(rpath, compression)) return false;
adb_unlink(lpath);
unique_fd lfd(adb_creat(lpath, 0644));
@@ -1040,9 +1093,24 @@
uint64_t bytes_copied = 0;
Block buffer(SYNC_DATA_MAX);
- BrotliDecoder decoder(std::span(buffer.data(), buffer.size()));
- bool reading = true;
- while (reading) {
+ std::variant<std::monostate, NullDecoder, BrotliDecoder> decoder_storage;
+ Decoder* decoder = nullptr;
+
+ std::span buffer_span(buffer.data(), buffer.size());
+ switch (compression) {
+ case CompressionType::None:
+ decoder = &decoder_storage.emplace<NullDecoder>(buffer_span);
+ break;
+
+ case CompressionType::Brotli:
+ decoder = &decoder_storage.emplace<BrotliDecoder>(buffer_span);
+ break;
+
+ case CompressionType::Any:
+ LOG(FATAL) << "unexpected CompressionType::Any";
+ }
+
+ while (true) {
syncmsg msg;
if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) {
adb_unlink(lpath);
@@ -1050,33 +1118,32 @@
}
if (msg.data.id == ID_DONE) {
- adb_unlink(lpath);
- sc.Error("unexpected ID_DONE");
- return false;
- }
-
- if (msg.data.id != ID_DATA) {
+ if (!decoder->Finish()) {
+ sc.Error("unexpected ID_DONE");
+ return false;
+ }
+ } else if (msg.data.id != ID_DATA) {
adb_unlink(lpath);
sc.ReportCopyFailure(rpath, lpath, msg);
return false;
- }
+ } else {
+ if (msg.data.size > sc.max) {
+ sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max);
+ adb_unlink(lpath);
+ return false;
+ }
- if (msg.data.size > sc.max) {
- sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max);
- adb_unlink(lpath);
- return false;
+ Block block(msg.data.size);
+ if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) {
+ adb_unlink(lpath);
+ return false;
+ }
+ decoder->Append(std::move(block));
}
- Block block(msg.data.size);
- if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) {
- adb_unlink(lpath);
- return false;
- }
- decoder.Append(std::move(block));
-
while (true) {
std::span<char> output;
- DecodeResult result = decoder.Decode(&output);
+ DecodeResult result = decoder->Decode(&output);
if (result == DecodeResult::Error) {
sc.Error("decompress failed");
@@ -1102,33 +1169,19 @@
} else if (result == DecodeResult::MoreOutput) {
continue;
} else if (result == DecodeResult::Done) {
- reading = false;
- break;
+ sc.RecordFilesTransferred(1);
+ return true;
} else {
LOG(FATAL) << "invalid DecodeResult: " << static_cast<int>(result);
}
}
}
-
- syncmsg msg;
- if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) {
- sc.Error("failed to read ID_DONE");
- return false;
- }
-
- if (msg.data.id != ID_DONE) {
- sc.Error("unexpected message after transfer: id = %d (expected ID_DONE)", msg.data.id);
- return false;
- }
-
- sc.RecordFilesTransferred(1);
- return true;
}
static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, const char* name,
- uint64_t expected_size, bool compressed) {
- if (sc.HaveSendRecv2() && compressed) {
- return sync_recv_v2(sc, rpath, lpath, name, expected_size);
+ uint64_t expected_size, CompressionType compression) {
+ if (sc.HaveSendRecv2()) {
+ return sync_recv_v2(sc, rpath, lpath, name, expected_size, compression);
} else {
return sync_recv_v1(sc, rpath, lpath, name, expected_size);
}
@@ -1210,7 +1263,8 @@
}
static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::string rpath,
- bool check_timestamps, bool list_only, bool compressed) {
+ bool check_timestamps, bool list_only,
+ CompressionType compression) {
sc.NewTransfer();
// Make sure that both directory paths end in a slash.
@@ -1292,7 +1346,7 @@
if (list_only) {
sc.Println("would push: %s -> %s", ci.lpath.c_str(), ci.rpath.c_str());
} else {
- if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compressed)) {
+ if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compression)) {
return false;
}
}
@@ -1308,7 +1362,7 @@
}
bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync,
- bool compressed) {
+ CompressionType compression) {
SyncConnection sc;
if (!sc.IsValid()) return false;
@@ -1373,7 +1427,7 @@
dst_dir.append(android::base::Basename(src_path));
}
- success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compressed);
+ success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compression);
continue;
} else if (!should_push_file(st.st_mode)) {
sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, st.st_mode);
@@ -1394,7 +1448,7 @@
sc.NewTransfer();
sc.SetExpectedTotalBytes(st.st_size);
- success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compressed);
+ success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compression);
sc.ReportTransferRate(src_path, TransferDirection::push);
}
@@ -1480,7 +1534,7 @@
}
static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::string lpath,
- bool copy_attrs, bool compressed) {
+ bool copy_attrs, CompressionType compression) {
sc.NewTransfer();
// Make sure that both directory paths end in a slash.
@@ -1510,7 +1564,7 @@
continue;
}
- if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compressed)) {
+ if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compression)) {
return false;
}
@@ -1528,7 +1582,7 @@
}
bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs,
- bool compressed, const char* name) {
+ CompressionType compression, const char* name) {
SyncConnection sc;
if (!sc.IsValid()) return false;
@@ -1602,7 +1656,7 @@
dst_dir.append(android::base::Basename(src_path));
}
- success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compressed);
+ success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compression);
continue;
} else if (!should_pull_file(src_st.st_mode)) {
sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, src_st.st_mode);
@@ -1621,7 +1675,7 @@
sc.NewTransfer();
sc.SetExpectedTotalBytes(src_st.st_size);
- if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compressed)) {
+ if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compression)) {
success = false;
continue;
}
@@ -1638,11 +1692,11 @@
}
bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only,
- bool compressed) {
+ CompressionType compression) {
SyncConnection sc;
if (!sc.IsValid()) return false;
- bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compressed);
+ bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compression);
if (!list_only) {
sc.ReportOverallTransferRate(TransferDirection::push);
}
diff --git a/adb/client/file_sync_client.h b/adb/client/file_sync_client.h
index de3f192..aab2e3f 100644
--- a/adb/client/file_sync_client.h
+++ b/adb/client/file_sync_client.h
@@ -19,11 +19,13 @@
#include <string>
#include <vector>
+#include "file_sync_protocol.h"
+
bool do_sync_ls(const char* path);
bool do_sync_push(const std::vector<const char*>& srcs, const char* dst, bool sync,
- bool compressed);
+ CompressionType compression);
bool do_sync_pull(const std::vector<const char*>& srcs, const char* dst, bool copy_attrs,
- bool compressed, const char* name = nullptr);
+ CompressionType compression, const char* name = nullptr);
bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only,
- bool compressed);
+ CompressionType compression);
diff --git a/adb/compression_utils.h b/adb/compression_utils.h
index c445095..f349697 100644
--- a/adb/compression_utils.h
+++ b/adb/compression_utils.h
@@ -16,8 +16,12 @@
#pragma once
+#include <algorithm>
+#include <memory>
#include <span>
+#include <android-base/logging.h>
+
#include <brotli/decode.h>
#include <brotli/encode.h>
@@ -37,15 +41,103 @@
MoreOutput,
};
-struct BrotliDecoder {
+struct Decoder {
+ void Append(Block&& block) { input_buffer_.append(std::move(block)); }
+ bool Finish() {
+ bool old = std::exchange(finished_, true);
+ if (old) {
+ LOG(FATAL) << "Decoder::Finish called while already finished?";
+ return false;
+ }
+ return true;
+ }
+
+ virtual DecodeResult Decode(std::span<char>* output) = 0;
+
+ protected:
+ Decoder(std::span<char> output_buffer) : output_buffer_(output_buffer) {}
+ ~Decoder() = default;
+
+ bool finished_ = false;
+ IOVector input_buffer_;
+ std::span<char> output_buffer_;
+};
+
+struct Encoder {
+ void Append(Block input) { input_buffer_.append(std::move(input)); }
+ bool Finish() {
+ bool old = std::exchange(finished_, true);
+ if (old) {
+ LOG(FATAL) << "Decoder::Finish called while already finished?";
+ return false;
+ }
+ return true;
+ }
+
+ virtual EncodeResult Encode(Block* output) = 0;
+
+ protected:
+ explicit Encoder(size_t output_block_size) : output_block_size_(output_block_size) {}
+ ~Encoder() = default;
+
+ const size_t output_block_size_;
+ bool finished_ = false;
+ IOVector input_buffer_;
+};
+
+struct NullDecoder final : public Decoder {
+ explicit NullDecoder(std::span<char> output_buffer) : Decoder(output_buffer) {}
+
+ DecodeResult Decode(std::span<char>* output) final {
+ size_t available_out = output_buffer_.size();
+ void* p = output_buffer_.data();
+ while (available_out > 0 && !input_buffer_.empty()) {
+ size_t len = std::min(available_out, input_buffer_.front_size());
+ p = mempcpy(p, input_buffer_.front_data(), len);
+ available_out -= len;
+ input_buffer_.drop_front(len);
+ }
+ *output = std::span(output_buffer_.data(), static_cast<char*>(p));
+ if (input_buffer_.empty()) {
+ return finished_ ? DecodeResult::Done : DecodeResult::NeedInput;
+ }
+ return DecodeResult::MoreOutput;
+ }
+};
+
+struct NullEncoder final : public Encoder {
+ explicit NullEncoder(size_t output_block_size) : Encoder(output_block_size) {}
+
+ EncodeResult Encode(Block* output) final {
+ output->clear();
+ output->resize(output_block_size_);
+
+ size_t available_out = output->size();
+ void* p = output->data();
+
+ while (available_out > 0 && !input_buffer_.empty()) {
+ size_t len = std::min(available_out, input_buffer_.front_size());
+ p = mempcpy(p, input_buffer_.front_data(), len);
+ available_out -= len;
+ input_buffer_.drop_front(len);
+ }
+
+ output->resize(output->size() - available_out);
+
+ if (input_buffer_.empty()) {
+ return finished_ ? EncodeResult::Done : EncodeResult::NeedInput;
+ }
+ return EncodeResult::MoreOutput;
+ }
+};
+
+struct BrotliDecoder final : public Decoder {
explicit BrotliDecoder(std::span<char> output_buffer)
- : output_buffer_(output_buffer),
+ : Decoder(output_buffer),
decoder_(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr),
BrotliDecoderDestroyInstance) {}
- void Append(Block&& block) { input_buffer_.append(std::move(block)); }
-
- DecodeResult Decode(std::span<char>* output) {
+ DecodeResult Decode(std::span<char>* output) final {
size_t available_in = input_buffer_.front_size();
const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());
@@ -63,7 +155,8 @@
switch (r) {
case BROTLI_DECODER_RESULT_SUCCESS:
- return DecodeResult::Done;
+ // We need to wait for ID_DONE from the other end.
+ return finished_ ? DecodeResult::Done : DecodeResult::NeedInput;
case BROTLI_DECODER_RESULT_ERROR:
return DecodeResult::Error;
case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
@@ -77,33 +170,29 @@
}
private:
- IOVector input_buffer_;
- std::span<char> output_buffer_;
std::unique_ptr<BrotliDecoderState, void (*)(BrotliDecoderState*)> decoder_;
};
-template <size_t OutputBlockSize>
-struct BrotliEncoder {
- explicit BrotliEncoder()
- : output_block_(OutputBlockSize),
- output_bytes_left_(OutputBlockSize),
+struct BrotliEncoder final : public Encoder {
+ explicit BrotliEncoder(size_t output_block_size)
+ : Encoder(output_block_size),
+ output_block_(output_block_size_),
+ output_bytes_left_(output_block_size_),
encoder_(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr),
BrotliEncoderDestroyInstance) {
BrotliEncoderSetParameter(encoder_.get(), BROTLI_PARAM_QUALITY, 1);
}
- void Append(Block input) { input_buffer_.append(std::move(input)); }
- void Finish() { finished_ = true; }
-
- EncodeResult Encode(Block* output) {
+ EncodeResult Encode(Block* output) final {
output->clear();
+
while (true) {
size_t available_in = input_buffer_.front_size();
const uint8_t* next_in = reinterpret_cast<const uint8_t*>(input_buffer_.front_data());
size_t available_out = output_bytes_left_;
- uint8_t* next_out = reinterpret_cast<uint8_t*>(output_block_.data() +
- (OutputBlockSize - output_bytes_left_));
+ uint8_t* next_out = reinterpret_cast<uint8_t*>(
+ output_block_.data() + (output_block_size_ - output_bytes_left_));
BrotliEncoderOperation op = BROTLI_OPERATION_PROCESS;
if (finished_) {
@@ -121,13 +210,13 @@
output_bytes_left_ = available_out;
if (BrotliEncoderIsFinished(encoder_.get())) {
- output_block_.resize(OutputBlockSize - output_bytes_left_);
+ output_block_.resize(output_block_size_ - output_bytes_left_);
*output = std::move(output_block_);
return EncodeResult::Done;
} else if (output_bytes_left_ == 0) {
*output = std::move(output_block_);
- output_block_.resize(OutputBlockSize);
- output_bytes_left_ = OutputBlockSize;
+ output_block_.resize(output_block_size_);
+ output_bytes_left_ = output_block_size_;
return EncodeResult::MoreOutput;
} else if (input_buffer_.empty()) {
return EncodeResult::NeedInput;
@@ -136,8 +225,6 @@
}
private:
- bool finished_ = false;
- IOVector input_buffer_;
Block output_block_;
size_t output_bytes_left_;
std::unique_ptr<BrotliEncoderState, void (*)(BrotliEncoderState*)> encoder_;
diff --git a/adb/daemon/file_sync_service.cpp b/adb/daemon/file_sync_service.cpp
index 5ccddea..dcd640b 100644
--- a/adb/daemon/file_sync_service.cpp
+++ b/adb/daemon/file_sync_service.cpp
@@ -35,6 +35,7 @@
#include <optional>
#include <span>
#include <string>
+#include <variant>
#include <vector>
#include <android-base/file.h>
@@ -266,29 +267,45 @@
return SendSyncFail(fd, StringPrintf("%s: %s", reason.c_str(), strerror(errno)));
}
-static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp) {
+static bool handle_send_file_data(borrowed_fd s, unique_fd fd, uint32_t* timestamp,
+ CompressionType compression) {
syncmsg msg;
- Block decode_buffer(SYNC_DATA_MAX);
- BrotliDecoder decoder(std::span(decode_buffer.data(), decode_buffer.size()));
+ Block buffer(SYNC_DATA_MAX);
+ std::span<char> buffer_span(buffer.data(), buffer.size());
+ std::variant<std::monostate, NullDecoder, BrotliDecoder> decoder_storage;
+ Decoder* decoder = nullptr;
+
+ switch (compression) {
+ case CompressionType::None:
+ decoder = &decoder_storage.emplace<NullDecoder>(buffer_span);
+ break;
+
+ case CompressionType::Brotli:
+ decoder = &decoder_storage.emplace<BrotliDecoder>(buffer_span);
+ break;
+
+ case CompressionType::Any:
+ LOG(FATAL) << "unexpected CompressionType::Any";
+ }
+
while (true) {
if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false;
- if (msg.data.id != ID_DATA) {
- if (msg.data.id == ID_DONE) {
- *timestamp = msg.data.size;
- return true;
- }
+ if (msg.data.id == ID_DONE) {
+ *timestamp = msg.data.size;
+ decoder->Finish();
+ } else if (msg.data.id == ID_DATA) {
+ Block block(msg.data.size);
+ if (!ReadFdExactly(s, block.data(), msg.data.size)) return false;
+ decoder->Append(std::move(block));
+ } else {
SendSyncFail(s, "invalid data message");
return false;
}
- Block block(msg.data.size);
- if (!ReadFdExactly(s, block.data(), msg.data.size)) return false;
- decoder.Append(std::move(block));
-
while (true) {
std::span<char> output;
- DecodeResult result = decoder.Decode(&output);
+ DecodeResult result = decoder->Decode(&output);
if (result == DecodeResult::Error) {
SendSyncFailErrno(s, "decompress failed");
return false;
@@ -304,7 +321,7 @@
} else if (result == DecodeResult::MoreOutput) {
continue;
} else if (result == DecodeResult::Done) {
- break;
+ return true;
} else {
LOG(FATAL) << "invalid DecodeResult: " << static_cast<int>(result);
}
@@ -314,37 +331,10 @@
__builtin_unreachable();
}
-static bool handle_send_file_uncompressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp,
- std::vector<char>& buffer) {
- syncmsg msg;
-
- while (true) {
- if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false;
-
- if (msg.data.id != ID_DATA) {
- if (msg.data.id == ID_DONE) {
- *timestamp = msg.data.size;
- return true;
- }
- SendSyncFail(s, "invalid data message");
- return false;
- }
-
- if (msg.data.size > buffer.size()) { // TODO: resize buffer?
- SendSyncFail(s, "oversize data message");
- return false;
- }
- if (!ReadFdExactly(s, &buffer[0], msg.data.size)) return false;
- if (!WriteFdExactly(fd, &buffer[0], msg.data.size)) {
- SendSyncFailErrno(s, "write failed");
- return false;
- }
- }
-}
-
static bool handle_send_file(borrowed_fd s, const char* path, uint32_t* timestamp, uid_t uid,
- gid_t gid, uint64_t capabilities, mode_t mode, bool compressed,
- std::vector<char>& buffer, bool do_unlink) {
+ gid_t gid, uint64_t capabilities, mode_t mode,
+ CompressionType compression, std::vector<char>& buffer,
+ bool do_unlink) {
int rc;
syncmsg msg;
@@ -389,14 +379,7 @@
D("[ Failed to fadvise: %s ]", strerror(rc));
}
- bool result;
- if (compressed) {
- result = handle_send_file_compressed(s, std::move(fd), timestamp);
- } else {
- result = handle_send_file_uncompressed(s, std::move(fd), timestamp, buffer);
- }
-
- if (!result) {
+ if (!handle_send_file_data(s, std::move(fd), timestamp, compression)) {
goto fail;
}
@@ -499,7 +482,7 @@
}
#endif
-static bool send_impl(int s, const std::string& path, mode_t mode, bool compressed,
+static bool send_impl(int s, const std::string& path, mode_t mode, CompressionType compression,
std::vector<char>& buffer) {
// Don't delete files before copying if they are not "regular" or symlinks.
struct stat st;
@@ -527,7 +510,7 @@
}
result = handle_send_file(s, path.c_str(), ×tamp, uid, gid, capabilities, mode,
- compressed, buffer, do_unlink);
+ compression, buffer, do_unlink);
}
if (!result) {
@@ -560,7 +543,7 @@
return false;
}
- return send_impl(s, path, mode, false, buffer);
+ return send_impl(s, path, mode, CompressionType::None, buffer);
}
static bool do_send_v2(int s, const std::string& path, std::vector<char>& buffer) {
@@ -574,45 +557,60 @@
PLOG(ERROR) << "failed to read send_v2 setup packet";
}
- bool compressed = false;
+ std::optional<CompressionType> compression;
if (msg.send_v2_setup.flags & kSyncFlagBrotli) {
msg.send_v2_setup.flags &= ~kSyncFlagBrotli;
- compressed = true;
+ if (compression) {
+ SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d",
+ msg.recv_v2_setup.flags));
+ return false;
+ }
+ compression = CompressionType::Brotli;
}
+
if (msg.send_v2_setup.flags) {
SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.send_v2_setup.flags));
return false;
}
errno = 0;
- return send_impl(s, path, msg.send_v2_setup.mode, compressed, buffer);
+ return send_impl(s, path, msg.send_v2_setup.mode, compression.value_or(CompressionType::None),
+ buffer);
}
-static bool recv_uncompressed(borrowed_fd s, unique_fd fd, std::vector<char>& buffer) {
- syncmsg msg;
- msg.data.id = ID_DATA;
- while (true) {
- int r = adb_read(fd.get(), &buffer[0], buffer.size() - sizeof(msg.data));
- if (r <= 0) {
- if (r == 0) break;
- SendSyncFailErrno(s, "read failed");
- return false;
- }
- msg.data.size = r;
+static bool recv_impl(borrowed_fd s, const char* path, CompressionType compression,
+ std::vector<char>& buffer) {
+ __android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path);
- if (!WriteFdExactly(s, &msg.data, sizeof(msg.data)) || !WriteFdExactly(s, &buffer[0], r)) {
- return false;
- }
+ unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC));
+ if (fd < 0) {
+ SendSyncFailErrno(s, "open failed");
+ return false;
}
- return true;
-}
+ int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE);
+ if (rc != 0) {
+ D("[ Failed to fadvise: %s ]", strerror(rc));
+ }
-static bool recv_compressed(borrowed_fd s, unique_fd fd) {
syncmsg msg;
msg.data.id = ID_DATA;
- BrotliEncoder<SYNC_DATA_MAX> encoder;
+ std::variant<std::monostate, NullEncoder, BrotliEncoder> encoder_storage;
+ Encoder* encoder;
+
+ switch (compression) {
+ case CompressionType::None:
+ encoder = &encoder_storage.emplace<NullEncoder>(SYNC_DATA_MAX);
+ break;
+
+ case CompressionType::Brotli:
+ encoder = &encoder_storage.emplace<BrotliEncoder>(SYNC_DATA_MAX);
+ break;
+
+ case CompressionType::Any:
+ LOG(FATAL) << "unexpected CompressionType::Any";
+ }
bool sending = true;
while (sending) {
@@ -624,15 +622,15 @@
}
if (r == 0) {
- encoder.Finish();
+ encoder->Finish();
} else {
input.resize(r);
- encoder.Append(std::move(input));
+ encoder->Append(std::move(input));
}
while (true) {
Block output;
- EncodeResult result = encoder.Encode(&output);
+ EncodeResult result = encoder->Encode(&output);
if (result == EncodeResult::Error) {
SendSyncFailErrno(s, "compress failed");
return false;
@@ -657,42 +655,13 @@
}
}
- return true;
-}
-
-static bool recv_impl(borrowed_fd s, const char* path, bool compressed, std::vector<char>& buffer) {
- __android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path);
-
- unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC));
- if (fd < 0) {
- SendSyncFailErrno(s, "open failed");
- return false;
- }
-
- int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE);
- if (rc != 0) {
- D("[ Failed to fadvise: %s ]", strerror(rc));
- }
-
- bool result;
- if (compressed) {
- result = recv_compressed(s, std::move(fd));
- } else {
- result = recv_uncompressed(s, std::move(fd), buffer);
- }
-
- if (!result) {
- return false;
- }
-
- syncmsg msg;
msg.data.id = ID_DONE;
msg.data.size = 0;
return WriteFdExactly(s, &msg.data, sizeof(msg.data));
}
static bool do_recv_v1(borrowed_fd s, const char* path, std::vector<char>& buffer) {
- return recv_impl(s, path, false, buffer);
+ return recv_impl(s, path, CompressionType::None, buffer);
}
static bool do_recv_v2(borrowed_fd s, const char* path, std::vector<char>& buffer) {
@@ -706,17 +675,23 @@
PLOG(ERROR) << "failed to read recv_v2 setup packet";
}
- bool compressed = false;
+ std::optional<CompressionType> compression;
if (msg.recv_v2_setup.flags & kSyncFlagBrotli) {
msg.recv_v2_setup.flags &= ~kSyncFlagBrotli;
- compressed = true;
+ if (compression) {
+ SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d",
+ msg.recv_v2_setup.flags));
+ return false;
+ }
+ compression = CompressionType::Brotli;
}
+
if (msg.recv_v2_setup.flags) {
SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.recv_v2_setup.flags));
return false;
}
- return recv_impl(s, path, compressed, buffer);
+ return recv_impl(s, path, compression.value_or(CompressionType::None), buffer);
}
static const char* sync_id_to_name(uint32_t id) {
diff --git a/adb/file_sync_protocol.h b/adb/file_sync_protocol.h
index fd9a516..70425f7 100644
--- a/adb/file_sync_protocol.h
+++ b/adb/file_sync_protocol.h
@@ -94,6 +94,12 @@
kSyncFlagBrotli = 1,
};
+enum class CompressionType {
+ None,
+ Any,
+ Brotli,
+};
+
// send_v1 sent the path in a buffer, followed by a comma and the mode as a string.
// send_v2 sends just the path in the first request, and then sends another syncmsg (with the
// same ID!) with details.
diff --git a/adb/test_device.py b/adb/test_device.py
index 6a9ff89..496a0ff 100755
--- a/adb/test_device.py
+++ b/adb/test_device.py
@@ -77,8 +77,7 @@
class DeviceTest(unittest.TestCase):
- def setUp(self):
- self.device = adb.get_device()
+ device = adb.get_device()
class AbbTest(DeviceTest):
@@ -753,535 +752,554 @@
return files
-class FileOperationsTest(DeviceTest):
- SCRATCH_DIR = '/data/local/tmp'
- DEVICE_TEMP_FILE = SCRATCH_DIR + '/adb_test_file'
- DEVICE_TEMP_DIR = SCRATCH_DIR + '/adb_test_dir'
+class FileOperationsTest:
+ class Base(DeviceTest):
+ SCRATCH_DIR = '/data/local/tmp'
+ DEVICE_TEMP_FILE = SCRATCH_DIR + '/adb_test_file'
+ DEVICE_TEMP_DIR = SCRATCH_DIR + '/adb_test_dir'
- def _verify_remote(self, checksum, remote_path):
- dev_md5, _ = self.device.shell([get_md5_prog(self.device),
- remote_path])[0].split()
- self.assertEqual(checksum, dev_md5)
+ def setUp(self):
+ self.previous_env = os.environ.get("ADB_COMPRESSION")
+ os.environ["ADB_COMPRESSION"] = self.compression
- def _verify_local(self, checksum, local_path):
- with open(local_path, 'rb') as host_file:
- host_md5 = compute_md5(host_file.read())
- self.assertEqual(host_md5, checksum)
+ def tearDown(self):
+ if self.previous_env is None:
+ del os.environ["ADB_COMPRESSION"]
+ else:
+ os.environ["ADB_COMPRESSION"] = self.previous_env
- def test_push(self):
- """Push a randomly generated file to specified device."""
- kbytes = 512
- tmp = tempfile.NamedTemporaryFile(mode='wb', delete=False)
- rand_str = os.urandom(1024 * kbytes)
- tmp.write(rand_str)
- tmp.close()
+ def _verify_remote(self, checksum, remote_path):
+ dev_md5, _ = self.device.shell([get_md5_prog(self.device),
+ remote_path])[0].split()
+ self.assertEqual(checksum, dev_md5)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE])
- self.device.push(local=tmp.name, remote=self.DEVICE_TEMP_FILE)
+ def _verify_local(self, checksum, local_path):
+ with open(local_path, 'rb') as host_file:
+ host_md5 = compute_md5(host_file.read())
+ self.assertEqual(host_md5, checksum)
- self._verify_remote(compute_md5(rand_str), self.DEVICE_TEMP_FILE)
- self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE])
+ def test_push(self):
+ """Push a randomly generated file to specified device."""
+ kbytes = 512
+ tmp = tempfile.NamedTemporaryFile(mode='wb', delete=False)
+ rand_str = os.urandom(1024 * kbytes)
+ tmp.write(rand_str)
+ tmp.close()
- os.remove(tmp.name)
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE])
+ self.device.push(local=tmp.name, remote=self.DEVICE_TEMP_FILE)
- def test_push_dir(self):
- """Push a randomly generated directory of files to the device."""
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
+ self._verify_remote(compute_md5(rand_str), self.DEVICE_TEMP_FILE)
+ self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE])
- try:
- host_dir = tempfile.mkdtemp()
+ os.remove(tmp.name)
- # Make sure the temp directory isn't setuid, or else adb will complain.
- os.chmod(host_dir, 0o700)
-
- # Create 32 random files.
- temp_files = make_random_host_files(in_dir=host_dir, num_files=32)
- self.device.push(host_dir, self.DEVICE_TEMP_DIR)
-
- for temp_file in temp_files:
- remote_path = posixpath.join(self.DEVICE_TEMP_DIR,
- os.path.basename(host_dir),
- temp_file.base_name)
- self._verify_remote(temp_file.checksum, remote_path)
+ def test_push_dir(self):
+ """Push a randomly generated directory of files to the device."""
self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
- def disabled_test_push_empty(self):
- """Push an empty directory to the device."""
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
+ try:
+ host_dir = tempfile.mkdtemp()
- try:
- host_dir = tempfile.mkdtemp()
+ # Make sure the temp directory isn't setuid, or else adb will complain.
+ os.chmod(host_dir, 0o700)
- # Make sure the temp directory isn't setuid, or else adb will complain.
- os.chmod(host_dir, 0o700)
+ # Create 32 random files.
+ temp_files = make_random_host_files(in_dir=host_dir, num_files=32)
+ self.device.push(host_dir, self.DEVICE_TEMP_DIR)
- # Create an empty directory.
- empty_dir_path = os.path.join(host_dir, 'empty')
- os.mkdir(empty_dir_path);
+ for temp_file in temp_files:
+ remote_path = posixpath.join(self.DEVICE_TEMP_DIR,
+ os.path.basename(host_dir),
+ temp_file.base_name)
+ self._verify_remote(temp_file.checksum, remote_path)
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
- self.device.push(empty_dir_path, self.DEVICE_TEMP_DIR)
-
- remote_path = os.path.join(self.DEVICE_TEMP_DIR, "empty")
- test_empty_cmd = ["[", "-d", remote_path, "]"]
- rc, _, _ = self.device.shell_nocheck(test_empty_cmd)
-
- self.assertEqual(rc, 0)
+ def disabled_test_push_empty(self):
+ """Push an empty directory to the device."""
self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
- @unittest.skipIf(sys.platform == "win32", "symlinks require elevated privileges on windows")
- def test_push_symlink(self):
- """Push a symlink.
+ try:
+ host_dir = tempfile.mkdtemp()
- Bug: http://b/31491920
- """
- try:
- host_dir = tempfile.mkdtemp()
+ # Make sure the temp directory isn't setuid, or else adb will complain.
+ os.chmod(host_dir, 0o700)
- # Make sure the temp directory isn't setuid, or else adb will
- # complain.
- os.chmod(host_dir, 0o700)
+ # Create an empty directory.
+ empty_dir_path = os.path.join(host_dir, 'empty')
+ os.mkdir(empty_dir_path);
- with open(os.path.join(host_dir, 'foo'), 'w') as f:
- f.write('foo')
+ self.device.push(empty_dir_path, self.DEVICE_TEMP_DIR)
- symlink_path = os.path.join(host_dir, 'symlink')
- os.symlink('foo', symlink_path)
+ remote_path = os.path.join(self.DEVICE_TEMP_DIR, "empty")
+ test_empty_cmd = ["[", "-d", remote_path, "]"]
+ rc, _, _ = self.device.shell_nocheck(test_empty_cmd)
+
+ self.assertEqual(rc, 0)
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ @unittest.skipIf(sys.platform == "win32", "symlinks require elevated privileges on windows")
+ def test_push_symlink(self):
+ """Push a symlink.
+
+ Bug: http://b/31491920
+ """
+ try:
+ host_dir = tempfile.mkdtemp()
+
+ # Make sure the temp directory isn't setuid, or else adb will
+ # complain.
+ os.chmod(host_dir, 0o700)
+
+ with open(os.path.join(host_dir, 'foo'), 'w') as f:
+ f.write('foo')
+
+ symlink_path = os.path.join(host_dir, 'symlink')
+ os.symlink('foo', symlink_path)
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
+ self.device.push(symlink_path, self.DEVICE_TEMP_DIR)
+ rc, out, _ = self.device.shell_nocheck(
+ ['cat', posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')])
+ self.assertEqual(0, rc)
+ self.assertEqual(out.strip(), 'foo')
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ def test_multiple_push(self):
+ """Push multiple files to the device in one adb push command.
+
+ Bug: http://b/25324823
+ """
self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
- self.device.push(symlink_path, self.DEVICE_TEMP_DIR)
- rc, out, _ = self.device.shell_nocheck(
- ['cat', posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')])
- self.assertEqual(0, rc)
- self.assertEqual(out.strip(), 'foo')
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
- def test_multiple_push(self):
- """Push multiple files to the device in one adb push command.
-
- Bug: http://b/25324823
- """
-
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', self.DEVICE_TEMP_DIR])
-
- try:
- host_dir = tempfile.mkdtemp()
-
- # Create some random files and a subdirectory containing more files.
- temp_files = make_random_host_files(in_dir=host_dir, num_files=4)
-
- subdir = os.path.join(host_dir, 'subdir')
- os.mkdir(subdir)
- subdir_temp_files = make_random_host_files(in_dir=subdir,
- num_files=4)
-
- paths = [x.full_path for x in temp_files]
- paths.append(subdir)
- self.device._simple_call(['push'] + paths + [self.DEVICE_TEMP_DIR])
-
- for temp_file in temp_files:
- remote_path = posixpath.join(self.DEVICE_TEMP_DIR,
- temp_file.base_name)
- self._verify_remote(temp_file.checksum, remote_path)
-
- for subdir_temp_file in subdir_temp_files:
- remote_path = posixpath.join(self.DEVICE_TEMP_DIR,
- # BROKEN: http://b/25394682
- # 'subdir';
- temp_file.base_name)
- self._verify_remote(temp_file.checksum, remote_path)
-
-
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
-
- @requires_non_root
- def test_push_error_reporting(self):
- """Make sure that errors that occur while pushing a file get reported
-
- Bug: http://b/26816782
- """
- with tempfile.NamedTemporaryFile() as tmp_file:
- tmp_file.write(b'\0' * 1024 * 1024)
- tmp_file.flush()
try:
- self.device.push(local=tmp_file.name, remote='/system/')
- self.fail('push should not have succeeded')
+ host_dir = tempfile.mkdtemp()
+
+ # Create some random files and a subdirectory containing more files.
+ temp_files = make_random_host_files(in_dir=host_dir, num_files=4)
+
+ subdir = os.path.join(host_dir, 'subdir')
+ os.mkdir(subdir)
+ subdir_temp_files = make_random_host_files(in_dir=subdir,
+ num_files=4)
+
+ paths = [x.full_path for x in temp_files]
+ paths.append(subdir)
+ self.device._simple_call(['push'] + paths + [self.DEVICE_TEMP_DIR])
+
+ for temp_file in temp_files:
+ remote_path = posixpath.join(self.DEVICE_TEMP_DIR,
+ temp_file.base_name)
+ self._verify_remote(temp_file.checksum, remote_path)
+
+ for subdir_temp_file in subdir_temp_files:
+ remote_path = posixpath.join(self.DEVICE_TEMP_DIR,
+ # BROKEN: http://b/25394682
+ # 'subdir';
+ temp_file.base_name)
+ self._verify_remote(temp_file.checksum, remote_path)
+
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ @requires_non_root
+ def test_push_error_reporting(self):
+ """Make sure that errors that occur while pushing a file get reported
+
+ Bug: http://b/26816782
+ """
+ with tempfile.NamedTemporaryFile() as tmp_file:
+ tmp_file.write(b'\0' * 1024 * 1024)
+ tmp_file.flush()
+ try:
+ self.device.push(local=tmp_file.name, remote='/system/')
+ self.fail('push should not have succeeded')
+ except subprocess.CalledProcessError as e:
+ output = e.output
+
+ self.assertTrue(b'Permission denied' in output or
+ b'Read-only file system' in output)
+
+ @requires_non_root
+ def test_push_directory_creation(self):
+ """Regression test for directory creation.
+
+ Bug: http://b/110953234
+ """
+ with tempfile.NamedTemporaryFile() as tmp_file:
+ tmp_file.write(b'\0' * 1024 * 1024)
+ tmp_file.flush()
+ remote_path = self.DEVICE_TEMP_DIR + '/test_push_directory_creation'
+ self.device.shell(['rm', '-rf', remote_path])
+
+ remote_path += '/filename'
+ self.device.push(local=tmp_file.name, remote=remote_path)
+
+ def disabled_test_push_multiple_slash_root(self):
+ """Regression test for pushing to //data/local/tmp.
+
+ Bug: http://b/141311284
+
+ Disabled because this broken on the adbd side as well: b/141943968
+ """
+ with tempfile.NamedTemporaryFile() as tmp_file:
+ tmp_file.write('\0' * 1024 * 1024)
+ tmp_file.flush()
+ remote_path = '/' + self.DEVICE_TEMP_DIR + '/test_push_multiple_slash_root'
+ self.device.shell(['rm', '-rf', remote_path])
+ self.device.push(local=tmp_file.name, remote=remote_path)
+
+ def _test_pull(self, remote_file, checksum):
+ tmp_write = tempfile.NamedTemporaryFile(mode='wb', delete=False)
+ tmp_write.close()
+ self.device.pull(remote=remote_file, local=tmp_write.name)
+ with open(tmp_write.name, 'rb') as tmp_read:
+ host_contents = tmp_read.read()
+ host_md5 = compute_md5(host_contents)
+ self.assertEqual(checksum, host_md5)
+ os.remove(tmp_write.name)
+
+ @requires_non_root
+ def test_pull_error_reporting(self):
+ self.device.shell(['touch', self.DEVICE_TEMP_FILE])
+ self.device.shell(['chmod', 'a-rwx', self.DEVICE_TEMP_FILE])
+
+ try:
+ output = self.device.pull(remote=self.DEVICE_TEMP_FILE, local='x')
except subprocess.CalledProcessError as e:
output = e.output
- self.assertTrue(b'Permission denied' in output or
- b'Read-only file system' in output)
+ self.assertIn(b'Permission denied', output)
- @requires_non_root
- def test_push_directory_creation(self):
- """Regression test for directory creation.
+ self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE])
- Bug: http://b/110953234
- """
- with tempfile.NamedTemporaryFile() as tmp_file:
- tmp_file.write(b'\0' * 1024 * 1024)
- tmp_file.flush()
- remote_path = self.DEVICE_TEMP_DIR + '/test_push_directory_creation'
- self.device.shell(['rm', '-rf', remote_path])
+ def test_pull(self):
+ """Pull a randomly generated file from specified device."""
+ kbytes = 512
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE])
+ cmd = ['dd', 'if=/dev/urandom',
+ 'of={}'.format(self.DEVICE_TEMP_FILE), 'bs=1024',
+ 'count={}'.format(kbytes)]
+ self.device.shell(cmd)
+ dev_md5, _ = self.device.shell(
+ [get_md5_prog(self.device), self.DEVICE_TEMP_FILE])[0].split()
+ self._test_pull(self.DEVICE_TEMP_FILE, dev_md5)
+ self.device.shell_nocheck(['rm', self.DEVICE_TEMP_FILE])
- remote_path += '/filename'
- self.device.push(local=tmp_file.name, remote=remote_path)
+ def test_pull_dir(self):
+ """Pull a randomly generated directory of files from the device."""
+ try:
+ host_dir = tempfile.mkdtemp()
- def disabled_test_push_multiple_slash_root(self):
- """Regression test for pushing to //data/local/tmp.
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
- Bug: http://b/141311284
+ # Populate device directory with random files.
+ temp_files = make_random_device_files(
+ self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
- Disabled because this broken on the adbd side as well: b/141943968
- """
- with tempfile.NamedTemporaryFile() as tmp_file:
- tmp_file.write('\0' * 1024 * 1024)
- tmp_file.flush()
- remote_path = '/' + self.DEVICE_TEMP_DIR + '/test_push_multiple_slash_root'
- self.device.shell(['rm', '-rf', remote_path])
- self.device.push(local=tmp_file.name, remote=remote_path)
+ self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir)
- def _test_pull(self, remote_file, checksum):
- tmp_write = tempfile.NamedTemporaryFile(mode='wb', delete=False)
- tmp_write.close()
- self.device.pull(remote=remote_file, local=tmp_write.name)
- with open(tmp_write.name, 'rb') as tmp_read:
- host_contents = tmp_read.read()
- host_md5 = compute_md5(host_contents)
- self.assertEqual(checksum, host_md5)
- os.remove(tmp_write.name)
+ for temp_file in temp_files:
+ host_path = os.path.join(
+ host_dir, posixpath.basename(self.DEVICE_TEMP_DIR),
+ temp_file.base_name)
+ self._verify_local(temp_file.checksum, host_path)
- @requires_non_root
- def test_pull_error_reporting(self):
- self.device.shell(['touch', self.DEVICE_TEMP_FILE])
- self.device.shell(['chmod', 'a-rwx', self.DEVICE_TEMP_FILE])
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
- try:
- output = self.device.pull(remote=self.DEVICE_TEMP_FILE, local='x')
- except subprocess.CalledProcessError as e:
- output = e.output
+ def test_pull_dir_symlink(self):
+ """Pull a directory into a symlink to a directory.
- self.assertIn(b'Permission denied', output)
+ Bug: http://b/27362811
+ """
+ if os.name != 'posix':
+ raise unittest.SkipTest('requires POSIX')
- self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE])
+ try:
+ host_dir = tempfile.mkdtemp()
+ real_dir = os.path.join(host_dir, 'dir')
+ symlink = os.path.join(host_dir, 'symlink')
+ os.mkdir(real_dir)
+ os.symlink(real_dir, symlink)
- def test_pull(self):
- """Pull a randomly generated file from specified device."""
- kbytes = 512
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE])
- cmd = ['dd', 'if=/dev/urandom',
- 'of={}'.format(self.DEVICE_TEMP_FILE), 'bs=1024',
- 'count={}'.format(kbytes)]
- self.device.shell(cmd)
- dev_md5, _ = self.device.shell(
- [get_md5_prog(self.device), self.DEVICE_TEMP_FILE])[0].split()
- self._test_pull(self.DEVICE_TEMP_FILE, dev_md5)
- self.device.shell_nocheck(['rm', self.DEVICE_TEMP_FILE])
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
- def test_pull_dir(self):
- """Pull a randomly generated directory of files from the device."""
- try:
- host_dir = tempfile.mkdtemp()
+ # Populate device directory with random files.
+ temp_files = make_random_device_files(
+ self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
+ self.device.pull(remote=self.DEVICE_TEMP_DIR, local=symlink)
- # Populate device directory with random files.
- temp_files = make_random_device_files(
- self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
+ for temp_file in temp_files:
+ host_path = os.path.join(
+ real_dir, posixpath.basename(self.DEVICE_TEMP_DIR),
+ temp_file.base_name)
+ self._verify_local(temp_file.checksum, host_path)
- self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir)
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+ def test_pull_dir_symlink_collision(self):
+ """Pull a directory into a colliding symlink to directory."""
+ if os.name != 'posix':
+ raise unittest.SkipTest('requires POSIX')
+
+ try:
+ host_dir = tempfile.mkdtemp()
+ real_dir = os.path.join(host_dir, 'real')
+ tmp_dirname = os.path.basename(self.DEVICE_TEMP_DIR)
+ symlink = os.path.join(host_dir, tmp_dirname)
+ os.mkdir(real_dir)
+ os.symlink(real_dir, symlink)
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
+
+ # Populate device directory with random files.
+ temp_files = make_random_device_files(
+ self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
+
+ self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir)
+
+ for temp_file in temp_files:
+ host_path = os.path.join(real_dir, temp_file.base_name)
+ self._verify_local(temp_file.checksum, host_path)
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ def test_pull_dir_nonexistent(self):
+ """Pull a directory of files from the device to a nonexistent path."""
+ try:
+ host_dir = tempfile.mkdtemp()
+ dest_dir = os.path.join(host_dir, 'dest')
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
+
+ # Populate device directory with random files.
+ temp_files = make_random_device_files(
+ self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
+
+ self.device.pull(remote=self.DEVICE_TEMP_DIR, local=dest_dir)
+
+ for temp_file in temp_files:
+ host_path = os.path.join(dest_dir, temp_file.base_name)
+ self._verify_local(temp_file.checksum, host_path)
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ # selinux prevents adbd from accessing symlinks on /data/local/tmp.
+ def disabled_test_pull_symlink_dir(self):
+ """Pull a symlink to a directory of symlinks to files."""
+ try:
+ host_dir = tempfile.mkdtemp()
+
+ remote_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'contents')
+ remote_links = posixpath.join(self.DEVICE_TEMP_DIR, 'links')
+ remote_symlink = posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', remote_dir, remote_links])
+ self.device.shell(['ln', '-s', remote_links, remote_symlink])
+
+ # Populate device directory with random files.
+ temp_files = make_random_device_files(
+ self.device, in_dir=remote_dir, num_files=32)
+
+ for temp_file in temp_files:
+ self.device.shell(
+ ['ln', '-s', '../contents/{}'.format(temp_file.base_name),
+ posixpath.join(remote_links, temp_file.base_name)])
+
+ self.device.pull(remote=remote_symlink, local=host_dir)
+
+ for temp_file in temp_files:
+ host_path = os.path.join(
+ host_dir, 'symlink', temp_file.base_name)
+ self._verify_local(temp_file.checksum, host_path)
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ def test_pull_empty(self):
+ """Pull a directory containing an empty directory from the device."""
+ try:
+ host_dir = tempfile.mkdtemp()
+
+ remote_empty_path = posixpath.join(self.DEVICE_TEMP_DIR, 'empty')
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', remote_empty_path])
+
+ self.device.pull(remote=remote_empty_path, local=host_dir)
+ self.assertTrue(os.path.isdir(os.path.join(host_dir, 'empty')))
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ def test_multiple_pull(self):
+ """Pull a randomly generated directory of files from the device."""
+
+ try:
+ host_dir = tempfile.mkdtemp()
+
+ subdir = posixpath.join(self.DEVICE_TEMP_DIR, 'subdir')
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ self.device.shell(['mkdir', '-p', subdir])
+
+ # Create some random files and a subdirectory containing more files.
+ temp_files = make_random_device_files(
+ self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=4)
+
+ subdir_temp_files = make_random_device_files(
+ self.device, in_dir=subdir, num_files=4, prefix='subdir_')
+
+ paths = [x.full_path for x in temp_files]
+ paths.append(subdir)
+ self.device._simple_call(['pull'] + paths + [host_dir])
+
+ for temp_file in temp_files:
+ local_path = os.path.join(host_dir, temp_file.base_name)
+ self._verify_local(temp_file.checksum, local_path)
+
+ for subdir_temp_file in subdir_temp_files:
+ local_path = os.path.join(host_dir,
+ 'subdir',
+ subdir_temp_file.base_name)
+ self._verify_local(subdir_temp_file.checksum, local_path)
+
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if host_dir is not None:
+ shutil.rmtree(host_dir)
+
+ def verify_sync(self, device, temp_files, device_dir):
+ """Verifies that a list of temp files was synced to the device."""
+ # Confirm that every file on the device mirrors that on the host.
for temp_file in temp_files:
- host_path = os.path.join(
- host_dir, posixpath.basename(self.DEVICE_TEMP_DIR),
- temp_file.base_name)
- self._verify_local(temp_file.checksum, host_path)
+ device_full_path = posixpath.join(
+ device_dir, temp_file.base_name)
+ dev_md5, _ = device.shell(
+ [get_md5_prog(self.device), device_full_path])[0].split()
+ self.assertEqual(temp_file.checksum, dev_md5)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ def test_sync(self):
+ """Sync a host directory to the data partition."""
- def test_pull_dir_symlink(self):
- """Pull a directory into a symlink to a directory.
+ try:
+ base_dir = tempfile.mkdtemp()
- Bug: http://b/27362811
- """
- if os.name != 'posix':
- raise unittest.SkipTest('requires POSIX')
+ # Create mirror device directory hierarchy within base_dir.
+ full_dir_path = base_dir + self.DEVICE_TEMP_DIR
+ os.makedirs(full_dir_path)
- try:
- host_dir = tempfile.mkdtemp()
- real_dir = os.path.join(host_dir, 'dir')
- symlink = os.path.join(host_dir, 'symlink')
- os.mkdir(real_dir)
- os.symlink(real_dir, symlink)
+ # Create 32 random files within the host mirror.
+ temp_files = make_random_host_files(
+ in_dir=full_dir_path, num_files=32)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
+ # Clean up any stale files on the device.
+ device = adb.get_device() # pylint: disable=no-member
+ device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- # Populate device directory with random files.
- temp_files = make_random_device_files(
- self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
+ old_product_out = os.environ.get('ANDROID_PRODUCT_OUT')
+ os.environ['ANDROID_PRODUCT_OUT'] = base_dir
+ device.sync('data')
+ if old_product_out is None:
+ del os.environ['ANDROID_PRODUCT_OUT']
+ else:
+ os.environ['ANDROID_PRODUCT_OUT'] = old_product_out
- self.device.pull(remote=self.DEVICE_TEMP_DIR, local=symlink)
+ self.verify_sync(device, temp_files, self.DEVICE_TEMP_DIR)
- for temp_file in temp_files:
- host_path = os.path.join(
- real_dir, posixpath.basename(self.DEVICE_TEMP_DIR),
- temp_file.base_name)
- self._verify_local(temp_file.checksum, host_path)
+ #self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if base_dir is not None:
+ shutil.rmtree(base_dir)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ def test_push_sync(self):
+ """Sync a host directory to a specific path."""
- def test_pull_dir_symlink_collision(self):
- """Pull a directory into a colliding symlink to directory."""
- if os.name != 'posix':
- raise unittest.SkipTest('requires POSIX')
+ try:
+ temp_dir = tempfile.mkdtemp()
+ temp_files = make_random_host_files(in_dir=temp_dir, num_files=32)
- try:
- host_dir = tempfile.mkdtemp()
- real_dir = os.path.join(host_dir, 'real')
- tmp_dirname = os.path.basename(self.DEVICE_TEMP_DIR)
- symlink = os.path.join(host_dir, tmp_dirname)
- os.mkdir(real_dir)
- os.symlink(real_dir, symlink)
+ device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'sync_src_dst')
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
+ # Clean up any stale files on the device.
+ device = adb.get_device() # pylint: disable=no-member
+ device.shell(['rm', '-rf', device_dir])
- # Populate device directory with random files.
- temp_files = make_random_device_files(
- self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
+ device.push(temp_dir, device_dir, sync=True)
- self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir)
+ self.verify_sync(device, temp_files, device_dir)
- for temp_file in temp_files:
- host_path = os.path.join(real_dir, temp_file.base_name)
- self._verify_local(temp_file.checksum, host_path)
+ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
+ finally:
+ if temp_dir is not None:
+ shutil.rmtree(temp_dir)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+ def test_unicode_paths(self):
+ """Ensure that we can support non-ASCII paths, even on Windows."""
+ name = u'로보카 폴리'
- def test_pull_dir_nonexistent(self):
- """Pull a directory of files from the device to a nonexistent path."""
- try:
- host_dir = tempfile.mkdtemp()
- dest_dir = os.path.join(host_dir, 'dest')
+ self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*'])
+ remote_path = u'/data/local/tmp/adb-test-{}'.format(name)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR])
+ ## push.
+ tf = tempfile.NamedTemporaryFile('wb', suffix=name, delete=False)
+ tf.close()
+ self.device.push(tf.name, remote_path)
+ os.remove(tf.name)
+ self.assertFalse(os.path.exists(tf.name))
- # Populate device directory with random files.
- temp_files = make_random_device_files(
- self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32)
+ # Verify that the device ended up with the expected UTF-8 path
+ output = self.device.shell(
+ ['ls', '/data/local/tmp/adb-test-*'])[0].strip()
+ self.assertEqual(remote_path, output)
- self.device.pull(remote=self.DEVICE_TEMP_DIR, local=dest_dir)
+ # pull.
+ self.device.pull(remote_path, tf.name)
+ self.assertTrue(os.path.exists(tf.name))
+ os.remove(tf.name)
+ self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*'])
- for temp_file in temp_files:
- host_path = os.path.join(dest_dir, temp_file.base_name)
- self._verify_local(temp_file.checksum, host_path)
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
+class FileOperationsTestUncompressed(FileOperationsTest.Base):
+ compression = "none"
- # selinux prevents adbd from accessing symlinks on /data/local/tmp.
- def disabled_test_pull_symlink_dir(self):
- """Pull a symlink to a directory of symlinks to files."""
- try:
- host_dir = tempfile.mkdtemp()
- remote_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'contents')
- remote_links = posixpath.join(self.DEVICE_TEMP_DIR, 'links')
- remote_symlink = posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')
-
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', remote_dir, remote_links])
- self.device.shell(['ln', '-s', remote_links, remote_symlink])
-
- # Populate device directory with random files.
- temp_files = make_random_device_files(
- self.device, in_dir=remote_dir, num_files=32)
-
- for temp_file in temp_files:
- self.device.shell(
- ['ln', '-s', '../contents/{}'.format(temp_file.base_name),
- posixpath.join(remote_links, temp_file.base_name)])
-
- self.device.pull(remote=remote_symlink, local=host_dir)
-
- for temp_file in temp_files:
- host_path = os.path.join(
- host_dir, 'symlink', temp_file.base_name)
- self._verify_local(temp_file.checksum, host_path)
-
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
-
- def test_pull_empty(self):
- """Pull a directory containing an empty directory from the device."""
- try:
- host_dir = tempfile.mkdtemp()
-
- remote_empty_path = posixpath.join(self.DEVICE_TEMP_DIR, 'empty')
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', remote_empty_path])
-
- self.device.pull(remote=remote_empty_path, local=host_dir)
- self.assertTrue(os.path.isdir(os.path.join(host_dir, 'empty')))
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
-
- def test_multiple_pull(self):
- """Pull a randomly generated directory of files from the device."""
-
- try:
- host_dir = tempfile.mkdtemp()
-
- subdir = posixpath.join(self.DEVICE_TEMP_DIR, 'subdir')
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- self.device.shell(['mkdir', '-p', subdir])
-
- # Create some random files and a subdirectory containing more files.
- temp_files = make_random_device_files(
- self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=4)
-
- subdir_temp_files = make_random_device_files(
- self.device, in_dir=subdir, num_files=4, prefix='subdir_')
-
- paths = [x.full_path for x in temp_files]
- paths.append(subdir)
- self.device._simple_call(['pull'] + paths + [host_dir])
-
- for temp_file in temp_files:
- local_path = os.path.join(host_dir, temp_file.base_name)
- self._verify_local(temp_file.checksum, local_path)
-
- for subdir_temp_file in subdir_temp_files:
- local_path = os.path.join(host_dir,
- 'subdir',
- subdir_temp_file.base_name)
- self._verify_local(subdir_temp_file.checksum, local_path)
-
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if host_dir is not None:
- shutil.rmtree(host_dir)
-
- def verify_sync(self, device, temp_files, device_dir):
- """Verifies that a list of temp files was synced to the device."""
- # Confirm that every file on the device mirrors that on the host.
- for temp_file in temp_files:
- device_full_path = posixpath.join(
- device_dir, temp_file.base_name)
- dev_md5, _ = device.shell(
- [get_md5_prog(self.device), device_full_path])[0].split()
- self.assertEqual(temp_file.checksum, dev_md5)
-
- def test_sync(self):
- """Sync a host directory to the data partition."""
-
- try:
- base_dir = tempfile.mkdtemp()
-
- # Create mirror device directory hierarchy within base_dir.
- full_dir_path = base_dir + self.DEVICE_TEMP_DIR
- os.makedirs(full_dir_path)
-
- # Create 32 random files within the host mirror.
- temp_files = make_random_host_files(
- in_dir=full_dir_path, num_files=32)
-
- # Clean up any stale files on the device.
- device = adb.get_device() # pylint: disable=no-member
- device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
-
- old_product_out = os.environ.get('ANDROID_PRODUCT_OUT')
- os.environ['ANDROID_PRODUCT_OUT'] = base_dir
- device.sync('data')
- if old_product_out is None:
- del os.environ['ANDROID_PRODUCT_OUT']
- else:
- os.environ['ANDROID_PRODUCT_OUT'] = old_product_out
-
- self.verify_sync(device, temp_files, self.DEVICE_TEMP_DIR)
-
- #self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if base_dir is not None:
- shutil.rmtree(base_dir)
-
- def test_push_sync(self):
- """Sync a host directory to a specific path."""
-
- try:
- temp_dir = tempfile.mkdtemp()
- temp_files = make_random_host_files(in_dir=temp_dir, num_files=32)
-
- device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'sync_src_dst')
-
- # Clean up any stale files on the device.
- device = adb.get_device() # pylint: disable=no-member
- device.shell(['rm', '-rf', device_dir])
-
- device.push(temp_dir, device_dir, sync=True)
-
- self.verify_sync(device, temp_files, device_dir)
-
- self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR])
- finally:
- if temp_dir is not None:
- shutil.rmtree(temp_dir)
-
- def test_unicode_paths(self):
- """Ensure that we can support non-ASCII paths, even on Windows."""
- name = u'로보카 폴리'
-
- self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*'])
- remote_path = u'/data/local/tmp/adb-test-{}'.format(name)
-
- ## push.
- tf = tempfile.NamedTemporaryFile('wb', suffix=name, delete=False)
- tf.close()
- self.device.push(tf.name, remote_path)
- os.remove(tf.name)
- self.assertFalse(os.path.exists(tf.name))
-
- # Verify that the device ended up with the expected UTF-8 path
- output = self.device.shell(
- ['ls', '/data/local/tmp/adb-test-*'])[0].strip()
- self.assertEqual(remote_path, output)
-
- # pull.
- self.device.pull(remote_path, tf.name)
- self.assertTrue(os.path.exists(tf.name))
- os.remove(tf.name)
- self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*'])
+class FileOperationsTestBrotli(FileOperationsTest.Base):
+ compression = "brotli"
class DeviceOfflineTest(DeviceTest):