Merge "BatteryMonitor: support Dock charging"
diff --git a/debuggerd/crash_dump.cpp b/debuggerd/crash_dump.cpp
index 5bb1d75..967b942 100644
--- a/debuggerd/crash_dump.cpp
+++ b/debuggerd/crash_dump.cpp
@@ -502,15 +502,24 @@
continue;
}
- struct iovec iov = {
+ struct iovec tagged_addr_iov = {
&info.tagged_addr_ctrl,
sizeof(info.tagged_addr_ctrl),
};
if (ptrace(PTRACE_GETREGSET, thread, NT_ARM_TAGGED_ADDR_CTRL,
- reinterpret_cast<void*>(&iov)) == -1) {
+ reinterpret_cast<void*>(&tagged_addr_iov)) == -1) {
info.tagged_addr_ctrl = -1;
}
+ struct iovec pac_enabled_keys_iov = {
+ &info.pac_enabled_keys,
+ sizeof(info.pac_enabled_keys),
+ };
+ if (ptrace(PTRACE_GETREGSET, thread, NT_ARM_PAC_ENABLED_KEYS,
+ reinterpret_cast<void*>(&pac_enabled_keys_iov)) == -1) {
+ info.pac_enabled_keys = -1;
+ }
+
if (thread == g_target_thread) {
// Read the thread's registers along with the rest of the crash info out of the pipe.
ReadCrashInfo(input_pipe, &siginfo, &info.registers, &process_info);
diff --git a/debuggerd/debuggerd_test.cpp b/debuggerd/debuggerd_test.cpp
index b107767..2cf5b18 100644
--- a/debuggerd/debuggerd_test.cpp
+++ b/debuggerd/debuggerd_test.cpp
@@ -173,6 +173,14 @@
*status = response.status;
}
+static bool pac_supported() {
+#if defined(__aarch64__)
+ return getauxval(AT_HWCAP) & HWCAP_PACA;
+#else
+ return false;
+#endif
+}
+
class CrasherTest : public ::testing::Test {
public:
pid_t crasher_pid = -1;
@@ -357,6 +365,12 @@
ASSERT_MATCH(result, R"(tagged_addr_ctrl: 000000000007fff3)"
R"( \(PR_TAGGED_ADDR_ENABLE, PR_MTE_TCF_SYNC, mask 0xfffe\))");
}
+
+ if (pac_supported()) {
+ // Test that the default PAC_ENABLED_KEYS value is set.
+ ASSERT_MATCH(result, R"(pac_enabled_keys: 000000000000000f)"
+ R"( \(PR_PAC_APIAKEY, PR_PAC_APIBKEY, PR_PAC_APDAKEY, PR_PAC_APDBKEY\))");
+ }
}
TEST_F(CrasherTest, tagged_fault_addr) {
diff --git a/debuggerd/libdebuggerd/gwp_asan.cpp b/debuggerd/libdebuggerd/gwp_asan.cpp
index 3ee309f..ed2b974 100644
--- a/debuggerd/libdebuggerd/gwp_asan.cpp
+++ b/debuggerd/libdebuggerd/gwp_asan.cpp
@@ -147,7 +147,7 @@
for (size_t i = 0; i != num_frames; ++i) {
unwindstack::FrameData frame_data = unwinder->BuildFrameFromPcOnly(frames[i]);
BacktraceFrame* f = heap_object->add_allocation_backtrace();
- fill_in_backtrace_frame(f, frame_data, unwinder->GetMaps());
+ fill_in_backtrace_frame(f, frame_data);
}
heap_object->set_deallocation_tid(__gwp_asan_get_deallocation_thread_id(responsible_allocation_));
@@ -156,7 +156,7 @@
for (size_t i = 0; i != num_frames; ++i) {
unwindstack::FrameData frame_data = unwinder->BuildFrameFromPcOnly(frames[i]);
BacktraceFrame* f = heap_object->add_deallocation_backtrace();
- fill_in_backtrace_frame(f, frame_data, unwinder->GetMaps());
+ fill_in_backtrace_frame(f, frame_data);
}
set_human_readable_cause(cause, crash_address_);
diff --git a/debuggerd/libdebuggerd/include/libdebuggerd/tombstone.h b/debuggerd/libdebuggerd/include/libdebuggerd/tombstone.h
index 2331f1e..7bf1688 100644
--- a/debuggerd/libdebuggerd/include/libdebuggerd/tombstone.h
+++ b/debuggerd/libdebuggerd/include/libdebuggerd/tombstone.h
@@ -37,7 +37,6 @@
namespace unwindstack {
struct FrameData;
-class Maps;
class Unwinder;
}
@@ -68,8 +67,7 @@
const Tombstone& tombstone,
std::function<void(const std::string& line, bool should_log)> callback);
-void fill_in_backtrace_frame(BacktraceFrame* f, const unwindstack::FrameData& frame,
- unwindstack::Maps* maps);
+void fill_in_backtrace_frame(BacktraceFrame* f, const unwindstack::FrameData& frame);
void set_human_readable_cause(Cause* cause, uint64_t fault_addr);
#endif // _DEBUGGERD_TOMBSTONE_H
diff --git a/debuggerd/libdebuggerd/include/libdebuggerd/types.h b/debuggerd/libdebuggerd/include/libdebuggerd/types.h
index 086dc97..a51e276 100644
--- a/debuggerd/libdebuggerd/include/libdebuggerd/types.h
+++ b/debuggerd/libdebuggerd/include/libdebuggerd/types.h
@@ -25,6 +25,7 @@
struct ThreadInfo {
std::unique_ptr<unwindstack::Regs> registers;
long tagged_addr_ctrl = -1;
+ long pac_enabled_keys = -1;
pid_t uid;
diff --git a/debuggerd/libdebuggerd/include/libdebuggerd/utility.h b/debuggerd/libdebuggerd/include/libdebuggerd/utility.h
index 002321f..63e142f 100644
--- a/debuggerd/libdebuggerd/include/libdebuggerd/utility.h
+++ b/debuggerd/libdebuggerd/include/libdebuggerd/utility.h
@@ -1,22 +1,20 @@
-/* system/debuggerd/utility.h
-**
-** Copyright 2008, The Android Open Source Project
-**
-** Licensed under the Apache License, Version 2.0 (the "License");
-** you may not use this file except in compliance with the License.
-** You may obtain a copy of the License at
-**
-** http://www.apache.org/licenses/LICENSE-2.0
-**
-** Unless required by applicable law or agreed to in writing, software
-** distributed under the License is distributed on an "AS IS" BASIS,
-** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-** See the License for the specific language governing permissions and
-** limitations under the License.
-*/
+/*
+ * Copyright 2008, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-#ifndef _DEBUGGERD_UTILITY_H
-#define _DEBUGGERD_UTILITY_H
+#pragma once
#include <inttypes.h>
#include <signal.h>
@@ -93,6 +91,7 @@
const char* get_signame(const siginfo_t*);
const char* get_sigcode(const siginfo_t*);
std::string describe_tagged_addr_ctrl(long ctrl);
+std::string describe_pac_enabled_keys(long keys);
// Number of bytes per MTE granule.
constexpr size_t kTagGranuleSize = 16;
@@ -100,5 +99,3 @@
// Number of rows and columns to display in an MTE tag dump.
constexpr size_t kNumTagColumns = 16;
constexpr size_t kNumTagRows = 16;
-
-#endif // _DEBUGGERD_UTILITY_H
diff --git a/debuggerd/libdebuggerd/scudo.cpp b/debuggerd/libdebuggerd/scudo.cpp
index a89f385..a2933f2 100644
--- a/debuggerd/libdebuggerd/scudo.cpp
+++ b/debuggerd/libdebuggerd/scudo.cpp
@@ -108,7 +108,7 @@
for (size_t i = 0; i < arraysize(report->allocation_trace) && report->allocation_trace[i]; ++i) {
unwindstack::FrameData frame_data = unwinder->BuildFrameFromPcOnly(report->allocation_trace[i]);
BacktraceFrame* f = heap_object->add_allocation_backtrace();
- fill_in_backtrace_frame(f, frame_data, unwinder->GetMaps());
+ fill_in_backtrace_frame(f, frame_data);
}
heap_object->set_deallocation_tid(report->deallocation_tid);
@@ -117,7 +117,7 @@
unwindstack::FrameData frame_data =
unwinder->BuildFrameFromPcOnly(report->deallocation_trace[i]);
BacktraceFrame* f = heap_object->add_deallocation_backtrace();
- fill_in_backtrace_frame(f, frame_data, unwinder->GetMaps());
+ fill_in_backtrace_frame(f, frame_data);
}
set_human_readable_cause(cause, untagged_fault_addr_);
diff --git a/debuggerd/libdebuggerd/test/utility_test.cpp b/debuggerd/libdebuggerd/test/utility_test.cpp
index 97328b7..dad3380 100644
--- a/debuggerd/libdebuggerd/test/utility_test.cpp
+++ b/debuggerd/libdebuggerd/test/utility_test.cpp
@@ -31,3 +31,12 @@
describe_tagged_addr_ctrl(0xf0000000 | PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC |
PR_MTE_TCF_ASYNC | (0xfffe << PR_MTE_TAG_SHIFT)));
}
+
+TEST(UtilityTest, describe_pac_enabled_keys) {
+ EXPECT_EQ("", describe_pac_enabled_keys(0));
+ EXPECT_EQ(" (PR_PAC_APIAKEY)", describe_pac_enabled_keys(PR_PAC_APIAKEY));
+ EXPECT_EQ(" (PR_PAC_APIAKEY, PR_PAC_APDBKEY)",
+ describe_pac_enabled_keys(PR_PAC_APIAKEY | PR_PAC_APDBKEY));
+ EXPECT_EQ(" (PR_PAC_APIAKEY, PR_PAC_APDBKEY, unknown 0x1000)",
+ describe_pac_enabled_keys(PR_PAC_APIAKEY | PR_PAC_APDBKEY | 0x1000));
+}
diff --git a/debuggerd/libdebuggerd/tombstone_proto.cpp b/debuggerd/libdebuggerd/tombstone_proto.cpp
index b1c4ef3..b7d5bc4 100644
--- a/debuggerd/libdebuggerd/tombstone_proto.cpp
+++ b/debuggerd/libdebuggerd/tombstone_proto.cpp
@@ -312,8 +312,7 @@
}
}
-void fill_in_backtrace_frame(BacktraceFrame* f, const unwindstack::FrameData& frame,
- unwindstack::Maps* maps) {
+void fill_in_backtrace_frame(BacktraceFrame* f, const unwindstack::FrameData& frame) {
f->set_rel_pc(frame.rel_pc);
f->set_pc(frame.pc);
f->set_sp(frame.sp);
@@ -331,21 +330,20 @@
f->set_function_offset(frame.function_offset);
- if (frame.map_start == frame.map_end) {
+ if (frame.map_info == nullptr) {
// No valid map associated with this frame.
f->set_file_name("<unknown>");
- } else if (!frame.map_name.empty()) {
- f->set_file_name(frame.map_name);
+ return;
+ }
+
+ if (!frame.map_info->name().empty()) {
+ f->set_file_name(frame.map_info->GetFullName());
} else {
- f->set_file_name(StringPrintf("<anonymous:%" PRIx64 ">", frame.map_start));
+ f->set_file_name(StringPrintf("<anonymous:%" PRIx64 ">", frame.map_info->start()));
}
+ f->set_file_map_offset(frame.map_info->elf_start_offset());
- f->set_file_map_offset(frame.map_elf_start_offset);
-
- auto map_info = maps->Find(frame.map_start);
- if (map_info.get() != nullptr) {
- f->set_build_id(map_info->GetPrintableBuildID());
- }
+ f->set_build_id(frame.map_info->GetPrintableBuildID());
}
static void dump_thread(Tombstone* tombstone, unwindstack::Unwinder* unwinder,
@@ -355,6 +353,7 @@
thread.set_id(thread_info.tid);
thread.set_name(thread_info.thread_name);
thread.set_tagged_addr_ctrl(thread_info.tagged_addr_ctrl);
+ thread.set_pac_enabled_keys(thread_info.pac_enabled_keys);
unwindstack::Maps* maps = unwinder->GetMaps();
unwindstack::Memory* memory = unwinder->GetProcessMemory().get();
@@ -434,7 +433,7 @@
unwinder->SetDisplayBuildID(true);
for (const auto& frame : unwinder->frames()) {
BacktraceFrame* f = thread.add_current_backtrace();
- fill_in_backtrace_frame(f, frame, maps);
+ fill_in_backtrace_frame(f, frame);
}
}
diff --git a/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp b/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp
index de86b0a..0265641 100644
--- a/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp
+++ b/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp
@@ -85,6 +85,10 @@
CB(should_log, "tagged_addr_ctrl: %016" PRIx64 "%s", thread.tagged_addr_ctrl(),
describe_tagged_addr_ctrl(thread.tagged_addr_ctrl()).c_str());
}
+ if (thread.pac_enabled_keys() != -1) {
+ CB(should_log, "pac_enabled_keys: %016" PRIx64 "%s", thread.pac_enabled_keys(),
+ describe_pac_enabled_keys(thread.pac_enabled_keys()).c_str());
+ }
}
static void print_register_row(CallbackType callback, int word_size,
diff --git a/debuggerd/libdebuggerd/utility.cpp b/debuggerd/libdebuggerd/utility.cpp
index 71f0c09..543a67c 100644
--- a/debuggerd/libdebuggerd/utility.cpp
+++ b/debuggerd/libdebuggerd/utility.cpp
@@ -446,31 +446,40 @@
return "?";
}
-std::string describe_tagged_addr_ctrl(long ctrl) {
+#define DESCRIBE_FLAG(flag) \
+ if (value & flag) { \
+ desc += ", "; \
+ desc += #flag; \
+ value &= ~flag; \
+ }
+
+static std::string describe_end(long value, std::string& desc) {
+ if (value) {
+ desc += StringPrintf(", unknown 0x%lx", value);
+ }
+ return desc.empty() ? "" : " (" + desc.substr(2) + ")";
+}
+
+std::string describe_tagged_addr_ctrl(long value) {
std::string desc;
- if (ctrl & PR_TAGGED_ADDR_ENABLE) {
- desc += ", PR_TAGGED_ADDR_ENABLE";
- ctrl &= ~PR_TAGGED_ADDR_ENABLE;
+ DESCRIBE_FLAG(PR_TAGGED_ADDR_ENABLE);
+ DESCRIBE_FLAG(PR_MTE_TCF_SYNC);
+ DESCRIBE_FLAG(PR_MTE_TCF_ASYNC);
+ if (value & PR_MTE_TAG_MASK) {
+ desc += StringPrintf(", mask 0x%04lx", (value & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT);
+ value &= ~PR_MTE_TAG_MASK;
}
- if (ctrl & PR_MTE_TCF_SYNC) {
- desc += ", PR_MTE_TCF_SYNC";
- ctrl &= ~PR_MTE_TCF_SYNC;
- }
- if (ctrl & PR_MTE_TCF_ASYNC) {
- desc += ", PR_MTE_TCF_ASYNC";
- ctrl &= ~PR_MTE_TCF_ASYNC;
- }
- if (ctrl & PR_MTE_TAG_MASK) {
- desc += StringPrintf(", mask 0x%04lx", (ctrl & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT);
- ctrl &= ~PR_MTE_TAG_MASK;
- }
- if (ctrl) {
- desc += StringPrintf(", unknown 0x%lx", ctrl);
- }
- if (desc.empty()) {
- return "";
- }
- return " (" + desc.substr(2) + ")";
+ return describe_end(value, desc);
+}
+
+std::string describe_pac_enabled_keys(long value) {
+ std::string desc;
+ DESCRIBE_FLAG(PR_PAC_APIAKEY);
+ DESCRIBE_FLAG(PR_PAC_APIBKEY);
+ DESCRIBE_FLAG(PR_PAC_APDAKEY);
+ DESCRIBE_FLAG(PR_PAC_APDBKEY);
+ DESCRIBE_FLAG(PR_PAC_APGAKEY);
+ return describe_end(value, desc);
}
void log_backtrace(log_t* log, unwindstack::Unwinder* unwinder, const char* prefix) {
diff --git a/debuggerd/proto/tombstone.proto b/debuggerd/proto/tombstone.proto
index a701212..40a942e 100644
--- a/debuggerd/proto/tombstone.proto
+++ b/debuggerd/proto/tombstone.proto
@@ -126,8 +126,9 @@
repeated BacktraceFrame current_backtrace = 4;
repeated MemoryDump memory_dump = 5;
int64 tagged_addr_ctrl = 6;
+ int64 pac_enabled_keys = 8;
- reserved 8 to 999;
+ reserved 9 to 999;
}
message BacktraceFrame {
diff --git a/fastboot/Android.bp b/fastboot/Android.bp
index 708a677..9ae2c37 100644
--- a/fastboot/Android.bp
+++ b/fastboot/Android.bp
@@ -15,7 +15,10 @@
// This is required because no Android.bp can include a library defined in an
// Android.mk. Eventually should kill libfastboot (defined in Android.mk)
package {
- default_applicable_licenses: ["system_core_fastboot_license"],
+ default_applicable_licenses: [
+ "system_core_fastboot_license",
+ "Android-Apache-2.0",
+ ],
}
// Added automatically by a large-scale-change that took the approach of
@@ -36,10 +39,9 @@
name: "system_core_fastboot_license",
visibility: [":__subpackages__"],
license_kinds: [
- "SPDX-license-identifier-Apache-2.0",
"SPDX-license-identifier-BSD",
],
- // large-scale-change unable to identify any license_text files
+ license_text: ["LICENSE"],
}
cc_library_host_static {
diff --git a/fastboot/LICENSE b/fastboot/LICENSE
new file mode 100644
index 0000000..f0a0e52
--- /dev/null
+++ b/fastboot/LICENSE
@@ -0,0 +1,23 @@
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ SUCH DAMAGE.
+
diff --git a/fastboot/device/flashing.cpp b/fastboot/device/flashing.cpp
index 7bef72a..44dc81f 100644
--- a/fastboot/device/flashing.cpp
+++ b/fastboot/device/flashing.cpp
@@ -119,9 +119,11 @@
}
int FlashSparseData(int fd, std::vector<char>& downloaded_data) {
- struct sparse_file* file = sparse_file_import_buf(downloaded_data.data(), true, false);
+ struct sparse_file* file = sparse_file_import_buf(downloaded_data.data(),
+ downloaded_data.size(), true, false);
if (!file) {
- return -ENOENT;
+ // Invalid sparse format
+ return -EINVAL;
}
return sparse_file_callback(file, false, false, WriteCallback, reinterpret_cast<void*>(fd));
}
diff --git a/fastboot/fuzzy_fastboot/main.cpp b/fastboot/fuzzy_fastboot/main.cpp
index 8593adc..074306b 100644
--- a/fastboot/fuzzy_fastboot/main.cpp
+++ b/fastboot/fuzzy_fastboot/main.cpp
@@ -874,6 +874,12 @@
<< "Device did not respond with FAIL for malformed download command '" << cmd << "'";
}
+TEST_F(Fuzz, DownloadInvalid9) {
+ std::string cmd("download:2PPPPPPPPPPPPPPPPPPPPPPPPPPPPPP");
+ EXPECT_EQ(fb->RawCommand(cmd), DEVICE_FAIL)
+ << "Device did not respond with FAIL for malformed download command '" << cmd << "'";
+}
+
TEST_F(Fuzz, GetVarAllSpam) {
auto start = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> elapsed;
@@ -977,6 +983,80 @@
}
}
+TEST_F(Fuzz, SparseZeroBlkSize) {
+ // handcrafted malform sparse file with zero as block size
+ const std::vector<char> buf = {
+ '\x3a', '\xff', '\x26', '\xed', '\x01', '\x00', '\x00', '\x00', '\x1c', '\x00', '\x0c', '\x00',
+ '\x00', '\x00', '\x00', '\x00', '\x01', '\x00', '\x00', '\x00', '\x01', '\x00', '\x00', '\x00',
+ '\x00', '\x00', '\x00', '\x00', '\xc2', '\xca', '\x00', '\x00', '\x01', '\x00', '\x00', '\x00',
+ '\x10', '\x00', '\x00', '\x00', '\x11', '\x22', '\x33', '\x44'
+ };
+
+ ASSERT_EQ(DownloadCommand(buf.size()), SUCCESS) << "Device rejected download command";
+ ASSERT_EQ(SendBuffer(buf), SUCCESS) << "Downloading payload failed";
+
+ // It can either reject this download or reject it during flash
+ if (HandleResponse() != DEVICE_FAIL) {
+ EXPECT_EQ(fb->Flash("userdata"), DEVICE_FAIL)
+ << "Flashing a zero block size in sparse file should fail";
+ }
+}
+
+TEST_F(Fuzz, SparseVeryLargeBlkSize) {
+ // handcrafted sparse file with block size of ~4GB and divisible 4
+ const std::vector<char> buf = {
+ '\x3a', '\xff', '\x26', '\xed', '\x01', '\x00', '\x00', '\x00',
+ '\x1c', '\x00', '\x0c', '\x00', '\xF0', '\xFF', '\xFF', '\xFF',
+ '\x01', '\x00', '\x00', '\x00', '\x01', '\x00', '\x00', '\x00',
+ '\x00', '\x00', '\x00', '\x00', '\xc3', '\xca', '\x00', '\x00',
+ '\x01', '\x00', '\x00', '\x00', '\x0c', '\x00', '\x00', '\x00',
+ '\x11', '\x22', '\x33', '\x44'
+ };
+
+ ASSERT_EQ(DownloadCommand(buf.size()), SUCCESS) << "Device rejected download command";
+ ASSERT_EQ(SendBuffer(buf), SUCCESS) << "Downloading payload failed";
+ ASSERT_EQ(HandleResponse(), SUCCESS) << "Not receive okay";
+ ASSERT_EQ(fb->Flash("userdata"), SUCCESS) << "Flashing sparse failed";
+}
+
+TEST_F(Fuzz, SparseTrimmed) {
+ // handcrafted malform sparse file which is trimmed
+ const std::vector<char> buf = {
+ '\x3a', '\xff', '\x26', '\xed', '\x01', '\x00', '\x00', '\x00', '\x1c', '\x00', '\x0c', '\x00',
+ '\x00', '\x10', '\x00', '\x00', '\x00', '\x00', '\x08', '\x00', '\x01', '\x00', '\x00', '\x00',
+ '\x00', '\x00', '\x00', '\x00', '\xc1', '\xca', '\x00', '\x00', '\x01', '\x00', '\x00', '\x00',
+ '\x00', '\x00', '\x00', '\x80', '\x11', '\x22', '\x33', '\x44'
+ };
+
+ ASSERT_EQ(DownloadCommand(buf.size()), SUCCESS) << "Device rejected download command";
+ ASSERT_EQ(SendBuffer(buf), SUCCESS) << "Downloading payload failed";
+
+ // It can either reject this download or reject it during flash
+ if (HandleResponse() != DEVICE_FAIL) {
+ EXPECT_EQ(fb->Flash("userdata"), DEVICE_FAIL)
+ << "Flashing a trimmed sparse file should fail";
+ }
+}
+
+TEST_F(Fuzz, SparseInvalidChurk) {
+ // handcrafted malform sparse file with invalid churk
+ const std::vector<char> buf = {
+ '\x3a', '\xff', '\x26', '\xed', '\x01', '\x00', '\x00', '\x00', '\x1c', '\x00', '\x0c', '\x00',
+ '\x00', '\x10', '\x00', '\x00', '\x00', '\x00', '\x08', '\x00', '\x01', '\x00', '\x00', '\x00',
+ '\x00', '\x00', '\x00', '\x00', '\xc1', '\xca', '\x00', '\x00', '\x01', '\x00', '\x00', '\x00',
+ '\x10', '\x00', '\x00', '\x00', '\x11', '\x22', '\x33', '\x44'
+ };
+
+ ASSERT_EQ(DownloadCommand(buf.size()), SUCCESS) << "Device rejected download command";
+ ASSERT_EQ(SendBuffer(buf), SUCCESS) << "Downloading payload failed";
+
+ // It can either reject this download or reject it during flash
+ if (HandleResponse() != DEVICE_FAIL) {
+ EXPECT_EQ(fb->Flash("userdata"), DEVICE_FAIL)
+ << "Flashing a sparse file with invalid churk should fail";
+ }
+}
+
TEST_F(Fuzz, SparseTooManyChunks) {
SparseWrapper sparse(4096, 4096); // 1 block, but we send two chunks that will use 2 blocks
ASSERT_TRUE(*sparse) << "Sparse image creation failed";
diff --git a/fs_mgr/libsnapshot/snapshot.cpp b/fs_mgr/libsnapshot/snapshot.cpp
index 18a9d22..f3de2b4 100644
--- a/fs_mgr/libsnapshot/snapshot.cpp
+++ b/fs_mgr/libsnapshot/snapshot.cpp
@@ -1467,6 +1467,14 @@
}
RemoveAllUpdateState(lock);
+
+ if (UpdateUsesUserSnapshots(lock) && !device()->IsTestDevice()) {
+ if (snapuserd_client_) {
+ snapuserd_client_->DetachSnapuserd();
+ snapuserd_client_->CloseConnection();
+ snapuserd_client_ = nullptr;
+ }
+ }
}
void SnapshotManager::AcknowledgeMergeFailure(MergeFailureCode failure_code) {
@@ -3200,7 +3208,7 @@
// Terminate stale daemon if any
std::unique_ptr<SnapuserdClient> snapuserd_client =
- SnapuserdClient::Connect(kSnapuserdSocket, 10s);
+ SnapuserdClient::Connect(kSnapuserdSocket, 5s);
if (snapuserd_client) {
snapuserd_client->DetachSnapuserd();
snapuserd_client->CloseConnection();
diff --git a/fs_mgr/libsnapshot/snapshot_test.cpp b/fs_mgr/libsnapshot/snapshot_test.cpp
index 11cebe1..d76558b 100644
--- a/fs_mgr/libsnapshot/snapshot_test.cpp
+++ b/fs_mgr/libsnapshot/snapshot_test.cpp
@@ -54,6 +54,8 @@
#include <libsnapshot/mock_snapshot.h>
DEFINE_string(force_config, "", "Force testing mode (dmsnap, vab, vabc) ignoring device config.");
+DEFINE_string(force_iouring_disable, "",
+ "Force testing mode (iouring_disabled) - disable io_uring");
namespace android {
namespace snapshot {
@@ -2769,10 +2771,22 @@
}
}
+ if (FLAGS_force_iouring_disable == "iouring_disabled") {
+ if (!android::base::SetProperty("snapuserd.test.io_uring.force_disable", "1")) {
+ return testing::AssertionFailure()
+ << "Failed to disable property: snapuserd.test.io_uring.disabled";
+ }
+ }
+
int ret = RUN_ALL_TESTS();
if (FLAGS_force_config == "dmsnap") {
android::base::SetProperty("snapuserd.test.dm.snapshots", "0");
}
+
+ if (FLAGS_force_iouring_disable == "iouring_disabled") {
+ android::base::SetProperty("snapuserd.test.io_uring.force_disable", "0");
+ }
+
return ret;
}
diff --git a/fs_mgr/libsnapshot/snapuserd/Android.bp b/fs_mgr/libsnapshot/snapuserd/Android.bp
index 84bcb94..bc2bceb 100644
--- a/fs_mgr/libsnapshot/snapuserd/Android.bp
+++ b/fs_mgr/libsnapshot/snapuserd/Android.bp
@@ -86,7 +86,9 @@
"libsnapshot_cow",
"libz",
"libext4_utils",
+ "liburing",
],
+ include_dirs: ["bionic/libc/kernel"],
}
cc_binary {
@@ -182,7 +184,10 @@
"libfs_mgr",
"libdm",
"libext4_utils",
+ "liburing",
+ "libgflags",
],
+ include_dirs: ["bionic/libc/kernel"],
header_libs: [
"libstorage_literals_headers",
"libfiemap_headers",
diff --git a/fs_mgr/libsnapshot/snapuserd/snapuserd_daemon.cpp b/fs_mgr/libsnapshot/snapuserd/snapuserd_daemon.cpp
index ddb1f79..a082742 100644
--- a/fs_mgr/libsnapshot/snapuserd/snapuserd_daemon.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/snapuserd_daemon.cpp
@@ -209,6 +209,8 @@
int main(int argc, char** argv) {
android::base::InitLogging(argv, &android::base::KernelLogger);
+ LOG(INFO) << "snapuserd daemon about to start";
+
android::snapshot::Daemon& daemon = android::snapshot::Daemon::Instance();
if (!daemon.StartDaemon(argc, argv)) {
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
index 95d95cd..5109d82 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.cpp
@@ -16,6 +16,10 @@
#include "snapuserd_core.h"
+#include <sys/utsname.h>
+
+#include <android-base/properties.h>
+#include <android-base/scopeguard.h>
#include <android-base/strings.h>
namespace android {
@@ -288,6 +292,136 @@
return ReadMetadata();
}
+void SnapshotHandler::FinalizeIouring() {
+ io_uring_queue_exit(ring_.get());
+}
+
+bool SnapshotHandler::InitializeIouring(int io_depth) {
+ ring_ = std::make_unique<struct io_uring>();
+
+ int ret = io_uring_queue_init(io_depth, ring_.get(), 0);
+ if (ret) {
+ LOG(ERROR) << "io_uring_queue_init failed with ret: " << ret;
+ return false;
+ }
+
+ LOG(INFO) << "io_uring_queue_init success with io_depth: " << io_depth;
+ return true;
+}
+
+bool SnapshotHandler::ReadBlocksAsync(const std::string& dm_block_device,
+ const std::string& partition_name, size_t size) {
+ // 64k block size with io_depth of 64 is optimal
+ // for a single thread. We just need a single thread
+ // to read all the blocks from all dynamic partitions.
+ size_t io_depth = 64;
+ size_t bs = (64 * 1024);
+
+ if (!InitializeIouring(io_depth)) {
+ return false;
+ }
+
+ LOG(INFO) << "ReadBlockAsync start "
+ << " Block-device: " << dm_block_device << " Partition-name: " << partition_name
+ << " Size: " << size;
+
+ auto scope_guard = android::base::make_scope_guard([this]() -> void { FinalizeIouring(); });
+
+ std::vector<std::unique_ptr<struct iovec>> vecs;
+ using AlignedBuf = std::unique_ptr<void, decltype(free)*>;
+ std::vector<AlignedBuf> alignedBufVector;
+
+ /*
+ * TODO: We need aligned memory for DIRECT-IO. However, if we do
+ * a DIRECT-IO and verify the blocks then we need to inform
+ * update-verifier that block verification has been done and
+ * there is no need to repeat the same. We are not there yet
+ * as we need to see if there are any boot time improvements doing
+ * a DIRECT-IO.
+ *
+ * Also, we could you the same function post merge for block verification;
+ * again, we can do a DIRECT-IO instead of thrashing page-cache and
+ * hurting other applications.
+ *
+ * For now, we will just create aligned buffers but rely on buffered
+ * I/O until we have perf numbers to justify DIRECT-IO.
+ */
+ for (int i = 0; i < io_depth; i++) {
+ auto iovec = std::make_unique<struct iovec>();
+ vecs.push_back(std::move(iovec));
+
+ struct iovec* iovec_ptr = vecs[i].get();
+
+ if (posix_memalign(&iovec_ptr->iov_base, BLOCK_SZ, bs)) {
+ LOG(ERROR) << "posix_memalign failed";
+ return false;
+ }
+
+ iovec_ptr->iov_len = bs;
+ alignedBufVector.push_back(
+ std::unique_ptr<void, decltype(free)*>(iovec_ptr->iov_base, free));
+ }
+
+ android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(dm_block_device.c_str(), O_RDONLY)));
+ if (fd.get() == -1) {
+ SNAP_PLOG(ERROR) << "File open failed - block-device " << dm_block_device
+ << " partition-name: " << partition_name;
+ return false;
+ }
+
+ loff_t offset = 0;
+ size_t remain = size;
+ size_t read_sz = io_depth * bs;
+
+ while (remain > 0) {
+ size_t to_read = std::min(remain, read_sz);
+ size_t queue_size = to_read / bs;
+
+ for (int i = 0; i < queue_size; i++) {
+ struct io_uring_sqe* sqe = io_uring_get_sqe(ring_.get());
+ if (!sqe) {
+ SNAP_LOG(ERROR) << "io_uring_get_sqe() failed";
+ return false;
+ }
+
+ struct iovec* iovec_ptr = vecs[i].get();
+
+ io_uring_prep_read(sqe, fd.get(), iovec_ptr->iov_base, iovec_ptr->iov_len, offset);
+ sqe->flags |= IOSQE_ASYNC;
+ offset += bs;
+ }
+
+ int ret = io_uring_submit(ring_.get());
+ if (ret != queue_size) {
+ SNAP_LOG(ERROR) << "submit got: " << ret << " wanted: " << queue_size;
+ return false;
+ }
+
+ for (int i = 0; i < queue_size; i++) {
+ struct io_uring_cqe* cqe;
+
+ int ret = io_uring_wait_cqe(ring_.get(), &cqe);
+ if (ret) {
+ SNAP_PLOG(ERROR) << "wait_cqe failed" << ret;
+ return false;
+ }
+
+ if (cqe->res < 0) {
+ SNAP_LOG(ERROR) << "io failed with res: " << cqe->res;
+ return false;
+ }
+ io_uring_cqe_seen(ring_.get(), cqe);
+ }
+
+ remain -= to_read;
+ }
+
+ LOG(INFO) << "ReadBlockAsync complete: "
+ << " Block-device: " << dm_block_device << " Partition-name: " << partition_name
+ << " Size: " << size;
+ return true;
+}
+
void SnapshotHandler::ReadBlocksToCache(const std::string& dm_block_device,
const std::string& partition_name, off_t offset,
size_t size) {
@@ -344,17 +478,22 @@
return;
}
- int num_threads = 2;
- size_t num_blocks = dev_sz >> BLOCK_SHIFT;
- size_t num_blocks_per_thread = num_blocks / num_threads;
- size_t read_sz_per_thread = num_blocks_per_thread << BLOCK_SHIFT;
- off_t offset = 0;
+ if (IsIouringSupported()) {
+ std::async(std::launch::async, &SnapshotHandler::ReadBlocksAsync, this, dm_block_device,
+ partition_name, dev_sz);
+ } else {
+ int num_threads = 2;
+ size_t num_blocks = dev_sz >> BLOCK_SHIFT;
+ size_t num_blocks_per_thread = num_blocks / num_threads;
+ size_t read_sz_per_thread = num_blocks_per_thread << BLOCK_SHIFT;
+ off_t offset = 0;
- for (int i = 0; i < num_threads; i++) {
- std::async(std::launch::async, &SnapshotHandler::ReadBlocksToCache, this, dm_block_device,
- partition_name, offset, read_sz_per_thread);
+ for (int i = 0; i < num_threads; i++) {
+ std::async(std::launch::async, &SnapshotHandler::ReadBlocksToCache, this,
+ dm_block_device, partition_name, offset, read_sz_per_thread);
- offset += read_sz_per_thread;
+ offset += read_sz_per_thread;
+ }
}
}
@@ -513,5 +652,33 @@
return ra_state;
}
+bool SnapshotHandler::IsIouringSupported() {
+ struct utsname uts;
+ unsigned int major, minor;
+
+ if (android::base::GetBoolProperty("snapuserd.test.io_uring.force_disable", false)) {
+ SNAP_LOG(INFO) << "io_uring disabled for testing";
+ return false;
+ }
+
+ if ((uname(&uts) != 0) || (sscanf(uts.release, "%u.%u", &major, &minor) != 2)) {
+ SNAP_LOG(ERROR) << "Could not parse the kernel version from uname. "
+ << " io_uring not supported";
+ return false;
+ }
+
+ // We will only support kernels from 5.6 onwards as IOSQE_ASYNC flag and
+ // IO_URING_OP_READ/WRITE opcodes were introduced only on 5.6 kernel
+ if (major >= 5) {
+ if (major == 5 && minor < 6) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+
+ return android::base::GetBoolProperty("ro.virtual_ab.io_uring.enabled", false);
+}
+
} // namespace snapshot
} // namespace android
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
index 1953316..b0f2d65 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_core.h
@@ -39,6 +39,7 @@
#include <libdm/dm.h>
#include <libsnapshot/cow_reader.h>
#include <libsnapshot/cow_writer.h>
+#include <liburing.h>
#include <snapuserd/snapuserd_buffer.h>
#include <snapuserd/snapuserd_kernel.h>
@@ -113,6 +114,19 @@
bool ReconstructDataFromCow();
void CheckOverlap(const CowOperation* cow_op);
+ bool ReadAheadAsyncIO();
+ bool ReapIoCompletions(int pending_ios_to_complete);
+ bool ReadXorData(size_t block_index, size_t xor_op_index,
+ std::vector<const CowOperation*>& xor_op_vec);
+ void ProcessXorData(size_t& block_xor_index, size_t& xor_index,
+ std::vector<const CowOperation*>& xor_op_vec, void* buffer,
+ loff_t& buffer_offset);
+ void UpdateScratchMetadata();
+
+ bool ReadAheadSyncIO();
+ bool InitializeIouring();
+ void FinalizeIouring();
+
void* read_ahead_buffer_;
void* metadata_buffer_;
@@ -131,7 +145,19 @@
std::unordered_set<uint64_t> dest_blocks_;
std::unordered_set<uint64_t> source_blocks_;
bool overlap_;
+ std::vector<uint64_t> blocks_;
+ int total_blocks_merged_ = 0;
+ std::unique_ptr<uint8_t[]> ra_temp_buffer_;
+ std::unique_ptr<uint8_t[]> ra_temp_meta_buffer_;
BufferSink bufsink_;
+
+ bool read_ahead_async_ = false;
+ // Queue depth of 32 seems optimal. We don't want
+ // to have a huge depth as it may put more memory pressure
+ // on the kernel worker threads given that we use
+ // IOSQE_ASYNC flag.
+ int queue_depth_ = 32;
+ std::unique_ptr<struct io_uring> ring_;
};
class Worker {
@@ -185,6 +211,7 @@
// Merge related ops
bool Merge();
bool MergeOrderedOps(const std::unique_ptr<ICowOpIter>& cowop_iter);
+ bool MergeOrderedOpsAsync(const std::unique_ptr<ICowOpIter>& cowop_iter);
bool MergeReplaceZeroOps(const std::unique_ptr<ICowOpIter>& cowop_iter);
int PrepareMerge(uint64_t* source_offset, int* pending_ops,
const std::unique_ptr<ICowOpIter>& cowop_iter,
@@ -193,6 +220,9 @@
sector_t ChunkToSector(chunk_t chunk) { return chunk << CHUNK_SHIFT; }
chunk_t SectorToChunk(sector_t sector) { return sector >> CHUNK_SHIFT; }
+ bool InitializeIouring();
+ void FinalizeIouring();
+
std::unique_ptr<CowReader> reader_;
BufferSink bufsink_;
XorSink xorsink_;
@@ -208,6 +238,14 @@
unique_fd base_path_merge_fd_;
unique_fd ctrl_fd_;
+ bool merge_async_ = false;
+ // Queue depth of 32 seems optimal. We don't want
+ // to have a huge depth as it may put more memory pressure
+ // on the kernel worker threads given that we use
+ // IOSQE_ASYNC flag.
+ int queue_depth_ = 32;
+ std::unique_ptr<struct io_uring> ring_;
+
std::shared_ptr<SnapshotHandler> snapuserd_;
};
@@ -292,6 +330,8 @@
bool GetRABuffer(std::unique_lock<std::mutex>* lock, uint64_t block, void* buffer);
MERGE_GROUP_STATE ProcessMergingBlock(uint64_t new_block, void* buffer);
+ bool IsIouringSupported();
+
private:
bool ReadMetadata();
sector_t ChunkToSector(chunk_t chunk) { return chunk << CHUNK_SHIFT; }
@@ -304,6 +344,11 @@
void ReadBlocksToCache(const std::string& dm_block_device, const std::string& partition_name,
off_t offset, size_t size);
+ bool InitializeIouring(int io_depth);
+ void FinalizeIouring();
+ bool ReadBlocksAsync(const std::string& dm_block_device, const std::string& partition_name,
+ size_t size);
+
// COW device
std::string cow_device_;
// Source device
@@ -352,6 +397,8 @@
bool attached_ = false;
bool is_socket_present_;
bool scratch_space_ = false;
+
+ std::unique_ptr<struct io_uring> ring_;
};
} // namespace snapshot
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.cpp
index fa055b7..d4d4efe 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_merge.cpp
@@ -72,16 +72,16 @@
}
bool Worker::MergeReplaceZeroOps(const std::unique_ptr<ICowOpIter>& cowop_iter) {
- // Flush every 2048 ops. Since all ops are independent and there is no
+ // Flush every 8192 ops. Since all ops are independent and there is no
// dependency between COW ops, we will flush the data and the number
- // of ops merged in COW file for every 2048 ops. If there is a crash,
+ // of ops merged in COW file for every 8192 ops. If there is a crash,
// we will end up replaying some of the COW ops which were already merged.
// That is ok.
//
- // Why 2048 ops ? We can probably increase this to bigger value but just
- // need to ensure that merge makes forward progress if there are
- // crashes repeatedly which is highly unlikely.
- int total_ops_merged_per_commit = (PAYLOAD_BUFFER_SZ / BLOCK_SZ) * 8;
+ // Why 8192 ops ? Increasing this may improve merge time 3-4 seconds but
+ // we need to make sure that we checkpoint; 8k ops seems optimal. In-case
+ // if there is a crash merge should always make forward progress.
+ int total_ops_merged_per_commit = (PAYLOAD_BUFFER_SZ / BLOCK_SZ) * 32;
int num_ops_merged = 0;
while (!cowop_iter->Done()) {
@@ -128,7 +128,7 @@
num_ops_merged += linear_blocks;
- if (num_ops_merged == total_ops_merged_per_commit) {
+ if (num_ops_merged >= total_ops_merged_per_commit) {
// Flush the data
if (fsync(base_path_merge_fd_.get()) < 0) {
SNAP_LOG(ERROR) << "Merge: ReplaceZeroOps: Failed to fsync merged data";
@@ -172,6 +172,173 @@
return true;
}
+bool Worker::MergeOrderedOpsAsync(const std::unique_ptr<ICowOpIter>& cowop_iter) {
+ void* mapped_addr = snapuserd_->GetMappedAddr();
+ void* read_ahead_buffer =
+ static_cast<void*>((char*)mapped_addr + snapuserd_->GetBufferDataOffset());
+ size_t block_index = 0;
+
+ SNAP_LOG(INFO) << "MergeOrderedOpsAsync started....";
+
+ while (!cowop_iter->Done()) {
+ const CowOperation* cow_op = &cowop_iter->Get();
+ if (!IsOrderedOp(*cow_op)) {
+ break;
+ }
+
+ SNAP_LOG(DEBUG) << "Waiting for merge begin...";
+ // Wait for RA thread to notify that the merge window
+ // is ready for merging.
+ if (!snapuserd_->WaitForMergeBegin()) {
+ snapuserd_->SetMergeFailed(block_index);
+ return false;
+ }
+
+ snapuserd_->SetMergeInProgress(block_index);
+
+ loff_t offset = 0;
+ int num_ops = snapuserd_->GetTotalBlocksToMerge();
+
+ int pending_sqe = queue_depth_;
+ int pending_ios_to_submit = 0;
+ bool flush_required = false;
+
+ SNAP_LOG(DEBUG) << "Merging copy-ops of size: " << num_ops;
+ while (num_ops) {
+ uint64_t source_offset;
+
+ int linear_blocks = PrepareMerge(&source_offset, &num_ops, cowop_iter);
+
+ if (linear_blocks != 0) {
+ size_t io_size = (linear_blocks * BLOCK_SZ);
+
+ // Get an SQE entry from the ring and populate the I/O variables
+ struct io_uring_sqe* sqe = io_uring_get_sqe(ring_.get());
+ if (!sqe) {
+ SNAP_PLOG(ERROR) << "io_uring_get_sqe failed during merge-ordered ops";
+ snapuserd_->SetMergeFailed(block_index);
+ return false;
+ }
+
+ io_uring_prep_write(sqe, base_path_merge_fd_.get(),
+ (char*)read_ahead_buffer + offset, io_size, source_offset);
+
+ offset += io_size;
+ num_ops -= linear_blocks;
+
+ pending_sqe -= 1;
+ pending_ios_to_submit += 1;
+ sqe->flags |= IOSQE_ASYNC;
+ }
+
+ // Ring is full or no more COW ops to be merged in this batch
+ if (pending_sqe == 0 || num_ops == 0 || (linear_blocks == 0 && pending_ios_to_submit)) {
+ // If this is a last set of COW ops to be merged in this batch, we need
+ // to sync the merged data. We will try to grab an SQE entry
+ // and set the FSYNC command; additionally, make sure that
+ // the fsync is done after all the I/O operations queued
+ // in the ring is completed by setting IOSQE_IO_DRAIN.
+ //
+ // If there is no space in the ring, we will flush it later
+ // by explicitly calling fsync() system call.
+ if (num_ops == 0 || (linear_blocks == 0 && pending_ios_to_submit)) {
+ if (pending_sqe != 0) {
+ struct io_uring_sqe* sqe = io_uring_get_sqe(ring_.get());
+ if (!sqe) {
+ // very unlikely but let's continue and not fail the
+ // merge - we will flush it later
+ SNAP_PLOG(ERROR) << "io_uring_get_sqe failed during merge-ordered ops";
+ flush_required = true;
+ } else {
+ io_uring_prep_fsync(sqe, base_path_merge_fd_.get(), 0);
+ // Drain the queue before fsync
+ io_uring_sqe_set_flags(sqe, IOSQE_IO_DRAIN);
+ pending_sqe -= 1;
+ flush_required = false;
+ pending_ios_to_submit += 1;
+ sqe->flags |= IOSQE_ASYNC;
+ }
+ } else {
+ flush_required = true;
+ }
+ }
+
+ // Submit the IO for all the COW ops in a single syscall
+ int ret = io_uring_submit(ring_.get());
+ if (ret != pending_ios_to_submit) {
+ SNAP_PLOG(ERROR)
+ << "io_uring_submit failed for read-ahead: "
+ << " io submit: " << ret << " expected: " << pending_ios_to_submit;
+ snapuserd_->SetMergeFailed(block_index);
+ return false;
+ }
+
+ int pending_ios_to_complete = pending_ios_to_submit;
+ pending_ios_to_submit = 0;
+
+ // Reap I/O completions
+ while (pending_ios_to_complete) {
+ struct io_uring_cqe* cqe;
+
+ ret = io_uring_wait_cqe(ring_.get(), &cqe);
+ if (ret) {
+ SNAP_LOG(ERROR) << "Read-ahead - io_uring_wait_cqe failed: " << ret;
+ snapuserd_->SetMergeFailed(block_index);
+ return false;
+ }
+
+ if (cqe->res < 0) {
+ SNAP_LOG(ERROR)
+ << "Read-ahead - io_uring_Wait_cqe failed with res: " << cqe->res;
+ snapuserd_->SetMergeFailed(block_index);
+ return false;
+ }
+
+ io_uring_cqe_seen(ring_.get(), cqe);
+ pending_ios_to_complete -= 1;
+ }
+
+ pending_sqe = queue_depth_;
+ }
+
+ if (linear_blocks == 0) {
+ break;
+ }
+ }
+
+ // Verify all ops are merged
+ CHECK(num_ops == 0);
+
+ // Flush the data
+ if (flush_required && (fsync(base_path_merge_fd_.get()) < 0)) {
+ SNAP_LOG(ERROR) << " Failed to fsync merged data";
+ snapuserd_->SetMergeFailed(block_index);
+ return false;
+ }
+
+ // Merge is done and data is on disk. Update the COW Header about
+ // the merge completion
+ if (!snapuserd_->CommitMerge(snapuserd_->GetTotalBlocksToMerge())) {
+ SNAP_LOG(ERROR) << " Failed to commit the merged block in the header";
+ snapuserd_->SetMergeFailed(block_index);
+ return false;
+ }
+
+ SNAP_LOG(DEBUG) << "Block commit of size: " << snapuserd_->GetTotalBlocksToMerge();
+ // Mark the block as merge complete
+ snapuserd_->SetMergeCompleted(block_index);
+
+ // Notify RA thread that the merge thread is ready to merge the next
+ // window
+ snapuserd_->NotifyRAForMergeReady();
+
+ // Get the next block
+ block_index += 1;
+ }
+
+ return true;
+}
+
bool Worker::MergeOrderedOps(const std::unique_ptr<ICowOpIter>& cowop_iter) {
void* mapped_addr = snapuserd_->GetMappedAddr();
void* read_ahead_buffer =
@@ -260,15 +427,23 @@
bool Worker::Merge() {
std::unique_ptr<ICowOpIter> cowop_iter = reader_->GetMergeOpIter();
- // Start with Copy and Xor ops
- if (!MergeOrderedOps(cowop_iter)) {
- SNAP_LOG(ERROR) << "Merge failed for ordered ops";
- snapuserd_->MergeFailed();
- return false;
+ if (merge_async_) {
+ if (!MergeOrderedOpsAsync(cowop_iter)) {
+ SNAP_LOG(ERROR) << "Merge failed for ordered ops";
+ snapuserd_->MergeFailed();
+ return false;
+ }
+ SNAP_LOG(INFO) << "MergeOrderedOpsAsync completed.....";
+ } else {
+ // Start with Copy and Xor ops
+ if (!MergeOrderedOps(cowop_iter)) {
+ SNAP_LOG(ERROR) << "Merge failed for ordered ops";
+ snapuserd_->MergeFailed();
+ return false;
+ }
+ SNAP_LOG(INFO) << "MergeOrderedOps completed.....";
}
- SNAP_LOG(INFO) << "MergeOrderedOps completed...";
-
// Replace and Zero ops
if (!MergeReplaceZeroOps(cowop_iter)) {
SNAP_LOG(ERROR) << "Merge failed for replace/zero ops";
@@ -281,6 +456,31 @@
return true;
}
+bool Worker::InitializeIouring() {
+ if (!snapuserd_->IsIouringSupported()) {
+ return false;
+ }
+
+ ring_ = std::make_unique<struct io_uring>();
+
+ int ret = io_uring_queue_init(queue_depth_, ring_.get(), 0);
+ if (ret) {
+ LOG(ERROR) << "Merge: io_uring_queue_init failed with ret: " << ret;
+ return false;
+ }
+
+ merge_async_ = true;
+
+ LOG(INFO) << "Merge: io_uring initialized with queue depth: " << queue_depth_;
+ return true;
+}
+
+void Worker::FinalizeIouring() {
+ if (merge_async_) {
+ io_uring_queue_exit(ring_.get());
+ }
+}
+
bool Worker::RunMergeThread() {
SNAP_LOG(DEBUG) << "Waiting for merge begin...";
if (!snapuserd_->WaitForMergeBegin()) {
@@ -296,10 +496,13 @@
return false;
}
+ InitializeIouring();
+
if (!Merge()) {
return false;
}
+ FinalizeIouring();
CloseFds();
reader_->CloseCowFd();
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
index 9e8ccfb..26c5f19 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_readahead.cpp
@@ -183,25 +183,311 @@
return true;
}
-bool ReadAhead::ReadAheadIOStart() {
- // Check if the data has to be constructed from the COW file.
- // This will be true only once during boot up after a crash
- // during merge.
- if (snapuserd_->ShouldReconstructDataFromCow()) {
- return ReconstructDataFromCow();
- }
+/*
+ * With io_uring, the data flow is slightly different.
+ *
+ * The data flow is as follows:
+ *
+ * 1: Queue the I/O requests to be read from backing source device.
+ * This is done by retrieving the SQE entry from ring and populating
+ * the SQE entry. Note that the I/O is not submitted yet.
+ *
+ * 2: Once the ring is full (aka queue_depth), we will submit all
+ * the queued I/O request with a single system call. This essentially
+ * cuts down "queue_depth" number of system calls to a single system call.
+ *
+ * 3: Once the I/O is submitted, user-space thread will now work
+ * on processing the XOR Operations. This happens in parallel when
+ * I/O requests are submitted to the kernel. This is ok because, for XOR
+ * operations, we first need to retrieve the compressed data form COW block
+ * device. Thus, we have offloaded the backing source I/O to the kernel
+ * and user-space is parallely working on fetching the data for XOR operations.
+ *
+ * 4: After the XOR operations are read from COW device, poll the completion
+ * queue for all the I/O submitted. If the I/O's were already completed,
+ * then user-space thread will just read the CQE requests from the ring
+ * without doing any system call. If none of the I/O were completed yet,
+ * user-space thread will do a system call and wait for I/O completions.
+ *
+ * Flow diagram:
+ * SQ-RING
+ * SQE1 <----------- Fetch SQE1 Entry ---------- |SQE1||SQE2|SQE3|
+ *
+ * SQE1 ------------ Populate SQE1 Entry ------> |SQE1-X||SQE2|SQE3|
+ *
+ * SQE2 <----------- Fetch SQE2 Entry ---------- |SQE1-X||SQE2|SQE3|
+ *
+ * SQE2 ------------ Populate SQE2 Entry ------> |SQE1-X||SQE2-X|SQE3|
+ *
+ * SQE3 <----------- Fetch SQE3 Entry ---------- |SQE1-X||SQE2-X|SQE3|
+ *
+ * SQE3 ------------ Populate SQE3 Entry ------> |SQE1-X||SQE2-X|SQE3-X|
+ *
+ * Submit-IO ---------------------------------> |SQE1-X||SQE2-X|SQE3-X|
+ * | |
+ * | Process I/O entries in kernel
+ * | |
+ * Retrieve XOR |
+ * data from COW |
+ * | |
+ * | |
+ * Fetch CQ completions
+ * | CQ-RING
+ * |CQE1-X||CQE2-X|CQE3-X|
+ * |
+ * CQE1 <------------Fetch CQE1 Entry |CQE1||CQE2-X|CQE3-X|
+ * CQE2 <------------Fetch CQE2 Entry |CQE1||CQE2-|CQE3-X|
+ * CQE3 <------------Fetch CQE3 Entry |CQE1||CQE2-|CQE3-|
+ * |
+ * |
+ * Continue Next set of operations in the RING
+ */
- std::vector<uint64_t> blocks;
-
+bool ReadAhead::ReadAheadAsyncIO() {
int num_ops = (snapuserd_->GetBufferDataSize()) / BLOCK_SZ;
loff_t buffer_offset = 0;
- int total_blocks_merged = 0;
+ total_blocks_merged_ = 0;
overlap_ = false;
dest_blocks_.clear();
source_blocks_.clear();
+ blocks_.clear();
std::vector<const CowOperation*> xor_op_vec;
- auto ra_temp_buffer = std::make_unique<uint8_t[]>(snapuserd_->GetBufferDataSize());
+ int pending_sqe = queue_depth_;
+ int pending_ios_to_submit = 0;
+
+ size_t xor_op_index = 0;
+ size_t block_index = 0;
+
+ loff_t offset = 0;
+
+ bufsink_.ResetBufferOffset();
+
+ // Number of ops to be merged in this window. This is a fixed size
+ // except for the last window wherein the number of ops can be less
+ // than the size of the RA window.
+ while (num_ops) {
+ uint64_t source_offset;
+ struct io_uring_sqe* sqe;
+
+ int linear_blocks = PrepareNextReadAhead(&source_offset, &num_ops, blocks_, xor_op_vec);
+
+ if (linear_blocks != 0) {
+ size_t io_size = (linear_blocks * BLOCK_SZ);
+
+ // Get an SQE entry from the ring and populate the I/O variables
+ sqe = io_uring_get_sqe(ring_.get());
+ if (!sqe) {
+ SNAP_PLOG(ERROR) << "io_uring_get_sqe failed during read-ahead";
+ snapuserd_->ReadAheadIOFailed();
+ return false;
+ }
+
+ io_uring_prep_read(sqe, backing_store_fd_.get(),
+ (char*)ra_temp_buffer_.get() + buffer_offset, io_size,
+ source_offset);
+
+ buffer_offset += io_size;
+ num_ops -= linear_blocks;
+ total_blocks_merged_ += linear_blocks;
+
+ pending_sqe -= 1;
+ pending_ios_to_submit += 1;
+ sqe->flags |= IOSQE_ASYNC;
+ }
+
+ // pending_sqe == 0 : Ring is full
+ //
+ // num_ops == 0 : All the COW ops in this batch are processed - Submit
+ // pending I/O requests in the ring
+ //
+ // linear_blocks == 0 : All the COW ops processing is done. Submit
+ // pending I/O requests in the ring
+ if (pending_sqe == 0 || num_ops == 0 || (linear_blocks == 0 && pending_ios_to_submit)) {
+ // Submit the IO for all the COW ops in a single syscall
+ int ret = io_uring_submit(ring_.get());
+ if (ret != pending_ios_to_submit) {
+ SNAP_PLOG(ERROR) << "io_uring_submit failed for read-ahead: "
+ << " io submit: " << ret << " expected: " << pending_ios_to_submit;
+ snapuserd_->ReadAheadIOFailed();
+ return false;
+ }
+
+ int pending_ios_to_complete = pending_ios_to_submit;
+ pending_ios_to_submit = 0;
+
+ bool xor_processing_required = (xor_op_vec.size() > 0);
+
+ // Read XOR data from COW file in parallel when I/O's are in-flight
+ if (xor_processing_required && !ReadXorData(block_index, xor_op_index, xor_op_vec)) {
+ SNAP_LOG(ERROR) << "ReadXorData failed";
+ snapuserd_->ReadAheadIOFailed();
+ return false;
+ }
+
+ // Fetch I/O completions
+ if (!ReapIoCompletions(pending_ios_to_complete)) {
+ SNAP_LOG(ERROR) << "ReapIoCompletions failed";
+ snapuserd_->ReadAheadIOFailed();
+ return false;
+ }
+
+ // Retrieve XOR'ed data
+ if (xor_processing_required) {
+ ProcessXorData(block_index, xor_op_index, xor_op_vec, ra_temp_buffer_.get(),
+ offset);
+ }
+
+ // All the I/O in the ring is processed.
+ pending_sqe = queue_depth_;
+ }
+
+ if (linear_blocks == 0) {
+ break;
+ }
+ }
+
+ // Done with merging ordered ops
+ if (RAIterDone() && total_blocks_merged_ == 0) {
+ return true;
+ }
+
+ CHECK(blocks_.size() == total_blocks_merged_);
+
+ UpdateScratchMetadata();
+
+ return true;
+}
+
+void ReadAhead::UpdateScratchMetadata() {
+ loff_t metadata_offset = 0;
+
+ struct ScratchMetadata* bm = reinterpret_cast<struct ScratchMetadata*>(
+ (char*)ra_temp_meta_buffer_.get() + metadata_offset);
+
+ bm->new_block = 0;
+ bm->file_offset = 0;
+
+ loff_t file_offset = snapuserd_->GetBufferDataOffset();
+
+ for (size_t block_index = 0; block_index < blocks_.size(); block_index++) {
+ uint64_t new_block = blocks_[block_index];
+ // Track the metadata blocks which are stored in scratch space
+ bm = reinterpret_cast<struct ScratchMetadata*>((char*)ra_temp_meta_buffer_.get() +
+ metadata_offset);
+
+ bm->new_block = new_block;
+ bm->file_offset = file_offset;
+
+ metadata_offset += sizeof(struct ScratchMetadata);
+ file_offset += BLOCK_SZ;
+ }
+
+ // This is important - explicitly set the contents to zero. This is used
+ // when re-constructing the data after crash. This indicates end of
+ // reading metadata contents when re-constructing the data
+ bm = reinterpret_cast<struct ScratchMetadata*>((char*)ra_temp_meta_buffer_.get() +
+ metadata_offset);
+ bm->new_block = 0;
+ bm->file_offset = 0;
+}
+
+bool ReadAhead::ReapIoCompletions(int pending_ios_to_complete) {
+ // Reap I/O completions
+ while (pending_ios_to_complete) {
+ struct io_uring_cqe* cqe;
+
+ int ret = io_uring_wait_cqe(ring_.get(), &cqe);
+ if (ret) {
+ SNAP_LOG(ERROR) << "Read-ahead - io_uring_wait_cqe failed: " << ret;
+ return false;
+ }
+
+ if (cqe->res < 0) {
+ SNAP_LOG(ERROR) << "Read-ahead - io_uring_Wait_cqe failed with res: " << cqe->res;
+ return false;
+ }
+
+ io_uring_cqe_seen(ring_.get(), cqe);
+ pending_ios_to_complete -= 1;
+ }
+
+ return true;
+}
+
+void ReadAhead::ProcessXorData(size_t& block_xor_index, size_t& xor_index,
+ std::vector<const CowOperation*>& xor_op_vec, void* buffer,
+ loff_t& buffer_offset) {
+ loff_t xor_buf_offset = 0;
+
+ while (block_xor_index < blocks_.size()) {
+ void* bufptr = static_cast<void*>((char*)buffer + buffer_offset);
+ uint64_t new_block = blocks_[block_xor_index];
+
+ if (xor_index < xor_op_vec.size()) {
+ const CowOperation* xor_op = xor_op_vec[xor_index];
+
+ // Check if this block is an XOR op
+ if (xor_op->new_block == new_block) {
+ // Pointer to the data read from base device
+ uint8_t* buffer = reinterpret_cast<uint8_t*>(bufptr);
+ // Get the xor'ed data read from COW device
+ uint8_t* xor_data = reinterpret_cast<uint8_t*>((char*)bufsink_.GetPayloadBufPtr() +
+ xor_buf_offset);
+
+ for (size_t byte_offset = 0; byte_offset < BLOCK_SZ; byte_offset++) {
+ buffer[byte_offset] ^= xor_data[byte_offset];
+ }
+
+ // Move to next XOR op
+ xor_index += 1;
+ xor_buf_offset += BLOCK_SZ;
+ }
+ }
+
+ buffer_offset += BLOCK_SZ;
+ block_xor_index += 1;
+ }
+
+ bufsink_.ResetBufferOffset();
+}
+
+bool ReadAhead::ReadXorData(size_t block_index, size_t xor_op_index,
+ std::vector<const CowOperation*>& xor_op_vec) {
+ // Process the XOR ops in parallel - We will be reading data
+ // from COW file for XOR ops processing.
+ while (block_index < blocks_.size()) {
+ uint64_t new_block = blocks_[block_index];
+
+ if (xor_op_index < xor_op_vec.size()) {
+ const CowOperation* xor_op = xor_op_vec[xor_op_index];
+ if (xor_op->new_block == new_block) {
+ if (!reader_->ReadData(*xor_op, &bufsink_)) {
+ SNAP_LOG(ERROR)
+ << " ReadAhead - XorOp Read failed for block: " << xor_op->new_block;
+ return false;
+ }
+
+ xor_op_index += 1;
+ bufsink_.UpdateBufferOffset(BLOCK_SZ);
+ }
+ }
+ block_index += 1;
+ }
+ return true;
+}
+
+bool ReadAhead::ReadAheadSyncIO() {
+ int num_ops = (snapuserd_->GetBufferDataSize()) / BLOCK_SZ;
+ loff_t buffer_offset = 0;
+ total_blocks_merged_ = 0;
+ overlap_ = false;
+ dest_blocks_.clear();
+ source_blocks_.clear();
+ blocks_.clear();
+ std::vector<const CowOperation*> xor_op_vec;
+
+ bufsink_.ResetBufferOffset();
// Number of ops to be merged in this window. This is a fixed size
// except for the last window wherein the number of ops can be less
@@ -209,7 +495,7 @@
while (num_ops) {
uint64_t source_offset;
- int linear_blocks = PrepareNextReadAhead(&source_offset, &num_ops, blocks, xor_op_vec);
+ int linear_blocks = PrepareNextReadAhead(&source_offset, &num_ops, blocks_, xor_op_vec);
if (linear_blocks == 0) {
// No more blocks to read
SNAP_LOG(DEBUG) << " Read-ahead completed....";
@@ -220,7 +506,7 @@
// Read from the base device consecutive set of blocks in one shot
if (!android::base::ReadFullyAtOffset(backing_store_fd_,
- (char*)ra_temp_buffer.get() + buffer_offset, io_size,
+ (char*)ra_temp_buffer_.get() + buffer_offset, io_size,
source_offset)) {
SNAP_PLOG(ERROR) << "Ordered-op failed. Read from backing store: "
<< backing_store_device_ << "at block :" << source_offset / BLOCK_SZ
@@ -233,21 +519,19 @@
}
buffer_offset += io_size;
- total_blocks_merged += linear_blocks;
+ total_blocks_merged_ += linear_blocks;
num_ops -= linear_blocks;
}
// Done with merging ordered ops
- if (RAIterDone() && total_blocks_merged == 0) {
+ if (RAIterDone() && total_blocks_merged_ == 0) {
return true;
}
loff_t metadata_offset = 0;
- auto ra_temp_meta_buffer = std::make_unique<uint8_t[]>(snapuserd_->GetBufferMetadataSize());
-
struct ScratchMetadata* bm = reinterpret_cast<struct ScratchMetadata*>(
- (char*)ra_temp_meta_buffer.get() + metadata_offset);
+ (char*)ra_temp_meta_buffer_.get() + metadata_offset);
bm->new_block = 0;
bm->file_offset = 0;
@@ -255,12 +539,15 @@
loff_t file_offset = snapuserd_->GetBufferDataOffset();
loff_t offset = 0;
- CHECK(blocks.size() == total_blocks_merged);
+ CHECK(blocks_.size() == total_blocks_merged_);
size_t xor_index = 0;
- for (size_t block_index = 0; block_index < blocks.size(); block_index++) {
- void* bufptr = static_cast<void*>((char*)ra_temp_buffer.get() + offset);
- uint64_t new_block = blocks[block_index];
+ BufferSink bufsink;
+ bufsink.Initialize(BLOCK_SZ * 2);
+
+ for (size_t block_index = 0; block_index < blocks_.size(); block_index++) {
+ void* bufptr = static_cast<void*>((char*)ra_temp_buffer_.get() + offset);
+ uint64_t new_block = blocks_[block_index];
if (xor_index < xor_op_vec.size()) {
const CowOperation* xor_op = xor_op_vec[xor_index];
@@ -268,17 +555,16 @@
// Check if this block is an XOR op
if (xor_op->new_block == new_block) {
// Read the xor'ed data from COW
- if (!reader_->ReadData(*xor_op, &bufsink_)) {
+ if (!reader_->ReadData(*xor_op, &bufsink)) {
SNAP_LOG(ERROR)
<< " ReadAhead - XorOp Read failed for block: " << xor_op->new_block;
snapuserd_->ReadAheadIOFailed();
return false;
}
-
// Pointer to the data read from base device
uint8_t* buffer = reinterpret_cast<uint8_t*>(bufptr);
// Get the xor'ed data read from COW device
- uint8_t* xor_data = reinterpret_cast<uint8_t*>(bufsink_.GetPayloadBufPtr());
+ uint8_t* xor_data = reinterpret_cast<uint8_t*>(bufsink.GetPayloadBufPtr());
// Retrieve the original data
for (size_t byte_offset = 0; byte_offset < BLOCK_SZ; byte_offset++) {
@@ -292,7 +578,7 @@
offset += BLOCK_SZ;
// Track the metadata blocks which are stored in scratch space
- bm = reinterpret_cast<struct ScratchMetadata*>((char*)ra_temp_meta_buffer.get() +
+ bm = reinterpret_cast<struct ScratchMetadata*>((char*)ra_temp_meta_buffer_.get() +
metadata_offset);
bm->new_block = new_block;
@@ -308,11 +594,34 @@
// This is important - explicitly set the contents to zero. This is used
// when re-constructing the data after crash. This indicates end of
// reading metadata contents when re-constructing the data
- bm = reinterpret_cast<struct ScratchMetadata*>((char*)ra_temp_meta_buffer.get() +
+ bm = reinterpret_cast<struct ScratchMetadata*>((char*)ra_temp_meta_buffer_.get() +
metadata_offset);
bm->new_block = 0;
bm->file_offset = 0;
+ return true;
+}
+
+bool ReadAhead::ReadAheadIOStart() {
+ // Check if the data has to be constructed from the COW file.
+ // This will be true only once during boot up after a crash
+ // during merge.
+ if (snapuserd_->ShouldReconstructDataFromCow()) {
+ return ReconstructDataFromCow();
+ }
+
+ if (read_ahead_async_) {
+ if (!ReadAheadAsyncIO()) {
+ SNAP_LOG(ERROR) << "ReadAheadAsyncIO failed - io_uring processing failure.";
+ return false;
+ }
+ } else {
+ if (!ReadAheadSyncIO()) {
+ SNAP_LOG(ERROR) << "ReadAheadSyncIO failed";
+ return false;
+ }
+ }
+
// Wait for the merge to finish for the previous RA window. We shouldn't
// be touching the scratch space until merge is complete of previous RA
// window. If there is a crash during this time frame, merge should resume
@@ -322,22 +631,22 @@
}
// Copy the data to scratch space
- memcpy(metadata_buffer_, ra_temp_meta_buffer.get(), snapuserd_->GetBufferMetadataSize());
- memcpy(read_ahead_buffer_, ra_temp_buffer.get(), total_blocks_merged * BLOCK_SZ);
+ memcpy(metadata_buffer_, ra_temp_meta_buffer_.get(), snapuserd_->GetBufferMetadataSize());
+ memcpy(read_ahead_buffer_, ra_temp_buffer_.get(), total_blocks_merged_ * BLOCK_SZ);
- offset = 0;
+ loff_t offset = 0;
std::unordered_map<uint64_t, void*>& read_ahead_buffer_map = snapuserd_->GetReadAheadMap();
read_ahead_buffer_map.clear();
- for (size_t block_index = 0; block_index < blocks.size(); block_index++) {
+ for (size_t block_index = 0; block_index < blocks_.size(); block_index++) {
void* bufptr = static_cast<void*>((char*)read_ahead_buffer_ + offset);
- uint64_t new_block = blocks[block_index];
+ uint64_t new_block = blocks_[block_index];
read_ahead_buffer_map[new_block] = bufptr;
offset += BLOCK_SZ;
}
- snapuserd_->SetMergedBlockCountForNextCommit(total_blocks_merged);
+ snapuserd_->SetMergedBlockCountForNextCommit(total_blocks_merged_);
// Flush the data only if we have a overlapping blocks in the region
// Notify the Merge thread to resume merging this window
@@ -350,6 +659,33 @@
return true;
}
+bool ReadAhead::InitializeIouring() {
+ if (!snapuserd_->IsIouringSupported()) {
+ return false;
+ }
+
+ ring_ = std::make_unique<struct io_uring>();
+
+ int ret = io_uring_queue_init(queue_depth_, ring_.get(), 0);
+ if (ret) {
+ SNAP_LOG(ERROR) << "io_uring_queue_init failed with ret: " << ret;
+ return false;
+ }
+
+ // For xor ops processing
+ bufsink_.Initialize(PAYLOAD_BUFFER_SZ * 2);
+ read_ahead_async_ = true;
+
+ SNAP_LOG(INFO) << "Read-ahead: io_uring initialized with queue depth: " << queue_depth_;
+ return true;
+}
+
+void ReadAhead::FinalizeIouring() {
+ if (read_ahead_async_) {
+ io_uring_queue_exit(ring_.get());
+ }
+}
+
bool ReadAhead::RunThread() {
if (!InitializeFds()) {
return false;
@@ -363,14 +699,18 @@
InitializeRAIter();
+ InitializeIouring();
+
while (!RAIterDone()) {
if (!ReadAheadIOStart()) {
break;
}
}
+ FinalizeIouring();
CloseFds();
reader_->CloseCowFd();
+
SNAP_LOG(INFO) << " ReadAhead thread terminating....";
return true;
}
@@ -434,8 +774,9 @@
metadata_buffer_ =
static_cast<void*>((char*)mapped_addr + snapuserd_->GetBufferMetadataOffset());
read_ahead_buffer_ = static_cast<void*>((char*)mapped_addr + snapuserd_->GetBufferDataOffset());
- // For xor ops
- bufsink_.Initialize(PAYLOAD_BUFFER_SZ);
+
+ ra_temp_buffer_ = std::make_unique<uint8_t[]>(snapuserd_->GetBufferDataSize());
+ ra_temp_meta_buffer_ = std::make_unique<uint8_t[]>(snapuserd_->GetBufferMetadataSize());
}
} // namespace snapshot
diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
index 1c3e04b..d670f1e 100644
--- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
+++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp
@@ -12,6 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <android-base/strings.h>
+#include <gflags/gflags.h>
+
#include <fcntl.h>
#include <linux/fs.h>
#include <linux/memfd.h>
@@ -27,6 +30,7 @@
#include <string_view>
#include <android-base/file.h>
+#include <android-base/properties.h>
#include <android-base/unique_fd.h>
#include <fs_mgr/file_wait.h>
#include <gtest/gtest.h>
@@ -38,6 +42,8 @@
#include "snapuserd_core.h"
+DEFINE_string(force_config, "", "Force testing mode with iouring disabled");
+
namespace android {
namespace snapshot {
@@ -857,5 +863,23 @@
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
+
+ gflags::ParseCommandLineFlags(&argc, &argv, false);
+
+ android::base::SetProperty("ctl.stop", "snapuserd");
+
+ if (FLAGS_force_config == "iouring_disabled") {
+ if (!android::base::SetProperty("snapuserd.test.io_uring.force_disable", "1")) {
+ return testing::AssertionFailure()
+ << "Failed to disable property: snapuserd.test.io_uring.disabled";
+ }
+ }
+
+ int ret = RUN_ALL_TESTS();
+
+ if (FLAGS_force_config == "iouring_disabled") {
+ android::base::SetProperty("snapuserd.test.io_uring.force_disable", "0");
+ }
+
+ return ret;
}
diff --git a/init/init.cpp b/init/init.cpp
index e3596cb..1df4c44 100644
--- a/init/init.cpp
+++ b/init/init.cpp
@@ -33,6 +33,7 @@
#define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_
#include <sys/_system_properties.h>
+#include <filesystem>
#include <functional>
#include <map>
#include <memory>
@@ -46,6 +47,7 @@
#include <android-base/logging.h>
#include <android-base/parseint.h>
#include <android-base/properties.h>
+#include <android-base/scopeguard.h>
#include <android-base/stringprintf.h>
#include <android-base/strings.h>
#include <backtrace/Backtrace.h>
@@ -773,6 +775,82 @@
return {};
}
+static bool SystemReadSmokeTest() {
+ std::string dev = "/dev/block/mapper/system"s + fs_mgr_get_slot_suffix();
+ android::base::unique_fd fd(open(dev.c_str(), O_RDONLY));
+ if (fd < 0) {
+ PLOG(ERROR) << "open " << dev << " failed, will not diangose snapuserd hangs";
+ return false;
+ }
+
+ for (size_t i = 1; i <= 100; i++) {
+ // Skip around the partition a bit.
+ size_t offset = i * 4096 * 512;
+
+ char b;
+ ssize_t n = TEMP_FAILURE_RETRY(pread(fd.get(), &b, 1, offset));
+ if (n < 0) {
+ PLOG(ERROR) << "snapuserd smoke test read failed";
+ return false;
+ }
+ }
+ return true;
+}
+
+static void DiagnoseSnapuserdHang(pid_t pid) {
+ bool succeeded = false;
+
+ std::mutex m;
+ std::condition_variable cv;
+
+ // Enforce an ordering between this and the thread startup, by taking the
+ // lock before we lanuch the thread.
+ std::unique_lock<std::mutex> cv_lock(m);
+
+ std::thread t([&]() -> void {
+ std::lock_guard<std::mutex> lock(m);
+ succeeded = SystemReadSmokeTest();
+ cv.notify_all();
+ });
+
+ auto join = android::base::make_scope_guard([&]() -> void {
+ // If the smoke test is hung, then this will too. We expect the device to
+ // automatically reboot once the watchdog kicks in.
+ t.join();
+ });
+
+ auto now = std::chrono::system_clock::now();
+ auto deadline = now + 10s;
+ auto status = cv.wait_until(cv_lock, deadline);
+ if (status == std::cv_status::timeout) {
+ LOG(ERROR) << "snapuserd smoke test timed out";
+ } else if (!succeeded) {
+ LOG(ERROR) << "snapuserd smoke test failed";
+ }
+
+ if (succeeded) {
+ LOG(INFO) << "snapuserd smoke test succeeded";
+ return;
+ }
+
+ while (true) {
+ LOG(ERROR) << "snapuserd problem detected, printing open fds";
+
+ std::error_code ec;
+ std::string proc_dir = "/proc/" + std::to_string(pid) + "/fd";
+ for (const auto& entry : std::filesystem::directory_iterator(proc_dir)) {
+ std::string target;
+ if (android::base::Readlink(entry.path(), &target)) {
+ LOG(ERROR) << "snapuserd opened: " << target;
+ } else {
+ LOG(ERROR) << "snapuserd opened: " << entry.path();
+ }
+ }
+
+ std::this_thread::sleep_for(10s);
+ }
+}
+
int SecondStageMain(int argc, char** argv) {
if (REBOOT_BOOTLOADER_ON_PANIC) {
InstallRebootSignalHandlers();
@@ -786,6 +864,11 @@
InitKernelLogging(argv);
LOG(INFO) << "init second stage started!";
+ if (auto pid = GetSnapuserdFirstStagePid()) {
+ std::thread t(DiagnoseSnapuserdHang, *pid);
+ t.detach();
+ }
+
// Update $PATH in the case the second stage init is newer than first stage init, where it is
// first set.
if (setenv("PATH", _PATH_DEFPATH, 1) != 0) {
diff --git a/init/perfboot.py b/init/perfboot.py
index 4b23ad2..968df38 100755
--- a/init/perfboot.py
+++ b/init/perfboot.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2015 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -39,7 +39,7 @@
import argparse
import atexit
-import cStringIO
+import io
import glob
import inspect
import logging
@@ -102,7 +102,7 @@
self._wait_cpu_cool_down(self._product, self._temp_paths)
else:
if self._waited:
- print 'Waiting for %d seconds' % self._interval
+ print('Waiting for %d seconds' % self._interval)
time.sleep(self._interval)
self._waited = True
@@ -119,9 +119,9 @@
threshold = IntervalAdjuster._CPU_COOL_DOWN_THRESHOLDS.get(
self._product)
if threshold is None:
- print 'No CPU temperature threshold is set for ' + self._product
- print ('Just wait %d seconds' %
- IntervalAdjuster._CPU_COOL_DOWN_WAIT_TIME_DEFAULT)
+ print('No CPU temperature threshold is set for ' + self._product)
+ print(('Just wait %d seconds' %
+ IntervalAdjuster._CPU_COOL_DOWN_WAIT_TIME_DEFAULT))
time.sleep(IntervalAdjuster._CPU_COOL_DOWN_WAIT_TIME_DEFAULT)
return
while True:
@@ -129,8 +129,8 @@
if temp < threshold:
logging.info('Current CPU temperature %s' % temp)
return
- print 'Waiting until CPU temperature (%d) falls below %d' % (
- temp, threshold)
+ print('Waiting until CPU temperature (%d) falls below %d' % (
+ temp, threshold))
time.sleep(IntervalAdjuster._CPU_COOL_DOWN_WAIT_INTERVAL)
@@ -260,7 +260,7 @@
def get_values(record, tag):
"""Gets values that matches |tag| from |record|."""
- keys = [key for key in record.keys() if key[0] == tag]
+ keys = [key for key in list(record.keys()) if key[0] == tag]
return [record[k] for k in sorted(keys)]
@@ -304,7 +304,7 @@
with open(filename, 'w') as f:
f.write('\t'.join(labels) + '\n')
for record in record_list:
- line = cStringIO.StringIO()
+ line = io.StringIO()
invalid_line = False
for i, tag in enumerate(tags):
if i != 0:
@@ -319,7 +319,7 @@
logging.error('Invalid record found: ' + line.getvalue())
line.write('\n')
f.write(line.getvalue())
- print 'Wrote: ' + filename
+ print(('Wrote: ' + filename))
def median(data):
@@ -349,9 +349,9 @@
# Filter out invalid data.
end_times = [get_last_value(record, end_tag) for record in record_list
if get_last_value(record, end_tag) != 0]
- print 'mean:', int(round(mean(end_times))), 'ms'
- print 'median:', int(round(median(end_times))), 'ms'
- print 'standard deviation:', int(round(stddev(end_times))), 'ms'
+ print(('mean:', int(round(mean(end_times))), 'ms'))
+ print(('median:', int(round(median(end_times))), 'ms'))
+ print(('standard deviation:', int(round(stddev(end_times))), 'ms'))
def do_iteration(device, interval_adjuster, event_tags_re, end_tag):
@@ -359,7 +359,7 @@
device.wait()
interval_adjuster.wait()
device.reboot()
- print 'Rebooted the device'
+ print('Rebooted the device, waiting for tag', end_tag)
record = {}
booted = False
while not booted:
@@ -372,7 +372,7 @@
stdout=subprocess.PIPE)
for line in readlines_unbuffered(p):
if t.is_timedout():
- print '*** Timed out ***'
+ print('*** Timed out ***')
return record
m = event_tags_re.search(line)
if not m:
@@ -381,8 +381,8 @@
event_time = int(m.group('time'))
pid = m.group('pid')
record[(tag, pid)] = event_time
- print 'Event log recorded: %s (%s) - %d ms' % (
- tag, pid, event_time)
+ print(('Event log recorded: %s (%s) - %d ms' % (
+ tag, pid, event_time)))
if tag == end_tag:
booted = True
t.cancel()
@@ -420,7 +420,7 @@
def install_apks(device, apk_dir):
for apk in glob.glob(os.path.join(apk_dir, '*.apk')):
- print 'Installing: ' + apk
+ print('Installing: ' + apk)
device.install(apk, replace=True)
@@ -452,7 +452,7 @@
event_tags_re = make_event_tags_re(event_tags)
for i in range(args.iterations):
- print 'Run #%d ' % i
+ print('Run #%d ' % i)
record = do_iteration(
device, interval_adjuster, event_tags_re, end_tag)
record_list.append(record)
diff --git a/libsparse/Android.bp b/libsparse/Android.bp
index 3f9aeb2..02bfee6 100644
--- a/libsparse/Android.bp
+++ b/libsparse/Android.bp
@@ -96,12 +96,14 @@
cc_fuzz {
name: "sparse_fuzzer",
- host_supported: false,
+ host_supported: true,
srcs: [
"sparse_fuzzer.cpp",
],
static_libs: [
"libsparse",
+ "libbase",
+ "libz",
"liblog",
],
}
diff --git a/libsparse/include/sparse/sparse.h b/libsparse/include/sparse/sparse.h
index 2f75349..9f91269 100644
--- a/libsparse/include/sparse/sparse.h
+++ b/libsparse/include/sparse/sparse.h
@@ -244,21 +244,6 @@
int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc);
/**
- * sparse_file_read_buf - read a buffer into a sparse file cookie
- *
- * @s - sparse file cookie
- * @buf - buffer to read from
- * @crc - verify the crc of a file in the Android sparse file format
- *
- * Reads a buffer into a sparse file cookie. The buffer must remain
- * valid until the sparse file cookie is freed. If crc is true, the
- * crc of the sparse file will be verified.
- *
- * Returns 0 on success, negative errno on error.
- */
-int sparse_file_read_buf(struct sparse_file *s, char *buf, bool crc);
-
-/**
* sparse_file_import - import an existing sparse file
*
* @fd - file descriptor to read from
@@ -277,6 +262,7 @@
* sparse_file_import_buf - import an existing sparse file from a buffer
*
* @buf - buffer to read from
+ * @len - length of buffer
* @verbose - print verbose errors while reading the sparse file
* @crc - verify the crc of a file in the Android sparse file format
*
@@ -286,7 +272,7 @@
*
* Returns a new sparse file cookie on success, NULL on error.
*/
-struct sparse_file *sparse_file_import_buf(char* buf, bool verbose, bool crc);
+struct sparse_file* sparse_file_import_buf(char* buf, size_t len, bool verbose, bool crc);
/**
* sparse_file_import_auto - import an existing sparse or normal file
diff --git a/libsparse/output_file.cpp b/libsparse/output_file.cpp
index b2c5407..cb5d730 100644
--- a/libsparse/output_file.cpp
+++ b/libsparse/output_file.cpp
@@ -54,6 +54,8 @@
#define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
#define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
+#define FILL_ZERO_BUFSIZE (2 * 1024 * 1024)
+
#define container_of(inner, outer_t, elem) ((outer_t*)((char*)(inner)-offsetof(outer_t, elem)))
struct output_file_ops {
@@ -391,13 +393,29 @@
ret = out->ops->write(out, data, len);
if (ret < 0) return -1;
if (zero_len) {
- ret = out->ops->write(out, out->zero_buf, zero_len);
- if (ret < 0) return -1;
+ uint64_t len = zero_len;
+ uint64_t write_len;
+ while (len) {
+ write_len = std::min(len, (uint64_t)FILL_ZERO_BUFSIZE);
+ ret = out->ops->write(out, out->zero_buf, write_len);
+ if (ret < 0) {
+ return ret;
+ }
+ len -= write_len;
+ }
}
if (out->use_crc) {
out->crc32 = sparse_crc32(out->crc32, data, len);
- if (zero_len) out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len);
+ if (zero_len) {
+ uint64_t len = zero_len;
+ uint64_t write_len;
+ while (len) {
+ write_len = std::min(len, (uint64_t)FILL_ZERO_BUFSIZE);
+ out->crc32 = sparse_crc32(out->crc32, out->zero_buf, write_len);
+ len -= write_len;
+ }
+ }
}
out->cur_out_ptr += rnd_up_len;
@@ -460,12 +478,12 @@
uint64_t write_len;
/* Initialize fill_buf with the fill_val */
- for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
+ for (i = 0; i < FILL_ZERO_BUFSIZE / sizeof(uint32_t); i++) {
out->fill_buf[i] = fill_val;
}
while (len) {
- write_len = std::min(len, (uint64_t)out->block_size);
+ write_len = std::min(len, (uint64_t)FILL_ZERO_BUFSIZE);
ret = out->ops->write(out, out->fill_buf, write_len);
if (ret < 0) {
return ret;
@@ -512,13 +530,15 @@
out->crc32 = 0;
out->use_crc = crc;
- out->zero_buf = reinterpret_cast<char*>(calloc(block_size, 1));
+ // don't use sparse format block size as it can takes up to 32GB
+ out->zero_buf = reinterpret_cast<char*>(calloc(FILL_ZERO_BUFSIZE, 1));
if (!out->zero_buf) {
error_errno("malloc zero_buf");
return -ENOMEM;
}
- out->fill_buf = reinterpret_cast<uint32_t*>(calloc(block_size, 1));
+ // don't use sparse format block size as it can takes up to 32GB
+ out->fill_buf = reinterpret_cast<uint32_t*>(calloc(FILL_ZERO_BUFSIZE, 1));
if (!out->fill_buf) {
error_errno("malloc fill_buf");
ret = -ENOMEM;
diff --git a/libsparse/simg_dump.py b/libsparse/simg_dump.py
index b0b3b22..8811a52 100755
--- a/libsparse/simg_dump.py
+++ b/libsparse/simg_dump.py
@@ -120,7 +120,7 @@
"output offset", "output blocks", "type", "hash"])
offset = 0
- for i in xrange(1, total_chunks + 1):
+ for i in range(1, total_chunks + 1):
header_bin = FH.read(12)
header = struct.unpack("<2H2I", header_bin)
chunk_type = header[0]
@@ -159,7 +159,7 @@
if showhash:
h = hashlib.sha1()
data = fill_bin * (blk_sz / 4);
- for block in xrange(chunk_sz):
+ for block in range(chunk_sz):
h.update(data)
curhash = h.hexdigest()
elif chunk_type == 0xCAC3:
diff --git a/libsparse/sparse_fuzzer.cpp b/libsparse/sparse_fuzzer.cpp
index 42f331f..235d15d 100644
--- a/libsparse/sparse_fuzzer.cpp
+++ b/libsparse/sparse_fuzzer.cpp
@@ -1,16 +1,27 @@
#include "include/sparse/sparse.h"
-extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
- if (size < 2 * sizeof(wchar_t)) return 0;
+static volatile int count;
- int64_t blocksize = 4096;
- struct sparse_file* file = sparse_file_new(size, blocksize);
- if (!file) {
+int WriteCallback(void* priv __attribute__((__unused__)), const void* data, size_t len) {
+ if (!data) {
+ return 0;
+ }
+ if (len == 0) {
return 0;
}
- unsigned int block = 1;
- sparse_file_add_data(file, &data, size, block);
- sparse_file_destroy(file);
+ const char* p = (const char*)data;
+ // Just to make sure the data is accessible
+ // We only check the head and tail to save time
+ count += *p;
+ count += *(p+len-1);
return 0;
}
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+ struct sparse_file* file = sparse_file_import_buf((char*)data, size, true, false);
+ if (!file) {
+ return 0;
+ }
+ return sparse_file_callback(file, false, false, WriteCallback, nullptr);
+}
diff --git a/libsparse/sparse_read.cpp b/libsparse/sparse_read.cpp
index c4c1823..0f39172 100644
--- a/libsparse/sparse_read.cpp
+++ b/libsparse/sparse_read.cpp
@@ -58,14 +58,15 @@
class SparseFileSource {
public:
- /* Seeks the source ahead by the given offset. */
- virtual void Seek(int64_t offset) = 0;
+ /* Seeks the source ahead by the given offset.
+ * Return 0 if successful. */
+ virtual int Seek(int64_t offset) = 0;
/* Return the current offset. */
virtual int64_t GetOffset() = 0;
- /* Set the current offset. Return 0 if successful. */
- virtual int SetOffset(int64_t offset) = 0;
+ /* Rewind to beginning. Return 0 if successful. */
+ virtual int Rewind() = 0;
/* Adds the given length from the current offset of the source to the file at the given block.
* Return 0 if successful. */
@@ -88,12 +89,14 @@
SparseFileFdSource(int fd) : fd(fd) {}
~SparseFileFdSource() override {}
- void Seek(int64_t off) override { lseek64(fd, off, SEEK_CUR); }
+ int Seek(int64_t off) override {
+ return lseek64(fd, off, SEEK_CUR) != -1 ? 0 : -errno;
+ }
int64_t GetOffset() override { return lseek64(fd, 0, SEEK_CUR); }
- int SetOffset(int64_t offset) override {
- return lseek64(fd, offset, SEEK_SET) == offset ? 0 : -errno;
+ int Rewind() override {
+ return lseek64(fd, 0, SEEK_SET) == 0 ? 0 : -errno;
}
int AddToSparseFile(struct sparse_file* s, int64_t len, unsigned int block) override {
@@ -120,39 +123,74 @@
class SparseFileBufSource : public SparseFileSource {
private:
+ char* buf_start;
+ char* buf_end;
char* buf;
int64_t offset;
+ int AccessOkay(int64_t len) {
+ if (len <= 0) return -EINVAL;
+ if (buf < buf_start) return -EOVERFLOW;
+ if (buf >= buf_end) return -EOVERFLOW;
+ if (len > buf_end - buf) return -EOVERFLOW;
+
+ return 0;
+ }
+
public:
- SparseFileBufSource(char* buf) : buf(buf), offset(0) {}
+ SparseFileBufSource(char* buf, uint64_t len) {
+ this->buf = buf;
+ this->offset = 0;
+ this->buf_start = buf;
+ this->buf_end = buf + len;
+ }
~SparseFileBufSource() override {}
- void Seek(int64_t off) override {
+ int Seek(int64_t off) override {
+ int ret = AccessOkay(off);
+ if (ret < 0) {
+ return ret;
+ }
buf += off;
offset += off;
+ return 0;
}
int64_t GetOffset() override { return offset; }
- int SetOffset(int64_t off) override {
- buf += off - offset;
- offset = off;
+ int Rewind() override {
+ buf = buf_start;
+ offset = 0;
return 0;
}
int AddToSparseFile(struct sparse_file* s, int64_t len, unsigned int block) override {
+ int ret = AccessOkay(len);
+ if (ret < 0) {
+ return ret;
+ }
return sparse_file_add_data(s, buf, len, block);
}
int ReadValue(void* ptr, int len) override {
+ int ret = AccessOkay(len);
+ if (ret < 0) {
+ return ret;
+ }
memcpy(ptr, buf, len);
- Seek(len);
+ buf += len;
+ offset += len;
return 0;
}
int GetCrc32(uint32_t* crc32, int64_t len) override {
+ int ret = AccessOkay(len);
+ if (ret < 0) {
+ return ret;
+ }
*crc32 = sparse_crc32(*crc32, buf, len);
- Seek(len);
+ buf += len;
+ offset += len;
return 0;
}
};
@@ -175,7 +213,7 @@
SparseFileSource* source, unsigned int blocks, unsigned int block,
uint32_t* crc32) {
int ret;
- int64_t len = blocks * s->block_size;
+ int64_t len = (int64_t)blocks * s->block_size;
if (chunk_size % s->block_size != 0) {
return -EINVAL;
@@ -196,7 +234,10 @@
return ret;
}
} else {
- source->Seek(len);
+ ret = source->Seek(len);
+ if (ret < 0) {
+ return ret;
+ }
}
return 0;
@@ -379,7 +420,10 @@
/* Skip the remaining bytes in a header that is longer than
* we expected.
*/
- source->Seek(sparse_header.file_hdr_sz - SPARSE_HEADER_LEN);
+ ret = source->Seek(sparse_header.file_hdr_sz - SPARSE_HEADER_LEN);
+ if (ret < 0) {
+ return ret;
+ }
}
for (i = 0; i < sparse_header.total_chunks; i++) {
@@ -392,7 +436,10 @@
/* Skip the remaining bytes in a header that is longer than
* we expected.
*/
- source->Seek(sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN);
+ ret = source->Seek(sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN);
+ if (ret < 0) {
+ return ret;
+ }
}
ret = process_chunk(s, source, sparse_header.chunk_hdr_sz, &chunk_header, cur_block, crc_ptr);
@@ -474,11 +521,6 @@
}
}
-int sparse_file_read_buf(struct sparse_file* s, char* buf, bool crc) {
- SparseFileBufSource source(buf);
- return sparse_file_read_sparse(s, &source, crc);
-}
-
static struct sparse_file* sparse_file_import_source(SparseFileSource* source, bool verbose,
bool crc) {
int ret;
@@ -510,6 +552,14 @@
return nullptr;
}
+ if (!sparse_header.blk_sz || (sparse_header.blk_sz % 4)) {
+ return nullptr;
+ }
+
+ if (!sparse_header.total_blks) {
+ return nullptr;
+ }
+
len = (int64_t)sparse_header.total_blks * sparse_header.blk_sz;
s = sparse_file_new(sparse_header.blk_sz, len);
if (!s) {
@@ -517,7 +567,7 @@
return nullptr;
}
- ret = source->SetOffset(0);
+ ret = source->Rewind();
if (ret < 0) {
verbose_error(verbose, ret, "seeking");
sparse_file_destroy(s);
@@ -540,8 +590,8 @@
return sparse_file_import_source(&source, verbose, crc);
}
-struct sparse_file* sparse_file_import_buf(char* buf, bool verbose, bool crc) {
- SparseFileBufSource source(buf);
+struct sparse_file* sparse_file_import_buf(char* buf, size_t len, bool verbose, bool crc) {
+ SparseFileBufSource source(buf, len);
return sparse_file_import_source(&source, verbose, crc);
}
diff --git a/libstats/pull_rust/Android.bp b/libstats/pull_rust/Android.bp
index f07e32b..4ffa98d 100644
--- a/libstats/pull_rust/Android.bp
+++ b/libstats/pull_rust/Android.bp
@@ -44,6 +44,11 @@
],
},
},
+ min_sdk_version: "apex_inherit",
+ apex_available: [
+ "//apex_available:platform",
+ "com.android.virt",
+ ]
}
rust_library {
diff --git a/libutils/Unicode_test.cpp b/libutils/Unicode_test.cpp
index b92eef8..8b994d9 100644
--- a/libutils/Unicode_test.cpp
+++ b/libutils/Unicode_test.cpp
@@ -100,7 +100,7 @@
0xF0, 0x90, 0x80, 0x80, // U+10000, 2 UTF-16 character
};
- char16_t output[1 + 1 + 1 + 2 + 1]; // Room for NULL
+ char16_t output[1 + 1 + 1 + 2 + 1]; // Room for null
utf8_to_utf16(str, sizeof(str), output, sizeof(output) / sizeof(output[0]));
@@ -114,8 +114,7 @@
<< "should be first half of surrogate U+10000";
EXPECT_EQ(0xDC00, output[4])
<< "should be second half of surrogate U+10000";
- EXPECT_EQ(NULL, output[5])
- << "should be NULL terminated";
+ EXPECT_EQ(0, output[5]) << "should be null terminated";
}
TEST_F(UnicodeTest, strstr16EmptyTarget) {
diff --git a/rootdir/Android.mk b/rootdir/Android.mk
index 2ed9eec..ce2ec0e 100644
--- a/rootdir/Android.mk
+++ b/rootdir/Android.mk
@@ -75,14 +75,9 @@
EXPORT_GLOBAL_GCOV_OPTIONS := export GCOV_PREFIX /data/misc/trace
endif
+EXPORT_GLOBAL_CLANG_COVERAGE_OPTIONS :=
ifeq ($(CLANG_COVERAGE),true)
- ifeq ($(BIONIC_COVERAGE),false)
- # http://b/210012154 Disable continuous coverage if instrumentation is on
- # for bionic/libc
- EXPORT_GLOBAL_CLANG_COVERAGE_OPTIONS := export LLVM_PROFILE_FILE /data/misc/trace/clang%c-%20m.profraw
- else
- EXPORT_GLOBAL_CLANG_COVERAGE_OPTIONS := export LLVM_PROFILE_FILE /data/misc/trace/clang-%20m.profraw
- endif
+ EXPORT_GLOBAL_CLANG_COVERAGE_OPTIONS := export LLVM_PROFILE_FILE /data/misc/trace/clang-%20m.profraw
endif
# Put it here instead of in init.rc module definition,
diff --git a/trusty/storage/proxy/Android.bp b/trusty/storage/proxy/Android.bp
index 38d8685..94f26d8 100644
--- a/trusty/storage/proxy/Android.bp
+++ b/trusty/storage/proxy/Android.bp
@@ -35,7 +35,10 @@
"liblog",
"libhardware_legacy",
],
- header_libs: ["libcutils_headers"],
+ header_libs: [
+ "libcutils_headers",
+ "libgsi_headers",
+ ],
static_libs: [
"libfstab",
diff --git a/trusty/storage/proxy/checkpoint_handling.cpp b/trusty/storage/proxy/checkpoint_handling.cpp
index 6c2fd36..3305d8d 100644
--- a/trusty/storage/proxy/checkpoint_handling.cpp
+++ b/trusty/storage/proxy/checkpoint_handling.cpp
@@ -18,9 +18,12 @@
#include "log.h"
#include <fstab/fstab.h>
+#include <unistd.h>
#include <cstring>
#include <string>
+#include <libgsi/libgsi.h>
+
namespace {
bool checkpointingDoneForever = false;
@@ -75,3 +78,15 @@
return 0;
}
+
+/**
+ * is_gsi_running() - Check if a GSI image is running via DSU.
+ *
+ * This function is equivalent to android::gsi::IsGsiRunning(), but this API is
+ * not yet vendor-accessible although the underlying metadata file is.
+ *
+ */
+bool is_gsi_running() {
+ /* TODO(b/210501710): Expose GSI image running state to vendor storageproxyd */
+ return !access(android::gsi::kGsiBootedIndicatorFile, F_OK);
+}
diff --git a/trusty/storage/proxy/checkpoint_handling.h b/trusty/storage/proxy/checkpoint_handling.h
index f1bf27c..dfe2947 100644
--- a/trusty/storage/proxy/checkpoint_handling.h
+++ b/trusty/storage/proxy/checkpoint_handling.h
@@ -32,6 +32,8 @@
*/
int is_data_checkpoint_active(bool* active);
+bool is_gsi_running();
+
#ifdef __cplusplus
}
#endif
diff --git a/trusty/storage/proxy/proxy.c b/trusty/storage/proxy/proxy.c
index c690a28..2620034 100644
--- a/trusty/storage/proxy/proxy.c
+++ b/trusty/storage/proxy/proxy.c
@@ -104,8 +104,11 @@
return -1;
}
- /* no-execute for user, no access for group and other */
- umask(S_IXUSR | S_IRWXG | S_IRWXO);
+ /*
+ * No access for group and other. We need execute access for user to create
+ * an accessible directory.
+ */
+ umask(S_IRWXG | S_IRWXO);
return 0;
}
diff --git a/trusty/storage/proxy/storage.c b/trusty/storage/proxy/storage.c
index 2fde30f..d74a708 100644
--- a/trusty/storage/proxy/storage.c
+++ b/trusty/storage/proxy/storage.c
@@ -16,6 +16,7 @@
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
+#include <libgen.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
@@ -24,13 +25,16 @@
#include <sys/types.h>
#include <unistd.h>
-#include "log.h"
+#include "checkpoint_handling.h"
#include "ipc.h"
+#include "log.h"
#include "storage.h"
#define FD_TBL_SIZE 64
#define MAX_READ_SIZE 4096
+#define ALTERNATE_DATA_DIR "alternate/"
+
enum sync_state {
SS_UNUSED = -1,
SS_CLEAN = 0,
@@ -44,6 +48,8 @@
static enum sync_state dir_state;
static enum sync_state fd_state[FD_TBL_SIZE];
+static bool alternate_mode;
+
static struct {
struct storage_file_read_resp hdr;
uint8_t data[MAX_READ_SIZE];
@@ -216,6 +222,7 @@
const void *r, size_t req_len)
{
char *path = NULL;
+ char* parent_path;
const struct storage_file_open_req *req = r;
struct storage_file_open_resp resp = {0};
@@ -234,6 +241,24 @@
goto err_response;
}
+ /*
+ * TODO(b/210501710): Expose GSI image running state to vendor
+ * storageproxyd. We want to control data file paths in vendor_init, but we
+ * don't have access to the necessary property there yet. When we have
+ * access to that property we can set the root data path read-only and only
+ * allow creation of files in alternate/. Checking paths here temporarily
+ * until that is fixed.
+ *
+ * We are just checking for "/" instead of "alternate/" because we still
+ * want to still allow access to "persist/" in alternate mode (for now, this
+ * may change in the future).
+ */
+ if (alternate_mode && !strchr(req->name, '/')) {
+ ALOGE("%s: Cannot open root data file \"%s\" in alternate mode\n", __func__, req->name);
+ msg->result = STORAGE_ERR_ACCESS;
+ goto err_response;
+ }
+
int rc = asprintf(&path, "%s/%s", ssdir_name, req->name);
if (rc < 0) {
ALOGE("%s: asprintf failed\n", __func__);
@@ -246,7 +271,23 @@
if (req->flags & STORAGE_FILE_OPEN_TRUNCATE)
open_flags |= O_TRUNC;
+ parent_path = dirname(path);
if (req->flags & STORAGE_FILE_OPEN_CREATE) {
+ /*
+ * Create the alternate parent dir if needed & allowed.
+ *
+ * TODO(b/210501710): Expose GSI image running state to vendor
+ * storageproxyd. This directory should be created by vendor_init, once
+ * it has access to the necessary bit of information.
+ */
+ if (strstr(req->name, ALTERNATE_DATA_DIR) == req->name) {
+ rc = mkdir(parent_path, S_IRWXU);
+ if (rc && errno != EEXIST) {
+ ALOGE("%s: Could not create parent directory \"%s\": %s\n", __func__, parent_path,
+ strerror(errno));
+ }
+ }
+
/* open or create */
if (req->flags & STORAGE_FILE_OPEN_CREATE_EXCLUSIVE) {
/* create exclusive */
@@ -467,6 +508,9 @@
int storage_init(const char *dirname)
{
+ /* If there is an active DSU image, use the alternate fs mode. */
+ alternate_mode = is_gsi_running();
+
fs_state = SS_CLEAN;
dir_state = SS_CLEAN;
for (uint i = 0; i < FD_TBL_SIZE; i++) {