Merge "Add debug logging for memtag level"
diff --git a/benchmarks/malloc_benchmark.cpp b/benchmarks/malloc_benchmark.cpp
index 18ba523..e733cd0 100644
--- a/benchmarks/malloc_benchmark.cpp
+++ b/benchmarks/malloc_benchmark.cpp
@@ -36,11 +36,11 @@
#if defined(__BIONIC__)
-static void BM_mallopt_purge(benchmark::State& state) {
+static void RunMalloptPurge(benchmark::State& state, int purge_value) {
static size_t sizes[] = {8, 16, 32, 64, 128, 1024, 4096, 16384, 65536, 131072, 1048576};
static int pagesize = getpagesize();
mallopt(M_DECAY_TIME, 1);
- mallopt(M_PURGE, 0);
+ mallopt(M_PURGE_ALL, 0);
for (auto _ : state) {
state.PauseTiming();
std::vector<void*> ptrs;
@@ -63,10 +63,19 @@
ptrs.clear();
state.ResumeTiming();
- mallopt(M_PURGE, 0);
+ mallopt(purge_value, 0);
}
mallopt(M_DECAY_TIME, 0);
}
+
+static void BM_mallopt_purge(benchmark::State& state) {
+ RunMalloptPurge(state, M_PURGE);
+}
BIONIC_BENCHMARK(BM_mallopt_purge);
+static void BM_mallopt_purge_all(benchmark::State& state) {
+ RunMalloptPurge(state, M_PURGE_ALL);
+}
+BIONIC_BENCHMARK(BM_mallopt_purge_all);
+
#endif
diff --git a/benchmarks/malloc_map_benchmark.cpp b/benchmarks/malloc_map_benchmark.cpp
index ba4d62c..5757325 100644
--- a/benchmarks/malloc_map_benchmark.cpp
+++ b/benchmarks/malloc_map_benchmark.cpp
@@ -69,7 +69,7 @@
for (auto _ : state) {
#if defined(__BIONIC__)
state.PauseTiming();
- mallopt(M_PURGE, 0);
+ mallopt(M_PURGE_ALL, 0);
uint64_t rss_bytes_before = 0;
Gather(&rss_bytes_before);
state.ResumeTiming();
@@ -80,7 +80,7 @@
}
#if defined(__BIONIC__)
state.PauseTiming();
- mallopt(M_PURGE, 0);
+ mallopt(M_PURGE_ALL, 0);
Gather(&rss_bytes);
// Try and record only the memory used in the map.
rss_bytes -= rss_bytes_before;
diff --git a/benchmarks/malloc_rss_benchmark.cpp b/benchmarks/malloc_rss_benchmark.cpp
index 58f61d9..4b34e72 100644
--- a/benchmarks/malloc_rss_benchmark.cpp
+++ b/benchmarks/malloc_rss_benchmark.cpp
@@ -112,7 +112,7 @@
// Do an explicit purge to ensure we will be more likely to get the actual
// in-use memory.
- mallopt(M_PURGE, 0);
+ mallopt(M_PURGE_ALL, 0);
android::meminfo::ProcMemInfo proc_mem(getpid());
const std::vector<android::meminfo::Vma>& maps = proc_mem.MapsWithoutUsageStats();
diff --git a/libc/Android.bp b/libc/Android.bp
index c101f45..99261f9 100644
--- a/libc/Android.bp
+++ b/libc/Android.bp
@@ -827,6 +827,7 @@
riscv64: {
srcs: [
"arch-riscv64/string/__memset_chk.S",
+ "arch-riscv64/string/__memcpy_chk.S",
],
},
},
@@ -2307,6 +2308,11 @@
"arch-arm64/string/__memcpy_chk.S",
],
},
+ riscv64: {
+ srcs: [
+ "arch-riscv64/string/__memcpy_chk.S",
+ ],
+ },
},
whole_static_libs: [
"libarm-optimized-routines-mem",
diff --git a/libc/arch-riscv64/string/__memcpy_chk.S b/libc/arch-riscv64/string/__memcpy_chk.S
new file mode 100644
index 0000000..4a2d13d
--- /dev/null
+++ b/libc/arch-riscv64/string/__memcpy_chk.S
@@ -0,0 +1,9 @@
+#include <private/bionic_asm.h>
+
+ENTRY(__memcpy_chk)
+ bleu a2, a3, 1f
+ call __memcpy_chk_fail
+
+1:
+ tail memcpy
+END(__memcpy_chk)
diff --git a/libc/bionic/fortify.cpp b/libc/bionic/fortify.cpp
index 73cf973..7dee5e3 100644
--- a/libc/bionic/fortify.cpp
+++ b/libc/bionic/fortify.cpp
@@ -489,9 +489,9 @@
return strcpy(dst, src);
}
-#if !defined(__arm__) && !defined(__aarch64__)
+#if !defined(__arm__) && !defined(__aarch64__) && !defined(__riscv)
// Runtime implementation of __memcpy_chk (used directly by compiler, not in headers).
-// arm32 and arm64 have assembler implementations, and don't need this C fallback.
+// arm32,arm64,riscv have assembler implementations, and don't need this C fallback.
extern "C" void* __memcpy_chk(void* dst, const void* src, size_t count, size_t dst_len) {
__check_count("memcpy", "count", count);
__check_buffer_access("memcpy", "write into", count, dst_len);
diff --git a/libc/bionic/jemalloc_wrapper.cpp b/libc/bionic/jemalloc_wrapper.cpp
index ef488ee..ce3f314 100644
--- a/libc/bionic/jemalloc_wrapper.cpp
+++ b/libc/bionic/jemalloc_wrapper.cpp
@@ -102,7 +102,7 @@
}
}
return 1;
- } else if (param == M_PURGE) {
+ } else if (param == M_PURGE || param == M_PURGE_ALL) {
// Only clear the current thread cache since there is no easy way to
// clear the caches of other threads.
// This must be done first so that cleared allocations get purged
diff --git a/libc/include/bits/elf_common.h b/libc/include/bits/elf_common.h
index ea833f4..b3c57a2 100644
--- a/libc/include/bits/elf_common.h
+++ b/libc/include/bits/elf_common.h
@@ -418,12 +418,12 @@
#define SHT_HIOS 0x6fffffff /* Last of OS specific semantics */
#define SHT_LOPROC 0x70000000 /* reserved range for processor */
#define SHT_X86_64_UNWIND 0x70000001 /* unwind information */
-#define SHT_AMD64_UNWIND SHT_X86_64_UNWIND
+#define SHT_AMD64_UNWIND SHT_X86_64_UNWIND
#define SHT_ARM_EXIDX 0x70000001 /* Exception index table. */
-#define SHT_ARM_PREEMPTMAP 0x70000002 /* BPABI DLL dynamic linking
+#define SHT_ARM_PREEMPTMAP 0x70000002 /* BPABI DLL dynamic linking
pre-emption map. */
-#define SHT_ARM_ATTRIBUTES 0x70000003 /* Object file compatibility
+#define SHT_ARM_ATTRIBUTES 0x70000003 /* Object file compatibility
attributes. */
#define SHT_ARM_DEBUGOVERLAY 0x70000004 /* See DBGOVL for details. */
#define SHT_ARM_OVERLAYSECTION 0x70000005 /* See DBGOVL for details. */
@@ -648,6 +648,11 @@
#define DT_AARCH64_BTI_PLT 0x70000001
#define DT_AARCH64_PAC_PLT 0x70000003
#define DT_AARCH64_VARIANT_PCS 0x70000005
+#define DT_AARCH64_MEMTAG_MODE 0x70000009
+#define DT_AARCH64_MEMTAG_HEAP 0x7000000b
+#define DT_AARCH64_MEMTAG_STACK 0x7000000c
+#define DT_AARCH64_MEMTAG_GLOBALS 0x7000000d
+#define DT_AARCH64_MEMTAG_GLOBALSSZ 0x7000000f
#define DT_ARM_SYMTABSZ 0x70000001
#define DT_ARM_PREEMPTMAP 0x70000002
diff --git a/libc/include/link.h b/libc/include/link.h
index bd430f5..a0a3d60 100644
--- a/libc/include/link.h
+++ b/libc/include/link.h
@@ -44,41 +44,41 @@
struct dl_phdr_info {
ElfW(Addr) dlpi_addr;
- const char* dlpi_name;
- const ElfW(Phdr)* dlpi_phdr;
+ const char* _Nullable dlpi_name;
+ const ElfW(Phdr)* _Nullable dlpi_phdr;
ElfW(Half) dlpi_phnum;
// These fields were added in Android R.
unsigned long long dlpi_adds;
unsigned long long dlpi_subs;
size_t dlpi_tls_modid;
- void* dlpi_tls_data;
+ void* _Nullable dlpi_tls_data;
};
#if defined(__arm__)
-int dl_iterate_phdr(int (*__callback)(struct dl_phdr_info*, size_t, void*), void* __data) __INTRODUCED_IN(21);
+int dl_iterate_phdr(int (* _Nonnull __callback)(struct dl_phdr_info* _Nonnull, size_t, void* _Nullable), void* _Nullable __data) __INTRODUCED_IN(21);
#else
-int dl_iterate_phdr(int (*__callback)(struct dl_phdr_info*, size_t, void*), void* __data);
+int dl_iterate_phdr(int (* _Nonnull __callback)(struct dl_phdr_info* _Nonnull, size_t, void*_Nullable ), void* _Nullable __data);
#endif
#ifdef __arm__
typedef uintptr_t _Unwind_Ptr;
-_Unwind_Ptr dl_unwind_find_exidx(_Unwind_Ptr, int*);
+_Unwind_Ptr dl_unwind_find_exidx(_Unwind_Ptr, int* _Nonnull);
#endif
/* Used by the dynamic linker to communicate with the debugger. */
struct link_map {
ElfW(Addr) l_addr;
- char* l_name;
- ElfW(Dyn)* l_ld;
- struct link_map* l_next;
- struct link_map* l_prev;
+ char* _Nullable l_name;
+ ElfW(Dyn)* _Nullable l_ld;
+ struct link_map* _Nullable l_next;
+ struct link_map* _Nullable l_prev;
};
/* Used by the dynamic linker to communicate with the debugger. */
struct r_debug {
int32_t r_version;
- struct link_map* r_map;
+ struct link_map* _Nullable r_map;
ElfW(Addr) r_brk;
enum {
RT_CONSISTENT,
diff --git a/libc/include/malloc.h b/libc/include/malloc.h
index 02bda60..6a2d380 100644
--- a/libc/include/malloc.h
+++ b/libc/include/malloc.h
@@ -183,7 +183,15 @@
* Available since API level 28.
*/
#define M_PURGE (-101)
-
+/**
+ * mallopt() option to immediately purge all possible memory back to
+ * the kernel. This call can take longer than a normal purge since it
+ * examines everything. In some cases, it can take more than twice the
+ * time of a M_PURGE call. The value is ignored.
+ *
+ * Available since API level 34.
+ */
+#define M_PURGE_ALL (-104)
/**
* mallopt() option to tune the allocator's choice of memory tags to
diff --git a/libc/include/pwd.h b/libc/include/pwd.h
index d481aac..2b17fbf 100644
--- a/libc/include/pwd.h
+++ b/libc/include/pwd.h
@@ -66,31 +66,31 @@
__BEGIN_DECLS
struct passwd {
- char* pw_name;
- char* pw_passwd;
+ char* _Nullable pw_name;
+ char* _Nullable pw_passwd;
uid_t pw_uid;
gid_t pw_gid;
#ifdef __LP64__
- char* pw_gecos;
+ char* _Nullable pw_gecos;
#else
/* Note: On LP32, we define pw_gecos to pw_passwd since they're both NULL. */
# define pw_gecos pw_passwd
#endif
- char* pw_dir;
- char* pw_shell;
+ char* _Nullable pw_dir;
+ char* _Nullable pw_shell;
};
-struct passwd* getpwnam(const char* __name);
-struct passwd* getpwuid(uid_t __uid);
+struct passwd* _Nullable getpwnam(const char* _Nonnull __name);
+struct passwd* _Nullable getpwuid(uid_t __uid);
/* Note: Android has thousands and thousands of ids to iterate through */
-struct passwd* getpwent(void) __INTRODUCED_IN(26);
+struct passwd* _Nullable getpwent(void) __INTRODUCED_IN(26);
void setpwent(void) __INTRODUCED_IN(26);
void endpwent(void) __INTRODUCED_IN(26);
-int getpwnam_r(const char* __name, struct passwd* __pwd, char* __buf, size_t __n, struct passwd** __result);
-int getpwuid_r(uid_t __uid, struct passwd* __pwd, char* __buf, size_t __n, struct passwd** __result);
+int getpwnam_r(const char* _Nonnull __name, struct passwd* _Nonnull __pwd, char* _Nonnull __buf, size_t __n, struct passwd* _Nullable * _Nonnull __result);
+int getpwuid_r(uid_t __uid, struct passwd* _Nonnull __pwd, char* _Nonnull __buf, size_t __n, struct passwd* _Nullable * _Nonnull __result);
__END_DECLS
diff --git a/libc/kernel/tools/defaults.py b/libc/kernel/tools/defaults.py
index d0fe157..c9c0a22 100644
--- a/libc/kernel/tools/defaults.py
+++ b/libc/kernel/tools/defaults.py
@@ -48,6 +48,7 @@
"in_addr": False,
"ip_mreq_source": False,
"ip_msfilter": False,
+ "timespec": False,
}
# define to true if you want to remove all defined(CONFIG_FOO) tests
diff --git a/libc/kernel/uapi/linux/time.h b/libc/kernel/uapi/linux/time.h
index df52295..55d0e6e 100644
--- a/libc/kernel/uapi/linux/time.h
+++ b/libc/kernel/uapi/linux/time.h
@@ -18,14 +18,11 @@
****************************************************************************/
#ifndef _UAPI_LINUX_TIME_H
#define _UAPI_LINUX_TIME_H
+#include <bits/timespec.h>
#include <linux/types.h>
#include <linux/time_types.h>
#ifndef _STRUCT_TIMESPEC
#define _STRUCT_TIMESPEC
-struct timespec {
- __kernel_old_time_t tv_sec;
- long tv_nsec;
-};
#endif
struct timeval {
__kernel_old_time_t tv_sec;
diff --git a/linker/linker.cpp b/linker/linker.cpp
index 6246f8c..c5a822a 100644
--- a/linker/linker.cpp
+++ b/linker/linker.cpp
@@ -3142,6 +3142,14 @@
case DT_AARCH64_VARIANT_PCS:
// Ignored: AArch64 processor-specific dynamic array tags.
break;
+ // TODO(mitchp): Add support to libc_init_mte to use these dynamic array entries instead of
+ // the Android-specific ELF note.
+ case DT_AARCH64_MEMTAG_MODE:
+ case DT_AARCH64_MEMTAG_HEAP:
+ case DT_AARCH64_MEMTAG_STACK:
+ case DT_AARCH64_MEMTAG_GLOBALS:
+ case DT_AARCH64_MEMTAG_GLOBALSSZ:
+ break;
#endif
default:
diff --git a/tests/malloc_test.cpp b/tests/malloc_test.cpp
index 63ad99d..4e7eb7b 100644
--- a/tests/malloc_test.cpp
+++ b/tests/malloc_test.cpp
@@ -36,7 +36,10 @@
#include <algorithm>
#include <atomic>
#include <functional>
+#include <string>
#include <thread>
+#include <unordered_map>
+#include <utility>
#include <vector>
#include <tinyxml2.h>
@@ -695,6 +698,44 @@
#endif
}
+TEST(malloc, mallopt_purge_all) {
+#if defined(__BIONIC__)
+ SKIP_WITH_HWASAN << "hwasan does not implement mallopt";
+ errno = 0;
+ ASSERT_EQ(1, mallopt(M_PURGE_ALL, 0));
+#else
+ GTEST_SKIP() << "bionic-only test";
+#endif
+}
+
+// Verify that all of the mallopt values are unique.
+TEST(malloc, mallopt_unique_params) {
+#if defined(__BIONIC__)
+ std::vector<std::pair<int, std::string>> params{
+ std::make_pair(M_DECAY_TIME, "M_DECAY_TIME"),
+ std::make_pair(M_PURGE, "M_PURGE"),
+ std::make_pair(M_PURGE_ALL, "M_PURGE_ALL"),
+ std::make_pair(M_MEMTAG_TUNING, "M_MEMTAG_TUNING"),
+ std::make_pair(M_THREAD_DISABLE_MEM_INIT, "M_THREAD_DISABLE_MEM_INIT"),
+ std::make_pair(M_CACHE_COUNT_MAX, "M_CACHE_COUNT_MAX"),
+ std::make_pair(M_CACHE_SIZE_MAX, "M_CACHE_SIZE_MAX"),
+ std::make_pair(M_TSDS_COUNT_MAX, "M_TSDS_COUNT_MAX"),
+ std::make_pair(M_BIONIC_ZERO_INIT, "M_BIONIC_ZERO_INIT"),
+ std::make_pair(M_BIONIC_SET_HEAP_TAGGING_LEVEL, "M_BIONIC_SET_HEAP_TAGGING_LEVEL"),
+ };
+
+ std::unordered_map<int, std::string> all_params;
+ for (const auto& param : params) {
+ EXPECT_TRUE(all_params.count(param.first) == 0)
+ << "mallopt params " << all_params[param.first] << " and " << param.second
+ << " have the same value " << param.first;
+ all_params.insert(param);
+ }
+#else
+ GTEST_SKIP() << "bionic-only test";
+#endif
+}
+
#if defined(__BIONIC__)
static void GetAllocatorVersion(bool* allocator_scudo) {
TemporaryFile tf;