Merge "Just say "arm" rather than "arm_eabi"." into main
diff --git a/libc/bionic/pthread_internal.cpp b/libc/bionic/pthread_internal.cpp
index e8a8ba2..14cc7da 100644
--- a/libc/bionic/pthread_internal.cpp
+++ b/libc/bionic/pthread_internal.cpp
@@ -36,9 +36,10 @@
#include <sys/prctl.h>
#include <async_safe/log.h>
+#include <bionic/mte.h>
#include <bionic/reserved_signals.h>
+#include <bionic/tls_defines.h>
-#include "bionic/tls_defines.h"
#include "private/ErrnoRestorer.h"
#include "private/ScopedRWLock.h"
#include "private/bionic_futex.h"
@@ -73,16 +74,12 @@
g_thread_list = thread->next;
}
}
-// N.B. that this is NOT the pagesize, but 4096. This is hardcoded in the codegen.
-// See
-// https://github.com/search?q=repo%3Allvm/llvm-project%20AArch64StackTagging%3A%3AinsertBaseTaggedPointer&type=code
-constexpr size_t kStackMteRingbufferSizeMultiplier = 4096;
static void __pthread_internal_free(pthread_internal_t* thread) {
#ifdef __aarch64__
if (void* stack_mte_tls = thread->bionic_tcb->tls_slot(TLS_SLOT_STACK_MTE)) {
size_t size =
- kStackMteRingbufferSizeMultiplier * (reinterpret_cast<uintptr_t>(stack_mte_tls) >> 56ULL);
+ stack_mte_ringbuffer_size_from_pointer(reinterpret_cast<uintptr_t>(stack_mte_tls));
void* ptr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(stack_mte_tls) &
((1ULL << 56ULL) - 1ULL));
munmap(ptr, size);
@@ -191,38 +188,8 @@
async_safe_fatal("stack not found in /proc/self/maps");
}
+#if defined(__aarch64__)
__LIBC_HIDDEN__ void* __allocate_stack_mte_ringbuffer(size_t n, pthread_internal_t* thread) {
- if (n > 7) async_safe_fatal("error: invalid mte stack ring buffer size");
- // Allocation needs to be aligned to 2*size to make the fancy code-gen work.
- // So we allocate 3*size - pagesz bytes, which will always contain size bytes
- // aligned to 2*size, and unmap the unneeded part.
- // See
- // https://github.com/search?q=repo%3Allvm/llvm-project%20AArch64StackTagging%3A%3AinsertBaseTaggedPointer&type=code
- //
- // In the worst case, we get an allocation that is one page past the properly
- // aligned address, in which case we have to unmap the previous
- // 2*size - pagesz bytes. In that case, we still have size properly aligned
- // bytes left.
- size_t size = (1 << n) * kStackMteRingbufferSizeMultiplier;
- size_t pgsize = page_size();
-
- size_t alloc_size = __BIONIC_ALIGN(3 * size - pgsize, pgsize);
- void* allocation_ptr =
- mmap(nullptr, alloc_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
- if (allocation_ptr == MAP_FAILED)
- async_safe_fatal("error: failed to allocate stack mte ring buffer");
- uintptr_t allocation = reinterpret_cast<uintptr_t>(allocation_ptr);
-
- size_t alignment = 2 * size;
- uintptr_t aligned_allocation = __BIONIC_ALIGN(allocation, alignment);
- if (allocation != aligned_allocation) {
- munmap(reinterpret_cast<void*>(allocation), aligned_allocation - allocation);
- }
- if (aligned_allocation + size != allocation + alloc_size) {
- munmap(reinterpret_cast<void*>(aligned_allocation + size),
- (allocation + alloc_size) - (aligned_allocation + size));
- }
-
const char* name;
if (thread == nullptr) {
name = "stack_mte_ring:main";
@@ -234,11 +201,11 @@
async_safe_format_buffer(name_buffer, arraysize(name_buffer), "stack_mte_ring:%d", thread->tid);
name = name_buffer;
}
- prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, reinterpret_cast<void*>(aligned_allocation), size, name);
-
- // We store the size in the top byte of the pointer (which is ignored)
- return reinterpret_cast<void*>(aligned_allocation | ((1ULL << n) << 56ULL));
+ void* ret = stack_mte_ringbuffer_allocate(n, name);
+ if (!ret) async_safe_fatal("error: failed to allocate stack mte ring buffer");
+ return ret;
}
+#endif
bool __pthread_internal_remap_stack_with_mte() {
#if defined(__aarch64__)
diff --git a/libc/bionic/pthread_internal.h b/libc/bionic/pthread_internal.h
index b270a06..5db42ab 100644
--- a/libc/bionic/pthread_internal.h
+++ b/libc/bionic/pthread_internal.h
@@ -213,7 +213,9 @@
__LIBC_HIDDEN__ void __pthread_internal_remove(pthread_internal_t* thread);
__LIBC_HIDDEN__ void __pthread_internal_remove_and_free(pthread_internal_t* thread);
__LIBC_HIDDEN__ void __find_main_stack_limits(uintptr_t* low, uintptr_t* high);
+#if defined(__aarch64__)
__LIBC_HIDDEN__ void* __allocate_stack_mte_ringbuffer(size_t n, pthread_internal_t* thread);
+#endif
static inline __always_inline bionic_tcb* __get_bionic_tcb() {
return reinterpret_cast<bionic_tcb*>(&__get_tls()[MIN_TLS_SLOT]);
diff --git a/libc/bionic/sys_msg.cpp b/libc/bionic/sys_msg.cpp
index 462c83b..9780d38 100644
--- a/libc/bionic/sys_msg.cpp
+++ b/libc/bionic/sys_msg.cpp
@@ -36,33 +36,17 @@
// Annoyingly, the kernel requires this for 32-bit but rejects it for 64-bit.
cmd |= IPC_64;
#endif
-#if defined(SYS_msgctl)
return syscall(SYS_msgctl, id, cmd, buf);
-#else
- return syscall(SYS_ipc, MSGCTL, id, cmd, 0, buf, 0);
-#endif
}
int msgget(key_t key, int flags) {
-#if defined(SYS_msgget)
return syscall(SYS_msgget, key, flags);
-#else
- return syscall(SYS_ipc, MSGGET, key, flags, 0, 0, 0);
-#endif
}
ssize_t msgrcv(int id, void* msg, size_t n, long type, int flags) {
-#if defined(SYS_msgrcv)
return syscall(SYS_msgrcv, id, msg, n, type, flags);
-#else
- return syscall(SYS_ipc, IPCCALL(1, MSGRCV), id, n, flags, msg, type);
-#endif
}
int msgsnd(int id, const void* msg, size_t n, int flags) {
-#if defined(SYS_msgsnd)
return syscall(SYS_msgsnd, id, msg, n, flags);
-#else
- return syscall(SYS_ipc, MSGSND, id, n, flags, msg, 0);
-#endif
}
diff --git a/libc/bionic/sys_sem.cpp b/libc/bionic/sys_sem.cpp
index 058cfef..b97bf46 100644
--- a/libc/bionic/sys_sem.cpp
+++ b/libc/bionic/sys_sem.cpp
@@ -41,19 +41,11 @@
va_start(ap, cmd);
semun arg = va_arg(ap, semun);
va_end(ap);
-#if defined(SYS_semctl)
return syscall(SYS_semctl, id, num, cmd, arg);
-#else
- return syscall(SYS_ipc, SEMCTL, id, num, cmd, &arg, 0);
-#endif
}
int semget(key_t key, int n, int flags) {
-#if defined(SYS_semget)
return syscall(SYS_semget, key, n, flags);
-#else
- return syscall(SYS_ipc, SEMGET, key, n, flags, 0, 0);
-#endif
}
int semop(int id, sembuf* ops, size_t op_count) {
@@ -64,6 +56,9 @@
#if defined(SYS_semtimedop)
return syscall(SYS_semtimedop, id, ops, op_count, ts);
#else
+ // 32-bit x86 -- the only architecture without semtimedop(2) -- only has
+ // semtimedop_time64(2), but since we don't have any timespec64 stuff,
+ // it's less painful for us to just stick with the legacy ipc(2) here.
return syscall(SYS_ipc, SEMTIMEDOP, id, op_count, 0, ops, ts);
#endif
}
diff --git a/libc/bionic/sys_shm.cpp b/libc/bionic/sys_shm.cpp
index f780e04..777b3ad 100644
--- a/libc/bionic/sys_shm.cpp
+++ b/libc/bionic/sys_shm.cpp
@@ -32,16 +32,7 @@
#include <unistd.h>
void* shmat(int id, const void* address, int flags) {
-#if defined(SYS_shmat)
return reinterpret_cast<void*>(syscall(SYS_shmat, id, address, flags));
-#else
- // See the kernel's ipc/syscall.c for the other side of this dance.
- void* result = nullptr;
- if (syscall(SYS_ipc, SHMAT, id, flags, &result, address, 0) == -1) {
- return reinterpret_cast<void*>(-1);
- }
- return result;
-#endif
}
int shmctl(int id, int cmd, struct shmid_ds* buf) {
@@ -49,25 +40,13 @@
// Annoyingly, the kernel requires this for 32-bit but rejects it for 64-bit.
cmd |= IPC_64;
#endif
-#if defined(SYS_shmctl)
return syscall(SYS_shmctl, id, cmd, buf);
-#else
- return syscall(SYS_ipc, SHMCTL, id, cmd, 0, buf, 0);
-#endif
}
int shmdt(const void* address) {
-#if defined(SYS_shmdt)
return syscall(SYS_shmdt, address);
-#else
- return syscall(SYS_ipc, SHMDT, 0, 0, 0, address, 0);
-#endif
}
int shmget(key_t key, size_t size, int flags) {
-#if defined(SYS_shmget)
return syscall(SYS_shmget, key, size, flags);
-#else
- return syscall(SYS_ipc, SHMGET, key, size, flags, 0, 0);
-#endif
}
diff --git a/libc/platform/bionic/mte.h b/libc/platform/bionic/mte.h
index 73cd821..98b3d27 100644
--- a/libc/platform/bionic/mte.h
+++ b/libc/platform/bionic/mte.h
@@ -29,8 +29,11 @@
#pragma once
#include <sys/auxv.h>
+#include <sys/mman.h>
#include <sys/prctl.h>
+#include "page.h"
+
// Note: Most PR_MTE_* constants come from the upstream kernel. This tag mask
// allows for the hardware to provision any nonzero tag. Zero tags are reserved
// for scudo to use for the chunk headers in order to prevent linear heap
@@ -63,6 +66,65 @@
}
}
};
+
+// N.B. that this is NOT the pagesize, but 4096. This is hardcoded in the codegen.
+// See
+// https://github.com/search?q=repo%3Allvm/llvm-project%20AArch64StackTagging%3A%3AinsertBaseTaggedPointer&type=code
+constexpr size_t kStackMteRingbufferSizeMultiplier = 4096;
+
+inline size_t stack_mte_ringbuffer_size(uintptr_t size_cls) {
+ return kStackMteRingbufferSizeMultiplier * (1 << size_cls);
+}
+
+inline size_t stack_mte_ringbuffer_size_from_pointer(uintptr_t ptr) {
+ // The size in the top byte is not the size_cls, but the number of "pages" (not OS pages, but
+ // kStackMteRingbufferSizeMultiplier).
+ return kStackMteRingbufferSizeMultiplier * (ptr >> 56ULL);
+}
+
+inline uintptr_t stack_mte_ringbuffer_size_add_to_pointer(uintptr_t ptr, uintptr_t size_cls) {
+ return ptr | ((1ULL << size_cls) << 56ULL);
+}
+
+inline void* stack_mte_ringbuffer_allocate(size_t n, const char* name) {
+ if (n > 7) return nullptr;
+ // Allocation needs to be aligned to 2*size to make the fancy code-gen work.
+ // So we allocate 3*size - pagesz bytes, which will always contain size bytes
+ // aligned to 2*size, and unmap the unneeded part.
+ // See
+ // https://github.com/search?q=repo%3Allvm/llvm-project%20AArch64StackTagging%3A%3AinsertBaseTaggedPointer&type=code
+ //
+ // In the worst case, we get an allocation that is one page past the properly
+ // aligned address, in which case we have to unmap the previous
+ // 2*size - pagesz bytes. In that case, we still have size properly aligned
+ // bytes left.
+ size_t size = stack_mte_ringbuffer_size(n);
+ size_t pgsize = page_size();
+
+ size_t alloc_size = __BIONIC_ALIGN(3 * size - pgsize, pgsize);
+ void* allocation_ptr =
+ mmap(nullptr, alloc_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (allocation_ptr == MAP_FAILED)
+ return nullptr;
+ uintptr_t allocation = reinterpret_cast<uintptr_t>(allocation_ptr);
+
+ size_t alignment = 2 * size;
+ uintptr_t aligned_allocation = __BIONIC_ALIGN(allocation, alignment);
+ if (allocation != aligned_allocation) {
+ munmap(reinterpret_cast<void*>(allocation), aligned_allocation - allocation);
+ }
+ if (aligned_allocation + size != allocation + alloc_size) {
+ munmap(reinterpret_cast<void*>(aligned_allocation + size),
+ (allocation + alloc_size) - (aligned_allocation + size));
+ }
+
+ if (name) {
+ prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, reinterpret_cast<void*>(aligned_allocation), size, name);
+ }
+
+ // We store the size in the top byte of the pointer (which is ignored)
+ return reinterpret_cast<void*>(stack_mte_ringbuffer_size_add_to_pointer(aligned_allocation, n));
+}
#else
struct ScopedDisableMTE {
// Silence unused variable warnings in non-aarch64 builds.