Remove PAGE_SIZE call sites.
To enable experiments with non-4KiB page sizes, introduce
an inline page_size() function that will either return the runtime
page size (if PAGE_SIZE is not 4096) or a constant 4096 (elsewhere).
This should ensure that there are no changes to the generated code on
unaffected platforms.
Test: source build/envsetup.sh
lunch aosp_cf_arm64_16k_phone-userdebug
m -j32 installclean
m -j32
Test: launch_cvd \
-kernel_path /path/to/out/android14-5.15/dist/Image \
-initramfs_path /path/to/out/android14-5.15/dist/initramfs.img \
-userdata_format=ext4
Bug: 277272383
Bug: 230790254
Change-Id: Ic0ed98b67f7c6b845804b90a4e16649f2fc94028
diff --git a/libc/bionic/atexit.cpp b/libc/bionic/atexit.cpp
index 5dbf322..dae15e0 100644
--- a/libc/bionic/atexit.cpp
+++ b/libc/bionic/atexit.cpp
@@ -73,8 +73,8 @@
// restart concurrent __cxa_finalize passes.
uint64_t total_appends_;
- static size_t page_start_of_index(size_t idx) { return PAGE_START(idx * sizeof(AtexitEntry)); }
- static size_t page_end_of_index(size_t idx) { return PAGE_END(idx * sizeof(AtexitEntry)); }
+ static size_t page_start_of_index(size_t idx) { return page_start(idx * sizeof(AtexitEntry)); }
+ static size_t page_end_of_index(size_t idx) { return page_end(idx * sizeof(AtexitEntry)); }
// Recompact the array if it will save at least one page of memory at the end.
bool needs_recompaction() const {
@@ -167,7 +167,7 @@
// than one.
bool AtexitArray::next_capacity(size_t capacity, size_t* result) {
if (capacity == 0) {
- *result = PAGE_END(sizeof(AtexitEntry)) / sizeof(AtexitEntry);
+ *result = page_end(sizeof(AtexitEntry)) / sizeof(AtexitEntry);
return true;
}
size_t num_bytes;
diff --git a/libc/bionic/bionic_allocator.cpp b/libc/bionic/bionic_allocator.cpp
index 98183d4..53f944f 100644
--- a/libc/bionic/bionic_allocator.cpp
+++ b/libc/bionic/bionic_allocator.cpp
@@ -95,12 +95,10 @@
return result;
}
-BionicSmallObjectAllocator::BionicSmallObjectAllocator(uint32_t type,
- size_t block_size)
+BionicSmallObjectAllocator::BionicSmallObjectAllocator(uint32_t type, size_t block_size)
: type_(type),
block_size_(block_size),
- blocks_per_page_((PAGE_SIZE - sizeof(small_object_page_info)) /
- block_size),
+ blocks_per_page_((page_size() - sizeof(small_object_page_info)) / block_size),
free_pages_cnt_(0),
page_list_(nullptr) {}
@@ -157,14 +155,13 @@
if (page_list_ == page) {
page_list_ = page->next_page;
}
- munmap(page, PAGE_SIZE);
+ munmap(page, page_size());
free_pages_cnt_--;
}
void BionicSmallObjectAllocator::free(void* ptr) {
small_object_page_info* const page =
- reinterpret_cast<small_object_page_info*>(
- PAGE_START(reinterpret_cast<uintptr_t>(ptr)));
+ reinterpret_cast<small_object_page_info*>(page_start(reinterpret_cast<uintptr_t>(ptr)));
if (reinterpret_cast<uintptr_t>(ptr) % block_size_ != 0) {
async_safe_fatal("invalid pointer: %p (block_size=%zd)", ptr, block_size_);
@@ -192,14 +189,13 @@
}
void BionicSmallObjectAllocator::alloc_page() {
- void* const map_ptr = mmap(nullptr, PAGE_SIZE, PROT_READ | PROT_WRITE,
- MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ void* const map_ptr =
+ mmap(nullptr, page_size(), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (map_ptr == MAP_FAILED) {
async_safe_fatal("mmap failed: %s", strerror(errno));
}
- prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, map_ptr, PAGE_SIZE,
- "bionic_alloc_small_objects");
+ prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, map_ptr, page_size(), "bionic_alloc_small_objects");
small_object_page_info* const page =
reinterpret_cast<small_object_page_info*>(map_ptr);
@@ -269,10 +265,10 @@
size_t header_size = __BIONIC_ALIGN(kPageInfoSize, align);
size_t allocated_size;
if (__builtin_add_overflow(header_size, size, &allocated_size) ||
- PAGE_END(allocated_size) < allocated_size) {
+ page_end(allocated_size) < allocated_size) {
async_safe_fatal("overflow trying to alloc %zu bytes", size);
}
- allocated_size = PAGE_END(allocated_size);
+ allocated_size = page_end(allocated_size);
void* map_ptr = mmap(nullptr, allocated_size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0);
@@ -317,7 +313,7 @@
void* BionicAllocator::memalign(size_t align, size_t size) {
// The Bionic allocator only supports alignment up to one page, which is good
// enough for ELF TLS.
- align = MIN(align, PAGE_SIZE);
+ align = MIN(align, page_size());
align = MAX(align, 16);
if (!powerof2(align)) {
align = BIONIC_ROUND_UP_POWER_OF_2(align);
@@ -327,7 +323,7 @@
}
inline page_info* BionicAllocator::get_page_info_unchecked(void* ptr) {
- uintptr_t header_page = PAGE_START(reinterpret_cast<size_t>(ptr) - kPageInfoSize);
+ uintptr_t header_page = page_start(reinterpret_cast<size_t>(ptr) - kPageInfoSize);
return reinterpret_cast<page_info*>(header_page);
}
diff --git a/libc/bionic/bionic_elf_tls.cpp b/libc/bionic/bionic_elf_tls.cpp
index 79893e3..077f310 100644
--- a/libc/bionic/bionic_elf_tls.cpp
+++ b/libc/bionic/bionic_elf_tls.cpp
@@ -34,10 +34,11 @@
#include <sys/param.h>
#include <unistd.h>
+#include "platform/bionic/macros.h"
+#include "platform/bionic/page.h"
#include "private/ScopedRWLock.h"
#include "private/ScopedSignalBlocker.h"
#include "private/bionic_globals.h"
-#include "platform/bionic/macros.h"
#include "private/bionic_tls.h"
#include "pthread_internal.h"
@@ -81,7 +82,7 @@
return false;
}
// Bionic only respects TLS alignment up to one page.
- *alignment = MIN(*alignment, PAGE_SIZE);
+ *alignment = MIN(*alignment, page_size());
return true;
}
diff --git a/libc/bionic/fdsan.cpp b/libc/bionic/fdsan.cpp
index 6433b59..3eb8c19 100644
--- a/libc/bionic/fdsan.cpp
+++ b/libc/bionic/fdsan.cpp
@@ -40,6 +40,7 @@
#include <unistd.h>
#include <async_safe/log.h>
+#include <platform/bionic/page.h>
#include <platform/bionic/reserved_signals.h>
#include <sys/system_properties.h>
@@ -80,7 +81,7 @@
size_t required_count = max - inline_fds;
size_t required_size = sizeof(FdTableOverflow) + required_count * sizeof(FdEntry);
- size_t aligned_size = __BIONIC_ALIGN(required_size, PAGE_SIZE);
+ size_t aligned_size = __BIONIC_ALIGN(required_size, page_size());
size_t aligned_count = (aligned_size - sizeof(FdTableOverflow)) / sizeof(FdEntry);
void* allocation =
diff --git a/libc/bionic/getpagesize.cpp b/libc/bionic/getpagesize.cpp
index 3a59900..da97633 100644
--- a/libc/bionic/getpagesize.cpp
+++ b/libc/bionic/getpagesize.cpp
@@ -27,9 +27,10 @@
*/
#include <unistd.h>
+#include "platform/bionic/page.h"
// Portable code should use sysconf(_SC_PAGE_SIZE) directly instead.
int getpagesize() {
// We dont use sysconf(3) here because that drags in stdio, which makes static binaries fat.
- return PAGE_SIZE;
+ return page_size();
}
diff --git a/libc/bionic/libc_init_static.cpp b/libc/bionic/libc_init_static.cpp
index d64d402..3b0ab82 100644
--- a/libc/bionic/libc_init_static.cpp
+++ b/libc/bionic/libc_init_static.cpp
@@ -129,8 +129,8 @@
continue;
}
- ElfW(Addr) seg_page_start = PAGE_START(phdr->p_vaddr);
- ElfW(Addr) seg_page_end = PAGE_END(phdr->p_vaddr + phdr->p_memsz);
+ ElfW(Addr) seg_page_start = page_start(phdr->p_vaddr);
+ ElfW(Addr) seg_page_end = page_end(phdr->p_vaddr + phdr->p_memsz);
// Check return value here? What do we do if we fail?
mprotect(reinterpret_cast<void*>(seg_page_start), seg_page_end - seg_page_start, PROT_READ);
@@ -354,9 +354,9 @@
__libc_shared_globals()->initial_memtag_stack = memtag_stack;
if (memtag_stack) {
- void* page_start =
- reinterpret_cast<void*>(PAGE_START(reinterpret_cast<uintptr_t>(stack_top)));
- if (mprotect(page_start, PAGE_SIZE, PROT_READ | PROT_WRITE | PROT_MTE | PROT_GROWSDOWN)) {
+ void* pg_start =
+ reinterpret_cast<void*>(page_start(reinterpret_cast<uintptr_t>(stack_top)));
+ if (mprotect(pg_start, PAGE_SIZE, PROT_READ | PROT_WRITE | PROT_MTE | PROT_GROWSDOWN)) {
async_safe_fatal("error: failed to set PROT_MTE on main thread stack: %s\n",
strerror(errno));
}
diff --git a/libc/bionic/mmap.cpp b/libc/bionic/mmap.cpp
index ed6b9c6..f05dcb8 100644
--- a/libc/bionic/mmap.cpp
+++ b/libc/bionic/mmap.cpp
@@ -32,6 +32,7 @@
#include <unistd.h>
#include "platform/bionic/macros.h"
+#include "platform/bionic/page.h"
#include "private/ErrnoRestorer.h"
// mmap2(2) is like mmap(2), but the offset is in 4096-byte blocks, not bytes.
@@ -46,7 +47,7 @@
}
// Prevent allocations large enough for `end - start` to overflow.
- size_t rounded = __BIONIC_ALIGN(size, PAGE_SIZE);
+ size_t rounded = __BIONIC_ALIGN(size, page_size());
if (rounded < size || rounded > PTRDIFF_MAX) {
errno = ENOMEM;
return MAP_FAILED;
diff --git a/libc/bionic/mremap.cpp b/libc/bionic/mremap.cpp
index d7c9353..88ec829 100644
--- a/libc/bionic/mremap.cpp
+++ b/libc/bionic/mremap.cpp
@@ -33,12 +33,13 @@
#include <unistd.h>
#include "platform/bionic/macros.h"
+#include "platform/bionic/page.h"
extern "C" void* __mremap(void*, size_t, size_t, int, void*);
void* mremap(void* old_address, size_t old_size, size_t new_size, int flags, ...) {
// prevent allocations large enough for `end - start` to overflow
- size_t rounded = __BIONIC_ALIGN(new_size, PAGE_SIZE);
+ size_t rounded = __BIONIC_ALIGN(new_size, page_size());
if (rounded < new_size || rounded > PTRDIFF_MAX) {
errno = ENOMEM;
return MAP_FAILED;
diff --git a/libc/bionic/pthread_attr.cpp b/libc/bionic/pthread_attr.cpp
index 89aa289..cdae72c 100644
--- a/libc/bionic/pthread_attr.cpp
+++ b/libc/bionic/pthread_attr.cpp
@@ -36,8 +36,9 @@
#include <async_safe/log.h>
-#include "private/bionic_defs.h"
+#include "platform/bionic/page.h"
#include "private/ErrnoRestorer.h"
+#include "private/bionic_defs.h"
#include "pthread_internal.h"
__BIONIC_WEAK_FOR_NATIVE_BRIDGE
@@ -143,10 +144,10 @@
__BIONIC_WEAK_FOR_NATIVE_BRIDGE
int pthread_attr_setstack(pthread_attr_t* attr, void* stack_base, size_t stack_size) {
- if ((stack_size & (PAGE_SIZE - 1) || stack_size < PTHREAD_STACK_MIN)) {
+ if ((stack_size & (page_size() - 1) || stack_size < PTHREAD_STACK_MIN)) {
return EINVAL;
}
- if (reinterpret_cast<uintptr_t>(stack_base) & (PAGE_SIZE - 1)) {
+ if (reinterpret_cast<uintptr_t>(stack_base) & (page_size() - 1)) {
return EINVAL;
}
attr->stack_base = stack_base;
diff --git a/libc/bionic/pthread_create.cpp b/libc/bionic/pthread_create.cpp
index 844f9d8..b46c6cf 100644
--- a/libc/bionic/pthread_create.cpp
+++ b/libc/bionic/pthread_create.cpp
@@ -42,6 +42,7 @@
#include "platform/bionic/macros.h"
#include "platform/bionic/mte.h"
+#include "platform/bionic/page.h"
#include "private/ErrnoRestorer.h"
#include "private/ScopedRWLock.h"
#include "private/bionic_constants.h"
@@ -71,7 +72,7 @@
// Allocate a temporary bionic_tls that the dynamic linker's main thread can
// use while it's loading the initial set of ELF modules.
bionic_tls* __allocate_temp_bionic_tls() {
- size_t allocation_size = __BIONIC_ALIGN(sizeof(bionic_tls), PAGE_SIZE);
+ size_t allocation_size = __BIONIC_ALIGN(sizeof(bionic_tls), page_size());
void* allocation = mmap(nullptr, allocation_size,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
@@ -84,7 +85,7 @@
}
void __free_temp_bionic_tls(bionic_tls* tls) {
- munmap(tls, __BIONIC_ALIGN(sizeof(bionic_tls), PAGE_SIZE));
+ munmap(tls, __BIONIC_ALIGN(sizeof(bionic_tls), page_size()));
}
static void __init_alternate_signal_stack(pthread_internal_t* thread) {
@@ -203,12 +204,11 @@
return 0;
}
-
// Allocate a thread's primary mapping. This mapping includes static TLS and
// optionally a stack. Static TLS includes ELF TLS segments and the bionic_tls
// struct.
//
-// The stack_guard_size must be a multiple of the PAGE_SIZE.
+// The stack_guard_size must be a multiple of the page_size().
ThreadMapping __allocate_thread_mapping(size_t stack_size, size_t stack_guard_size) {
const StaticTlsLayout& layout = __libc_shared_globals()->static_tls_layout;
@@ -220,7 +220,7 @@
// Align the result to a page size.
const size_t unaligned_size = mmap_size;
- mmap_size = __BIONIC_ALIGN(mmap_size, PAGE_SIZE);
+ mmap_size = __BIONIC_ALIGN(mmap_size, page_size());
if (mmap_size < unaligned_size) return {};
// Create a new private anonymous map. Make the entire mapping PROT_NONE, then carve out a
@@ -271,9 +271,9 @@
if (attr->stack_base == nullptr) {
// The caller didn't provide a stack, so allocate one.
- // Make sure the guard size is a multiple of PAGE_SIZE.
+ // Make sure the guard size is a multiple of page_size().
const size_t unaligned_guard_size = attr->guard_size;
- attr->guard_size = __BIONIC_ALIGN(attr->guard_size, PAGE_SIZE);
+ attr->guard_size = __BIONIC_ALIGN(attr->guard_size, page_size());
if (attr->guard_size < unaligned_guard_size) return EAGAIN;
mapping = __allocate_thread_mapping(attr->stack_size, attr->guard_size);
diff --git a/libc/bionic/sysconf.cpp b/libc/bionic/sysconf.cpp
index 1c06c9e..3906e2e 100644
--- a/libc/bionic/sysconf.cpp
+++ b/libc/bionic/sysconf.cpp
@@ -38,6 +38,7 @@
#include <time.h>
#include <unistd.h>
+#include "platform/bionic/page.h"
#include "private/bionic_tls.h"
static long __sysconf_rlimit(int resource) {
diff --git a/libc/bionic/sysinfo.cpp b/libc/bionic/sysinfo.cpp
index 1e4a0e8..79fd55e 100644
--- a/libc/bionic/sysinfo.cpp
+++ b/libc/bionic/sysinfo.cpp
@@ -33,8 +33,9 @@
#include <string.h>
#include <unistd.h>
-#include "private/get_cpu_count_from_string.h"
+#include "platform/bionic/page.h"
#include "private/ScopedReaddir.h"
+#include "private/get_cpu_count_from_string.h"
int __get_cpu_count(const char* sys_file) {
int cpu_count = 1;
@@ -64,11 +65,11 @@
long get_phys_pages() {
struct sysinfo si;
sysinfo(&si);
- return (static_cast<int64_t>(si.totalram) * si.mem_unit) / PAGE_SIZE;
+ return (static_cast<int64_t>(si.totalram) * si.mem_unit) / page_size();
}
long get_avphys_pages() {
struct sysinfo si;
sysinfo(&si);
- return ((static_cast<int64_t>(si.freeram) + si.bufferram) * si.mem_unit) / PAGE_SIZE;
+ return ((static_cast<int64_t>(si.freeram) + si.bufferram) * si.mem_unit) / page_size();
}
diff --git a/libc/platform/bionic/page.h b/libc/platform/bionic/page.h
index d063a98..5f4bb6f 100644
--- a/libc/platform/bionic/page.h
+++ b/libc/platform/bionic/page.h
@@ -16,15 +16,39 @@
#pragma once
-// Get PAGE_SIZE and PAGE_MASK.
+#include <stddef.h>
+#include <stdint.h>
+#include <sys/auxv.h>
+
+// For PAGE_SIZE.
#include <sys/user.h>
+inline size_t page_size() {
+ /*
+ * PAGE_SIZE defines the maximum supported page size. Since 4096 is the
+ * minimum supported page size we can just let it be constant folded if it's
+ * also the maximum.
+ */
+#if PAGE_SIZE == 4096
+ return PAGE_SIZE;
+#else
+ static size_t size = getauxval(AT_PAGESZ);
+ return size;
+#endif
+}
+
// Returns the address of the page containing address 'x'.
-#define PAGE_START(x) ((x) & PAGE_MASK)
+inline uintptr_t page_start(uintptr_t x) {
+ return x & ~(page_size() - 1);
+}
// Returns the offset of address 'x' in its page.
-#define PAGE_OFFSET(x) ((x) & ~PAGE_MASK)
+inline uintptr_t page_offset(uintptr_t x) {
+ return x & (page_size() - 1);
+}
// Returns the address of the next page after address 'x', unless 'x' is
// itself at the start of a page.
-#define PAGE_END(x) PAGE_START((x) + (PAGE_SIZE-1))
+inline uintptr_t page_end(uintptr_t x) {
+ return page_start(x + page_size() - 1);
+}