Merge "Microoptimize the strtol() family." into main
diff --git a/libc/Android.bp b/libc/Android.bp
index 6f36aad..fdf34eb 100644
--- a/libc/Android.bp
+++ b/libc/Android.bp
@@ -67,9 +67,6 @@
// __dso_handle needs to have default visibility on ARM32. See b/73485611.
"-Wexit-time-destructors",
- // GWP-ASan requires platform TLS.
- "-fno-emulated-tls",
-
// We know clang does a lot of harm by rewriting what we've said, and sadly
// never see any good it does, so let's just ask it to do what we say...
// (The specific motivating example was clang turning a loop that would only
diff --git a/libc/bionic/bionic_futex.cpp b/libc/bionic/bionic_futex.cpp
index 0ac1f6e..aa8234f 100644
--- a/libc/bionic/bionic_futex.cpp
+++ b/libc/bionic/bionic_futex.cpp
@@ -28,6 +28,7 @@
#include "private/bionic_futex.h"
+#include <stdatomic.h>
#include <time.h>
#include "private/bionic_time_conversions.h"
@@ -35,7 +36,6 @@
static inline __always_inline int FutexWithTimeout(volatile void* ftx, int op, int value,
bool use_realtime_clock,
const timespec* abs_timeout, int bitset) {
- const timespec* futex_abs_timeout = abs_timeout;
// pthread's and semaphore's default behavior is to use CLOCK_REALTIME, however this behavior is
// essentially never intended, as that clock is prone to change discontinuously.
//
@@ -46,16 +46,26 @@
// We have seen numerous bugs directly attributable to this difference. Therefore, we provide
// this general workaround to always use CLOCK_MONOTONIC for waiting, regardless of what the input
// timespec is.
- timespec converted_monotonic_abs_timeout;
- if (abs_timeout && use_realtime_clock) {
- monotonic_time_from_realtime_time(converted_monotonic_abs_timeout, *abs_timeout);
- if (converted_monotonic_abs_timeout.tv_sec < 0) {
+ timespec converted_timeout;
+ if (abs_timeout) {
+ if ((op & FUTEX_CMD_MASK) == FUTEX_LOCK_PI) {
+ if (!use_realtime_clock) {
+ realtime_time_from_monotonic_time(converted_timeout, *abs_timeout);
+ abs_timeout = &converted_timeout;
+ }
+ } else {
+ op &= ~FUTEX_CLOCK_REALTIME;
+ if (use_realtime_clock) {
+ monotonic_time_from_realtime_time(converted_timeout, *abs_timeout);
+ abs_timeout = &converted_timeout;
+ }
+ }
+ if (abs_timeout->tv_sec < 0) {
return -ETIMEDOUT;
}
- futex_abs_timeout = &converted_monotonic_abs_timeout;
}
- return __futex(ftx, op, value, futex_abs_timeout, bitset);
+ return __futex(ftx, op, value, abs_timeout, bitset);
}
int __futex_wait_ex(volatile void* ftx, bool shared, int value, bool use_realtime_clock,
@@ -66,6 +76,22 @@
int __futex_pi_lock_ex(volatile void* ftx, bool shared, bool use_realtime_clock,
const timespec* abs_timeout) {
- return FutexWithTimeout(ftx, (shared ? FUTEX_LOCK_PI : FUTEX_LOCK_PI_PRIVATE), 0,
- use_realtime_clock, abs_timeout, 0);
+ // We really want FUTEX_LOCK_PI2 which is default CLOCK_MONOTONIC, but that isn't supported
+ // on linux before 5.14. FUTEX_LOCK_PI uses CLOCK_REALTIME. Here we verify support.
+
+ static atomic_int lock_op = 0;
+ int op = atomic_load_explicit(&lock_op, memory_order_relaxed);
+ if (op == 0) {
+ uint32_t tmp = 0;
+ if (__futex(&tmp, FUTEX_LOCK_PI2, 0, nullptr, 0) == 0) {
+ __futex(&tmp, FUTEX_UNLOCK_PI, 0, nullptr, 0);
+ op = FUTEX_LOCK_PI2;
+ } else {
+ op = FUTEX_LOCK_PI;
+ }
+ atomic_store_explicit(&lock_op, op, memory_order_relaxed);
+ }
+
+ if (!shared) op |= FUTEX_PRIVATE_FLAG;
+ return FutexWithTimeout(ftx, op, 0 /* value */, use_realtime_clock, abs_timeout, 0 /* bitset */);
}
diff --git a/libc/bionic/bionic_time_conversions.cpp b/libc/bionic/bionic_time_conversions.cpp
index d21e12e..9f3c50d 100644
--- a/libc/bionic/bionic_time_conversions.cpp
+++ b/libc/bionic/bionic_time_conversions.cpp
@@ -52,23 +52,32 @@
tv.tv_usec = ts.tv_nsec / 1000;
}
-void monotonic_time_from_realtime_time(timespec& monotonic_time, const timespec& realtime_time) {
- monotonic_time = realtime_time;
+static void convert_timespec_clocks(timespec& new_time, clockid_t new_clockbase,
+ const timespec& old_time, clockid_t old_clockbase) {
+ // get reference clocks
+ timespec new_clock;
+ clock_gettime(new_clockbase, &new_clock);
+ timespec old_clock;
+ clock_gettime(old_clockbase, &old_clock);
- timespec cur_monotonic_time;
- clock_gettime(CLOCK_MONOTONIC, &cur_monotonic_time);
- timespec cur_realtime_time;
- clock_gettime(CLOCK_REALTIME, &cur_realtime_time);
+ // compute new time by moving old delta to the new clock.
+ new_time.tv_sec = old_time.tv_sec - old_clock.tv_sec + new_clock.tv_sec;
+ new_time.tv_nsec = old_time.tv_nsec - old_clock.tv_nsec + new_clock.tv_nsec;
- monotonic_time.tv_nsec -= cur_realtime_time.tv_nsec;
- monotonic_time.tv_nsec += cur_monotonic_time.tv_nsec;
- if (monotonic_time.tv_nsec >= NS_PER_S) {
- monotonic_time.tv_nsec -= NS_PER_S;
- monotonic_time.tv_sec += 1;
- } else if (monotonic_time.tv_nsec < 0) {
- monotonic_time.tv_nsec += NS_PER_S;
- monotonic_time.tv_sec -= 1;
+ // correct nsec to second wrap.
+ if (new_time.tv_nsec >= NS_PER_S) {
+ new_time.tv_nsec -= NS_PER_S;
+ new_time.tv_sec += 1;
+ } else if (new_time.tv_nsec < 0) {
+ new_time.tv_nsec += NS_PER_S;
+ new_time.tv_sec -= 1;
}
- monotonic_time.tv_sec -= cur_realtime_time.tv_sec;
- monotonic_time.tv_sec += cur_monotonic_time.tv_sec;
+}
+
+void monotonic_time_from_realtime_time(timespec& monotonic_time, const timespec& realtime_time) {
+ convert_timespec_clocks(monotonic_time, CLOCK_MONOTONIC, realtime_time, CLOCK_REALTIME);
+}
+
+void realtime_time_from_monotonic_time(timespec& realtime_time, const timespec& monotonic_time) {
+ convert_timespec_clocks(realtime_time, CLOCK_REALTIME, monotonic_time, CLOCK_MONOTONIC);
}
diff --git a/libc/bionic/vdso.cpp b/libc/bionic/vdso.cpp
index d0f01d0..0a9a9e5 100644
--- a/libc/bionic/vdso.cpp
+++ b/libc/bionic/vdso.cpp
@@ -131,6 +131,7 @@
for (size_t i = 0; i < vdso_ehdr->e_shnum; ++i) {
if (vdso_shdr[i].sh_type == SHT_DYNSYM) {
symbol_count = vdso_shdr[i].sh_size / sizeof(ElfW(Sym));
+ break;
}
}
if (symbol_count == 0) {
@@ -147,6 +148,7 @@
} else if (vdso_phdr[i].p_type == PT_LOAD) {
vdso_addr = vdso_ehdr_addr + vdso_phdr[i].p_offset - vdso_phdr[i].p_vaddr;
}
+ if (vdso_addr && vdso_dyn) break;
}
if (vdso_addr == 0 || vdso_dyn == nullptr) {
return;
@@ -161,16 +163,18 @@
} else if (d->d_tag == DT_SYMTAB) {
symtab = reinterpret_cast<ElfW(Sym)*>(vdso_addr + d->d_un.d_ptr);
}
+ if (strtab && symtab) break;
}
if (strtab == nullptr || symtab == nullptr) {
return;
}
// Are there any symbols we want?
- for (size_t i = 0; i < symbol_count; ++i) {
- for (size_t j = 0; j < VDSO_END; ++j) {
- if (strcmp(vdso[j].name, strtab + symtab[i].st_name) == 0) {
- vdso[j].fn = reinterpret_cast<void*>(vdso_addr + symtab[i].st_value);
+ for (size_t i = 0; i < VDSO_END; ++i) {
+ for (size_t j = 0; j < symbol_count; ++j) {
+ if (strcmp(vdso[i].name, strtab + symtab[j].st_name) == 0) {
+ vdso[i].fn = reinterpret_cast<void*>(vdso_addr + symtab[j].st_value);
+ break;
}
}
}
diff --git a/libc/platform/bionic/tls_defines.h b/libc/platform/bionic/tls_defines.h
index 3e2efa3..8fe8701 100644
--- a/libc/platform/bionic/tls_defines.h
+++ b/libc/platform/bionic/tls_defines.h
@@ -68,6 +68,11 @@
//
// - TLS_SLOT_APP: Available for use by apps in Android Q and later. (This slot
// was used for errno in P and earlier.)
+//
+// - TLS_SLOT_NATIVE_BRIDGE_GUEST_STATE: Pointer to the guest state for native
+// bridge implementations. It is (to be) used by debuggerd to access this
+// state for guest aware crash reporting of the binary translated code.
+// (Introduced in V)
#if defined(__arm__) || defined(__aarch64__)
@@ -80,7 +85,8 @@
// [1] "Addenda to, and Errata in, the ABI for the ARM Architecture". Section 3.
// http://infocenter.arm.com/help/topic/com.arm.doc.ihi0045e/IHI0045E_ABI_addenda.pdf
-#define MIN_TLS_SLOT (-1) // update this value when reserving a slot
+#define MIN_TLS_SLOT (-2) // update this value when reserving a slot
+#define TLS_SLOT_NATIVE_BRIDGE_GUEST_STATE (-2)
#define TLS_SLOT_BIONIC_TLS (-1)
#define TLS_SLOT_DTV 0
#define TLS_SLOT_THREAD_ID 1
@@ -112,7 +118,8 @@
#define TLS_SLOT_ART_THREAD_SELF 7
#define TLS_SLOT_DTV 8
#define TLS_SLOT_BIONIC_TLS 9
-#define MAX_TLS_SLOT 9 // update this value when reserving a slot
+#define TLS_SLOT_NATIVE_BRIDGE_GUEST_STATE 10
+#define MAX_TLS_SLOT 10 // update this value when reserving a slot
#elif defined(__riscv)
@@ -123,8 +130,9 @@
// [1]: RISC-V ELF Specification. Section: Thread Local Storage
// https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#thread-local-storage
-#define MIN_TLS_SLOT (-9) // update this value when reserving a slot
+#define MIN_TLS_SLOT (-10) // update this value when reserving a slot
+#define TLS_SLOT_NATIVE_BRIDGE_GUEST_STATE (-10)
#define TLS_SLOT_BIONIC_TLS (-9)
#define TLS_SLOT_DTV (-8)
#define TLS_SLOT_THREAD_ID (-7)
diff --git a/libc/private/bionic_constants.h b/libc/private/bionic_constants.h
index 05914f4..6274fe2 100644
--- a/libc/private/bionic_constants.h
+++ b/libc/private/bionic_constants.h
@@ -16,7 +16,7 @@
#pragma once
-#define NS_PER_S 1000000000
+#define NS_PER_S 1'000'000'000LL
// Size of the shadow call stack. This can be small because these stacks only
// contain return addresses. This must be a power of 2 so the mask trick works.
diff --git a/libc/private/bionic_time_conversions.h b/libc/private/bionic_time_conversions.h
index fb049f2..c6b3c78 100644
--- a/libc/private/bionic_time_conversions.h
+++ b/libc/private/bionic_time_conversions.h
@@ -45,6 +45,9 @@
__LIBC_HIDDEN__ void monotonic_time_from_realtime_time(timespec& monotonic_time,
const timespec& realtime_time);
+__LIBC_HIDDEN__ void realtime_time_from_monotonic_time(timespec& realtime_time,
+ const timespec& monotonic_time);
+
__END_DECLS
static inline int check_timespec(const timespec* ts, bool null_allowed) {
diff --git a/libc/system_properties/prop_area.cpp b/libc/system_properties/prop_area.cpp
index 4668eed..d5693b9 100644
--- a/libc/system_properties/prop_area.cpp
+++ b/libc/system_properties/prop_area.cpp
@@ -41,7 +41,7 @@
#include <async_safe/log.h>
-constexpr size_t PA_SIZE = 128 * 1024;
+constexpr size_t PA_SIZE = 256 * 1024;
constexpr uint32_t PROP_AREA_MAGIC = 0x504f5250;
constexpr uint32_t PROP_AREA_VERSION = 0xfc6ed0ab;
diff --git a/tests/pthread_test.cpp b/tests/pthread_test.cpp
index 749d687..1a882be 100644
--- a/tests/pthread_test.cpp
+++ b/tests/pthread_test.cpp
@@ -2398,6 +2398,20 @@
ts.tv_sec = -1;
ASSERT_EQ(ETIMEDOUT, lock_function(&m, &ts));
+ // check we wait long enough for the lock.
+ ASSERT_EQ(0, clock_gettime(clock, &ts));
+ const int64_t start_ns = ts.tv_sec * NS_PER_S + ts.tv_nsec;
+
+ // add a second to get deadline.
+ ts.tv_sec += 1;
+
+ ASSERT_EQ(ETIMEDOUT, lock_function(&m, &ts));
+
+ // The timedlock must have waited at least 1 second before returning.
+ clock_gettime(clock, &ts);
+ const int64_t end_ns = ts.tv_sec * NS_PER_S + ts.tv_nsec;
+ ASSERT_GT(end_ns - start_ns, NS_PER_S);
+
// If the mutex is unlocked, pthread_mutex_timedlock should succeed.
ASSERT_EQ(0, pthread_mutex_unlock(&m));
@@ -2443,7 +2457,11 @@
timespec ts;
clock_gettime(clock, &ts);
+ const int64_t start_ns = ts.tv_sec * NS_PER_S + ts.tv_nsec;
+
+ // add a second to get deadline.
ts.tv_sec += 1;
+
ASSERT_EQ(0, lock_function(&m.lock, &ts));
struct ThreadArgs {
@@ -2472,6 +2490,12 @@
void* result;
ASSERT_EQ(0, pthread_join(thread, &result));
ASSERT_EQ(ETIMEDOUT, reinterpret_cast<intptr_t>(result));
+
+ // The timedlock must have waited at least 1 second before returning.
+ clock_gettime(clock, &ts);
+ const int64_t end_ns = ts.tv_sec * NS_PER_S + ts.tv_nsec;
+ ASSERT_GT(end_ns - start_ns, NS_PER_S);
+
ASSERT_EQ(0, pthread_mutex_unlock(&m.lock));
}
diff --git a/tests/system_properties_test.cpp b/tests/system_properties_test.cpp
index 0b7f5ae..dff92ea 100644
--- a/tests/system_properties_test.cpp
+++ b/tests/system_properties_test.cpp
@@ -593,7 +593,7 @@
ASSERT_TRUE(system_properties.valid());
auto name = "ro.super_long_property"s;
- auto value = std::string(128 * 1024 + 1, 'x');
+ auto value = std::string(256 * 1024 + 1, 'x');
ASSERT_NE(0, system_properties.Add(name.c_str(), name.size(), value.c_str(), value.size()));
#else // __BIONIC__