Merge "[MTE] cleanup: factor out BIONIC_MEMTAG_UPGRADE_SECS handling" into main
diff --git a/docs/status.md b/docs/status.md
index 2919471..bc8ab6a 100644
--- a/docs/status.md
+++ b/docs/status.md
@@ -397,22 +397,25 @@
 overrun is detected, the program is safely aborted as in this
 [example](https://source.android.com/devices/tech/debug/native-crash#fortify).
 
-Note that in recent releases Android's FORTIFY has been extended to
-cover other issues. It can now detect, for example, passing `O_CREAT`
-to open(2) without specifying a mode. It also performs some checking
-regardless of whether the caller was built with FORTIFY enabled. In P,
-for example, calling a `pthread_mutex_` function on a destroyed mutex,
-calling a `<dirent.h>` function on a null pointer, using `%n` with the
-printf(3) family, or using the scanf(3) `m` modifier incorrectly will
-all result in FORTIFY failures even for code not built with FORTIFY.
+Note that Android's FORTIFY has been extended to cover other issues. It can
+detect, for example, passing `O_CREAT` to open(2) without specifying a mode. It
+also performs some checking regardless of whether the caller was built with
+FORTIFY enabled. From API level 28, for example, calling a `pthread_mutex_`
+function on a destroyed mutex, calling a `<dirent.h>` function on a null
+pointer, using `%n` with the printf(3) family, or using the scanf(3) `m`
+modifier incorrectly will all result in FORTIFY failures even for code not built
+with FORTIFY.
 
 More background information is available in our
 [FORTIFY in Android](https://android-developers.googleblog.com/2017/04/fortify-in-android.html)
-blog post.
+blog post, and there's more detail about the implementation in
+[The Anatomy of Clang FORTIFY](clang_fortify_anatomy.md).
 
-The Android platform is built with `-D_FORTIFY_SOURCE=2`, but NDK users
-need to manually enable FORTIFY by setting that themselves in whatever
-build system they're using. The exact subset of FORTIFY available to
+The Android platform is built with `-D_FORTIFY_SOURCE=2`. Users of ndk-build
+or the NDK's CMake toolchain file also get this by default with NDK r21 or
+newer. Users of other build systems
+need to manually enable FORTIFY by setting `_FORTIFY_SOURCE` themselves in
+whatever build system they're using. The exact subset of FORTIFY available to
 NDK users will depend on their target ABI level, because when a FORTIFY
 check can't be guaranteed at compile-time, a call to a run-time `_chk`
 function is added.
diff --git a/libc/Android.bp b/libc/Android.bp
index 84fa498..2efca68 100644
--- a/libc/Android.bp
+++ b/libc/Android.bp
@@ -55,7 +55,9 @@
 cc_defaults {
     name: "libc_defaults",
     defaults: ["linux_bionic_supported"],
-    cflags: libc_common_flags,
+    cflags: libc_common_flags + [
+        "-DUSE_SCUDO",
+    ],
     asflags: libc_common_flags,
     conlyflags: ["-std=gnu99"],
     cppflags: [],
@@ -98,8 +100,8 @@
         malloc_pattern_fill_contents: {
             cflags: ["-DSCUDO_PATTERN_FILL_CONTENTS"],
         },
-        malloc_not_svelte: {
-            cflags: ["-DUSE_SCUDO"],
+        malloc_low_memory: {
+            cflags: ["-UUSE_SCUDO"],
         },
     },
 
@@ -112,32 +114,31 @@
     tidy_disabled_srcs: ["upstream-*/**/*.c"],
 }
 
-libc_scudo_product_variables = {
-    malloc_not_svelte: {
-        cflags: ["-DUSE_SCUDO"],
-        whole_static_libs: ["libscudo"],
-        exclude_static_libs: [
-            "libjemalloc5",
-            "libc_jemalloc_wrapper",
-        ],
-    },
-}
-
 // Defaults for native allocator libs/includes to make it
 // easier to change.
-// To disable scudo for the non-svelte config remove the line:
-//     product_variables: libc_scudo_product_variables,
-// in the cc_defaults below.
 // ========================================================
 cc_defaults {
     name: "libc_native_allocator_defaults",
 
     whole_static_libs: [
-        "libjemalloc5",
-        "libc_jemalloc_wrapper",
+        "libscudo",
+    ],
+    cflags: [
+        "-DUSE_SCUDO",
     ],
     header_libs: ["gwp_asan_headers"],
-    product_variables: libc_scudo_product_variables,
+    product_variables: {
+        malloc_low_memory: {
+            cflags: ["-UUSE_SCUDO"],
+            whole_static_libs: [
+                "libjemalloc5",
+                "libc_jemalloc_wrapper",
+            ],
+            exclude_static_libs: [
+                "libscudo",
+            ],
+        },
+    },
 }
 
 // Functions not implemented by jemalloc directly, or that need to
@@ -2990,3 +2991,8 @@
     name: "versioner-dependencies",
     srcs: ["versioner-dependencies/**/*"],
 }
+
+filegroup {
+    name: "linux_capability_header",
+    srcs: ["kernel/uapi/linux/capability.h"],
+}
diff --git a/libc/bionic/gwp_asan_wrappers.cpp b/libc/bionic/gwp_asan_wrappers.cpp
index 11f7ced..2124f51 100644
--- a/libc/bionic/gwp_asan_wrappers.cpp
+++ b/libc/bionic/gwp_asan_wrappers.cpp
@@ -57,7 +57,7 @@
 static gwp_asan::GuardedPoolAllocator GuardedAlloc;
 static const MallocDispatch* prev_dispatch;
 
-using Action = android_mallopt_gwp_asan_options_t::Action;
+using Mode = android_mallopt_gwp_asan_options_t::Mode;
 using Options = gwp_asan::options::Options;
 
 // basename() is a mess, see the manpage. Let's be explicit what handling we
@@ -261,8 +261,8 @@
   options->Recoverable = true;
   GwpAsanRecoverable = true;
 
-  if (mallopt_options.desire == Action::TURN_ON_WITH_SAMPLING ||
-      mallopt_options.desire == Action::TURN_ON_FOR_APP_SAMPLED_NON_CRASHING) {
+  if (mallopt_options.mode == Mode::SYSTEM_PROCESS_OR_SYSTEM_APP ||
+      mallopt_options.mode == Mode::APP_MANIFEST_DEFAULT) {
     *process_sample_rate = kDefaultProcessSampling;
   } else {
     *process_sample_rate = 1;
@@ -285,7 +285,7 @@
   // be used. Tests still continue to use the environment variable though.
   if (*basename != '\0') {
     const char* default_sysprop = system_sysprop;
-    if (mallopt_options.desire == Action::TURN_ON_FOR_APP) {
+    if (mallopt_options.mode == Mode::APP_MANIFEST_ALWAYS) {
       default_sysprop = app_sysprop;
     }
     async_safe_format_buffer(&program_specific_sysprop[0], kSyspropMaxLen, "%s%s",
@@ -425,7 +425,7 @@
   Options options;
   unsigned process_sample_rate = kDefaultProcessSampling;
   if (!GetGwpAsanOptions(&options, &process_sample_rate, mallopt_options) &&
-      mallopt_options.desire == Action::DONT_TURN_ON_UNLESS_OVERRIDDEN) {
+      mallopt_options.mode == Mode::APP_MANIFEST_NEVER) {
     return false;
   }
 
@@ -492,7 +492,7 @@
 
   android_mallopt_gwp_asan_options_t mallopt_options;
   mallopt_options.program_name = progname;
-  mallopt_options.desire = Action::TURN_ON_WITH_SAMPLING;
+  mallopt_options.mode = Mode::SYSTEM_PROCESS_OR_SYSTEM_APP;
 
   return MaybeInitGwpAsan(globals, mallopt_options);
 }
diff --git a/libc/bionic/heap_tagging.cpp b/libc/bionic/heap_tagging.cpp
index 4d1981c..c8a025f 100644
--- a/libc/bionic/heap_tagging.cpp
+++ b/libc/bionic/heap_tagging.cpp
@@ -38,6 +38,11 @@
 extern "C" void scudo_malloc_disable_memory_tagging();
 extern "C" void scudo_malloc_set_track_allocation_stacks(int);
 
+extern "C" const char* __scudo_get_stack_depot_addr();
+extern "C" const char* __scudo_get_ring_buffer_addr();
+extern "C" size_t __scudo_get_ring_buffer_size();
+extern "C" size_t __scudo_get_stack_depot_size();
+
 // Protected by `g_heap_tagging_lock`.
 static HeapTaggingLevel heap_tagging_level = M_HEAP_TAGGING_LEVEL_NONE;
 
@@ -158,6 +163,10 @@
         set_tcf_on_all_threads(PR_MTE_TCF_SYNC);
 #if defined(USE_SCUDO) && !__has_feature(hwaddress_sanitizer)
         scudo_malloc_set_track_allocation_stacks(1);
+        __libc_shared_globals()->scudo_ring_buffer = __scudo_get_ring_buffer_addr();
+        __libc_shared_globals()->scudo_ring_buffer_size = __scudo_get_ring_buffer_size();
+        __libc_shared_globals()->scudo_stack_depot = __scudo_get_stack_depot_addr();
+        __libc_shared_globals()->scudo_stack_depot_size = __scudo_get_stack_depot_size();
 #endif
       }
       break;
diff --git a/libc/bionic/sys_statvfs.cpp b/libc/bionic/sys_statvfs.cpp
index 3a05c3f..b3a0aca 100644
--- a/libc/bionic/sys_statvfs.cpp
+++ b/libc/bionic/sys_statvfs.cpp
@@ -17,7 +17,7 @@
 #include <sys/statfs.h>
 #include <sys/statvfs.h>
 
-static inline void __bionic_statfs_to_statvfs(const struct statfs* src, struct statvfs* dst) {
+static __inline void __bionic_statfs_to_statvfs(const struct statfs* src, struct statvfs* dst) {
   dst->f_bsize = src->f_bsize;
   dst->f_frsize = src->f_frsize;
   dst->f_blocks = src->f_blocks;
diff --git a/libc/dns/resolv/res_cache.c b/libc/dns/resolv/res_cache.c
index d6416e5..38de84b 100644
--- a/libc/dns/resolv/res_cache.c
+++ b/libc/dns/resolv/res_cache.c
@@ -1166,23 +1166,19 @@
     }
 }
 
-static inline void
-entry_mru_remove( Entry*  e )
-{
-    e->mru_prev->mru_next = e->mru_next;
-    e->mru_next->mru_prev = e->mru_prev;
+static __inline__ void entry_mru_remove(Entry* e) {
+  e->mru_prev->mru_next = e->mru_next;
+  e->mru_next->mru_prev = e->mru_prev;
 }
 
-static inline void
-entry_mru_add( Entry*  e, Entry*  list )
-{
-    Entry*  first = list->mru_next;
+static __inline__ void entry_mru_add(Entry* e, Entry* list) {
+  Entry* first = list->mru_next;
 
-    e->mru_next = first;
-    e->mru_prev = list;
+  e->mru_next = first;
+  e->mru_prev = list;
 
-    list->mru_next  = e;
-    first->mru_prev = e;
+  list->mru_next = e;
+  first->mru_prev = e;
 }
 
 /* compute the hash of a given entry, this is a hash of most
diff --git a/libc/include/android/api-level.h b/libc/include/android/api-level.h
index 113897c..77ec653 100644
--- a/libc/include/android/api-level.h
+++ b/libc/include/android/api-level.h
@@ -191,7 +191,7 @@
 #if __ANDROID_API__ < 29
 
 /* android_get_device_api_level is a static inline before API level 29. */
-#define __BIONIC_GET_DEVICE_API_LEVEL_INLINE static inline
+#define __BIONIC_GET_DEVICE_API_LEVEL_INLINE static __inline
 #include <bits/get_device_api_level_inlines.h>
 #undef __BIONIC_GET_DEVICE_API_LEVEL_INLINE
 
diff --git a/libc/include/android/legacy_stdlib_inlines.h b/libc/include/android/legacy_stdlib_inlines.h
index 0ca1022..f0985fe 100644
--- a/libc/include/android/legacy_stdlib_inlines.h
+++ b/libc/include/android/legacy_stdlib_inlines.h
@@ -38,15 +38,15 @@
 
 __BEGIN_DECLS
 
-static inline double strtod_l(const char* _Nonnull __s, char* _Nullable * _Nullable __end_ptr, locale_t _Nonnull __l) {
+static __inline double strtod_l(const char* _Nonnull __s, char* _Nullable * _Nullable __end_ptr, locale_t _Nonnull __l) {
   return strtod(__s, __end_ptr);
 }
 
-static inline float strtof_l(const char* _Nonnull __s, char* _Nullable * _Nullable __end_ptr, locale_t _Nonnull __l) {
+static __inline float strtof_l(const char* _Nonnull __s, char* _Nullable * _Nullable __end_ptr, locale_t _Nonnull __l) {
   return strtof(__s, __end_ptr);
 }
 
-static inline long strtol_l(const char* _Nonnull __s, char* _Nullable * _Nullable __end_ptr, int __base, locale_t _Nonnull __l) {
+static __inline long strtol_l(const char* _Nonnull __s, char* _Nullable * _Nullable __end_ptr, int __base, locale_t _Nonnull __l) {
   return strtol(__s, __end_ptr, __base);
 }
 
diff --git a/libc/include/android/legacy_termios_inlines.h b/libc/include/android/legacy_termios_inlines.h
index e557525..a816b40 100644
--- a/libc/include/android/legacy_termios_inlines.h
+++ b/libc/include/android/legacy_termios_inlines.h
@@ -39,14 +39,14 @@
 #include <sys/ioctl.h>
 #include <sys/types.h>
 
-#define __BIONIC_TERMIOS_INLINE static inline
+#define __BIONIC_TERMIOS_INLINE static __inline
 #include <bits/termios_inlines.h>
 
 #endif
 
 #if __ANDROID_API__ < 35
 
-#define __BIONIC_TERMIOS_WINSIZE_INLINE static inline
+#define __BIONIC_TERMIOS_WINSIZE_INLINE static __inline
 #include <bits/termios_winsize_inlines.h>
 
 #endif
diff --git a/libc/include/android/legacy_threads_inlines.h b/libc/include/android/legacy_threads_inlines.h
index 06e7438..c614cd0 100644
--- a/libc/include/android/legacy_threads_inlines.h
+++ b/libc/include/android/legacy_threads_inlines.h
@@ -32,7 +32,7 @@
 
 #if __ANDROID_API__ < 30
 
-#define __BIONIC_THREADS_INLINE static inline
+#define __BIONIC_THREADS_INLINE static __inline
 #include <bits/threads_inlines.h>
 
 #endif
diff --git a/libc/include/android/legacy_unistd_inlines.h b/libc/include/android/legacy_unistd_inlines.h
index ac9f3b3..4a5206b 100644
--- a/libc/include/android/legacy_unistd_inlines.h
+++ b/libc/include/android/legacy_unistd_inlines.h
@@ -32,7 +32,7 @@
 
 #if __ANDROID_API__ < 28
 
-#define __BIONIC_SWAB_INLINE static inline
+#define __BIONIC_SWAB_INLINE static __inline
 #include <bits/swab.h>
 
 #endif
diff --git a/libc/include/android/versioning.h b/libc/include/android/versioning.h
index cd61f33..64528e1 100644
--- a/libc/include/android/versioning.h
+++ b/libc/include/android/versioning.h
@@ -22,8 +22,8 @@
 
 #define __INTRODUCED_IN(api_level) __attribute__((__annotate__("introduced_in=" #api_level)))
 #define __INTRODUCED_IN_NO_GUARD_FOR_NDK(api_level) __attribute__((__annotate__("introduced_in=" #api_level))) __VERSIONER_NO_GUARD
-#define __DEPRECATED_IN(api_level) __attribute__((__annotate__("deprecated_in=" #api_level)))
-#define __REMOVED_IN(api_level) __attribute__((__annotate__("obsoleted_in=" #api_level)))
+#define __DEPRECATED_IN(api_level, ...) __attribute__((__annotate__("deprecated_in=" #api_level)))
+#define __REMOVED_IN(api_level, ...) __attribute__((__annotate__("obsoleted_in=" #api_level)))
 #define __INTRODUCED_IN_32(api_level) __attribute__((__annotate__("introduced_in_32=" #api_level)))
 #define __INTRODUCED_IN_64(api_level) __attribute__((__annotate__("introduced_in_64=" #api_level)))
 
@@ -47,16 +47,16 @@
 // libc++ doesn't currently guard these calls. There's no risk to the apps though because using
 // those APIs will still cause a link error.
 #if defined(__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__)
-#define __BIONIC_AVAILABILITY(__what) __attribute__((__availability__(android,__what)))
+#define __BIONIC_AVAILABILITY(__what, ...) __attribute__((__availability__(android,__what __VA_OPT__(,) __VA_ARGS__)))
 #define __INTRODUCED_IN_NO_GUARD_FOR_NDK(api_level) __INTRODUCED_IN(api_level)
 #else
-#define __BIONIC_AVAILABILITY(__what) __attribute__((__availability__(android,strict,__what)))
+#define __BIONIC_AVAILABILITY(__what, ...) __attribute__((__availability__(android,strict,__what __VA_OPT__(,) __VA_ARGS__)))
 #define __INTRODUCED_IN_NO_GUARD_FOR_NDK(api_level)
 #endif
 
 #define __INTRODUCED_IN(api_level) __BIONIC_AVAILABILITY(introduced=api_level)
-#define __DEPRECATED_IN(api_level) __BIONIC_AVAILABILITY(deprecated=api_level)
-#define __REMOVED_IN(api_level) __BIONIC_AVAILABILITY(obsoleted=api_level)
+#define __DEPRECATED_IN(api_level, ...) __BIONIC_AVAILABILITY(deprecated=api_level __VA_OPT__(,message=) __VA_ARGS__)
+#define __REMOVED_IN(api_level, ...) __BIONIC_AVAILABILITY(obsoleted=api_level __VA_OPT__(,message=) __VA_ARGS__)
 
 // The same availability attribute can't be annotated multiple times. Therefore, the macros are
 // defined for the configuration that it is valid for so that declarations like the below doesn't
@@ -80,5 +80,5 @@
 // Vendor modules do not follow SDK versioning. Ignore NDK guards for vendor modules.
 #if defined(__ANDROID_VENDOR__)
 #undef __BIONIC_AVAILABILITY
-#define __BIONIC_AVAILABILITY(x)
+#define __BIONIC_AVAILABILITY(api_level, ...)
 #endif // defined(__ANDROID_VENDOR__)
diff --git a/libc/include/bits/stdatomic.h b/libc/include/bits/stdatomic.h
index 8df86e2..c74eafd 100644
--- a/libc/include/bits/stdatomic.h
+++ b/libc/include/bits/stdatomic.h
@@ -138,11 +138,11 @@
  * 7.17.4 Fences.
  */
 
-static inline void atomic_thread_fence(memory_order __order __attribute__((__unused__))) {
+static __inline void atomic_thread_fence(memory_order __order __attribute__((__unused__))) {
 	__c11_atomic_thread_fence(__order);
 }
 
-static inline void atomic_signal_fence(memory_order __order __attribute__((__unused__))) {
+static __inline void atomic_signal_fence(memory_order __order __attribute__((__unused__))) {
 	__c11_atomic_signal_fence(__order);
 }
 
@@ -269,18 +269,18 @@
 
 #define	ATOMIC_FLAG_INIT		{ ATOMIC_VAR_INIT(false) }
 
-static inline bool atomic_flag_test_and_set_explicit(volatile atomic_flag * _Nonnull __object, memory_order __order) {
+static __inline bool atomic_flag_test_and_set_explicit(volatile atomic_flag * _Nonnull __object, memory_order __order) {
 	return (atomic_exchange_explicit(&__object->__flag, 1, __order));
 }
 
-static inline void atomic_flag_clear_explicit(volatile atomic_flag * _Nonnull __object, memory_order __order) {
+static __inline void atomic_flag_clear_explicit(volatile atomic_flag * _Nonnull __object, memory_order __order) {
 	atomic_store_explicit(&__object->__flag, 0, __order);
 }
 
-static inline bool atomic_flag_test_and_set(volatile atomic_flag * _Nonnull __object) {
+static __inline bool atomic_flag_test_and_set(volatile atomic_flag * _Nonnull __object) {
 	return (atomic_flag_test_and_set_explicit(__object, memory_order_seq_cst));
 }
 
-static inline void atomic_flag_clear(volatile atomic_flag * _Nonnull __object) {
+static __inline void atomic_flag_clear(volatile atomic_flag * _Nonnull __object) {
 	atomic_flag_clear_explicit(__object, memory_order_seq_cst);
 }
diff --git a/libc/include/bits/swab.h b/libc/include/bits/swab.h
index ebb7c74..9591c2e 100644
--- a/libc/include/bits/swab.h
+++ b/libc/include/bits/swab.h
@@ -33,7 +33,7 @@
 #include <sys/types.h>
 
 #if !defined(__BIONIC_SWAB_INLINE)
-#define __BIONIC_SWAB_INLINE static inline
+#define __BIONIC_SWAB_INLINE static __inline
 #endif
 
 __BEGIN_DECLS
diff --git a/libc/include/bits/termios_inlines.h b/libc/include/bits/termios_inlines.h
index 702f433..a884b59 100644
--- a/libc/include/bits/termios_inlines.h
+++ b/libc/include/bits/termios_inlines.h
@@ -37,7 +37,7 @@
 #include <linux/termios.h>
 
 #if !defined(__BIONIC_TERMIOS_INLINE)
-#define __BIONIC_TERMIOS_INLINE static inline
+#define __BIONIC_TERMIOS_INLINE static __inline
 #endif
 
 __BEGIN_DECLS
@@ -45,7 +45,7 @@
 // Supporting separate input and output speeds would require an ABI
 // change for `struct termios`.
 
-static inline speed_t cfgetspeed(const struct termios* _Nonnull s) {
+static __inline speed_t cfgetspeed(const struct termios* _Nonnull s) {
   return __BIONIC_CAST(static_cast, speed_t, s->c_cflag & CBAUD);
 }
 
diff --git a/libc/include/bits/termios_winsize_inlines.h b/libc/include/bits/termios_winsize_inlines.h
index 0d188e7..ae246e4 100644
--- a/libc/include/bits/termios_winsize_inlines.h
+++ b/libc/include/bits/termios_winsize_inlines.h
@@ -36,7 +36,7 @@
 #include <linux/termios.h>
 
 #if !defined(__BIONIC_TERMIOS_WINSIZE_INLINE)
-#define __BIONIC_TERMIOS_WINSIZE_INLINE static inline
+#define __BIONIC_TERMIOS_WINSIZE_INLINE static __inline
 #endif
 
 __BEGIN_DECLS
diff --git a/libc/include/bits/threads_inlines.h b/libc/include/bits/threads_inlines.h
index 074e1ca..5878e0a 100644
--- a/libc/include/bits/threads_inlines.h
+++ b/libc/include/bits/threads_inlines.h
@@ -38,7 +38,7 @@
 
 __BEGIN_DECLS
 
-static inline int __bionic_thrd_error(int __pthread_code) {
+static __inline int __bionic_thrd_error(int __pthread_code) {
   switch (__pthread_code) {
     case 0: return 0;
     case ENOMEM: return thrd_nomem;
diff --git a/libc/include/ctype.h b/libc/include/ctype.h
index 5cad412..c15ee56 100644
--- a/libc/include/ctype.h
+++ b/libc/include/ctype.h
@@ -42,7 +42,7 @@
  * also provide actual symbols for any caller that needs them.
  */
 #if !defined(__BIONIC_CTYPE_INLINE)
-#define __BIONIC_CTYPE_INLINE static inline
+#define __BIONIC_CTYPE_INLINE static __inline
 #endif
 
 /** Internal implementation detail. Do not use. */
diff --git a/libc/include/stdio.h b/libc/include/stdio.h
index 32264d6..78114c3 100644
--- a/libc/include/stdio.h
+++ b/libc/include/stdio.h
@@ -69,7 +69,7 @@
 #define stderr stderr
 #else
 /* Before M the actual symbols for stdin and friends had different names. */
-extern FILE __sF[] __REMOVED_IN(23);
+extern FILE __sF[] __REMOVED_IN(23, "Use stdin/stdout/stderr");
 
 #define stdin (&__sF[0])
 #define stdout (&__sF[1])
diff --git a/libc/include/strings.h b/libc/include/strings.h
index 6ec3bdf..2f4f764 100644
--- a/libc/include/strings.h
+++ b/libc/include/strings.h
@@ -50,7 +50,7 @@
 #include <bits/strcasecmp.h>
 
 #if !defined(__BIONIC_STRINGS_INLINE)
-#define __BIONIC_STRINGS_INLINE static inline
+#define __BIONIC_STRINGS_INLINE static __inline
 #endif
 
 #undef ffs
@@ -61,13 +61,13 @@
 
 /** Deprecated. Use memmove() instead. */
 #define bcopy(b1, b2, len) __bionic_bcopy((b1), (b2), (len))
-static inline __always_inline void __bionic_bcopy(const void* _Nonnull b1, void* _Nonnull b2, size_t len) {
+static __inline__ __always_inline void __bionic_bcopy(const void* _Nonnull b1, void* _Nonnull b2, size_t len) {
   __builtin_memmove(b2, b1, len);
 }
 
 /** Deprecated. Use memset() instead. */
 #define bzero(b, len) __bionic_bzero((b), (len))
-static inline __always_inline void __bionic_bzero(void* _Nonnull b, size_t len) {
+static __inline__ __always_inline void __bionic_bzero(void* _Nonnull b, size_t len) {
   __builtin_memset(b, 0, len);
 }
 
diff --git a/libc/include/sys/cdefs.h b/libc/include/sys/cdefs.h
index 3218d15..7625d38 100644
--- a/libc/include/sys/cdefs.h
+++ b/libc/include/sys/cdefs.h
@@ -260,7 +260,7 @@
  * them available externally. FORTIFY'ed functions try to be as close to possible as 'invisible';
  * having stack protectors detracts from that (b/182948263).
  */
-#  define __BIONIC_FORTIFY_INLINE static inline __attribute__((__no_stack_protector__)) \
+#  define __BIONIC_FORTIFY_INLINE static __inline__ __attribute__((__no_stack_protector__)) \
       __always_inline __VERSIONER_FORTIFY_INLINE
 /*
  * We should use __BIONIC_FORTIFY_VARIADIC instead of __BIONIC_FORTIFY_INLINE
@@ -268,7 +268,7 @@
  * The __always_inline attribute is useless, misleading, and could trigger
  * clang compiler bug to incorrectly inline variadic functions.
  */
-#  define __BIONIC_FORTIFY_VARIADIC static inline
+#  define __BIONIC_FORTIFY_VARIADIC static __inline__
 /* Error functions don't have bodies, so they can just be static. */
 #  define __BIONIC_ERROR_FUNCTION_VISIBILITY static __attribute__((__unused__))
 #else
diff --git a/libc/kernel/tools/cpp.py b/libc/kernel/tools/cpp.py
index 0fd6e46..08b786a 100755
--- a/libc/kernel/tools/cpp.py
+++ b/libc/kernel/tools/cpp.py
@@ -2345,11 +2345,11 @@
 
     def test_function_keep_attribute_structs(self):
         text = """\
-static inline struct some_struct1 * function(struct some_struct2 * e) {
+static __inline__ struct some_struct1 * function(struct some_struct2 * e) {
 }
 """
         expected = """\
-static inline struct some_struct1 * function(struct some_struct2 * e) {
+static __inline__ struct some_struct1 * function(struct some_struct2 * e) {
 }
 """
         self.assertEqual(self.parse(text, set(["function"])), expected)
diff --git a/libc/platform/bionic/malloc.h b/libc/platform/bionic/malloc.h
index ffc6d4a..da85cf5 100644
--- a/libc/platform/bionic/malloc.h
+++ b/libc/platform/bionic/malloc.h
@@ -130,36 +130,55 @@
   // Worth noting, the "libc.debug.gwp_asan.*.app_default" sysprops *do not*
   // apply to system apps. They use the "libc.debug.gwp_asan.*.system_default"
   // sysprops.
-  enum Action {
-    // Enable GWP-ASan. This is used by apps that have `gwpAsanMode=always` in
-    // the manifest.
-    TURN_ON_FOR_APP,
-    // Enable GWP-ASan, but only a small percentage of the time. This is used by
-    // system processes and system apps, and we use a lottery to determine which
-    // processes have GWP-ASan enabled. This allows us to mitigate system-wide
-    // memory overhead concerns, as each GWP-ASan enabled process uses ~70KiB of
-    // extra memory.
-    TURN_ON_WITH_SAMPLING,
-    // Don't enable GWP-ASan, unless overwritten by a system property or
-    // environment variable. This is used by apps that have `gwpAsanMode=never`
-    // in the manifest. Prior to Android 14, this also was used by non-system
-    // apps that didn't specify a `gwpAsanMode` in their manifest.
-    DONT_TURN_ON_UNLESS_OVERRIDDEN,
-    // Enable GWP-ASan, but only a small percentage of the time, and enable it
-    // in the non-crashing ("recoverable") mode. In Android 14, this is used by
-    // apps that don't specify `gwpAsanMode` (or use `gwpAsanMode=default`) in
-    // their manifest. GWP-ASan will detect heap memory safety bugs in this
-    // mode, and bug reports will be created by debuggerd, however the process
-    // will recover and continue to function as if the memory safety bug wasn't
-    // detected.
+  //
+  // In recoverable mode, GWP-ASan will detect heap memory safety bugs, and bug
+  // reports will be created by debuggerd, however the process will recover and
+  // continue to function as if the memory safety bug wasn't detected. This
+  // prevents any user-visible impact as apps and processes don't crash, and
+  // probably saves us some CPU time in restarting the process.
+  //
+  // Process sampling enables GWP-ASan, but only a small percentage of the time
+  // (~1%). This helps mitigate any recurring high-frequency problems in certain
+  // processes, as it's highly likely the next restart of said process won't
+  // have GWP-ASan. In addition, for system processes and system apps, this
+  // allows us to mitigate system-wide memory overhead concerns, as each
+  // GWP-ASan enabled process uses ~70KiB of extra memory.
+  enum Mode {
+    // Used by default for apps, or by those that have an explicit
+    // `gwpAsanMode=default` in the manifest.
     //
-    // In Android 15, this is the same as TURN_ON_WITH_SAMPLING, as GWP-ASan is
-    // only ever used in non-crashing mode (even for platform executables and
-    // system apps).
-    TURN_ON_FOR_APP_SAMPLED_NON_CRASHING,
+    // Result:
+    //  - Android 13 and before: GWP-ASan is not enabled.
+    //  - Android 14 and after: Enables GWP-ASan with process sampling in
+    //    recoverable mode.
+    APP_MANIFEST_DEFAULT = 3,
+    // This is used by apps that have `gwpAsanMode=always` in the manifest.
+    //
+    // Result:
+    //  - Android 14 and before: Enables GWP-ASan in non-recoverable mode,
+    //    without process sampling.
+    //  - Android 15 and after: Enables GWP-ASan in recoverable mode, without
+    //    process sampling.
+    APP_MANIFEST_ALWAYS = 0,
+    // This is used by apps that have `gwpAsanMode=never` in the manifest.
+    //
+    // Result:
+    //  - GWP-ASan is not enabled, unless it's force-enabled by a system
+    //    property or environment variable.
+    APP_MANIFEST_NEVER = 2,
+    // Used by system processes and system apps.
+    //
+    // Result:
+    //  - Android 14 and before: Enables GWP-ASan with process sampling in
+    //    non-recoverable mode.
+    //  - Android 15 and after: Enables GWP-ASan with process sampling in
+    //    recoverable mode.
+    SYSTEM_PROCESS_OR_SYSTEM_APP = 1,
+    // Next enum value = 4. Numbered non-sequentially above to preserve ABI
+    // stability, but now ordered more logically.
   };
 
-  Action desire = DONT_TURN_ON_UNLESS_OVERRIDDEN;
+  Mode mode = APP_MANIFEST_NEVER;
 } android_mallopt_gwp_asan_options_t;
 #pragma clang diagnostic pop
 // Manipulates bionic-specific handling of memory allocation APIs such as
diff --git a/libc/system_properties/system_properties.cpp b/libc/system_properties/system_properties.cpp
index 1dedb61..e0d38a8 100644
--- a/libc/system_properties/system_properties.cpp
+++ b/libc/system_properties/system_properties.cpp
@@ -337,31 +337,42 @@
 
 int SystemProperties::Add(const char* name, unsigned int namelen, const char* value,
                           unsigned int valuelen) {
-  if (valuelen >= PROP_VALUE_MAX && !is_read_only(name)) {
+  if (namelen < 1) {
+    async_safe_format_log(ANDROID_LOG_ERROR, "libc",
+                          "__system_property_add failed: name length 0");
     return -1;
   }
 
-  if (namelen < 1) {
+  if (valuelen >= PROP_VALUE_MAX && !is_read_only(name)) {
+    async_safe_format_log(ANDROID_LOG_ERROR, "libc",
+                          "__system_property_add failed: \"%s\" value too long: %d >= PROP_VALUE_MAX",
+                          name, valuelen);
     return -1;
   }
 
   if (!initialized_) {
+    async_safe_format_log(ANDROID_LOG_ERROR, "libc",
+                          "__system_property_add failed: properties not initialized");
     return -1;
   }
 
   prop_area* serial_pa = contexts_->GetSerialPropArea();
   if (serial_pa == nullptr) {
+    async_safe_format_log(ANDROID_LOG_ERROR, "libc",
+                          "__system_property_add failed: property area not found");
     return -1;
   }
 
   prop_area* pa = contexts_->GetPropAreaForName(name);
   if (!pa) {
-    async_safe_format_log(ANDROID_LOG_ERROR, "libc", "Access denied adding property \"%s\"", name);
+    async_safe_format_log(ANDROID_LOG_ERROR, "libc",
+                          "__system_property_add failed: access denied for \"%s\"", name);
     return -1;
   }
 
-  bool ret = pa->add(name, namelen, value, valuelen);
-  if (!ret) {
+  if (!pa->add(name, namelen, value, valuelen)) {
+    async_safe_format_log(ANDROID_LOG_ERROR, "libc",
+                          "__system_property_add failed: add failed for \"%s\"", name);
     return -1;
   }
 
diff --git a/linker/linker_crt_pad_segment_test.cpp b/linker/linker_crt_pad_segment_test.cpp
index 5a219f8..c11df50 100644
--- a/linker/linker_crt_pad_segment_test.cpp
+++ b/linker/linker_crt_pad_segment_test.cpp
@@ -72,13 +72,22 @@
 };  // anonymous namespace
 
 TEST(crt_pad_segment, note_absent) {
+  if (!page_size_migration_supported()) {
+    GTEST_SKIP() << "Kernel does not support page size migration";
+  }
   ASSERT_FALSE(GetPadSegment("no_crt_pad_segment.so"));
 }
 
 TEST(crt_pad_segment, note_present_and_enabled) {
+  if (!page_size_migration_supported()) {
+    GTEST_SKIP() << "Kernel does not support page size migration";
+  }
   ASSERT_TRUE(GetPadSegment("crt_pad_segment_enabled.so"));
 }
 
 TEST(crt_pad_segment, note_present_and_disabled) {
+  if (!page_size_migration_supported()) {
+    GTEST_SKIP() << "Kernel does not support page size migration";
+  }
   ASSERT_FALSE(GetPadSegment("crt_pad_segment_disabled.so"));
 }
diff --git a/linker/linker_phdr.cpp b/linker/linker_phdr.cpp
index ef7671c..fa712a1 100644
--- a/linker/linker_phdr.cpp
+++ b/linker/linker_phdr.cpp
@@ -46,6 +46,8 @@
 #include "private/CFIShadow.h" // For kLibraryAlignment
 #include "private/elf_note.h"
 
+#include <android-base/file.h>
+
 static int GetTargetElfMachine() {
 #if defined(__arm__)
   return EM_ARM;
@@ -707,8 +709,28 @@
   return true;
 }
 
+/*
+ * Returns true if the kernel supports page size migration, else false.
+ */
+bool page_size_migration_supported() {
+  static bool pgsize_migration_enabled = []() {
+    std::string enabled;
+    if (!android::base::ReadFileToString("/sys/kernel/mm/pgsize_migration/enabled", &enabled)) {
+      return false;
+    }
+    return enabled.find("1") != std::string::npos;
+  }();
+  return pgsize_migration_enabled;
+}
+
 // Find the ELF note of type NT_ANDROID_TYPE_PAD_SEGMENT and check that the desc value is 1.
 bool ElfReader::ReadPadSegmentNote() {
+  if (!page_size_migration_supported()) {
+    // Don't attempt to read the note, since segment extension isn't
+    // supported; but return true so that loading can continue normally.
+    return true;
+  }
+
   // The ELF can have multiple PT_NOTE's, check them all
   for (size_t i = 0; i < phdr_num_; ++i) {
     const ElfW(Phdr)* phdr = &phdr_table_[i];
@@ -773,7 +795,16 @@
   const ElfW(Phdr)* next = nullptr;
   size_t next_idx = phdr_idx + 1;
 
-  if (phdr->p_align == kPageSize || !should_pad_segments) {
+  // Don't do segment extension for p_align > 64KiB, such ELFs already existed in the
+  // field e.g. 2MiB p_align for THPs and are relatively small in number.
+  //
+  // The kernel can only represent padding for p_align up to 64KiB. This is because
+  // the kernel uses 4 available bits in the vm_area_struct to represent padding
+  // extent; and so cannot enable mitigations to avoid breaking app compatibility for
+  // p_aligns > 64KiB.
+  //
+  // Don't perform segment extension on these to avoid app compatibility issues.
+  if (phdr->p_align <= kPageSize || phdr->p_align > 64*1024 || !should_pad_segments) {
     return;
   }
 
@@ -887,10 +918,28 @@
     //   2) Break the COW backing, faulting in new anon pages for a region
     //      that will not be used.
 
-    // _seg_file_end = unextended seg_file_end
-    uint64_t _seg_file_end = seg_start + phdr->p_filesz;
-    if ((phdr->p_flags & PF_W) != 0 && page_offset(_seg_file_end) > 0) {
-      memset(reinterpret_cast<void*>(_seg_file_end), 0, kPageSize - page_offset(_seg_file_end));
+    uint64_t unextended_seg_file_end = seg_start + phdr->p_filesz;
+    if ((phdr->p_flags & PF_W) != 0 && page_offset(unextended_seg_file_end) > 0) {
+      memset(reinterpret_cast<void*>(unextended_seg_file_end), 0,
+             kPageSize - page_offset(unextended_seg_file_end));
+    }
+
+    // Pages may be brought in due to readahead.
+    // Drop the padding (zero) pages, to avoid reclaim work later.
+    //
+    // NOTE: The madvise() here is special, as it also serves to hint to the
+    // kernel the portion of the LOAD segment that is padding.
+    //
+    // See: [1] https://android-review.googlesource.com/c/kernel/common/+/3032411
+    //      [2] https://android-review.googlesource.com/c/kernel/common/+/3048835
+    uint64_t pad_start = page_end(unextended_seg_file_end);
+    uint64_t pad_end = page_end(seg_file_end);
+    CHECK(pad_start <= pad_end);
+    uint64_t pad_len = pad_end - pad_start;
+    if (page_size_migration_supported() && pad_len > 0 &&
+        madvise(reinterpret_cast<void*>(pad_start), pad_len, MADV_DONTNEED)) {
+      DL_WARN("\"%s\": madvise(0x%" PRIx64 ", 0x%" PRIx64 ", MADV_DONTNEED) failed: %m",
+              name_.c_str(), pad_start, pad_len);
     }
 
     seg_file_end = page_end(seg_file_end);
diff --git a/linker/linker_phdr.h b/linker/linker_phdr.h
index 61242eb..aab9018 100644
--- a/linker/linker_phdr.h
+++ b/linker/linker_phdr.h
@@ -154,3 +154,5 @@
 
 const char* phdr_table_get_interpreter_name(const ElfW(Phdr)* phdr_table, size_t phdr_count,
                                             ElfW(Addr) load_bias);
+
+bool page_size_migration_supported();
diff --git a/tests/clang_fortify_tests.cpp b/tests/clang_fortify_tests.cpp
index 544af43..f4ef4ac 100644
--- a/tests/clang_fortify_tests.cpp
+++ b/tests/clang_fortify_tests.cpp
@@ -164,9 +164,7 @@
     const char large_string[] = "Hello!!!";
     static_assert(sizeof(large_string) > sizeof(small_buffer), "");
 
-#if __clang_major__ > 13
-    // expected-error@+3{{will always overflow}}
-#endif
+    // expected-error@+2{{will always overflow}}
     // expected-error@+1{{string bigger than buffer}}
     EXPECT_FORTIFY_DEATH(strcpy(small_buffer, large_string));
     // expected-error@+1{{string bigger than buffer}}
@@ -204,9 +202,7 @@
     static_assert(sizeof(small_string) > sizeof(split.tiny_buffer), "");
 
 #if _FORTIFY_SOURCE > 1
-#if __clang_major__ > 13
-    // expected-error@+4{{will always overflow}}
-#endif
+    // expected-error@+3{{will always overflow}}
     // expected-error@+2{{string bigger than buffer}}
 #endif
     EXPECT_FORTIFY_DEATH_STRUCT(strcpy(split.tiny_buffer, small_string));
diff --git a/tests/malloc_test.cpp b/tests/malloc_test.cpp
index bd17b82..a5916d3 100644
--- a/tests/malloc_test.cpp
+++ b/tests/malloc_test.cpp
@@ -1412,15 +1412,15 @@
 }
 
 #if defined(__BIONIC__)
-using Action = android_mallopt_gwp_asan_options_t::Action;
+using Mode = android_mallopt_gwp_asan_options_t::Mode;
 TEST(android_mallopt, DISABLED_multiple_enable_gwp_asan) {
   android_mallopt_gwp_asan_options_t options;
   options.program_name = "";  // Don't infer GWP-ASan options from sysprops.
-  options.desire = Action::DONT_TURN_ON_UNLESS_OVERRIDDEN;
+  options.mode = Mode::APP_MANIFEST_NEVER;
   // GWP-ASan should already be enabled. Trying to enable or disable it should
   // always pass.
   ASSERT_TRUE(android_mallopt(M_INITIALIZE_GWP_ASAN, &options, sizeof(options)));
-  options.desire = Action::TURN_ON_WITH_SAMPLING;
+  options.mode = Mode::APP_MANIFEST_DEFAULT;
   ASSERT_TRUE(android_mallopt(M_INITIALIZE_GWP_ASAN, &options, sizeof(options)));
 }
 #endif  // defined(__BIONIC__)
diff --git a/tests/setjmp_test.cpp b/tests/setjmp_test.cpp
index 6ae8bfd..0de0a01 100644
--- a/tests/setjmp_test.cpp
+++ b/tests/setjmp_test.cpp
@@ -174,31 +174,23 @@
   }
 }
 
-#if defined(__aarch64__)
+#if defined(__arm__)
+#define SET_FREG(n, v) asm volatile("vmov.f64 d"#n ", #"#v : : : "d"#n)
+#define GET_FREG(n) ({ double _r; asm volatile("fcpyd %P0, d"#n : "=w"(_r) : :); _r;})
+#define CLEAR_FREG(n) asm volatile("vmov.i64 d"#n ", #0x0" : : : "d"#n)
+#elif defined(__aarch64__)
 #define SET_FREG(n, v) asm volatile("fmov d"#n ", "#v : : : "d"#n)
+#define GET_FREG(n) ({ double _r; asm volatile("fmov %0, d"#n : "=r"(_r) : :); _r; })
 #define CLEAR_FREG(n) asm volatile("fmov d"#n ", xzr" : : : "d"#n)
+#endif
+
+#if defined(__arm__) || defined(__aarch64__)
 #define SET_FREGS \
   SET_FREG(8, 8.0); SET_FREG(9, 9.0); SET_FREG(10, 10.0); SET_FREG(11, 11.0); \
   SET_FREG(12, 12.0); SET_FREG(13, 13.0); SET_FREG(14, 14.0); SET_FREG(15, 15.0);
 #define CLEAR_FREGS \
   CLEAR_FREG(8); CLEAR_FREG(9); CLEAR_FREG(10); CLEAR_FREG(11); \
   CLEAR_FREG(12); CLEAR_FREG(13); CLEAR_FREG(14); CLEAR_FREG(15);
-#define GET_FREG(n) ({ double _r; asm volatile("fmov %0, d"#n : "=r"(_r) : :); _r; })
-#define CHECK_FREGS \
-    EXPECT_EQ(8.0, GET_FREG(8)); EXPECT_EQ(9.0, GET_FREG(9)); \
-    EXPECT_EQ(10.0, GET_FREG(10)); EXPECT_EQ(11.0, GET_FREG(11)); \
-    EXPECT_EQ(12.0, GET_FREG(12)); EXPECT_EQ(13.0, GET_FREG(13)); \
-    EXPECT_EQ(14.0, GET_FREG(14)); EXPECT_EQ(15.0, GET_FREG(15));
-#elif defined(__arm__)
-#define SET_FREG(n, v) \
-  ({ const double _v{v}; asm volatile("fcpyd d"#n ", %P0" : : "w"(_v) : "d"#n); })
-#define SET_FREGS \
-  SET_FREG(8, 8); SET_FREG(9, 9); SET_FREG(10, 10); SET_FREG(11, 11); \
-  SET_FREG(12, 12); SET_FREG(13, 13); SET_FREG(14, 14); SET_FREG(15, 15);
-#define CLEAR_FREGS \
-  SET_FREG(8, 0); SET_FREG(9, 0); SET_FREG(10, 0); SET_FREG(11, 0); \
-  SET_FREG(12, 0); SET_FREG(13, 0); SET_FREG(14, 0); SET_FREG(15, 0);
-#define GET_FREG(n) ({ double _r; asm volatile("fcpyd %P0, d"#n : "=w"(_r) : :); _r;})
 #define CHECK_FREGS \
     EXPECT_EQ(8.0, GET_FREG(8)); EXPECT_EQ(9.0, GET_FREG(9)); \
     EXPECT_EQ(10.0, GET_FREG(10)); EXPECT_EQ(11.0, GET_FREG(11)); \
diff --git a/tests/stack_protector_test.cpp b/tests/stack_protector_test.cpp
index c4be78c..aea791c 100644
--- a/tests/stack_protector_test.cpp
+++ b/tests/stack_protector_test.cpp
@@ -136,7 +136,7 @@
   if (stack_mte_enabled()) {
     GTEST_SKIP() << "Stack MTE is enabled, stack protector is not available";
   } else if (hwasan_enabled()) {
-    ASSERT_EXIT(modify_stack_protector_test(), testing::KilledBySignal(SIGABRT), "tag-mismatch");
+    GTEST_SKIP() << "HWASan is enabled, stack protector is not testable";
   } else {
     ASSERT_EXIT(modify_stack_protector_test(), testing::KilledBySignal(SIGABRT),
                 "stack corruption detected");