Merge "Add implementation notes for MTE" into main
diff --git a/libc/Android.bp b/libc/Android.bp
index a4f2c69..08d2606 100644
--- a/libc/Android.bp
+++ b/libc/Android.bp
@@ -264,6 +264,7 @@
name: "libc_init_static",
defaults: ["libc_defaults"],
srcs: [
+ "bionic/libc_init_mte.cpp",
"bionic/libc_init_static.cpp",
":elf_note_sources",
],
@@ -353,7 +354,6 @@
"-Wno-unused-parameter",
"-include netbsd-compat.h",
"-Wframe-larger-than=66000",
- "-include private/bsd_sys_param.h",
],
local_include_dirs: [
@@ -418,20 +418,6 @@
"upstream-freebsd/lib/libc/string/wmemmove.c",
"upstream-freebsd/lib/libc/string/wmemset.c",
],
- arch: {
- x86: {
- exclude_srcs: [
- "upstream-freebsd/lib/libc/string/wcschr.c",
- "upstream-freebsd/lib/libc/string/wcscmp.c",
- "upstream-freebsd/lib/libc/string/wcslen.c",
- "upstream-freebsd/lib/libc/string/wcsrchr.c",
- "upstream-freebsd/lib/libc/string/wmemcmp.c",
- "upstream-freebsd/lib/libc/string/wcscat.c",
- "upstream-freebsd/lib/libc/string/wcscpy.c",
- "upstream-freebsd/lib/libc/string/wmemcmp.c",
- ],
- },
- },
cflags: [
"-Wno-sign-compare",
@@ -1181,12 +1167,9 @@
"arch-x86/bionic/vfork.S",
"arch-x86/bionic/__x86.get_pc_thunk.S",
- "arch-x86/generic/string/memcmp.S",
-
"arch-x86/string/sse2-memchr-atom.S",
"arch-x86/string/sse2-memmove-slm.S",
"arch-x86/string/sse2-memrchr-atom.S",
- "arch-x86/string/sse2-memset-atom.S",
"arch-x86/string/sse2-memset-slm.S",
"arch-x86/string/sse2-stpcpy-slm.S",
"arch-x86/string/sse2-stpncpy-slm.S",
@@ -1196,28 +1179,16 @@
"arch-x86/string/sse2-strncpy-slm.S",
"arch-x86/string/sse2-strnlen-atom.S",
"arch-x86/string/sse2-strrchr-atom.S",
- "arch-x86/string/sse2-wcschr-atom.S",
- "arch-x86/string/sse2-wcsrchr-atom.S",
- "arch-x86/string/sse2-wcslen-atom.S",
- "arch-x86/string/sse2-wcscmp-atom.S",
- "arch-x86/string/sse2-strlen-atom.S",
"arch-x86/string/ssse3-memcmp-atom.S",
- "arch-x86/string/ssse3-memmove-atom.S",
"arch-x86/string/ssse3-strcat-atom.S",
"arch-x86/string/ssse3-strcmp-atom.S",
- "arch-x86/string/ssse3-strcpy-atom.S",
"arch-x86/string/ssse3-strlcat-atom.S",
"arch-x86/string/ssse3-strlcpy-atom.S",
"arch-x86/string/ssse3-strncat-atom.S",
"arch-x86/string/ssse3-strncmp-atom.S",
- "arch-x86/string/ssse3-strncpy-atom.S",
- "arch-x86/string/ssse3-wcscat-atom.S",
- "arch-x86/string/ssse3-wcscpy-atom.S",
- "arch-x86/string/ssse3-wmemcmp-atom.S",
"arch-x86/string/sse4-memcmp-slm.S",
- "arch-x86/string/sse4-wmemcmp-slm.S",
"bionic/strchrnul.cpp",
],
@@ -1790,6 +1761,7 @@
name: "libc_uapi_headers",
visibility: [
"//external/musl",
+ "//external/rust/crates/v4l2r/android",
],
llndk: {
llndk_headers: true,
@@ -2025,15 +1997,6 @@
cc_defaults {
name: "crt_so_defaults",
defaults: ["crt_defaults"],
-
- arch: {
- x86: {
- cflags: ["-fPIC"],
- },
- x86_64: {
- cflags: ["-fPIC"],
- },
- },
stl: "none",
}
@@ -2173,6 +2136,7 @@
// async_safe_fatal_va_list
cc_library_static {
name: "librust_baremetal",
+ defaults: ["cc_baremetal_defaults"],
header_libs: ["libc_headers"],
include_dirs: [
"bionic/libc/async_safe/include",
diff --git a/libc/NOTICE b/libc/NOTICE
index 1a84d3c..1fa036a 100644
--- a/libc/NOTICE
+++ b/libc/NOTICE
@@ -3862,36 +3862,6 @@
-------------------------------------------------------------------
-Copyright (c) 2011 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
--------------------------------------------------------------------
-
Copyright (c) 2011 Martin Pieuchot <mpi@openbsd.org>
Permission to use, copy, modify, and distribute this software for any
@@ -3937,36 +3907,6 @@
-------------------------------------------------------------------
-Copyright (c) 2011, 2012, 2013 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
--------------------------------------------------------------------
-
Copyright (c) 2011, Intel Corporation
All rights reserved.
diff --git a/libc/arch-common/bionic/crt_pad_segment.S b/libc/arch-common/bionic/crt_pad_segment.S
index 86c730d..2fbe0b9 100644
--- a/libc/arch-common/bionic/crt_pad_segment.S
+++ b/libc/arch-common/bionic/crt_pad_segment.S
@@ -26,6 +26,12 @@
* SUCH DAMAGE.
*/
+#if defined(__aarch64__)
+#include <private/bionic_asm_arm64.h>
+
+__bionic_asm_custom_note_gnu_section()
+#endif
+
#include <private/bionic_asm_note.h>
.section ".note.android.pad_segment", "a", %note
diff --git a/libc/arch-x86/dynamic_function_dispatch.cpp b/libc/arch-x86/dynamic_function_dispatch.cpp
index e6cc5fb..240fcdf 100644
--- a/libc/arch-x86/dynamic_function_dispatch.cpp
+++ b/libc/arch-x86/dynamic_function_dispatch.cpp
@@ -33,65 +33,9 @@
DEFINE_IFUNC_FOR(memcmp) {
__builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(memcmp_func_t, memcmp_atom);
if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(memcmp_func_t, memcmp_sse4);
- RETURN_FUNC(memcmp_func_t, memcmp_generic);
+ RETURN_FUNC(memcmp_func_t, memcmp_atom);
}
MEMCMP_SHIM()
-DEFINE_IFUNC_FOR(memset) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(memset_func_t, memset_atom);
- RETURN_FUNC(memset_func_t, memset_generic);
-}
-MEMSET_SHIM()
-
-DEFINE_IFUNC_FOR(__memset_chk) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(__memset_chk_func_t, __memset_chk_atom);
- RETURN_FUNC(__memset_chk_func_t, __memset_chk_generic);
-}
-__MEMSET_CHK_SHIM()
-
-DEFINE_IFUNC_FOR(memmove) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(memmove_func_t, memmove_atom);
- RETURN_FUNC(memmove_func_t, memmove_generic);
-}
-MEMMOVE_SHIM()
-
-DEFINE_IFUNC_FOR(memcpy) { return memmove_resolver(); }
-MEMCPY_SHIM()
-
-DEFINE_IFUNC_FOR(strcpy) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(strcpy_func_t, strcpy_atom);
- RETURN_FUNC(strcpy_func_t, strcpy_generic);
-}
-STRCPY_SHIM()
-
-DEFINE_IFUNC_FOR(strncpy) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(strncpy_func_t, strncpy_atom);
- RETURN_FUNC(strncpy_func_t, strncpy_generic);
-}
-STRNCPY_SHIM()
-
-DEFINE_IFUNC_FOR(strlen) {
- __builtin_cpu_init();
- if (__builtin_cpu_is("atom")) RETURN_FUNC(strlen_func_t, strlen_atom);
- RETURN_FUNC(strlen_func_t, strlen_generic);
-}
-STRLEN_SHIM()
-
-typedef int wmemcmp_func_t(const wchar_t*, const wchar_t*, size_t);
-DEFINE_IFUNC_FOR(wmemcmp) {
- __builtin_cpu_init();
- if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(wmemcmp_func_t, wmemcmp_sse4);
- RETURN_FUNC(wmemcmp_func_t, wmemcmp_atom);
-}
-DEFINE_STATIC_SHIM(int wmemcmp(const wchar_t* lhs, const wchar_t* rhs, size_t n) {
- FORWARD(wmemcmp)(lhs, rhs, n);
-})
-
} // extern "C"
diff --git a/libc/arch-x86/generic/string/memcmp.S b/libc/arch-x86/generic/string/memcmp.S
deleted file mode 100644
index 1d327c7..0000000
--- a/libc/arch-x86/generic/string/memcmp.S
+++ /dev/null
@@ -1,44 +0,0 @@
-/* $OpenBSD: memcmp.S,v 1.4 2005/08/07 11:30:38 espie Exp $ */
-/*
- * Written by J.T. Conklin <jtc@netbsd.org>.
- * Public domain.
- */
-
-#include <private/bionic_asm.h>
-
-ENTRY(memcmp_generic)
- pushl %edi
- pushl %esi
- movl 12(%esp),%edi
- movl 16(%esp),%esi
- cld /* set compare direction forward */
-
- movl 20(%esp),%ecx /* compare by words */
- shrl $2,%ecx
- repe
- cmpsl
- jne L5 /* do we match so far? */
-
- movl 20(%esp),%ecx /* compare remainder by bytes */
- andl $3,%ecx
- repe
- cmpsb
- jne L6 /* do we match? */
-
- xorl %eax,%eax /* we match, return zero */
- popl %esi
- popl %edi
- ret
-
-L5: movl $4,%ecx /* We know that one of the next */
- subl %ecx,%edi /* four pairs of bytes do not */
- subl %ecx,%esi /* match. */
- repe
- cmpsb
-L6: movzbl -1(%edi),%eax /* Perform unsigned comparison */
- movzbl -1(%esi),%edx
- subl %edx,%eax
- popl %esi
- popl %edi
- ret
-END(memcmp_generic)
diff --git a/libc/arch-x86/string/sse2-memmove-slm.S b/libc/arch-x86/string/sse2-memmove-slm.S
index 7f42374..2ed4e7b 100644
--- a/libc/arch-x86/string/sse2-memmove-slm.S
+++ b/libc/arch-x86/string/sse2-memmove-slm.S
@@ -31,7 +31,7 @@
#define FOR_SILVERMONT
#ifndef MEMMOVE
-# define MEMMOVE memmove_generic
+# define MEMMOVE memmove
#endif
#ifndef L
@@ -551,3 +551,9 @@
jmp L(mm_recalc_len)
END (MEMMOVE)
+
+// N.B., `private/bionic_asm.h` provides ALIAS_SYMBOL, but that file provides
+// conflicting definitions for some macros in this file. Since ALIAS_SYMBOL is
+// small, inline it here.
+.globl memcpy;
+.equ memcpy, MEMMOVE
diff --git a/libc/arch-x86/string/sse2-memset-atom.S b/libc/arch-x86/string/sse2-memset-atom.S
deleted file mode 100644
index e43ead0..0000000
--- a/libc/arch-x86/string/sse2-memset-atom.S
+++ /dev/null
@@ -1,841 +0,0 @@
-/*
-Copyright (c) 2010, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#include <private/bionic_asm.h>
-
-#define FOR_ATOM
-
-#ifndef L
-# define L(label) .L##label
-#endif
-
-#ifndef ALIGN
-# define ALIGN(n) .p2align n
-#endif
-
-#define CFI_PUSH(REG) \
- .cfi_adjust_cfa_offset 4; \
- .cfi_rel_offset REG, 0
-
-#define CFI_POP(REG) \
- .cfi_adjust_cfa_offset -4; \
- .cfi_restore REG
-
-#define PUSH(REG) pushl REG; CFI_PUSH(REG)
-#define POP(REG) popl REG; CFI_POP(REG)
-
-#define PARMS 8 /* Preserve EBX. */
-#define DST PARMS
-#define CHR (DST+4)
-#define LEN (CHR+4)
-#define CHK_DST_LEN (LEN+4)
-#define SETRTNVAL movl DST(%esp), %eax
-
-#define ENTRANCE PUSH(%ebx);
-#define RETURN_END POP(%ebx); ret
-#define RETURN RETURN_END; CFI_PUSH(%ebx)
-#define JMPTBL(I, B) I - B
-
-#define SETUP_PIC_REG(x) call __x86.get_pc_thunk.x
-
-/* Load an entry in a jump table into EBX and branch to it. TABLE is a
- jump table with relative offsets. */
-# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \
- /* We first load PC into EBX. */ \
- call __x86.get_pc_thunk.bx; \
- /* Get the address of the jump table. */ \
- add $(TABLE - .), %ebx; \
- /* Get the entry and convert the relative offset to the \
- absolute address. */ \
- add (%ebx,%ecx,4), %ebx; \
- add %ecx, %edx; \
- /* We loaded the jump table and adjusted EDX. Go. */ \
- jmp *%ebx
-
-ENTRY(__memset_chk_atom)
- ENTRANCE
-
- movl LEN(%esp), %ecx
- cmpl CHK_DST_LEN(%esp), %ecx
- jna L(memset_length_loaded)
-
- POP(%ebx) // Undo ENTRANCE without returning.
- jmp __memset_chk_fail
-END(__memset_chk_atom)
-
- .section .text.sse2,"ax",@progbits
- ALIGN(4)
-ENTRY(memset_atom)
- ENTRANCE
-
- movl LEN(%esp), %ecx
-L(memset_length_loaded):
- movzbl CHR(%esp), %eax
- movb %al, %ah
- /* Fill the whole EAX with pattern. */
- movl %eax, %edx
- shl $16, %eax
- or %edx, %eax
- movl DST(%esp), %edx
- cmp $32, %ecx
- jae L(32bytesormore)
-
-L(write_less32bytes):
- BRANCH_TO_JMPTBL_ENTRY(L(table_less_32bytes))
-
-
- .pushsection .rodata.sse2,"a",@progbits
- ALIGN(2)
-L(table_less_32bytes):
- .int JMPTBL(L(write_0bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_1bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_2bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_3bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_4bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_5bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_6bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_7bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_8bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_9bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_10bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_11bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_12bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_13bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_14bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_15bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_16bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_17bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_18bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_19bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_20bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_21bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_22bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_23bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_24bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_25bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_26bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_27bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_28bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_29bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_30bytes), L(table_less_32bytes))
- .int JMPTBL(L(write_31bytes), L(table_less_32bytes))
- .popsection
-
- ALIGN(4)
-L(write_28bytes):
- movl %eax, -28(%edx)
-L(write_24bytes):
- movl %eax, -24(%edx)
-L(write_20bytes):
- movl %eax, -20(%edx)
-L(write_16bytes):
- movl %eax, -16(%edx)
-L(write_12bytes):
- movl %eax, -12(%edx)
-L(write_8bytes):
- movl %eax, -8(%edx)
-L(write_4bytes):
- movl %eax, -4(%edx)
-L(write_0bytes):
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(write_29bytes):
- movl %eax, -29(%edx)
-L(write_25bytes):
- movl %eax, -25(%edx)
-L(write_21bytes):
- movl %eax, -21(%edx)
-L(write_17bytes):
- movl %eax, -17(%edx)
-L(write_13bytes):
- movl %eax, -13(%edx)
-L(write_9bytes):
- movl %eax, -9(%edx)
-L(write_5bytes):
- movl %eax, -5(%edx)
-L(write_1bytes):
- movb %al, -1(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(write_30bytes):
- movl %eax, -30(%edx)
-L(write_26bytes):
- movl %eax, -26(%edx)
-L(write_22bytes):
- movl %eax, -22(%edx)
-L(write_18bytes):
- movl %eax, -18(%edx)
-L(write_14bytes):
- movl %eax, -14(%edx)
-L(write_10bytes):
- movl %eax, -10(%edx)
-L(write_6bytes):
- movl %eax, -6(%edx)
-L(write_2bytes):
- movw %ax, -2(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(write_31bytes):
- movl %eax, -31(%edx)
-L(write_27bytes):
- movl %eax, -27(%edx)
-L(write_23bytes):
- movl %eax, -23(%edx)
-L(write_19bytes):
- movl %eax, -19(%edx)
-L(write_15bytes):
- movl %eax, -15(%edx)
-L(write_11bytes):
- movl %eax, -11(%edx)
-L(write_7bytes):
- movl %eax, -7(%edx)
-L(write_3bytes):
- movw %ax, -3(%edx)
- movb %al, -1(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-/* ECX > 32 and EDX is 4 byte aligned. */
-L(32bytesormore):
- /* Fill xmm0 with the pattern. */
- movd %eax, %xmm0
- pshufd $0, %xmm0, %xmm0
- testl $0xf, %edx
- jz L(aligned_16)
-/* ECX > 32 and EDX is not 16 byte aligned. */
-L(not_aligned_16):
- movdqu %xmm0, (%edx)
- movl %edx, %eax
- and $-16, %edx
- add $16, %edx
- sub %edx, %eax
- add %eax, %ecx
- movd %xmm0, %eax
-
- ALIGN(4)
-L(aligned_16):
- cmp $128, %ecx
- jae L(128bytesormore)
-
-L(aligned_16_less128bytes):
- BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
-
- ALIGN(4)
-L(128bytesormore):
- PUSH(%ebx)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- mov __x86_shared_cache_size@GOTOFF(%ebx), %ebx
- cmp %ebx, %ecx
- jae L(128bytesormore_nt_start)
-
-
- POP(%ebx)
-# define RESTORE_EBX_STATE CFI_PUSH(%ebx)
- PUSH(%ebx)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size@GOTOFF(%ebx), %ecx
- POP(%ebx)
-
- jae L(128bytes_L2_normal)
- subl $128, %ecx
-L(128bytesormore_normal):
- sub $128, %ecx
- movdqa %xmm0, (%edx)
- movdqa %xmm0, 0x10(%edx)
- movdqa %xmm0, 0x20(%edx)
- movdqa %xmm0, 0x30(%edx)
- movdqa %xmm0, 0x40(%edx)
- movdqa %xmm0, 0x50(%edx)
- movdqa %xmm0, 0x60(%edx)
- movdqa %xmm0, 0x70(%edx)
- lea 128(%edx), %edx
- jb L(128bytesless_normal)
-
-
- sub $128, %ecx
- movdqa %xmm0, (%edx)
- movdqa %xmm0, 0x10(%edx)
- movdqa %xmm0, 0x20(%edx)
- movdqa %xmm0, 0x30(%edx)
- movdqa %xmm0, 0x40(%edx)
- movdqa %xmm0, 0x50(%edx)
- movdqa %xmm0, 0x60(%edx)
- movdqa %xmm0, 0x70(%edx)
- lea 128(%edx), %edx
- jae L(128bytesormore_normal)
-
-L(128bytesless_normal):
- add $128, %ecx
- BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
-
- ALIGN(4)
-L(128bytes_L2_normal):
- prefetcht0 0x380(%edx)
- prefetcht0 0x3c0(%edx)
- sub $128, %ecx
- movdqa %xmm0, (%edx)
- movaps %xmm0, 0x10(%edx)
- movaps %xmm0, 0x20(%edx)
- movaps %xmm0, 0x30(%edx)
- movaps %xmm0, 0x40(%edx)
- movaps %xmm0, 0x50(%edx)
- movaps %xmm0, 0x60(%edx)
- movaps %xmm0, 0x70(%edx)
- add $128, %edx
- cmp $128, %ecx
- jae L(128bytes_L2_normal)
-
-L(128bytesless_L2_normal):
- BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
-
- RESTORE_EBX_STATE
-L(128bytesormore_nt_start):
- sub %ebx, %ecx
- mov %ebx, %eax
- and $0x7f, %eax
- add %eax, %ecx
- movd %xmm0, %eax
- ALIGN(4)
-L(128bytesormore_shared_cache_loop):
- prefetcht0 0x3c0(%edx)
- prefetcht0 0x380(%edx)
- sub $0x80, %ebx
- movdqa %xmm0, (%edx)
- movdqa %xmm0, 0x10(%edx)
- movdqa %xmm0, 0x20(%edx)
- movdqa %xmm0, 0x30(%edx)
- movdqa %xmm0, 0x40(%edx)
- movdqa %xmm0, 0x50(%edx)
- movdqa %xmm0, 0x60(%edx)
- movdqa %xmm0, 0x70(%edx)
- add $0x80, %edx
- cmp $0x80, %ebx
- jae L(128bytesormore_shared_cache_loop)
- cmp $0x80, %ecx
- jb L(shared_cache_loop_end)
- ALIGN(4)
-L(128bytesormore_nt):
- sub $0x80, %ecx
- movntdq %xmm0, (%edx)
- movntdq %xmm0, 0x10(%edx)
- movntdq %xmm0, 0x20(%edx)
- movntdq %xmm0, 0x30(%edx)
- movntdq %xmm0, 0x40(%edx)
- movntdq %xmm0, 0x50(%edx)
- movntdq %xmm0, 0x60(%edx)
- movntdq %xmm0, 0x70(%edx)
- add $0x80, %edx
- cmp $0x80, %ecx
- jae L(128bytesormore_nt)
- sfence
-L(shared_cache_loop_end):
- POP(%ebx)
- BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
-
-
- .pushsection .rodata.sse2,"a",@progbits
- ALIGN(2)
-L(table_16_128bytes):
- .int JMPTBL(L(aligned_16_0bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_1bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_2bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_3bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_4bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_5bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_6bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_7bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_8bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_9bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_10bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_11bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_12bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_13bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_14bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_15bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_16bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_17bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_18bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_19bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_20bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_21bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_22bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_23bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_24bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_25bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_26bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_27bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_28bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_29bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_30bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_31bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_32bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_33bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_34bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_35bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_36bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_37bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_38bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_39bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_40bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_41bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_42bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_43bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_44bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_45bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_46bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_47bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_48bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_49bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_50bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_51bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_52bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_53bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_54bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_55bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_56bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_57bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_58bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_59bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_60bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_61bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_62bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_63bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_64bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_65bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_66bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_67bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_68bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_69bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_70bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_71bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_72bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_73bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_74bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_75bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_76bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_77bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_78bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_79bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_80bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_81bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_82bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_83bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_84bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_85bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_86bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_87bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_88bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_89bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_90bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_91bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_92bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_93bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_94bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_95bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_96bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_97bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_98bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_99bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_100bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_101bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_102bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_103bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_104bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_105bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_106bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_107bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_108bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_109bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_110bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_111bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_112bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_113bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_114bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_115bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_116bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_117bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_118bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_119bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_120bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_121bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_122bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_123bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_124bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_125bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_126bytes), L(table_16_128bytes))
- .int JMPTBL(L(aligned_16_127bytes), L(table_16_128bytes))
- .popsection
-
- ALIGN(4)
-L(aligned_16_112bytes):
- movdqa %xmm0, -112(%edx)
-L(aligned_16_96bytes):
- movdqa %xmm0, -96(%edx)
-L(aligned_16_80bytes):
- movdqa %xmm0, -80(%edx)
-L(aligned_16_64bytes):
- movdqa %xmm0, -64(%edx)
-L(aligned_16_48bytes):
- movdqa %xmm0, -48(%edx)
-L(aligned_16_32bytes):
- movdqa %xmm0, -32(%edx)
-L(aligned_16_16bytes):
- movdqa %xmm0, -16(%edx)
-L(aligned_16_0bytes):
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_113bytes):
- movdqa %xmm0, -113(%edx)
-L(aligned_16_97bytes):
- movdqa %xmm0, -97(%edx)
-L(aligned_16_81bytes):
- movdqa %xmm0, -81(%edx)
-L(aligned_16_65bytes):
- movdqa %xmm0, -65(%edx)
-L(aligned_16_49bytes):
- movdqa %xmm0, -49(%edx)
-L(aligned_16_33bytes):
- movdqa %xmm0, -33(%edx)
-L(aligned_16_17bytes):
- movdqa %xmm0, -17(%edx)
-L(aligned_16_1bytes):
- movb %al, -1(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_114bytes):
- movdqa %xmm0, -114(%edx)
-L(aligned_16_98bytes):
- movdqa %xmm0, -98(%edx)
-L(aligned_16_82bytes):
- movdqa %xmm0, -82(%edx)
-L(aligned_16_66bytes):
- movdqa %xmm0, -66(%edx)
-L(aligned_16_50bytes):
- movdqa %xmm0, -50(%edx)
-L(aligned_16_34bytes):
- movdqa %xmm0, -34(%edx)
-L(aligned_16_18bytes):
- movdqa %xmm0, -18(%edx)
-L(aligned_16_2bytes):
- movw %ax, -2(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_115bytes):
- movdqa %xmm0, -115(%edx)
-L(aligned_16_99bytes):
- movdqa %xmm0, -99(%edx)
-L(aligned_16_83bytes):
- movdqa %xmm0, -83(%edx)
-L(aligned_16_67bytes):
- movdqa %xmm0, -67(%edx)
-L(aligned_16_51bytes):
- movdqa %xmm0, -51(%edx)
-L(aligned_16_35bytes):
- movdqa %xmm0, -35(%edx)
-L(aligned_16_19bytes):
- movdqa %xmm0, -19(%edx)
-L(aligned_16_3bytes):
- movw %ax, -3(%edx)
- movb %al, -1(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_116bytes):
- movdqa %xmm0, -116(%edx)
-L(aligned_16_100bytes):
- movdqa %xmm0, -100(%edx)
-L(aligned_16_84bytes):
- movdqa %xmm0, -84(%edx)
-L(aligned_16_68bytes):
- movdqa %xmm0, -68(%edx)
-L(aligned_16_52bytes):
- movdqa %xmm0, -52(%edx)
-L(aligned_16_36bytes):
- movdqa %xmm0, -36(%edx)
-L(aligned_16_20bytes):
- movdqa %xmm0, -20(%edx)
-L(aligned_16_4bytes):
- movl %eax, -4(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_117bytes):
- movdqa %xmm0, -117(%edx)
-L(aligned_16_101bytes):
- movdqa %xmm0, -101(%edx)
-L(aligned_16_85bytes):
- movdqa %xmm0, -85(%edx)
-L(aligned_16_69bytes):
- movdqa %xmm0, -69(%edx)
-L(aligned_16_53bytes):
- movdqa %xmm0, -53(%edx)
-L(aligned_16_37bytes):
- movdqa %xmm0, -37(%edx)
-L(aligned_16_21bytes):
- movdqa %xmm0, -21(%edx)
-L(aligned_16_5bytes):
- movl %eax, -5(%edx)
- movb %al, -1(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_118bytes):
- movdqa %xmm0, -118(%edx)
-L(aligned_16_102bytes):
- movdqa %xmm0, -102(%edx)
-L(aligned_16_86bytes):
- movdqa %xmm0, -86(%edx)
-L(aligned_16_70bytes):
- movdqa %xmm0, -70(%edx)
-L(aligned_16_54bytes):
- movdqa %xmm0, -54(%edx)
-L(aligned_16_38bytes):
- movdqa %xmm0, -38(%edx)
-L(aligned_16_22bytes):
- movdqa %xmm0, -22(%edx)
-L(aligned_16_6bytes):
- movl %eax, -6(%edx)
- movw %ax, -2(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_119bytes):
- movdqa %xmm0, -119(%edx)
-L(aligned_16_103bytes):
- movdqa %xmm0, -103(%edx)
-L(aligned_16_87bytes):
- movdqa %xmm0, -87(%edx)
-L(aligned_16_71bytes):
- movdqa %xmm0, -71(%edx)
-L(aligned_16_55bytes):
- movdqa %xmm0, -55(%edx)
-L(aligned_16_39bytes):
- movdqa %xmm0, -39(%edx)
-L(aligned_16_23bytes):
- movdqa %xmm0, -23(%edx)
-L(aligned_16_7bytes):
- movl %eax, -7(%edx)
- movw %ax, -3(%edx)
- movb %al, -1(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_120bytes):
- movdqa %xmm0, -120(%edx)
-L(aligned_16_104bytes):
- movdqa %xmm0, -104(%edx)
-L(aligned_16_88bytes):
- movdqa %xmm0, -88(%edx)
-L(aligned_16_72bytes):
- movdqa %xmm0, -72(%edx)
-L(aligned_16_56bytes):
- movdqa %xmm0, -56(%edx)
-L(aligned_16_40bytes):
- movdqa %xmm0, -40(%edx)
-L(aligned_16_24bytes):
- movdqa %xmm0, -24(%edx)
-L(aligned_16_8bytes):
- movq %xmm0, -8(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_121bytes):
- movdqa %xmm0, -121(%edx)
-L(aligned_16_105bytes):
- movdqa %xmm0, -105(%edx)
-L(aligned_16_89bytes):
- movdqa %xmm0, -89(%edx)
-L(aligned_16_73bytes):
- movdqa %xmm0, -73(%edx)
-L(aligned_16_57bytes):
- movdqa %xmm0, -57(%edx)
-L(aligned_16_41bytes):
- movdqa %xmm0, -41(%edx)
-L(aligned_16_25bytes):
- movdqa %xmm0, -25(%edx)
-L(aligned_16_9bytes):
- movq %xmm0, -9(%edx)
- movb %al, -1(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_122bytes):
- movdqa %xmm0, -122(%edx)
-L(aligned_16_106bytes):
- movdqa %xmm0, -106(%edx)
-L(aligned_16_90bytes):
- movdqa %xmm0, -90(%edx)
-L(aligned_16_74bytes):
- movdqa %xmm0, -74(%edx)
-L(aligned_16_58bytes):
- movdqa %xmm0, -58(%edx)
-L(aligned_16_42bytes):
- movdqa %xmm0, -42(%edx)
-L(aligned_16_26bytes):
- movdqa %xmm0, -26(%edx)
-L(aligned_16_10bytes):
- movq %xmm0, -10(%edx)
- movw %ax, -2(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_123bytes):
- movdqa %xmm0, -123(%edx)
-L(aligned_16_107bytes):
- movdqa %xmm0, -107(%edx)
-L(aligned_16_91bytes):
- movdqa %xmm0, -91(%edx)
-L(aligned_16_75bytes):
- movdqa %xmm0, -75(%edx)
-L(aligned_16_59bytes):
- movdqa %xmm0, -59(%edx)
-L(aligned_16_43bytes):
- movdqa %xmm0, -43(%edx)
-L(aligned_16_27bytes):
- movdqa %xmm0, -27(%edx)
-L(aligned_16_11bytes):
- movq %xmm0, -11(%edx)
- movw %ax, -3(%edx)
- movb %al, -1(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_124bytes):
- movdqa %xmm0, -124(%edx)
-L(aligned_16_108bytes):
- movdqa %xmm0, -108(%edx)
-L(aligned_16_92bytes):
- movdqa %xmm0, -92(%edx)
-L(aligned_16_76bytes):
- movdqa %xmm0, -76(%edx)
-L(aligned_16_60bytes):
- movdqa %xmm0, -60(%edx)
-L(aligned_16_44bytes):
- movdqa %xmm0, -44(%edx)
-L(aligned_16_28bytes):
- movdqa %xmm0, -28(%edx)
-L(aligned_16_12bytes):
- movq %xmm0, -12(%edx)
- movl %eax, -4(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_125bytes):
- movdqa %xmm0, -125(%edx)
-L(aligned_16_109bytes):
- movdqa %xmm0, -109(%edx)
-L(aligned_16_93bytes):
- movdqa %xmm0, -93(%edx)
-L(aligned_16_77bytes):
- movdqa %xmm0, -77(%edx)
-L(aligned_16_61bytes):
- movdqa %xmm0, -61(%edx)
-L(aligned_16_45bytes):
- movdqa %xmm0, -45(%edx)
-L(aligned_16_29bytes):
- movdqa %xmm0, -29(%edx)
-L(aligned_16_13bytes):
- movq %xmm0, -13(%edx)
- movl %eax, -5(%edx)
- movb %al, -1(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_126bytes):
- movdqa %xmm0, -126(%edx)
-L(aligned_16_110bytes):
- movdqa %xmm0, -110(%edx)
-L(aligned_16_94bytes):
- movdqa %xmm0, -94(%edx)
-L(aligned_16_78bytes):
- movdqa %xmm0, -78(%edx)
-L(aligned_16_62bytes):
- movdqa %xmm0, -62(%edx)
-L(aligned_16_46bytes):
- movdqa %xmm0, -46(%edx)
-L(aligned_16_30bytes):
- movdqa %xmm0, -30(%edx)
-L(aligned_16_14bytes):
- movq %xmm0, -14(%edx)
- movl %eax, -6(%edx)
- movw %ax, -2(%edx)
- SETRTNVAL
- RETURN
-
- ALIGN(4)
-L(aligned_16_127bytes):
- movdqa %xmm0, -127(%edx)
-L(aligned_16_111bytes):
- movdqa %xmm0, -111(%edx)
-L(aligned_16_95bytes):
- movdqa %xmm0, -95(%edx)
-L(aligned_16_79bytes):
- movdqa %xmm0, -79(%edx)
-L(aligned_16_63bytes):
- movdqa %xmm0, -63(%edx)
-L(aligned_16_47bytes):
- movdqa %xmm0, -47(%edx)
-L(aligned_16_31bytes):
- movdqa %xmm0, -31(%edx)
-L(aligned_16_15bytes):
- movq %xmm0, -15(%edx)
- movl %eax, -7(%edx)
- movw %ax, -3(%edx)
- movb %al, -1(%edx)
- SETRTNVAL
- RETURN_END
-
-END(memset_atom)
diff --git a/libc/arch-x86/string/sse2-memset-slm.S b/libc/arch-x86/string/sse2-memset-slm.S
index e4c8fa1..ec2ee52 100644
--- a/libc/arch-x86/string/sse2-memset-slm.S
+++ b/libc/arch-x86/string/sse2-memset-slm.S
@@ -79,7 +79,7 @@
/* We loaded the jump table and adjusted EDX. Go. */ \
jmp *%ebx
-ENTRY(__memset_chk_generic)
+ENTRY(__memset_chk)
ENTRANCE
movl LEN(%esp), %ecx
@@ -88,11 +88,11 @@
POP(%ebx) // Undo ENTRANCE without returning.
jmp __memset_chk_fail
-END(__memset_chk_generic)
+END(__memset_chk)
.section .text.sse2,"ax",@progbits
ALIGN(4)
-ENTRY(memset_generic)
+ENTRY(memset)
ENTRANCE
movl LEN(%esp), %ecx
@@ -755,4 +755,4 @@
SETRTNVAL
RETURN_END
-END(memset_generic)
+END(memset)
diff --git a/libc/arch-x86/string/sse2-stpcpy-slm.S b/libc/arch-x86/string/sse2-stpcpy-slm.S
old mode 100755
new mode 100644
diff --git a/libc/arch-x86/string/sse2-strcpy-slm.S b/libc/arch-x86/string/sse2-strcpy-slm.S
old mode 100755
new mode 100644
index 22ceeab..b5d84b5
--- a/libc/arch-x86/string/sse2-strcpy-slm.S
+++ b/libc/arch-x86/string/sse2-strcpy-slm.S
@@ -79,7 +79,7 @@
#define POP(REG) popl REG; CFI_POP (REG)
#ifndef STRCPY
-# define STRCPY strcpy_generic
+# define STRCPY strcpy
#endif
#ifdef USE_AS_STPNCPY
diff --git a/libc/arch-x86/string/sse2-strlen-slm.S b/libc/arch-x86/string/sse2-strlen-slm.S
old mode 100755
new mode 100644
index b805ad6..27cc025
--- a/libc/arch-x86/string/sse2-strlen-slm.S
+++ b/libc/arch-x86/string/sse2-strlen-slm.S
@@ -29,7 +29,7 @@
*/
#ifndef STRLEN
-# define STRLEN strlen_generic
+# define STRLEN strlen
#endif
#ifndef L
diff --git a/libc/arch-x86/string/sse2-strncpy-slm.S b/libc/arch-x86/string/sse2-strncpy-slm.S
old mode 100755
new mode 100644
index aff7fb9..591419f
--- a/libc/arch-x86/string/sse2-strncpy-slm.S
+++ b/libc/arch-x86/string/sse2-strncpy-slm.S
@@ -29,5 +29,5 @@
*/
#define USE_AS_STRNCPY
-#define STRCPY strncpy_generic
+#define STRCPY strncpy
#include "sse2-strcpy-slm.S"
diff --git a/libc/arch-x86/string/sse2-wcschr-atom.S b/libc/arch-x86/string/sse2-wcschr-atom.S
deleted file mode 100644
index 729302b..0000000
--- a/libc/arch-x86/string/sse2-wcschr-atom.S
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
-Copyright (c) 2011 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef L
-# define L(label) .L##label
-#endif
-
-#ifndef cfi_startproc
-# define cfi_startproc .cfi_startproc
-#endif
-
-#ifndef cfi_endproc
-# define cfi_endproc .cfi_endproc
-#endif
-
-#ifndef cfi_rel_offset
-# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off
-#endif
-
-#ifndef cfi_restore
-# define cfi_restore(reg) .cfi_restore reg
-#endif
-
-#ifndef cfi_adjust_cfa_offset
-# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off
-#endif
-
-#ifndef ENTRY
-# define ENTRY(name) \
- .type name, @function; \
- .globl name; \
- .p2align 4; \
-name: \
- cfi_startproc
-#endif
-
-#ifndef END
-# define END(name) \
- cfi_endproc; \
- .size name, .-name
-#endif
-
-#define CFI_PUSH(REG) \
- cfi_adjust_cfa_offset (4); \
- cfi_rel_offset (REG, 0)
-
-#define CFI_POP(REG) \
- cfi_adjust_cfa_offset (-4); \
- cfi_restore (REG)
-
-#define PUSH(REG) pushl REG; CFI_PUSH (REG)
-#define POP(REG) popl REG; CFI_POP (REG)
-
-#define PARMS 4
-
-
-#define STR1 PARMS
-#define STR2 STR1+4
-
- .text
-ENTRY (wcschr)
-
- mov STR1(%esp), %ecx
- movd STR2(%esp), %xmm1
-
- mov %ecx, %eax
- punpckldq %xmm1, %xmm1
- pxor %xmm2, %xmm2
- punpckldq %xmm1, %xmm1
-
- and $63, %eax
- cmp $48, %eax
- ja L(cross_cache)
-
- movdqu (%ecx), %xmm0
- pcmpeqd %xmm0, %xmm2
- pcmpeqd %xmm1, %xmm0
- pmovmskb %xmm2, %edx
- pmovmskb %xmm0, %eax
- or %eax, %edx
- jnz L(matches)
- and $-16, %ecx
- jmp L(loop)
-
- .p2align 4
-L(cross_cache):
- PUSH (%edi)
- mov %ecx, %edi
- mov %eax, %ecx
- and $-16, %edi
- and $15, %ecx
- movdqa (%edi), %xmm0
- pcmpeqd %xmm0, %xmm2
- pcmpeqd %xmm1, %xmm0
- pmovmskb %xmm2, %edx
- pmovmskb %xmm0, %eax
-
- sarl %cl, %edx
- sarl %cl, %eax
- test %eax, %eax
- jz L(unaligned_no_match)
-
- add %edi, %ecx
- POP (%edi)
-
- test %edx, %edx
- jz L(match_case1)
- test %al, %al
- jz L(match_higth_case2)
- test $15, %al
- jnz L(match_case2_4)
- test $15, %dl
- jnz L(return_null)
- lea 4(%ecx), %eax
- ret
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(unaligned_no_match):
- mov %edi, %ecx
- POP (%edi)
-
- test %edx, %edx
- jnz L(return_null)
-
- pxor %xmm2, %xmm2
-
-/* Loop start on aligned string. */
- .p2align 4
-L(loop):
- add $16, %ecx
- movdqa (%ecx), %xmm0
- pcmpeqd %xmm0, %xmm2
- pcmpeqd %xmm1, %xmm0
- pmovmskb %xmm2, %edx
- pmovmskb %xmm0, %eax
- or %eax, %edx
- jnz L(matches)
- add $16, %ecx
-
- movdqa (%ecx), %xmm0
- pcmpeqd %xmm0, %xmm2
- pcmpeqd %xmm1, %xmm0
- pmovmskb %xmm2, %edx
- pmovmskb %xmm0, %eax
- or %eax, %edx
- jnz L(matches)
- add $16, %ecx
-
- movdqa (%ecx), %xmm0
- pcmpeqd %xmm0, %xmm2
- pcmpeqd %xmm1, %xmm0
- pmovmskb %xmm2, %edx
- pmovmskb %xmm0, %eax
- or %eax, %edx
- jnz L(matches)
- add $16, %ecx
-
- movdqa (%ecx), %xmm0
- pcmpeqd %xmm0, %xmm2
- pcmpeqd %xmm1, %xmm0
- pmovmskb %xmm2, %edx
- pmovmskb %xmm0, %eax
- or %eax, %edx
- jz L(loop)
-
- .p2align 4
-L(matches):
- pmovmskb %xmm2, %edx
- test %eax, %eax
- jz L(return_null)
- test %edx, %edx
- jz L(match_case1)
-
- .p2align 4
-L(match_case2):
- test %al, %al
- jz L(match_higth_case2)
- test $15, %al
- jnz L(match_case2_4)
- test $15, %dl
- jnz L(return_null)
- lea 4(%ecx), %eax
- ret
-
- .p2align 4
-L(match_case2_4):
- mov %ecx, %eax
- ret
-
- .p2align 4
-L(match_higth_case2):
- test %dl, %dl
- jnz L(return_null)
- test $15, %ah
- jnz L(match_case2_12)
- test $15, %dh
- jnz L(return_null)
- lea 12(%ecx), %eax
- ret
-
- .p2align 4
-L(match_case2_12):
- lea 8(%ecx), %eax
- ret
-
- .p2align 4
-L(match_case1):
- test %al, %al
- jz L(match_higth_case1)
-
- test $0x01, %al
- jnz L(exit0)
- lea 4(%ecx), %eax
- ret
-
- .p2align 4
-L(match_higth_case1):
- test $0x01, %ah
- jnz L(exit3)
- lea 12(%ecx), %eax
- ret
-
- .p2align 4
-L(exit0):
- mov %ecx, %eax
- ret
-
- .p2align 4
-L(exit3):
- lea 8(%ecx), %eax
- ret
-
- .p2align 4
-L(return_null):
- xor %eax, %eax
- ret
-
-END (wcschr)
diff --git a/libc/arch-x86/string/sse2-wcscmp-atom.S b/libc/arch-x86/string/sse2-wcscmp-atom.S
deleted file mode 100644
index 8867d28..0000000
--- a/libc/arch-x86/string/sse2-wcscmp-atom.S
+++ /dev/null
@@ -1,1062 +0,0 @@
-/*
-Copyright (c) 2011 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef L
-# define L(label) .L##label
-#endif
-
-#ifndef cfi_startproc
-# define cfi_startproc .cfi_startproc
-#endif
-
-#ifndef cfi_endproc
-# define cfi_endproc .cfi_endproc
-#endif
-
-#ifndef cfi_rel_offset
-# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off
-#endif
-
-#ifndef cfi_restore
-# define cfi_restore(reg) .cfi_restore reg
-#endif
-
-#ifndef cfi_adjust_cfa_offset
-# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off
-#endif
-
-#ifndef ENTRY
-# define ENTRY(name) \
- .type name, @function; \
- .globl name; \
- .p2align 4; \
-name: \
- cfi_startproc
-#endif
-
-#ifndef END
-# define END(name) \
- cfi_endproc; \
- .size name, .-name
-#endif
-
-#define CFI_PUSH(REG) \
- cfi_adjust_cfa_offset (4); \
- cfi_rel_offset (REG, 0)
-
-#define CFI_POP(REG) \
- cfi_adjust_cfa_offset (-4); \
- cfi_restore (REG)
-
-#define PUSH(REG) pushl REG; CFI_PUSH (REG)
-#define POP(REG) popl REG; CFI_POP (REG)
-
-#define ENTRANCE PUSH(%esi); PUSH(%edi)
-#define RETURN POP(%edi); POP(%esi); ret; CFI_PUSH(%esi); CFI_PUSH(%edi);
-#define PARMS 4
-#define STR1 PARMS
-#define STR2 STR1+4
-
- .text
-ENTRY (wcscmp)
-/*
- * This implementation uses SSE to compare up to 16 bytes at a time.
-*/
- mov STR1(%esp), %edx
- mov STR2(%esp), %eax
-
- mov (%eax), %ecx
- cmp %ecx, (%edx)
- jne L(neq)
- test %ecx, %ecx
- jz L(eq)
-
- mov 4(%eax), %ecx
- cmp %ecx, 4(%edx)
- jne L(neq)
- test %ecx, %ecx
- jz L(eq)
-
- mov 8(%eax), %ecx
- cmp %ecx, 8(%edx)
- jne L(neq)
- test %ecx, %ecx
- jz L(eq)
-
- mov 12(%eax), %ecx
- cmp %ecx, 12(%edx)
- jne L(neq)
- test %ecx, %ecx
- jz L(eq)
-
- ENTRANCE
- add $16, %eax
- add $16, %edx
-
- mov %eax, %esi
- mov %edx, %edi
- pxor %xmm0, %xmm0 /* clear %xmm0 for null char checks */
- mov %al, %ch
- mov %dl, %cl
- and $63, %eax /* esi alignment in cache line */
- and $63, %edx /* edi alignment in cache line */
- and $15, %cl
- jz L(continue_00)
- cmp $16, %edx
- jb L(continue_0)
- cmp $32, %edx
- jb L(continue_16)
- cmp $48, %edx
- jb L(continue_32)
-
-L(continue_48):
- and $15, %ch
- jz L(continue_48_00)
- cmp $16, %eax
- jb L(continue_0_48)
- cmp $32, %eax
- jb L(continue_16_48)
- cmp $48, %eax
- jb L(continue_32_48)
-
- .p2align 4
-L(continue_48_48):
- mov (%esi), %ecx
- cmp %ecx, (%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 4(%esi), %ecx
- cmp %ecx, 4(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 8(%esi), %ecx
- cmp %ecx, 8(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 12(%esi), %ecx
- cmp %ecx, 12(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- movdqu 16(%edi), %xmm1
- movdqu 16(%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- movdqu 32(%edi), %xmm1
- movdqu 32(%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_32)
-
- movdqu 48(%edi), %xmm1
- movdqu 48(%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_48)
-
- add $64, %esi
- add $64, %edi
- jmp L(continue_48_48)
-
-L(continue_0):
- and $15, %ch
- jz L(continue_0_00)
- cmp $16, %eax
- jb L(continue_0_0)
- cmp $32, %eax
- jb L(continue_0_16)
- cmp $48, %eax
- jb L(continue_0_32)
-
- .p2align 4
-L(continue_0_48):
- mov (%esi), %ecx
- cmp %ecx, (%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 4(%esi), %ecx
- cmp %ecx, 4(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 8(%esi), %ecx
- cmp %ecx, 8(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 12(%esi), %ecx
- cmp %ecx, 12(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- movdqu 16(%edi), %xmm1
- movdqu 16(%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- movdqu 32(%edi), %xmm1
- movdqu 32(%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_32)
-
- mov 48(%esi), %ecx
- cmp %ecx, 48(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 52(%esi), %ecx
- cmp %ecx, 52(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 56(%esi), %ecx
- cmp %ecx, 56(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 60(%esi), %ecx
- cmp %ecx, 60(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- add $64, %esi
- add $64, %edi
- jmp L(continue_0_48)
-
- .p2align 4
-L(continue_00):
- and $15, %ch
- jz L(continue_00_00)
- cmp $16, %eax
- jb L(continue_00_0)
- cmp $32, %eax
- jb L(continue_00_16)
- cmp $48, %eax
- jb L(continue_00_32)
-
- .p2align 4
-L(continue_00_48):
- pcmpeqd (%edi), %xmm0
- mov (%edi), %eax
- pmovmskb %xmm0, %ecx
- test %ecx, %ecx
- jnz L(less4_double_words1)
-
- cmp (%esi), %eax
- jne L(nequal)
-
- mov 4(%edi), %eax
- cmp 4(%esi), %eax
- jne L(nequal)
-
- mov 8(%edi), %eax
- cmp 8(%esi), %eax
- jne L(nequal)
-
- mov 12(%edi), %eax
- cmp 12(%esi), %eax
- jne L(nequal)
-
- movdqu 16(%esi), %xmm2
- pcmpeqd %xmm2, %xmm0 /* Any null double_word? */
- pcmpeqd 16(%edi), %xmm2 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm2 /* packed sub of comparison results*/
- pmovmskb %xmm2, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- movdqu 32(%esi), %xmm2
- pcmpeqd %xmm2, %xmm0 /* Any null double_word? */
- pcmpeqd 32(%edi), %xmm2 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm2 /* packed sub of comparison results*/
- pmovmskb %xmm2, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_32)
-
- movdqu 48(%esi), %xmm2
- pcmpeqd %xmm2, %xmm0 /* Any null double_word? */
- pcmpeqd 48(%edi), %xmm2 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm2 /* packed sub of comparison results*/
- pmovmskb %xmm2, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_48)
-
- add $64, %esi
- add $64, %edi
- jmp L(continue_00_48)
-
- .p2align 4
-L(continue_32):
- and $15, %ch
- jz L(continue_32_00)
- cmp $16, %eax
- jb L(continue_0_32)
- cmp $32, %eax
- jb L(continue_16_32)
- cmp $48, %eax
- jb L(continue_32_32)
-
- .p2align 4
-L(continue_32_48):
- mov (%esi), %ecx
- cmp %ecx, (%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 4(%esi), %ecx
- cmp %ecx, 4(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 8(%esi), %ecx
- cmp %ecx, 8(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 12(%esi), %ecx
- cmp %ecx, 12(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 16(%esi), %ecx
- cmp %ecx, 16(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 20(%esi), %ecx
- cmp %ecx, 20(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 24(%esi), %ecx
- cmp %ecx, 24(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 28(%esi), %ecx
- cmp %ecx, 28(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- movdqu 32(%edi), %xmm1
- movdqu 32(%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_32)
-
- movdqu 48(%edi), %xmm1
- movdqu 48(%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_48)
-
- add $64, %esi
- add $64, %edi
- jmp L(continue_32_48)
-
- .p2align 4
-L(continue_16):
- and $15, %ch
- jz L(continue_16_00)
- cmp $16, %eax
- jb L(continue_0_16)
- cmp $32, %eax
- jb L(continue_16_16)
- cmp $48, %eax
- jb L(continue_16_32)
-
- .p2align 4
-L(continue_16_48):
- mov (%esi), %ecx
- cmp %ecx, (%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 4(%esi), %ecx
- cmp %ecx, 4(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 8(%esi), %ecx
- cmp %ecx, 8(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 12(%esi), %ecx
- cmp %ecx, 12(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- movdqu 16(%edi), %xmm1
- movdqu 16(%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- mov 32(%esi), %ecx
- cmp %ecx, 32(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 36(%esi), %ecx
- cmp %ecx, 36(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 40(%esi), %ecx
- cmp %ecx, 40(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 44(%esi), %ecx
- cmp %ecx, 44(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- movdqu 48(%edi), %xmm1
- movdqu 48(%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_48)
-
- add $64, %esi
- add $64, %edi
- jmp L(continue_16_48)
-
- .p2align 4
-L(continue_00_00):
- movdqa (%edi), %xmm1
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd (%esi), %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- movdqa 16(%edi), %xmm3
- pcmpeqd %xmm3, %xmm0 /* Any null double_word? */
- pcmpeqd 16(%esi), %xmm3 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm3 /* packed sub of comparison results*/
- pmovmskb %xmm3, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- movdqa 32(%edi), %xmm5
- pcmpeqd %xmm5, %xmm0 /* Any null double_word? */
- pcmpeqd 32(%esi), %xmm5 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm5 /* packed sub of comparison results*/
- pmovmskb %xmm5, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_32)
-
- movdqa 48(%edi), %xmm1
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd 48(%esi), %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_48)
-
- add $64, %esi
- add $64, %edi
- jmp L(continue_00_00)
-
- .p2align 4
-L(continue_00_32):
- movdqu (%esi), %xmm2
- pcmpeqd %xmm2, %xmm0 /* Any null double_word? */
- pcmpeqd (%edi), %xmm2 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm2 /* packed sub of comparison results*/
- pmovmskb %xmm2, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- add $16, %esi
- add $16, %edi
- jmp L(continue_00_48)
-
- .p2align 4
-L(continue_00_16):
- movdqu (%esi), %xmm2
- pcmpeqd %xmm2, %xmm0 /* Any null double_word? */
- pcmpeqd (%edi), %xmm2 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm2 /* packed sub of comparison results*/
- pmovmskb %xmm2, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- movdqu 16(%esi), %xmm2
- pcmpeqd %xmm2, %xmm0 /* Any null double_word? */
- pcmpeqd 16(%edi), %xmm2 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm2 /* packed sub of comparison results*/
- pmovmskb %xmm2, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- add $32, %esi
- add $32, %edi
- jmp L(continue_00_48)
-
- .p2align 4
-L(continue_00_0):
- movdqu (%esi), %xmm2
- pcmpeqd %xmm2, %xmm0 /* Any null double_word? */
- pcmpeqd (%edi), %xmm2 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm2 /* packed sub of comparison results*/
- pmovmskb %xmm2, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- movdqu 16(%esi), %xmm2
- pcmpeqd %xmm2, %xmm0 /* Any null double_word? */
- pcmpeqd 16(%edi), %xmm2 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm2 /* packed sub of comparison results*/
- pmovmskb %xmm2, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- movdqu 32(%esi), %xmm2
- pcmpeqd %xmm2, %xmm0 /* Any null double_word? */
- pcmpeqd 32(%edi), %xmm2 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm2 /* packed sub of comparison results*/
- pmovmskb %xmm2, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_32)
-
- add $48, %esi
- add $48, %edi
- jmp L(continue_00_48)
-
- .p2align 4
-L(continue_48_00):
- pcmpeqd (%esi), %xmm0
- mov (%edi), %eax
- pmovmskb %xmm0, %ecx
- test %ecx, %ecx
- jnz L(less4_double_words1)
-
- cmp (%esi), %eax
- jne L(nequal)
-
- mov 4(%edi), %eax
- cmp 4(%esi), %eax
- jne L(nequal)
-
- mov 8(%edi), %eax
- cmp 8(%esi), %eax
- jne L(nequal)
-
- mov 12(%edi), %eax
- cmp 12(%esi), %eax
- jne L(nequal)
-
- movdqu 16(%edi), %xmm1
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd 16(%esi), %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- movdqu 32(%edi), %xmm1
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd 32(%esi), %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_32)
-
- movdqu 48(%edi), %xmm1
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd 48(%esi), %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_48)
-
- add $64, %esi
- add $64, %edi
- jmp L(continue_48_00)
-
- .p2align 4
-L(continue_32_00):
- movdqu (%edi), %xmm1
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd (%esi), %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- add $16, %esi
- add $16, %edi
- jmp L(continue_48_00)
-
- .p2align 4
-L(continue_16_00):
- movdqu (%edi), %xmm1
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd (%esi), %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- movdqu 16(%edi), %xmm1
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd 16(%esi), %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- add $32, %esi
- add $32, %edi
- jmp L(continue_48_00)
-
- .p2align 4
-L(continue_0_00):
- movdqu (%edi), %xmm1
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd (%esi), %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- movdqu 16(%edi), %xmm1
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd 16(%esi), %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- movdqu 32(%edi), %xmm1
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd 32(%esi), %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_32)
-
- add $48, %esi
- add $48, %edi
- jmp L(continue_48_00)
-
- .p2align 4
-L(continue_32_32):
- movdqu (%edi), %xmm1
- movdqu (%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- add $16, %esi
- add $16, %edi
- jmp L(continue_48_48)
-
- .p2align 4
-L(continue_16_16):
- movdqu (%edi), %xmm1
- movdqu (%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- movdqu 16(%edi), %xmm3
- movdqu 16(%esi), %xmm4
- pcmpeqd %xmm3, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm4, %xmm3 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm3 /* packed sub of comparison results*/
- pmovmskb %xmm3, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- add $32, %esi
- add $32, %edi
- jmp L(continue_48_48)
-
- .p2align 4
-L(continue_0_0):
- movdqu (%edi), %xmm1
- movdqu (%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- movdqu 16(%edi), %xmm3
- movdqu 16(%esi), %xmm4
- pcmpeqd %xmm3, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm4, %xmm3 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm3 /* packed sub of comparison results*/
- pmovmskb %xmm3, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- movdqu 32(%edi), %xmm1
- movdqu 32(%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_32)
-
- add $48, %esi
- add $48, %edi
- jmp L(continue_48_48)
-
- .p2align 4
-L(continue_0_16):
- movdqu (%edi), %xmm1
- movdqu (%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- movdqu 16(%edi), %xmm1
- movdqu 16(%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words_16)
-
- add $32, %esi
- add $32, %edi
- jmp L(continue_32_48)
-
- .p2align 4
-L(continue_0_32):
- movdqu (%edi), %xmm1
- movdqu (%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- add $16, %esi
- add $16, %edi
- jmp L(continue_16_48)
-
- .p2align 4
-L(continue_16_32):
- movdqu (%edi), %xmm1
- movdqu (%esi), %xmm2
- pcmpeqd %xmm1, %xmm0 /* Any null double_word? */
- pcmpeqd %xmm2, %xmm1 /* compare first 4 double_words for equality */
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
- pmovmskb %xmm1, %edx
- sub $0xffff, %edx /* if first 4 double_words are same, edx == 0xffff */
- jnz L(less4_double_words)
-
- add $16, %esi
- add $16, %edi
- jmp L(continue_32_48)
-
- .p2align 4
-L(less4_double_words1):
- cmp (%esi), %eax
- jne L(nequal)
- test %eax, %eax
- jz L(equal)
-
- mov 4(%esi), %ecx
- cmp %ecx, 4(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 8(%esi), %ecx
- cmp %ecx, 8(%edi)
- jne L(nequal)
- test %ecx, %ecx
- jz L(equal)
-
- mov 12(%esi), %ecx
- cmp %ecx, 12(%edi)
- jne L(nequal)
- xor %eax, %eax
- RETURN
-
- .p2align 4
-L(less4_double_words):
- xor %eax, %eax
- test %dl, %dl
- jz L(next_two_double_words)
- and $15, %dl
- jz L(second_double_word)
- mov (%esi), %ecx
- cmp %ecx, (%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(second_double_word):
- mov 4(%esi), %ecx
- cmp %ecx, 4(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(next_two_double_words):
- and $15, %dh
- jz L(fourth_double_word)
- mov 8(%esi), %ecx
- cmp %ecx, 8(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(fourth_double_word):
- mov 12(%esi), %ecx
- cmp %ecx, 12(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(less4_double_words_16):
- xor %eax, %eax
- test %dl, %dl
- jz L(next_two_double_words_16)
- and $15, %dl
- jz L(second_double_word_16)
- mov 16(%esi), %ecx
- cmp %ecx, 16(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(second_double_word_16):
- mov 20(%esi), %ecx
- cmp %ecx, 20(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(next_two_double_words_16):
- and $15, %dh
- jz L(fourth_double_word_16)
- mov 24(%esi), %ecx
- cmp %ecx, 24(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(fourth_double_word_16):
- mov 28(%esi), %ecx
- cmp %ecx, 28(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(less4_double_words_32):
- xor %eax, %eax
- test %dl, %dl
- jz L(next_two_double_words_32)
- and $15, %dl
- jz L(second_double_word_32)
- mov 32(%esi), %ecx
- cmp %ecx, 32(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(second_double_word_32):
- mov 36(%esi), %ecx
- cmp %ecx, 36(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(next_two_double_words_32):
- and $15, %dh
- jz L(fourth_double_word_32)
- mov 40(%esi), %ecx
- cmp %ecx, 40(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(fourth_double_word_32):
- mov 44(%esi), %ecx
- cmp %ecx, 44(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(less4_double_words_48):
- xor %eax, %eax
- test %dl, %dl
- jz L(next_two_double_words_48)
- and $15, %dl
- jz L(second_double_word_48)
- mov 48(%esi), %ecx
- cmp %ecx, 48(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(second_double_word_48):
- mov 52(%esi), %ecx
- cmp %ecx, 52(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(next_two_double_words_48):
- and $15, %dh
- jz L(fourth_double_word_48)
- mov 56(%esi), %ecx
- cmp %ecx, 56(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(fourth_double_word_48):
- mov 60(%esi), %ecx
- cmp %ecx, 60(%edi)
- jne L(nequal)
- RETURN
-
- .p2align 4
-L(nequal):
- mov $1, %eax
- jg L(return)
- neg %eax
- RETURN
-
- .p2align 4
-L(return):
- RETURN
-
- .p2align 4
-L(equal):
- xorl %eax, %eax
- RETURN
-
- CFI_POP (%edi)
- CFI_POP (%esi)
-
- .p2align 4
-L(neq):
- mov $1, %eax
- jg L(neq_bigger)
- neg %eax
-
-L(neq_bigger):
- ret
-
- .p2align 4
-L(eq):
- xorl %eax, %eax
- ret
-
-END (wcscmp)
-
diff --git a/libc/arch-x86/string/sse2-wcslen-atom.S b/libc/arch-x86/string/sse2-wcslen-atom.S
deleted file mode 100644
index 2f10db4..0000000
--- a/libc/arch-x86/string/sse2-wcslen-atom.S
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
-Copyright (c) 2011 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef USE_AS_WCSCAT
-
-# ifndef L
-# define L(label) .L##label
-# endif
-
-# ifndef cfi_startproc
-# define cfi_startproc .cfi_startproc
-# endif
-
-# ifndef cfi_endproc
-# define cfi_endproc .cfi_endproc
-# endif
-
-# ifndef ENTRY
-# define ENTRY(name) \
- .type name, @function; \
- .globl name; \
- .p2align 4; \
-name: \
- cfi_startproc
-# endif
-
-# ifndef END
-# define END(name) \
- cfi_endproc; \
- .size name, .-name
-# endif
-
-# define PARMS 4
-# define STR PARMS
-# define RETURN ret
-
- .text
-ENTRY (wcslen)
- mov STR(%esp), %edx
-#endif
- cmpl $0, (%edx)
- jz L(exit_tail0)
- cmpl $0, 4(%edx)
- jz L(exit_tail1)
- cmpl $0, 8(%edx)
- jz L(exit_tail2)
- cmpl $0, 12(%edx)
- jz L(exit_tail3)
- cmpl $0, 16(%edx)
- jz L(exit_tail4)
- cmpl $0, 20(%edx)
- jz L(exit_tail5)
- cmpl $0, 24(%edx)
- jz L(exit_tail6)
- cmpl $0, 28(%edx)
- jz L(exit_tail7)
-
- pxor %xmm0, %xmm0
-
- lea 32(%edx), %eax
- lea -16(%eax), %ecx
- and $-16, %eax
-
- pcmpeqd (%eax), %xmm0
- pmovmskb %xmm0, %edx
- pxor %xmm1, %xmm1
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm1
- pmovmskb %xmm1, %edx
- pxor %xmm2, %xmm2
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm2
- pmovmskb %xmm2, %edx
- pxor %xmm3, %xmm3
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm3
- pmovmskb %xmm3, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm0
- pmovmskb %xmm0, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm1
- pmovmskb %xmm1, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm2
- pmovmskb %xmm2, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm3
- pmovmskb %xmm3, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm0
- pmovmskb %xmm0, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm1
- pmovmskb %xmm1, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm2
- pmovmskb %xmm2, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm3
- pmovmskb %xmm3, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm0
- pmovmskb %xmm0, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm1
- pmovmskb %xmm1, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm2
- pmovmskb %xmm2, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd (%eax), %xmm3
- pmovmskb %xmm3, %edx
- lea 16(%eax), %eax
- test %edx, %edx
- jnz L(exit)
-
- and $-0x40, %eax
-
- .p2align 4
-L(aligned_64_loop):
- movaps (%eax), %xmm0
- movaps 16(%eax), %xmm1
- movaps 32(%eax), %xmm2
- movaps 48(%eax), %xmm6
-
- pminub %xmm1, %xmm0
- pminub %xmm6, %xmm2
- pminub %xmm0, %xmm2
- pcmpeqd %xmm3, %xmm2
- pmovmskb %xmm2, %edx
- lea 64(%eax), %eax
- test %edx, %edx
- jz L(aligned_64_loop)
-
- pcmpeqd -64(%eax), %xmm3
- pmovmskb %xmm3, %edx
- lea 48(%ecx), %ecx
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd %xmm1, %xmm3
- pmovmskb %xmm3, %edx
- lea -16(%ecx), %ecx
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd -32(%eax), %xmm3
- pmovmskb %xmm3, %edx
- lea -16(%ecx), %ecx
- test %edx, %edx
- jnz L(exit)
-
- pcmpeqd %xmm6, %xmm3
- pmovmskb %xmm3, %edx
- lea -16(%ecx), %ecx
- test %edx, %edx
- jnz L(exit)
-
- jmp L(aligned_64_loop)
-
- .p2align 4
-L(exit):
- sub %ecx, %eax
- shr $2, %eax
- test %dl, %dl
- jz L(exit_high)
-
- mov %dl, %cl
- and $15, %cl
- jz L(exit_1)
- RETURN
-
- .p2align 4
-L(exit_high):
- mov %dh, %ch
- and $15, %ch
- jz L(exit_3)
- add $2, %eax
- RETURN
-
- .p2align 4
-L(exit_1):
- add $1, %eax
- RETURN
-
- .p2align 4
-L(exit_3):
- add $3, %eax
- RETURN
-
- .p2align 4
-L(exit_tail0):
- xor %eax, %eax
- RETURN
-
- .p2align 4
-L(exit_tail1):
- mov $1, %eax
- RETURN
-
- .p2align 4
-L(exit_tail2):
- mov $2, %eax
- RETURN
-
- .p2align 4
-L(exit_tail3):
- mov $3, %eax
- RETURN
-
- .p2align 4
-L(exit_tail4):
- mov $4, %eax
- RETURN
-
- .p2align 4
-L(exit_tail5):
- mov $5, %eax
- RETURN
-
- .p2align 4
-L(exit_tail6):
- mov $6, %eax
- RETURN
-
- .p2align 4
-L(exit_tail7):
- mov $7, %eax
-#ifndef USE_AS_WCSCAT
- RETURN
-
-END (wcslen)
-#endif
diff --git a/libc/arch-x86/string/sse2-wcsrchr-atom.S b/libc/arch-x86/string/sse2-wcsrchr-atom.S
deleted file mode 100644
index 1a55df2..0000000
--- a/libc/arch-x86/string/sse2-wcsrchr-atom.S
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
-Copyright (c) 2011 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef L
-# define L(label) .L##label
-#endif
-
-#ifndef cfi_startproc
-# define cfi_startproc .cfi_startproc
-#endif
-
-#ifndef cfi_endproc
-# define cfi_endproc .cfi_endproc
-#endif
-
-#ifndef cfi_rel_offset
-# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off
-#endif
-
-#ifndef cfi_restore
-# define cfi_restore(reg) .cfi_restore reg
-#endif
-
-#ifndef cfi_adjust_cfa_offset
-# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off
-#endif
-
-#ifndef ENTRY
-# define ENTRY(name) \
- .type name, @function; \
- .globl name; \
- .p2align 4; \
-name: \
- cfi_startproc
-#endif
-
-#ifndef END
-# define END(name) \
- cfi_endproc; \
- .size name, .-name
-#endif
-
-#define CFI_PUSH(REG) \
- cfi_adjust_cfa_offset (4); \
- cfi_rel_offset (REG, 0)
-
-#define CFI_POP(REG) \
- cfi_adjust_cfa_offset (-4); \
- cfi_restore (REG)
-
-#define PUSH(REG) pushl REG; CFI_PUSH (REG)
-#define POP(REG) popl REG; CFI_POP (REG)
-
-#define PARMS 8
-#define ENTRANCE PUSH(%edi);
-#define RETURN POP(%edi); ret; CFI_PUSH(%edi);
-
-#define STR1 PARMS
-#define STR2 STR1+4
-
- .text
-ENTRY (wcsrchr)
-
- ENTRANCE
- mov STR1(%esp), %ecx
- movd STR2(%esp), %xmm1
-
- mov %ecx, %edi
- punpckldq %xmm1, %xmm1
- pxor %xmm2, %xmm2
- punpckldq %xmm1, %xmm1
-
-/* ECX has OFFSET. */
- and $63, %ecx
- cmp $48, %ecx
- ja L(crosscache)
-
-/* unaligned string. */
- movdqu (%edi), %xmm0
- pcmpeqd %xmm0, %xmm2
- pcmpeqd %xmm1, %xmm0
-/* Find where NULL is. */
- pmovmskb %xmm2, %ecx
-/* Check if there is a match. */
- pmovmskb %xmm0, %eax
- add $16, %edi
-
- test %eax, %eax
- jnz L(unaligned_match1)
-
- test %ecx, %ecx
- jnz L(return_null)
-
- and $-16, %edi
-
- PUSH (%esi)
-
- xor %edx, %edx
- jmp L(loop)
-
- CFI_POP (%esi)
-
- .p2align 4
-L(unaligned_match1):
- test %ecx, %ecx
- jnz L(prolog_find_zero_1)
-
- PUSH (%esi)
-
-/* Save current match */
- mov %eax, %edx
- mov %edi, %esi
- and $-16, %edi
- jmp L(loop)
-
- CFI_POP (%esi)
-
- .p2align 4
-L(crosscache):
-/* Hancle unaligned string. */
- and $15, %ecx
- and $-16, %edi
- pxor %xmm3, %xmm3
- movdqa (%edi), %xmm0
- pcmpeqd %xmm0, %xmm3
- pcmpeqd %xmm1, %xmm0
-/* Find where NULL is. */
- pmovmskb %xmm3, %edx
-/* Check if there is a match. */
- pmovmskb %xmm0, %eax
-/* Remove the leading bytes. */
- shr %cl, %edx
- shr %cl, %eax
- add $16, %edi
-
- test %eax, %eax
- jnz L(unaligned_match)
-
- test %edx, %edx
- jnz L(return_null)
-
- PUSH (%esi)
-
- xor %edx, %edx
- jmp L(loop)
-
- CFI_POP (%esi)
-
- .p2align 4
-L(unaligned_match):
- test %edx, %edx
- jnz L(prolog_find_zero)
-
- PUSH (%esi)
-
- mov %eax, %edx
- lea (%edi, %ecx), %esi
-
-/* Loop start on aligned string. */
- .p2align 4
-L(loop):
- movdqa (%edi), %xmm0
- pcmpeqd %xmm0, %xmm2
- add $16, %edi
- pcmpeqd %xmm1, %xmm0
- pmovmskb %xmm2, %ecx
- pmovmskb %xmm0, %eax
- or %eax, %ecx
- jnz L(matches)
-
- movdqa (%edi), %xmm3
- pcmpeqd %xmm3, %xmm2
- add $16, %edi
- pcmpeqd %xmm1, %xmm3
- pmovmskb %xmm2, %ecx
- pmovmskb %xmm3, %eax
- or %eax, %ecx
- jnz L(matches)
-
- movdqa (%edi), %xmm4
- pcmpeqd %xmm4, %xmm2
- add $16, %edi
- pcmpeqd %xmm1, %xmm4
- pmovmskb %xmm2, %ecx
- pmovmskb %xmm4, %eax
- or %eax, %ecx
- jnz L(matches)
-
- movdqa (%edi), %xmm5
- pcmpeqd %xmm5, %xmm2
- add $16, %edi
- pcmpeqd %xmm1, %xmm5
- pmovmskb %xmm2, %ecx
- pmovmskb %xmm5, %eax
- or %eax, %ecx
- jz L(loop)
-
- .p2align 4
-L(matches):
- test %eax, %eax
- jnz L(match)
-L(return_value):
- test %edx, %edx
- jz L(return_null_1)
- mov %edx, %eax
- mov %esi, %edi
-
- POP (%esi)
-
- test %ah, %ah
- jnz L(match_third_or_fourth_wchar)
- test $15 << 4, %al
- jnz L(match_second_wchar)
- lea -16(%edi), %eax
- RETURN
-
- CFI_PUSH (%esi)
-
- .p2align 4
-L(return_null_1):
- POP (%esi)
-
- xor %eax, %eax
- RETURN
-
- CFI_PUSH (%esi)
-
- .p2align 4
-L(match):
- pmovmskb %xmm2, %ecx
- test %ecx, %ecx
- jnz L(find_zero)
-/* save match info */
- mov %eax, %edx
- mov %edi, %esi
- jmp L(loop)
-
- .p2align 4
-L(find_zero):
- test %cl, %cl
- jz L(find_zero_in_third_or_fourth_wchar)
- test $15, %cl
- jz L(find_zero_in_second_wchar)
- and $1, %eax
- jz L(return_value)
-
- POP (%esi)
-
- lea -16(%edi), %eax
- RETURN
-
- CFI_PUSH (%esi)
-
- .p2align 4
-L(find_zero_in_second_wchar):
- and $(1 << 5) - 1, %eax
- jz L(return_value)
-
- POP (%esi)
-
- test $15 << 4, %al
- jnz L(match_second_wchar)
- lea -16(%edi), %eax
- RETURN
-
- CFI_PUSH (%esi)
-
- .p2align 4
-L(find_zero_in_third_or_fourth_wchar):
- test $15, %ch
- jz L(find_zero_in_fourth_wchar)
- and $(1 << 9) - 1, %eax
- jz L(return_value)
-
- POP (%esi)
-
- test %ah, %ah
- jnz L(match_third_wchar)
- test $15 << 4, %al
- jnz L(match_second_wchar)
- lea -16(%edi), %eax
- RETURN
-
- CFI_PUSH (%esi)
-
- .p2align 4
-L(find_zero_in_fourth_wchar):
-
- POP (%esi)
-
- test %ah, %ah
- jnz L(match_third_or_fourth_wchar)
- test $15 << 4, %al
- jnz L(match_second_wchar)
- lea -16(%edi), %eax
- RETURN
-
- CFI_PUSH (%esi)
-
- .p2align 4
-L(match_second_wchar):
- lea -12(%edi), %eax
- RETURN
-
- .p2align 4
-L(match_third_or_fourth_wchar):
- test $15 << 4, %ah
- jnz L(match_fourth_wchar)
- lea -8(%edi), %eax
- RETURN
-
- .p2align 4
-L(match_third_wchar):
- lea -8(%edi), %eax
- RETURN
-
- .p2align 4
-L(match_fourth_wchar):
- lea -4(%edi), %eax
- RETURN
-
- .p2align 4
-L(return_null):
- xor %eax, %eax
- RETURN
-
- .p2align 4
-L(prolog_find_zero):
- add %ecx, %edi
- mov %edx, %ecx
-L(prolog_find_zero_1):
- test %cl, %cl
- jz L(prolog_find_zero_in_third_or_fourth_wchar)
- test $15, %cl
- jz L(prolog_find_zero_in_second_wchar)
- and $1, %eax
- jz L(return_null)
-
- lea -16(%edi), %eax
- RETURN
-
- .p2align 4
-L(prolog_find_zero_in_second_wchar):
- and $(1 << 5) - 1, %eax
- jz L(return_null)
-
- test $15 << 4, %al
- jnz L(match_second_wchar)
- lea -16(%edi), %eax
- RETURN
-
- .p2align 4
-L(prolog_find_zero_in_third_or_fourth_wchar):
- test $15, %ch
- jz L(prolog_find_zero_in_fourth_wchar)
- and $(1 << 9) - 1, %eax
- jz L(return_null)
-
- test %ah, %ah
- jnz L(match_third_wchar)
- test $15 << 4, %al
- jnz L(match_second_wchar)
- lea -16(%edi), %eax
- RETURN
-
- .p2align 4
-L(prolog_find_zero_in_fourth_wchar):
- test %ah, %ah
- jnz L(match_third_or_fourth_wchar)
- test $15 << 4, %al
- jnz L(match_second_wchar)
- lea -16(%edi), %eax
- RETURN
-
-END (wcsrchr)
diff --git a/libc/arch-x86/string/sse4-memcmp-slm.S b/libc/arch-x86/string/sse4-memcmp-slm.S
old mode 100755
new mode 100644
diff --git a/libc/arch-x86/string/sse4-wmemcmp-slm.S b/libc/arch-x86/string/sse4-wmemcmp-slm.S
deleted file mode 100755
index 2bf92f5..0000000
--- a/libc/arch-x86/string/sse4-wmemcmp-slm.S
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-Copyright (c) 2014, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#define USE_AS_WMEMCMP
-#define MEMCMP wmemcmp_sse4
-#include "sse4-memcmp-slm.S"
diff --git a/libc/arch-x86/string/ssse3-memcpy-atom.S b/libc/arch-x86/string/ssse3-memcpy-atom.S
deleted file mode 100644
index 83e1985..0000000
--- a/libc/arch-x86/string/ssse3-memcpy-atom.S
+++ /dev/null
@@ -1,3124 +0,0 @@
-/*
-Copyright (c) 2010, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#define FOR_ATOM
-
-#ifndef MEMCPY
-# define MEMCPY memcpy_atom
-#endif
-
-#ifndef L
-# define L(label) .L##label
-#endif
-
-#ifndef cfi_startproc
-# define cfi_startproc .cfi_startproc
-#endif
-
-#ifndef cfi_endproc
-# define cfi_endproc .cfi_endproc
-#endif
-
-#ifndef cfi_rel_offset
-# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off
-#endif
-
-#ifndef cfi_restore
-# define cfi_restore(reg) .cfi_restore reg
-#endif
-
-#ifndef cfi_adjust_cfa_offset
-# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off
-#endif
-
-#ifndef ENTRY
-# define ENTRY(name) \
- .type name, @function; \
- .globl name; \
- .p2align 4; \
-name: \
- cfi_startproc
-#endif
-
-#ifndef END
-# define END(name) \
- cfi_endproc; \
- .size name, .-name
-#endif
-
-#define DEST PARMS
-#define SRC DEST+4
-#define LEN SRC+4
-
-#define CFI_PUSH(REG) \
- cfi_adjust_cfa_offset (4); \
- cfi_rel_offset (REG, 0)
-
-#define CFI_POP(REG) \
- cfi_adjust_cfa_offset (-4); \
- cfi_restore (REG)
-
-#define PUSH(REG) pushl REG; CFI_PUSH (REG)
-#define POP(REG) popl REG; CFI_POP (REG)
-
-#if (defined SHARED || defined __PIC__)
-# define PARMS 8 /* Preserve EBX. */
-# define ENTRANCE PUSH (%ebx);
-# define RETURN_END POP (%ebx); ret
-# define RETURN RETURN_END; CFI_PUSH (%ebx)
-# define JMPTBL(I, B) I - B
-
-# define SETUP_PIC_REG(x) call __x86.get_pc_thunk.x
-
-/* Load an entry in a jump table into EBX and branch to it. TABLE is a
- jump table with relative offsets. INDEX is a register contains the
- index into the jump table. SCALE is the scale of INDEX. */
-
-# define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \
- /* We first load PC into EBX. */ \
- SETUP_PIC_REG(bx); \
- /* Get the address of the jump table. */ \
- addl $(TABLE - .), %ebx; \
- /* Get the entry and convert the relative offset to the \
- absolute address. */ \
- addl (%ebx, INDEX, SCALE), %ebx; \
- /* We loaded the jump table. Go. */ \
- jmp *%ebx
-#else
-
-# define PARMS 4
-# define ENTRANCE
-# define RETURN_END ret
-# define RETURN RETURN_END
-# define JMPTBL(I, B) I
-
-/* Branch to an entry in a jump table. TABLE is a jump table with
- absolute offsets. INDEX is a register contains the index into the
- jump table. SCALE is the scale of INDEX. */
-
-# define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \
- jmp *TABLE(, INDEX, SCALE)
-#endif
-
- .section .text.ssse3,"ax",@progbits
-ENTRY (MEMCPY)
- ENTRANCE
- movl LEN(%esp), %ecx
- movl SRC(%esp), %eax
- movl DEST(%esp), %edx
-
-#ifdef USE_AS_MEMMOVE
- cmp %eax, %edx
- jb L(copy_forward)
- je L(fwd_write_0bytes)
- cmp $32, %ecx
- jae L(memmove_bwd)
- jmp L(bk_write_less32bytes_2)
-
- .p2align 4
-L(memmove_bwd):
- add %ecx, %eax
- cmp %eax, %edx
- movl SRC(%esp), %eax
- jb L(copy_backward)
-
-L(copy_forward):
-#endif
- cmp $48, %ecx
- jae L(48bytesormore)
-
-L(fwd_write_less32bytes):
-#ifndef USE_AS_MEMMOVE
- cmp %dl, %al
- jb L(bk_write)
-#endif
- add %ecx, %edx
- add %ecx, %eax
- BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
-#ifndef USE_AS_MEMMOVE
- .p2align 4
-L(bk_write):
- BRANCH_TO_JMPTBL_ENTRY (L(table_48_bytes_bwd), %ecx, 4)
-#endif
-
- .p2align 4
-L(48bytesormore):
-#ifndef USE_AS_MEMMOVE
- movlpd (%eax), %xmm0
- movlpd 8(%eax), %xmm1
- movlpd %xmm0, (%edx)
- movlpd %xmm1, 8(%edx)
-#else
- movdqu (%eax), %xmm0
-#endif
- PUSH (%edi)
- movl %edx, %edi
- and $-16, %edx
- add $16, %edx
- sub %edx, %edi
- add %edi, %ecx
- sub %edi, %eax
-
-#ifdef SHARED_CACHE_SIZE_HALF
- cmp $SHARED_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_shared_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_shared_cache_size_half, %ecx
-# endif
-#endif
-
- mov %eax, %edi
- jae L(large_page)
- and $0xf, %edi
- jz L(shl_0)
- BRANCH_TO_JMPTBL_ENTRY (L(shl_table), %edi, 4)
-
- .p2align 4
-L(shl_0):
-#ifdef USE_AS_MEMMOVE
- movl DEST+4(%esp), %edi
- movdqu %xmm0, (%edi)
-#endif
- xor %edi, %edi
- cmp $127, %ecx
- ja L(shl_0_gobble)
- lea -32(%ecx), %ecx
-
- .p2align 4
-L(shl_0_loop):
- movdqa (%eax, %edi), %xmm0
- movdqa 16(%eax, %edi), %xmm1
- sub $32, %ecx
- movdqa %xmm0, (%edx, %edi)
- movdqa %xmm1, 16(%edx, %edi)
- lea 32(%edi), %edi
- jb L(shl_0_end)
-
- movdqa (%eax, %edi), %xmm0
- movdqa 16(%eax, %edi), %xmm1
- sub $32, %ecx
- movdqa %xmm0, (%edx, %edi)
- movdqa %xmm1, 16(%edx, %edi)
- lea 32(%edi), %edi
- jb L(shl_0_end)
-
- movdqa (%eax, %edi), %xmm0
- movdqa 16(%eax, %edi), %xmm1
- sub $32, %ecx
- movdqa %xmm0, (%edx, %edi)
- movdqa %xmm1, 16(%edx, %edi)
- lea 32(%edi), %edi
- jb L(shl_0_end)
-
- movdqa (%eax, %edi), %xmm0
- movdqa 16(%eax, %edi), %xmm1
- sub $32, %ecx
- movdqa %xmm0, (%edx, %edi)
- movdqa %xmm1, 16(%edx, %edi)
- lea 32(%edi), %edi
-
-L(shl_0_end):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- add %edi, %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd_align), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_0_gobble):
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- POP (%edi)
- lea -128(%ecx), %ecx
- jae L(shl_0_gobble_mem_loop)
-
- .p2align 4
-L(shl_0_gobble_cache_loop):
- movdqa (%eax), %xmm0
- movdqa 0x10(%eax), %xmm1
- movdqa 0x20(%eax), %xmm2
- movdqa 0x30(%eax), %xmm3
- movdqa 0x40(%eax), %xmm4
- movdqa 0x50(%eax), %xmm5
- movdqa 0x60(%eax), %xmm6
- movdqa 0x70(%eax), %xmm7
- lea 0x80(%eax), %eax
- sub $128, %ecx
- movdqa %xmm0, (%edx)
- movdqa %xmm1, 0x10(%edx)
- movdqa %xmm2, 0x20(%edx)
- movdqa %xmm3, 0x30(%edx)
- movdqa %xmm4, 0x40(%edx)
- movdqa %xmm5, 0x50(%edx)
- movdqa %xmm6, 0x60(%edx)
- movdqa %xmm7, 0x70(%edx)
- lea 0x80(%edx), %edx
-
- jae L(shl_0_gobble_cache_loop)
- cmp $-0x40, %ecx
- lea 0x80(%ecx), %ecx
- jl L(shl_0_cache_less_64bytes)
-
- movdqa (%eax), %xmm0
- sub $0x40, %ecx
- movdqa 0x10(%eax), %xmm1
- movdqa %xmm0, (%edx)
- movdqa %xmm1, 0x10(%edx)
- movdqa 0x20(%eax), %xmm0
- movdqa 0x30(%eax), %xmm1
- add $0x40, %eax
- movdqa %xmm0, 0x20(%edx)
- movdqa %xmm1, 0x30(%edx)
- add $0x40, %edx
-
-L(shl_0_cache_less_64bytes):
- cmp $0x20, %ecx
- jb L(shl_0_cache_less_32bytes)
- movdqa (%eax), %xmm0
- sub $0x20, %ecx
- movdqa 0x10(%eax), %xmm1
- add $0x20, %eax
- movdqa %xmm0, (%edx)
- movdqa %xmm1, 0x10(%edx)
- add $0x20, %edx
-
-L(shl_0_cache_less_32bytes):
- cmp $0x10, %ecx
- jb L(shl_0_cache_less_16bytes)
- sub $0x10, %ecx
- movdqa (%eax), %xmm0
- add $0x10, %eax
- movdqa %xmm0, (%edx)
- add $0x10, %edx
-
-L(shl_0_cache_less_16bytes):
- add %ecx, %edx
- add %ecx, %eax
- BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
-
- .p2align 4
-L(shl_0_gobble_mem_loop):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x280(%eax)
- prefetcht0 0x1c0(%edx)
-
- movdqa (%eax), %xmm0
- movdqa 0x10(%eax), %xmm1
- movdqa 0x20(%eax), %xmm2
- movdqa 0x30(%eax), %xmm3
- movdqa 0x40(%eax), %xmm4
- movdqa 0x50(%eax), %xmm5
- movdqa 0x60(%eax), %xmm6
- movdqa 0x70(%eax), %xmm7
- lea 0x80(%eax), %eax
- sub $0x80, %ecx
- movdqa %xmm0, (%edx)
- movdqa %xmm1, 0x10(%edx)
- movdqa %xmm2, 0x20(%edx)
- movdqa %xmm3, 0x30(%edx)
- movdqa %xmm4, 0x40(%edx)
- movdqa %xmm5, 0x50(%edx)
- movdqa %xmm6, 0x60(%edx)
- movdqa %xmm7, 0x70(%edx)
- lea 0x80(%edx), %edx
-
- jae L(shl_0_gobble_mem_loop)
- cmp $-0x40, %ecx
- lea 0x80(%ecx), %ecx
- jl L(shl_0_mem_less_64bytes)
-
- movdqa (%eax), %xmm0
- sub $0x40, %ecx
- movdqa 0x10(%eax), %xmm1
-
- movdqa %xmm0, (%edx)
- movdqa %xmm1, 0x10(%edx)
-
- movdqa 0x20(%eax), %xmm0
- movdqa 0x30(%eax), %xmm1
- add $0x40, %eax
-
- movdqa %xmm0, 0x20(%edx)
- movdqa %xmm1, 0x30(%edx)
- add $0x40, %edx
-
-L(shl_0_mem_less_64bytes):
- cmp $0x20, %ecx
- jb L(shl_0_mem_less_32bytes)
- movdqa (%eax), %xmm0
- sub $0x20, %ecx
- movdqa 0x10(%eax), %xmm1
- add $0x20, %eax
- movdqa %xmm0, (%edx)
- movdqa %xmm1, 0x10(%edx)
- add $0x20, %edx
-
-L(shl_0_mem_less_32bytes):
- cmp $0x10, %ecx
- jb L(shl_0_mem_less_16bytes)
- sub $0x10, %ecx
- movdqa (%eax), %xmm0
- add $0x10, %eax
- movdqa %xmm0, (%edx)
- add $0x10, %edx
-
-L(shl_0_mem_less_16bytes):
- add %ecx, %edx
- add %ecx, %eax
- BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd_align), %ecx, 4)
-
- .p2align 4
-L(shl_1):
-#ifndef USE_AS_MEMMOVE
- movaps -1(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -1(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_1_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl1LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 15(%eax), %xmm2
- movaps 31(%eax), %xmm3
- movaps 47(%eax), %xmm4
- movaps 63(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $1, %xmm4, %xmm5
- palignr $1, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $1, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $1, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl1LoopStart)
-
-L(Shl1LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 15(%eax), %xmm2
- movaps 31(%eax), %xmm3
- palignr $1, %xmm2, %xmm3
- palignr $1, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_1_no_prefetch):
- lea -32(%ecx), %ecx
- lea -1(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_1_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $1, %xmm2, %xmm3
- palignr $1, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jb L(sh_1_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $1, %xmm2, %xmm3
- palignr $1, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jae L(sh_1_no_prefetch_loop)
-
-L(sh_1_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 1(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_2):
-#ifndef USE_AS_MEMMOVE
- movaps -2(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -2(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_2_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl2LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 14(%eax), %xmm2
- movaps 30(%eax), %xmm3
- movaps 46(%eax), %xmm4
- movaps 62(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $2, %xmm4, %xmm5
- palignr $2, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $2, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $2, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl2LoopStart)
-
-L(Shl2LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 14(%eax), %xmm2
- movaps 30(%eax), %xmm3
- palignr $2, %xmm2, %xmm3
- palignr $2, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_2_no_prefetch):
- lea -32(%ecx), %ecx
- lea -2(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_2_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $2, %xmm2, %xmm3
- palignr $2, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jb L(sh_2_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $2, %xmm2, %xmm3
- palignr $2, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jae L(sh_2_no_prefetch_loop)
-
-L(sh_2_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 2(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_3):
-#ifndef USE_AS_MEMMOVE
- movaps -3(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -3(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_3_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl3LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 13(%eax), %xmm2
- movaps 29(%eax), %xmm3
- movaps 45(%eax), %xmm4
- movaps 61(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $3, %xmm4, %xmm5
- palignr $3, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $3, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $3, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl3LoopStart)
-
-L(Shl3LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 13(%eax), %xmm2
- movaps 29(%eax), %xmm3
- palignr $3, %xmm2, %xmm3
- palignr $3, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_3_no_prefetch):
- lea -32(%ecx), %ecx
- lea -3(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_3_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $3, %xmm2, %xmm3
- palignr $3, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
-
- jb L(sh_3_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $3, %xmm2, %xmm3
- palignr $3, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
-
- jae L(sh_3_no_prefetch_loop)
-
-L(sh_3_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 3(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_4):
-#ifndef USE_AS_MEMMOVE
- movaps -4(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -4(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_4_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl4LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 12(%eax), %xmm2
- movaps 28(%eax), %xmm3
- movaps 44(%eax), %xmm4
- movaps 60(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $4, %xmm4, %xmm5
- palignr $4, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $4, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $4, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl4LoopStart)
-
-L(Shl4LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 12(%eax), %xmm2
- movaps 28(%eax), %xmm3
- palignr $4, %xmm2, %xmm3
- palignr $4, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_4_no_prefetch):
- lea -32(%ecx), %ecx
- lea -4(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_4_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $4, %xmm2, %xmm3
- palignr $4, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
-
- jb L(sh_4_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $4, %xmm2, %xmm3
- palignr $4, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
-
- jae L(sh_4_no_prefetch_loop)
-
-L(sh_4_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 4(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_5):
-#ifndef USE_AS_MEMMOVE
- movaps -5(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -5(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_5_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl5LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 11(%eax), %xmm2
- movaps 27(%eax), %xmm3
- movaps 43(%eax), %xmm4
- movaps 59(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $5, %xmm4, %xmm5
- palignr $5, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $5, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $5, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl5LoopStart)
-
-L(Shl5LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 11(%eax), %xmm2
- movaps 27(%eax), %xmm3
- palignr $5, %xmm2, %xmm3
- palignr $5, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_5_no_prefetch):
- lea -32(%ecx), %ecx
- lea -5(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_5_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $5, %xmm2, %xmm3
- palignr $5, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
-
- jb L(sh_5_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $5, %xmm2, %xmm3
- palignr $5, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
-
- jae L(sh_5_no_prefetch_loop)
-
-L(sh_5_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 5(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_6):
-#ifndef USE_AS_MEMMOVE
- movaps -6(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -6(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_6_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl6LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 10(%eax), %xmm2
- movaps 26(%eax), %xmm3
- movaps 42(%eax), %xmm4
- movaps 58(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $6, %xmm4, %xmm5
- palignr $6, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $6, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $6, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl6LoopStart)
-
-L(Shl6LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 10(%eax), %xmm2
- movaps 26(%eax), %xmm3
- palignr $6, %xmm2, %xmm3
- palignr $6, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_6_no_prefetch):
- lea -32(%ecx), %ecx
- lea -6(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_6_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $6, %xmm2, %xmm3
- palignr $6, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
-
- jb L(sh_6_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $6, %xmm2, %xmm3
- palignr $6, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
-
- jae L(sh_6_no_prefetch_loop)
-
-L(sh_6_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 6(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_7):
-#ifndef USE_AS_MEMMOVE
- movaps -7(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -7(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_7_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl7LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 9(%eax), %xmm2
- movaps 25(%eax), %xmm3
- movaps 41(%eax), %xmm4
- movaps 57(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $7, %xmm4, %xmm5
- palignr $7, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $7, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $7, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl7LoopStart)
-
-L(Shl7LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 9(%eax), %xmm2
- movaps 25(%eax), %xmm3
- palignr $7, %xmm2, %xmm3
- palignr $7, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_7_no_prefetch):
- lea -32(%ecx), %ecx
- lea -7(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_7_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $7, %xmm2, %xmm3
- palignr $7, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jb L(sh_7_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $7, %xmm2, %xmm3
- palignr $7, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jae L(sh_7_no_prefetch_loop)
-
-L(sh_7_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 7(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_8):
-#ifndef USE_AS_MEMMOVE
- movaps -8(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -8(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_8_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl8LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 8(%eax), %xmm2
- movaps 24(%eax), %xmm3
- movaps 40(%eax), %xmm4
- movaps 56(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $8, %xmm4, %xmm5
- palignr $8, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $8, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $8, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl8LoopStart)
-
-L(LoopLeave8):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 8(%eax), %xmm2
- movaps 24(%eax), %xmm3
- palignr $8, %xmm2, %xmm3
- palignr $8, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_8_no_prefetch):
- lea -32(%ecx), %ecx
- lea -8(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_8_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $8, %xmm2, %xmm3
- palignr $8, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jb L(sh_8_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $8, %xmm2, %xmm3
- palignr $8, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jae L(sh_8_no_prefetch_loop)
-
-L(sh_8_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 8(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_9):
-#ifndef USE_AS_MEMMOVE
- movaps -9(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -9(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_9_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl9LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 7(%eax), %xmm2
- movaps 23(%eax), %xmm3
- movaps 39(%eax), %xmm4
- movaps 55(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $9, %xmm4, %xmm5
- palignr $9, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $9, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $9, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl9LoopStart)
-
-L(Shl9LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 7(%eax), %xmm2
- movaps 23(%eax), %xmm3
- palignr $9, %xmm2, %xmm3
- palignr $9, %xmm1, %xmm2
-
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_9_no_prefetch):
- lea -32(%ecx), %ecx
- lea -9(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_9_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $9, %xmm2, %xmm3
- palignr $9, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jb L(sh_9_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $9, %xmm2, %xmm3
- palignr $9, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jae L(sh_9_no_prefetch_loop)
-
-L(sh_9_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 9(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_10):
-#ifndef USE_AS_MEMMOVE
- movaps -10(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -10(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_10_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl10LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 6(%eax), %xmm2
- movaps 22(%eax), %xmm3
- movaps 38(%eax), %xmm4
- movaps 54(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $10, %xmm4, %xmm5
- palignr $10, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $10, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $10, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl10LoopStart)
-
-L(Shl10LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 6(%eax), %xmm2
- movaps 22(%eax), %xmm3
- palignr $10, %xmm2, %xmm3
- palignr $10, %xmm1, %xmm2
-
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_10_no_prefetch):
- lea -32(%ecx), %ecx
- lea -10(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_10_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $10, %xmm2, %xmm3
- palignr $10, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jb L(sh_10_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $10, %xmm2, %xmm3
- palignr $10, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jae L(sh_10_no_prefetch_loop)
-
-L(sh_10_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 10(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_11):
-#ifndef USE_AS_MEMMOVE
- movaps -11(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -11(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_11_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl11LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 5(%eax), %xmm2
- movaps 21(%eax), %xmm3
- movaps 37(%eax), %xmm4
- movaps 53(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $11, %xmm4, %xmm5
- palignr $11, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $11, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $11, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl11LoopStart)
-
-L(Shl11LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 5(%eax), %xmm2
- movaps 21(%eax), %xmm3
- palignr $11, %xmm2, %xmm3
- palignr $11, %xmm1, %xmm2
-
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_11_no_prefetch):
- lea -32(%ecx), %ecx
- lea -11(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_11_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $11, %xmm2, %xmm3
- palignr $11, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jb L(sh_11_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $11, %xmm2, %xmm3
- palignr $11, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jae L(sh_11_no_prefetch_loop)
-
-L(sh_11_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 11(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_12):
-#ifndef USE_AS_MEMMOVE
- movaps -12(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -12(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_12_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl12LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 4(%eax), %xmm2
- movaps 20(%eax), %xmm3
- movaps 36(%eax), %xmm4
- movaps 52(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $12, %xmm4, %xmm5
- palignr $12, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $12, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $12, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl12LoopStart)
-
-L(Shl12LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 4(%eax), %xmm2
- movaps 20(%eax), %xmm3
- palignr $12, %xmm2, %xmm3
- palignr $12, %xmm1, %xmm2
-
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_12_no_prefetch):
- lea -32(%ecx), %ecx
- lea -12(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_12_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $12, %xmm2, %xmm3
- palignr $12, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jb L(sh_12_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $12, %xmm2, %xmm3
- palignr $12, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jae L(sh_12_no_prefetch_loop)
-
-L(sh_12_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 12(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_13):
-#ifndef USE_AS_MEMMOVE
- movaps -13(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -13(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_13_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl13LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 3(%eax), %xmm2
- movaps 19(%eax), %xmm3
- movaps 35(%eax), %xmm4
- movaps 51(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $13, %xmm4, %xmm5
- palignr $13, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $13, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $13, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl13LoopStart)
-
-L(Shl13LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 3(%eax), %xmm2
- movaps 19(%eax), %xmm3
- palignr $13, %xmm2, %xmm3
- palignr $13, %xmm1, %xmm2
-
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_13_no_prefetch):
- lea -32(%ecx), %ecx
- lea -13(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_13_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $13, %xmm2, %xmm3
- palignr $13, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jb L(sh_13_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $13, %xmm2, %xmm3
- palignr $13, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jae L(sh_13_no_prefetch_loop)
-
-L(sh_13_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 13(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_14):
-#ifndef USE_AS_MEMMOVE
- movaps -14(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -14(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_14_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl14LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 2(%eax), %xmm2
- movaps 18(%eax), %xmm3
- movaps 34(%eax), %xmm4
- movaps 50(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $14, %xmm4, %xmm5
- palignr $14, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $14, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $14, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl14LoopStart)
-
-L(Shl14LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 2(%eax), %xmm2
- movaps 18(%eax), %xmm3
- palignr $14, %xmm2, %xmm3
- palignr $14, %xmm1, %xmm2
-
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_14_no_prefetch):
- lea -32(%ecx), %ecx
- lea -14(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_14_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $14, %xmm2, %xmm3
- palignr $14, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jb L(sh_14_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $14, %xmm2, %xmm3
- palignr $14, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jae L(sh_14_no_prefetch_loop)
-
-L(sh_14_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 14(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_15):
-#ifndef USE_AS_MEMMOVE
- movaps -15(%eax), %xmm1
-#else
- movl DEST+4(%esp), %edi
- movaps -15(%eax), %xmm1
- movdqu %xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
- cmp $DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
- SETUP_PIC_REG(bx)
- add $_GLOBAL_OFFSET_TABLE_, %ebx
- cmp __x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
- cmp __x86_data_cache_size_half, %ecx
-# endif
-#endif
- jb L(sh_15_no_prefetch)
-
- lea -64(%ecx), %ecx
-
- .p2align 4
-L(Shl15LoopStart):
- prefetcht0 0x1c0(%eax)
- prefetcht0 0x1c0(%edx)
- movaps 1(%eax), %xmm2
- movaps 17(%eax), %xmm3
- movaps 33(%eax), %xmm4
- movaps 49(%eax), %xmm5
- movaps %xmm5, %xmm7
- palignr $15, %xmm4, %xmm5
- palignr $15, %xmm3, %xmm4
- movaps %xmm5, 48(%edx)
- palignr $15, %xmm2, %xmm3
- lea 64(%eax), %eax
- palignr $15, %xmm1, %xmm2
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm7, %xmm1
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- sub $64, %ecx
- ja L(Shl15LoopStart)
-
-L(Shl15LoopLeave):
- add $32, %ecx
- jle L(shl_end_0)
-
- movaps 1(%eax), %xmm2
- movaps 17(%eax), %xmm3
- palignr $15, %xmm2, %xmm3
- palignr $15, %xmm1, %xmm2
-
- movaps %xmm2, (%edx)
- movaps %xmm3, 16(%edx)
- lea 32(%edx, %ecx), %edx
- lea 32(%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(sh_15_no_prefetch):
- lea -32(%ecx), %ecx
- lea -15(%eax), %eax
- xor %edi, %edi
-
- .p2align 4
-L(sh_15_no_prefetch_loop):
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm4
- palignr $15, %xmm2, %xmm3
- palignr $15, %xmm1, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jb L(sh_15_end_no_prefetch_loop)
-
- movdqa 16(%eax, %edi), %xmm2
- sub $32, %ecx
- movdqa 32(%eax, %edi), %xmm3
- movdqa %xmm3, %xmm1
- palignr $15, %xmm2, %xmm3
- palignr $15, %xmm4, %xmm2
- lea 32(%edi), %edi
- movdqa %xmm2, -32(%edx, %edi)
- movdqa %xmm3, -16(%edx, %edi)
- jae L(sh_15_no_prefetch_loop)
-
-L(sh_15_end_no_prefetch_loop):
- lea 32(%ecx), %ecx
- add %ecx, %edi
- add %edi, %edx
- lea 15(%edi, %eax), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(shl_end_0):
- lea 32(%ecx), %ecx
- lea (%edx, %ecx), %edx
- lea (%eax, %ecx), %eax
- POP (%edi)
- BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
- .p2align 4
-L(fwd_write_44bytes):
- movq -44(%eax), %xmm0
- movq %xmm0, -44(%edx)
-L(fwd_write_36bytes):
- movq -36(%eax), %xmm0
- movq %xmm0, -36(%edx)
-L(fwd_write_28bytes):
- movq -28(%eax), %xmm0
- movq %xmm0, -28(%edx)
-L(fwd_write_20bytes):
- movq -20(%eax), %xmm0
- movq %xmm0, -20(%edx)
-L(fwd_write_12bytes):
- movq -12(%eax), %xmm0
- movq %xmm0, -12(%edx)
-L(fwd_write_4bytes):
- movl -4(%eax), %ecx
- movl %ecx, -4(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_40bytes):
- movq -40(%eax), %xmm0
- movq %xmm0, -40(%edx)
-L(fwd_write_32bytes):
- movq -32(%eax), %xmm0
- movq %xmm0, -32(%edx)
-L(fwd_write_24bytes):
- movq -24(%eax), %xmm0
- movq %xmm0, -24(%edx)
-L(fwd_write_16bytes):
- movq -16(%eax), %xmm0
- movq %xmm0, -16(%edx)
-L(fwd_write_8bytes):
- movq -8(%eax), %xmm0
- movq %xmm0, -8(%edx)
-L(fwd_write_0bytes):
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_5bytes):
- movl -5(%eax), %ecx
- movl -4(%eax), %eax
- movl %ecx, -5(%edx)
- movl %eax, -4(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_45bytes):
- movq -45(%eax), %xmm0
- movq %xmm0, -45(%edx)
-L(fwd_write_37bytes):
- movq -37(%eax), %xmm0
- movq %xmm0, -37(%edx)
-L(fwd_write_29bytes):
- movq -29(%eax), %xmm0
- movq %xmm0, -29(%edx)
-L(fwd_write_21bytes):
- movq -21(%eax), %xmm0
- movq %xmm0, -21(%edx)
-L(fwd_write_13bytes):
- movq -13(%eax), %xmm0
- movq %xmm0, -13(%edx)
- movl -5(%eax), %ecx
- movl %ecx, -5(%edx)
- movzbl -1(%eax), %ecx
- movb %cl, -1(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_41bytes):
- movq -41(%eax), %xmm0
- movq %xmm0, -41(%edx)
-L(fwd_write_33bytes):
- movq -33(%eax), %xmm0
- movq %xmm0, -33(%edx)
-L(fwd_write_25bytes):
- movq -25(%eax), %xmm0
- movq %xmm0, -25(%edx)
-L(fwd_write_17bytes):
- movq -17(%eax), %xmm0
- movq %xmm0, -17(%edx)
-L(fwd_write_9bytes):
- movq -9(%eax), %xmm0
- movq %xmm0, -9(%edx)
-L(fwd_write_1bytes):
- movzbl -1(%eax), %ecx
- movb %cl, -1(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_46bytes):
- movq -46(%eax), %xmm0
- movq %xmm0, -46(%edx)
-L(fwd_write_38bytes):
- movq -38(%eax), %xmm0
- movq %xmm0, -38(%edx)
-L(fwd_write_30bytes):
- movq -30(%eax), %xmm0
- movq %xmm0, -30(%edx)
-L(fwd_write_22bytes):
- movq -22(%eax), %xmm0
- movq %xmm0, -22(%edx)
-L(fwd_write_14bytes):
- movq -14(%eax), %xmm0
- movq %xmm0, -14(%edx)
-L(fwd_write_6bytes):
- movl -6(%eax), %ecx
- movl %ecx, -6(%edx)
- movzwl -2(%eax), %ecx
- movw %cx, -2(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_42bytes):
- movq -42(%eax), %xmm0
- movq %xmm0, -42(%edx)
-L(fwd_write_34bytes):
- movq -34(%eax), %xmm0
- movq %xmm0, -34(%edx)
-L(fwd_write_26bytes):
- movq -26(%eax), %xmm0
- movq %xmm0, -26(%edx)
-L(fwd_write_18bytes):
- movq -18(%eax), %xmm0
- movq %xmm0, -18(%edx)
-L(fwd_write_10bytes):
- movq -10(%eax), %xmm0
- movq %xmm0, -10(%edx)
-L(fwd_write_2bytes):
- movzwl -2(%eax), %ecx
- movw %cx, -2(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_47bytes):
- movq -47(%eax), %xmm0
- movq %xmm0, -47(%edx)
-L(fwd_write_39bytes):
- movq -39(%eax), %xmm0
- movq %xmm0, -39(%edx)
-L(fwd_write_31bytes):
- movq -31(%eax), %xmm0
- movq %xmm0, -31(%edx)
-L(fwd_write_23bytes):
- movq -23(%eax), %xmm0
- movq %xmm0, -23(%edx)
-L(fwd_write_15bytes):
- movq -15(%eax), %xmm0
- movq %xmm0, -15(%edx)
-L(fwd_write_7bytes):
- movl -7(%eax), %ecx
- movl %ecx, -7(%edx)
- movzwl -3(%eax), %ecx
- movzbl -1(%eax), %eax
- movw %cx, -3(%edx)
- movb %al, -1(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_43bytes):
- movq -43(%eax), %xmm0
- movq %xmm0, -43(%edx)
-L(fwd_write_35bytes):
- movq -35(%eax), %xmm0
- movq %xmm0, -35(%edx)
-L(fwd_write_27bytes):
- movq -27(%eax), %xmm0
- movq %xmm0, -27(%edx)
-L(fwd_write_19bytes):
- movq -19(%eax), %xmm0
- movq %xmm0, -19(%edx)
-L(fwd_write_11bytes):
- movq -11(%eax), %xmm0
- movq %xmm0, -11(%edx)
-L(fwd_write_3bytes):
- movzwl -3(%eax), %ecx
- movzbl -1(%eax), %eax
- movw %cx, -3(%edx)
- movb %al, -1(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_40bytes_align):
- movdqa -40(%eax), %xmm0
- movdqa %xmm0, -40(%edx)
-L(fwd_write_24bytes_align):
- movdqa -24(%eax), %xmm0
- movdqa %xmm0, -24(%edx)
-L(fwd_write_8bytes_align):
- movq -8(%eax), %xmm0
- movq %xmm0, -8(%edx)
-L(fwd_write_0bytes_align):
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_32bytes_align):
- movdqa -32(%eax), %xmm0
- movdqa %xmm0, -32(%edx)
-L(fwd_write_16bytes_align):
- movdqa -16(%eax), %xmm0
- movdqa %xmm0, -16(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_5bytes_align):
- movl -5(%eax), %ecx
- movl -4(%eax), %eax
- movl %ecx, -5(%edx)
- movl %eax, -4(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_45bytes_align):
- movdqa -45(%eax), %xmm0
- movdqa %xmm0, -45(%edx)
-L(fwd_write_29bytes_align):
- movdqa -29(%eax), %xmm0
- movdqa %xmm0, -29(%edx)
-L(fwd_write_13bytes_align):
- movq -13(%eax), %xmm0
- movq %xmm0, -13(%edx)
- movl -5(%eax), %ecx
- movl %ecx, -5(%edx)
- movzbl -1(%eax), %ecx
- movb %cl, -1(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_37bytes_align):
- movdqa -37(%eax), %xmm0
- movdqa %xmm0, -37(%edx)
-L(fwd_write_21bytes_align):
- movdqa -21(%eax), %xmm0
- movdqa %xmm0, -21(%edx)
- movl -5(%eax), %ecx
- movl %ecx, -5(%edx)
- movzbl -1(%eax), %ecx
- movb %cl, -1(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_41bytes_align):
- movdqa -41(%eax), %xmm0
- movdqa %xmm0, -41(%edx)
-L(fwd_write_25bytes_align):
- movdqa -25(%eax), %xmm0
- movdqa %xmm0, -25(%edx)
-L(fwd_write_9bytes_align):
- movq -9(%eax), %xmm0
- movq %xmm0, -9(%edx)
-L(fwd_write_1bytes_align):
- movzbl -1(%eax), %ecx
- movb %cl, -1(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_33bytes_align):
- movdqa -33(%eax), %xmm0
- movdqa %xmm0, -33(%edx)
-L(fwd_write_17bytes_align):
- movdqa -17(%eax), %xmm0
- movdqa %xmm0, -17(%edx)
- movzbl -1(%eax), %ecx
- movb %cl, -1(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_46bytes_align):
- movdqa -46(%eax), %xmm0
- movdqa %xmm0, -46(%edx)
-L(fwd_write_30bytes_align):
- movdqa -30(%eax), %xmm0
- movdqa %xmm0, -30(%edx)
-L(fwd_write_14bytes_align):
- movq -14(%eax), %xmm0
- movq %xmm0, -14(%edx)
-L(fwd_write_6bytes_align):
- movl -6(%eax), %ecx
- movl %ecx, -6(%edx)
- movzwl -2(%eax), %ecx
- movw %cx, -2(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_38bytes_align):
- movdqa -38(%eax), %xmm0
- movdqa %xmm0, -38(%edx)
-L(fwd_write_22bytes_align):
- movdqa -22(%eax), %xmm0
- movdqa %xmm0, -22(%edx)
- movl -6(%eax), %ecx
- movl %ecx, -6(%edx)
- movzwl -2(%eax), %ecx
- movw %cx, -2(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_42bytes_align):
- movdqa -42(%eax), %xmm0
- movdqa %xmm0, -42(%edx)
-L(fwd_write_26bytes_align):
- movdqa -26(%eax), %xmm0
- movdqa %xmm0, -26(%edx)
-L(fwd_write_10bytes_align):
- movq -10(%eax), %xmm0
- movq %xmm0, -10(%edx)
-L(fwd_write_2bytes_align):
- movzwl -2(%eax), %ecx
- movw %cx, -2(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_34bytes_align):
- movdqa -34(%eax), %xmm0
- movdqa %xmm0, -34(%edx)
-L(fwd_write_18bytes_align):
- movdqa -18(%eax), %xmm0
- movdqa %xmm0, -18(%edx)
- movzwl -2(%eax), %ecx
- movw %cx, -2(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_47bytes_align):
- movdqa -47(%eax), %xmm0
- movdqa %xmm0, -47(%edx)
-L(fwd_write_31bytes_align):
- movdqa -31(%eax), %xmm0
- movdqa %xmm0, -31(%edx)
-L(fwd_write_15bytes_align):
- movq -15(%eax), %xmm0
- movq %xmm0, -15(%edx)
-L(fwd_write_7bytes_align):
- movl -7(%eax), %ecx
- movl %ecx, -7(%edx)
- movzwl -3(%eax), %ecx
- movzbl -1(%eax), %eax
- movw %cx, -3(%edx)
- movb %al, -1(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_39bytes_align):
- movdqa -39(%eax), %xmm0
- movdqa %xmm0, -39(%edx)
-L(fwd_write_23bytes_align):
- movdqa -23(%eax), %xmm0
- movdqa %xmm0, -23(%edx)
- movl -7(%eax), %ecx
- movl %ecx, -7(%edx)
- movzwl -3(%eax), %ecx
- movzbl -1(%eax), %eax
- movw %cx, -3(%edx)
- movb %al, -1(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_43bytes_align):
- movdqa -43(%eax), %xmm0
- movdqa %xmm0, -43(%edx)
-L(fwd_write_27bytes_align):
- movdqa -27(%eax), %xmm0
- movdqa %xmm0, -27(%edx)
-L(fwd_write_11bytes_align):
- movq -11(%eax), %xmm0
- movq %xmm0, -11(%edx)
-L(fwd_write_3bytes_align):
- movzwl -3(%eax), %ecx
- movzbl -1(%eax), %eax
- movw %cx, -3(%edx)
- movb %al, -1(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_35bytes_align):
- movdqa -35(%eax), %xmm0
- movdqa %xmm0, -35(%edx)
-L(fwd_write_19bytes_align):
- movdqa -19(%eax), %xmm0
- movdqa %xmm0, -19(%edx)
- movzwl -3(%eax), %ecx
- movzbl -1(%eax), %eax
- movw %cx, -3(%edx)
- movb %al, -1(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_44bytes_align):
- movdqa -44(%eax), %xmm0
- movdqa %xmm0, -44(%edx)
-L(fwd_write_28bytes_align):
- movdqa -28(%eax), %xmm0
- movdqa %xmm0, -28(%edx)
-L(fwd_write_12bytes_align):
- movq -12(%eax), %xmm0
- movq %xmm0, -12(%edx)
-L(fwd_write_4bytes_align):
- movl -4(%eax), %ecx
- movl %ecx, -4(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN
-
- .p2align 4
-L(fwd_write_36bytes_align):
- movdqa -36(%eax), %xmm0
- movdqa %xmm0, -36(%edx)
-L(fwd_write_20bytes_align):
- movdqa -20(%eax), %xmm0
- movdqa %xmm0, -20(%edx)
- movl -4(%eax), %ecx
- movl %ecx, -4(%edx)
-#ifdef USE_AS_MEMPCPY
- movl %edx, %eax
-#else
- movl DEST(%esp), %eax
-#endif
- RETURN_END
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(large_page):
- movdqu (%eax), %xmm1
-#ifdef USE_AS_MEMMOVE
- movl DEST+4(%esp), %edi
- movdqu %xmm0, (%edi)
-#endif
- lea 16(%eax), %eax
- movntdq %xmm1, (%edx)
- lea 16(%edx), %edx
- lea -0x90(%ecx), %ecx
- POP (%edi)
-
- .p2align 4
-L(large_page_loop):
- movdqu (%eax), %xmm0
- movdqu 0x10(%eax), %xmm1
- movdqu 0x20(%eax), %xmm2
- movdqu 0x30(%eax), %xmm3
- movdqu 0x40(%eax), %xmm4
- movdqu 0x50(%eax), %xmm5
- movdqu 0x60(%eax), %xmm6
- movdqu 0x70(%eax), %xmm7
- lea 0x80(%eax), %eax
-
- sub $0x80, %ecx
- movntdq %xmm0, (%edx)
- movntdq %xmm1, 0x10(%edx)
- movntdq %xmm2, 0x20(%edx)
- movntdq %xmm3, 0x30(%edx)
- movntdq %xmm4, 0x40(%edx)
- movntdq %xmm5, 0x50(%edx)
- movntdq %xmm6, 0x60(%edx)
- movntdq %xmm7, 0x70(%edx)
- lea 0x80(%edx), %edx
- jae L(large_page_loop)
- cmp $-0x40, %ecx
- lea 0x80(%ecx), %ecx
- jl L(large_page_less_64bytes)
-
- movdqu (%eax), %xmm0
- movdqu 0x10(%eax), %xmm1
- movdqu 0x20(%eax), %xmm2
- movdqu 0x30(%eax), %xmm3
- lea 0x40(%eax), %eax
-
- movntdq %xmm0, (%edx)
- movntdq %xmm1, 0x10(%edx)
- movntdq %xmm2, 0x20(%edx)
- movntdq %xmm3, 0x30(%edx)
- lea 0x40(%edx), %edx
- sub $0x40, %ecx
-L(large_page_less_64bytes):
- cmp $32, %ecx
- jb L(large_page_less_32bytes)
- movdqu (%eax), %xmm0
- movdqu 0x10(%eax), %xmm1
- lea 0x20(%eax), %eax
- movntdq %xmm0, (%edx)
- movntdq %xmm1, 0x10(%edx)
- lea 0x20(%edx), %edx
- sub $0x20, %ecx
-L(large_page_less_32bytes):
- add %ecx, %edx
- add %ecx, %eax
- sfence
- BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
-
- .p2align 4
-L(bk_write_44bytes):
- movq 36(%eax), %xmm0
- movq %xmm0, 36(%edx)
-L(bk_write_36bytes):
- movq 28(%eax), %xmm0
- movq %xmm0, 28(%edx)
-L(bk_write_28bytes):
- movq 20(%eax), %xmm0
- movq %xmm0, 20(%edx)
-L(bk_write_20bytes):
- movq 12(%eax), %xmm0
- movq %xmm0, 12(%edx)
-L(bk_write_12bytes):
- movq 4(%eax), %xmm0
- movq %xmm0, 4(%edx)
-L(bk_write_4bytes):
- movl (%eax), %ecx
- movl %ecx, (%edx)
-L(bk_write_0bytes):
- movl DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
- movl LEN(%esp), %ecx
- add %ecx, %eax
-#endif
- RETURN
-
- .p2align 4
-L(bk_write_40bytes):
- movq 32(%eax), %xmm0
- movq %xmm0, 32(%edx)
-L(bk_write_32bytes):
- movq 24(%eax), %xmm0
- movq %xmm0, 24(%edx)
-L(bk_write_24bytes):
- movq 16(%eax), %xmm0
- movq %xmm0, 16(%edx)
-L(bk_write_16bytes):
- movq 8(%eax), %xmm0
- movq %xmm0, 8(%edx)
-L(bk_write_8bytes):
- movq (%eax), %xmm0
- movq %xmm0, (%edx)
- movl DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
- movl LEN(%esp), %ecx
- add %ecx, %eax
-#endif
- RETURN
-
- .p2align 4
-L(bk_write_45bytes):
- movq 37(%eax), %xmm0
- movq %xmm0, 37(%edx)
-L(bk_write_37bytes):
- movq 29(%eax), %xmm0
- movq %xmm0, 29(%edx)
-L(bk_write_29bytes):
- movq 21(%eax), %xmm0
- movq %xmm0, 21(%edx)
-L(bk_write_21bytes):
- movq 13(%eax), %xmm0
- movq %xmm0, 13(%edx)
-L(bk_write_13bytes):
- movq 5(%eax), %xmm0
- movq %xmm0, 5(%edx)
-L(bk_write_5bytes):
- movl 1(%eax), %ecx
- movl %ecx, 1(%edx)
-L(bk_write_1bytes):
- movzbl (%eax), %ecx
- movb %cl, (%edx)
- movl DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
- movl LEN(%esp), %ecx
- add %ecx, %eax
-#endif
- RETURN
-
- .p2align 4
-L(bk_write_41bytes):
- movq 33(%eax), %xmm0
- movq %xmm0, 33(%edx)
-L(bk_write_33bytes):
- movq 25(%eax), %xmm0
- movq %xmm0, 25(%edx)
-L(bk_write_25bytes):
- movq 17(%eax), %xmm0
- movq %xmm0, 17(%edx)
-L(bk_write_17bytes):
- movq 9(%eax), %xmm0
- movq %xmm0, 9(%edx)
-L(bk_write_9bytes):
- movq 1(%eax), %xmm0
- movq %xmm0, 1(%edx)
- movzbl (%eax), %ecx
- movb %cl, (%edx)
- movl DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
- movl LEN(%esp), %ecx
- add %ecx, %eax
-#endif
- RETURN
-
- .p2align 4
-L(bk_write_46bytes):
- movq 38(%eax), %xmm0
- movq %xmm0, 38(%edx)
-L(bk_write_38bytes):
- movq 30(%eax), %xmm0
- movq %xmm0, 30(%edx)
-L(bk_write_30bytes):
- movq 22(%eax), %xmm0
- movq %xmm0, 22(%edx)
-L(bk_write_22bytes):
- movq 14(%eax), %xmm0
- movq %xmm0, 14(%edx)
-L(bk_write_14bytes):
- movq 6(%eax), %xmm0
- movq %xmm0, 6(%edx)
-L(bk_write_6bytes):
- movl 2(%eax), %ecx
- movl %ecx, 2(%edx)
- movzwl (%eax), %ecx
- movw %cx, (%edx)
- movl DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
- movl LEN(%esp), %ecx
- add %ecx, %eax
-#endif
- RETURN
-
- .p2align 4
-L(bk_write_42bytes):
- movq 34(%eax), %xmm0
- movq %xmm0, 34(%edx)
-L(bk_write_34bytes):
- movq 26(%eax), %xmm0
- movq %xmm0, 26(%edx)
-L(bk_write_26bytes):
- movq 18(%eax), %xmm0
- movq %xmm0, 18(%edx)
-L(bk_write_18bytes):
- movq 10(%eax), %xmm0
- movq %xmm0, 10(%edx)
-L(bk_write_10bytes):
- movq 2(%eax), %xmm0
- movq %xmm0, 2(%edx)
-L(bk_write_2bytes):
- movzwl (%eax), %ecx
- movw %cx, (%edx)
- movl DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
- movl LEN(%esp), %ecx
- add %ecx, %eax
-#endif
- RETURN
-
- .p2align 4
-L(bk_write_47bytes):
- movq 39(%eax), %xmm0
- movq %xmm0, 39(%edx)
-L(bk_write_39bytes):
- movq 31(%eax), %xmm0
- movq %xmm0, 31(%edx)
-L(bk_write_31bytes):
- movq 23(%eax), %xmm0
- movq %xmm0, 23(%edx)
-L(bk_write_23bytes):
- movq 15(%eax), %xmm0
- movq %xmm0, 15(%edx)
-L(bk_write_15bytes):
- movq 7(%eax), %xmm0
- movq %xmm0, 7(%edx)
-L(bk_write_7bytes):
- movl 3(%eax), %ecx
- movl %ecx, 3(%edx)
- movzwl 1(%eax), %ecx
- movw %cx, 1(%edx)
- movzbl (%eax), %eax
- movb %al, (%edx)
- movl DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
- movl LEN(%esp), %ecx
- add %ecx, %eax
-#endif
- RETURN
-
- .p2align 4
-L(bk_write_43bytes):
- movq 35(%eax), %xmm0
- movq %xmm0, 35(%edx)
-L(bk_write_35bytes):
- movq 27(%eax), %xmm0
- movq %xmm0, 27(%edx)
-L(bk_write_27bytes):
- movq 19(%eax), %xmm0
- movq %xmm0, 19(%edx)
-L(bk_write_19bytes):
- movq 11(%eax), %xmm0
- movq %xmm0, 11(%edx)
-L(bk_write_11bytes):
- movq 3(%eax), %xmm0
- movq %xmm0, 3(%edx)
-L(bk_write_3bytes):
- movzwl 1(%eax), %ecx
- movw %cx, 1(%edx)
- movzbl (%eax), %eax
- movb %al, (%edx)
- movl DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
- movl LEN(%esp), %ecx
- add %ecx, %eax
-#endif
- RETURN_END
-
-
- .pushsection .rodata.ssse3,"a",@progbits
- .p2align 2
-L(table_48bytes_fwd):
- .int JMPTBL (L(fwd_write_0bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_1bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_2bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_3bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_4bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_5bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_6bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_7bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_8bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_9bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_10bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_11bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_12bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_13bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_14bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_15bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_16bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_17bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_18bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_19bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_20bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_21bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_22bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_23bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_24bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_25bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_26bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_27bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_28bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_29bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_30bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_31bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_32bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_33bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_34bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_35bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_36bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_37bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_38bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_39bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_40bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_41bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_42bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_43bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_44bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_45bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_46bytes), L(table_48bytes_fwd))
- .int JMPTBL (L(fwd_write_47bytes), L(table_48bytes_fwd))
-
- .p2align 2
-L(table_48bytes_fwd_align):
- .int JMPTBL (L(fwd_write_0bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_1bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_2bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_3bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_4bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_5bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_6bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_7bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_8bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_9bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_10bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_11bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_12bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_13bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_14bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_15bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_16bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_17bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_18bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_19bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_20bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_21bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_22bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_23bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_24bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_25bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_26bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_27bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_28bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_29bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_30bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_31bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_32bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_33bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_34bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_35bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_36bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_37bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_38bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_39bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_40bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_41bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_42bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_43bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_44bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_45bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_46bytes_align), L(table_48bytes_fwd_align))
- .int JMPTBL (L(fwd_write_47bytes_align), L(table_48bytes_fwd_align))
-
- .p2align 2
-L(shl_table):
- .int JMPTBL (L(shl_0), L(shl_table))
- .int JMPTBL (L(shl_1), L(shl_table))
- .int JMPTBL (L(shl_2), L(shl_table))
- .int JMPTBL (L(shl_3), L(shl_table))
- .int JMPTBL (L(shl_4), L(shl_table))
- .int JMPTBL (L(shl_5), L(shl_table))
- .int JMPTBL (L(shl_6), L(shl_table))
- .int JMPTBL (L(shl_7), L(shl_table))
- .int JMPTBL (L(shl_8), L(shl_table))
- .int JMPTBL (L(shl_9), L(shl_table))
- .int JMPTBL (L(shl_10), L(shl_table))
- .int JMPTBL (L(shl_11), L(shl_table))
- .int JMPTBL (L(shl_12), L(shl_table))
- .int JMPTBL (L(shl_13), L(shl_table))
- .int JMPTBL (L(shl_14), L(shl_table))
- .int JMPTBL (L(shl_15), L(shl_table))
-
- .p2align 2
-L(table_48_bytes_bwd):
- .int JMPTBL (L(bk_write_0bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_1bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_2bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_3bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_4bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_5bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_6bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_7bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_8bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_9bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_10bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_11bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_12bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_13bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_14bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_15bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_16bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_17bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_18bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_19bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_20bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_21bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_22bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_23bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_24bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_25bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_26bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_27bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_28bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_29bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_30bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_31bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_32bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_33bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_34bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_35bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_36bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_37bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_38bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_39bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_40bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_41bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_42bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_43bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_44bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_45bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_46bytes), L(table_48_bytes_bwd))
- .int JMPTBL (L(bk_write_47bytes), L(table_48_bytes_bwd))
-
- .popsection
-
-#ifdef USE_AS_MEMMOVE
- .p2align 4
-L(copy_backward):
- PUSH (%edi)
- movl %eax, %edi
- lea (%ecx,%edx,1),%edx
- lea (%ecx,%edi,1),%edi
- testl $0x3, %edx
- jnz L(bk_align)
-
-L(bk_aligned_4):
- cmp $64, %ecx
- jae L(bk_write_more64bytes)
-
-L(bk_write_64bytesless):
- cmp $32, %ecx
- jb L(bk_write_less32bytes)
-
-L(bk_write_more32bytes):
- /* Copy 32 bytes at a time. */
- sub $32, %ecx
- movq -8(%edi), %xmm0
- movq %xmm0, -8(%edx)
- movq -16(%edi), %xmm0
- movq %xmm0, -16(%edx)
- movq -24(%edi), %xmm0
- movq %xmm0, -24(%edx)
- movq -32(%edi), %xmm0
- movq %xmm0, -32(%edx)
- sub $32, %edx
- sub $32, %edi
-
-L(bk_write_less32bytes):
- movl %edi, %eax
- sub %ecx, %edx
- sub %ecx, %eax
- POP (%edi)
-L(bk_write_less32bytes_2):
- BRANCH_TO_JMPTBL_ENTRY (L(table_48_bytes_bwd), %ecx, 4)
-
- CFI_PUSH (%edi)
-
- .p2align 4
-L(bk_align):
- cmp $8, %ecx
- jbe L(bk_write_less32bytes)
- testl $1, %edx
- /* We get here only if (EDX & 3 ) != 0 so if (EDX & 1) ==0,
- then (EDX & 2) must be != 0. */
- jz L(bk_got2)
- sub $1, %edi
- sub $1, %ecx
- sub $1, %edx
- movzbl (%edi), %eax
- movb %al, (%edx)
-
- testl $2, %edx
- jz L(bk_aligned_4)
-
-L(bk_got2):
- sub $2, %edi
- sub $2, %ecx
- sub $2, %edx
- movzwl (%edi), %eax
- movw %ax, (%edx)
- jmp L(bk_aligned_4)
-
- .p2align 4
-L(bk_write_more64bytes):
- /* Check alignment of last byte. */
- testl $15, %edx
- jz L(bk_ssse3_cpy_pre)
-
-/* EDX is aligned 4 bytes, but not 16 bytes. */
-L(bk_ssse3_align):
- sub $4, %edi
- sub $4, %ecx
- sub $4, %edx
- movl (%edi), %eax
- movl %eax, (%edx)
-
- testl $15, %edx
- jz L(bk_ssse3_cpy_pre)
-
- sub $4, %edi
- sub $4, %ecx
- sub $4, %edx
- movl (%edi), %eax
- movl %eax, (%edx)
-
- testl $15, %edx
- jz L(bk_ssse3_cpy_pre)
-
- sub $4, %edi
- sub $4, %ecx
- sub $4, %edx
- movl (%edi), %eax
- movl %eax, (%edx)
-
-L(bk_ssse3_cpy_pre):
- cmp $64, %ecx
- jb L(bk_write_more32bytes)
-
- .p2align 4
-L(bk_ssse3_cpy):
- sub $64, %edi
- sub $64, %ecx
- sub $64, %edx
- movdqu 0x30(%edi), %xmm3
- movdqa %xmm3, 0x30(%edx)
- movdqu 0x20(%edi), %xmm2
- movdqa %xmm2, 0x20(%edx)
- movdqu 0x10(%edi), %xmm1
- movdqa %xmm1, 0x10(%edx)
- movdqu (%edi), %xmm0
- movdqa %xmm0, (%edx)
- cmp $64, %ecx
- jae L(bk_ssse3_cpy)
- jmp L(bk_write_64bytesless)
-
-#endif
-
-END (MEMCPY)
diff --git a/libc/arch-x86/string/ssse3-memmove-atom.S b/libc/arch-x86/string/ssse3-memmove-atom.S
deleted file mode 100644
index 3572eac..0000000
--- a/libc/arch-x86/string/ssse3-memmove-atom.S
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
-Copyright (c) 2010, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-
-#define MEMCPY memmove_atom
-#define USE_AS_MEMMOVE
-#include "ssse3-memcpy-atom.S"
diff --git a/libc/arch-x86/string/ssse3-strncpy-atom.S b/libc/arch-x86/string/ssse3-strncpy-atom.S
deleted file mode 100644
index 0c27ffe..0000000
--- a/libc/arch-x86/string/ssse3-strncpy-atom.S
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-Copyright (c) 2011, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#define USE_AS_STRNCPY
-#define STRCPY strncpy_atom
-#include "ssse3-strcpy-atom.S"
diff --git a/libc/arch-x86/string/ssse3-wcscat-atom.S b/libc/arch-x86/string/ssse3-wcscat-atom.S
deleted file mode 100644
index 8a389a3..0000000
--- a/libc/arch-x86/string/ssse3-wcscat-atom.S
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
-Copyright (c) 2011 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef L
-# define L(label) .L##label
-#endif
-
-#ifndef cfi_startproc
-# define cfi_startproc .cfi_startproc
-#endif
-
-#ifndef cfi_endproc
-# define cfi_endproc .cfi_endproc
-#endif
-
-#ifndef cfi_rel_offset
-# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off
-#endif
-
-#ifndef cfi_restore
-# define cfi_restore(reg) .cfi_restore reg
-#endif
-
-#ifndef cfi_adjust_cfa_offset
-# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off
-#endif
-
-#ifndef ENTRY
-# define ENTRY(name) \
- .type name, @function; \
- .globl name; \
- .p2align 4; \
-name: \
- cfi_startproc
-#endif
-
-#ifndef END
-# define END(name) \
- cfi_endproc; \
- .size name, .-name
-#endif
-
-#define CFI_PUSH(REG) \
- cfi_adjust_cfa_offset (4); \
- cfi_rel_offset (REG, 0)
-
-#define CFI_POP(REG) \
- cfi_adjust_cfa_offset (-4); \
- cfi_restore (REG)
-
-#define PUSH(REG) pushl REG; CFI_PUSH (REG)
-#define POP(REG) popl REG; CFI_POP (REG)
-
-#define PARMS 4
-#define STR1 PARMS+4
-#define STR2 STR1+4
-
-#define USE_AS_WCSCAT
-
-.text
-ENTRY (wcscat)
- PUSH (%edi)
- mov STR1(%esp), %edi
- mov %edi, %edx
-
-#define RETURN jmp L(WcscpyAtom)
-#include "sse2-wcslen-atom.S"
-
-L(WcscpyAtom):
- shl $2, %eax
- mov STR2(%esp), %ecx
- lea (%edi, %eax), %edx
-
- cmpl $0, (%ecx)
- jz L(Exit4)
- cmpl $0, 4(%ecx)
- jz L(Exit8)
- cmpl $0, 8(%ecx)
- jz L(Exit12)
- cmpl $0, 12(%ecx)
- jz L(Exit16)
-
-#undef RETURN
-#define RETURN POP(%edi); ret; CFI_PUSH(%edi)
-#include "ssse3-wcscpy-atom.S"
-
-END (wcscat)
diff --git a/libc/arch-x86/string/ssse3-wcscpy-atom.S b/libc/arch-x86/string/ssse3-wcscpy-atom.S
deleted file mode 100644
index 27cb61e..0000000
--- a/libc/arch-x86/string/ssse3-wcscpy-atom.S
+++ /dev/null
@@ -1,652 +0,0 @@
-/*
-Copyright (c) 2011, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef USE_AS_WCSCAT
-
-# ifndef L
-# define L(label) .L##label
-# endif
-
-# ifndef cfi_startproc
-# define cfi_startproc .cfi_startproc
-# endif
-
-# ifndef cfi_endproc
-# define cfi_endproc .cfi_endproc
-# endif
-
-# ifndef cfi_rel_offset
-# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off
-# endif
-
-# ifndef cfi_restore
-# define cfi_restore(reg) .cfi_restore reg
-# endif
-
-# ifndef cfi_adjust_cfa_offset
-# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off
-# endif
-
-# ifndef ENTRY
-# define ENTRY(name) \
- .type name, @function; \
- .globl name; \
- .p2align 4; \
-name: \
- cfi_startproc
-# endif
-
-# ifndef END
-# define END(name) \
- cfi_endproc; \
- .size name, .-name
-# endif
-
-# define CFI_PUSH(REG) \
- cfi_adjust_cfa_offset (4); \
- cfi_rel_offset (REG, 0)
-
-# define CFI_POP(REG) \
- cfi_adjust_cfa_offset (-4); \
- cfi_restore (REG)
-
-# define PUSH(REG) pushl REG; CFI_PUSH (REG)
-# define POP(REG) popl REG; CFI_POP (REG)
-
-# define PARMS 4
-# define RETURN POP (%edi); ret; CFI_PUSH (%edi)
-
-# define STR1 PARMS
-# define STR2 STR1+4
-# define LEN STR2+4
-
-.text
-ENTRY (wcscpy)
- mov STR1(%esp), %edx
- mov STR2(%esp), %ecx
-
- cmpl $0, (%ecx)
- jz L(ExitTail4)
- cmpl $0, 4(%ecx)
- jz L(ExitTail8)
- cmpl $0, 8(%ecx)
- jz L(ExitTail12)
- cmpl $0, 12(%ecx)
- jz L(ExitTail16)
-
- PUSH (%edi)
- mov %edx, %edi
-#endif
- PUSH (%esi)
- lea 16(%ecx), %esi
-
- and $-16, %esi
-
- pxor %xmm0, %xmm0
- pcmpeqd (%esi), %xmm0
- movdqu (%ecx), %xmm1
- movdqu %xmm1, (%edx)
-
- pmovmskb %xmm0, %eax
- sub %ecx, %esi
-
- test %eax, %eax
- jnz L(CopyFrom1To16Bytes)
-
- mov %edx, %eax
- lea 16(%edx), %edx
- and $-16, %edx
- sub %edx, %eax
-
- sub %eax, %ecx
- mov %ecx, %eax
- and $0xf, %eax
- mov $0, %esi
-
- jz L(Align16Both)
- cmp $4, %eax
- je L(Shl4)
- cmp $8, %eax
- je L(Shl8)
- jmp L(Shl12)
-
-L(Align16Both):
- movaps (%ecx), %xmm1
- movaps 16(%ecx), %xmm2
- movaps %xmm1, (%edx)
- pcmpeqd %xmm2, %xmm0
- pmovmskb %xmm0, %eax
- lea 16(%esi), %esi
-
- test %eax, %eax
- jnz L(CopyFrom1To16Bytes)
-
- movaps 16(%ecx, %esi), %xmm3
- movaps %xmm2, (%edx, %esi)
- pcmpeqd %xmm3, %xmm0
- pmovmskb %xmm0, %eax
- lea 16(%esi), %esi
-
- test %eax, %eax
- jnz L(CopyFrom1To16Bytes)
-
- movaps 16(%ecx, %esi), %xmm4
- movaps %xmm3, (%edx, %esi)
- pcmpeqd %xmm4, %xmm0
- pmovmskb %xmm0, %eax
- lea 16(%esi), %esi
-
- test %eax, %eax
- jnz L(CopyFrom1To16Bytes)
-
- movaps 16(%ecx, %esi), %xmm1
- movaps %xmm4, (%edx, %esi)
- pcmpeqd %xmm1, %xmm0
- pmovmskb %xmm0, %eax
- lea 16(%esi), %esi
-
- test %eax, %eax
- jnz L(CopyFrom1To16Bytes)
-
- movaps 16(%ecx, %esi), %xmm2
- movaps %xmm1, (%edx, %esi)
- pcmpeqd %xmm2, %xmm0
- pmovmskb %xmm0, %eax
- lea 16(%esi), %esi
-
- test %eax, %eax
- jnz L(CopyFrom1To16Bytes)
-
- movaps 16(%ecx, %esi), %xmm3
- movaps %xmm2, (%edx, %esi)
- pcmpeqd %xmm3, %xmm0
- pmovmskb %xmm0, %eax
- lea 16(%esi), %esi
-
- test %eax, %eax
- jnz L(CopyFrom1To16Bytes)
-
- movaps %xmm3, (%edx, %esi)
- mov %ecx, %eax
- lea 16(%ecx, %esi), %ecx
- and $-0x40, %ecx
- sub %ecx, %eax
- sub %eax, %edx
-
- mov $-0x40, %esi
-
-L(Aligned64Loop):
- movaps (%ecx), %xmm2
- movaps 32(%ecx), %xmm3
- movaps %xmm2, %xmm4
- movaps 16(%ecx), %xmm5
- movaps %xmm3, %xmm6
- movaps 48(%ecx), %xmm7
- pminub %xmm5, %xmm2
- pminub %xmm7, %xmm3
- pminub %xmm2, %xmm3
- lea 64(%edx), %edx
- pcmpeqd %xmm0, %xmm3
- lea 64(%ecx), %ecx
- pmovmskb %xmm3, %eax
-
- test %eax, %eax
- jnz L(Aligned64Leave)
- movaps %xmm4, -64(%edx)
- movaps %xmm5, -48(%edx)
- movaps %xmm6, -32(%edx)
- movaps %xmm7, -16(%edx)
- jmp L(Aligned64Loop)
-
-L(Aligned64Leave):
- pcmpeqd %xmm4, %xmm0
- pmovmskb %xmm0, %eax
- test %eax, %eax
- jnz L(CopyFrom1To16Bytes)
-
- pcmpeqd %xmm5, %xmm0
- pmovmskb %xmm0, %eax
- movaps %xmm4, -64(%edx)
- lea 16(%esi), %esi
- test %eax, %eax
- jnz L(CopyFrom1To16Bytes)
-
- pcmpeqd %xmm6, %xmm0
- pmovmskb %xmm0, %eax
- movaps %xmm5, -48(%edx)
- lea 16(%esi), %esi
- test %eax, %eax
- jnz L(CopyFrom1To16Bytes)
-
- movaps %xmm6, -32(%edx)
- pcmpeqd %xmm7, %xmm0
- pmovmskb %xmm0, %eax
- lea 16(%esi), %esi
- test %eax, %eax
- jnz L(CopyFrom1To16Bytes)
-
- mov $-0x40, %esi
- movaps %xmm7, -16(%edx)
- jmp L(Aligned64Loop)
-
- .p2align 4
-L(Shl4):
- movaps -4(%ecx), %xmm1
- movaps 12(%ecx), %xmm2
-L(Shl4Start):
- pcmpeqd %xmm2, %xmm0
- pmovmskb %xmm0, %eax
- movaps %xmm2, %xmm3
-
- test %eax, %eax
- jnz L(Shl4LoopExit)
-
- palignr $4, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps 28(%ecx), %xmm2
-
- pcmpeqd %xmm2, %xmm0
- lea 16(%edx), %edx
- pmovmskb %xmm0, %eax
- lea 16(%ecx), %ecx
- movaps %xmm2, %xmm1
-
- test %eax, %eax
- jnz L(Shl4LoopExit)
-
- palignr $4, %xmm3, %xmm2
- movaps %xmm2, (%edx)
- movaps 28(%ecx), %xmm2
-
- pcmpeqd %xmm2, %xmm0
- lea 16(%edx), %edx
- pmovmskb %xmm0, %eax
- lea 16(%ecx), %ecx
- movaps %xmm2, %xmm3
-
- test %eax, %eax
- jnz L(Shl4LoopExit)
-
- palignr $4, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps 28(%ecx), %xmm2
-
- pcmpeqd %xmm2, %xmm0
- lea 16(%edx), %edx
- pmovmskb %xmm0, %eax
- lea 16(%ecx), %ecx
-
- test %eax, %eax
- jnz L(Shl4LoopExit)
-
- palignr $4, %xmm3, %xmm2
- movaps %xmm2, (%edx)
- lea 28(%ecx), %ecx
- lea 16(%edx), %edx
-
- mov %ecx, %eax
- and $-0x40, %ecx
- sub %ecx, %eax
- lea -12(%ecx), %ecx
- sub %eax, %edx
-
- movaps -4(%ecx), %xmm1
-
-L(Shl4LoopStart):
- movaps 12(%ecx), %xmm2
- movaps 28(%ecx), %xmm3
- movaps %xmm3, %xmm6
- movaps 44(%ecx), %xmm4
- movaps %xmm4, %xmm7
- movaps 60(%ecx), %xmm5
- pminub %xmm2, %xmm6
- pminub %xmm5, %xmm7
- pminub %xmm6, %xmm7
- pcmpeqd %xmm0, %xmm7
- pmovmskb %xmm7, %eax
- movaps %xmm5, %xmm7
- palignr $4, %xmm4, %xmm5
- palignr $4, %xmm3, %xmm4
- test %eax, %eax
- jnz L(Shl4Start)
-
- palignr $4, %xmm2, %xmm3
- lea 64(%ecx), %ecx
- palignr $4, %xmm1, %xmm2
- movaps %xmm7, %xmm1
- movaps %xmm5, 48(%edx)
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- jmp L(Shl4LoopStart)
-
-L(Shl4LoopExit):
- movlpd (%ecx), %xmm0
- movl 8(%ecx), %esi
- movlpd %xmm0, (%edx)
- movl %esi, 8(%edx)
- POP (%esi)
- add $12, %edx
- add $12, %ecx
- test %al, %al
- jz L(ExitHigh)
- test $0x01, %al
- jnz L(Exit4)
- movlpd (%ecx), %xmm0
- movlpd %xmm0, (%edx)
- movl %edi, %eax
- RETURN
-
- CFI_PUSH (%esi)
-
- .p2align 4
-L(Shl8):
- movaps -8(%ecx), %xmm1
- movaps 8(%ecx), %xmm2
-L(Shl8Start):
- pcmpeqd %xmm2, %xmm0
- pmovmskb %xmm0, %eax
- movaps %xmm2, %xmm3
-
- test %eax, %eax
- jnz L(Shl8LoopExit)
-
- palignr $8, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps 24(%ecx), %xmm2
-
- pcmpeqd %xmm2, %xmm0
- lea 16(%edx), %edx
- pmovmskb %xmm0, %eax
- lea 16(%ecx), %ecx
- movaps %xmm2, %xmm1
-
- test %eax, %eax
- jnz L(Shl8LoopExit)
-
- palignr $8, %xmm3, %xmm2
- movaps %xmm2, (%edx)
- movaps 24(%ecx), %xmm2
-
- pcmpeqd %xmm2, %xmm0
- lea 16(%edx), %edx
- pmovmskb %xmm0, %eax
- lea 16(%ecx), %ecx
- movaps %xmm2, %xmm3
-
- test %eax, %eax
- jnz L(Shl8LoopExit)
-
- palignr $8, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps 24(%ecx), %xmm2
-
- pcmpeqd %xmm2, %xmm0
- lea 16(%edx), %edx
- pmovmskb %xmm0, %eax
- lea 16(%ecx), %ecx
-
- test %eax, %eax
- jnz L(Shl8LoopExit)
-
- palignr $8, %xmm3, %xmm2
- movaps %xmm2, (%edx)
- lea 24(%ecx), %ecx
- lea 16(%edx), %edx
-
- mov %ecx, %eax
- and $-0x40, %ecx
- sub %ecx, %eax
- lea -8(%ecx), %ecx
- sub %eax, %edx
-
- movaps -8(%ecx), %xmm1
-
-L(Shl8LoopStart):
- movaps 8(%ecx), %xmm2
- movaps 24(%ecx), %xmm3
- movaps %xmm3, %xmm6
- movaps 40(%ecx), %xmm4
- movaps %xmm4, %xmm7
- movaps 56(%ecx), %xmm5
- pminub %xmm2, %xmm6
- pminub %xmm5, %xmm7
- pminub %xmm6, %xmm7
- pcmpeqd %xmm0, %xmm7
- pmovmskb %xmm7, %eax
- movaps %xmm5, %xmm7
- palignr $8, %xmm4, %xmm5
- palignr $8, %xmm3, %xmm4
- test %eax, %eax
- jnz L(Shl8Start)
-
- palignr $8, %xmm2, %xmm3
- lea 64(%ecx), %ecx
- palignr $8, %xmm1, %xmm2
- movaps %xmm7, %xmm1
- movaps %xmm5, 48(%edx)
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- jmp L(Shl8LoopStart)
-
-L(Shl8LoopExit):
- movlpd (%ecx), %xmm0
- movlpd %xmm0, (%edx)
- POP (%esi)
- add $8, %edx
- add $8, %ecx
- test %al, %al
- jz L(ExitHigh)
- test $0x01, %al
- jnz L(Exit4)
- movlpd (%ecx), %xmm0
- movlpd %xmm0, (%edx)
- movl %edi, %eax
- RETURN
-
- CFI_PUSH (%esi)
-
- .p2align 4
-L(Shl12):
- movaps -12(%ecx), %xmm1
- movaps 4(%ecx), %xmm2
-L(Shl12Start):
- pcmpeqd %xmm2, %xmm0
- pmovmskb %xmm0, %eax
- movaps %xmm2, %xmm3
-
- test %eax, %eax
- jnz L(Shl12LoopExit)
-
- palignr $12, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps 20(%ecx), %xmm2
-
- pcmpeqd %xmm2, %xmm0
- lea 16(%edx), %edx
- pmovmskb %xmm0, %eax
- lea 16(%ecx), %ecx
- movaps %xmm2, %xmm1
-
- test %eax, %eax
- jnz L(Shl12LoopExit)
-
- palignr $12, %xmm3, %xmm2
- movaps %xmm2, (%edx)
- movaps 20(%ecx), %xmm2
-
- pcmpeqd %xmm2, %xmm0
- lea 16(%edx), %edx
- pmovmskb %xmm0, %eax
- lea 16(%ecx), %ecx
- movaps %xmm2, %xmm3
-
- test %eax, %eax
- jnz L(Shl12LoopExit)
-
- palignr $12, %xmm1, %xmm2
- movaps %xmm2, (%edx)
- movaps 20(%ecx), %xmm2
-
- pcmpeqd %xmm2, %xmm0
- lea 16(%edx), %edx
- pmovmskb %xmm0, %eax
- lea 16(%ecx), %ecx
-
- test %eax, %eax
- jnz L(Shl12LoopExit)
-
- palignr $12, %xmm3, %xmm2
- movaps %xmm2, (%edx)
- lea 20(%ecx), %ecx
- lea 16(%edx), %edx
-
- mov %ecx, %eax
- and $-0x40, %ecx
- sub %ecx, %eax
- lea -4(%ecx), %ecx
- sub %eax, %edx
-
- movaps -12(%ecx), %xmm1
-
-L(Shl12LoopStart):
- movaps 4(%ecx), %xmm2
- movaps 20(%ecx), %xmm3
- movaps %xmm3, %xmm6
- movaps 36(%ecx), %xmm4
- movaps %xmm4, %xmm7
- movaps 52(%ecx), %xmm5
- pminub %xmm2, %xmm6
- pminub %xmm5, %xmm7
- pminub %xmm6, %xmm7
- pcmpeqd %xmm0, %xmm7
- pmovmskb %xmm7, %eax
- movaps %xmm5, %xmm7
- palignr $12, %xmm4, %xmm5
- palignr $12, %xmm3, %xmm4
- test %eax, %eax
- jnz L(Shl12Start)
-
- palignr $12, %xmm2, %xmm3
- lea 64(%ecx), %ecx
- palignr $12, %xmm1, %xmm2
- movaps %xmm7, %xmm1
- movaps %xmm5, 48(%edx)
- movaps %xmm4, 32(%edx)
- movaps %xmm3, 16(%edx)
- movaps %xmm2, (%edx)
- lea 64(%edx), %edx
- jmp L(Shl12LoopStart)
-
-L(Shl12LoopExit):
- movl (%ecx), %esi
- movl %esi, (%edx)
- mov $4, %esi
-
- .p2align 4
-L(CopyFrom1To16Bytes):
- add %esi, %edx
- add %esi, %ecx
-
- POP (%esi)
- test %al, %al
- jz L(ExitHigh)
- test $0x01, %al
- jnz L(Exit4)
-L(Exit8):
- movlpd (%ecx), %xmm0
- movlpd %xmm0, (%edx)
- movl %edi, %eax
- RETURN
-
- .p2align 4
-L(ExitHigh):
- test $0x01, %ah
- jnz L(Exit12)
-L(Exit16):
- movdqu (%ecx), %xmm0
- movdqu %xmm0, (%edx)
- movl %edi, %eax
- RETURN
-
- .p2align 4
-L(Exit4):
- movl (%ecx), %eax
- movl %eax, (%edx)
- movl %edi, %eax
- RETURN
-
- .p2align 4
-L(Exit12):
- movlpd (%ecx), %xmm0
- movlpd %xmm0, (%edx)
- movl 8(%ecx), %eax
- movl %eax, 8(%edx)
- movl %edi, %eax
- RETURN
-
-CFI_POP (%edi)
-
- .p2align 4
-L(ExitTail4):
- movl (%ecx), %eax
- movl %eax, (%edx)
- movl %edx, %eax
- ret
-
- .p2align 4
-L(ExitTail8):
- movlpd (%ecx), %xmm0
- movlpd %xmm0, (%edx)
- movl %edx, %eax
- ret
-
- .p2align 4
-L(ExitTail12):
- movlpd (%ecx), %xmm0
- movlpd %xmm0, (%edx)
- movl 8(%ecx), %eax
- movl %eax, 8(%edx)
- movl %edx, %eax
- ret
-
- .p2align 4
-L(ExitTail16):
- movdqu (%ecx), %xmm0
- movdqu %xmm0, (%edx)
- movl %edx, %eax
- ret
-
-#ifndef USE_AS_WCSCAT
-END (wcscpy)
-#endif
diff --git a/libc/arch-x86/string/ssse3-wmemcmp-atom.S b/libc/arch-x86/string/ssse3-wmemcmp-atom.S
deleted file mode 100644
index a81b78b..0000000
--- a/libc/arch-x86/string/ssse3-wmemcmp-atom.S
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
-Copyright (c) 2011, 2012, 2013 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
-
- * Neither the name of Intel Corporation nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#define MEMCMP wmemcmp_atom
-
-#define USE_WCHAR
-#define USE_AS_WMEMCMP 1
-#include "ssse3-memcmp-atom.S"
diff --git a/libc/bionic/fts.c b/libc/bionic/fts.c
index 1287267..c36835e 100644
--- a/libc/bionic/fts.c
+++ b/libc/bionic/fts.c
@@ -29,7 +29,6 @@
* SUCH DAMAGE.
*/
-#include <sys/param.h> /* ALIGN */
#include <sys/stat.h>
#include <dirent.h>
@@ -37,6 +36,7 @@
#include <fcntl.h>
#include <fts.h>
#include <limits.h>
+#include <stdalign.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
@@ -912,10 +912,14 @@
* be careful that the stat structure is reasonably aligned. Since the
* fts_name field is declared to be of size 1, the fts_name pointer is
* namelen + 2 before the first possible address of the stat structure.
+ *
+ * We can't use the same trick FreeBSD uses here because our fts_name
+ * is a char[1] rather than a char*. This is also the reason we don't
+ * need to say `namelen + 1`. We just assume the worst alignment.
*/
len = sizeof(FTSENT) + namelen;
if (!ISSET(FTS_NOSTAT))
- len += sizeof(struct stat) + ALIGNBYTES;
+ len += alignof(struct stat) + sizeof(struct stat);
if ((p = calloc(1, len)) == NULL)
return (NULL);
@@ -923,7 +927,7 @@
p->fts_namelen = namelen;
p->fts_instr = FTS_NOINSTR;
if (!ISSET(FTS_NOSTAT))
- p->fts_statp = (struct stat *)ALIGN(p->fts_name + namelen + 2);
+ p->fts_statp = (struct stat *)__builtin_align_up(p->fts_name + namelen + 2, alignof(struct stat));
memcpy(p->fts_name, name, namelen);
return (p);
diff --git a/libc/bionic/libc_init_mte.cpp b/libc/bionic/libc_init_mte.cpp
new file mode 100644
index 0000000..3c8ef7d
--- /dev/null
+++ b/libc/bionic/libc_init_mte.cpp
@@ -0,0 +1,325 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <android/api-level.h>
+#include <elf.h>
+#include <errno.h>
+#include <malloc.h>
+#include <signal.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/auxv.h>
+#include <sys/mman.h>
+
+#include "async_safe/log.h"
+#include "heap_tagging.h"
+#include "libc_init_common.h"
+#include "platform/bionic/macros.h"
+#include "platform/bionic/mte.h"
+#include "platform/bionic/page.h"
+#include "platform/bionic/reserved_signals.h"
+#include "private/KernelArgumentBlock.h"
+#include "private/bionic_asm.h"
+#include "private/bionic_asm_note.h"
+#include "private/bionic_call_ifunc_resolver.h"
+#include "private/bionic_elf_tls.h"
+#include "private/bionic_globals.h"
+#include "private/bionic_tls.h"
+#include "private/elf_note.h"
+#include "pthread_internal.h"
+#include "sys/system_properties.h"
+#include "sysprop_helpers.h"
+
+#ifdef __aarch64__
+extern "C" const char* __gnu_basename(const char* path);
+
+static HeapTaggingLevel __get_memtag_level_from_note(const ElfW(Phdr) * phdr_start, size_t phdr_ct,
+ const ElfW(Addr) load_bias, bool* stack) {
+ const ElfW(Nhdr) * note;
+ const char* desc;
+ if (!__find_elf_note(NT_ANDROID_TYPE_MEMTAG, "Android", phdr_start, phdr_ct, ¬e, &desc,
+ load_bias)) {
+ return M_HEAP_TAGGING_LEVEL_TBI;
+ }
+
+ // Previously (in Android 12), if the note was != 4 bytes, we check-failed
+ // here. Let's be more permissive to allow future expansion.
+ if (note->n_descsz < 4) {
+ async_safe_fatal("unrecognized android.memtag note: n_descsz = %d, expected >= 4",
+ note->n_descsz);
+ }
+
+ // `desc` is always aligned due to ELF requirements, enforced in __find_elf_note().
+ ElfW(Word) note_val = *reinterpret_cast<const ElfW(Word)*>(desc);
+ *stack = (note_val & NT_MEMTAG_STACK) != 0;
+
+ // Warning: In Android 12, any value outside of bits [0..3] resulted in a check-fail.
+ if (!(note_val & (NT_MEMTAG_HEAP | NT_MEMTAG_STACK))) {
+ async_safe_format_log(ANDROID_LOG_INFO, "libc",
+ "unrecognised memtag note_val did not specificy heap or stack: %u",
+ note_val);
+ return M_HEAP_TAGGING_LEVEL_TBI;
+ }
+
+ unsigned mode = note_val & NT_MEMTAG_LEVEL_MASK;
+ switch (mode) {
+ case NT_MEMTAG_LEVEL_NONE:
+ // Note, previously (in Android 12), NT_MEMTAG_LEVEL_NONE was
+ // NT_MEMTAG_LEVEL_DEFAULT, which implied SYNC mode. This was never used
+ // by anyone, but we note it (heh) here for posterity, in case the zero
+ // level becomes meaningful, and binaries with this note can be executed
+ // on Android 12 devices.
+ return M_HEAP_TAGGING_LEVEL_TBI;
+ case NT_MEMTAG_LEVEL_ASYNC:
+ return M_HEAP_TAGGING_LEVEL_ASYNC;
+ case NT_MEMTAG_LEVEL_SYNC:
+ default:
+ // We allow future extensions to specify mode 3 (currently unused), with
+ // the idea that it might be used for ASYMM mode or something else. On
+ // this version of Android, it falls back to SYNC mode.
+ return M_HEAP_TAGGING_LEVEL_SYNC;
+ }
+}
+
+// Returns true if there's an environment setting (either sysprop or env var)
+// that should overwrite the ELF note, and places the equivalent heap tagging
+// level into *level.
+static bool get_environment_memtag_setting(HeapTaggingLevel* level) {
+ static const char kMemtagPrognameSyspropPrefix[] = "arm64.memtag.process.";
+ static const char kMemtagGlobalSysprop[] = "persist.arm64.memtag.default";
+ static const char kMemtagOverrideSyspropPrefix[] =
+ "persist.device_config.memory_safety_native.mode_override.process.";
+
+ const char* progname = __libc_shared_globals()->init_progname;
+ if (progname == nullptr) return false;
+
+ const char* basename = __gnu_basename(progname);
+
+ char options_str[PROP_VALUE_MAX];
+ char sysprop_name[512];
+ async_safe_format_buffer(sysprop_name, sizeof(sysprop_name), "%s%s", kMemtagPrognameSyspropPrefix,
+ basename);
+ char remote_sysprop_name[512];
+ async_safe_format_buffer(remote_sysprop_name, sizeof(remote_sysprop_name), "%s%s",
+ kMemtagOverrideSyspropPrefix, basename);
+ const char* sys_prop_names[] = {sysprop_name, remote_sysprop_name, kMemtagGlobalSysprop};
+
+ if (!get_config_from_env_or_sysprops("MEMTAG_OPTIONS", sys_prop_names, arraysize(sys_prop_names),
+ options_str, sizeof(options_str))) {
+ return false;
+ }
+
+ if (strcmp("sync", options_str) == 0) {
+ *level = M_HEAP_TAGGING_LEVEL_SYNC;
+ } else if (strcmp("async", options_str) == 0) {
+ *level = M_HEAP_TAGGING_LEVEL_ASYNC;
+ } else if (strcmp("off", options_str) == 0) {
+ *level = M_HEAP_TAGGING_LEVEL_TBI;
+ } else {
+ async_safe_format_log(
+ ANDROID_LOG_ERROR, "libc",
+ "unrecognized memtag level: \"%s\" (options are \"sync\", \"async\", or \"off\").",
+ options_str);
+ return false;
+ }
+
+ return true;
+}
+
+// Returns the initial heap tagging level. Note: This function will never return
+// M_HEAP_TAGGING_LEVEL_NONE, if MTE isn't enabled for this process we enable
+// M_HEAP_TAGGING_LEVEL_TBI.
+static HeapTaggingLevel __get_tagging_level(const memtag_dynamic_entries_t* memtag_dynamic_entries,
+ const void* phdr_start, size_t phdr_ct,
+ uintptr_t load_bias, bool* stack) {
+ HeapTaggingLevel level = M_HEAP_TAGGING_LEVEL_TBI;
+
+ // If the dynamic entries exist, use those. Otherwise, fall back to the old
+ // Android note, which is still used for fully static executables. When
+ // -fsanitize=memtag* is used in newer toolchains, currently both the dynamic
+ // entries and the old note are created, but we'd expect to move to just the
+ // dynamic entries for dynamically linked executables in the future. In
+ // addition, there's still some cleanup of the build system (that uses a
+ // manually-constructed note) needed. For more information about the dynamic
+ // entries, see:
+ // https://github.com/ARM-software/abi-aa/blob/main/memtagabielf64/memtagabielf64.rst#dynamic-section
+ if (memtag_dynamic_entries && memtag_dynamic_entries->has_memtag_mode) {
+ switch (memtag_dynamic_entries->memtag_mode) {
+ case 0:
+ level = M_HEAP_TAGGING_LEVEL_SYNC;
+ break;
+ case 1:
+ level = M_HEAP_TAGGING_LEVEL_ASYNC;
+ break;
+ default:
+ async_safe_format_log(ANDROID_LOG_INFO, "libc",
+ "unrecognised DT_AARCH64_MEMTAG_MODE value: %u",
+ memtag_dynamic_entries->memtag_mode);
+ }
+ *stack = memtag_dynamic_entries->memtag_stack;
+ } else {
+ level = __get_memtag_level_from_note(reinterpret_cast<const ElfW(Phdr)*>(phdr_start), phdr_ct,
+ load_bias, stack);
+ }
+
+ // We can't short-circuit the environment override, as `stack` is still inherited from the
+ // binary's settings.
+ get_environment_memtag_setting(&level);
+ return level;
+}
+
+static void __enable_mte_signal_handler(int, siginfo_t* info, void*) {
+ if (info->si_code != SI_TIMER) {
+ async_safe_format_log(ANDROID_LOG_ERROR, "libc", "Got BIONIC_ENABLE_MTE not from SI_TIMER");
+ return;
+ }
+ int tagged_addr_ctrl = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
+ if (tagged_addr_ctrl < 0) {
+ async_safe_fatal("failed to PR_GET_TAGGED_ADDR_CTRL: %m");
+ }
+ if ((tagged_addr_ctrl & PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE) {
+ return;
+ }
+ async_safe_format_log(ANDROID_LOG_INFO, "libc",
+ "Re-enabling MTE, value: %x (tagged_addr_ctrl %lu)",
+ info->si_value.sival_int, info->si_value.sival_int & PR_MTE_TCF_MASK);
+ tagged_addr_ctrl =
+ (tagged_addr_ctrl & ~PR_MTE_TCF_MASK) | (info->si_value.sival_int & PR_MTE_TCF_MASK);
+ if (prctl(PR_SET_TAGGED_ADDR_CTRL, tagged_addr_ctrl, 0, 0, 0) < 0) {
+ async_safe_fatal("failed to PR_SET_TAGGED_ADDR_CTRL %d: %m", tagged_addr_ctrl);
+ }
+}
+
+static int64_t __get_memtag_upgrade_secs() {
+ char* env = getenv("BIONIC_MEMTAG_UPGRADE_SECS");
+ if (!env) return 0;
+ int64_t timed_upgrade = 0;
+ static const char kAppProcessName[] = "app_process64";
+ const char* progname = __libc_shared_globals()->init_progname;
+ progname = progname ? __gnu_basename(progname) : nullptr;
+ // disable timed upgrade for zygote, as the thread spawned will violate the requirement
+ // that it be single-threaded.
+ if (!progname || strncmp(progname, kAppProcessName, sizeof(kAppProcessName)) != 0) {
+ char* endptr;
+ timed_upgrade = strtoll(env, &endptr, 10);
+ if (*endptr != '\0' || timed_upgrade < 0) {
+ async_safe_format_log(ANDROID_LOG_ERROR, "libc",
+ "Invalid value for BIONIC_MEMTAG_UPGRADE_SECS: %s", env);
+ timed_upgrade = 0;
+ }
+ }
+ // Make sure that this does not get passed to potential processes inheriting
+ // this environment.
+ unsetenv("BIONIC_MEMTAG_UPGRADE_SECS");
+ return timed_upgrade;
+}
+
+// Figure out the desired memory tagging mode (sync/async, heap/globals/stack) for this executable.
+// This function is called from the linker before the main executable is relocated.
+__attribute__((no_sanitize("hwaddress", "memtag"))) void __libc_init_mte(
+ const memtag_dynamic_entries_t* memtag_dynamic_entries, const void* phdr_start, size_t phdr_ct,
+ uintptr_t load_bias) {
+ bool memtag_stack = false;
+ HeapTaggingLevel level =
+ __get_tagging_level(memtag_dynamic_entries, phdr_start, phdr_ct, load_bias, &memtag_stack);
+ if (memtag_stack) __libc_shared_globals()->initial_memtag_stack_abi = true;
+
+ if (int64_t timed_upgrade = __get_memtag_upgrade_secs()) {
+ if (level == M_HEAP_TAGGING_LEVEL_ASYNC) {
+ async_safe_format_log(ANDROID_LOG_INFO, "libc",
+ "Attempting timed MTE upgrade from async to sync.");
+ __libc_shared_globals()->heap_tagging_upgrade_timer_sec = timed_upgrade;
+ level = M_HEAP_TAGGING_LEVEL_SYNC;
+ } else if (level != M_HEAP_TAGGING_LEVEL_SYNC) {
+ async_safe_format_log(ANDROID_LOG_ERROR, "libc",
+ "Requested timed MTE upgrade from invalid %s to sync. Ignoring.",
+ DescribeTaggingLevel(level));
+ }
+ }
+ if (level == M_HEAP_TAGGING_LEVEL_SYNC || level == M_HEAP_TAGGING_LEVEL_ASYNC) {
+ unsigned long prctl_arg = PR_TAGGED_ADDR_ENABLE | PR_MTE_TAG_SET_NONZERO;
+ prctl_arg |= (level == M_HEAP_TAGGING_LEVEL_SYNC) ? PR_MTE_TCF_SYNC : PR_MTE_TCF_ASYNC;
+
+ // When entering ASYNC mode, specify that we want to allow upgrading to SYNC by OR'ing in the
+ // SYNC flag. But if the kernel doesn't support specifying multiple TCF modes, fall back to
+ // specifying a single mode.
+ if (prctl(PR_SET_TAGGED_ADDR_CTRL, prctl_arg | PR_MTE_TCF_SYNC, 0, 0, 0) == 0 ||
+ prctl(PR_SET_TAGGED_ADDR_CTRL, prctl_arg, 0, 0, 0) == 0) {
+ __libc_shared_globals()->initial_heap_tagging_level = level;
+
+ struct sigaction action = {};
+ action.sa_flags = SA_SIGINFO | SA_RESTART;
+ action.sa_sigaction = __enable_mte_signal_handler;
+ sigaction(BIONIC_ENABLE_MTE, &action, nullptr);
+ return;
+ }
+ }
+
+ // MTE was either not enabled, or wasn't supported on this device. Try and use
+ // TBI.
+ if (prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0, 0, 0) == 0) {
+ __libc_shared_globals()->initial_heap_tagging_level = M_HEAP_TAGGING_LEVEL_TBI;
+ }
+ // We did not enable MTE, so we do not need to arm the upgrade timer.
+ __libc_shared_globals()->heap_tagging_upgrade_timer_sec = 0;
+}
+
+// Figure out whether we need to map the stack as PROT_MTE.
+// For dynamic executables, this has to be called after loading all
+// DT_NEEDED libraries, in case one of them needs stack MTE.
+__attribute__((no_sanitize("hwaddress", "memtag"))) void __libc_init_mte_stack(void* stack_top) {
+ if (!__libc_shared_globals()->initial_memtag_stack_abi) {
+ return;
+ }
+
+ // Even if the device doesn't support MTE, we have to allocate stack
+ // history buffers for code compiled for stack MTE. That is because the
+ // codegen expects a buffer to be present in TLS_SLOT_STACK_MTE either
+ // way.
+ __get_bionic_tcb()->tls_slot(TLS_SLOT_STACK_MTE) = __allocate_stack_mte_ringbuffer(0, nullptr);
+
+ if (__libc_mte_enabled()) {
+ __libc_shared_globals()->initial_memtag_stack = true;
+ void* pg_start = reinterpret_cast<void*>(page_start(reinterpret_cast<uintptr_t>(stack_top)));
+ if (mprotect(pg_start, page_size(), PROT_READ | PROT_WRITE | PROT_MTE | PROT_GROWSDOWN)) {
+ async_safe_fatal("error: failed to set PROT_MTE on main thread stack: %m");
+ }
+ }
+}
+
+#else // __aarch64__
+void __libc_init_mte(const memtag_dynamic_entries_t*, const void*, size_t, uintptr_t) {}
+void __libc_init_mte_stack(void*) {}
+#endif // __aarch64__
+
+bool __libc_mte_enabled() {
+ HeapTaggingLevel lvl = __libc_shared_globals()->initial_heap_tagging_level;
+ return lvl == M_HEAP_TAGGING_LEVEL_SYNC || lvl == M_HEAP_TAGGING_LEVEL_ASYNC;
+}
diff --git a/libc/bionic/libc_init_static.cpp b/libc/bionic/libc_init_static.cpp
index 7c46113..553741d 100644
--- a/libc/bionic/libc_init_static.cpp
+++ b/libc/bionic/libc_init_static.cpp
@@ -157,260 +157,6 @@
layout.finish_layout();
}
-
-#ifdef __aarch64__
-static HeapTaggingLevel __get_memtag_level_from_note(const ElfW(Phdr) * phdr_start, size_t phdr_ct,
- const ElfW(Addr) load_bias, bool* stack) {
- const ElfW(Nhdr) * note;
- const char* desc;
- if (!__find_elf_note(NT_ANDROID_TYPE_MEMTAG, "Android", phdr_start, phdr_ct, ¬e, &desc,
- load_bias)) {
- return M_HEAP_TAGGING_LEVEL_TBI;
- }
-
- // Previously (in Android 12), if the note was != 4 bytes, we check-failed
- // here. Let's be more permissive to allow future expansion.
- if (note->n_descsz < 4) {
- async_safe_fatal("unrecognized android.memtag note: n_descsz = %d, expected >= 4",
- note->n_descsz);
- }
-
- // `desc` is always aligned due to ELF requirements, enforced in __find_elf_note().
- ElfW(Word) note_val = *reinterpret_cast<const ElfW(Word)*>(desc);
- *stack = (note_val & NT_MEMTAG_STACK) != 0;
-
- // Warning: In Android 12, any value outside of bits [0..3] resulted in a check-fail.
- if (!(note_val & (NT_MEMTAG_HEAP | NT_MEMTAG_STACK))) {
- async_safe_format_log(ANDROID_LOG_INFO, "libc",
- "unrecognised memtag note_val did not specificy heap or stack: %u",
- note_val);
- return M_HEAP_TAGGING_LEVEL_TBI;
- }
-
- unsigned mode = note_val & NT_MEMTAG_LEVEL_MASK;
- switch (mode) {
- case NT_MEMTAG_LEVEL_NONE:
- // Note, previously (in Android 12), NT_MEMTAG_LEVEL_NONE was
- // NT_MEMTAG_LEVEL_DEFAULT, which implied SYNC mode. This was never used
- // by anyone, but we note it (heh) here for posterity, in case the zero
- // level becomes meaningful, and binaries with this note can be executed
- // on Android 12 devices.
- return M_HEAP_TAGGING_LEVEL_TBI;
- case NT_MEMTAG_LEVEL_ASYNC:
- return M_HEAP_TAGGING_LEVEL_ASYNC;
- case NT_MEMTAG_LEVEL_SYNC:
- default:
- // We allow future extensions to specify mode 3 (currently unused), with
- // the idea that it might be used for ASYMM mode or something else. On
- // this version of Android, it falls back to SYNC mode.
- return M_HEAP_TAGGING_LEVEL_SYNC;
- }
-}
-
-// Returns true if there's an environment setting (either sysprop or env var)
-// that should overwrite the ELF note, and places the equivalent heap tagging
-// level into *level.
-static bool get_environment_memtag_setting(HeapTaggingLevel* level) {
- static const char kMemtagPrognameSyspropPrefix[] = "arm64.memtag.process.";
- static const char kMemtagGlobalSysprop[] = "persist.arm64.memtag.default";
- static const char kMemtagOverrideSyspropPrefix[] =
- "persist.device_config.memory_safety_native.mode_override.process.";
-
- const char* progname = __libc_shared_globals()->init_progname;
- if (progname == nullptr) return false;
-
- const char* basename = __gnu_basename(progname);
-
- char options_str[PROP_VALUE_MAX];
- char sysprop_name[512];
- async_safe_format_buffer(sysprop_name, sizeof(sysprop_name), "%s%s", kMemtagPrognameSyspropPrefix,
- basename);
- char remote_sysprop_name[512];
- async_safe_format_buffer(remote_sysprop_name, sizeof(remote_sysprop_name), "%s%s",
- kMemtagOverrideSyspropPrefix, basename);
- const char* sys_prop_names[] = {sysprop_name, remote_sysprop_name, kMemtagGlobalSysprop};
-
- if (!get_config_from_env_or_sysprops("MEMTAG_OPTIONS", sys_prop_names, arraysize(sys_prop_names),
- options_str, sizeof(options_str))) {
- return false;
- }
-
- if (strcmp("sync", options_str) == 0) {
- *level = M_HEAP_TAGGING_LEVEL_SYNC;
- } else if (strcmp("async", options_str) == 0) {
- *level = M_HEAP_TAGGING_LEVEL_ASYNC;
- } else if (strcmp("off", options_str) == 0) {
- *level = M_HEAP_TAGGING_LEVEL_TBI;
- } else {
- async_safe_format_log(
- ANDROID_LOG_ERROR, "libc",
- "unrecognized memtag level: \"%s\" (options are \"sync\", \"async\", or \"off\").",
- options_str);
- return false;
- }
-
- return true;
-}
-
-// Returns the initial heap tagging level. Note: This function will never return
-// M_HEAP_TAGGING_LEVEL_NONE, if MTE isn't enabled for this process we enable
-// M_HEAP_TAGGING_LEVEL_TBI.
-static HeapTaggingLevel __get_tagging_level(const memtag_dynamic_entries_t* memtag_dynamic_entries,
- const void* phdr_start, size_t phdr_ct,
- uintptr_t load_bias, bool* stack) {
- HeapTaggingLevel level = M_HEAP_TAGGING_LEVEL_TBI;
-
- // If the dynamic entries exist, use those. Otherwise, fall back to the old
- // Android note, which is still used for fully static executables. When
- // -fsanitize=memtag* is used in newer toolchains, currently both the dynamic
- // entries and the old note are created, but we'd expect to move to just the
- // dynamic entries for dynamically linked executables in the future. In
- // addition, there's still some cleanup of the build system (that uses a
- // manually-constructed note) needed. For more information about the dynamic
- // entries, see:
- // https://github.com/ARM-software/abi-aa/blob/main/memtagabielf64/memtagabielf64.rst#dynamic-section
- if (memtag_dynamic_entries && memtag_dynamic_entries->has_memtag_mode) {
- switch (memtag_dynamic_entries->memtag_mode) {
- case 0:
- level = M_HEAP_TAGGING_LEVEL_SYNC;
- break;
- case 1:
- level = M_HEAP_TAGGING_LEVEL_ASYNC;
- break;
- default:
- async_safe_format_log(ANDROID_LOG_INFO, "libc",
- "unrecognised DT_AARCH64_MEMTAG_MODE value: %u",
- memtag_dynamic_entries->memtag_mode);
- }
- *stack = memtag_dynamic_entries->memtag_stack;
- } else {
- level = __get_memtag_level_from_note(reinterpret_cast<const ElfW(Phdr)*>(phdr_start), phdr_ct,
- load_bias, stack);
- }
-
- // We can't short-circuit the environment override, as `stack` is still inherited from the
- // binary's settings.
- get_environment_memtag_setting(&level);
- return level;
-}
-
-static void __enable_mte_signal_handler(int, siginfo_t* info, void*) {
- if (info->si_code != SI_TIMER) {
- async_safe_format_log(ANDROID_LOG_ERROR, "libc", "Got BIONIC_ENABLE_MTE not from SI_TIMER");
- return;
- }
- int tagged_addr_ctrl = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
- if (tagged_addr_ctrl < 0) {
- async_safe_fatal("failed to PR_GET_TAGGED_ADDR_CTRL: %m");
- }
- if ((tagged_addr_ctrl & PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE) {
- return;
- }
- async_safe_format_log(ANDROID_LOG_INFO, "libc",
- "Re-enabling MTE, value: %x (tagged_addr_ctrl %lu)",
- info->si_value.sival_int, info->si_value.sival_int & PR_MTE_TCF_MASK);
- tagged_addr_ctrl =
- (tagged_addr_ctrl & ~PR_MTE_TCF_MASK) | (info->si_value.sival_int & PR_MTE_TCF_MASK);
- if (prctl(PR_SET_TAGGED_ADDR_CTRL, tagged_addr_ctrl, 0, 0, 0) < 0) {
- async_safe_fatal("failed to PR_SET_TAGGED_ADDR_CTRL %d: %m", tagged_addr_ctrl);
- }
-}
-
-static int64_t __get_memtag_upgrade_secs() {
- char* env = getenv("BIONIC_MEMTAG_UPGRADE_SECS");
- if (!env) return 0;
- int64_t timed_upgrade = 0;
- static const char kAppProcessName[] = "app_process64";
- const char* progname = __libc_shared_globals()->init_progname;
- progname = progname ? __gnu_basename(progname) : nullptr;
- // disable timed upgrade for zygote, as the thread spawned will violate the requirement
- // that it be single-threaded.
- if (!progname || strncmp(progname, kAppProcessName, sizeof(kAppProcessName)) != 0) {
- char* endptr;
- timed_upgrade = strtoll(env, &endptr, 10);
- if (*endptr != '\0' || timed_upgrade < 0) {
- async_safe_format_log(ANDROID_LOG_ERROR, "libc",
- "Invalid value for BIONIC_MEMTAG_UPGRADE_SECS: %s", env);
- timed_upgrade = 0;
- }
- }
- // Make sure that this does not get passed to potential processes inheriting
- // this environment.
- unsetenv("BIONIC_MEMTAG_UPGRADE_SECS");
- return timed_upgrade;
-}
-
-// Figure out the desired memory tagging mode (sync/async, heap/globals/stack) for this executable.
-// This function is called from the linker before the main executable is relocated.
-__attribute__((no_sanitize("hwaddress", "memtag"))) void __libc_init_mte(
- const memtag_dynamic_entries_t* memtag_dynamic_entries, const void* phdr_start, size_t phdr_ct,
- uintptr_t load_bias, void* stack_top) {
- bool memtag_stack = false;
- HeapTaggingLevel level =
- __get_tagging_level(memtag_dynamic_entries, phdr_start, phdr_ct, load_bias, &memtag_stack);
- // initial_memtag_stack is used by the linker (in linker.cpp) to communicate than any library
- // linked by this executable enables memtag-stack.
- // memtag_stack is also set for static executables if they request memtag stack via the note,
- // in which case it will differ from initial_memtag_stack.
- if (__libc_shared_globals()->initial_memtag_stack || memtag_stack) {
- memtag_stack = true;
- __libc_shared_globals()->initial_memtag_stack_abi = true;
- __get_bionic_tcb()->tls_slot(TLS_SLOT_STACK_MTE) = __allocate_stack_mte_ringbuffer(0, nullptr);
- }
- if (int64_t timed_upgrade = __get_memtag_upgrade_secs()) {
- if (level == M_HEAP_TAGGING_LEVEL_ASYNC) {
- async_safe_format_log(ANDROID_LOG_INFO, "libc",
- "Attempting timed MTE upgrade from async to sync.");
- __libc_shared_globals()->heap_tagging_upgrade_timer_sec = timed_upgrade;
- level = M_HEAP_TAGGING_LEVEL_SYNC;
- } else if (level != M_HEAP_TAGGING_LEVEL_SYNC) {
- async_safe_format_log(
- ANDROID_LOG_ERROR, "libc",
- "Requested timed MTE upgrade from invalid %s to sync. Ignoring.",
- DescribeTaggingLevel(level));
- }
- }
- if (level == M_HEAP_TAGGING_LEVEL_SYNC || level == M_HEAP_TAGGING_LEVEL_ASYNC) {
- unsigned long prctl_arg = PR_TAGGED_ADDR_ENABLE | PR_MTE_TAG_SET_NONZERO;
- prctl_arg |= (level == M_HEAP_TAGGING_LEVEL_SYNC) ? PR_MTE_TCF_SYNC : PR_MTE_TCF_ASYNC;
-
- // When entering ASYNC mode, specify that we want to allow upgrading to SYNC by OR'ing in the
- // SYNC flag. But if the kernel doesn't support specifying multiple TCF modes, fall back to
- // specifying a single mode.
- if (prctl(PR_SET_TAGGED_ADDR_CTRL, prctl_arg | PR_MTE_TCF_SYNC, 0, 0, 0) == 0 ||
- prctl(PR_SET_TAGGED_ADDR_CTRL, prctl_arg, 0, 0, 0) == 0) {
- __libc_shared_globals()->initial_heap_tagging_level = level;
- __libc_shared_globals()->initial_memtag_stack = memtag_stack;
-
- if (memtag_stack) {
- void* pg_start =
- reinterpret_cast<void*>(page_start(reinterpret_cast<uintptr_t>(stack_top)));
- if (mprotect(pg_start, page_size(), PROT_READ | PROT_WRITE | PROT_MTE | PROT_GROWSDOWN)) {
- async_safe_fatal("error: failed to set PROT_MTE on main thread stack: %m");
- }
- }
- struct sigaction action = {};
- action.sa_flags = SA_SIGINFO | SA_RESTART;
- action.sa_sigaction = __enable_mte_signal_handler;
- sigaction(BIONIC_ENABLE_MTE, &action, nullptr);
- return;
- }
- }
-
- // MTE was either not enabled, or wasn't supported on this device. Try and use
- // TBI.
- if (prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0, 0, 0) == 0) {
- __libc_shared_globals()->initial_heap_tagging_level = M_HEAP_TAGGING_LEVEL_TBI;
- }
- // We did not enable MTE, so we do not need to arm the upgrade timer.
- __libc_shared_globals()->heap_tagging_upgrade_timer_sec = 0;
- // We also didn't enable memtag_stack.
- __libc_shared_globals()->initial_memtag_stack = false;
-}
-#else // __aarch64__
-void __libc_init_mte(const memtag_dynamic_entries_t*, const void*, size_t, uintptr_t, void*) {}
-#endif // __aarch64__
-
void __libc_init_profiling_handlers() {
// The dynamic variant of this function is more interesting, but this
// at least ensures that static binaries aren't killed by the kernel's
@@ -436,7 +182,8 @@
__libc_init_common();
__libc_init_mte(/*memtag_dynamic_entries=*/nullptr,
reinterpret_cast<ElfW(Phdr)*>(getauxval(AT_PHDR)), getauxval(AT_PHNUM),
- /*load_bias = */ 0, /*stack_top = */ raw_args);
+ /*load_bias = */ 0);
+ __libc_init_mte_stack(/*stack_top = */ raw_args);
__libc_init_scudo();
__libc_init_profiling_handlers();
__libc_init_fork_handler();
diff --git a/libc/bionic/stdlib_l.cpp b/libc/bionic/stdlib_l.cpp
index a636d08..58a9079 100644
--- a/libc/bionic/stdlib_l.cpp
+++ b/libc/bionic/stdlib_l.cpp
@@ -26,17 +26,11 @@
* SUCH DAMAGE.
*/
+#define __BIONIC_STDLIB_INLINE /* Out of line. */
#include <stdlib.h>
-#include <xlocale.h>
+#include <bits/stdlib_inlines.h>
-double strtod_l(const char* s, char** end_ptr, locale_t) {
- return strtod(s, end_ptr);
-}
-
-float strtof_l(const char* s, char** end_ptr, locale_t) {
- return strtof(s, end_ptr);
-}
-
+// strtold_l was introduced in API level 21, so it isn't polyfilled any more.
long double strtold_l(const char* s, char** end_ptr, locale_t) {
return strtold(s, end_ptr);
}
diff --git a/libc/dns/net/gethnamaddr.c b/libc/dns/net/gethnamaddr.c
index add124f..1ffabfa 100644
--- a/libc/dns/net/gethnamaddr.c
+++ b/libc/dns/net/gethnamaddr.c
@@ -495,7 +495,7 @@
*he = NO_RECOVERY;
return NULL;
success:
- bp = (char *)ALIGN(bp);
+ bp = __builtin_align_up(bp, sizeof(uintptr_t));
n = (int)(ap - aliases);
qlen = (n + 1) * sizeof(*hent->h_aliases);
if ((size_t)(ep - bp) < qlen)
@@ -616,7 +616,7 @@
}
// Fix alignment after variable-length data.
- ptr = (char*)ALIGN(ptr);
+ ptr = __builtin_align_up(ptr, sizeof(uintptr_t));
int aliases_len = ((int)(aliases - aliases_ptrs) + 1) * sizeof(*hp->h_aliases);
if (ptr + aliases_len > hbuf_end) {
@@ -653,7 +653,7 @@
}
// Fix alignment after variable-length data.
- ptr = (char*)ALIGN(ptr);
+ ptr = __builtin_align_up(ptr, sizeof(uintptr_t));
int addrs_len = ((int)(addr_p - addr_ptrs) + 1) * sizeof(*hp->h_addr_list);
if (ptr + addrs_len > hbuf_end) {
diff --git a/libc/dns/net/sethostent.c b/libc/dns/net/sethostent.c
index 5c4bdb5..8ea4315 100644
--- a/libc/dns/net/sethostent.c
+++ b/libc/dns/net/sethostent.c
@@ -198,7 +198,7 @@
HENT_SCOPY(aliases[anum], hp->h_aliases[anum],
ptr, len);
}
- ptr = (void *)ALIGN(ptr);
+ ptr = __builtin_align_up(ptr, sizeof(uintptr_t));
if ((size_t)(ptr - buf) >= info->buflen)
goto nospc;
}
diff --git a/libc/execinfo/include/execinfo.h b/libc/execinfo/include/execinfo.h
index e092c00..c8f9e21 100644
--- a/libc/execinfo/include/execinfo.h
+++ b/libc/execinfo/include/execinfo.h
@@ -30,8 +30,10 @@
/*
* This file is exported as part of libexecinfo for use with musl, which doesn't
- * define __INTRODUCED_IN. Stub it out.
+ * define __INTRODUCED_IN or __BIONIC_AVAILABILITY_GUARD. Stub them out.
*/
#define __INTRODUCED_IN(x)
+#define __BIONIC_AVAILABILITY_GUARD(x) 1
#include <bionic/execinfo.h>
+#undef __BIONIC_AVAILABILITY_GUARD
#undef __INTRODUCED_IN
diff --git a/libc/include/android/api-level.h b/libc/include/android/api-level.h
index 1bde3a5..c9536c1 100644
--- a/libc/include/android/api-level.h
+++ b/libc/include/android/api-level.h
@@ -189,7 +189,11 @@
*
* Available since API level 24.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
int android_get_application_target_sdk_version() __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
#if __ANDROID_API__ < 29
diff --git a/libc/include/android/crash_detail.h b/libc/include/android/crash_detail.h
index 946a3ab..fd1312a 100644
--- a/libc/include/android/crash_detail.h
+++ b/libc/include/android/crash_detail.h
@@ -33,9 +33,10 @@
* @brief Attach extra information to android crashes.
*/
-#include <stddef.h>
#include <sys/cdefs.h>
+#include <stddef.h>
+
__BEGIN_DECLS
typedef struct crash_detail_t crash_detail_t;
@@ -79,6 +80,8 @@
*
* \return a handle to the extra crash detail.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(35)
crash_detail_t* _Nullable android_crash_detail_register(
const void* _Nonnull name, size_t name_size, const void* _Nullable data, size_t data_size) __INTRODUCED_IN(35);
@@ -122,5 +125,7 @@
* \param name_size number of bytes of the buffer pointed to by name
*/
void android_crash_detail_replace_name(crash_detail_t* _Nonnull crash_detail, const void* _Nonnull name, size_t name_size) __INTRODUCED_IN(35);
+#endif /* __BIONIC_AVAILABILITY_GUARD(35) */
+
__END_DECLS
diff --git a/libc/include/android/dlext.h b/libc/include/android/dlext.h
index 842ceea..d8d2752 100644
--- a/libc/include/android/dlext.h
+++ b/libc/include/android/dlext.h
@@ -16,10 +16,11 @@
#pragma once
+#include <sys/cdefs.h>
+
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
-#include <sys/cdefs.h>
#include <sys/types.h> /* for off64_t */
/**
diff --git a/libc/include/android/fdsan.h b/libc/include/android/fdsan.h
index 4540498..a04fc7e 100644
--- a/libc/include/android/fdsan.h
+++ b/libc/include/android/fdsan.h
@@ -28,9 +28,10 @@
#pragma once
+#include <sys/cdefs.h>
+
#include <stdbool.h>
#include <stdint.h>
-#include <sys/cdefs.h>
__BEGIN_DECLS
@@ -134,6 +135,8 @@
/*
* Create an owner tag with the specified type and least significant 56 bits of tag.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(29)
uint64_t android_fdsan_create_owner_tag(enum android_fdsan_owner_type type, uint64_t tag) __INTRODUCED_IN(29) __attribute__((__weak__));
/*
@@ -168,6 +171,8 @@
* Get an owner tag's value, with the type masked off.
*/
uint64_t android_fdsan_get_tag_value(uint64_t tag) __INTRODUCED_IN(29);
+#endif /* __BIONIC_AVAILABILITY_GUARD(29) */
+
enum android_fdsan_error_level {
// No errors.
@@ -186,6 +191,8 @@
/*
* Get the error level.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(29)
enum android_fdsan_error_level android_fdsan_get_error_level() __INTRODUCED_IN(29) __attribute__((__weak__));
/*
@@ -203,9 +210,15 @@
* (e.g. postfork).
*/
enum android_fdsan_error_level android_fdsan_set_error_level(enum android_fdsan_error_level new_level) __INTRODUCED_IN(29) __attribute__((__weak__));
+#endif /* __BIONIC_AVAILABILITY_GUARD(29) */
+
/*
* Set the error level to the global setting if available, or a default value.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(30)
enum android_fdsan_error_level android_fdsan_set_error_level_from_property(enum android_fdsan_error_level default_level) __INTRODUCED_IN(30) __attribute__((__weak__));
+#endif /* __BIONIC_AVAILABILITY_GUARD(30) */
+
__END_DECLS
diff --git a/libc/include/android/legacy_stdlib_inlines.h b/libc/include/android/legacy_stdlib_inlines.h
index a5a07ef..d228e67 100644
--- a/libc/include/android/legacy_stdlib_inlines.h
+++ b/libc/include/android/legacy_stdlib_inlines.h
@@ -30,22 +30,9 @@
#include <sys/cdefs.h>
-
#if __ANDROID_API__ < 26
-#include <stdlib.h>
-#include <xlocale.h>
-
-__BEGIN_DECLS
-
-static __inline double strtod_l(const char* _Nonnull __s, char* _Nullable * _Nullable __end_ptr, locale_t _Nonnull __l) {
- return strtod(__s, __end_ptr);
-}
-
-static __inline float strtof_l(const char* _Nonnull __s, char* _Nullable * _Nullable __end_ptr, locale_t _Nonnull __l) {
- return strtof(__s, __end_ptr);
-}
-
-__END_DECLS
+#define __BIONIC_THREADS_INLINE static __inline
+#include <bits/stdlib_inlines.h>
#endif
diff --git a/libc/include/android/set_abort_message.h b/libc/include/android/set_abort_message.h
index a778057..6ad5678 100644
--- a/libc/include/android/set_abort_message.h
+++ b/libc/include/android/set_abort_message.h
@@ -33,10 +33,11 @@
* @brief The android_set_abort_message() function.
*/
+#include <sys/cdefs.h>
+
#include <stddef.h>
#include <stdint.h>
#include <string.h>
-#include <sys/cdefs.h>
__BEGIN_DECLS
diff --git a/libc/include/android/versioning.h b/libc/include/android/versioning.h
index fe9264d..1676a72 100644
--- a/libc/include/android/versioning.h
+++ b/libc/include/android/versioning.h
@@ -32,6 +32,11 @@
// load even on systems too old to contain the API, but calls must be guarded
// with `__builtin_available(android api_level, *)` to avoid segfaults.
#define __BIONIC_AVAILABILITY(__what, ...) __attribute__((__availability__(android,__what __VA_OPT__(,) __VA_ARGS__)))
+
+// When the caller is using weak API references, we should expose the decls for
+// APIs which are not available in the caller's minSdkVersion, otherwise there's
+// no way to take advantage of the weak references.
+#define __BIONIC_AVAILABILITY_GUARD(api_level) 1
#else
// The 'strict' flag is required for NDK clients where the code was not written
// to handle the case where the API was available at build-time but not at
@@ -40,9 +45,25 @@
// compile in this mode (or worse, if the build doesn't use
// -Werror=unguarded-availability, it would build but crash at runtime).
#define __BIONIC_AVAILABILITY(__what, ...) __attribute__((__availability__(android,strict,__what __VA_OPT__(,) __VA_ARGS__)))
+
+// When the caller is using strict API references, we hide APIs which are not
+// available in the caller's minSdkVersion. This is a bionic-only deviation in
+// behavior from the rest of the NDK headers, but it's necessary to maintain
+// source compatibility with 3p libraries that either can't correctly detect API
+// availability (either incorrectly detecting as always-available or as
+// never-available, but neither is true), or define their own polyfills which
+// conflict with our declarations.
+//
+// https://github.com/android/ndk/issues/2081
+#define __BIONIC_AVAILABILITY_GUARD(api_level) (__ANDROID_MIN_SDK_VERSION__ >= (api_level))
#endif
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wc23-extensions"
+// Passing no argument for the '...' parameter of a variadic macro is a C23 extension
#define __INTRODUCED_IN(api_level) __BIONIC_AVAILABILITY(introduced=api_level)
+#pragma clang diagnostic pop
+
#define __DEPRECATED_IN(api_level, msg) __BIONIC_AVAILABILITY(deprecated=api_level, message=msg)
#define __REMOVED_IN(api_level, msg) __BIONIC_AVAILABILITY(obsoleted=api_level, message=msg)
@@ -60,8 +81,9 @@
#define __INTRODUCED_IN_64(api_level) __BIONIC_AVAILABILITY(introduced=api_level)
#endif
-// Vendor modules do not follow SDK versioning. Ignore NDK guards for vendor modules.
-#if defined(__ANDROID_VENDOR__)
+// Vendor and product modules do not follow SDK versioning. Ignore NDK guards for these modules.
+#if defined(__ANDROID_VNDK__)
#undef __BIONIC_AVAILABILITY
#define __BIONIC_AVAILABILITY(api_level, ...)
-#endif // defined(__ANDROID_VENDOR__)
+#define __BIONIC_AVAILABILITY_GUARD(api_level) 1
+#endif // defined(__ANDROID_VNDK__)
diff --git a/libc/include/arpa/ftp.h b/libc/include/arpa/ftp.h
index 081c037..fecbf7f 100644
--- a/libc/include/arpa/ftp.h
+++ b/libc/include/arpa/ftp.h
@@ -34,6 +34,8 @@
#ifndef _ARPA_FTP_H_
#define _ARPA_FTP_H_
+#include <sys/cdefs.h>
+
/* Definitions for FTP; see RFC-765. */
/*
diff --git a/libc/include/arpa/inet.h b/libc/include/arpa/inet.h
index f00f2c1..ce9dd93 100644
--- a/libc/include/arpa/inet.h
+++ b/libc/include/arpa/inet.h
@@ -29,9 +29,10 @@
#ifndef _ARPA_INET_H_
#define _ARPA_INET_H_
+#include <sys/cdefs.h>
+
#include <netinet/in.h>
#include <stdint.h>
-#include <sys/cdefs.h>
#include <sys/types.h>
__BEGIN_DECLS
diff --git a/libc/include/arpa/nameser.h b/libc/include/arpa/nameser.h
index 97109ee..3e0025e 100644
--- a/libc/include/arpa/nameser.h
+++ b/libc/include/arpa/nameser.h
@@ -55,9 +55,10 @@
#define BIND_4_COMPAT
-#include <sys/types.h>
#include <sys/cdefs.h>
+#include <sys/types.h>
+
/*
* Revision information. This is the release date in YYYYMMDD format.
* It can change every day so the right thing to do with it is use it
@@ -547,6 +548,8 @@
#define ns_sprintrrf __ns_sprintrrf
#endif
+
+#if __BIONIC_AVAILABILITY_GUARD(22)
int ns_msg_getflag(ns_msg __handle, int __flag) __INTRODUCED_IN(22);
uint16_t ns_get16(const u_char* _Nonnull __src) __INTRODUCED_IN(22);
uint32_t ns_get32(const u_char* _Nonnull __src) __INTRODUCED_IN(22);
@@ -570,6 +573,8 @@
int ns_makecanon(const char* _Nonnull __src, char* _Nonnull __dst, size_t __dst_size) __INTRODUCED_IN(22);
int ns_samename(const char* _Nonnull __lhs, const char* _Nonnull __rhs) __INTRODUCED_IN(22);
+#endif /* __BIONIC_AVAILABILITY_GUARD(22) */
+
__END_DECLS
diff --git a/libc/include/arpa/nameser_compat.h b/libc/include/arpa/nameser_compat.h
index e4e9335..027e5ca 100644
--- a/libc/include/arpa/nameser_compat.h
+++ b/libc/include/arpa/nameser_compat.h
@@ -40,9 +40,10 @@
#ifndef _ARPA_NAMESER_COMPAT_
#define _ARPA_NAMESER_COMPAT_
-#include <endian.h>
#include <sys/cdefs.h>
+#include <endian.h>
+
#define __BIND 19950621 /* (DEAD) interface version stamp. */
/*
diff --git a/libc/include/arpa/telnet.h b/libc/include/arpa/telnet.h
index 758e9b8..30d8f21 100644
--- a/libc/include/arpa/telnet.h
+++ b/libc/include/arpa/telnet.h
@@ -33,6 +33,8 @@
#ifndef _ARPA_TELNET_H_
#define _ARPA_TELNET_H_
+#include <sys/cdefs.h>
+
/*
* Definitions for the TELNET protocol.
*/
diff --git a/libc/include/bits/bionic_multibyte_result.h b/libc/include/bits/bionic_multibyte_result.h
index 0d5cf21..930e67c 100644
--- a/libc/include/bits/bionic_multibyte_result.h
+++ b/libc/include/bits/bionic_multibyte_result.h
@@ -34,9 +34,10 @@
* conversion APIs defined by C.
*/
-#include <stddef.h>
#include <sys/cdefs.h>
+#include <stddef.h>
+
__BEGIN_DECLS
/**
diff --git a/libc/include/bits/fortify/poll.h b/libc/include/bits/fortify/poll.h
index f2e27d7..1b4a5bf 100644
--- a/libc/include/bits/fortify/poll.h
+++ b/libc/include/bits/fortify/poll.h
@@ -30,9 +30,17 @@
#error "Never include this file directly; instead, include <poll.h>"
#endif
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int __poll_chk(struct pollfd* _Nullable, nfds_t, int, size_t) __INTRODUCED_IN(23);
int __ppoll_chk(struct pollfd* _Nullable, nfds_t, const struct timespec* _Nullable, const sigset_t* _Nullable, size_t) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int __ppoll64_chk(struct pollfd* _Nullable, nfds_t, const struct timespec* _Nullable, const sigset64_t* _Nullable, size_t) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
#if defined(__BIONIC_FORTIFY)
#define __bos_fd_count_trivially_safe(bos_val, fds, fd_count) \
diff --git a/libc/include/bits/fortify/socket.h b/libc/include/bits/fortify/socket.h
index 1c3605b..bd626f9 100644
--- a/libc/include/bits/fortify/socket.h
+++ b/libc/include/bits/fortify/socket.h
@@ -30,7 +30,11 @@
#error "Never include this file directly; instead, include <sys/socket.h>"
#endif
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
ssize_t __sendto_chk(int, const void* _Nonnull, size_t, size_t, int, const struct sockaddr* _Nullable, socklen_t) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
ssize_t __recvfrom_chk(int, void* _Nullable, size_t, size_t, int, struct sockaddr* _Nullable, socklen_t* _Nullable);
#if defined(__BIONIC_FORTIFY)
diff --git a/libc/include/bits/fortify/stdio.h b/libc/include/bits/fortify/stdio.h
index e4607e0..f9faeba 100644
--- a/libc/include/bits/fortify/stdio.h
+++ b/libc/include/bits/fortify/stdio.h
@@ -31,8 +31,12 @@
#endif
char* _Nullable __fgets_chk(char* _Nonnull, int, FILE* _Nonnull, size_t);
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
size_t __fread_chk(void* _Nonnull, size_t, size_t, FILE* _Nonnull, size_t) __INTRODUCED_IN(24);
size_t __fwrite_chk(const void* _Nonnull, size_t, size_t, FILE* _Nonnull, size_t) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
#if defined(__BIONIC_FORTIFY) && !defined(__BIONIC_NO_STDIO_FORTIFY)
diff --git a/libc/include/bits/fortify/string.h b/libc/include/bits/fortify/string.h
index 4d32b04..6f0ee4a 100644
--- a/libc/include/bits/fortify/string.h
+++ b/libc/include/bits/fortify/string.h
@@ -30,8 +30,12 @@
#error "Never include this file directly; instead, include <string.h>"
#endif
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
void* _Nullable __memchr_chk(const void* _Nonnull, int, size_t, size_t) __INTRODUCED_IN(23);
void* _Nullable __memrchr_chk(const void* _Nonnull, int, size_t, size_t) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
char* _Nonnull __stpncpy_chk2(char* _Nonnull, const char* _Nonnull, size_t, size_t, size_t);
char* _Nonnull __strncpy_chk2(char* _Nonnull, const char* _Nonnull, size_t, size_t, size_t);
size_t __strlcpy_chk(char* _Nonnull, const char* _Nonnull, size_t, size_t);
@@ -220,8 +224,13 @@
}
#if __BIONIC_FORTIFY_RUNTIME_CHECKS_ENABLED
-__BIONIC_FORTIFY_INLINE
-size_t strlen(const char* _Nonnull const s __pass_object_size0) __overloadable {
+/*
+ * Clang, when parsing C, can fold strlen to a constant without LLVM's help.
+ * This doesn't apply to overloads of strlen, so write this differently. We
+ * can't use `__pass_object_size0` here, but that's fine: it doesn't help much
+ * on __always_inline functions.
+ */
+extern __always_inline __inline__ __attribute__((gnu_inline)) size_t strlen(const char* _Nonnull s) {
return __strlen_chk(s, __bos0(s));
}
#endif
diff --git a/libc/include/bits/fortify/unistd.h b/libc/include/bits/fortify/unistd.h
index 7eda1a6..9acb942 100644
--- a/libc/include/bits/fortify/unistd.h
+++ b/libc/include/bits/fortify/unistd.h
@@ -29,24 +29,52 @@
#error "Never include this file directly; instead, include <unistd.h>"
#endif
-char* _Nullable __getcwd_chk(char* _Nullable, size_t, size_t) __INTRODUCED_IN(24);
+#if __BIONIC_AVAILABILITY_GUARD(24)
+char* _Nullable __getcwd_chk(char* _Nullable, size_t, size_t) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
+
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
ssize_t __pread_chk(int, void* _Nonnull, size_t, off_t, size_t) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
ssize_t __pread_real(int, void* _Nonnull, size_t, off_t) __RENAME(pread);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
ssize_t __pread64_chk(int, void* _Nonnull, size_t, off64_t, size_t) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
ssize_t __pread64_real(int, void* _Nonnull, size_t, off64_t) __RENAME(pread64);
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
ssize_t __pwrite_chk(int, const void* _Nonnull, size_t, off_t, size_t) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
ssize_t __pwrite_real(int, const void* _Nonnull, size_t, off_t) __RENAME(pwrite);
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
ssize_t __pwrite64_chk(int, const void* _Nonnull, size_t, off64_t, size_t) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
ssize_t __pwrite64_real(int, const void* _Nonnull, size_t, off64_t) __RENAME(pwrite64);
ssize_t __read_chk(int, void* __BIONIC_COMPLICATED_NULLNESS, size_t, size_t);
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
ssize_t __write_chk(int, const void* __BIONIC_COMPLICATED_NULLNESS, size_t, size_t) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
ssize_t __readlink_chk(const char* _Nonnull, char* _Nonnull, size_t, size_t) __INTRODUCED_IN(23);
ssize_t __readlinkat_chk(int dirfd, const char* _Nonnull, char* _Nonnull, size_t, size_t) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
#if defined(__BIONIC_FORTIFY)
diff --git a/libc/include/bits/getentropy.h b/libc/include/bits/getentropy.h
index 98d8879..c878470 100644
--- a/libc/include/bits/getentropy.h
+++ b/libc/include/bits/getentropy.h
@@ -48,6 +48,10 @@
*
* See also arc4random_buf() which is available in all API levels.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
__nodiscard int getentropy(void* _Nonnull __buffer, size_t __buffer_size) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
__END_DECLS
diff --git a/libc/include/bits/glibc-syscalls.h b/libc/include/bits/glibc-syscalls.h
index 7b171e8..8c5a91d 100644
--- a/libc/include/bits/glibc-syscalls.h
+++ b/libc/include/bits/glibc-syscalls.h
@@ -36,9 +36,6 @@
#if defined(__NR_arch_prctl)
#define SYS_arch_prctl __NR_arch_prctl
#endif
-#if defined(__NR_arch_specific_syscall)
- #define SYS_arch_specific_syscall __NR_arch_specific_syscall
-#endif
#if defined(__NR_arm_fadvise64_64)
#define SYS_arm_fadvise64_64 __NR_arm_fadvise64_64
#endif
@@ -1272,9 +1269,6 @@
#if defined(__NR_syscall)
#define SYS_syscall __NR_syscall
#endif
-#if defined(__NR_syscalls)
- #define SYS_syscalls __NR_syscalls
-#endif
#if defined(__NR_sysfs)
#define SYS_sysfs __NR_sysfs
#endif
@@ -1371,6 +1365,9 @@
#if defined(__NR_unshare)
#define SYS_unshare __NR_unshare
#endif
+#if defined(__NR_uretprobe)
+ #define SYS_uretprobe __NR_uretprobe
+#endif
#if defined(__NR_uselib)
#define SYS_uselib __NR_uselib
#endif
diff --git a/libc/include/bits/lockf.h b/libc/include/bits/lockf.h
index 195b34a..8f922b9 100644
--- a/libc/include/bits/lockf.h
+++ b/libc/include/bits/lockf.h
@@ -56,6 +56,8 @@
*
* See also flock().
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
int lockf(int __fd, int __op, off_t __length) __RENAME_IF_FILE_OFFSET64(lockf64) __INTRODUCED_IN(24);
/**
@@ -63,5 +65,7 @@
* even from a 32-bit process without `_FILE_OFFSET_BITS=64`.
*/
int lockf64(int __fd, int __op, off64_t __length) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
__END_DECLS
diff --git a/libc/include/bits/seek_constants.h b/libc/include/bits/seek_constants.h
index bfc02a8..a4fffb2 100644
--- a/libc/include/bits/seek_constants.h
+++ b/libc/include/bits/seek_constants.h
@@ -33,6 +33,8 @@
* @brief The `SEEK_` constants.
*/
+#include <sys/cdefs.h>
+
/** Seek to an absolute offset. */
#define SEEK_SET 0
/** Seek relative to the current offset. */
diff --git a/libc/include/bits/stdatomic.h b/libc/include/bits/stdatomic.h
index c74eafd..ebdc9e5 100644
--- a/libc/include/bits/stdatomic.h
+++ b/libc/include/bits/stdatomic.h
@@ -134,6 +134,8 @@
memory_order_seq_cst = __ATOMIC_SEQ_CST
} memory_order;
+#define kill_dependency(y) (y)
+
/*
* 7.17.4 Fences.
*/
diff --git a/libc/include/bits/stdlib_inlines.h b/libc/include/bits/stdlib_inlines.h
new file mode 100644
index 0000000..fffca19
--- /dev/null
+++ b/libc/include/bits/stdlib_inlines.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <xlocale.h>
+#include <sys/cdefs.h>
+
+#if !defined(__BIONIC_STDLIB_INLINE)
+#define __BIONIC_STDLIB_INLINE static __inline
+#endif
+
+__BEGIN_DECLS
+
+__BIONIC_STDLIB_INLINE double strtod_l(const char* _Nonnull __s, char* _Nullable * _Nullable __end_ptr, locale_t _Nonnull __l) {
+ return strtod(__s, __end_ptr);
+}
+
+__BIONIC_STDLIB_INLINE float strtof_l(const char* _Nonnull __s, char* _Nullable * _Nullable __end_ptr, locale_t _Nonnull __l) {
+ return strtof(__s, __end_ptr);
+}
+
+__END_DECLS
diff --git a/libc/include/bits/strcasecmp.h b/libc/include/bits/strcasecmp.h
index be910ad..d76cec9 100644
--- a/libc/include/bits/strcasecmp.h
+++ b/libc/include/bits/strcasecmp.h
@@ -51,7 +51,11 @@
/**
* Like strcasecmp() but taking a `locale_t`.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int strcasecmp_l(const char* _Nonnull __s1, const char* _Nonnull __s2, locale_t _Nonnull __l) __attribute_pure__ __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
/**
* [strncasecmp(3)](https://man7.org/linux/man-pages/man3/strncasecmp.3.html) compares the first
@@ -66,6 +70,10 @@
/**
* Like strncasecmp() but taking a `locale_t`.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int strncasecmp_l(const char* _Nonnull __s1, const char* _Nonnull __s2, size_t __n, locale_t _Nonnull __l) __attribute_pure__ __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
__END_DECLS
diff --git a/libc/include/bits/swab.h b/libc/include/bits/swab.h
index 9591c2e..da2865a 100644
--- a/libc/include/bits/swab.h
+++ b/libc/include/bits/swab.h
@@ -28,8 +28,9 @@
#pragma once
-#include <stdint.h>
#include <sys/cdefs.h>
+
+#include <stdint.h>
#include <sys/types.h>
#if !defined(__BIONIC_SWAB_INLINE)
diff --git a/libc/include/bits/termios_inlines.h b/libc/include/bits/termios_inlines.h
index a884b59..bb04e4d 100644
--- a/libc/include/bits/termios_inlines.h
+++ b/libc/include/bits/termios_inlines.h
@@ -29,8 +29,9 @@
#ifndef _BITS_TERMIOS_INLINES_H_
#define _BITS_TERMIOS_INLINES_H_
-#include <errno.h>
#include <sys/cdefs.h>
+
+#include <errno.h>
#include <sys/ioctl.h>
#include <sys/types.h>
diff --git a/libc/include/bits/termios_winsize_inlines.h b/libc/include/bits/termios_winsize_inlines.h
index ae246e4..86777b0 100644
--- a/libc/include/bits/termios_winsize_inlines.h
+++ b/libc/include/bits/termios_winsize_inlines.h
@@ -28,8 +28,9 @@
#pragma once
-#include <errno.h>
#include <sys/cdefs.h>
+
+#include <errno.h>
#include <sys/ioctl.h>
#include <sys/types.h>
diff --git a/libc/include/bits/threads_inlines.h b/libc/include/bits/threads_inlines.h
index 05b785a..ab294c1 100644
--- a/libc/include/bits/threads_inlines.h
+++ b/libc/include/bits/threads_inlines.h
@@ -28,6 +28,8 @@
#pragma once
+#include <sys/cdefs.h>
+
#include <threads.h>
#include <errno.h>
diff --git a/libc/include/bits/wctype.h b/libc/include/bits/wctype.h
index 13a4254..d0cffec 100644
--- a/libc/include/bits/wctype.h
+++ b/libc/include/bits/wctype.h
@@ -58,8 +58,12 @@
int iswctype(wint_t __wc, wctype_t __type);
typedef const void* wctrans_t;
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
wint_t towctrans(wint_t __wc, wctrans_t _Nonnull __transform) __INTRODUCED_IN(26);
wctrans_t _Nullable wctrans(const char* _Nonnull __name) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
__END_DECLS
diff --git a/libc/include/complex.h b/libc/include/complex.h
index f205abd..1115862 100644
--- a/libc/include/complex.h
+++ b/libc/include/complex.h
@@ -53,76 +53,190 @@
/* 7.3.5 Trigonometric functions */
/* 7.3.5.1 The cacos functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex cacos(double complex __z) __INTRODUCED_IN(23);
float complex cacosf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex cacosl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.5.2 The casin functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex casin(double complex __z) __INTRODUCED_IN(23);
float complex casinf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex casinl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.5.1 The catan functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex catan(double complex __z) __INTRODUCED_IN(23);
float complex catanf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex catanl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.5.1 The ccos functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex ccos(double complex __z) __INTRODUCED_IN(23);
float complex ccosf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex ccosl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.5.1 The csin functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex csin(double complex __z) __INTRODUCED_IN(23);
float complex csinf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex csinl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.5.1 The ctan functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex ctan(double complex __z) __INTRODUCED_IN(23);
float complex ctanf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex ctanl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.6 Hyperbolic functions */
/* 7.3.6.1 The cacosh functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex cacosh(double complex __z) __INTRODUCED_IN(23);
float complex cacoshf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex cacoshl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.6.2 The casinh functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex casinh(double complex __z) __INTRODUCED_IN(23);
float complex casinhf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex casinhl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.6.3 The catanh functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex catanh(double complex __z) __INTRODUCED_IN(23);
float complex catanhf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex catanhl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.6.4 The ccosh functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex ccosh(double complex __z) __INTRODUCED_IN(23);
float complex ccoshf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex ccoshl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.6.5 The csinh functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex csinh(double complex __z) __INTRODUCED_IN(23);
float complex csinhf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex csinhl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.6.6 The ctanh functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex ctanh(double complex __z) __INTRODUCED_IN(23);
float complex ctanhf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex ctanhl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.7 Exponential and logarithmic functions */
/* 7.3.7.1 The cexp functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex cexp(double complex __z) __INTRODUCED_IN(23);
float complex cexpf(float complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
long double complex cexpl(long double complex __z) __INTRODUCED_IN(26);
/* 7.3.7.2 The clog functions */
double complex clog(double complex __z) __INTRODUCED_IN(26);
float complex clogf(float complex __z) __INTRODUCED_IN(26);
long double complex clogl(long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.8 Power and absolute-value functions */
/* 7.3.8.1 The cabs functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double cabs(double complex __z) __INTRODUCED_IN(23);
float cabsf(float complex __z) __INTRODUCED_IN(23);
long double cabsl(long double complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
/* 7.3.8.2 The cpow functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
double complex cpow(double complex __x, double complex __z) __INTRODUCED_IN(26);
float complex cpowf(float complex __x, float complex __z) __INTRODUCED_IN(26);
long double complex cpowl(long double complex __x, long double complex __z) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/* 7.3.8.3 The csqrt functions */
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
double complex csqrt(double complex __z) __INTRODUCED_IN(23);
float complex csqrtf(float complex __z) __INTRODUCED_IN(23);
long double complex csqrtl(long double complex __z) __INTRODUCED_IN(23);
@@ -148,6 +262,8 @@
double creal(double complex __z) __INTRODUCED_IN(23);
float crealf(float complex __z) __INTRODUCED_IN(23);
long double creall(long double complex __z) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
__END_DECLS
diff --git a/libc/include/dirent.h b/libc/include/dirent.h
index 5333d78..8058cfb 100644
--- a/libc/include/dirent.h
+++ b/libc/include/dirent.h
@@ -33,8 +33,9 @@
* @brief Directory entry iteration.
*/
-#include <stdint.h>
#include <sys/cdefs.h>
+
+#include <stdint.h>
#include <sys/types.h>
__BEGIN_DECLS
@@ -149,6 +150,8 @@
*
* Available since API level 23.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
void seekdir(DIR* _Nonnull __dir, long __location) __INTRODUCED_IN(23);
/**
@@ -161,6 +164,8 @@
* Available since API level 23.
*/
long telldir(DIR* _Nonnull __dir) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
/**
* [dirfd(3)](https://man7.org/linux/man-pages/man3/dirfd.3.html)
@@ -221,6 +226,8 @@
*
* Available since API level 24.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
int scandirat64(int __dir_fd, const char* _Nonnull __path, struct dirent64* _Nonnull * _Nonnull * _Nonnull __name_list, int (* _Nullable __filter)(const struct dirent64* _Nonnull), int (* _Nullable __comparator)(const struct dirent64* _Nonnull * _Nonnull, const struct dirent64* _Nonnull * _Nonnull)) __INTRODUCED_IN(24);
/**
@@ -237,6 +244,8 @@
* Available since API level 24.
*/
int scandirat(int __dir_fd, const char* _Nonnull __path, struct dirent* _Nonnull * _Nonnull * _Nonnull __name_list, int (* _Nullable __filter)(const struct dirent* _Nonnull), int (* _Nullable __comparator)(const struct dirent* _Nonnull * _Nonnull, const struct dirent* _Nonnull * _Nonnull)) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
#endif
diff --git a/libc/include/dlfcn.h b/libc/include/dlfcn.h
index 071d50a..67759b8 100644
--- a/libc/include/dlfcn.h
+++ b/libc/include/dlfcn.h
@@ -28,9 +28,10 @@
#pragma once
-#include <stdint.h>
#include <sys/cdefs.h>
+#include <stdint.h>
+
__BEGIN_DECLS
/**
@@ -116,7 +117,11 @@
* Returns the address of the symbol on success, and returns NULL on failure,
* in which case dlerror() can be used to retrieve the specific error.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
void* _Nullable dlvsym(void* __BIONIC_COMPLICATED_NULLNESS __handle, const char* _Nullable __symbol, const char* _Nullable __version) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
/**
* [dladdr(3)](https://man7.org/linux/man-pages/man3/dladdr.3.html)
diff --git a/libc/include/err.h b/libc/include/err.h
index d8122d7..4a1841b 100644
--- a/libc/include/err.h
+++ b/libc/include/err.h
@@ -36,8 +36,9 @@
* @brief BSD error reporting functions. See `<error.h>` for the GNU equivalent.
*/
-#include <stdarg.h>
#include <sys/cdefs.h>
+
+#include <stdarg.h>
#include <sys/types.h>
__BEGIN_DECLS
diff --git a/libc/include/error.h b/libc/include/error.h
index cb867cd..a9bdc24 100644
--- a/libc/include/error.h
+++ b/libc/include/error.h
@@ -44,6 +44,8 @@
*
* Available since API level 23.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
extern void (* _Nullable error_print_progname)(void) __INTRODUCED_IN(23);
/**
@@ -81,5 +83,7 @@
* Available since API level 23.
*/
void error_at_line(int __status, int __errno, const char* _Nonnull __filename, unsigned int __line_number, const char* _Nonnull __fmt, ...) __printflike(5, 6) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
__END_DECLS
diff --git a/libc/include/execinfo.h b/libc/include/execinfo.h
index 88f4ae7..84b637c 100644
--- a/libc/include/execinfo.h
+++ b/libc/include/execinfo.h
@@ -47,6 +47,8 @@
*
* Available since API level 33.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(33)
int backtrace(void* _Nonnull * _Nonnull buffer, int size) __INTRODUCED_IN(33);
/**
@@ -70,5 +72,7 @@
* Available since API level 33.
*/
void backtrace_symbols_fd(void* _Nonnull const* _Nonnull buffer, int size, int fd) __INTRODUCED_IN(33);
+#endif /* __BIONIC_AVAILABILITY_GUARD(33) */
+
__END_DECLS
diff --git a/libc/include/fcntl.h b/libc/include/fcntl.h
index 1e9a285..2bd1fc6 100644
--- a/libc/include/fcntl.h
+++ b/libc/include/fcntl.h
@@ -227,7 +227,11 @@
*
* Returns 0 on success and returns -1 and sets `errno` on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int sync_file_range(int __fd, off64_t __offset, off64_t __length, unsigned int __flags) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
#endif
diff --git a/libc/include/glob.h b/libc/include/glob.h
index 2c2b8d1..ccdf2e9 100644
--- a/libc/include/glob.h
+++ b/libc/include/glob.h
@@ -92,8 +92,12 @@
__BEGIN_DECLS
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int glob(const char* _Nonnull __pattern, int __flags, int (* _Nullable __error_callback)(const char* _Nonnull __failure_path, int __failure_errno), glob_t* _Nonnull __result_ptr) __INTRODUCED_IN(28);
void globfree(glob_t* _Nonnull __result_ptr) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
__END_DECLS
diff --git a/libc/include/grp.h b/libc/include/grp.h
index 2451db5..a48c046 100644
--- a/libc/include/grp.h
+++ b/libc/include/grp.h
@@ -51,12 +51,20 @@
struct group* _Nullable getgrnam(const char* _Nonnull __name);
/* Note: Android has thousands and thousands of ids to iterate through. */
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
struct group* _Nullable getgrent(void) __INTRODUCED_IN(26);
void setgrent(void) __INTRODUCED_IN(26);
void endgrent(void) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
int getgrgid_r(gid_t __gid, struct group* __BIONIC_COMPLICATED_NULLNESS __group, char* _Nonnull __buf, size_t __n, struct group* _Nullable * _Nonnull __result) __INTRODUCED_IN(24);
int getgrnam_r(const char* _Nonnull __name, struct group* __BIONIC_COMPLICATED_NULLNESS __group, char* _Nonnull __buf, size_t __n, struct group* _Nullable *_Nonnull __result) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
int getgrouplist(const char* _Nonnull __user, gid_t __group, gid_t* __BIONIC_COMPLICATED_NULLNESS __groups, int* _Nonnull __group_count);
int initgroups(const char* _Nonnull __user, gid_t __group);
diff --git a/libc/include/iconv.h b/libc/include/iconv.h
index 9da46b4..35328ee 100644
--- a/libc/include/iconv.h
+++ b/libc/include/iconv.h
@@ -60,6 +60,8 @@
*
* Available since API level 28.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
iconv_t _Nonnull iconv_open(const char* _Nonnull __dst_encoding, const char* _Nonnull __src_encoding) __INTRODUCED_IN(28);
/**
@@ -82,5 +84,7 @@
* Available since API level 28.
*/
int iconv_close(iconv_t _Nonnull __converter) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
__END_DECLS
diff --git a/libc/include/ifaddrs.h b/libc/include/ifaddrs.h
index c4d0e10..87d2947 100644
--- a/libc/include/ifaddrs.h
+++ b/libc/include/ifaddrs.h
@@ -80,6 +80,8 @@
*
* Available since API level 24.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
int getifaddrs(struct ifaddrs* _Nullable * _Nonnull __list_ptr) __INTRODUCED_IN(24);
/**
@@ -89,5 +91,7 @@
* Available since API level 24.
*/
void freeifaddrs(struct ifaddrs* _Nullable __ptr) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
__END_DECLS
diff --git a/libc/include/inttypes.h b/libc/include/inttypes.h
index 9fcd9f3..790030e 100644
--- a/libc/include/inttypes.h
+++ b/libc/include/inttypes.h
@@ -19,8 +19,8 @@
#ifndef _INTTYPES_H_
#define _INTTYPES_H_
-#include <stdint.h>
#include <sys/cdefs.h>
+#include <stdint.h>
#ifdef __LP64__
#define __PRI_64_prefix "l"
diff --git a/libc/include/langinfo.h b/libc/include/langinfo.h
index 2b43892..b9d695c 100644
--- a/libc/include/langinfo.h
+++ b/libc/include/langinfo.h
@@ -92,8 +92,12 @@
#define NOEXPR 54
#define CRNCYSTR 55
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
char* _Nonnull nl_langinfo(nl_item __item) __INTRODUCED_IN(26);
char* _Nonnull nl_langinfo_l(nl_item __item, locale_t _Nonnull __l) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
__END_DECLS
diff --git a/libc/include/limits.h b/libc/include/limits.h
index e1f566c..5e9ce59 100644
--- a/libc/include/limits.h
+++ b/libc/include/limits.h
@@ -1,6 +1,3 @@
-/* $OpenBSD: limits.h,v 1.13 2005/12/31 19:29:38 millert Exp $ */
-/* $NetBSD: limits.h,v 1.7 1994/10/26 00:56:00 cgd Exp $ */
-
/*
* Copyright (c) 1988 The Regents of the University of California.
* All rights reserved.
@@ -32,108 +29,101 @@
* @(#)limits.h 5.9 (Berkeley) 4/3/91
*/
-#ifndef _LIMITS_H_
-#define _LIMITS_H_
+#pragma once
+
+/**
+ * @file limits.h
+ * @brief Constants relating to implementation limits.
+ *
+ * This file is included via `#include_next` from the clang header of the same
+ * name that provides all the limits that the compiler is responsible for,
+ * primarily those relating to integer types defined by the C standard.
+ * This file defines the additional limits defined by POSIX.
+ */
+
+/*
+ * The Android build system has bionic _before_ the clang headers,
+ * so although the claim above that clang does an `#include_next`
+ * of this file is true for the NDK, it's not true for the OS,
+ * and we need to paper over that difference here until/unless
+ * the OS build changes.
+ */
+#if __has_include_next(<limits.h>)
+#include_next <limits.h>
+#endif
#include <sys/cdefs.h>
/* Historically bionic exposed the content of <float.h> from <limits.h> and <sys/limits.h> too. */
#include <float.h>
+/* Many of the POSIX limits come from the kernel. */
#include <linux/limits.h>
-#define PASS_MAX 128 /* _PASSWORD_LEN from <pwd.h> */
-
-#define NL_ARGMAX 9
-#define NL_LANGMAX 14
-#define NL_MSGMAX 32767
-#define NL_NMAX 1
-#define NL_SETMAX 255
-#define NL_TEXTMAX 255
-
-#define TMP_MAX 308915776
-
-/* TODO: get all these from the compiler's <limits.h>? */
-
-#define CHAR_BIT 8
-#ifdef __LP64__
-# define LONG_BIT 64
-#else
-# define LONG_BIT 32
+/*
+ * bionic always exposed these alternative names,
+ * but clang's <limits.h> considers them GNU extensions,
+ * and may or may not have defined them.
+ */
+#ifndef LONG_LONG_MIN
+/** Non-portable synonym; use LLONG_MIN directly instead. */
+#define LONG_LONG_MIN LLONG_MIN
#endif
+#ifndef LONG_LONG_MAX
+/** Non-portable synonym; use LLONG_MAX directly instead. */
+#define LONG_LONG_MAX LLONG_MAX
+#endif
+#ifndef ULONG_LONG_MAX
+/** Non-portable synonym; use ULLONG_MAX directly instead. */
+#define ULONG_LONG_MAX ULLONG_MAX
+#endif
+
+/** Maximum number of positional arguments in a printf()/scanf() format string. */
+#define NL_ARGMAX 9
+/** Maximum number of bytes in a $LANG name. */
+#define NL_LANGMAX 14
+/** Irrelevant with Android's <nl_types.h>. */
+#define NL_MSGMAX 32767
+/** Obsolete; removed from POSIX. */
+#define NL_NMAX 1
+/** Irrelevant with Android's <nl_types.h>. */
+#define NL_SETMAX 255
+/** Irrelevant with Android's <nl_types.h>. */
+#define NL_TEXTMAX 255
+
+/** Obsolete; removed from POSIX. */
+#define PASS_MAX 128
+/** Obsolete; removed from POSIX. */
+#define TMP_MAX 308915776
+
+/** Number of bits in a `long` (POSIX). */
+#if __LP64__
+#define LONG_BIT 64
+#else
+#define LONG_BIT 32
+#endif
+/** Number of bits in a "word" of `int` (POSIX). */
#define WORD_BIT 32
-#define SCHAR_MAX 0x7f /* max value for a signed char */
-#define SCHAR_MIN (-0x7f-1) /* min value for a signed char */
-
-#define UCHAR_MAX 0xffU /* max value for an unsigned char */
-#ifdef __CHAR_UNSIGNED__
-# define CHAR_MIN 0 /* min value for a char */
-# define CHAR_MAX 0xff /* max value for a char */
-#else
-# define CHAR_MAX 0x7f
-# define CHAR_MIN (-0x7f-1)
-#endif
-
-#define USHRT_MAX 0xffffU /* max value for an unsigned short */
-#define SHRT_MAX 0x7fff /* max value for a short */
-#define SHRT_MIN (-0x7fff-1) /* min value for a short */
-
-#define UINT_MAX 0xffffffffU /* max value for an unsigned int */
-#define INT_MAX 0x7fffffff /* max value for an int */
-#define INT_MIN (-0x7fffffff-1) /* min value for an int */
-
-#ifdef __LP64__
-# define ULONG_MAX 0xffffffffffffffffUL /* max value for unsigned long */
-# define LONG_MAX 0x7fffffffffffffffL /* max value for a signed long */
-# define LONG_MIN (-0x7fffffffffffffffL-1) /* min value for a signed long */
-#else
-# define ULONG_MAX 0xffffffffUL /* max value for an unsigned long */
-# define LONG_MAX 0x7fffffffL /* max value for a long */
-# define LONG_MIN (-0x7fffffffL-1)/* min value for a long */
-#endif
-
-# define ULLONG_MAX 0xffffffffffffffffULL /* max value for unsigned long long */
-# define LLONG_MAX 0x7fffffffffffffffLL /* max value for a signed long long */
-# define LLONG_MIN (-0x7fffffffffffffffLL-1) /* min value for a signed long long */
-
-/* GLibc compatibility definitions.
- Note that these are defined by GCC's <limits.h>
- only when __GNU_LIBRARY__ is defined, i.e. when
- targetting GLibc. */
-#ifndef LONG_LONG_MIN
-#define LONG_LONG_MIN LLONG_MIN
-#endif
-
-#ifndef LONG_LONG_MAX
-#define LONG_LONG_MAX LLONG_MAX
-#endif
-
-#ifndef ULONG_LONG_MAX
-#define ULONG_LONG_MAX ULLONG_MAX
-#endif
-
-#if defined(__USE_BSD) || defined(__BIONIC__) /* Historically bionic exposed these. */
-# define UID_MAX UINT_MAX /* max value for a uid_t */
-# define GID_MAX UINT_MAX /* max value for a gid_t */
-#if defined(__LP64__)
+/** Maximum value of a uid_t. */
+#define UID_MAX UINT_MAX
+/** Maximum value of a gid_t. */
+#define GID_MAX UINT_MAX
+/** Maximum value of a size_t. */
#define SIZE_T_MAX ULONG_MAX
-#else
-#define SIZE_T_MAX UINT_MAX
-#endif
-#endif
-
-#if defined(__LP64__)
+/** Maximum value of a ssize_t. */
#define SSIZE_MAX LONG_MAX
-#else
-#define SSIZE_MAX INT_MAX
-#endif
+/** Maximum number of bytes in a multibyte character. */
#define MB_LEN_MAX 4
+/** Default process priority. */
#define NZERO 20
+/** Maximum number of struct iovec that can be passed in a single readv()/writev(). */
#define IOV_MAX 1024
+
+/** Maximum value for a semaphore. */
#define SEM_VALUE_MAX 0x3fffffff
/** Do not use: prefer getline() or asprintf() rather than hard-coding an arbitrary size. */
@@ -142,12 +132,17 @@
/* POSIX says these belong in <unistd.h> but BSD has some in <limits.h>. */
#include <bits/posix_limits.h>
+/** Maximum length of a hostname returned by gethostname(). */
#define HOST_NAME_MAX _POSIX_HOST_NAME_MAX
+
+/** Maximum length of a login name. */
#define LOGIN_NAME_MAX 256
+
+/** Maximum length of terminal device name. */
#define TTY_NAME_MAX 32
-/* >= _POSIX_THREAD_DESTRUCTOR_ITERATIONS */
-#define PTHREAD_DESTRUCTOR_ITERATIONS 4
+/** Maximum number of attempts to destroy thread-specific data when a thread exits. */
+#define PTHREAD_DESTRUCTOR_ITERATIONS _POSIX_THREAD_DESTRUCTOR_ITERATIONS
/**
* The number of calls to pthread_key_create() without intervening calls to
@@ -156,7 +151,5 @@
*/
#define PTHREAD_KEYS_MAX 128
-/** bionic has no specific limit on the number of threads. */
+/** bionic has no fixed limit on the number of threads. */
#undef PTHREAD_THREADS_MAX
-
-#endif /* !_LIMITS_H_ */
diff --git a/libc/include/link.h b/libc/include/link.h
index 216502e..331070e 100644
--- a/libc/include/link.h
+++ b/libc/include/link.h
@@ -33,8 +33,9 @@
* @brief Extra dynamic linker functionality (see also <dlfcn.h>).
*/
-#include <stdint.h>
#include <sys/cdefs.h>
+
+#include <stdint.h>
#include <sys/types.h>
#include <elf.h>
diff --git a/libc/include/malloc.h b/libc/include/malloc.h
index 2fa4b49..dc2ca2b 100644
--- a/libc/include/malloc.h
+++ b/libc/include/malloc.h
@@ -195,7 +195,11 @@
*
* Available since API level 23.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int malloc_info(int __must_be_zero, FILE* _Nonnull __fp) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
/**
* mallopt() option to set the decay time. Valid values are -1, 0 and 1.
@@ -368,7 +372,11 @@
*
* Available since API level 26.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int mallopt(int __option, int __value) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/**
* [__malloc_hook(3)](https://man7.org/linux/man-pages/man3/__malloc_hook.3.html)
@@ -379,6 +387,8 @@
*
* See also: [extra documentation](https://android.googlesource.com/platform/bionic/+/main/libc/malloc_hooks/README.md)
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
extern void* _Nonnull (*volatile _Nonnull __malloc_hook)(size_t __byte_count, const void* _Nonnull __caller) __INTRODUCED_IN(28);
/**
@@ -413,5 +423,7 @@
* See also: [extra documentation](https://android.googlesource.com/platform/bionic/+/main/libc/malloc_hooks/README.md)
*/
extern void* _Nonnull (*volatile _Nonnull __memalign_hook)(size_t __alignment, size_t __byte_count, const void* _Nonnull __caller) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
__END_DECLS
diff --git a/libc/include/math.h b/libc/include/math.h
index 343ab98..59161bf 100644
--- a/libc/include/math.h
+++ b/libc/include/math.h
@@ -350,7 +350,11 @@
double gamma_r(double __x, int* _Nonnull __sign);
double lgamma_r(double __x, int* _Nonnull __sign);
double significand(double __x);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
long double lgammal_r(long double __x, int* _Nonnull __sign) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
long double significandl(long double __x);
float dremf(float __x, float __y);
int finitef(float __x) __attribute_const__;
diff --git a/libc/include/mntent.h b/libc/include/mntent.h
index 9a31838..4c03602 100644
--- a/libc/include/mntent.h
+++ b/libc/include/mntent.h
@@ -29,8 +29,9 @@
#ifndef _MNTENT_H_
#define _MNTENT_H_
-#include <stdio.h>
#include <sys/cdefs.h>
+
+#include <stdio.h>
#include <paths.h> /* for _PATH_MOUNTED */
#define MOUNTED _PATH_MOUNTED
@@ -61,7 +62,11 @@
struct mntent* _Nullable getmntent(FILE* _Nonnull __fp);
struct mntent* _Nullable getmntent_r(FILE* _Nonnull __fp, struct mntent* _Nonnull __entry, char* _Nonnull __buf, int __size);
FILE* _Nullable setmntent(const char* _Nonnull __filename, const char* _Nonnull __type);
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
char* _Nullable hasmntopt(const struct mntent* _Nonnull __entry, const char* _Nonnull __option) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
__END_DECLS
diff --git a/libc/include/net/if.h b/libc/include/net/if.h
index 79b4195..50bc74c 100644
--- a/libc/include/net/if.h
+++ b/libc/include/net/if.h
@@ -29,9 +29,10 @@
#ifndef _NET_IF_H_
#define _NET_IF_H_
+#include <sys/cdefs.h>
+
#include <sys/socket.h>
#include <linux/if.h>
-#include <sys/cdefs.h>
#ifndef IF_NAMESIZE
#define IF_NAMESIZE IFNAMSIZ
@@ -46,8 +47,12 @@
char* _Nullable if_indextoname(unsigned __index, char* _Nonnull __buf);
unsigned if_nametoindex(const char* _Nonnull __name);
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
struct if_nameindex* _Nullable if_nameindex(void) __INTRODUCED_IN(24);
void if_freenameindex(struct if_nameindex* _Nullable __ptr) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
__END_DECLS
diff --git a/libc/include/netdb.h b/libc/include/netdb.h
index 88214d5..04aaf5c 100644
--- a/libc/include/netdb.h
+++ b/libc/include/netdb.h
@@ -212,28 +212,52 @@
void herror(const char* _Nonnull __s);
const char* _Nonnull hstrerror(int __error);
struct hostent* _Nullable gethostbyaddr(const void* _Nonnull __addr, socklen_t __length, int __type);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int gethostbyaddr_r(const void* _Nonnull __addr, socklen_t __length, int __type, struct hostent* _Nonnull __ret, char* _Nonnull __buf, size_t __buf_size, struct hostent* _Nullable * _Nonnull __result, int* _Nonnull __h_errno_ptr) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
struct hostent* _Nullable gethostbyname(const char* _Nonnull __name);
int gethostbyname_r(const char* _Nonnull __name, struct hostent* _Nonnull __ret, char* _Nonnull __buf, size_t __buf_size, struct hostent* _Nullable * _Nonnull __result, int* _Nonnull __h_errno_ptr);
struct hostent* _Nullable gethostbyname2(const char* _Nonnull __name, int __af);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int gethostbyname2_r(const char* _Nonnull __name, int __af, struct hostent* _Nonnull __ret, char* _Nonnull __buf, size_t __buf_size, struct hostent* _Nullable * _Nonnull __result, int* _Nonnull __h_errno_ptr) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
void endhostent(void) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
struct hostent* _Nullable gethostent(void);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
void sethostent(int __stay_open) __INTRODUCED_IN(28);
/* These functions are obsolete. None of these functions return anything but nullptr. */
void endnetent(void) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
struct netent* _Nullable getnetbyaddr(uint32_t __net, int __type);
struct netent* _Nullable getnetbyname(const char* _Nonnull __name);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
struct netent* _Nullable getnetent(void) __INTRODUCED_IN(28);
void setnetent(int __stay_open) __INTRODUCED_IN(28);
/* None of these functions return anything but nullptr. */
void endprotoent(void) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
struct protoent* _Nullable getprotobyname(const char* _Nonnull __name);
struct protoent* _Nullable getprotobynumber(int __proto);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
struct protoent* _Nullable getprotoent(void) __INTRODUCED_IN(28);
void setprotoent(int __stay_open) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
/* These functions return entries from a built-in database. */
void endservent(void);
diff --git a/libc/include/netinet/icmp6.h b/libc/include/netinet/icmp6.h
index 2b237a8..ebd9f6c 100644
--- a/libc/include/netinet/icmp6.h
+++ b/libc/include/netinet/icmp6.h
@@ -65,9 +65,10 @@
#ifndef _NETINET_ICMP6_H_
#define _NETINET_ICMP6_H_
-#include <netinet/in.h> /* android-added: glibc source compatibility. */
#include <sys/cdefs.h>
+#include <netinet/in.h> /* android-added: glibc source compatibility. */
+
#define ICMPV6_PLD_MAXLEN 1232 /* IPV6_MMTU - sizeof(struct ip6_hdr)
- sizeof(struct icmp6_hdr) */
diff --git a/libc/include/netinet/in.h b/libc/include/netinet/in.h
index 163e614..d4ce302 100644
--- a/libc/include/netinet/in.h
+++ b/libc/include/netinet/in.h
@@ -28,9 +28,10 @@
#pragma once
+#include <sys/cdefs.h>
+
#include <endian.h>
#include <netinet/in6.h>
-#include <sys/cdefs.h>
#include <sys/socket.h>
#include <linux/in.h>
diff --git a/libc/include/nl_types.h b/libc/include/nl_types.h
index 6c9935d..172d80d 100644
--- a/libc/include/nl_types.h
+++ b/libc/include/nl_types.h
@@ -62,6 +62,8 @@
*
* Available since API level 28.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
nl_catd _Nonnull catopen(const char* _Nonnull __name, int __flag) __INTRODUCED_IN(26);
/**
@@ -80,5 +82,7 @@
* On Android, this always returns -1 with `errno` set to `EBADF`.
*/
int catclose(nl_catd _Nonnull __catalog) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
__END_DECLS
diff --git a/libc/include/poll.h b/libc/include/poll.h
index 0dda3da..e57f812 100644
--- a/libc/include/poll.h
+++ b/libc/include/poll.h
@@ -64,7 +64,11 @@
/**
* Like ppoll() but allows setting a signal mask with RT signals even from a 32-bit process.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int ppoll64(struct pollfd* _Nullable __fds, nfds_t __count, const struct timespec* _Nullable __timeout, const sigset64_t* _Nullable __mask) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
#if defined(__BIONIC_INCLUDE_FORTIFY_HEADERS)
#define _POLL_H_
diff --git a/libc/include/pthread.h b/libc/include/pthread.h
index d718b40..8b78a74 100644
--- a/libc/include/pthread.h
+++ b/libc/include/pthread.h
@@ -33,11 +33,12 @@
* @brief POSIX threads.
*/
+#include <sys/cdefs.h>
+
#include <limits.h>
#include <bits/page_size.h>
#include <bits/pthread_types.h>
#include <sched.h>
-#include <sys/cdefs.h>
#include <sys/types.h>
#include <time.h>
@@ -98,7 +99,11 @@
int pthread_attr_destroy(pthread_attr_t* _Nonnull __attr);
int pthread_attr_getdetachstate(const pthread_attr_t* _Nonnull __attr, int* _Nonnull __state);
int pthread_attr_getguardsize(const pthread_attr_t* _Nonnull __attr, size_t* _Nonnull __size);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int pthread_attr_getinheritsched(const pthread_attr_t* _Nonnull __attr, int* _Nonnull __flag) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int pthread_attr_getschedparam(const pthread_attr_t* _Nonnull __attr, struct sched_param* _Nonnull __param);
int pthread_attr_getschedpolicy(const pthread_attr_t* _Nonnull __attr, int* _Nonnull __policy);
int pthread_attr_getscope(const pthread_attr_t* _Nonnull __attr, int* _Nonnull __scope);
@@ -107,7 +112,11 @@
int pthread_attr_init(pthread_attr_t* _Nonnull __attr);
int pthread_attr_setdetachstate(pthread_attr_t* _Nonnull __attr, int __state);
int pthread_attr_setguardsize(pthread_attr_t* _Nonnull __attr, size_t __size);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int pthread_attr_setinheritsched(pthread_attr_t* _Nonnull __attr, int __flag) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int pthread_attr_setschedparam(pthread_attr_t* _Nonnull __attr, const struct sched_param* _Nonnull __param);
int pthread_attr_setschedpolicy(pthread_attr_t* _Nonnull __attr, int __policy);
int pthread_attr_setscope(pthread_attr_t* _Nonnull __attr, int __scope);
@@ -122,8 +131,12 @@
int pthread_condattr_setpshared(pthread_condattr_t* _Nonnull __attr, int __shared);
int pthread_cond_broadcast(pthread_cond_t* _Nonnull __cond);
+
+#if __BIONIC_AVAILABILITY_GUARD(30)
int pthread_cond_clockwait(pthread_cond_t* _Nonnull __cond, pthread_mutex_t* _Nonnull __mutex, clockid_t __clock,
const struct timespec* _Nullable __timeout) __INTRODUCED_IN(30);
+#endif /* __BIONIC_AVAILABILITY_GUARD(30) */
+
int pthread_cond_destroy(pthread_cond_t* _Nonnull __cond);
int pthread_cond_init(pthread_cond_t* _Nonnull __cond, const pthread_condattr_t* _Nullable __attr);
int pthread_cond_signal(pthread_cond_t* _Nonnull __cond);
@@ -138,8 +151,12 @@
* Note that pthread_cond_clockwait() allows specifying an arbitrary clock and has superseded this
* function.
*/
+
+#if (!defined(__LP64__)) || (defined(__LP64__) && __ANDROID_API__ >= 28)
int pthread_cond_timedwait_monotonic_np(pthread_cond_t* _Nonnull __cond, pthread_mutex_t* _Nonnull __mutex,
const struct timespec* _Nullable __timeout) __INTRODUCED_IN_64(28);
+#endif /* (!defined(__LP64__)) || (defined(__LP64__) && __ANDROID_API__ >= 28) */
+
int pthread_cond_wait(pthread_cond_t* _Nonnull __cond, pthread_mutex_t* _Nonnull __mutex);
int pthread_create(pthread_t* _Nonnull __pthread_ptr, pthread_attr_t const* _Nullable __attr, void* _Nullable (* _Nonnull __start_routine)(void* _Nullable), void* _Nullable);
@@ -187,14 +204,26 @@
int pthread_mutexattr_destroy(pthread_mutexattr_t* _Nonnull __attr);
int pthread_mutexattr_getpshared(const pthread_mutexattr_t* _Nonnull __attr, int* _Nonnull __shared);
int pthread_mutexattr_gettype(const pthread_mutexattr_t* _Nonnull __attr, int* _Nonnull __type);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int pthread_mutexattr_getprotocol(const pthread_mutexattr_t* _Nonnull __attr, int* _Nonnull __protocol) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int pthread_mutexattr_init(pthread_mutexattr_t* _Nonnull __attr);
int pthread_mutexattr_setpshared(pthread_mutexattr_t* _Nonnull __attr, int __shared);
int pthread_mutexattr_settype(pthread_mutexattr_t* _Nonnull __attr, int __type);
-int pthread_mutexattr_setprotocol(pthread_mutexattr_t* _Nonnull __attr, int __protocol) __INTRODUCED_IN(28);
+#if __BIONIC_AVAILABILITY_GUARD(28)
+int pthread_mutexattr_setprotocol(pthread_mutexattr_t* _Nonnull __attr, int __protocol) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
+
+
+#if __BIONIC_AVAILABILITY_GUARD(30)
int pthread_mutex_clocklock(pthread_mutex_t* _Nonnull __mutex, clockid_t __clock,
const struct timespec* _Nullable __abstime) __INTRODUCED_IN(30);
+#endif /* __BIONIC_AVAILABILITY_GUARD(30) */
+
int pthread_mutex_destroy(pthread_mutex_t* _Nonnull __mutex);
int pthread_mutex_init(pthread_mutex_t* _Nonnull __mutex, const pthread_mutexattr_t* _Nullable __attr);
int pthread_mutex_lock(pthread_mutex_t* _Nonnull __mutex);
@@ -209,8 +238,12 @@
* Note that pthread_mutex_clocklock() allows specifying an arbitrary clock and has superseded this
* function.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int pthread_mutex_timedlock_monotonic_np(pthread_mutex_t* _Nonnull __mutex, const struct timespec* _Nullable __timeout)
__INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int pthread_mutex_trylock(pthread_mutex_t* _Nonnull __mutex);
int pthread_mutex_unlock(pthread_mutex_t* _Nonnull __mutex);
@@ -220,30 +253,48 @@
int pthread_rwlockattr_destroy(pthread_rwlockattr_t* _Nonnull __attr);
int pthread_rwlockattr_getpshared(const pthread_rwlockattr_t* _Nonnull __attr, int* _Nonnull __shared);
int pthread_rwlockattr_setpshared(pthread_rwlockattr_t* _Nonnull __attr, int __shared);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int pthread_rwlockattr_getkind_np(const pthread_rwlockattr_t* _Nonnull __attr, int* _Nonnull __kind)
__INTRODUCED_IN(23);
int pthread_rwlockattr_setkind_np(pthread_rwlockattr_t* _Nonnull __attr, int __kind) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(30)
int pthread_rwlock_clockrdlock(pthread_rwlock_t* _Nonnull __rwlock, clockid_t __clock,
const struct timespec* _Nullable __timeout) __INTRODUCED_IN(30);
int pthread_rwlock_clockwrlock(pthread_rwlock_t* _Nonnull __rwlock, clockid_t __clock,
const struct timespec* _Nullable __timeout) __INTRODUCED_IN(30);
+#endif /* __BIONIC_AVAILABILITY_GUARD(30) */
+
int pthread_rwlock_destroy(pthread_rwlock_t* _Nonnull __rwlock);
int pthread_rwlock_init(pthread_rwlock_t* _Nonnull __rwlock, const pthread_rwlockattr_t* _Nullable __attr);
int pthread_rwlock_rdlock(pthread_rwlock_t* _Nonnull __rwlock);
int pthread_rwlock_timedrdlock(pthread_rwlock_t* _Nonnull __rwlock, const struct timespec* _Nullable __timeout);
/* See the comment on pthread_mutex_timedlock_monotonic_np for usage of this function. */
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int pthread_rwlock_timedrdlock_monotonic_np(pthread_rwlock_t* _Nonnull __rwlock,
const struct timespec* _Nullable __timeout) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int pthread_rwlock_timedwrlock(pthread_rwlock_t* _Nonnull __rwlock, const struct timespec* _Nullable __timeout);
/* See the comment on pthread_mutex_timedlock_monotonic_np for usage of this function. */
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int pthread_rwlock_timedwrlock_monotonic_np(pthread_rwlock_t* _Nonnull __rwlock,
const struct timespec* _Nullable __timeout) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int pthread_rwlock_tryrdlock(pthread_rwlock_t* _Nonnull __rwlock);
int pthread_rwlock_trywrlock(pthread_rwlock_t* _Nonnull __rwlock);
int pthread_rwlock_unlock(pthread_rwlock_t* _Nonnull __rwlock);
int pthread_rwlock_wrlock(pthread_rwlock_t* _Nonnull __rwlock);
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
int pthread_barrierattr_init(pthread_barrierattr_t* _Nonnull __attr) __INTRODUCED_IN(24);
int pthread_barrierattr_destroy(pthread_barrierattr_t* _Nonnull __attr) __INTRODUCED_IN(24);
int pthread_barrierattr_getpshared(const pthread_barrierattr_t* _Nonnull __attr, int* _Nonnull __shared) __INTRODUCED_IN(24);
@@ -258,11 +309,17 @@
int pthread_spin_lock(pthread_spinlock_t* _Nonnull __spinlock) __INTRODUCED_IN(24);
int pthread_spin_trylock(pthread_spinlock_t* _Nonnull __spinlock) __INTRODUCED_IN(24);
int pthread_spin_unlock(pthread_spinlock_t* _Nonnull __spinlock) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
pthread_t pthread_self(void) __attribute_const__;
#if defined(__USE_GNU)
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int pthread_getname_np(pthread_t __pthread, char* _Nonnull __buf, size_t __n) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
#endif
/* TODO: this should be __USE_GNU too. */
int pthread_setname_np(pthread_t __pthread, const char* _Nonnull __name);
@@ -301,7 +358,11 @@
*
* Available since API level 28.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int pthread_setschedprio(pthread_t __pthread, int __priority) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int pthread_setspecific(pthread_key_t __key, const void* _Nullable __value);
diff --git a/libc/include/pty.h b/libc/include/pty.h
index 1cfb772..92d7fbb 100644
--- a/libc/include/pty.h
+++ b/libc/include/pty.h
@@ -49,6 +49,8 @@
*
* Available since API level 23.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int openpty(int* _Nonnull __pty_fd, int* _Nonnull __tty_fd, char* _Nullable __tty_name, const struct termios* _Nullable __termios_ptr, const struct winsize* _Nullable __winsize_ptr) __INTRODUCED_IN(23);
/**
@@ -61,5 +63,7 @@
* Available since API level 23.
*/
int forkpty(int* _Nonnull __parent_pty_fd, char* _Nullable __child_tty_name, const struct termios* _Nullable __termios_ptr, const struct winsize* _Nullable __winsize_ptr) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
__END_DECLS
diff --git a/libc/include/pwd.h b/libc/include/pwd.h
index 2b17fbf..09592bc 100644
--- a/libc/include/pwd.h
+++ b/libc/include/pwd.h
@@ -84,10 +84,14 @@
struct passwd* _Nullable getpwuid(uid_t __uid);
/* Note: Android has thousands and thousands of ids to iterate through */
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
struct passwd* _Nullable getpwent(void) __INTRODUCED_IN(26);
void setpwent(void) __INTRODUCED_IN(26);
void endpwent(void) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
int getpwnam_r(const char* _Nonnull __name, struct passwd* _Nonnull __pwd, char* _Nonnull __buf, size_t __n, struct passwd* _Nullable * _Nonnull __result);
int getpwuid_r(uid_t __uid, struct passwd* _Nonnull __pwd, char* _Nonnull __buf, size_t __n, struct passwd* _Nullable * _Nonnull __result);
diff --git a/libc/include/resolv.h b/libc/include/resolv.h
index f25484a..c49cefc 100644
--- a/libc/include/resolv.h
+++ b/libc/include/resolv.h
@@ -29,9 +29,10 @@
#ifndef _RESOLV_H_
#define _RESOLV_H_
+#include <sys/cdefs.h>
+
#include <sys/param.h>
#include <sys/types.h>
-#include <sys/cdefs.h>
#include <sys/socket.h>
#include <stdio.h>
#include <arpa/nameser.h>
@@ -60,7 +61,11 @@
int res_search(const char* _Nonnull __name, int __class, int __type, u_char* _Nonnull __answer, int __answer_size);
#define res_randomid __res_randomid
+
+#if __BIONIC_AVAILABILITY_GUARD(29)
u_int __res_randomid(void) __INTRODUCED_IN(29);
+#endif /* __BIONIC_AVAILABILITY_GUARD(29) */
+
__END_DECLS
diff --git a/libc/include/sched.h b/libc/include/sched.h
index e8f7736..7a2dcad 100644
--- a/libc/include/sched.h
+++ b/libc/include/sched.h
@@ -33,9 +33,10 @@
* @brief Thread execution scheduling.
*/
+#include <sys/cdefs.h>
+
#include <bits/timespec.h>
#include <linux/sched.h>
-#include <sys/cdefs.h>
__BEGIN_DECLS
diff --git a/libc/include/search.h b/libc/include/search.h
index 85e31ee..2f43d91 100644
--- a/libc/include/search.h
+++ b/libc/include/search.h
@@ -85,6 +85,8 @@
*
* Available since API level 28.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int hcreate(size_t __n) __INTRODUCED_IN(28);
/**
@@ -109,6 +111,8 @@
* Available since API level 28.
*/
ENTRY* _Nullable hsearch(ENTRY __entry, ACTION __action) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
#if defined(__USE_BSD) || defined(__USE_GNU)
@@ -120,6 +124,8 @@
*
* Available since API level 28.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int hcreate_r(size_t __n, struct hsearch_data* _Nonnull __table) __INTRODUCED_IN(28);
/**
@@ -140,6 +146,8 @@
* Available since API level 28.
*/
int hsearch_r(ENTRY __entry, ACTION __action, ENTRY* _Nullable * _Nonnull __result, struct hsearch_data* _Nonnull __table) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
#endif
diff --git a/libc/include/semaphore.h b/libc/include/semaphore.h
index 6ad9ea3..9c4702d 100644
--- a/libc/include/semaphore.h
+++ b/libc/include/semaphore.h
@@ -45,7 +45,11 @@
#define SEM_FAILED __BIONIC_CAST(reinterpret_cast, sem_t*, 0)
+
+#if __BIONIC_AVAILABILITY_GUARD(30)
int sem_clockwait(sem_t* _Nonnull __sem, clockid_t __clock, const struct timespec* _Nonnull __ts) __INTRODUCED_IN(30);
+#endif /* __BIONIC_AVAILABILITY_GUARD(30) */
+
int sem_destroy(sem_t* _Nonnull __sem);
int sem_getvalue(sem_t* _Nonnull __sem, int* _Nonnull __value);
int sem_init(sem_t* _Nonnull __sem, int __shared, unsigned int __value);
@@ -59,7 +63,11 @@
* Note that sem_clockwait() allows specifying an arbitrary clock and has superseded this
* function.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int sem_timedwait_monotonic_np(sem_t* _Nonnull __sem, const struct timespec* _Nonnull __ts) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int sem_trywait(sem_t* _Nonnull __sem);
int sem_wait(sem_t* _Nonnull __sem);
diff --git a/libc/include/signal.h b/libc/include/signal.h
index 893fa9d..38dcbde 100644
--- a/libc/include/signal.h
+++ b/libc/include/signal.h
@@ -60,31 +60,73 @@
#define si_timerid si_tid /* glibc compatibility. */
int sigaction(int __signal, const struct sigaction* _Nullable __new_action, struct sigaction* _Nullable __old_action);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int sigaction64(int __signal, const struct sigaction64* _Nullable __new_action, struct sigaction64* _Nullable __old_action) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int siginterrupt(int __signal, int __flag);
sighandler_t _Nonnull signal(int __signal, sighandler_t _Nullable __handler);
int sigaddset(sigset_t* _Nonnull __set, int __signal);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int sigaddset64(sigset64_t* _Nonnull __set, int __signal) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int sigdelset(sigset_t* _Nonnull __set, int __signal);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int sigdelset64(sigset64_t* _Nonnull __set, int __signal) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int sigemptyset(sigset_t* _Nonnull __set);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int sigemptyset64(sigset64_t* _Nonnull __set) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int sigfillset(sigset_t* _Nonnull __set);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int sigfillset64(sigset64_t* _Nonnull __set) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int sigismember(const sigset_t* _Nonnull __set, int __signal);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int sigismember64(const sigset64_t* _Nonnull __set, int __signal) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int sigpending(sigset_t* _Nonnull __set);
-int sigpending64(sigset64_t* _Nonnull __set) __INTRODUCED_IN(28);
-int sigprocmask(int __how, const sigset_t* _Nullable __new_set, sigset_t* _Nullable __old_set);
-int sigprocmask64(int __how, const sigset64_t* _Nullable __new_set, sigset64_t* _Nullable __old_set) __INTRODUCED_IN(28);
-int sigsuspend(const sigset_t* _Nonnull __mask);
-int sigsuspend64(const sigset64_t* _Nonnull __mask) __INTRODUCED_IN(28);
-int sigwait(const sigset_t* _Nonnull __set, int* _Nonnull __signal);
-int sigwait64(const sigset64_t* _Nonnull __set, int* _Nonnull __signal) __INTRODUCED_IN(28);
+#if __BIONIC_AVAILABILITY_GUARD(28)
+int sigpending64(sigset64_t* _Nonnull __set) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
+int sigprocmask(int __how, const sigset_t* _Nullable __new_set, sigset_t* _Nullable __old_set);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
+int sigprocmask64(int __how, const sigset64_t* _Nullable __new_set, sigset64_t* _Nullable __old_set) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
+int sigsuspend(const sigset_t* _Nonnull __mask);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
+int sigsuspend64(const sigset64_t* _Nonnull __mask) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
+int sigwait(const sigset_t* _Nonnull __set, int* _Nonnull __signal);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
+int sigwait64(const sigset64_t* _Nonnull __set, int* _Nonnull __signal) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
+
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int sighold(int __signal)
__attribute__((__deprecated__("use sigprocmask() or pthread_sigmask() instead")))
__INTRODUCED_IN(26);
@@ -97,6 +139,8 @@
__INTRODUCED_IN(26);
sighandler_t _Nonnull sigset(int __signal, sighandler_t _Nullable __handler)
__attribute__((__deprecated__("use sigaction() instead"))) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
int raise(int __signal);
int kill(pid_t __pid, int __signal);
@@ -110,17 +154,41 @@
int pthread_kill(pthread_t __pthread, int __signal);
#if defined(__USE_GNU)
+
+#if __BIONIC_AVAILABILITY_GUARD(29)
int pthread_sigqueue(pthread_t __pthread, int __signal, const union sigval __value) __INTRODUCED_IN(29);
+#endif /* __BIONIC_AVAILABILITY_GUARD(29) */
+
#endif
int pthread_sigmask(int __how, const sigset_t* _Nullable __new_set, sigset_t* _Nullable __old_set);
-int pthread_sigmask64(int __how, const sigset64_t* _Nullable __new_set, sigset64_t* _Nullable __old_set) __INTRODUCED_IN(28);
+#if __BIONIC_AVAILABILITY_GUARD(28)
+int pthread_sigmask64(int __how, const sigset64_t* _Nullable __new_set, sigset64_t* _Nullable __old_set) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
+
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int sigqueue(pid_t __pid, int __signal, const union sigval __value) __INTRODUCED_IN(23);
int sigtimedwait(const sigset_t* _Nonnull __set, siginfo_t* _Nullable __info, const struct timespec* _Nullable __timeout) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int sigtimedwait64(const sigset64_t* _Nonnull __set, siginfo_t* _Nullable __info, const struct timespec* _Nullable __timeout) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int sigwaitinfo(const sigset_t* _Nonnull __set, siginfo_t* _Nullable __info) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int sigwaitinfo64(const sigset64_t* _Nonnull __set, siginfo_t* _Nullable __info) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
/**
* Buffer size suitable for any call to sig2str().
@@ -137,6 +205,8 @@
*
* Available since API level 36.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(36)
int sig2str(int __signal, char* _Nonnull __buf) __INTRODUCED_IN(36);
/**
@@ -149,6 +219,8 @@
* Available since API level 36.
*/
int str2sig(const char* _Nonnull __name, int* _Nonnull __signal) __INTRODUCED_IN(36);
+#endif /* __BIONIC_AVAILABILITY_GUARD(36) */
+
__END_DECLS
diff --git a/libc/include/spawn.h b/libc/include/spawn.h
index f366239..b105754 100644
--- a/libc/include/spawn.h
+++ b/libc/include/spawn.h
@@ -55,6 +55,8 @@
typedef struct __posix_spawnattr* posix_spawnattr_t;
typedef struct __posix_spawn_file_actions* posix_spawn_file_actions_t;
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int posix_spawn(pid_t* _Nullable __pid, const char* _Nonnull __path, const posix_spawn_file_actions_t _Nullable * _Nullable __actions, const posix_spawnattr_t _Nullable * _Nullable __attr, char* const _Nullable __argv[_Nullable], char* const _Nullable __env[_Nullable]) __INTRODUCED_IN(28);
int posix_spawnp(pid_t* _Nullable __pid, const char* _Nonnull __file, const posix_spawn_file_actions_t _Nullable * _Nullable __actions, const posix_spawnattr_t _Nullable * _Nullable __attr, char* const _Nullable __argv[_Nullable], char* const _Nullable __env[_Nullable]) __INTRODUCED_IN(28);
@@ -89,9 +91,15 @@
int posix_spawn_file_actions_addopen(posix_spawn_file_actions_t _Nonnull * _Nonnull __actions, int __fd, const char* _Nonnull __path, int __flags, mode_t __mode) __INTRODUCED_IN(28);
int posix_spawn_file_actions_addclose(posix_spawn_file_actions_t _Nonnull * _Nonnull __actions, int __fd) __INTRODUCED_IN(28);
int posix_spawn_file_actions_adddup2(posix_spawn_file_actions_t _Nonnull * _Nonnull __actions, int __fd, int __new_fd) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(34)
int posix_spawn_file_actions_addchdir_np(posix_spawn_file_actions_t _Nonnull * _Nonnull __actions, const char* _Nonnull __path) __INTRODUCED_IN(34);
int posix_spawn_file_actions_addfchdir_np(posix_spawn_file_actions_t _Nonnull * _Nonnull __actions, int __fd) __INTRODUCED_IN(34);
+#endif /* __BIONIC_AVAILABILITY_GUARD(34) */
+
__END_DECLS
diff --git a/libc/include/stdint.h b/libc/include/stdint.h
index 322a81c..772fe8b 100644
--- a/libc/include/stdint.h
+++ b/libc/include/stdint.h
@@ -29,9 +29,10 @@
#ifndef _STDINT_H
#define _STDINT_H
+#include <sys/cdefs.h>
+
#include <bits/wchar_limits.h>
#include <stddef.h>
-#include <sys/cdefs.h>
typedef signed char __int8_t;
typedef unsigned char __uint8_t;
diff --git a/libc/include/stdio.h b/libc/include/stdio.h
index d24f6af..2c2dc01 100644
--- a/libc/include/stdio.h
+++ b/libc/include/stdio.h
@@ -196,7 +196,11 @@
*
* Returns 0 on success, and returns -1 and sets `errno` on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(30)
int renameat2(int __old_dir_fd, const char* _Nonnull __old_path, int __new_dir_fd, const char* _Nonnull __new_path, unsigned __flags) __INTRODUCED_IN(30);
+#endif /* __BIONIC_AVAILABILITY_GUARD(30) */
+
#endif
@@ -205,17 +209,25 @@
/* See https://android.googlesource.com/platform/bionic/+/main/docs/32-bit-abi.md */
#if defined(__USE_FILE_OFFSET64)
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
int fgetpos(FILE* _Nonnull __fp, fpos_t* _Nonnull __pos) __RENAME(fgetpos64) __INTRODUCED_IN(24);
int fsetpos(FILE* _Nonnull __fp, const fpos_t* _Nonnull __pos) __RENAME(fsetpos64) __INTRODUCED_IN(24);
int fseeko(FILE* _Nonnull __fp, off_t __offset, int __whence) __RENAME(fseeko64) __INTRODUCED_IN(24);
__nodiscard off_t ftello(FILE* _Nonnull __fp) __RENAME(ftello64) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
# if defined(__USE_BSD)
/* If __read_fn and __write_fn are both nullptr, it will cause EINVAL */
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
__nodiscard FILE* _Nullable funopen(const void* _Nullable __cookie,
int (* __BIONIC_COMPLICATED_NULLNESS __read_fn)(void* _Nonnull, char* _Nonnull, int),
int (* __BIONIC_COMPLICATED_NULLNESS __write_fn)(void* _Nonnull, const char* _Nonnull, int),
fpos_t (* _Nullable __seek_fn)(void* _Nonnull, fpos_t, int),
int (* _Nullable __close_fn)(void* _Nonnull)) __RENAME(funopen64) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
# endif
#else
int fgetpos(FILE* _Nonnull __fp, fpos_t* _Nonnull __pos);
@@ -231,25 +243,45 @@
int (* _Nullable __close_fn)(void* _Nonnull));
# endif
#endif
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
int fgetpos64(FILE* _Nonnull __fp, fpos64_t* _Nonnull __pos) __INTRODUCED_IN(24);
int fsetpos64(FILE* _Nonnull __fp, const fpos64_t* _Nonnull __pos) __INTRODUCED_IN(24);
int fseeko64(FILE* _Nonnull __fp, off64_t __offset, int __whence) __INTRODUCED_IN(24);
__nodiscard off64_t ftello64(FILE* _Nonnull __fp) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
#if defined(__USE_BSD)
/* If __read_fn and __write_fn are both nullptr, it will cause EINVAL */
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
__nodiscard FILE* _Nullable funopen64(const void* _Nullable __cookie,
int (* __BIONIC_COMPLICATED_NULLNESS __read_fn)(void* _Nonnull, char* _Nonnull, int),
int (* __BIONIC_COMPLICATED_NULLNESS __write_fn)(void* _Nonnull, const char* _Nonnull, int),
fpos64_t (* _Nullable __seek_fn)(void* _Nonnull, fpos64_t, int),
int (* _Nullable __close_fn)(void* _Nonnull)) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
#endif
__nodiscard FILE* _Nullable fopen(const char* _Nonnull __path, const char* _Nonnull __mode);
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
__nodiscard FILE* _Nullable fopen64(const char* _Nonnull __path, const char* _Nonnull __mode) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
FILE* _Nullable freopen(const char* _Nullable __path, const char* _Nonnull __mode, FILE* _Nonnull __fp);
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
FILE* _Nullable freopen64(const char* _Nullable __path, const char* _Nonnull __mode, FILE* _Nonnull __fp) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
__nodiscard FILE* _Nullable tmpfile(void);
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
__nodiscard FILE* _Nullable tmpfile64(void) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
int snprintf(char* __BIONIC_COMPLICATED_NULLNESS __buf, size_t __size, const char* _Nonnull __fmt, ...) __printflike(3, 4);
int vfscanf(FILE* _Nonnull __fp, const char* _Nonnull __fmt, va_list __args) __scanflike(2, 0);
@@ -258,7 +290,11 @@
int vsscanf(const char* _Nonnull __s, const char* _Nonnull __fmt, va_list __args) __scanflike(2, 0);
#define L_ctermid 1024 /* size for ctermid() */
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
char* _Nonnull ctermid(char* _Nullable __buf) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
__nodiscard FILE* _Nullable fdopen(int __fd, const char* _Nonnull __mode);
__nodiscard int fileno(FILE* _Nonnull __fp);
@@ -272,8 +308,12 @@
int putc_unlocked(int __ch, FILE* _Nonnull __fp);
int putchar_unlocked(int __ch);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
__nodiscard FILE* _Nullable fmemopen(void* _Nullable __buf, size_t __size, const char* _Nonnull __mode) __INTRODUCED_IN(23);
__nodiscard FILE* _Nullable open_memstream(char* _Nonnull * _Nonnull __ptr, size_t* _Nonnull __size_ptr) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
#if defined(__USE_BSD) || defined(__BIONIC__) /* Historically bionic exposed these. */
int asprintf(char* _Nullable * _Nonnull __s_ptr, const char* _Nonnull __fmt, ...) __printflike(2, 3);
@@ -282,25 +322,41 @@
void setbuffer(FILE* _Nonnull __fp, char* _Nullable __buf, int __size);
int setlinebuf(FILE* _Nonnull __fp);
int vasprintf(char* _Nullable * _Nonnull __s_ptr, const char* _Nonnull __fmt, va_list __args) __printflike(2, 0);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
void clearerr_unlocked(FILE* _Nonnull __fp) __INTRODUCED_IN(23);
__nodiscard int feof_unlocked(FILE* _Nonnull __fp) __INTRODUCED_IN(23);
__nodiscard int ferror_unlocked(FILE* _Nonnull __fp) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
__nodiscard int fileno_unlocked(FILE* _Nonnull __fp) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
#define fropen(cookie, fn) funopen(cookie, fn, 0, 0, 0)
#define fwopen(cookie, fn) funopen(cookie, 0, fn, 0, 0)
#endif
#if defined(__USE_BSD)
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int fflush_unlocked(FILE* _Nullable __fp) __INTRODUCED_IN(28);
__nodiscard int fgetc_unlocked(FILE* _Nonnull __fp) __INTRODUCED_IN(28);
int fputc_unlocked(int __ch, FILE* _Nonnull __fp) __INTRODUCED_IN(28);
size_t fread_unlocked(void* _Nonnull __buf, size_t __size, size_t __count, FILE* _Nonnull __fp) __INTRODUCED_IN(28);
size_t fwrite_unlocked(const void* _Nonnull __buf, size_t __size, size_t __count, FILE* _Nonnull __fp) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
#endif
#if defined(__USE_GNU)
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int fputs_unlocked(const char* _Nonnull __s, FILE* _Nonnull __fp) __INTRODUCED_IN(28);
char* _Nullable fgets_unlocked(char* _Nonnull __buf, int __size, FILE* _Nonnull __fp) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
#endif
#if defined(__BIONIC_INCLUDE_FORTIFY_HEADERS)
diff --git a/libc/include/stdio_ext.h b/libc/include/stdio_ext.h
index d426a4a..9ff07da 100644
--- a/libc/include/stdio_ext.h
+++ b/libc/include/stdio_ext.h
@@ -44,6 +44,8 @@
*
* Available since API level 23.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
size_t __fbufsize(FILE* _Nonnull __fp) __INTRODUCED_IN(23);
/**
@@ -53,6 +55,8 @@
* Available since API level 23.
*/
int __freadable(FILE* _Nonnull __fp) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
/**
* [__freading(3)](https://man7.org/linux/man-pages/man3/__freading.3.html) returns non-zero if
@@ -60,7 +64,11 @@
*
* Available since API level 28.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int __freading(FILE* _Nonnull __fp) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
/**
* [__fwritable(3)](https://man7.org/linux/man-pages/man3/__fwritable.3.html) returns non-zero if
@@ -68,7 +76,11 @@
*
* Available since API level 23.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int __fwritable(FILE* _Nonnull __fp) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
/**
* [__fwriting(3)](https://man7.org/linux/man-pages/man3/__fwriting.3.html) returns non-zero if
@@ -76,7 +88,11 @@
*
* Available since API level 28.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int __fwriting(FILE* _Nonnull __fp) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
/**
* [__flbf(3)](https://man7.org/linux/man-pages/man3/__flbf.3.html) returns non-zero if
@@ -84,7 +100,11 @@
*
* Available since API level 23.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int __flbf(FILE* _Nonnull __fp) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
/**
* [__fpurge(3)](https://man7.org/linux/man-pages/man3/__fpurge.3.html) discards the contents of
@@ -98,7 +118,11 @@
*
* Available since API level 23.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
size_t __fpending(FILE* _Nonnull __fp) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
/**
* __freadahead(3) returns the number of bytes in the input buffer.
@@ -106,7 +130,11 @@
*
* Available since API level 34.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(34)
size_t __freadahead(FILE* _Nonnull __fp) __INTRODUCED_IN(34);
+#endif /* __BIONIC_AVAILABILITY_GUARD(34) */
+
/**
* [_flushlbf(3)](https://man7.org/linux/man-pages/man3/_flushlbf.3.html) flushes all
@@ -114,7 +142,11 @@
*
* Available since API level 23.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
void _flushlbf(void) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
/**
* `__fseterr` sets the
@@ -122,7 +154,11 @@
*
* Available since API level 28.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
void __fseterr(FILE* _Nonnull __fp) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
/** __fsetlocking() constant to query locking type. */
#define FSETLOCKING_QUERY 0
@@ -139,6 +175,10 @@
*
* Available since API level 23.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int __fsetlocking(FILE* _Nonnull __fp, int __type) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
__END_DECLS
diff --git a/libc/include/stdlib.h b/libc/include/stdlib.h
index 076a978..7081d7c 100644
--- a/libc/include/stdlib.h
+++ b/libc/include/stdlib.h
@@ -29,11 +29,12 @@
#ifndef _STDLIB_H
#define _STDLIB_H
+#include <sys/cdefs.h>
+
#include <alloca.h>
#include <bits/wait.h>
#include <malloc.h>
#include <stddef.h>
-#include <sys/cdefs.h>
#include <xlocale.h>
__BEGIN_DECLS
@@ -59,13 +60,21 @@
char* _Nullable mkdtemp(char* _Nonnull __template);
char* _Nullable mktemp(char* _Nonnull __template) __attribute__((__deprecated__("mktemp is unsafe, use mkstemp or tmpfile instead")));
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int mkostemp64(char* _Nonnull __template, int __flags) __INTRODUCED_IN(23);
int mkostemp(char* _Nonnull __template, int __flags) __INTRODUCED_IN(23);
int mkostemps64(char* _Nonnull __template, int __suffix_length, int __flags) __INTRODUCED_IN(23);
int mkostemps(char* _Nonnull __template, int __suffix_length, int __flags) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
int mkstemp64(char* _Nonnull __template);
int mkstemp(char* _Nonnull __template);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int mkstemps64(char* _Nonnull __template, int __flags) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
int mkstemps(char* _Nonnull __template, int __flags);
int posix_memalign(void* _Nullable * _Nullable __memptr, size_t __alignment, size_t __size);
@@ -79,7 +88,11 @@
*
* Available since API level 28.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
__nodiscard void* _Nullable aligned_alloc(size_t __alignment, size_t __size) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
__nodiscard char* _Nullable realpath(const char* _Nonnull __path, char* _Nullable __resolved);
@@ -122,7 +135,11 @@
*
* Available since API level 36.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(36)
void qsort_r(void* _Nullable __array, size_t __n, size_t __size, int (* _Nonnull __comparator)(const void* _Nullable __lhs, const void* _Nullable __rhs, void* _Nullable __context), void* _Nullable __context) __INTRODUCED_IN(36);
+#endif /* __BIONIC_AVAILABILITY_GUARD(36) */
+
uint32_t arc4random(void);
uint32_t arc4random_uniform(uint32_t __upper_bound);
@@ -135,7 +152,11 @@
double drand48(void);
double erand48(unsigned short __xsubi[_Nonnull 3]);
long jrand48(unsigned short __xsubi[_Nonnull 3]);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
void lcong48(unsigned short __param[_Nonnull 7]) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
long lrand48(void);
long mrand48(void);
long nrand48(unsigned short __xsubi[_Nonnull 3]);
@@ -151,7 +172,11 @@
int ptsname_r(int __fd, char* _Nonnull __buf, size_t __n);
int unlockpt(int __fd);
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int getsubopt(char* _Nonnull * _Nonnull __option, char* _Nonnull const* _Nonnull __tokens, char* _Nullable * _Nonnull __value_ptr) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
typedef struct {
int quot;
@@ -181,13 +206,21 @@
*
* Returns the number of samples written to `__averages` (at most 3), and returns -1 on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(29)
int getloadavg(double __averages[_Nonnull], int __n) __INTRODUCED_IN(29);
+#endif /* __BIONIC_AVAILABILITY_GUARD(29) */
+
/* BSD compatibility. */
const char* _Nullable getprogname(void);
void setprogname(const char* _Nonnull __name);
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int mblen(const char* _Nullable __s, size_t __n) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
size_t mbstowcs(wchar_t* _Nullable __dst, const char* _Nullable __src, size_t __n);
int mbtowc(wchar_t* _Nullable __wc_ptr, const char* _Nullable __s, size_t __n);
int wctomb(char* _Nullable __dst, wchar_t __wc);
diff --git a/libc/include/string.h b/libc/include/string.h
index 7c1c3be..79aac91 100644
--- a/libc/include/string.h
+++ b/libc/include/string.h
@@ -52,7 +52,11 @@
int memcmp(const void* _Nonnull __lhs, const void* _Nonnull __rhs, size_t __n) __attribute_pure__;
void* _Nonnull memcpy(void* _Nonnull, const void* _Nonnull, size_t);
#if defined(__USE_GNU)
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
void* _Nonnull mempcpy(void* _Nonnull __dst, const void* _Nonnull __src, size_t __n) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
#endif
void* _Nonnull memmove(void* _Nonnull __dst, const void* _Nonnull __src, size_t __n);
@@ -71,7 +75,11 @@
*
* Returns `dst`.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(34)
void* _Nonnull memset_explicit(void* _Nonnull __dst, int __ch, size_t __n) __INTRODUCED_IN(34);
+#endif /* __BIONIC_AVAILABILITY_GUARD(34) */
+
void* _Nullable memmem(const void* _Nonnull __haystack, size_t __haystack_size, const void* _Nonnull __needle, size_t __needle_size) __attribute_pure__;
@@ -79,10 +87,18 @@
char* _Nullable __strchr_chk(const char* _Nonnull __s, int __ch, size_t __n);
#if defined(__USE_GNU)
#if defined(__cplusplus)
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
extern "C++" char* _Nonnull strchrnul(char* _Nonnull __s, int __ch) __RENAME(strchrnul) __attribute_pure__ __INTRODUCED_IN(24);
extern "C++" const char* _Nonnull strchrnul(const char* _Nonnull __s, int __ch) __RENAME(strchrnul) __attribute_pure__ __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
#else
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
char* _Nonnull strchrnul(const char* _Nonnull __s, int __ch) __attribute_pure__ __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
#endif
#endif
@@ -156,7 +172,11 @@
* Available since API level 35.
*/
#if defined(__USE_GNU)
+
+#if __BIONIC_AVAILABILITY_GUARD(35)
const char* _Nullable strerrorname_np(int __errno_value) __INTRODUCED_IN(35);
+#endif /* __BIONIC_AVAILABILITY_GUARD(35) */
+
#endif
/**
@@ -199,10 +219,18 @@
* It doesn't modify its argument, and in C++ it's const-correct.
*/
#if defined(__cplusplus)
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
extern "C++" char* _Nonnull basename(char* _Nullable __path) __RENAME(__gnu_basename) __INTRODUCED_IN(23);
extern "C++" const char* _Nonnull basename(const char* _Nonnull __path) __RENAME(__gnu_basename) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
#else
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
char* _Nonnull basename(const char* _Nonnull __path) __RENAME(__gnu_basename) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
#endif
#endif
diff --git a/libc/include/strings.h b/libc/include/strings.h
index d203bd2..7543edc 100644
--- a/libc/include/strings.h
+++ b/libc/include/strings.h
@@ -43,8 +43,9 @@
* @brief Extra string functions.
*/
-#include <sys/types.h>
#include <sys/cdefs.h>
+
+#include <sys/types.h>
#include <xlocale.h>
#include <bits/strcasecmp.h>
diff --git a/libc/include/sys/cachectl.h b/libc/include/sys/cachectl.h
index b5fabe3..d06d683 100644
--- a/libc/include/sys/cachectl.h
+++ b/libc/include/sys/cachectl.h
@@ -48,7 +48,7 @@
/**
* __riscv_flush_icache(2) flushes the instruction cache for the given range of addresses.
- * The address range is currently (Linux 6.4) ignored, so both pointers may be null.
+ * The address range is currently (Linux 6.12) ignored, so both pointers may be null.
*
* Returns 0 on success, and returns -1 and sets `errno` on failure.
*/
diff --git a/libc/include/sys/epoll.h b/libc/include/sys/epoll.h
index a5e3c14..bec7c64 100644
--- a/libc/include/sys/epoll.h
+++ b/libc/include/sys/epoll.h
@@ -88,13 +88,19 @@
*
* Available since API level 28.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int epoll_pwait64(int __epoll_fd, struct epoll_event* _Nonnull __events, int __event_count, int __timeout_ms, const sigset64_t* _Nullable __mask) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
/**
* Like epoll_pwait() but with a `struct timespec` timeout, for nanosecond resolution.
*
* Available since API level 35.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(35)
int epoll_pwait2(int __epoll_fd, struct epoll_event* _Nonnull __events, int __event_count, const struct timespec* _Nullable __timeout, const sigset_t* _Nullable __mask) __INTRODUCED_IN(35);
/**
@@ -103,5 +109,7 @@
* Available since API level 35.
*/
int epoll_pwait2_64(int __epoll_fd, struct epoll_event* _Nonnull __events, int __event_count, const struct timespec* _Nullable __timeout, const sigset64_t* _Nullable __mask) __INTRODUCED_IN(35);
+#endif /* __BIONIC_AVAILABILITY_GUARD(35) */
+
__END_DECLS
diff --git a/libc/include/sys/mman.h b/libc/include/sys/mman.h
index 1a0e7f6..38cbf2f 100644
--- a/libc/include/sys/mman.h
+++ b/libc/include/sys/mman.h
@@ -133,7 +133,11 @@
*
* Returns 0 on success, and returns -1 and sets `errno` on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(30)
int mlock2(const void* _Nonnull __addr, size_t __size, int __flags) __INTRODUCED_IN(30);
+#endif /* __BIONIC_AVAILABILITY_GUARD(30) */
+
/**
* [munlock(2)](https://man7.org/linux/man-pages/man2/munlock.2.html)
@@ -171,7 +175,11 @@
*
* Returns the number of bytes advised on success, and returns -1 and sets `errno` on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(31)
ssize_t process_madvise(int __pid_fd, const struct iovec* _Nonnull __iov, size_t __count, int __advice, unsigned __flags) __INTRODUCED_IN(31);
+#endif /* __BIONIC_AVAILABILITY_GUARD(31) */
+
#if defined(__USE_GNU)
@@ -183,7 +191,11 @@
*
* Returns an fd on success, and returns -1 and sets `errno` on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(30)
int memfd_create(const char* _Nonnull __name, unsigned __flags) __INTRODUCED_IN(30);
+#endif /* __BIONIC_AVAILABILITY_GUARD(30) */
+
#endif
@@ -220,7 +232,11 @@
*
* Returns 0 on success, and returns a positive error number on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int posix_madvise(void* _Nonnull __addr, size_t __size, int __advice) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
/**
* [mseal(2)](https://man7.org/linux/man-pages/man2/mseal.2.html)
@@ -232,6 +248,10 @@
*
* Returns 0 on success, and returns -1 and sets `errno` on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(36)
int mseal(void* _Nonnull __addr, size_t __size, unsigned long __flags) __INTRODUCED_IN(36);
+#endif /* __BIONIC_AVAILABILITY_GUARD(36) */
+
__END_DECLS
diff --git a/libc/include/sys/msg.h b/libc/include/sys/msg.h
index 26071b1..8b619be 100644
--- a/libc/include/sys/msg.h
+++ b/libc/include/sys/msg.h
@@ -46,6 +46,8 @@
typedef __kernel_ulong_t msglen_t;
/** Not useful on Android; disallowed by SELinux. */
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int msgctl(int __msg_id, int __op, struct msqid_ds* _Nullable __buf) __INTRODUCED_IN(26);
/** Not useful on Android; disallowed by SELinux. */
int msgget(key_t __key, int __flags) __INTRODUCED_IN(26);
@@ -53,5 +55,7 @@
ssize_t msgrcv(int __msg_id, void* _Nonnull __msgbuf_ptr, size_t __size, long __type, int __flags) __INTRODUCED_IN(26);
/** Not useful on Android; disallowed by SELinux. */
int msgsnd(int __msg_id, const void* _Nonnull __msgbuf_ptr, size_t __size, int __flags) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
__END_DECLS
diff --git a/libc/include/sys/param.h b/libc/include/sys/param.h
index 1c991ae..99b6a07 100644
--- a/libc/include/sys/param.h
+++ b/libc/include/sys/param.h
@@ -33,10 +33,11 @@
* @brief Various macros.
*/
+#include <sys/cdefs.h>
+
#include <endian.h>
#include <limits.h>
#include <linux/param.h>
-#include <sys/cdefs.h>
/** The unit of `st_blocks` in `struct stat`. */
#define DEV_BSIZE 512
diff --git a/libc/include/sys/pidfd.h b/libc/include/sys/pidfd.h
index 30455bb..aaf49c9 100644
--- a/libc/include/sys/pidfd.h
+++ b/libc/include/sys/pidfd.h
@@ -49,6 +49,8 @@
*
* Available since API level 31.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(31)
int pidfd_open(pid_t __pid, unsigned int __flags) __INTRODUCED_IN(31);
/**
@@ -72,5 +74,7 @@
* Available since API level 31.
*/
int pidfd_send_signal(int __pidfd, int __sig, siginfo_t * _Nullable __info, unsigned int __flags) __INTRODUCED_IN(31);
+#endif /* __BIONIC_AVAILABILITY_GUARD(31) */
+
__END_DECLS
diff --git a/libc/include/sys/quota.h b/libc/include/sys/quota.h
index 6e32705..af09674 100644
--- a/libc/include/sys/quota.h
+++ b/libc/include/sys/quota.h
@@ -51,6 +51,10 @@
*
* Available since API level 26.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int quotactl(int __op, const char* _Nullable __special, int __id, char* __BIONIC_COMPLICATED_NULLNESS __addr) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
__END_DECLS
diff --git a/libc/include/sys/random.h b/libc/include/sys/random.h
index b4a9993..23d2c3a 100644
--- a/libc/include/sys/random.h
+++ b/libc/include/sys/random.h
@@ -52,6 +52,10 @@
*
* See also arc4random_buf() which is available in all API levels.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
__nodiscard ssize_t getrandom(void* _Nonnull __buffer, size_t __buffer_size, unsigned int __flags) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
__END_DECLS
diff --git a/libc/include/sys/resource.h b/libc/include/sys/resource.h
index 6743343..05ef2c2 100644
--- a/libc/include/sys/resource.h
+++ b/libc/include/sys/resource.h
@@ -54,7 +54,11 @@
int getrusage(int __who, struct rusage* _Nonnull __usage);
+
+#if (!defined(__LP64__) && __ANDROID_API__ >= 24) || (defined(__LP64__))
int prlimit(pid_t __pid, int __resource, const struct rlimit* _Nullable __new_limit, struct rlimit* _Nullable __old_limit) __INTRODUCED_IN_32(24) __INTRODUCED_IN_64(21);
+#endif /* (!defined(__LP64__) && __ANDROID_API__ >= 24) || (defined(__LP64__)) */
+
int prlimit64(pid_t __pid, int __resource, const struct rlimit64* _Nullable __new_limit, struct rlimit64* _Nullable __old_limit);
__END_DECLS
diff --git a/libc/include/sys/select.h b/libc/include/sys/select.h
index d5b3495..a7227b0 100644
--- a/libc/include/sys/select.h
+++ b/libc/include/sys/select.h
@@ -119,6 +119,10 @@
*
* Available since API level 28.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int pselect64(int __max_fd_plus_one, fd_set* _Nullable __read_fds, fd_set* _Nullable __write_fds, fd_set* _Nullable __exception_fds, const struct timespec* _Nullable __timeout, const sigset64_t* _Nullable __mask) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
__END_DECLS
diff --git a/libc/include/sys/sem.h b/libc/include/sys/sem.h
index 5682282..72f567e 100644
--- a/libc/include/sys/sem.h
+++ b/libc/include/sys/sem.h
@@ -51,12 +51,20 @@
void* _Nullable __pad;
};
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int semctl(int __sem_id, int __sem_num, int __op, ...) __INTRODUCED_IN(26);
int semget(key_t __key, int __sem_count, int __flags) __INTRODUCED_IN(26);
int semop(int __sem_id, struct sembuf* _Nonnull __ops, size_t __op_count) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
#if defined(__USE_GNU)
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int semtimedop(int __sem_id, struct sembuf* _Nonnull __ops, size_t __op_count, const struct timespec* _Nullable __timeout) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
#endif
__END_DECLS
diff --git a/libc/include/sys/shm.h b/libc/include/sys/shm.h
index 8ab3d9a..a960781 100644
--- a/libc/include/sys/shm.h
+++ b/libc/include/sys/shm.h
@@ -48,6 +48,8 @@
typedef unsigned long shmatt_t;
/** Not useful on Android; disallowed by SELinux. */
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
void* _Nonnull shmat(int __shm_id, const void* _Nullable __addr, int __flags) __INTRODUCED_IN(26);
/** Not useful on Android; disallowed by SELinux. */
int shmctl(int __shm_id, int __op, struct shmid_ds* _Nullable __buf) __INTRODUCED_IN(26);
@@ -55,5 +57,7 @@
int shmdt(const void* _Nonnull __addr) __INTRODUCED_IN(26);
/** Not useful on Android; disallowed by SELinux. */
int shmget(key_t __key, size_t __size, int __flags) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
__END_DECLS
diff --git a/libc/include/sys/signalfd.h b/libc/include/sys/signalfd.h
index 5568b7d..eaea525 100644
--- a/libc/include/sys/signalfd.h
+++ b/libc/include/sys/signalfd.h
@@ -51,6 +51,10 @@
/**
* Like signalfd() but allows setting a signal mask with RT signals even from a 32-bit process.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int signalfd64(int __fd, const sigset64_t* _Nonnull __mask, int __flags) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
__END_DECLS
diff --git a/libc/include/sys/stat.h b/libc/include/sys/stat.h
index 2633b69..0b4b248 100644
--- a/libc/include/sys/stat.h
+++ b/libc/include/sys/stat.h
@@ -33,9 +33,10 @@
* @brief File status.
*/
+#include <sys/cdefs.h>
+
#include <bits/timespec.h>
#include <linux/stat.h>
-#include <sys/cdefs.h>
#include <sys/types.h>
__BEGIN_DECLS
@@ -176,7 +177,11 @@
*
* Returns 0 on success and returns -1 and sets `errno` on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(36)
int lchmod(const char* _Nonnull __path, mode_t __mode) __INTRODUCED_IN(36);
+#endif /* __BIONIC_AVAILABILITY_GUARD(36) */
+
/**
* [mkdir(2)](https://man7.org/linux/man-pages/man2/mkdir.2.html)
@@ -280,7 +285,11 @@
*
* Returns 0 on success and returns -1 and sets `errno` on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int mkfifoat(int __dir_fd, const char* _Nonnull __path, mode_t __mode) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
/**
* Used in the tv_nsec field of an argument to utimensat()/futimens()
@@ -331,7 +340,11 @@
*
* Available since API level 30.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(30)
int statx(int __dir_fd, const char* _Nullable __path, int __flags, unsigned __mask, struct statx* _Nonnull __buf) __INTRODUCED_IN(30);
+#endif /* __BIONIC_AVAILABILITY_GUARD(30) */
+
#endif
__END_DECLS
diff --git a/libc/include/sys/statvfs.h b/libc/include/sys/statvfs.h
index 2feca81..860824b 100644
--- a/libc/include/sys/statvfs.h
+++ b/libc/include/sys/statvfs.h
@@ -21,8 +21,9 @@
* @brief Filesystem statistics.
*/
-#include <stdint.h>
#include <sys/cdefs.h>
+
+#include <stdint.h>
#include <sys/types.h>
__BEGIN_DECLS
diff --git a/libc/include/sys/syscall.h b/libc/include/sys/syscall.h
index a49323d..9341ffb 100644
--- a/libc/include/sys/syscall.h
+++ b/libc/include/sys/syscall.h
@@ -29,9 +29,10 @@
#ifndef _SYS_SYSCALL_H_
#define _SYS_SYSCALL_H_
+#include <sys/cdefs.h>
+
#include <asm/unistd.h> /* Linux kernel __NR_* names. */
#include <bits/glibc-syscalls.h> /* glibc-compatible SYS_* aliases. */
-#include <sys/cdefs.h>
/* The syscall function itself is declared in <unistd.h>, not here. */
diff --git a/libc/include/sys/sysinfo.h b/libc/include/sys/sysinfo.h
index 5956feb..ed6a007 100644
--- a/libc/include/sys/sysinfo.h
+++ b/libc/include/sys/sysinfo.h
@@ -53,6 +53,8 @@
*
* See also sysconf().
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int get_nprocs_conf(void) __INTRODUCED_IN(23);
/**
@@ -84,5 +86,7 @@
* See also sysconf().
*/
long get_avphys_pages(void) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
__END_DECLS
diff --git a/libc/include/sys/system_properties.h b/libc/include/sys/system_properties.h
index e8b6e34..1303079 100644
--- a/libc/include/sys/system_properties.h
+++ b/libc/include/sys/system_properties.h
@@ -71,9 +71,13 @@
*
* Available since API level 26.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
void __system_property_read_callback(const prop_info* _Nonnull __pi,
void (* _Nonnull __callback)(void* _Nullable __cookie, const char* _Nonnull __name, const char* _Nonnull __value, uint32_t __serial),
void* _Nullable __cookie) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/**
* Passes a `prop_info` for each system property to the provided
@@ -101,8 +105,12 @@
* Available since API level 26.
*/
struct timespec;
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
bool __system_property_wait(const prop_info* _Nullable __pi, uint32_t __old_serial, uint32_t* _Nonnull __new_serial_ptr, const struct timespec* _Nullable __relative_timeout)
__INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/**
* Deprecated: there's no limit on the length of a property name since
@@ -233,7 +241,11 @@
*
* Available since API level 35.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(35)
int __system_properties_zygote_reload(void) __INTRODUCED_IN(35);
+#endif /* __BIONIC_AVAILABILITY_GUARD(35) */
+
/**
* Deprecated: previously for testing, but now that SystemProperties is its own
diff --git a/libc/include/sys/thread_properties.h b/libc/include/sys/thread_properties.h
index efd212a..b6214ee 100644
--- a/libc/include/sys/thread_properties.h
+++ b/libc/include/sys/thread_properties.h
@@ -50,6 +50,8 @@
*
* Available since API level 31.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(31)
void __libc_get_static_tls_bounds(void* _Nonnull * _Nonnull __static_tls_begin,
void* _Nonnull * _Nonnull __static_tls_end) __INTRODUCED_IN(31);
@@ -93,5 +95,7 @@
void* _Nonnull __dynamic_tls_end),
void (* _Nonnull __on_destruction)(void* _Nonnull __dynamic_tls_begin,
void* _Nonnull __dynamic_tls_end)) __INTRODUCED_IN(31);
+#endif /* __BIONIC_AVAILABILITY_GUARD(31) */
+
__END_DECLS
diff --git a/libc/include/sys/time.h b/libc/include/sys/time.h
index 6ba7a37..d12c306 100644
--- a/libc/include/sys/time.h
+++ b/libc/include/sys/time.h
@@ -47,8 +47,12 @@
int utimes(const char* _Nonnull __path, const struct timeval __times[_Nullable 2]);
#if defined(__USE_BSD)
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int futimes(int __fd, const struct timeval __times[_Nullable 2]) __INTRODUCED_IN(26);
int lutimes(const char* _Nonnull __path, const struct timeval __times[_Nullable 2]) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
#endif
#if defined(__USE_GNU)
@@ -65,7 +69,11 @@
*
* Available since API level 26.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int futimesat(int __dir_fd, const char* __BIONIC_COMPLICATED_NULLNESS __path, const struct timeval __times[_Nullable 2]) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
#endif
#define timerclear(a) \
diff --git a/libc/include/sys/timerfd.h b/libc/include/sys/timerfd.h
index bfa9a55..f7f1ffa 100644
--- a/libc/include/sys/timerfd.h
+++ b/libc/include/sys/timerfd.h
@@ -33,10 +33,11 @@
* @brief Timer file descriptors.
*/
+#include <sys/cdefs.h>
+
#include <fcntl.h>
#include <linux/timerfd.h>
#include <time.h>
-#include <sys/cdefs.h>
#include <sys/types.h>
__BEGIN_DECLS
diff --git a/libc/include/sys/timex.h b/libc/include/sys/timex.h
index 828eb47..6fb58e4 100644
--- a/libc/include/sys/timex.h
+++ b/libc/include/sys/timex.h
@@ -46,6 +46,8 @@
*
* Available since API level 24.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
int adjtimex(struct timex* _Nonnull __buf) __INTRODUCED_IN(24);
/**
@@ -56,5 +58,7 @@
* Available since API level 24.
*/
int clock_adjtime(clockid_t __clock, struct timex* _Nonnull __tx) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
__END_DECLS
diff --git a/libc/include/sys/types.h b/libc/include/sys/types.h
index 4622a4e..0446260 100644
--- a/libc/include/sys/types.h
+++ b/libc/include/sys/types.h
@@ -29,9 +29,10 @@
#ifndef _SYS_TYPES_H_
#define _SYS_TYPES_H_
+#include <sys/cdefs.h>
+
#include <stddef.h>
#include <stdint.h>
-#include <sys/cdefs.h>
#include <linux/types.h>
#include <linux/posix_types.h>
diff --git a/libc/include/sys/uio.h b/libc/include/sys/uio.h
index d3e6561..eff3b14 100644
--- a/libc/include/sys/uio.h
+++ b/libc/include/sys/uio.h
@@ -69,6 +69,8 @@
*
* Available since API level 24.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(24)
ssize_t preadv(int __fd, const struct iovec* _Nonnull __iov, int __count, off_t __offset) __RENAME_IF_FILE_OFFSET64(preadv64) __INTRODUCED_IN(24);
/**
@@ -96,6 +98,8 @@
* Available since API level 24.
*/
ssize_t pwritev64(int __fd, const struct iovec* _Nonnull __iov, int __count, off64_t __offset) __INTRODUCED_IN(24);
+#endif /* __BIONIC_AVAILABILITY_GUARD(24) */
+
/**
* [preadv2(2)](https://man7.org/linux/man-pages/man2/preadv2.2.html) reads
@@ -107,6 +111,8 @@
*
* Available since API level 33.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(33)
ssize_t preadv2(int __fd, const struct iovec* _Nonnull __iov, int __count, off_t __offset, int __flags) __RENAME_IF_FILE_OFFSET64(preadv64v2) __INTRODUCED_IN(33);
/**
@@ -134,6 +140,8 @@
* Available since API level 33.
*/
ssize_t pwritev64v2(int __fd, const struct iovec* _Nonnull __iov, int __count, off64_t __offset, int __flags) __INTRODUCED_IN(33);
+#endif /* __BIONIC_AVAILABILITY_GUARD(33) */
+
/**
* [process_vm_readv(2)](https://man7.org/linux/man-pages/man2/process_vm_readv.2.html)
@@ -144,6 +152,8 @@
*
* Available since API level 23.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
ssize_t process_vm_readv(pid_t __pid, const struct iovec* __BIONIC_COMPLICATED_NULLNESS __local_iov, unsigned long __local_iov_count, const struct iovec* __BIONIC_COMPLICATED_NULLNESS __remote_iov, unsigned long __remote_iov_count, unsigned long __flags) __INTRODUCED_IN(23);
/**
@@ -156,6 +166,8 @@
* Available since API level 23.
*/
ssize_t process_vm_writev(pid_t __pid, const struct iovec* __BIONIC_COMPLICATED_NULLNESS __local_iov, unsigned long __local_iov_count, const struct iovec* __BIONIC_COMPLICATED_NULLNESS __remote_iov, unsigned long __remote_iov_count, unsigned long __flags) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
#endif
diff --git a/libc/include/sys/un.h b/libc/include/sys/un.h
index 83c1d17..c2bfcb0 100644
--- a/libc/include/sys/un.h
+++ b/libc/include/sys/un.h
@@ -33,9 +33,10 @@
* @brief Unix domain sockets.
*/
+#include <sys/cdefs.h>
+
#include <bits/sa_family_t.h>
#include <linux/un.h>
-#include <sys/cdefs.h>
#if defined(__USE_BSD) || defined(__USE_GNU)
#include <string.h>
diff --git a/libc/include/sys/vfs.h b/libc/include/sys/vfs.h
index 1a640ba..5d078be 100644
--- a/libc/include/sys/vfs.h
+++ b/libc/include/sys/vfs.h
@@ -29,8 +29,9 @@
#ifndef _SYS_VFS_H_
#define _SYS_VFS_H_
-#include <stdint.h>
#include <sys/cdefs.h>
+
+#include <stdint.h>
#include <sys/types.h>
__BEGIN_DECLS
diff --git a/libc/include/sys/wait.h b/libc/include/sys/wait.h
index 5208366..632aa43 100644
--- a/libc/include/sys/wait.h
+++ b/libc/include/sys/wait.h
@@ -28,8 +28,9 @@
#pragma once
-#include <bits/wait.h>
#include <sys/cdefs.h>
+
+#include <bits/wait.h>
#include <sys/types.h>
#include <sys/resource.h>
#include <linux/wait.h>
diff --git a/libc/include/sys/xattr.h b/libc/include/sys/xattr.h
index 38c11e2..ebe4eb8 100644
--- a/libc/include/sys/xattr.h
+++ b/libc/include/sys/xattr.h
@@ -33,8 +33,9 @@
* @brief Extended attribute functions.
*/
-#include <linux/xattr.h>
#include <sys/cdefs.h>
+
+#include <linux/xattr.h>
#include <sys/types.h>
__BEGIN_DECLS
diff --git a/libc/include/syslog.h b/libc/include/syslog.h
index 33979f0..7a594f1 100644
--- a/libc/include/syslog.h
+++ b/libc/include/syslog.h
@@ -56,8 +56,9 @@
#pragma once
-#include <stdio.h>
#include <sys/cdefs.h>
+
+#include <stdio.h>
#include <stdarg.h>
__BEGIN_DECLS
diff --git a/libc/include/time.h b/libc/include/time.h
index e9d6569..6c9b761 100644
--- a/libc/include/time.h
+++ b/libc/include/time.h
@@ -166,7 +166,11 @@
*
* Available since API level 35.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(35)
time_t mktime_z(timezone_t _Nonnull __tz, struct tm* _Nonnull __tm) __INTRODUCED_IN(35);
+#endif /* __BIONIC_AVAILABILITY_GUARD(35) */
+
/**
* [localtime(3)](https://man7.org/linux/man-pages/man3/localtime.3p.html) converts
@@ -200,7 +204,11 @@
*
* Available since API level 35.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(35)
struct tm* _Nullable localtime_rz(timezone_t _Nonnull __tz, const time_t* _Nonnull __t, struct tm* _Nonnull __tm) __INTRODUCED_IN(35);
+#endif /* __BIONIC_AVAILABILITY_GUARD(35) */
+
/**
* Inverse of localtime().
@@ -314,6 +322,8 @@
*
* Available since API level 35.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(35)
timezone_t _Nullable tzalloc(const char* _Nullable __id) __INTRODUCED_IN(35);
/**
@@ -326,6 +336,8 @@
* Available since API level 35.
*/
void tzfree(timezone_t _Nullable __tz) __INTRODUCED_IN(35);
+#endif /* __BIONIC_AVAILABILITY_GUARD(35) */
+
/**
* [clock(3)](https://man7.org/linux/man-pages/man3/clock.3.html)
@@ -345,7 +357,11 @@
*
* Returns 0 on success, and returns -1 and returns an error number on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int clock_getcpuclockid(pid_t __pid, clockid_t* _Nonnull __clock) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
/**
* [clock_getres(2)](https://man7.org/linux/man-pages/man2/clock_getres.2.html)
@@ -459,7 +475,11 @@
* Available since API level 29 for TIME_UTC; other bases arrived later.
* Code for Android should prefer clock_gettime().
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(29)
int timespec_get(struct timespec* _Nonnull __ts, int __base) __INTRODUCED_IN(29);
+#endif /* __BIONIC_AVAILABILITY_GUARD(29) */
+
/**
* timespec_getres(3) is equivalent to clock_getres() for the clock corresponding to the given base.
@@ -469,6 +489,10 @@
* Available since API level 35.
* Code for Android should prefer clock_gettime().
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(35)
int timespec_getres(struct timespec* _Nonnull __ts, int __base) __INTRODUCED_IN(35);
+#endif /* __BIONIC_AVAILABILITY_GUARD(35) */
+
__END_DECLS
diff --git a/libc/include/uchar.h b/libc/include/uchar.h
index 55a36e7..94efb2d 100644
--- a/libc/include/uchar.h
+++ b/libc/include/uchar.h
@@ -33,9 +33,10 @@
* @brief Unicode functions.
*/
-#include <stddef.h>
#include <sys/cdefs.h>
+#include <stddef.h>
+
#include <bits/bionic_multibyte_result.h>
#include <bits/mbstate_t.h>
diff --git a/libc/include/unistd.h b/libc/include/unistd.h
index e1c268f..e623339 100644
--- a/libc/include/unistd.h
+++ b/libc/include/unistd.h
@@ -28,8 +28,9 @@
#pragma once
-#include <stddef.h>
#include <sys/cdefs.h>
+
+#include <stddef.h>
#include <sys/types.h>
#include <sys/select.h>
@@ -100,7 +101,11 @@
*
* Available since API level 35.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(35)
pid_t _Fork(void) __INTRODUCED_IN(35);
+#endif /* __BIONIC_AVAILABILITY_GUARD(35) */
+
/**
* [vfork(2)](https://man7.org/linux/man-pages/man2/vfork.2.html) creates a new
@@ -145,7 +150,11 @@
int execlp(const char* _Nonnull __file, const char* _Nullable __arg0, ...) __attribute__((__sentinel__));
int execle(const char* _Nonnull __path, const char* _Nullable __arg0, ... /*, char* const* __envp */)
__attribute__((__sentinel__(1)));
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int fexecve(int __fd, char* _Nullable const* _Nullable __argv, char* _Nullable const* _Nullable __envp) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
int nice(int __incr);
@@ -246,7 +255,11 @@
int getresuid(uid_t* _Nonnull __ruid, uid_t* _Nonnull __euid, uid_t* _Nonnull __suid);
int getresgid(gid_t* _Nonnull __rgid, gid_t* _Nonnull __egid, gid_t* _Nonnull __sgid);
char* _Nullable getlogin(void);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int getlogin_r(char* _Nonnull __buffer, size_t __buffer_size) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
long fpathconf(int __fd, int __name);
long pathconf(const char* _Nonnull __path, int __name);
@@ -257,8 +270,29 @@
int linkat(int __old_dir_fd, const char* _Nonnull __old_path, int __new_dir_fd, const char* _Nonnull __new_path, int __flags);
int unlink(const char* _Nonnull __path);
int unlinkat(int __dirfd, const char* _Nonnull __path, int __flags);
+
+/**
+ * [chdir(2)](https://man7.org/linux/man-pages/man2/chdir.2.html) changes
+ * the current working directory to the given path.
+ *
+ * This function affects all threads in the process, so is generally a bad idea
+ * on Android where most code will be running in a multi-threaded context.
+ *
+ * Returns 0 on success, and returns -1 and sets `errno` on failure.
+ */
int chdir(const char* _Nonnull __path);
+
+/**
+ * [fchdir(2)](https://man7.org/linux/man-pages/man2/chdir.2.html) changes
+ * the current working directory to the given fd.
+ *
+ * This function affects all threads in the process, so is generally a bad idea
+ * on Android where most code will be running in a multi-threaded context.
+ *
+ * Returns 0 on success, and returns -1 and sets `errno` on failure.
+ */
int fchdir(int __fd);
+
int rmdir(const char* _Nonnull __path);
int pipe(int __fds[_Nonnull 2]);
#if defined(__USE_GNU)
@@ -277,7 +311,11 @@
void sync(void);
#if defined(__USE_GNU)
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
int syncfs(int __fd) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
#endif
int close(int __fd);
@@ -339,7 +377,11 @@
int usleep(useconds_t __microseconds);
int gethostname(char* _Nonnull _buf, size_t __buf_size);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int sethostname(const char* _Nonnull __name, size_t __n) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
int brk(void* _Nonnull __addr);
void* _Nullable sbrk(ptrdiff_t __increment);
@@ -382,8 +424,12 @@
} while (_rc == -1 && errno == EINTR); \
_rc; })
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
int getdomainname(char* _Nonnull __buf, size_t __buf_size) __INTRODUCED_IN(26);
int setdomainname(const char* _Nonnull __name, size_t __n) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
/**
* [copy_file_range(2)](https://man7.org/linux/man-pages/man2/copy_file_range.2.html) copies
@@ -394,7 +440,11 @@
* Returns the number of bytes copied on success, and returns -1 and sets
* `errno` on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(34)
ssize_t copy_file_range(int __fd_in, off64_t* _Nullable __off_in, int __fd_out, off64_t* _Nullable __off_out, size_t __length, unsigned int __flags) __INTRODUCED_IN(34);
+#endif /* __BIONIC_AVAILABILITY_GUARD(34) */
+
#if __ANDROID_API__ >= 28
void swab(const void* _Nonnull __src, void* _Nonnull __dst, ssize_t __byte_count) __INTRODUCED_IN(28);
@@ -414,7 +464,11 @@
*
* Returns 0 on success, and returns -1 and sets `errno` on failure.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(34)
int close_range(unsigned int __min_fd, unsigned int __max_fd, int __flags) __INTRODUCED_IN(34);
+#endif /* __BIONIC_AVAILABILITY_GUARD(34) */
+
#if defined(__BIONIC_INCLUDE_FORTIFY_HEADERS)
#define _UNISTD_H_
diff --git a/libc/include/utmp.h b/libc/include/utmp.h
index d249f8a..1674491 100644
--- a/libc/include/utmp.h
+++ b/libc/include/utmp.h
@@ -131,6 +131,10 @@
*
* Available since API level 23.
*/
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int login_tty(int __fd) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
__END_DECLS
diff --git a/libc/include/wchar.h b/libc/include/wchar.h
index e86f94d..56594dc 100644
--- a/libc/include/wchar.h
+++ b/libc/include/wchar.h
@@ -75,7 +75,11 @@
wchar_t* _Nonnull wcpncpy(wchar_t* _Nonnull __dst, const wchar_t* _Nonnull __src, size_t __n);
size_t wcrtomb(char* _Nullable __buf, wchar_t __wc, mbstate_t* _Nullable __ps);
int wcscasecmp(const wchar_t* _Nonnull __lhs, const wchar_t* _Nonnull __rhs);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int wcscasecmp_l(const wchar_t* _Nonnull __lhs, const wchar_t* _Nonnull __rhs, locale_t _Nonnull __l) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
wchar_t* _Nonnull wcscat(wchar_t* _Nonnull __dst, const wchar_t* _Nonnull __src);
wchar_t* _Nullable wcschr(const wchar_t * _Nonnull __s, wchar_t __wc);
int wcscmp(const wchar_t* _Nonnull __lhs, const wchar_t* _Nonnull __rhs);
@@ -83,10 +87,18 @@
wchar_t* _Nonnull wcscpy(wchar_t* _Nonnull __dst, const wchar_t* _Nonnull __src);
size_t wcscspn(const wchar_t* _Nonnull __s, const wchar_t* _Nonnull __accept);
size_t wcsftime(wchar_t* _Nonnull __buf, size_t __n, const wchar_t* _Nullable __fmt, const struct tm* _Nonnull __tm);
+
+#if __BIONIC_AVAILABILITY_GUARD(28)
size_t wcsftime_l(wchar_t* _Nonnull __buf, size_t __n, const wchar_t* _Nullable __fmt, const struct tm* _Nonnull __tm, locale_t _Nonnull __l) __INTRODUCED_IN(28);
+#endif /* __BIONIC_AVAILABILITY_GUARD(28) */
+
size_t wcslen(const wchar_t* _Nonnull __s);
int wcsncasecmp(const wchar_t* _Nonnull __lhs, const wchar_t* _Nonnull __rhs, size_t __n);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
int wcsncasecmp_l(const wchar_t* _Nonnull __lhs, const wchar_t* _Nonnull __rhs, size_t __n, locale_t _Nonnull __l) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
wchar_t* _Nonnull wcsncat(wchar_t* _Nonnull __dst, const wchar_t* _Nonnull __src, size_t __n);
int wcsncmp(const wchar_t* _Nonnull __lhs, const wchar_t* _Nonnull __rhs, size_t __n);
wchar_t* _Nonnull wcsncpy(wchar_t* _Nonnull __dst, const wchar_t* _Nonnull __src, size_t __n);
@@ -117,7 +129,11 @@
int wmemcmp(const wchar_t* _Nullable __lhs, const wchar_t* _Nullable __rhs, size_t __n);
wchar_t* _Nonnull wmemcpy(wchar_t* _Nonnull __dst, const wchar_t* _Nonnull __src, size_t __n);
#if defined(__USE_GNU)
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
wchar_t* _Nonnull wmempcpy(wchar_t* _Nonnull __dst, const wchar_t* _Nonnull __src, size_t __n) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
#endif
wchar_t* _Nonnull wmemmove(wchar_t* _Nonnull __dst, const wchar_t* _Nonnull __src, size_t __n);
wchar_t* _Nonnull wmemset(wchar_t* _Nonnull __dst, wchar_t __wc, size_t __n);
@@ -133,7 +149,11 @@
size_t wcslcat(wchar_t* _Nonnull __dst, const wchar_t* _Nonnull __src, size_t __n);
size_t wcslcpy(wchar_t* _Nonnull __dst, const wchar_t* _Nonnull __src, size_t __n);
+
+#if __BIONIC_AVAILABILITY_GUARD(23)
FILE* _Nullable open_wmemstream(wchar_t* _Nonnull * _Nonnull __ptr, size_t* _Nonnull __size_ptr) __INTRODUCED_IN(23);
+#endif /* __BIONIC_AVAILABILITY_GUARD(23) */
+
wchar_t* _Nullable wcsdup(const wchar_t* _Nonnull __s);
size_t wcsnlen(const wchar_t* _Nonnull __s, size_t __n);
diff --git a/libc/include/wctype.h b/libc/include/wctype.h
index 4f6f81f..30ec04f 100644
--- a/libc/include/wctype.h
+++ b/libc/include/wctype.h
@@ -29,8 +29,9 @@
#ifndef _WCTYPE_H_
#define _WCTYPE_H_
-#include <bits/wctype.h>
#include <sys/cdefs.h>
+
+#include <bits/wctype.h>
#include <xlocale.h>
__BEGIN_DECLS
@@ -51,8 +52,12 @@
wint_t towlower_l(wint_t __wc, locale_t _Nonnull __l);
wint_t towupper_l(wint_t __wc, locale_t _Nonnull __l);
+
+#if __BIONIC_AVAILABILITY_GUARD(26)
wint_t towctrans_l(wint_t __wc, wctrans_t _Nonnull __transform, locale_t _Nonnull __l) __INTRODUCED_IN(26);
wctrans_t _Nonnull wctrans_l(const char* _Nonnull __name, locale_t _Nonnull __l) __INTRODUCED_IN(26);
+#endif /* __BIONIC_AVAILABILITY_GUARD(26) */
+
wctype_t wctype_l(const char* _Nonnull __name, locale_t _Nonnull __l);
int iswctype_l(wint_t __wc, wctype_t __transform, locale_t _Nonnull __l);
diff --git a/libc/kernel/android/scsi/scsi/scsi_proto.h b/libc/kernel/android/scsi/scsi/scsi_proto.h
index d873fad..754e12a 100644
--- a/libc/kernel/android/scsi/scsi/scsi_proto.h
+++ b/libc/kernel/android/scsi/scsi/scsi_proto.h
@@ -109,6 +109,7 @@
#define WRITE_SAME_16 0x93
#define ZBC_OUT 0x94
#define ZBC_IN 0x95
+#define WRITE_ATOMIC_16 0x9c
#define SERVICE_ACTION_BIDIRECTIONAL 0x9d
#define SERVICE_ACTION_IN_16 0x9e
#define SERVICE_ACTION_OUT_16 0x9f
diff --git a/libc/kernel/tools/cpp.py b/libc/kernel/tools/cpp.py
index 08b786a..df50806 100755
--- a/libc/kernel/tools/cpp.py
+++ b/libc/kernel/tools/cpp.py
@@ -1430,7 +1430,7 @@
state = VAR_DECL
elif state == NORMAL and token_id in ['struct', 'typedef',
'enum', 'union',
- '__extension__']:
+ '__extension__', '=']:
state = OTHER_DECL
state_token = token_id
elif block.tokens[i].kind == TokenKind.IDENTIFIER:
@@ -2057,7 +2057,7 @@
struct timeval val2;
};
"""
- self.assertEqual(self.parse(text, {"remove": True}), expected)
+ self.assertEqual(self.parse(text, {"remove": None}), expected)
def test_remove_struct_from_end(self):
text = """\
@@ -2076,7 +2076,7 @@
struct timeval val2;
};
"""
- self.assertEqual(self.parse(text, {"remove": True}), expected)
+ self.assertEqual(self.parse(text, {"remove": None}), expected)
def test_remove_minimal_struct(self):
text = """\
@@ -2084,7 +2084,7 @@
};
"""
expected = "";
- self.assertEqual(self.parse(text, {"remove": True}), expected)
+ self.assertEqual(self.parse(text, {"remove": None}), expected)
def test_remove_struct_with_struct_fields(self):
text = """\
@@ -2104,7 +2104,7 @@
struct remove val2;
};
"""
- self.assertEqual(self.parse(text, {"remove": True}), expected)
+ self.assertEqual(self.parse(text, {"remove": None}), expected)
def test_remove_consecutive_structs(self):
text = """\
@@ -2136,7 +2136,7 @@
struct timeval val2;
};
"""
- self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
+ self.assertEqual(self.parse(text, {"remove1": None, "remove2": None}), expected)
def test_remove_multiple_structs(self):
text = """\
@@ -2169,7 +2169,7 @@
int val;
};
"""
- self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
+ self.assertEqual(self.parse(text, {"remove1": None, "remove2": None}), expected)
def test_remove_struct_with_inline_structs(self):
text = """\
@@ -2194,7 +2194,7 @@
struct timeval val2;
};
"""
- self.assertEqual(self.parse(text, {"remove": True}), expected)
+ self.assertEqual(self.parse(text, {"remove": None}), expected)
def test_remove_struct_across_blocks(self):
text = """\
@@ -2219,7 +2219,7 @@
struct timeval val2;
};
"""
- self.assertEqual(self.parse(text, {"remove": True}), expected)
+ self.assertEqual(self.parse(text, {"remove": None}), expected)
def test_remove_struct_across_blocks_multiple_structs(self):
text = """\
@@ -2246,7 +2246,7 @@
struct timeval val2;
};
"""
- self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
+ self.assertEqual(self.parse(text, {"remove1": None, "remove2": None}), expected)
def test_remove_multiple_struct_and_add_includes(self):
text = """\
@@ -2263,7 +2263,7 @@
#include <bits/remove1.h>
#include <bits/remove2.h>
"""
- self.assertEqual(self.parse(text, {"remove1": False, "remove2": False}), expected)
+ self.assertEqual(self.parse(text, {"remove1": "bits/remove1.h", "remove2": "bits/remove2.h"}), expected)
class FullPathTest(unittest.TestCase):
@@ -2580,6 +2580,71 @@
struct timeval timeval;
struct itimerval itimerval;
};
+#include <linux/time.h>
+"""
+ self.assertEqual(self.parse(text), expected)
+
+ def test_var_definition(self):
+ # If we're definining the whole thing, it's probably worth keeping.
+ text = """\
+static const char *kString = "hello world";
+static const int kInteger = 42;
+"""
+ expected = """\
+static const char * kString = "hello world";
+static const int kInteger = 42;
+"""
+ self.assertEqual(self.parse(text), expected)
+
+ def test_struct_array_definition(self):
+ text = """\
+struct descriptor {
+ int args;
+ int size;
+};
+static const struct descriptor[] = {
+ {0, 0},
+ {1, 12},
+ {0, 42},
+};
+"""
+ expected = """\
+struct descriptor {
+ int args;
+ int size;
+};
+static const struct descriptor[] = {
+ {
+ 0, 0
+ }
+ , {
+ 1, 12
+ }
+ , {
+ 0, 42
+ }
+ ,
+};
+"""
+ self.assertEqual(self.parse(text), expected)
+
+ def test_array_definition(self):
+ text = """\
+static const char *arr[] = {
+ "foo",
+ "bar",
+ "baz",
+};
+
+static int another_arr[5] = { 1, 2, 3, 4, 5};
+"""
+ expected = """\
+static const char * arr[] = {
+ "foo", "bar", "baz",
+};
+static int another_arr[5] = {
+ 1, 2, 3, 4, 5
+};
"""
self.assertEqual(self.parse(text), expected)
diff --git a/libc/kernel/tools/defaults.py b/libc/kernel/tools/defaults.py
index 06afb25..a71318e 100644
--- a/libc/kernel/tools/defaults.py
+++ b/libc/kernel/tools/defaults.py
@@ -133,6 +133,9 @@
# These are required to support the above functions.
"__fswahw32",
"__fswahb32",
+ # As are these, for ILP32.
+ "__arch_swab32",
+ "__arch_swab64",
# This is used by various macros in <linux/ioprio.h>.
"ioprio_value",
diff --git a/libc/kernel/tools/update_all.py b/libc/kernel/tools/update_all.py
index ae89a80..331a957 100755
--- a/libc/kernel/tools/update_all.py
+++ b/libc/kernel/tools/update_all.py
@@ -88,15 +88,8 @@
# Collect the set of all syscalls for all architectures.
syscalls = set()
pattern = re.compile(r'^\s*#\s*define\s*__NR_([a-z_]\S+)')
- for unistd_h in ['kernel/uapi/asm-generic/unistd.h',
- 'kernel/uapi/asm-arm/asm/unistd.h',
- 'kernel/uapi/asm-arm/asm/unistd-eabi.h',
- 'kernel/uapi/asm-arm/asm/unistd-oabi.h',
- 'kernel/uapi/asm-riscv/asm/unistd.h',
- 'kernel/uapi/asm-x86/asm/unistd_32.h',
- 'kernel/uapi/asm-x86/asm/unistd_64.h',
- 'kernel/uapi/asm-x86/asm/unistd_x32.h']:
- for line in open(os.path.join(libc_root, unistd_h)):
+ for unistd_h in glob.glob('%s/kernel/uapi/asm-*/asm/unistd*.h' % libc_root):
+ for line in open(unistd_h):
m = re.search(pattern, line)
if m:
nr_name = m.group(1)
diff --git a/libc/kernel/uapi/asm-arm/asm/swab.h b/libc/kernel/uapi/asm-arm/asm/swab.h
index 7684c22..3fff953 100644
--- a/libc/kernel/uapi/asm-arm/asm/swab.h
+++ b/libc/kernel/uapi/asm-arm/asm/swab.h
@@ -11,7 +11,18 @@
#ifndef __STRICT_ANSI__
#define __SWAB_64_THRU_32__
#endif
+static inline __attribute__((__const__)) __u32 __arch_swab32(__u32 x) {
+ __u32 t;
#ifndef __thumb__
+ if(! __builtin_constant_p(x)) {
+ asm("eor\t%0, %1, %1, ror #16" : "=r" (t) : "r" (x));
+ } else
#endif
+ t = x ^ ((x << 16) | (x >> 16));
+ x = (x << 24) | (x >> 8);
+ t &= ~0x00FF0000;
+ x ^= (t >> 8);
+ return x;
+}
#define __arch_swab32 __arch_swab32
#endif
diff --git a/libc/kernel/uapi/asm-arm64/asm/unistd.h b/libc/kernel/uapi/asm-arm64/asm/unistd.h
index 7457ebc..178578f 100644
--- a/libc/kernel/uapi/asm-arm64/asm/unistd.h
+++ b/libc/kernel/uapi/asm-arm64/asm/unistd.h
@@ -4,10 +4,4 @@
* See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
* for more information.
*/
-#define __ARCH_WANT_RENAMEAT
-#define __ARCH_WANT_NEW_STAT
-#define __ARCH_WANT_SET_GET_RLIMIT
-#define __ARCH_WANT_TIME32_SYSCALLS
-#define __ARCH_WANT_SYS_CLONE3
-#define __ARCH_WANT_MEMFD_SECRET
-#include <asm-generic/unistd.h>
+#include <asm/unistd_64.h>
diff --git a/libc/kernel/uapi/asm-arm64/asm/unistd_64.h b/libc/kernel/uapi/asm-arm64/asm/unistd_64.h
new file mode 100644
index 0000000..0a0a1c0
--- /dev/null
+++ b/libc/kernel/uapi/asm-arm64/asm/unistd_64.h
@@ -0,0 +1,327 @@
+/*
+ * This file is auto-generated. Modifications will be lost.
+ *
+ * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
+ * for more information.
+ */
+#ifndef _UAPI_ASM_UNISTD_64_H
+#define _UAPI_ASM_UNISTD_64_H
+#define __NR_io_setup 0
+#define __NR_io_destroy 1
+#define __NR_io_submit 2
+#define __NR_io_cancel 3
+#define __NR_io_getevents 4
+#define __NR_setxattr 5
+#define __NR_lsetxattr 6
+#define __NR_fsetxattr 7
+#define __NR_getxattr 8
+#define __NR_lgetxattr 9
+#define __NR_fgetxattr 10
+#define __NR_listxattr 11
+#define __NR_llistxattr 12
+#define __NR_flistxattr 13
+#define __NR_removexattr 14
+#define __NR_lremovexattr 15
+#define __NR_fremovexattr 16
+#define __NR_getcwd 17
+#define __NR_lookup_dcookie 18
+#define __NR_eventfd2 19
+#define __NR_epoll_create1 20
+#define __NR_epoll_ctl 21
+#define __NR_epoll_pwait 22
+#define __NR_dup 23
+#define __NR_dup3 24
+#define __NR_fcntl 25
+#define __NR_inotify_init1 26
+#define __NR_inotify_add_watch 27
+#define __NR_inotify_rm_watch 28
+#define __NR_ioctl 29
+#define __NR_ioprio_set 30
+#define __NR_ioprio_get 31
+#define __NR_flock 32
+#define __NR_mknodat 33
+#define __NR_mkdirat 34
+#define __NR_unlinkat 35
+#define __NR_symlinkat 36
+#define __NR_linkat 37
+#define __NR_renameat 38
+#define __NR_umount2 39
+#define __NR_mount 40
+#define __NR_pivot_root 41
+#define __NR_nfsservctl 42
+#define __NR_statfs 43
+#define __NR_fstatfs 44
+#define __NR_truncate 45
+#define __NR_ftruncate 46
+#define __NR_fallocate 47
+#define __NR_faccessat 48
+#define __NR_chdir 49
+#define __NR_fchdir 50
+#define __NR_chroot 51
+#define __NR_fchmod 52
+#define __NR_fchmodat 53
+#define __NR_fchownat 54
+#define __NR_fchown 55
+#define __NR_openat 56
+#define __NR_close 57
+#define __NR_vhangup 58
+#define __NR_pipe2 59
+#define __NR_quotactl 60
+#define __NR_getdents64 61
+#define __NR_lseek 62
+#define __NR_read 63
+#define __NR_write 64
+#define __NR_readv 65
+#define __NR_writev 66
+#define __NR_pread64 67
+#define __NR_pwrite64 68
+#define __NR_preadv 69
+#define __NR_pwritev 70
+#define __NR_sendfile 71
+#define __NR_pselect6 72
+#define __NR_ppoll 73
+#define __NR_signalfd4 74
+#define __NR_vmsplice 75
+#define __NR_splice 76
+#define __NR_tee 77
+#define __NR_readlinkat 78
+#define __NR_newfstatat 79
+#define __NR_fstat 80
+#define __NR_sync 81
+#define __NR_fsync 82
+#define __NR_fdatasync 83
+#define __NR_sync_file_range 84
+#define __NR_timerfd_create 85
+#define __NR_timerfd_settime 86
+#define __NR_timerfd_gettime 87
+#define __NR_utimensat 88
+#define __NR_acct 89
+#define __NR_capget 90
+#define __NR_capset 91
+#define __NR_personality 92
+#define __NR_exit 93
+#define __NR_exit_group 94
+#define __NR_waitid 95
+#define __NR_set_tid_address 96
+#define __NR_unshare 97
+#define __NR_futex 98
+#define __NR_set_robust_list 99
+#define __NR_get_robust_list 100
+#define __NR_nanosleep 101
+#define __NR_getitimer 102
+#define __NR_setitimer 103
+#define __NR_kexec_load 104
+#define __NR_init_module 105
+#define __NR_delete_module 106
+#define __NR_timer_create 107
+#define __NR_timer_gettime 108
+#define __NR_timer_getoverrun 109
+#define __NR_timer_settime 110
+#define __NR_timer_delete 111
+#define __NR_clock_settime 112
+#define __NR_clock_gettime 113
+#define __NR_clock_getres 114
+#define __NR_clock_nanosleep 115
+#define __NR_syslog 116
+#define __NR_ptrace 117
+#define __NR_sched_setparam 118
+#define __NR_sched_setscheduler 119
+#define __NR_sched_getscheduler 120
+#define __NR_sched_getparam 121
+#define __NR_sched_setaffinity 122
+#define __NR_sched_getaffinity 123
+#define __NR_sched_yield 124
+#define __NR_sched_get_priority_max 125
+#define __NR_sched_get_priority_min 126
+#define __NR_sched_rr_get_interval 127
+#define __NR_restart_syscall 128
+#define __NR_kill 129
+#define __NR_tkill 130
+#define __NR_tgkill 131
+#define __NR_sigaltstack 132
+#define __NR_rt_sigsuspend 133
+#define __NR_rt_sigaction 134
+#define __NR_rt_sigprocmask 135
+#define __NR_rt_sigpending 136
+#define __NR_rt_sigtimedwait 137
+#define __NR_rt_sigqueueinfo 138
+#define __NR_rt_sigreturn 139
+#define __NR_setpriority 140
+#define __NR_getpriority 141
+#define __NR_reboot 142
+#define __NR_setregid 143
+#define __NR_setgid 144
+#define __NR_setreuid 145
+#define __NR_setuid 146
+#define __NR_setresuid 147
+#define __NR_getresuid 148
+#define __NR_setresgid 149
+#define __NR_getresgid 150
+#define __NR_setfsuid 151
+#define __NR_setfsgid 152
+#define __NR_times 153
+#define __NR_setpgid 154
+#define __NR_getpgid 155
+#define __NR_getsid 156
+#define __NR_setsid 157
+#define __NR_getgroups 158
+#define __NR_setgroups 159
+#define __NR_uname 160
+#define __NR_sethostname 161
+#define __NR_setdomainname 162
+#define __NR_getrlimit 163
+#define __NR_setrlimit 164
+#define __NR_getrusage 165
+#define __NR_umask 166
+#define __NR_prctl 167
+#define __NR_getcpu 168
+#define __NR_gettimeofday 169
+#define __NR_settimeofday 170
+#define __NR_adjtimex 171
+#define __NR_getpid 172
+#define __NR_getppid 173
+#define __NR_getuid 174
+#define __NR_geteuid 175
+#define __NR_getgid 176
+#define __NR_getegid 177
+#define __NR_gettid 178
+#define __NR_sysinfo 179
+#define __NR_mq_open 180
+#define __NR_mq_unlink 181
+#define __NR_mq_timedsend 182
+#define __NR_mq_timedreceive 183
+#define __NR_mq_notify 184
+#define __NR_mq_getsetattr 185
+#define __NR_msgget 186
+#define __NR_msgctl 187
+#define __NR_msgrcv 188
+#define __NR_msgsnd 189
+#define __NR_semget 190
+#define __NR_semctl 191
+#define __NR_semtimedop 192
+#define __NR_semop 193
+#define __NR_shmget 194
+#define __NR_shmctl 195
+#define __NR_shmat 196
+#define __NR_shmdt 197
+#define __NR_socket 198
+#define __NR_socketpair 199
+#define __NR_bind 200
+#define __NR_listen 201
+#define __NR_accept 202
+#define __NR_connect 203
+#define __NR_getsockname 204
+#define __NR_getpeername 205
+#define __NR_sendto 206
+#define __NR_recvfrom 207
+#define __NR_setsockopt 208
+#define __NR_getsockopt 209
+#define __NR_shutdown 210
+#define __NR_sendmsg 211
+#define __NR_recvmsg 212
+#define __NR_readahead 213
+#define __NR_brk 214
+#define __NR_munmap 215
+#define __NR_mremap 216
+#define __NR_add_key 217
+#define __NR_request_key 218
+#define __NR_keyctl 219
+#define __NR_clone 220
+#define __NR_execve 221
+#define __NR_mmap 222
+#define __NR_fadvise64 223
+#define __NR_swapon 224
+#define __NR_swapoff 225
+#define __NR_mprotect 226
+#define __NR_msync 227
+#define __NR_mlock 228
+#define __NR_munlock 229
+#define __NR_mlockall 230
+#define __NR_munlockall 231
+#define __NR_mincore 232
+#define __NR_madvise 233
+#define __NR_remap_file_pages 234
+#define __NR_mbind 235
+#define __NR_get_mempolicy 236
+#define __NR_set_mempolicy 237
+#define __NR_migrate_pages 238
+#define __NR_move_pages 239
+#define __NR_rt_tgsigqueueinfo 240
+#define __NR_perf_event_open 241
+#define __NR_accept4 242
+#define __NR_recvmmsg 243
+#define __NR_wait4 260
+#define __NR_prlimit64 261
+#define __NR_fanotify_init 262
+#define __NR_fanotify_mark 263
+#define __NR_name_to_handle_at 264
+#define __NR_open_by_handle_at 265
+#define __NR_clock_adjtime 266
+#define __NR_syncfs 267
+#define __NR_setns 268
+#define __NR_sendmmsg 269
+#define __NR_process_vm_readv 270
+#define __NR_process_vm_writev 271
+#define __NR_kcmp 272
+#define __NR_finit_module 273
+#define __NR_sched_setattr 274
+#define __NR_sched_getattr 275
+#define __NR_renameat2 276
+#define __NR_seccomp 277
+#define __NR_getrandom 278
+#define __NR_memfd_create 279
+#define __NR_bpf 280
+#define __NR_execveat 281
+#define __NR_userfaultfd 282
+#define __NR_membarrier 283
+#define __NR_mlock2 284
+#define __NR_copy_file_range 285
+#define __NR_preadv2 286
+#define __NR_pwritev2 287
+#define __NR_pkey_mprotect 288
+#define __NR_pkey_alloc 289
+#define __NR_pkey_free 290
+#define __NR_statx 291
+#define __NR_io_pgetevents 292
+#define __NR_rseq 293
+#define __NR_kexec_file_load 294
+#define __NR_pidfd_send_signal 424
+#define __NR_io_uring_setup 425
+#define __NR_io_uring_enter 426
+#define __NR_io_uring_register 427
+#define __NR_open_tree 428
+#define __NR_move_mount 429
+#define __NR_fsopen 430
+#define __NR_fsconfig 431
+#define __NR_fsmount 432
+#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_close_range 436
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
+#define __NR_process_madvise 440
+#define __NR_epoll_pwait2 441
+#define __NR_mount_setattr 442
+#define __NR_quotactl_fd 443
+#define __NR_landlock_create_ruleset 444
+#define __NR_landlock_add_rule 445
+#define __NR_landlock_restrict_self 446
+#define __NR_memfd_secret 447
+#define __NR_process_mrelease 448
+#define __NR_futex_waitv 449
+#define __NR_set_mempolicy_home_node 450
+#define __NR_cachestat 451
+#define __NR_fchmodat2 452
+#define __NR_map_shadow_stack 453
+#define __NR_futex_wake 454
+#define __NR_futex_wait 455
+#define __NR_futex_requeue 456
+#define __NR_statmount 457
+#define __NR_listmount 458
+#define __NR_lsm_get_self_attr 459
+#define __NR_lsm_set_self_attr 460
+#define __NR_lsm_list_modules 461
+#define __NR_mseal 462
+#endif
diff --git a/libc/kernel/uapi/asm-generic/unistd.h b/libc/kernel/uapi/asm-generic/unistd.h
index 7eaa89a..652e7a2 100644
--- a/libc/kernel/uapi/asm-generic/unistd.h
+++ b/libc/kernel/uapi/asm-generic/unistd.h
@@ -381,9 +381,7 @@
#define __NR_fsmount 432
#define __NR_fspick 433
#define __NR_pidfd_open 434
-#ifdef __ARCH_WANT_SYS_CLONE3
#define __NR_clone3 435
-#endif
#define __NR_close_range 436
#define __NR_openat2 437
#define __NR_pidfd_getfd 438
diff --git a/libc/kernel/uapi/asm-riscv/asm/hwprobe.h b/libc/kernel/uapi/asm-riscv/asm/hwprobe.h
index 3f30c88..2e5f9a4 100644
--- a/libc/kernel/uapi/asm-riscv/asm/hwprobe.h
+++ b/libc/kernel/uapi/asm-riscv/asm/hwprobe.h
@@ -54,6 +54,18 @@
#define RISCV_HWPROBE_EXT_ZACAS (1ULL << 34)
#define RISCV_HWPROBE_EXT_ZICOND (1ULL << 35)
#define RISCV_HWPROBE_EXT_ZIHINTPAUSE (1ULL << 36)
+#define RISCV_HWPROBE_EXT_ZVE32X (1ULL << 37)
+#define RISCV_HWPROBE_EXT_ZVE32F (1ULL << 38)
+#define RISCV_HWPROBE_EXT_ZVE64X (1ULL << 39)
+#define RISCV_HWPROBE_EXT_ZVE64F (1ULL << 40)
+#define RISCV_HWPROBE_EXT_ZVE64D (1ULL << 41)
+#define RISCV_HWPROBE_EXT_ZIMOP (1ULL << 42)
+#define RISCV_HWPROBE_EXT_ZCA (1ULL << 43)
+#define RISCV_HWPROBE_EXT_ZCB (1ULL << 44)
+#define RISCV_HWPROBE_EXT_ZCD (1ULL << 45)
+#define RISCV_HWPROBE_EXT_ZCF (1ULL << 46)
+#define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47)
+#define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48)
#define RISCV_HWPROBE_KEY_CPUPERF_0 5
#define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0)
#define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0)
@@ -62,5 +74,13 @@
#define RISCV_HWPROBE_MISALIGNED_UNSUPPORTED (4 << 0)
#define RISCV_HWPROBE_MISALIGNED_MASK (7 << 0)
#define RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE 6
+#define RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS 7
+#define RISCV_HWPROBE_KEY_TIME_CSR_FREQ 8
+#define RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF 9
+#define RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN 0
+#define RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED 1
+#define RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW 2
+#define RISCV_HWPROBE_MISALIGNED_SCALAR_FAST 3
+#define RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED 4
#define RISCV_HWPROBE_WHICH_CPUS (1 << 0)
#endif
diff --git a/libc/kernel/uapi/asm-riscv/asm/kvm.h b/libc/kernel/uapi/asm-riscv/asm/kvm.h
index 12d8f61..51f4977 100644
--- a/libc/kernel/uapi/asm-riscv/asm/kvm.h
+++ b/libc/kernel/uapi/asm-riscv/asm/kvm.h
@@ -128,6 +128,13 @@
KVM_RISCV_ISA_EXT_ZTSO,
KVM_RISCV_ISA_EXT_ZACAS,
KVM_RISCV_ISA_EXT_SSCOFPMF,
+ KVM_RISCV_ISA_EXT_ZIMOP,
+ KVM_RISCV_ISA_EXT_ZCA,
+ KVM_RISCV_ISA_EXT_ZCB,
+ KVM_RISCV_ISA_EXT_ZCD,
+ KVM_RISCV_ISA_EXT_ZCF,
+ KVM_RISCV_ISA_EXT_ZCMOP,
+ KVM_RISCV_ISA_EXT_ZAWRS,
KVM_RISCV_ISA_EXT_MAX,
};
enum KVM_RISCV_SBI_EXT_ID {
diff --git a/libc/kernel/uapi/asm-riscv/asm/unistd.h b/libc/kernel/uapi/asm-riscv/asm/unistd.h
index 0c58887..f395f71 100644
--- a/libc/kernel/uapi/asm-riscv/asm/unistd.h
+++ b/libc/kernel/uapi/asm-riscv/asm/unistd.h
@@ -4,18 +4,9 @@
* See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
* for more information.
*/
-#if defined(__LP64__) && !defined(__SYSCALL_COMPAT)
-#define __ARCH_WANT_NEW_STAT
-#define __ARCH_WANT_SET_GET_RLIMIT
+#include <asm/bitsperlong.h>
+#if __BITS_PER_LONG == 64
+#include <asm/unistd_64.h>
+#else
+#include <asm/unistd_32.h>
#endif
-#define __ARCH_WANT_SYS_CLONE3
-#define __ARCH_WANT_MEMFD_SECRET
-#include <asm-generic/unistd.h>
-#ifndef __NR_riscv_flush_icache
-#define __NR_riscv_flush_icache (__NR_arch_specific_syscall + 15)
-#endif
-__SYSCALL(__NR_riscv_flush_icache, sys_riscv_flush_icache)
-#ifndef __NR_riscv_hwprobe
-#define __NR_riscv_hwprobe (__NR_arch_specific_syscall + 14)
-#endif
-__SYSCALL(__NR_riscv_hwprobe, sys_riscv_hwprobe)
diff --git a/libc/kernel/uapi/asm-riscv/asm/unistd_32.h b/libc/kernel/uapi/asm-riscv/asm/unistd_32.h
new file mode 100644
index 0000000..864a556
--- /dev/null
+++ b/libc/kernel/uapi/asm-riscv/asm/unistd_32.h
@@ -0,0 +1,318 @@
+/*
+ * This file is auto-generated. Modifications will be lost.
+ *
+ * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
+ * for more information.
+ */
+#ifndef _UAPI_ASM_UNISTD_32_H
+#define _UAPI_ASM_UNISTD_32_H
+#define __NR_io_setup 0
+#define __NR_io_destroy 1
+#define __NR_io_submit 2
+#define __NR_io_cancel 3
+#define __NR_setxattr 5
+#define __NR_lsetxattr 6
+#define __NR_fsetxattr 7
+#define __NR_getxattr 8
+#define __NR_lgetxattr 9
+#define __NR_fgetxattr 10
+#define __NR_listxattr 11
+#define __NR_llistxattr 12
+#define __NR_flistxattr 13
+#define __NR_removexattr 14
+#define __NR_lremovexattr 15
+#define __NR_fremovexattr 16
+#define __NR_getcwd 17
+#define __NR_lookup_dcookie 18
+#define __NR_eventfd2 19
+#define __NR_epoll_create1 20
+#define __NR_epoll_ctl 21
+#define __NR_epoll_pwait 22
+#define __NR_dup 23
+#define __NR_dup3 24
+#define __NR_fcntl64 25
+#define __NR_inotify_init1 26
+#define __NR_inotify_add_watch 27
+#define __NR_inotify_rm_watch 28
+#define __NR_ioctl 29
+#define __NR_ioprio_set 30
+#define __NR_ioprio_get 31
+#define __NR_flock 32
+#define __NR_mknodat 33
+#define __NR_mkdirat 34
+#define __NR_unlinkat 35
+#define __NR_symlinkat 36
+#define __NR_linkat 37
+#define __NR_umount2 39
+#define __NR_mount 40
+#define __NR_pivot_root 41
+#define __NR_nfsservctl 42
+#define __NR_statfs64 43
+#define __NR_fstatfs64 44
+#define __NR_truncate64 45
+#define __NR_ftruncate64 46
+#define __NR_fallocate 47
+#define __NR_faccessat 48
+#define __NR_chdir 49
+#define __NR_fchdir 50
+#define __NR_chroot 51
+#define __NR_fchmod 52
+#define __NR_fchmodat 53
+#define __NR_fchownat 54
+#define __NR_fchown 55
+#define __NR_openat 56
+#define __NR_close 57
+#define __NR_vhangup 58
+#define __NR_pipe2 59
+#define __NR_quotactl 60
+#define __NR_getdents64 61
+#define __NR_llseek 62
+#define __NR_read 63
+#define __NR_write 64
+#define __NR_readv 65
+#define __NR_writev 66
+#define __NR_pread64 67
+#define __NR_pwrite64 68
+#define __NR_preadv 69
+#define __NR_pwritev 70
+#define __NR_sendfile64 71
+#define __NR_signalfd4 74
+#define __NR_vmsplice 75
+#define __NR_splice 76
+#define __NR_tee 77
+#define __NR_readlinkat 78
+#define __NR_sync 81
+#define __NR_fsync 82
+#define __NR_fdatasync 83
+#define __NR_sync_file_range 84
+#define __NR_timerfd_create 85
+#define __NR_acct 89
+#define __NR_capget 90
+#define __NR_capset 91
+#define __NR_personality 92
+#define __NR_exit 93
+#define __NR_exit_group 94
+#define __NR_waitid 95
+#define __NR_set_tid_address 96
+#define __NR_unshare 97
+#define __NR_set_robust_list 99
+#define __NR_get_robust_list 100
+#define __NR_getitimer 102
+#define __NR_setitimer 103
+#define __NR_kexec_load 104
+#define __NR_init_module 105
+#define __NR_delete_module 106
+#define __NR_timer_create 107
+#define __NR_timer_getoverrun 109
+#define __NR_timer_delete 111
+#define __NR_syslog 116
+#define __NR_ptrace 117
+#define __NR_sched_setparam 118
+#define __NR_sched_setscheduler 119
+#define __NR_sched_getscheduler 120
+#define __NR_sched_getparam 121
+#define __NR_sched_setaffinity 122
+#define __NR_sched_getaffinity 123
+#define __NR_sched_yield 124
+#define __NR_sched_get_priority_max 125
+#define __NR_sched_get_priority_min 126
+#define __NR_restart_syscall 128
+#define __NR_kill 129
+#define __NR_tkill 130
+#define __NR_tgkill 131
+#define __NR_sigaltstack 132
+#define __NR_rt_sigsuspend 133
+#define __NR_rt_sigaction 134
+#define __NR_rt_sigprocmask 135
+#define __NR_rt_sigpending 136
+#define __NR_rt_sigqueueinfo 138
+#define __NR_rt_sigreturn 139
+#define __NR_setpriority 140
+#define __NR_getpriority 141
+#define __NR_reboot 142
+#define __NR_setregid 143
+#define __NR_setgid 144
+#define __NR_setreuid 145
+#define __NR_setuid 146
+#define __NR_setresuid 147
+#define __NR_getresuid 148
+#define __NR_setresgid 149
+#define __NR_getresgid 150
+#define __NR_setfsuid 151
+#define __NR_setfsgid 152
+#define __NR_times 153
+#define __NR_setpgid 154
+#define __NR_getpgid 155
+#define __NR_getsid 156
+#define __NR_setsid 157
+#define __NR_getgroups 158
+#define __NR_setgroups 159
+#define __NR_uname 160
+#define __NR_sethostname 161
+#define __NR_setdomainname 162
+#define __NR_getrusage 165
+#define __NR_umask 166
+#define __NR_prctl 167
+#define __NR_getcpu 168
+#define __NR_getpid 172
+#define __NR_getppid 173
+#define __NR_getuid 174
+#define __NR_geteuid 175
+#define __NR_getgid 176
+#define __NR_getegid 177
+#define __NR_gettid 178
+#define __NR_sysinfo 179
+#define __NR_mq_open 180
+#define __NR_mq_unlink 181
+#define __NR_mq_notify 184
+#define __NR_mq_getsetattr 185
+#define __NR_msgget 186
+#define __NR_msgctl 187
+#define __NR_msgrcv 188
+#define __NR_msgsnd 189
+#define __NR_semget 190
+#define __NR_semctl 191
+#define __NR_semop 193
+#define __NR_shmget 194
+#define __NR_shmctl 195
+#define __NR_shmat 196
+#define __NR_shmdt 197
+#define __NR_socket 198
+#define __NR_socketpair 199
+#define __NR_bind 200
+#define __NR_listen 201
+#define __NR_accept 202
+#define __NR_connect 203
+#define __NR_getsockname 204
+#define __NR_getpeername 205
+#define __NR_sendto 206
+#define __NR_recvfrom 207
+#define __NR_setsockopt 208
+#define __NR_getsockopt 209
+#define __NR_shutdown 210
+#define __NR_sendmsg 211
+#define __NR_recvmsg 212
+#define __NR_readahead 213
+#define __NR_brk 214
+#define __NR_munmap 215
+#define __NR_mremap 216
+#define __NR_add_key 217
+#define __NR_request_key 218
+#define __NR_keyctl 219
+#define __NR_clone 220
+#define __NR_execve 221
+#define __NR_mmap2 222
+#define __NR_fadvise64_64 223
+#define __NR_swapon 224
+#define __NR_swapoff 225
+#define __NR_mprotect 226
+#define __NR_msync 227
+#define __NR_mlock 228
+#define __NR_munlock 229
+#define __NR_mlockall 230
+#define __NR_munlockall 231
+#define __NR_mincore 232
+#define __NR_madvise 233
+#define __NR_remap_file_pages 234
+#define __NR_mbind 235
+#define __NR_get_mempolicy 236
+#define __NR_set_mempolicy 237
+#define __NR_migrate_pages 238
+#define __NR_move_pages 239
+#define __NR_rt_tgsigqueueinfo 240
+#define __NR_perf_event_open 241
+#define __NR_accept4 242
+#define __NR_riscv_hwprobe 258
+#define __NR_riscv_flush_icache 259
+#define __NR_prlimit64 261
+#define __NR_fanotify_init 262
+#define __NR_fanotify_mark 263
+#define __NR_name_to_handle_at 264
+#define __NR_open_by_handle_at 265
+#define __NR_syncfs 267
+#define __NR_setns 268
+#define __NR_sendmmsg 269
+#define __NR_process_vm_readv 270
+#define __NR_process_vm_writev 271
+#define __NR_kcmp 272
+#define __NR_finit_module 273
+#define __NR_sched_setattr 274
+#define __NR_sched_getattr 275
+#define __NR_renameat2 276
+#define __NR_seccomp 277
+#define __NR_getrandom 278
+#define __NR_memfd_create 279
+#define __NR_bpf 280
+#define __NR_execveat 281
+#define __NR_userfaultfd 282
+#define __NR_membarrier 283
+#define __NR_mlock2 284
+#define __NR_copy_file_range 285
+#define __NR_preadv2 286
+#define __NR_pwritev2 287
+#define __NR_pkey_mprotect 288
+#define __NR_pkey_alloc 289
+#define __NR_pkey_free 290
+#define __NR_statx 291
+#define __NR_rseq 293
+#define __NR_kexec_file_load 294
+#define __NR_clock_gettime64 403
+#define __NR_clock_settime64 404
+#define __NR_clock_adjtime64 405
+#define __NR_clock_getres_time64 406
+#define __NR_clock_nanosleep_time64 407
+#define __NR_timer_gettime64 408
+#define __NR_timer_settime64 409
+#define __NR_timerfd_gettime64 410
+#define __NR_timerfd_settime64 411
+#define __NR_utimensat_time64 412
+#define __NR_pselect6_time64 413
+#define __NR_ppoll_time64 414
+#define __NR_io_pgetevents_time64 416
+#define __NR_recvmmsg_time64 417
+#define __NR_mq_timedsend_time64 418
+#define __NR_mq_timedreceive_time64 419
+#define __NR_semtimedop_time64 420
+#define __NR_rt_sigtimedwait_time64 421
+#define __NR_futex_time64 422
+#define __NR_sched_rr_get_interval_time64 423
+#define __NR_pidfd_send_signal 424
+#define __NR_io_uring_setup 425
+#define __NR_io_uring_enter 426
+#define __NR_io_uring_register 427
+#define __NR_open_tree 428
+#define __NR_move_mount 429
+#define __NR_fsopen 430
+#define __NR_fsconfig 431
+#define __NR_fsmount 432
+#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_close_range 436
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
+#define __NR_process_madvise 440
+#define __NR_epoll_pwait2 441
+#define __NR_mount_setattr 442
+#define __NR_quotactl_fd 443
+#define __NR_landlock_create_ruleset 444
+#define __NR_landlock_add_rule 445
+#define __NR_landlock_restrict_self 446
+#define __NR_memfd_secret 447
+#define __NR_process_mrelease 448
+#define __NR_futex_waitv 449
+#define __NR_set_mempolicy_home_node 450
+#define __NR_cachestat 451
+#define __NR_fchmodat2 452
+#define __NR_map_shadow_stack 453
+#define __NR_futex_wake 454
+#define __NR_futex_wait 455
+#define __NR_futex_requeue 456
+#define __NR_statmount 457
+#define __NR_listmount 458
+#define __NR_lsm_get_self_attr 459
+#define __NR_lsm_set_self_attr 460
+#define __NR_lsm_list_modules 461
+#define __NR_mseal 462
+#endif
diff --git a/libc/kernel/uapi/asm-riscv/asm/unistd_64.h b/libc/kernel/uapi/asm-riscv/asm/unistd_64.h
new file mode 100644
index 0000000..f15b65b
--- /dev/null
+++ b/libc/kernel/uapi/asm-riscv/asm/unistd_64.h
@@ -0,0 +1,328 @@
+/*
+ * This file is auto-generated. Modifications will be lost.
+ *
+ * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
+ * for more information.
+ */
+#ifndef _UAPI_ASM_UNISTD_64_H
+#define _UAPI_ASM_UNISTD_64_H
+#define __NR_io_setup 0
+#define __NR_io_destroy 1
+#define __NR_io_submit 2
+#define __NR_io_cancel 3
+#define __NR_io_getevents 4
+#define __NR_setxattr 5
+#define __NR_lsetxattr 6
+#define __NR_fsetxattr 7
+#define __NR_getxattr 8
+#define __NR_lgetxattr 9
+#define __NR_fgetxattr 10
+#define __NR_listxattr 11
+#define __NR_llistxattr 12
+#define __NR_flistxattr 13
+#define __NR_removexattr 14
+#define __NR_lremovexattr 15
+#define __NR_fremovexattr 16
+#define __NR_getcwd 17
+#define __NR_lookup_dcookie 18
+#define __NR_eventfd2 19
+#define __NR_epoll_create1 20
+#define __NR_epoll_ctl 21
+#define __NR_epoll_pwait 22
+#define __NR_dup 23
+#define __NR_dup3 24
+#define __NR_fcntl 25
+#define __NR_inotify_init1 26
+#define __NR_inotify_add_watch 27
+#define __NR_inotify_rm_watch 28
+#define __NR_ioctl 29
+#define __NR_ioprio_set 30
+#define __NR_ioprio_get 31
+#define __NR_flock 32
+#define __NR_mknodat 33
+#define __NR_mkdirat 34
+#define __NR_unlinkat 35
+#define __NR_symlinkat 36
+#define __NR_linkat 37
+#define __NR_umount2 39
+#define __NR_mount 40
+#define __NR_pivot_root 41
+#define __NR_nfsservctl 42
+#define __NR_statfs 43
+#define __NR_fstatfs 44
+#define __NR_truncate 45
+#define __NR_ftruncate 46
+#define __NR_fallocate 47
+#define __NR_faccessat 48
+#define __NR_chdir 49
+#define __NR_fchdir 50
+#define __NR_chroot 51
+#define __NR_fchmod 52
+#define __NR_fchmodat 53
+#define __NR_fchownat 54
+#define __NR_fchown 55
+#define __NR_openat 56
+#define __NR_close 57
+#define __NR_vhangup 58
+#define __NR_pipe2 59
+#define __NR_quotactl 60
+#define __NR_getdents64 61
+#define __NR_lseek 62
+#define __NR_read 63
+#define __NR_write 64
+#define __NR_readv 65
+#define __NR_writev 66
+#define __NR_pread64 67
+#define __NR_pwrite64 68
+#define __NR_preadv 69
+#define __NR_pwritev 70
+#define __NR_sendfile 71
+#define __NR_pselect6 72
+#define __NR_ppoll 73
+#define __NR_signalfd4 74
+#define __NR_vmsplice 75
+#define __NR_splice 76
+#define __NR_tee 77
+#define __NR_readlinkat 78
+#define __NR_newfstatat 79
+#define __NR_fstat 80
+#define __NR_sync 81
+#define __NR_fsync 82
+#define __NR_fdatasync 83
+#define __NR_sync_file_range 84
+#define __NR_timerfd_create 85
+#define __NR_timerfd_settime 86
+#define __NR_timerfd_gettime 87
+#define __NR_utimensat 88
+#define __NR_acct 89
+#define __NR_capget 90
+#define __NR_capset 91
+#define __NR_personality 92
+#define __NR_exit 93
+#define __NR_exit_group 94
+#define __NR_waitid 95
+#define __NR_set_tid_address 96
+#define __NR_unshare 97
+#define __NR_futex 98
+#define __NR_set_robust_list 99
+#define __NR_get_robust_list 100
+#define __NR_nanosleep 101
+#define __NR_getitimer 102
+#define __NR_setitimer 103
+#define __NR_kexec_load 104
+#define __NR_init_module 105
+#define __NR_delete_module 106
+#define __NR_timer_create 107
+#define __NR_timer_gettime 108
+#define __NR_timer_getoverrun 109
+#define __NR_timer_settime 110
+#define __NR_timer_delete 111
+#define __NR_clock_settime 112
+#define __NR_clock_gettime 113
+#define __NR_clock_getres 114
+#define __NR_clock_nanosleep 115
+#define __NR_syslog 116
+#define __NR_ptrace 117
+#define __NR_sched_setparam 118
+#define __NR_sched_setscheduler 119
+#define __NR_sched_getscheduler 120
+#define __NR_sched_getparam 121
+#define __NR_sched_setaffinity 122
+#define __NR_sched_getaffinity 123
+#define __NR_sched_yield 124
+#define __NR_sched_get_priority_max 125
+#define __NR_sched_get_priority_min 126
+#define __NR_sched_rr_get_interval 127
+#define __NR_restart_syscall 128
+#define __NR_kill 129
+#define __NR_tkill 130
+#define __NR_tgkill 131
+#define __NR_sigaltstack 132
+#define __NR_rt_sigsuspend 133
+#define __NR_rt_sigaction 134
+#define __NR_rt_sigprocmask 135
+#define __NR_rt_sigpending 136
+#define __NR_rt_sigtimedwait 137
+#define __NR_rt_sigqueueinfo 138
+#define __NR_rt_sigreturn 139
+#define __NR_setpriority 140
+#define __NR_getpriority 141
+#define __NR_reboot 142
+#define __NR_setregid 143
+#define __NR_setgid 144
+#define __NR_setreuid 145
+#define __NR_setuid 146
+#define __NR_setresuid 147
+#define __NR_getresuid 148
+#define __NR_setresgid 149
+#define __NR_getresgid 150
+#define __NR_setfsuid 151
+#define __NR_setfsgid 152
+#define __NR_times 153
+#define __NR_setpgid 154
+#define __NR_getpgid 155
+#define __NR_getsid 156
+#define __NR_setsid 157
+#define __NR_getgroups 158
+#define __NR_setgroups 159
+#define __NR_uname 160
+#define __NR_sethostname 161
+#define __NR_setdomainname 162
+#define __NR_getrlimit 163
+#define __NR_setrlimit 164
+#define __NR_getrusage 165
+#define __NR_umask 166
+#define __NR_prctl 167
+#define __NR_getcpu 168
+#define __NR_gettimeofday 169
+#define __NR_settimeofday 170
+#define __NR_adjtimex 171
+#define __NR_getpid 172
+#define __NR_getppid 173
+#define __NR_getuid 174
+#define __NR_geteuid 175
+#define __NR_getgid 176
+#define __NR_getegid 177
+#define __NR_gettid 178
+#define __NR_sysinfo 179
+#define __NR_mq_open 180
+#define __NR_mq_unlink 181
+#define __NR_mq_timedsend 182
+#define __NR_mq_timedreceive 183
+#define __NR_mq_notify 184
+#define __NR_mq_getsetattr 185
+#define __NR_msgget 186
+#define __NR_msgctl 187
+#define __NR_msgrcv 188
+#define __NR_msgsnd 189
+#define __NR_semget 190
+#define __NR_semctl 191
+#define __NR_semtimedop 192
+#define __NR_semop 193
+#define __NR_shmget 194
+#define __NR_shmctl 195
+#define __NR_shmat 196
+#define __NR_shmdt 197
+#define __NR_socket 198
+#define __NR_socketpair 199
+#define __NR_bind 200
+#define __NR_listen 201
+#define __NR_accept 202
+#define __NR_connect 203
+#define __NR_getsockname 204
+#define __NR_getpeername 205
+#define __NR_sendto 206
+#define __NR_recvfrom 207
+#define __NR_setsockopt 208
+#define __NR_getsockopt 209
+#define __NR_shutdown 210
+#define __NR_sendmsg 211
+#define __NR_recvmsg 212
+#define __NR_readahead 213
+#define __NR_brk 214
+#define __NR_munmap 215
+#define __NR_mremap 216
+#define __NR_add_key 217
+#define __NR_request_key 218
+#define __NR_keyctl 219
+#define __NR_clone 220
+#define __NR_execve 221
+#define __NR_mmap 222
+#define __NR_fadvise64 223
+#define __NR_swapon 224
+#define __NR_swapoff 225
+#define __NR_mprotect 226
+#define __NR_msync 227
+#define __NR_mlock 228
+#define __NR_munlock 229
+#define __NR_mlockall 230
+#define __NR_munlockall 231
+#define __NR_mincore 232
+#define __NR_madvise 233
+#define __NR_remap_file_pages 234
+#define __NR_mbind 235
+#define __NR_get_mempolicy 236
+#define __NR_set_mempolicy 237
+#define __NR_migrate_pages 238
+#define __NR_move_pages 239
+#define __NR_rt_tgsigqueueinfo 240
+#define __NR_perf_event_open 241
+#define __NR_accept4 242
+#define __NR_recvmmsg 243
+#define __NR_riscv_hwprobe 258
+#define __NR_riscv_flush_icache 259
+#define __NR_wait4 260
+#define __NR_prlimit64 261
+#define __NR_fanotify_init 262
+#define __NR_fanotify_mark 263
+#define __NR_name_to_handle_at 264
+#define __NR_open_by_handle_at 265
+#define __NR_clock_adjtime 266
+#define __NR_syncfs 267
+#define __NR_setns 268
+#define __NR_sendmmsg 269
+#define __NR_process_vm_readv 270
+#define __NR_process_vm_writev 271
+#define __NR_kcmp 272
+#define __NR_finit_module 273
+#define __NR_sched_setattr 274
+#define __NR_sched_getattr 275
+#define __NR_renameat2 276
+#define __NR_seccomp 277
+#define __NR_getrandom 278
+#define __NR_memfd_create 279
+#define __NR_bpf 280
+#define __NR_execveat 281
+#define __NR_userfaultfd 282
+#define __NR_membarrier 283
+#define __NR_mlock2 284
+#define __NR_copy_file_range 285
+#define __NR_preadv2 286
+#define __NR_pwritev2 287
+#define __NR_pkey_mprotect 288
+#define __NR_pkey_alloc 289
+#define __NR_pkey_free 290
+#define __NR_statx 291
+#define __NR_io_pgetevents 292
+#define __NR_rseq 293
+#define __NR_kexec_file_load 294
+#define __NR_pidfd_send_signal 424
+#define __NR_io_uring_setup 425
+#define __NR_io_uring_enter 426
+#define __NR_io_uring_register 427
+#define __NR_open_tree 428
+#define __NR_move_mount 429
+#define __NR_fsopen 430
+#define __NR_fsconfig 431
+#define __NR_fsmount 432
+#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_close_range 436
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
+#define __NR_process_madvise 440
+#define __NR_epoll_pwait2 441
+#define __NR_mount_setattr 442
+#define __NR_quotactl_fd 443
+#define __NR_landlock_create_ruleset 444
+#define __NR_landlock_add_rule 445
+#define __NR_landlock_restrict_self 446
+#define __NR_memfd_secret 447
+#define __NR_process_mrelease 448
+#define __NR_futex_waitv 449
+#define __NR_set_mempolicy_home_node 450
+#define __NR_cachestat 451
+#define __NR_fchmodat2 452
+#define __NR_map_shadow_stack 453
+#define __NR_futex_wake 454
+#define __NR_futex_wait 455
+#define __NR_futex_requeue 456
+#define __NR_statmount 457
+#define __NR_listmount 458
+#define __NR_lsm_get_self_attr 459
+#define __NR_lsm_set_self_attr 460
+#define __NR_lsm_list_modules 461
+#define __NR_mseal 462
+#endif
diff --git a/libc/kernel/uapi/asm-x86/asm/kvm.h b/libc/kernel/uapi/asm-x86/asm/kvm.h
index 17b1c5d..cd647b6 100644
--- a/libc/kernel/uapi/asm-x86/asm/kvm.h
+++ b/libc/kernel/uapi/asm-x86/asm/kvm.h
@@ -94,6 +94,7 @@
#define KVM_NR_IRQCHIPS 3
#define KVM_RUN_X86_SMM (1 << 0)
#define KVM_RUN_X86_BUS_LOCK (1 << 1)
+#define KVM_RUN_X86_GUEST_MODE (1 << 2)
struct kvm_regs {
__u64 rax, rbx, rcx, rdx;
__u64 rsi, rdi, rsp, rbp;
@@ -532,6 +533,9 @@
KVM_SEV_GET_ATTESTATION_REPORT,
KVM_SEV_SEND_CANCEL,
KVM_SEV_INIT2,
+ KVM_SEV_SNP_LAUNCH_START = 100,
+ KVM_SEV_SNP_LAUNCH_UPDATE,
+ KVM_SEV_SNP_LAUNCH_FINISH,
KVM_SEV_NR_MAX,
};
struct kvm_sev_cmd {
@@ -644,6 +648,42 @@
__u32 trans_len;
__u32 pad2;
};
+struct kvm_sev_snp_launch_start {
+ __u64 policy;
+ __u8 gosvw[16];
+ __u16 flags;
+ __u8 pad0[6];
+ __u64 pad1[4];
+};
+#define KVM_SEV_SNP_PAGE_TYPE_NORMAL 0x1
+#define KVM_SEV_SNP_PAGE_TYPE_ZERO 0x3
+#define KVM_SEV_SNP_PAGE_TYPE_UNMEASURED 0x4
+#define KVM_SEV_SNP_PAGE_TYPE_SECRETS 0x5
+#define KVM_SEV_SNP_PAGE_TYPE_CPUID 0x6
+struct kvm_sev_snp_launch_update {
+ __u64 gfn_start;
+ __u64 uaddr;
+ __u64 len;
+ __u8 type;
+ __u8 pad0;
+ __u16 flags;
+ __u32 pad1;
+ __u64 pad2[4];
+};
+#define KVM_SEV_SNP_ID_BLOCK_SIZE 96
+#define KVM_SEV_SNP_ID_AUTH_SIZE 4096
+#define KVM_SEV_SNP_FINISH_DATA_SIZE 32
+struct kvm_sev_snp_launch_finish {
+ __u64 id_block_uaddr;
+ __u64 id_auth_uaddr;
+ __u8 id_block_en;
+ __u8 auth_key_en;
+ __u8 vcek_disabled;
+ __u8 host_data[KVM_SEV_SNP_FINISH_DATA_SIZE];
+ __u8 pad0[3];
+ __u16 flags;
+ __u64 pad1[4];
+};
#define KVM_X2APIC_API_USE_32BIT_IDS (1ULL << 0)
#define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK (1ULL << 1)
struct kvm_hyperv_eventfd {
@@ -667,4 +707,5 @@
#define KVM_X86_SW_PROTECTED_VM 1
#define KVM_X86_SEV_VM 2
#define KVM_X86_SEV_ES_VM 3
+#define KVM_X86_SNP_VM 4
#endif
diff --git a/libc/kernel/uapi/asm-x86/asm/svm.h b/libc/kernel/uapi/asm-x86/asm/svm.h
index ffbf0b3..4f165fa 100644
--- a/libc/kernel/uapi/asm-x86/asm/svm.h
+++ b/libc/kernel/uapi/asm-x86/asm/svm.h
@@ -117,6 +117,7 @@
#define SVM_VMGEXIT_AP_CREATE_ON_INIT 0
#define SVM_VMGEXIT_AP_CREATE 1
#define SVM_VMGEXIT_AP_DESTROY 2
+#define SVM_VMGEXIT_SNP_RUN_VMPL 0x80000018
#define SVM_VMGEXIT_HV_FEATURES 0x8000fffd
#define SVM_VMGEXIT_TERM_REQUEST 0x8000fffe
#define SVM_VMGEXIT_TERM_REASON(reason_set,reason_code) (((((u64) reason_set) & 0xf)) | ((((u64) reason_code) & 0xff) << 4))
diff --git a/libc/kernel/uapi/asm-x86/asm/swab.h b/libc/kernel/uapi/asm-x86/asm/swab.h
index 31c850d..ce43658 100644
--- a/libc/kernel/uapi/asm-x86/asm/swab.h
+++ b/libc/kernel/uapi/asm-x86/asm/swab.h
@@ -8,9 +8,27 @@
#define _ASM_X86_SWAB_H
#include <linux/types.h>
#include <linux/compiler.h>
+static inline __attribute__((__const__)) __u32 __arch_swab32(__u32 val) {
+ asm("bswapl %0" : "=r" (val) : "0" (val));
+ return val;
+}
#define __arch_swab32 __arch_swab32
+static inline __attribute__((__const__)) __u64 __arch_swab64(__u64 val) {
#ifdef __i386__
+ union {
+ struct {
+ __u32 a;
+ __u32 b;
+ } s;
+ __u64 u;
+ } v;
+ v.u = val;
+ asm("bswapl %0; bswapl %1; xchgl %0,%1" : "=r" (v.s.a), "=r" (v.s.b) : "0" (v.s.a), "1" (v.s.b));
+ return v.u;
#else
+ asm("bswapq %0" : "=r" (val) : "0" (val));
+ return val;
#endif
+}
#define __arch_swab64 __arch_swab64
#endif
diff --git a/libc/kernel/uapi/asm-x86/asm/unistd_64.h b/libc/kernel/uapi/asm-x86/asm/unistd_64.h
index 5dd666c..d5408a3 100644
--- a/libc/kernel/uapi/asm-x86/asm/unistd_64.h
+++ b/libc/kernel/uapi/asm-x86/asm/unistd_64.h
@@ -341,6 +341,7 @@
#define __NR_statx 332
#define __NR_io_pgetevents 333
#define __NR_rseq 334
+#define __NR_uretprobe 335
#define __NR_pidfd_send_signal 424
#define __NR_io_uring_setup 425
#define __NR_io_uring_enter 426
diff --git a/libc/kernel/uapi/asm-x86/asm/unistd_x32.h b/libc/kernel/uapi/asm-x86/asm/unistd_x32.h
index a2ff6f4..fdcf7e6 100644
--- a/libc/kernel/uapi/asm-x86/asm/unistd_x32.h
+++ b/libc/kernel/uapi/asm-x86/asm/unistd_x32.h
@@ -294,6 +294,7 @@
#define __NR_statx (__X32_SYSCALL_BIT + 332)
#define __NR_io_pgetevents (__X32_SYSCALL_BIT + 333)
#define __NR_rseq (__X32_SYSCALL_BIT + 334)
+#define __NR_uretprobe (__X32_SYSCALL_BIT + 335)
#define __NR_pidfd_send_signal (__X32_SYSCALL_BIT + 424)
#define __NR_io_uring_setup (__X32_SYSCALL_BIT + 425)
#define __NR_io_uring_enter (__X32_SYSCALL_BIT + 426)
diff --git a/libc/kernel/uapi/drm/amdgpu_drm.h b/libc/kernel/uapi/drm/amdgpu_drm.h
index 0ad0bc2..7bbd5de 100644
--- a/libc/kernel/uapi/drm/amdgpu_drm.h
+++ b/libc/kernel/uapi/drm/amdgpu_drm.h
@@ -65,6 +65,7 @@
#define AMDGPU_GEM_CREATE_COHERENT (1 << 13)
#define AMDGPU_GEM_CREATE_UNCACHED (1 << 14)
#define AMDGPU_GEM_CREATE_EXT_COHERENT (1 << 15)
+#define AMDGPU_GEM_CREATE_GFX12_DCC (1 << 16)
struct drm_amdgpu_gem_create_in {
__u64 bo_size;
__u64 alignment;
@@ -216,6 +217,14 @@
#define AMDGPU_TILING_DCC_INDEPENDENT_128B_MASK 0x1
#define AMDGPU_TILING_SCANOUT_SHIFT 63
#define AMDGPU_TILING_SCANOUT_MASK 0x1
+#define AMDGPU_TILING_GFX12_SWIZZLE_MODE_SHIFT 0
+#define AMDGPU_TILING_GFX12_SWIZZLE_MODE_MASK 0x7
+#define AMDGPU_TILING_GFX12_DCC_MAX_COMPRESSED_BLOCK_SHIFT 3
+#define AMDGPU_TILING_GFX12_DCC_MAX_COMPRESSED_BLOCK_MASK 0x3
+#define AMDGPU_TILING_GFX12_DCC_NUMBER_TYPE_SHIFT 5
+#define AMDGPU_TILING_GFX12_DCC_NUMBER_TYPE_MASK 0x7
+#define AMDGPU_TILING_GFX12_DCC_DATA_FORMAT_SHIFT 8
+#define AMDGPU_TILING_GFX12_DCC_DATA_FORMAT_MASK 0x3f
#define AMDGPU_TILING_SET(field,value) (((__u64) (value) & AMDGPU_TILING_ ##field ##_MASK) << AMDGPU_TILING_ ##field ##_SHIFT)
#define AMDGPU_TILING_GET(value,field) (((__u64) (value) >> AMDGPU_TILING_ ##field ##_SHIFT) & AMDGPU_TILING_ ##field ##_MASK)
#define AMDGPU_GEM_METADATA_OP_SET_METADATA 1
@@ -744,6 +753,10 @@
#define AMDGPU_FAMILY_GC_10_3_6 149
#define AMDGPU_FAMILY_GC_10_3_7 151
#define AMDGPU_FAMILY_GC_11_5_0 150
+#define AMDGPU_FAMILY_GC_12_0_0 152
+struct drm_color_ctm_3x4 {
+ __u64 matrix[12];
+};
#ifdef __cplusplus
}
#endif
diff --git a/libc/kernel/uapi/drm/drm_fourcc.h b/libc/kernel/uapi/drm/drm_fourcc.h
index 6fd2eb8..4902d6c 100644
--- a/libc/kernel/uapi/drm/drm_fourcc.h
+++ b/libc/kernel/uapi/drm/drm_fourcc.h
@@ -255,12 +255,17 @@
#define AMD_FMT_MOD_TILE_VER_GFX10 2
#define AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS 3
#define AMD_FMT_MOD_TILE_VER_GFX11 4
+#define AMD_FMT_MOD_TILE_VER_GFX12 5
#define AMD_FMT_MOD_TILE_GFX9_64K_S 9
#define AMD_FMT_MOD_TILE_GFX9_64K_D 10
#define AMD_FMT_MOD_TILE_GFX9_64K_S_X 25
#define AMD_FMT_MOD_TILE_GFX9_64K_D_X 26
#define AMD_FMT_MOD_TILE_GFX9_64K_R_X 27
#define AMD_FMT_MOD_TILE_GFX11_256K_R_X 31
+#define AMD_FMT_MOD_TILE_GFX12_256B_2D 1
+#define AMD_FMT_MOD_TILE_GFX12_4K_2D 2
+#define AMD_FMT_MOD_TILE_GFX12_64K_2D 3
+#define AMD_FMT_MOD_TILE_GFX12_256K_2D 4
#define AMD_FMT_MOD_DCC_BLOCK_64B 0
#define AMD_FMT_MOD_DCC_BLOCK_128B 1
#define AMD_FMT_MOD_DCC_BLOCK_256B 2
diff --git a/libc/kernel/uapi/drm/drm_mode.h b/libc/kernel/uapi/drm/drm_mode.h
index 8fccdaf..06c91c5 100644
--- a/libc/kernel/uapi/drm/drm_mode.h
+++ b/libc/kernel/uapi/drm/drm_mode.h
@@ -357,9 +357,6 @@
struct drm_color_ctm {
__u64 matrix[9];
};
-struct drm_color_ctm_3x4 {
- __u64 matrix[12];
-};
struct drm_color_lut {
__u16 red;
__u16 green;
diff --git a/libc/kernel/uapi/drm/i915_drm.h b/libc/kernel/uapi/drm/i915_drm.h
index 13eda7c..b43d8df 100644
--- a/libc/kernel/uapi/drm/i915_drm.h
+++ b/libc/kernel/uapi/drm/i915_drm.h
@@ -745,6 +745,7 @@
#define I915_CONTEXT_PARAM_RINGSIZE 0xc
#define I915_CONTEXT_PARAM_PROTECTED_CONTENT 0xd
#define I915_CONTEXT_PARAM_LOW_LATENCY 0xe
+#define I915_CONTEXT_PARAM_CONTEXT_IMAGE 0xf
__u64 value;
};
struct drm_i915_gem_context_param_sseu {
@@ -799,6 +800,14 @@
} __attribute__((packed));
#define I915_DEFINE_CONTEXT_PARAM_ENGINES(name__,N__) struct { __u64 extensions; struct i915_engine_class_instance engines[N__]; \
} __attribute__((packed)) name__
+struct i915_gem_context_param_context_image {
+ struct i915_engine_class_instance engine;
+ __u32 flags;
+#define I915_CONTEXT_IMAGE_FLAG_ENGINE_INDEX (1u << 0)
+ __u32 size;
+ __u32 mbz;
+ __u64 image;
+} __attribute__((packed));
struct drm_i915_gem_context_create_ext_setparam {
struct i915_user_extension base;
struct drm_i915_gem_context_param param;
diff --git a/libc/kernel/uapi/drm/ivpu_accel.h b/libc/kernel/uapi/drm/ivpu_accel.h
index fcbf6f7..960bd43 100644
--- a/libc/kernel/uapi/drm/ivpu_accel.h
+++ b/libc/kernel/uapi/drm/ivpu_accel.h
@@ -18,12 +18,20 @@
#define DRM_IVPU_BO_INFO 0x03
#define DRM_IVPU_SUBMIT 0x05
#define DRM_IVPU_BO_WAIT 0x06
+#define DRM_IVPU_METRIC_STREAMER_START 0x07
+#define DRM_IVPU_METRIC_STREAMER_STOP 0x08
+#define DRM_IVPU_METRIC_STREAMER_GET_DATA 0x09
+#define DRM_IVPU_METRIC_STREAMER_GET_INFO 0x0a
#define DRM_IOCTL_IVPU_GET_PARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_GET_PARAM, struct drm_ivpu_param)
#define DRM_IOCTL_IVPU_SET_PARAM DRM_IOW(DRM_COMMAND_BASE + DRM_IVPU_SET_PARAM, struct drm_ivpu_param)
#define DRM_IOCTL_IVPU_BO_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_BO_CREATE, struct drm_ivpu_bo_create)
#define DRM_IOCTL_IVPU_BO_INFO DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_BO_INFO, struct drm_ivpu_bo_info)
#define DRM_IOCTL_IVPU_SUBMIT DRM_IOW(DRM_COMMAND_BASE + DRM_IVPU_SUBMIT, struct drm_ivpu_submit)
#define DRM_IOCTL_IVPU_BO_WAIT DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_BO_WAIT, struct drm_ivpu_bo_wait)
+#define DRM_IOCTL_IVPU_METRIC_STREAMER_START DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_METRIC_STREAMER_START, struct drm_ivpu_metric_streamer_start)
+#define DRM_IOCTL_IVPU_METRIC_STREAMER_STOP DRM_IOW(DRM_COMMAND_BASE + DRM_IVPU_METRIC_STREAMER_STOP, struct drm_ivpu_metric_streamer_stop)
+#define DRM_IOCTL_IVPU_METRIC_STREAMER_GET_DATA DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_METRIC_STREAMER_GET_DATA, struct drm_ivpu_metric_streamer_get_data)
+#define DRM_IOCTL_IVPU_METRIC_STREAMER_GET_INFO DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_METRIC_STREAMER_GET_INFO, struct drm_ivpu_metric_streamer_get_data)
#define DRM_IVPU_PARAM_DEVICE_ID 0
#define DRM_IVPU_PARAM_DEVICE_REVISION 1
#define DRM_IVPU_PARAM_PLATFORM_TYPE 2
@@ -96,6 +104,22 @@
__u32 job_status;
__u32 pad;
};
+struct drm_ivpu_metric_streamer_start {
+ __u64 metric_group_mask;
+ __u64 sampling_period_ns;
+ __u32 read_period_samples;
+ __u32 sample_size;
+ __u32 max_data_size;
+};
+struct drm_ivpu_metric_streamer_get_data {
+ __u64 metric_group_mask;
+ __u64 buffer_ptr;
+ __u64 buffer_size;
+ __u64 data_size;
+};
+struct drm_ivpu_metric_streamer_stop {
+ __u64 metric_group_mask;
+};
#ifdef __cplusplus
}
#endif
diff --git a/libc/kernel/uapi/drm/msm_drm.h b/libc/kernel/uapi/drm/msm_drm.h
index 4d83744..7ec5ed2 100644
--- a/libc/kernel/uapi/drm/msm_drm.h
+++ b/libc/kernel/uapi/drm/msm_drm.h
@@ -37,6 +37,7 @@
#define MSM_PARAM_VA_START 0x0e
#define MSM_PARAM_VA_SIZE 0x0f
#define MSM_PARAM_HIGHEST_BANK_BIT 0x10
+#define MSM_PARAM_RAYTRACING 0x11
#define MSM_PARAM_NR_RINGS MSM_PARAM_PRIORITIES
struct drm_msm_param {
__u32 pipe;
diff --git a/libc/kernel/uapi/drm/v3d_drm.h b/libc/kernel/uapi/drm/v3d_drm.h
index 4000fd3..b7aca21 100644
--- a/libc/kernel/uapi/drm/v3d_drm.h
+++ b/libc/kernel/uapi/drm/v3d_drm.h
@@ -22,6 +22,7 @@
#define DRM_V3D_PERFMON_DESTROY 0x09
#define DRM_V3D_PERFMON_GET_VALUES 0x0a
#define DRM_V3D_SUBMIT_CPU 0x0b
+#define DRM_V3D_PERFMON_GET_COUNTER 0x0c
#define DRM_IOCTL_V3D_SUBMIT_CL DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_SUBMIT_CL, struct drm_v3d_submit_cl)
#define DRM_IOCTL_V3D_WAIT_BO DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_WAIT_BO, struct drm_v3d_wait_bo)
#define DRM_IOCTL_V3D_CREATE_BO DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_CREATE_BO, struct drm_v3d_create_bo)
@@ -34,6 +35,7 @@
#define DRM_IOCTL_V3D_PERFMON_DESTROY DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_PERFMON_DESTROY, struct drm_v3d_perfmon_destroy)
#define DRM_IOCTL_V3D_PERFMON_GET_VALUES DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_PERFMON_GET_VALUES, struct drm_v3d_perfmon_get_values)
#define DRM_IOCTL_V3D_SUBMIT_CPU DRM_IOW(DRM_COMMAND_BASE + DRM_V3D_SUBMIT_CPU, struct drm_v3d_submit_cpu)
+#define DRM_IOCTL_V3D_PERFMON_GET_COUNTER DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_PERFMON_GET_COUNTER, struct drm_v3d_perfmon_get_counter)
#define DRM_V3D_SUBMIT_CL_FLUSH_CACHE 0x01
#define DRM_V3D_SUBMIT_EXTENSION 0x02
struct drm_v3d_extension {
@@ -119,6 +121,7 @@
DRM_V3D_PARAM_SUPPORTS_PERFMON,
DRM_V3D_PARAM_SUPPORTS_MULTISYNC_EXT,
DRM_V3D_PARAM_SUPPORTS_CPU_QUEUE,
+ DRM_V3D_PARAM_MAX_PERF_COUNTERS,
};
struct drm_v3d_get_param {
__u32 param;
@@ -324,6 +327,16 @@
__u32 pad;
__u64 values_ptr;
};
+#define DRM_V3D_PERFCNT_MAX_NAME 64
+#define DRM_V3D_PERFCNT_MAX_CATEGORY 32
+#define DRM_V3D_PERFCNT_MAX_DESCRIPTION 256
+struct drm_v3d_perfmon_get_counter {
+ __u8 counter;
+ __u8 name[DRM_V3D_PERFCNT_MAX_NAME];
+ __u8 category[DRM_V3D_PERFCNT_MAX_CATEGORY];
+ __u8 description[DRM_V3D_PERFCNT_MAX_DESCRIPTION];
+ __u8 reserved[7];
+};
#ifdef __cplusplus
}
#endif
diff --git a/libc/kernel/uapi/drm/xe_drm.h b/libc/kernel/uapi/drm/xe_drm.h
index d1b6dad..a034b29 100644
--- a/libc/kernel/uapi/drm/xe_drm.h
+++ b/libc/kernel/uapi/drm/xe_drm.h
@@ -21,6 +21,7 @@
#define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x08
#define DRM_XE_EXEC 0x09
#define DRM_XE_WAIT_USER_FENCE 0x0a
+#define DRM_XE_OBSERVATION 0x0b
#define DRM_IOCTL_XE_DEVICE_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_DEVICE_QUERY, struct drm_xe_device_query)
#define DRM_IOCTL_XE_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_CREATE, struct drm_xe_gem_create)
#define DRM_IOCTL_XE_GEM_MMAP_OFFSET DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_MMAP_OFFSET, struct drm_xe_gem_mmap_offset)
@@ -32,6 +33,7 @@
#define DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_GET_PROPERTY, struct drm_xe_exec_queue_get_property)
#define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec)
#define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence)
+#define DRM_IOCTL_XE_OBSERVATION DRM_IOW(DRM_COMMAND_BASE + DRM_XE_OBSERVATION, struct drm_xe_observation_param)
struct drm_xe_user_extension {
__u64 next_extension;
__u32 name;
@@ -120,6 +122,7 @@
__u16 gt_id;
#define DRM_XE_TOPO_DSS_GEOMETRY 1
#define DRM_XE_TOPO_DSS_COMPUTE 2
+#define DRM_XE_TOPO_L3_BANK 3
#define DRM_XE_TOPO_EU_PER_DSS 4
__u16 type;
__u32 num_bytes;
@@ -155,6 +158,7 @@
#define DRM_XE_DEVICE_QUERY_GT_TOPOLOGY 5
#define DRM_XE_DEVICE_QUERY_ENGINE_CYCLES 6
#define DRM_XE_DEVICE_QUERY_UC_FW_VERSION 7
+#define DRM_XE_DEVICE_QUERY_OA_UNITS 8
__u32 query;
__u32 size;
__u64 data;
@@ -309,6 +313,92 @@
__u32 pad2;
__u64 reserved[2];
};
+enum drm_xe_observation_type {
+ DRM_XE_OBSERVATION_TYPE_OA,
+};
+enum drm_xe_observation_op {
+ DRM_XE_OBSERVATION_OP_STREAM_OPEN,
+ DRM_XE_OBSERVATION_OP_ADD_CONFIG,
+ DRM_XE_OBSERVATION_OP_REMOVE_CONFIG,
+};
+struct drm_xe_observation_param {
+ __u64 extensions;
+ __u64 observation_type;
+ __u64 observation_op;
+ __u64 param;
+};
+enum drm_xe_observation_ioctls {
+ DRM_XE_OBSERVATION_IOCTL_ENABLE = _IO('i', 0x0),
+ DRM_XE_OBSERVATION_IOCTL_DISABLE = _IO('i', 0x1),
+ DRM_XE_OBSERVATION_IOCTL_CONFIG = _IO('i', 0x2),
+ DRM_XE_OBSERVATION_IOCTL_STATUS = _IO('i', 0x3),
+ DRM_XE_OBSERVATION_IOCTL_INFO = _IO('i', 0x4),
+};
+enum drm_xe_oa_unit_type {
+ DRM_XE_OA_UNIT_TYPE_OAG,
+ DRM_XE_OA_UNIT_TYPE_OAM,
+};
+struct drm_xe_oa_unit {
+ __u64 extensions;
+ __u32 oa_unit_id;
+ __u32 oa_unit_type;
+ __u64 capabilities;
+#define DRM_XE_OA_CAPS_BASE (1 << 0)
+ __u64 oa_timestamp_freq;
+ __u64 reserved[4];
+ __u64 num_engines;
+ struct drm_xe_engine_class_instance eci[];
+};
+struct drm_xe_query_oa_units {
+ __u64 extensions;
+ __u32 num_oa_units;
+ __u32 pad;
+ __u64 oa_units[];
+};
+enum drm_xe_oa_format_type {
+ DRM_XE_OA_FMT_TYPE_OAG,
+ DRM_XE_OA_FMT_TYPE_OAR,
+ DRM_XE_OA_FMT_TYPE_OAM,
+ DRM_XE_OA_FMT_TYPE_OAC,
+ DRM_XE_OA_FMT_TYPE_OAM_MPEC,
+ DRM_XE_OA_FMT_TYPE_PEC,
+};
+enum drm_xe_oa_property_id {
+#define DRM_XE_OA_EXTENSION_SET_PROPERTY 0
+ DRM_XE_OA_PROPERTY_OA_UNIT_ID = 1,
+ DRM_XE_OA_PROPERTY_SAMPLE_OA,
+ DRM_XE_OA_PROPERTY_OA_METRIC_SET,
+ DRM_XE_OA_PROPERTY_OA_FORMAT,
+#define DRM_XE_OA_FORMAT_MASK_FMT_TYPE (0xffu << 0)
+#define DRM_XE_OA_FORMAT_MASK_COUNTER_SEL (0xffu << 8)
+#define DRM_XE_OA_FORMAT_MASK_COUNTER_SIZE (0xffu << 16)
+#define DRM_XE_OA_FORMAT_MASK_BC_REPORT (0xffu << 24)
+ DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT,
+ DRM_XE_OA_PROPERTY_OA_DISABLED,
+ DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID,
+ DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE,
+ DRM_XE_OA_PROPERTY_NO_PREEMPT,
+};
+struct drm_xe_oa_config {
+ __u64 extensions;
+ char uuid[36];
+ __u32 n_regs;
+ __u64 regs_ptr;
+};
+struct drm_xe_oa_stream_status {
+ __u64 extensions;
+ __u64 oa_status;
+#define DRM_XE_OASTATUS_MMIO_TRG_Q_FULL (1 << 3)
+#define DRM_XE_OASTATUS_COUNTER_OVERFLOW (1 << 2)
+#define DRM_XE_OASTATUS_BUFFER_OVERFLOW (1 << 1)
+#define DRM_XE_OASTATUS_REPORT_LOST (1 << 0)
+ __u64 reserved[3];
+};
+struct drm_xe_oa_stream_info {
+ __u64 extensions;
+ __u64 oa_buf_size;
+ __u64 reserved[3];
+};
#ifdef __cplusplus
}
#endif
diff --git a/libc/kernel/uapi/linux/bpf.h b/libc/kernel/uapi/linux/bpf.h
index c732920..8d64816 100644
--- a/libc/kernel/uapi/linux/bpf.h
+++ b/libc/kernel/uapi/linux/bpf.h
@@ -367,6 +367,7 @@
#define BPF_F_QUERY_EFFECTIVE (1U << 0)
#define BPF_F_TEST_RUN_ON_CPU (1U << 0)
#define BPF_F_TEST_XDP_LIVE_FRAMES (1U << 1)
+#define BPF_F_TEST_SKB_CHECKSUM_COMPLETE (1U << 2)
enum bpf_stats_type {
BPF_STATS_RUN_TIME = 0,
};
@@ -773,8 +774,11 @@
#define __bpf_md_ptr(type,name) union { type name; __u64 : 64; \
} __attribute__((aligned(8)))
enum {
- BPF_SKB_TSTAMP_UNSPEC,
- BPF_SKB_TSTAMP_DELIVERY_MONO,
+ BPF_SKB_TSTAMP_UNSPEC = 0,
+ BPF_SKB_TSTAMP_DELIVERY_MONO = 1,
+ BPF_SKB_CLOCK_REALTIME = 0,
+ BPF_SKB_CLOCK_MONOTONIC = 1,
+ BPF_SKB_CLOCK_TAI = 2,
};
struct __sk_buff {
__u32 len;
diff --git a/libc/kernel/uapi/linux/btrfs_tree.h b/libc/kernel/uapi/linux/btrfs_tree.h
index ea33eee..88f44d9 100644
--- a/libc/kernel/uapi/linux/btrfs_tree.h
+++ b/libc/kernel/uapi/linux/btrfs_tree.h
@@ -279,18 +279,8 @@
__le64 devid;
__le64 physical;
} __attribute__((__packed__));
-#define BTRFS_STRIPE_RAID0 1
-#define BTRFS_STRIPE_RAID1 2
-#define BTRFS_STRIPE_DUP 3
-#define BTRFS_STRIPE_RAID10 4
-#define BTRFS_STRIPE_RAID5 5
-#define BTRFS_STRIPE_RAID6 6
-#define BTRFS_STRIPE_RAID1C3 7
-#define BTRFS_STRIPE_RAID1C4 8
struct btrfs_stripe_extent {
- __u8 encoding;
- __u8 reserved[7];
- struct btrfs_raid_stride strides[];
+ __DECLARE_FLEX_ARRAY(struct btrfs_raid_stride, strides);
} __attribute__((__packed__));
#define BTRFS_HEADER_FLAG_WRITTEN (1ULL << 0)
#define BTRFS_HEADER_FLAG_RELOC (1ULL << 1)
@@ -300,6 +290,9 @@
#define BTRFS_SUPER_FLAG_METADUMP_V2 (1ULL << 34)
#define BTRFS_SUPER_FLAG_CHANGING_FSID (1ULL << 35)
#define BTRFS_SUPER_FLAG_CHANGING_FSID_V2 (1ULL << 36)
+#define BTRFS_SUPER_FLAG_CHANGING_BG_TREE (1ULL << 38)
+#define BTRFS_SUPER_FLAG_CHANGING_DATA_CSUM (1ULL << 39)
+#define BTRFS_SUPER_FLAG_CHANGING_META_CSUM (1ULL << 40)
struct btrfs_extent_item {
__le64 refs;
__le64 generation;
diff --git a/libc/kernel/uapi/linux/dlm.h b/libc/kernel/uapi/linux/dlm.h
index 3c73908..1344f9b 100644
--- a/libc/kernel/uapi/linux/dlm.h
+++ b/libc/kernel/uapi/linux/dlm.h
@@ -20,4 +20,5 @@
};
#define DLM_LSFL_TIMEWARN 0x00000002
#define DLM_LSFL_NEWEXCL 0x00000008
+#define __DLM_LSFL_RESERVED0 0x00000010
#endif
diff --git a/libc/kernel/uapi/linux/dma-heap.h b/libc/kernel/uapi/linux/dma-heap.h
index 467e336..ac86b59 100644
--- a/libc/kernel/uapi/linux/dma-heap.h
+++ b/libc/kernel/uapi/linux/dma-heap.h
@@ -9,7 +9,7 @@
#include <linux/ioctl.h>
#include <linux/types.h>
#define DMA_HEAP_VALID_FD_FLAGS (O_CLOEXEC | O_ACCMODE)
-#define DMA_HEAP_VALID_HEAP_FLAGS (0)
+#define DMA_HEAP_VALID_HEAP_FLAGS (0ULL)
struct dma_heap_allocation_data {
__u64 len;
__u32 fd;
diff --git a/libc/kernel/uapi/linux/ethtool.h b/libc/kernel/uapi/linux/ethtool.h
index 334c788..e213ba1 100644
--- a/libc/kernel/uapi/linux/ethtool.h
+++ b/libc/kernel/uapi/linux/ethtool.h
@@ -270,6 +270,56 @@
ETHTOOL_MODULE_POWER_MODE_LOW = 1,
ETHTOOL_MODULE_POWER_MODE_HIGH,
};
+enum ethtool_c33_pse_ext_state {
+ ETHTOOL_C33_PSE_EXT_STATE_ERROR_CONDITION = 1,
+ ETHTOOL_C33_PSE_EXT_STATE_MR_MPS_VALID,
+ ETHTOOL_C33_PSE_EXT_STATE_MR_PSE_ENABLE,
+ ETHTOOL_C33_PSE_EXT_STATE_OPTION_DETECT_TED,
+ ETHTOOL_C33_PSE_EXT_STATE_OPTION_VPORT_LIM,
+ ETHTOOL_C33_PSE_EXT_STATE_OVLD_DETECTED,
+ ETHTOOL_C33_PSE_EXT_STATE_PD_DLL_POWER_TYPE,
+ ETHTOOL_C33_PSE_EXT_STATE_POWER_NOT_AVAILABLE,
+ ETHTOOL_C33_PSE_EXT_STATE_SHORT_DETECTED,
+};
+enum ethtool_c33_pse_ext_substate_mr_mps_valid {
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_MR_MPS_VALID_DETECTED_UNDERLOAD = 1,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_MR_MPS_VALID_CONNECTION_OPEN,
+};
+enum ethtool_c33_pse_ext_substate_error_condition {
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_NON_EXISTING_PORT = 1,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_UNDEFINED_PORT,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_INTERNAL_HW_FAULT,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_COMM_ERROR_AFTER_FORCE_ON,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_UNKNOWN_PORT_STATUS,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_HOST_CRASH_TURN_OFF,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_HOST_CRASH_FORCE_SHUTDOWN,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_CONFIG_CHANGE,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_DETECTED_OVER_TEMP,
+};
+enum ethtool_c33_pse_ext_substate_mr_pse_enable {
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_MR_PSE_ENABLE_DISABLE_PIN_ACTIVE = 1,
+};
+enum ethtool_c33_pse_ext_substate_option_detect_ted {
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_OPTION_DETECT_TED_DET_IN_PROCESS = 1,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_OPTION_DETECT_TED_CONNECTION_CHECK_ERROR,
+};
+enum ethtool_c33_pse_ext_substate_option_vport_lim {
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_OPTION_VPORT_LIM_HIGH_VOLTAGE = 1,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_OPTION_VPORT_LIM_LOW_VOLTAGE,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_OPTION_VPORT_LIM_VOLTAGE_INJECTION,
+};
+enum ethtool_c33_pse_ext_substate_ovld_detected {
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_OVLD_DETECTED_OVERLOAD = 1,
+};
+enum ethtool_c33_pse_ext_substate_power_not_available {
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_POWER_NOT_AVAILABLE_BUDGET_EXCEEDED = 1,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_POWER_NOT_AVAILABLE_PORT_PW_LIMIT_EXCEEDS_CONTROLLER_BUDGET,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_POWER_NOT_AVAILABLE_PD_REQUEST_EXCEEDS_PORT_LIMIT,
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_POWER_NOT_AVAILABLE_HW_PW_LIMIT,
+};
+enum ethtool_c33_pse_ext_substate_short_detected {
+ ETHTOOL_C33_PSE_EXT_SUBSTATE_SHORT_DETECTED_SHORT_CONDITION = 1,
+};
enum ethtool_pse_types {
ETHTOOL_PSE_UNKNOWN = 1 << 0,
ETHTOOL_PSE_PODL = 1 << 1,
@@ -311,6 +361,12 @@
ETHTOOL_MM_VERIFY_STATUS_FAILED,
ETHTOOL_MM_VERIFY_STATUS_DISABLED,
};
+enum ethtool_module_fw_flash_status {
+ ETHTOOL_MODULE_FW_FLASH_STATUS_STARTED = 1,
+ ETHTOOL_MODULE_FW_FLASH_STATUS_IN_PROGRESS,
+ ETHTOOL_MODULE_FW_FLASH_STATUS_COMPLETED,
+ ETHTOOL_MODULE_FW_FLASH_STATUS_ERROR,
+};
struct ethtool_gstrings {
__u32 cmd;
__u32 string_set;
@@ -749,6 +805,7 @@
ETHTOOL_LINK_MODE_10baseT1S_Full_BIT = 99,
ETHTOOL_LINK_MODE_10baseT1S_Half_BIT = 100,
ETHTOOL_LINK_MODE_10baseT1S_P2MP_Half_BIT = 101,
+ ETHTOOL_LINK_MODE_10baseT1BRR_Full_BIT = 102,
__ETHTOOL_LINK_MODE_MASK_NBITS
};
#define __ETHTOOL_LINK_MODE_LEGACY_MASK(base_name) (1UL << (ETHTOOL_LINK_MODE_ ##base_name ##_BIT))
diff --git a/libc/kernel/uapi/linux/ethtool_netlink.h b/libc/kernel/uapi/linux/ethtool_netlink.h
index 6757ef5..ac6391a 100644
--- a/libc/kernel/uapi/linux/ethtool_netlink.h
+++ b/libc/kernel/uapi/linux/ethtool_netlink.h
@@ -52,6 +52,7 @@
ETHTOOL_MSG_PLCA_GET_STATUS,
ETHTOOL_MSG_MM_GET,
ETHTOOL_MSG_MM_SET,
+ ETHTOOL_MSG_MODULE_FW_FLASH_ACT,
__ETHTOOL_MSG_USER_CNT,
ETHTOOL_MSG_USER_MAX = __ETHTOOL_MSG_USER_CNT - 1
};
@@ -100,6 +101,7 @@
ETHTOOL_MSG_PLCA_NTF,
ETHTOOL_MSG_MM_GET_REPLY,
ETHTOOL_MSG_MM_NTF,
+ ETHTOOL_MSG_MODULE_FW_FLASH_NTF,
__ETHTOOL_MSG_KERNEL_CNT,
ETHTOOL_MSG_KERNEL_MAX = __ETHTOOL_MSG_KERNEL_CNT - 1
};
@@ -316,10 +318,26 @@
ETHTOOL_A_COALESCE_TX_AGGR_MAX_BYTES,
ETHTOOL_A_COALESCE_TX_AGGR_MAX_FRAMES,
ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS,
+ ETHTOOL_A_COALESCE_RX_PROFILE,
+ ETHTOOL_A_COALESCE_TX_PROFILE,
__ETHTOOL_A_COALESCE_CNT,
ETHTOOL_A_COALESCE_MAX = (__ETHTOOL_A_COALESCE_CNT - 1)
};
enum {
+ ETHTOOL_A_PROFILE_UNSPEC,
+ ETHTOOL_A_PROFILE_IRQ_MODERATION,
+ __ETHTOOL_A_PROFILE_CNT,
+ ETHTOOL_A_PROFILE_MAX = (__ETHTOOL_A_PROFILE_CNT - 1)
+};
+enum {
+ ETHTOOL_A_IRQ_MODERATION_UNSPEC,
+ ETHTOOL_A_IRQ_MODERATION_USEC,
+ ETHTOOL_A_IRQ_MODERATION_PKTS,
+ ETHTOOL_A_IRQ_MODERATION_COMPS,
+ __ETHTOOL_A_IRQ_MODERATION_CNT,
+ ETHTOOL_A_IRQ_MODERATION_MAX = (__ETHTOOL_A_IRQ_MODERATION_CNT - 1)
+};
+enum {
ETHTOOL_A_PAUSE_UNSPEC,
ETHTOOL_A_PAUSE_HEADER,
ETHTOOL_A_PAUSE_AUTONEG,
@@ -635,6 +653,11 @@
ETHTOOL_A_MODULE_MAX = (__ETHTOOL_A_MODULE_CNT - 1)
};
enum {
+ ETHTOOL_A_C33_PSE_PW_LIMIT_UNSPEC,
+ ETHTOOL_A_C33_PSE_PW_LIMIT_MIN,
+ ETHTOOL_A_C33_PSE_PW_LIMIT_MAX,
+};
+enum {
ETHTOOL_A_PSE_UNSPEC,
ETHTOOL_A_PSE_HEADER,
ETHTOOL_A_PODL_PSE_ADMIN_STATE,
@@ -643,6 +666,12 @@
ETHTOOL_A_C33_PSE_ADMIN_STATE,
ETHTOOL_A_C33_PSE_ADMIN_CONTROL,
ETHTOOL_A_C33_PSE_PW_D_STATUS,
+ ETHTOOL_A_C33_PSE_PW_CLASS,
+ ETHTOOL_A_C33_PSE_ACTUAL_PW,
+ ETHTOOL_A_C33_PSE_EXT_STATE,
+ ETHTOOL_A_C33_PSE_EXT_SUBSTATE,
+ ETHTOOL_A_C33_PSE_AVAIL_PW_LIMIT,
+ ETHTOOL_A_C33_PSE_PW_LIMIT_RANGES,
__ETHTOOL_A_PSE_CNT,
ETHTOOL_A_PSE_MAX = (__ETHTOOL_A_PSE_CNT - 1)
};
@@ -699,6 +728,18 @@
__ETHTOOL_A_MM_CNT,
ETHTOOL_A_MM_MAX = (__ETHTOOL_A_MM_CNT - 1)
};
+enum {
+ ETHTOOL_A_MODULE_FW_FLASH_UNSPEC,
+ ETHTOOL_A_MODULE_FW_FLASH_HEADER,
+ ETHTOOL_A_MODULE_FW_FLASH_FILE_NAME,
+ ETHTOOL_A_MODULE_FW_FLASH_PASSWORD,
+ ETHTOOL_A_MODULE_FW_FLASH_STATUS,
+ ETHTOOL_A_MODULE_FW_FLASH_STATUS_MSG,
+ ETHTOOL_A_MODULE_FW_FLASH_DONE,
+ ETHTOOL_A_MODULE_FW_FLASH_TOTAL,
+ __ETHTOOL_A_MODULE_FW_FLASH_CNT,
+ ETHTOOL_A_MODULE_FW_FLASH_MAX = (__ETHTOOL_A_MODULE_FW_FLASH_CNT - 1)
+};
#define ETHTOOL_GENL_NAME "ethtool"
#define ETHTOOL_GENL_VERSION 1
#define ETHTOOL_MCGRP_MONITOR_NAME "monitor"
diff --git a/libc/kernel/uapi/linux/fs.h b/libc/kernel/uapi/linux/fs.h
index 38a2d9e..adab56f 100644
--- a/libc/kernel/uapi/linux/fs.h
+++ b/libc/kernel/uapi/linux/fs.h
@@ -194,8 +194,10 @@
#define RWF_NOWAIT (( __kernel_rwf_t) 0x00000008)
#define RWF_APPEND (( __kernel_rwf_t) 0x00000010)
#define RWF_NOAPPEND (( __kernel_rwf_t) 0x00000020)
-#define RWF_SUPPORTED (RWF_HIPRI | RWF_DSYNC | RWF_SYNC | RWF_NOWAIT | RWF_APPEND | RWF_NOAPPEND)
-#define PAGEMAP_SCAN _IOWR('f', 16, struct pm_scan_arg)
+#define RWF_ATOMIC (( __kernel_rwf_t) 0x00000040)
+#define RWF_SUPPORTED (RWF_HIPRI | RWF_DSYNC | RWF_SYNC | RWF_NOWAIT | RWF_APPEND | RWF_NOAPPEND | RWF_ATOMIC)
+#define PROCFS_IOCTL_MAGIC 'f'
+#define PAGEMAP_SCAN _IOWR(PROCFS_IOCTL_MAGIC, 16, struct pm_scan_arg)
#define PAGE_IS_WPALLOWED (1 << 0)
#define PAGE_IS_WRITTEN (1 << 1)
#define PAGE_IS_FILE (1 << 2)
@@ -225,4 +227,30 @@
__u64 category_anyof_mask;
__u64 return_mask;
};
+#define PROCMAP_QUERY _IOWR(PROCFS_IOCTL_MAGIC, 17, struct procmap_query)
+enum procmap_query_flags {
+ PROCMAP_QUERY_VMA_READABLE = 0x01,
+ PROCMAP_QUERY_VMA_WRITABLE = 0x02,
+ PROCMAP_QUERY_VMA_EXECUTABLE = 0x04,
+ PROCMAP_QUERY_VMA_SHARED = 0x08,
+ PROCMAP_QUERY_COVERING_OR_NEXT_VMA = 0x10,
+ PROCMAP_QUERY_FILE_BACKED_VMA = 0x20,
+};
+struct procmap_query {
+ __u64 size;
+ __u64 query_flags;
+ __u64 query_addr;
+ __u64 vma_start;
+ __u64 vma_end;
+ __u64 vma_flags;
+ __u64 vma_page_size;
+ __u64 vma_offset;
+ __u64 inode;
+ __u32 dev_major;
+ __u32 dev_minor;
+ __u32 vma_name_size;
+ __u32 build_id_size;
+ __u64 vma_name_addr;
+ __u64 build_id_addr;
+};
#endif
diff --git a/libc/kernel/uapi/linux/if_xdp.h b/libc/kernel/uapi/linux/if_xdp.h
index b7eec87..7201e06 100644
--- a/libc/kernel/uapi/linux/if_xdp.h
+++ b/libc/kernel/uapi/linux/if_xdp.h
@@ -14,6 +14,7 @@
#define XDP_USE_SG (1 << 4)
#define XDP_UMEM_UNALIGNED_CHUNK_FLAG (1 << 0)
#define XDP_UMEM_TX_SW_CSUM (1 << 1)
+#define XDP_UMEM_TX_METADATA_LEN (1 << 2)
struct sockaddr_xdp {
__u16 sxdp_family;
__u16 sxdp_flags;
diff --git a/libc/kernel/uapi/linux/iio/buffer.h b/libc/kernel/uapi/linux/iio/buffer.h
index 45c6f65..7e03a8c 100644
--- a/libc/kernel/uapi/linux/iio/buffer.h
+++ b/libc/kernel/uapi/linux/iio/buffer.h
@@ -6,5 +6,16 @@
*/
#ifndef _UAPI_IIO_BUFFER_H_
#define _UAPI_IIO_BUFFER_H_
+#include <linux/types.h>
+#define IIO_BUFFER_DMABUF_CYCLIC (1 << 0)
+#define IIO_BUFFER_DMABUF_SUPPORTED_FLAGS 0x00000001
+struct iio_dmabuf {
+ __u32 fd;
+ __u32 flags;
+ __u64 bytes_used;
+};
#define IIO_BUFFER_GET_FD_IOCTL _IOWR('i', 0x91, int)
+#define IIO_BUFFER_DMABUF_ATTACH_IOCTL _IOW('i', 0x92, int)
+#define IIO_BUFFER_DMABUF_DETACH_IOCTL _IOW('i', 0x93, int)
+#define IIO_BUFFER_DMABUF_ENQUEUE_IOCTL _IOW('i', 0x94, struct iio_dmabuf)
#endif
diff --git a/libc/kernel/uapi/linux/in.h b/libc/kernel/uapi/linux/in.h
index 44efdd8..97bf493 100644
--- a/libc/kernel/uapi/linux/in.h
+++ b/libc/kernel/uapi/linux/in.h
@@ -69,6 +69,8 @@
#define IPPROTO_ETHERNET IPPROTO_ETHERNET
IPPROTO_RAW = 255,
#define IPPROTO_RAW IPPROTO_RAW
+ IPPROTO_SMC = 256,
+#define IPPROTO_SMC IPPROTO_SMC
IPPROTO_MPTCP = 262,
#define IPPROTO_MPTCP IPPROTO_MPTCP
IPPROTO_MAX
diff --git a/libc/kernel/uapi/linux/io_uring.h b/libc/kernel/uapi/linux/io_uring.h
index 5505963..6b4f2ea 100644
--- a/libc/kernel/uapi/linux/io_uring.h
+++ b/libc/kernel/uapi/linux/io_uring.h
@@ -176,6 +176,8 @@
IORING_OP_FUTEX_WAITV,
IORING_OP_FIXED_FD_INSTALL,
IORING_OP_FTRUNCATE,
+ IORING_OP_BIND,
+ IORING_OP_LISTEN,
IORING_OP_LAST,
};
#define IORING_URING_CMD_FIXED (1U << 0)
diff --git a/libc/kernel/uapi/linux/iommufd.h b/libc/kernel/uapi/linux/iommufd.h
index 2570628..6f663b4 100644
--- a/libc/kernel/uapi/linux/iommufd.h
+++ b/libc/kernel/uapi/linux/iommufd.h
@@ -12,19 +12,20 @@
enum {
IOMMUFD_CMD_BASE = 0x80,
IOMMUFD_CMD_DESTROY = IOMMUFD_CMD_BASE,
- IOMMUFD_CMD_IOAS_ALLOC,
- IOMMUFD_CMD_IOAS_ALLOW_IOVAS,
- IOMMUFD_CMD_IOAS_COPY,
- IOMMUFD_CMD_IOAS_IOVA_RANGES,
- IOMMUFD_CMD_IOAS_MAP,
- IOMMUFD_CMD_IOAS_UNMAP,
- IOMMUFD_CMD_OPTION,
- IOMMUFD_CMD_VFIO_IOAS,
- IOMMUFD_CMD_HWPT_ALLOC,
- IOMMUFD_CMD_GET_HW_INFO,
- IOMMUFD_CMD_HWPT_SET_DIRTY_TRACKING,
- IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP,
- IOMMUFD_CMD_HWPT_INVALIDATE,
+ IOMMUFD_CMD_IOAS_ALLOC = 0x81,
+ IOMMUFD_CMD_IOAS_ALLOW_IOVAS = 0x82,
+ IOMMUFD_CMD_IOAS_COPY = 0x83,
+ IOMMUFD_CMD_IOAS_IOVA_RANGES = 0x84,
+ IOMMUFD_CMD_IOAS_MAP = 0x85,
+ IOMMUFD_CMD_IOAS_UNMAP = 0x86,
+ IOMMUFD_CMD_OPTION = 0x87,
+ IOMMUFD_CMD_VFIO_IOAS = 0x88,
+ IOMMUFD_CMD_HWPT_ALLOC = 0x89,
+ IOMMUFD_CMD_GET_HW_INFO = 0x8a,
+ IOMMUFD_CMD_HWPT_SET_DIRTY_TRACKING = 0x8b,
+ IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP = 0x8c,
+ IOMMUFD_CMD_HWPT_INVALIDATE = 0x8d,
+ IOMMUFD_CMD_FAULT_QUEUE_ALLOC = 0x8e,
};
struct iommu_destroy {
__u32 size;
@@ -122,6 +123,7 @@
enum iommufd_hwpt_alloc_flags {
IOMMU_HWPT_ALLOC_NEST_PARENT = 1 << 0,
IOMMU_HWPT_ALLOC_DIRTY_TRACKING = 1 << 1,
+ IOMMU_HWPT_FAULT_ID_VALID = 1 << 2,
};
enum iommu_hwpt_vtd_s1_flags {
IOMMU_VTD_S1_SRE = 1 << 0,
@@ -135,8 +137,8 @@
__u32 __reserved;
};
enum iommu_hwpt_data_type {
- IOMMU_HWPT_DATA_NONE,
- IOMMU_HWPT_DATA_VTD_S1,
+ IOMMU_HWPT_DATA_NONE = 0,
+ IOMMU_HWPT_DATA_VTD_S1 = 1,
};
struct iommu_hwpt_alloc {
__u32 size;
@@ -148,6 +150,8 @@
__u32 data_type;
__u32 data_len;
__aligned_u64 data_uptr;
+ __u32 fault_id;
+ __u32 __reserved2;
};
#define IOMMU_HWPT_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_ALLOC)
enum iommu_hw_info_vtd_flags {
@@ -160,8 +164,8 @@
__aligned_u64 ecap_reg;
};
enum iommu_hw_info_type {
- IOMMU_HW_INFO_TYPE_NONE,
- IOMMU_HW_INFO_TYPE_INTEL_VTD,
+ IOMMU_HW_INFO_TYPE_NONE = 0,
+ IOMMU_HW_INFO_TYPE_INTEL_VTD = 1,
};
enum iommufd_hw_capabilities {
IOMMU_HW_CAP_DIRTY_TRACKING = 1 << 0,
@@ -202,7 +206,7 @@
};
#define IOMMU_HWPT_GET_DIRTY_BITMAP _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP)
enum iommu_hwpt_invalidate_data_type {
- IOMMU_HWPT_INVALIDATE_DATA_VTD_S1,
+ IOMMU_HWPT_INVALIDATE_DATA_VTD_S1 = 0,
};
enum iommu_hwpt_vtd_s1_invalidate_flags {
IOMMU_VTD_INV_FLAGS_LEAF = 1 << 0,
@@ -223,4 +227,39 @@
__u32 __reserved;
};
#define IOMMU_HWPT_INVALIDATE _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_INVALIDATE)
+enum iommu_hwpt_pgfault_flags {
+ IOMMU_PGFAULT_FLAGS_PASID_VALID = (1 << 0),
+ IOMMU_PGFAULT_FLAGS_LAST_PAGE = (1 << 1),
+};
+enum iommu_hwpt_pgfault_perm {
+ IOMMU_PGFAULT_PERM_READ = (1 << 0),
+ IOMMU_PGFAULT_PERM_WRITE = (1 << 1),
+ IOMMU_PGFAULT_PERM_EXEC = (1 << 2),
+ IOMMU_PGFAULT_PERM_PRIV = (1 << 3),
+};
+struct iommu_hwpt_pgfault {
+ __u32 flags;
+ __u32 dev_id;
+ __u32 pasid;
+ __u32 grpid;
+ __u32 perm;
+ __u64 addr;
+ __u32 length;
+ __u32 cookie;
+};
+enum iommufd_page_response_code {
+ IOMMUFD_PAGE_RESP_SUCCESS = 0,
+ IOMMUFD_PAGE_RESP_INVALID = 1,
+};
+struct iommu_hwpt_page_response {
+ __u32 cookie;
+ __u32 code;
+};
+struct iommu_fault_alloc {
+ __u32 size;
+ __u32 flags;
+ __u32 out_fault_id;
+ __u32 out_fault_fd;
+};
+#define IOMMU_FAULT_QUEUE_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_FAULT_QUEUE_ALLOC)
#endif
diff --git a/libc/kernel/uapi/linux/kfd_ioctl.h b/libc/kernel/uapi/linux/kfd_ioctl.h
index 62c9872..193dd8e 100644
--- a/libc/kernel/uapi/linux/kfd_ioctl.h
+++ b/libc/kernel/uapi/linux/kfd_ioctl.h
@@ -9,7 +9,7 @@
#include <drm/drm.h>
#include <linux/ioctl.h>
#define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 15
+#define KFD_IOCTL_MINOR_VERSION 16
struct kfd_ioctl_get_version_args {
__u32 major_version;
__u32 minor_version;
@@ -269,6 +269,7 @@
#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26)
#define KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED (1 << 25)
#define KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT (1 << 24)
+#define KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS (1 << 23)
struct kfd_ioctl_alloc_memory_of_gpu_args {
__u64 va_addr;
__u64 size;
@@ -465,6 +466,7 @@
};
enum kfd_dbg_trap_flags {
KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP = 1,
+ KFD_DBG_TRAP_FLAG_SINGLE_ALU_OP = 2,
};
enum kfd_dbg_trap_exception_code {
EC_NONE = 0,
diff --git a/libc/kernel/uapi/linux/kfd_sysfs.h b/libc/kernel/uapi/linux/kfd_sysfs.h
index e538cf2..7771582 100644
--- a/libc/kernel/uapi/linux/kfd_sysfs.h
+++ b/libc/kernel/uapi/linux/kfd_sysfs.h
@@ -35,7 +35,8 @@
#define HSA_CAP_SVMAPI_SUPPORTED 0x08000000
#define HSA_CAP_FLAGS_COHERENTHOSTACCESS 0x10000000
#define HSA_CAP_TRAP_DEBUG_FIRMWARE_SUPPORTED 0x20000000
-#define HSA_CAP_RESERVED 0xe00f8000
+#define HSA_CAP_TRAP_DEBUG_PRECISE_ALU_OPERATIONS_SUPPORTED 0x40000000
+#define HSA_CAP_RESERVED 0x800f8000
#define HSA_DBG_WATCH_ADDR_MASK_LO_BIT_MASK 0x0000000f
#define HSA_DBG_WATCH_ADDR_MASK_LO_BIT_SHIFT 0
#define HSA_DBG_WATCH_ADDR_MASK_HI_BIT_MASK 0x000003f0
diff --git a/libc/kernel/uapi/linux/kvm.h b/libc/kernel/uapi/linux/kvm.h
index ffaf5e6..297a09d 100644
--- a/libc/kernel/uapi/linux/kvm.h
+++ b/libc/kernel/uapi/linux/kvm.h
@@ -149,9 +149,10 @@
#define KVM_INTERNAL_ERROR_DELIVERY_EV 3
#define KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON 4
#define KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES (1ULL << 0)
+#define HINT_UNSAFE_IN_KVM(_symbol) _symbol
struct kvm_run {
__u8 request_interrupt_window;
- __u8 immediate_exit;
+ __u8 HINT_UNSAFE_IN_KVM(immediate_exit);
__u8 padding1[6];
__u32 exit_reason;
__u8 ready_for_interrupt_injection;
@@ -719,6 +720,9 @@
#define KVM_CAP_MEMORY_ATTRIBUTES 233
#define KVM_CAP_GUEST_MEMFD 234
#define KVM_CAP_VM_TYPES 235
+#define KVM_CAP_PRE_FAULT_MEMORY 236
+#define KVM_CAP_X86_APIC_BUS_CYCLES_NS 237
+#define KVM_CAP_X86_GUEST_MODE 238
struct kvm_irq_routing_irqchip {
__u32 irqchip;
__u32 pin;
@@ -1109,4 +1113,11 @@
__u64 flags;
__u64 reserved[6];
};
+#define KVM_PRE_FAULT_MEMORY _IOWR(KVMIO, 0xd5, struct kvm_pre_fault_memory)
+struct kvm_pre_fault_memory {
+ __u64 gpa;
+ __u64 size;
+ __u64 flags;
+ __u64 padding[5];
+};
#endif
diff --git a/libc/kernel/uapi/linux/media/raspberrypi/pisp_be_config.h b/libc/kernel/uapi/linux/media/raspberrypi/pisp_be_config.h
new file mode 100644
index 0000000..2e981ad
--- /dev/null
+++ b/libc/kernel/uapi/linux/media/raspberrypi/pisp_be_config.h
@@ -0,0 +1,418 @@
+/*
+ * This file is auto-generated. Modifications will be lost.
+ *
+ * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
+ * for more information.
+ */
+#ifndef _UAPI_PISP_BE_CONFIG_H_
+#define _UAPI_PISP_BE_CONFIG_H_
+#include <linux/types.h>
+#include "pisp_common.h"
+#define PISP_BACK_END_INPUT_ALIGN 4u
+#define PISP_BACK_END_COMPRESSED_ALIGN 8u
+#define PISP_BACK_END_OUTPUT_MIN_ALIGN 16u
+#define PISP_BACK_END_OUTPUT_MAX_ALIGN 64u
+#define PISP_BACK_END_MIN_TILE_WIDTH 16u
+#define PISP_BACK_END_MIN_TILE_HEIGHT 16u
+#define PISP_BACK_END_NUM_OUTPUTS 2
+#define PISP_BACK_END_HOG_OUTPUT 1
+#define PISP_BACK_END_NUM_TILES 64
+enum pisp_be_bayer_enable {
+ PISP_BE_BAYER_ENABLE_INPUT = 0x000001,
+ PISP_BE_BAYER_ENABLE_DECOMPRESS = 0x000002,
+ PISP_BE_BAYER_ENABLE_DPC = 0x000004,
+ PISP_BE_BAYER_ENABLE_GEQ = 0x000008,
+ PISP_BE_BAYER_ENABLE_TDN_INPUT = 0x000010,
+ PISP_BE_BAYER_ENABLE_TDN_DECOMPRESS = 0x000020,
+ PISP_BE_BAYER_ENABLE_TDN = 0x000040,
+ PISP_BE_BAYER_ENABLE_TDN_COMPRESS = 0x000080,
+ PISP_BE_BAYER_ENABLE_TDN_OUTPUT = 0x000100,
+ PISP_BE_BAYER_ENABLE_SDN = 0x000200,
+ PISP_BE_BAYER_ENABLE_BLC = 0x000400,
+ PISP_BE_BAYER_ENABLE_STITCH_INPUT = 0x000800,
+ PISP_BE_BAYER_ENABLE_STITCH_DECOMPRESS = 0x001000,
+ PISP_BE_BAYER_ENABLE_STITCH = 0x002000,
+ PISP_BE_BAYER_ENABLE_STITCH_COMPRESS = 0x004000,
+ PISP_BE_BAYER_ENABLE_STITCH_OUTPUT = 0x008000,
+ PISP_BE_BAYER_ENABLE_WBG = 0x010000,
+ PISP_BE_BAYER_ENABLE_CDN = 0x020000,
+ PISP_BE_BAYER_ENABLE_LSC = 0x040000,
+ PISP_BE_BAYER_ENABLE_TONEMAP = 0x080000,
+ PISP_BE_BAYER_ENABLE_CAC = 0x100000,
+ PISP_BE_BAYER_ENABLE_DEBIN = 0x200000,
+ PISP_BE_BAYER_ENABLE_DEMOSAIC = 0x400000,
+};
+enum pisp_be_rgb_enable {
+ PISP_BE_RGB_ENABLE_INPUT = 0x000001,
+ PISP_BE_RGB_ENABLE_CCM = 0x000002,
+ PISP_BE_RGB_ENABLE_SAT_CONTROL = 0x000004,
+ PISP_BE_RGB_ENABLE_YCBCR = 0x000008,
+ PISP_BE_RGB_ENABLE_FALSE_COLOUR = 0x000010,
+ PISP_BE_RGB_ENABLE_SHARPEN = 0x000020,
+ PISP_BE_RGB_ENABLE_YCBCR_INVERSE = 0x000080,
+ PISP_BE_RGB_ENABLE_GAMMA = 0x000100,
+ PISP_BE_RGB_ENABLE_CSC0 = 0x000200,
+ PISP_BE_RGB_ENABLE_CSC1 = 0x000400,
+ PISP_BE_RGB_ENABLE_DOWNSCALE0 = 0x001000,
+ PISP_BE_RGB_ENABLE_DOWNSCALE1 = 0x002000,
+ PISP_BE_RGB_ENABLE_RESAMPLE0 = 0x008000,
+ PISP_BE_RGB_ENABLE_RESAMPLE1 = 0x010000,
+ PISP_BE_RGB_ENABLE_OUTPUT0 = 0x040000,
+ PISP_BE_RGB_ENABLE_OUTPUT1 = 0x080000,
+ PISP_BE_RGB_ENABLE_HOG = 0x200000
+};
+#define PISP_BE_RGB_ENABLE_CSC(i) (PISP_BE_RGB_ENABLE_CSC0 << (i))
+#define PISP_BE_RGB_ENABLE_DOWNSCALE(i) (PISP_BE_RGB_ENABLE_DOWNSCALE0 << (i))
+#define PISP_BE_RGB_ENABLE_RESAMPLE(i) (PISP_BE_RGB_ENABLE_RESAMPLE0 << (i))
+#define PISP_BE_RGB_ENABLE_OUTPUT(i) (PISP_BE_RGB_ENABLE_OUTPUT0 << (i))
+enum pisp_be_dirty {
+ PISP_BE_DIRTY_GLOBAL = 0x0001,
+ PISP_BE_DIRTY_SH_FC_COMBINE = 0x0002,
+ PISP_BE_DIRTY_CROP = 0x0004
+};
+struct pisp_be_global_config {
+ __u32 bayer_enables;
+ __u32 rgb_enables;
+ __u8 bayer_order;
+ __u8 pad[3];
+} __attribute__((packed));
+struct pisp_be_input_buffer_config {
+ __u32 addr[3][2];
+} __attribute__((packed));
+struct pisp_be_dpc_config {
+ __u8 coeff_level;
+ __u8 coeff_range;
+ __u8 pad;
+#define PISP_BE_DPC_FLAG_FOLDBACK 1
+ __u8 flags;
+} __attribute__((packed));
+struct pisp_be_geq_config {
+ __u16 offset;
+#define PISP_BE_GEQ_SHARPER (1U << 15)
+#define PISP_BE_GEQ_SLOPE ((1 << 10) - 1)
+ __u16 slope_sharper;
+ __u16 min;
+ __u16 max;
+} __attribute__((packed));
+struct pisp_be_tdn_input_buffer_config {
+ __u32 addr[2];
+} __attribute__((packed));
+struct pisp_be_tdn_config {
+ __u16 black_level;
+ __u16 ratio;
+ __u16 noise_constant;
+ __u16 noise_slope;
+ __u16 threshold;
+ __u8 reset;
+ __u8 pad;
+} __attribute__((packed));
+struct pisp_be_tdn_output_buffer_config {
+ __u32 addr[2];
+} __attribute__((packed));
+struct pisp_be_sdn_config {
+ __u16 black_level;
+ __u8 leakage;
+ __u8 pad;
+ __u16 noise_constant;
+ __u16 noise_slope;
+ __u16 noise_constant2;
+ __u16 noise_slope2;
+} __attribute__((packed));
+struct pisp_be_stitch_input_buffer_config {
+ __u32 addr[2];
+} __attribute__((packed));
+#define PISP_BE_STITCH_STREAMING_LONG 0x8000
+#define PISP_BE_STITCH_EXPOSURE_RATIO_MASK 0x7fff
+struct pisp_be_stitch_config {
+ __u16 threshold_lo;
+ __u8 threshold_diff_power;
+ __u8 pad;
+ __u16 exposure_ratio;
+ __u8 motion_threshold_256;
+ __u8 motion_threshold_recip;
+} __attribute__((packed));
+struct pisp_be_stitch_output_buffer_config {
+ __u32 addr[2];
+} __attribute__((packed));
+struct pisp_be_cdn_config {
+ __u16 thresh;
+ __u8 iir_strength;
+ __u8 g_adjust;
+} __attribute__((packed));
+#define PISP_BE_LSC_LOG_GRID_SIZE 5
+#define PISP_BE_LSC_GRID_SIZE (1 << PISP_BE_LSC_LOG_GRID_SIZE)
+#define PISP_BE_LSC_STEP_PRECISION 18
+struct pisp_be_lsc_config {
+ __u16 grid_step_x;
+ __u16 grid_step_y;
+#define PISP_BE_LSC_LUT_SIZE (PISP_BE_LSC_GRID_SIZE + 1)
+ __u32 lut_packed[PISP_BE_LSC_LUT_SIZE][PISP_BE_LSC_LUT_SIZE];
+} __attribute__((packed));
+struct pisp_be_lsc_extra {
+ __u16 offset_x;
+ __u16 offset_y;
+} __attribute__((packed));
+#define PISP_BE_CAC_LOG_GRID_SIZE 3
+#define PISP_BE_CAC_GRID_SIZE (1 << PISP_BE_CAC_LOG_GRID_SIZE)
+#define PISP_BE_CAC_STEP_PRECISION 20
+struct pisp_be_cac_config {
+ __u16 grid_step_x;
+ __u16 grid_step_y;
+#define PISP_BE_CAC_LUT_SIZE (PISP_BE_CAC_GRID_SIZE + 1)
+ __s8 lut[PISP_BE_CAC_LUT_SIZE][PISP_BE_CAC_LUT_SIZE][2][2];
+} __attribute__((packed));
+struct pisp_be_cac_extra {
+ __u16 offset_x;
+ __u16 offset_y;
+} __attribute__((packed));
+#define PISP_BE_DEBIN_NUM_COEFFS 4
+struct pisp_be_debin_config {
+ __s8 coeffs[PISP_BE_DEBIN_NUM_COEFFS];
+ __s8 h_enable;
+ __s8 v_enable;
+ __s8 pad[2];
+} __attribute__((packed));
+#define PISP_BE_TONEMAP_LUT_SIZE 64
+struct pisp_be_tonemap_config {
+ __u16 detail_constant;
+ __u16 detail_slope;
+ __u16 iir_strength;
+ __u16 strength;
+ __u32 lut[PISP_BE_TONEMAP_LUT_SIZE];
+} __attribute__((packed));
+struct pisp_be_demosaic_config {
+ __u8 sharper;
+ __u8 fc_mode;
+ __u8 pad[2];
+} __attribute__((packed));
+struct pisp_be_ccm_config {
+ __s16 coeffs[9];
+ __u8 pad[2];
+ __s32 offsets[3];
+} __attribute__((packed));
+struct pisp_be_sat_control_config {
+ __u8 shift_r;
+ __u8 shift_g;
+ __u8 shift_b;
+ __u8 pad;
+} __attribute__((packed));
+struct pisp_be_false_colour_config {
+ __u8 distance;
+ __u8 pad[3];
+} __attribute__((packed));
+#define PISP_BE_SHARPEN_SIZE 5
+#define PISP_BE_SHARPEN_FUNC_NUM_POINTS 9
+struct pisp_be_sharpen_config {
+ __s8 kernel0[PISP_BE_SHARPEN_SIZE * PISP_BE_SHARPEN_SIZE];
+ __s8 pad0[3];
+ __s8 kernel1[PISP_BE_SHARPEN_SIZE * PISP_BE_SHARPEN_SIZE];
+ __s8 pad1[3];
+ __s8 kernel2[PISP_BE_SHARPEN_SIZE * PISP_BE_SHARPEN_SIZE];
+ __s8 pad2[3];
+ __s8 kernel3[PISP_BE_SHARPEN_SIZE * PISP_BE_SHARPEN_SIZE];
+ __s8 pad3[3];
+ __s8 kernel4[PISP_BE_SHARPEN_SIZE * PISP_BE_SHARPEN_SIZE];
+ __s8 pad4[3];
+ __u16 threshold_offset0;
+ __u16 threshold_slope0;
+ __u16 scale0;
+ __u16 pad5;
+ __u16 threshold_offset1;
+ __u16 threshold_slope1;
+ __u16 scale1;
+ __u16 pad6;
+ __u16 threshold_offset2;
+ __u16 threshold_slope2;
+ __u16 scale2;
+ __u16 pad7;
+ __u16 threshold_offset3;
+ __u16 threshold_slope3;
+ __u16 scale3;
+ __u16 pad8;
+ __u16 threshold_offset4;
+ __u16 threshold_slope4;
+ __u16 scale4;
+ __u16 pad9;
+ __u16 positive_strength;
+ __u16 positive_pre_limit;
+ __u16 positive_func[PISP_BE_SHARPEN_FUNC_NUM_POINTS];
+ __u16 positive_limit;
+ __u16 negative_strength;
+ __u16 negative_pre_limit;
+ __u16 negative_func[PISP_BE_SHARPEN_FUNC_NUM_POINTS];
+ __u16 negative_limit;
+ __u8 enables;
+ __u8 white;
+ __u8 black;
+ __u8 grey;
+} __attribute__((packed));
+struct pisp_be_sh_fc_combine_config {
+ __u8 y_factor;
+ __u8 c1_factor;
+ __u8 c2_factor;
+ __u8 pad;
+} __attribute__((packed));
+#define PISP_BE_GAMMA_LUT_SIZE 64
+struct pisp_be_gamma_config {
+ __u32 lut[PISP_BE_GAMMA_LUT_SIZE];
+} __attribute__((packed));
+struct pisp_be_crop_config {
+ __u16 offset_x, offset_y;
+ __u16 width, height;
+} __attribute__((packed));
+#define PISP_BE_RESAMPLE_FILTER_SIZE 96
+struct pisp_be_resample_config {
+ __u16 scale_factor_h, scale_factor_v;
+ __s16 coef[PISP_BE_RESAMPLE_FILTER_SIZE];
+} __attribute__((packed));
+struct pisp_be_resample_extra {
+ __u16 scaled_width;
+ __u16 scaled_height;
+ __s16 initial_phase_h[3];
+ __s16 initial_phase_v[3];
+} __attribute__((packed));
+struct pisp_be_downscale_config {
+ __u16 scale_factor_h;
+ __u16 scale_factor_v;
+ __u16 scale_recip_h;
+ __u16 scale_recip_v;
+} __attribute__((packed));
+struct pisp_be_downscale_extra {
+ __u16 scaled_width;
+ __u16 scaled_height;
+} __attribute__((packed));
+struct pisp_be_hog_config {
+ __u8 compute_signed;
+ __u8 channel_mix[3];
+ __u32 stride;
+} __attribute__((packed));
+struct pisp_be_axi_config {
+ __u8 r_qos;
+ __u8 r_cache_prot;
+ __u8 w_qos;
+ __u8 w_cache_prot;
+} __attribute__((packed));
+enum pisp_be_transform {
+ PISP_BE_TRANSFORM_NONE = 0x0,
+ PISP_BE_TRANSFORM_HFLIP = 0x1,
+ PISP_BE_TRANSFORM_VFLIP = 0x2,
+ PISP_BE_TRANSFORM_ROT180 = (PISP_BE_TRANSFORM_HFLIP | PISP_BE_TRANSFORM_VFLIP)
+};
+struct pisp_be_output_format_config {
+ struct pisp_image_format_config image;
+ __u8 transform;
+ __u8 pad[3];
+ __u16 lo;
+ __u16 hi;
+ __u16 lo2;
+ __u16 hi2;
+} __attribute__((packed));
+struct pisp_be_output_buffer_config {
+ __u32 addr[3][2];
+} __attribute__((packed));
+struct pisp_be_hog_buffer_config {
+ __u32 addr[2];
+} __attribute__((packed));
+struct pisp_be_config {
+ struct pisp_be_input_buffer_config input_buffer;
+ struct pisp_be_tdn_input_buffer_config tdn_input_buffer;
+ struct pisp_be_stitch_input_buffer_config stitch_input_buffer;
+ struct pisp_be_tdn_output_buffer_config tdn_output_buffer;
+ struct pisp_be_stitch_output_buffer_config stitch_output_buffer;
+ struct pisp_be_output_buffer_config output_buffer[PISP_BACK_END_NUM_OUTPUTS];
+ struct pisp_be_hog_buffer_config hog_buffer;
+ struct pisp_be_global_config global;
+ struct pisp_image_format_config input_format;
+ struct pisp_decompress_config decompress;
+ struct pisp_be_dpc_config dpc;
+ struct pisp_be_geq_config geq;
+ struct pisp_image_format_config tdn_input_format;
+ struct pisp_decompress_config tdn_decompress;
+ struct pisp_be_tdn_config tdn;
+ struct pisp_compress_config tdn_compress;
+ struct pisp_image_format_config tdn_output_format;
+ struct pisp_be_sdn_config sdn;
+ struct pisp_bla_config blc;
+ struct pisp_compress_config stitch_compress;
+ struct pisp_image_format_config stitch_output_format;
+ struct pisp_image_format_config stitch_input_format;
+ struct pisp_decompress_config stitch_decompress;
+ struct pisp_be_stitch_config stitch;
+ struct pisp_be_lsc_config lsc;
+ struct pisp_wbg_config wbg;
+ struct pisp_be_cdn_config cdn;
+ struct pisp_be_cac_config cac;
+ struct pisp_be_debin_config debin;
+ struct pisp_be_tonemap_config tonemap;
+ struct pisp_be_demosaic_config demosaic;
+ struct pisp_be_ccm_config ccm;
+ struct pisp_be_sat_control_config sat_control;
+ struct pisp_be_ccm_config ycbcr;
+ struct pisp_be_sharpen_config sharpen;
+ struct pisp_be_false_colour_config false_colour;
+ struct pisp_be_sh_fc_combine_config sh_fc_combine;
+ struct pisp_be_ccm_config ycbcr_inverse;
+ struct pisp_be_gamma_config gamma;
+ struct pisp_be_ccm_config csc[PISP_BACK_END_NUM_OUTPUTS];
+ struct pisp_be_downscale_config downscale[PISP_BACK_END_NUM_OUTPUTS];
+ struct pisp_be_resample_config resample[PISP_BACK_END_NUM_OUTPUTS];
+ struct pisp_be_output_format_config output_format[PISP_BACK_END_NUM_OUTPUTS];
+ struct pisp_be_hog_config hog;
+ struct pisp_be_axi_config axi;
+ struct pisp_be_lsc_extra lsc_extra;
+ struct pisp_be_cac_extra cac_extra;
+ struct pisp_be_downscale_extra downscale_extra[PISP_BACK_END_NUM_OUTPUTS];
+ struct pisp_be_resample_extra resample_extra[PISP_BACK_END_NUM_OUTPUTS];
+ struct pisp_be_crop_config crop;
+ struct pisp_image_format_config hog_format;
+ __u32 dirty_flags_bayer;
+ __u32 dirty_flags_rgb;
+ __u32 dirty_flags_extra;
+} __attribute__((packed));
+enum pisp_tile_edge {
+ PISP_LEFT_EDGE = (1 << 0),
+ PISP_RIGHT_EDGE = (1 << 1),
+ PISP_TOP_EDGE = (1 << 2),
+ PISP_BOTTOM_EDGE = (1 << 3)
+};
+struct pisp_tile {
+ __u8 edge;
+ __u8 pad0[3];
+ __u32 input_addr_offset;
+ __u32 input_addr_offset2;
+ __u16 input_offset_x;
+ __u16 input_offset_y;
+ __u16 input_width;
+ __u16 input_height;
+ __u32 tdn_input_addr_offset;
+ __u32 tdn_output_addr_offset;
+ __u32 stitch_input_addr_offset;
+ __u32 stitch_output_addr_offset;
+ __u32 lsc_grid_offset_x;
+ __u32 lsc_grid_offset_y;
+ __u32 cac_grid_offset_x;
+ __u32 cac_grid_offset_y;
+ __u16 crop_x_start[PISP_BACK_END_NUM_OUTPUTS];
+ __u16 crop_x_end[PISP_BACK_END_NUM_OUTPUTS];
+ __u16 crop_y_start[PISP_BACK_END_NUM_OUTPUTS];
+ __u16 crop_y_end[PISP_BACK_END_NUM_OUTPUTS];
+ __u16 downscale_phase_x[3 * PISP_BACK_END_NUM_OUTPUTS];
+ __u16 downscale_phase_y[3 * PISP_BACK_END_NUM_OUTPUTS];
+ __u16 resample_in_width[PISP_BACK_END_NUM_OUTPUTS];
+ __u16 resample_in_height[PISP_BACK_END_NUM_OUTPUTS];
+ __u16 resample_phase_x[3 * PISP_BACK_END_NUM_OUTPUTS];
+ __u16 resample_phase_y[3 * PISP_BACK_END_NUM_OUTPUTS];
+ __u16 output_offset_x[PISP_BACK_END_NUM_OUTPUTS];
+ __u16 output_offset_y[PISP_BACK_END_NUM_OUTPUTS];
+ __u16 output_width[PISP_BACK_END_NUM_OUTPUTS];
+ __u16 output_height[PISP_BACK_END_NUM_OUTPUTS];
+ __u32 output_addr_offset[PISP_BACK_END_NUM_OUTPUTS];
+ __u32 output_addr_offset2[PISP_BACK_END_NUM_OUTPUTS];
+ __u32 output_hog_addr_offset;
+} __attribute__((packed));
+struct pisp_be_tiles_config {
+ struct pisp_be_config config;
+ struct pisp_tile tiles[PISP_BACK_END_NUM_TILES];
+ __u32 num_tiles;
+} __attribute__((packed));
+#endif
diff --git a/libc/kernel/uapi/linux/media/raspberrypi/pisp_common.h b/libc/kernel/uapi/linux/media/raspberrypi/pisp_common.h
new file mode 100644
index 0000000..0e0b23f
--- /dev/null
+++ b/libc/kernel/uapi/linux/media/raspberrypi/pisp_common.h
@@ -0,0 +1,119 @@
+/*
+ * This file is auto-generated. Modifications will be lost.
+ *
+ * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
+ * for more information.
+ */
+#ifndef _UAPI_PISP_COMMON_H_
+#define _UAPI_PISP_COMMON_H_
+#include <linux/types.h>
+struct pisp_image_format_config {
+ __u16 width;
+ __u16 height;
+ __u32 format;
+ __s32 stride;
+ __s32 stride2;
+} __attribute__((packed));
+enum pisp_bayer_order {
+ PISP_BAYER_ORDER_RGGB = 0,
+ PISP_BAYER_ORDER_GBRG = 1,
+ PISP_BAYER_ORDER_BGGR = 2,
+ PISP_BAYER_ORDER_GRBG = 3,
+ PISP_BAYER_ORDER_GREYSCALE = 128
+};
+enum pisp_image_format {
+ PISP_IMAGE_FORMAT_BPS_8 = 0x00000000,
+ PISP_IMAGE_FORMAT_BPS_10 = 0x00000001,
+ PISP_IMAGE_FORMAT_BPS_12 = 0x00000002,
+ PISP_IMAGE_FORMAT_BPS_16 = 0x00000003,
+ PISP_IMAGE_FORMAT_BPS_MASK = 0x00000003,
+ PISP_IMAGE_FORMAT_PLANARITY_INTERLEAVED = 0x00000000,
+ PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR = 0x00000010,
+ PISP_IMAGE_FORMAT_PLANARITY_PLANAR = 0x00000020,
+ PISP_IMAGE_FORMAT_PLANARITY_MASK = 0x00000030,
+ PISP_IMAGE_FORMAT_SAMPLING_444 = 0x00000000,
+ PISP_IMAGE_FORMAT_SAMPLING_422 = 0x00000100,
+ PISP_IMAGE_FORMAT_SAMPLING_420 = 0x00000200,
+ PISP_IMAGE_FORMAT_SAMPLING_MASK = 0x00000300,
+ PISP_IMAGE_FORMAT_ORDER_NORMAL = 0x00000000,
+ PISP_IMAGE_FORMAT_ORDER_SWAPPED = 0x00001000,
+ PISP_IMAGE_FORMAT_SHIFT_0 = 0x00000000,
+ PISP_IMAGE_FORMAT_SHIFT_1 = 0x00010000,
+ PISP_IMAGE_FORMAT_SHIFT_2 = 0x00020000,
+ PISP_IMAGE_FORMAT_SHIFT_3 = 0x00030000,
+ PISP_IMAGE_FORMAT_SHIFT_4 = 0x00040000,
+ PISP_IMAGE_FORMAT_SHIFT_5 = 0x00050000,
+ PISP_IMAGE_FORMAT_SHIFT_6 = 0x00060000,
+ PISP_IMAGE_FORMAT_SHIFT_7 = 0x00070000,
+ PISP_IMAGE_FORMAT_SHIFT_8 = 0x00080000,
+ PISP_IMAGE_FORMAT_SHIFT_MASK = 0x000f0000,
+ PISP_IMAGE_FORMAT_BPP_32 = 0x00100000,
+ PISP_IMAGE_FORMAT_UNCOMPRESSED = 0x00000000,
+ PISP_IMAGE_FORMAT_COMPRESSION_MODE_1 = 0x01000000,
+ PISP_IMAGE_FORMAT_COMPRESSION_MODE_2 = 0x02000000,
+ PISP_IMAGE_FORMAT_COMPRESSION_MODE_3 = 0x03000000,
+ PISP_IMAGE_FORMAT_COMPRESSION_MASK = 0x03000000,
+ PISP_IMAGE_FORMAT_HOG_SIGNED = 0x04000000,
+ PISP_IMAGE_FORMAT_HOG_UNSIGNED = 0x08000000,
+ PISP_IMAGE_FORMAT_INTEGRAL_IMAGE = 0x10000000,
+ PISP_IMAGE_FORMAT_WALLPAPER_ROLL = 0x20000000,
+ PISP_IMAGE_FORMAT_THREE_CHANNEL = 0x40000000,
+ PISP_IMAGE_FORMAT_SINGLE_16 = PISP_IMAGE_FORMAT_BPS_16,
+ PISP_IMAGE_FORMAT_THREE_16 = PISP_IMAGE_FORMAT_BPS_16 | PISP_IMAGE_FORMAT_THREE_CHANNEL
+};
+#define PISP_IMAGE_FORMAT_BPS_8(fmt) (((fmt) & PISP_IMAGE_FORMAT_BPS_MASK) == PISP_IMAGE_FORMAT_BPS_8)
+#define PISP_IMAGE_FORMAT_BPS_10(fmt) (((fmt) & PISP_IMAGE_FORMAT_BPS_MASK) == PISP_IMAGE_FORMAT_BPS_10)
+#define PISP_IMAGE_FORMAT_BPS_12(fmt) (((fmt) & PISP_IMAGE_FORMAT_BPS_MASK) == PISP_IMAGE_FORMAT_BPS_12)
+#define PISP_IMAGE_FORMAT_BPS_16(fmt) (((fmt) & PISP_IMAGE_FORMAT_BPS_MASK) == PISP_IMAGE_FORMAT_BPS_16)
+#define PISP_IMAGE_FORMAT_BPS(fmt) (((fmt) & PISP_IMAGE_FORMAT_BPS_MASK) ? 8 + (2 << (((fmt) & PISP_IMAGE_FORMAT_BPS_MASK) - 1)) : 8)
+#define PISP_IMAGE_FORMAT_SHIFT(fmt) (((fmt) & PISP_IMAGE_FORMAT_SHIFT_MASK) / PISP_IMAGE_FORMAT_SHIFT_1)
+#define PISP_IMAGE_FORMAT_THREE_CHANNEL(fmt) ((fmt) & PISP_IMAGE_FORMAT_THREE_CHANNEL)
+#define PISP_IMAGE_FORMAT_SINGLE_CHANNEL(fmt) (! ((fmt) & PISP_IMAGE_FORMAT_THREE_CHANNEL))
+#define PISP_IMAGE_FORMAT_COMPRESSED(fmt) (((fmt) & PISP_IMAGE_FORMAT_COMPRESSION_MASK) != PISP_IMAGE_FORMAT_UNCOMPRESSED)
+#define PISP_IMAGE_FORMAT_SAMPLING_444(fmt) (((fmt) & PISP_IMAGE_FORMAT_SAMPLING_MASK) == PISP_IMAGE_FORMAT_SAMPLING_444)
+#define PISP_IMAGE_FORMAT_SAMPLING_422(fmt) (((fmt) & PISP_IMAGE_FORMAT_SAMPLING_MASK) == PISP_IMAGE_FORMAT_SAMPLING_422)
+#define PISP_IMAGE_FORMAT_SAMPLING_420(fmt) (((fmt) & PISP_IMAGE_FORMAT_SAMPLING_MASK) == PISP_IMAGE_FORMAT_SAMPLING_420)
+#define PISP_IMAGE_FORMAT_ORDER_NORMAL(fmt) (! ((fmt) & PISP_IMAGE_FORMAT_ORDER_SWAPPED))
+#define PISP_IMAGE_FORMAT_ORDER_SWAPPED(fmt) ((fmt) & PISP_IMAGE_FORMAT_ORDER_SWAPPED)
+#define PISP_IMAGE_FORMAT_INTERLEAVED(fmt) (((fmt) & PISP_IMAGE_FORMAT_PLANARITY_MASK) == PISP_IMAGE_FORMAT_PLANARITY_INTERLEAVED)
+#define PISP_IMAGE_FORMAT_SEMIPLANAR(fmt) (((fmt) & PISP_IMAGE_FORMAT_PLANARITY_MASK) == PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR)
+#define PISP_IMAGE_FORMAT_PLANAR(fmt) (((fmt) & PISP_IMAGE_FORMAT_PLANARITY_MASK) == PISP_IMAGE_FORMAT_PLANARITY_PLANAR)
+#define PISP_IMAGE_FORMAT_WALLPAPER(fmt) ((fmt) & PISP_IMAGE_FORMAT_WALLPAPER_ROLL)
+#define PISP_IMAGE_FORMAT_BPP_32(fmt) ((fmt) & PISP_IMAGE_FORMAT_BPP_32)
+#define PISP_IMAGE_FORMAT_HOG(fmt) ((fmt) & (PISP_IMAGE_FORMAT_HOG_SIGNED | PISP_IMAGE_FORMAT_HOG_UNSIGNED))
+#define PISP_WALLPAPER_WIDTH 128
+struct pisp_bla_config {
+ __u16 black_level_r;
+ __u16 black_level_gr;
+ __u16 black_level_gb;
+ __u16 black_level_b;
+ __u16 output_black_level;
+ __u8 pad[2];
+} __attribute__((packed));
+struct pisp_wbg_config {
+ __u16 gain_r;
+ __u16 gain_g;
+ __u16 gain_b;
+ __u8 pad[2];
+} __attribute__((packed));
+struct pisp_compress_config {
+ __u16 offset;
+ __u8 pad;
+ __u8 mode;
+} __attribute__((packed));
+struct pisp_decompress_config {
+ __u16 offset;
+ __u8 pad;
+ __u8 mode;
+} __attribute__((packed));
+enum pisp_axi_flags {
+ PISP_AXI_FLAG_ALIGN = 128,
+ PISP_AXI_FLAG_PAD = 64,
+ PISP_AXI_FLAG_PANIC = 32,
+};
+struct pisp_axi_config {
+ __u8 maxlen_flags;
+ __u8 cache_prot;
+ __u16 qos;
+} __attribute__((packed));
+#endif
diff --git a/libc/kernel/uapi/linux/mman.h b/libc/kernel/uapi/linux/mman.h
index cf1e978..f50b51c 100644
--- a/libc/kernel/uapi/linux/mman.h
+++ b/libc/kernel/uapi/linux/mman.h
@@ -18,6 +18,7 @@
#define MAP_SHARED 0x01
#define MAP_PRIVATE 0x02
#define MAP_SHARED_VALIDATE 0x03
+#define MAP_DROPPABLE 0x08
#define MAP_HUGE_SHIFT HUGETLB_FLAG_ENCODE_SHIFT
#define MAP_HUGE_MASK HUGETLB_FLAG_ENCODE_MASK
#define MAP_HUGE_16KB HUGETLB_FLAG_ENCODE_16KB
diff --git a/libc/kernel/uapi/linux/mount.h b/libc/kernel/uapi/linux/mount.h
index 1166a7a..c4278b5 100644
--- a/libc/kernel/uapi/linux/mount.h
+++ b/libc/kernel/uapi/linux/mount.h
@@ -90,7 +90,7 @@
#define MOUNT_ATTR_SIZE_VER0 32
struct statmount {
__u32 size;
- __u32 __spare1;
+ __u32 mnt_opts;
__u64 mask;
__u32 sb_dev_major;
__u32 sb_dev_minor;
@@ -108,7 +108,8 @@
__u64 propagate_from;
__u32 mnt_root;
__u32 mnt_point;
- __u64 __spare2[50];
+ __u64 mnt_ns_id;
+ __u64 __spare2[49];
char str[];
};
struct mnt_id_req {
@@ -116,13 +117,18 @@
__u32 spare;
__u64 mnt_id;
__u64 param;
+ __u64 mnt_ns_id;
};
#define MNT_ID_REQ_SIZE_VER0 24
+#define MNT_ID_REQ_SIZE_VER1 32
#define STATMOUNT_SB_BASIC 0x00000001U
#define STATMOUNT_MNT_BASIC 0x00000002U
#define STATMOUNT_PROPAGATE_FROM 0x00000004U
#define STATMOUNT_MNT_ROOT 0x00000008U
#define STATMOUNT_MNT_POINT 0x00000010U
#define STATMOUNT_FS_TYPE 0x00000020U
+#define STATMOUNT_MNT_NS_ID 0x00000040U
+#define STATMOUNT_MNT_OPTS 0x00000080U
#define LSMT_ROOT 0xffffffffffffffff
+#define LISTMOUNT_REVERSE (1 << 0)
#endif
diff --git a/libc/kernel/uapi/linux/netfilter/nf_tables.h b/libc/kernel/uapi/linux/netfilter/nf_tables.h
index 7922147..bfc6e25 100644
--- a/libc/kernel/uapi/linux/netfilter/nf_tables.h
+++ b/libc/kernel/uapi/linux/netfilter/nf_tables.h
@@ -695,7 +695,7 @@
__NFTA_SECMARK_MAX,
};
#define NFTA_SECMARK_MAX (__NFTA_SECMARK_MAX - 1)
-#define NFT_SECMARK_CTX_MAXLEN 256
+#define NFT_SECMARK_CTX_MAXLEN 4096
enum nft_reject_types {
NFT_REJECT_ICMP_UNREACH,
NFT_REJECT_TCP_RST,
diff --git a/libc/kernel/uapi/linux/nfs4.h b/libc/kernel/uapi/linux/nfs4.h
index 21f1103..6512901 100644
--- a/libc/kernel/uapi/linux/nfs4.h
+++ b/libc/kernel/uapi/linux/nfs4.h
@@ -34,6 +34,7 @@
#define NFS4_OPEN_RESULT_CONFIRM 0x0002
#define NFS4_OPEN_RESULT_LOCKTYPE_POSIX 0x0004
#define NFS4_OPEN_RESULT_PRESERVE_UNLINKED 0x0008
+#define NFS4_OPEN_RESULT_NO_OPEN_STATEID 0x0010
#define NFS4_OPEN_RESULT_MAY_NOTIFY_LOCK 0x0020
#define NFS4_SHARE_ACCESS_MASK 0x000F
#define NFS4_SHARE_ACCESS_READ 0x0001
@@ -52,6 +53,8 @@
#define NFS4_SHARE_WHEN_MASK 0xF0000
#define NFS4_SHARE_SIGNAL_DELEG_WHEN_RESRC_AVAIL 0x10000
#define NFS4_SHARE_PUSH_DELEG_WHEN_UNCONTENDED 0x20000
+#define NFS4_SHARE_WANT_DELEG_TIMESTAMPS 0x100000
+#define NFS4_SHARE_WANT_OPEN_XOR_DELEGATION 0x200000
#define NFS4_CDFC4_FORE 0x1
#define NFS4_CDFC4_BACK 0x2
#define NFS4_CDFC4_BOTH 0x3
diff --git a/libc/kernel/uapi/linux/nfsd_netlink.h b/libc/kernel/uapi/linux/nfsd_netlink.h
index 45cb504..bd3f02c 100644
--- a/libc/kernel/uapi/linux/nfsd_netlink.h
+++ b/libc/kernel/uapi/linux/nfsd_netlink.h
@@ -58,6 +58,12 @@
NFSD_A_SERVER_SOCK_MAX = (__NFSD_A_SERVER_SOCK_MAX - 1)
};
enum {
+ NFSD_A_POOL_MODE_MODE = 1,
+ NFSD_A_POOL_MODE_NPOOLS,
+ __NFSD_A_POOL_MODE_MAX,
+ NFSD_A_POOL_MODE_MAX = (__NFSD_A_POOL_MODE_MAX - 1)
+};
+enum {
NFSD_CMD_RPC_STATUS_GET = 1,
NFSD_CMD_THREADS_SET,
NFSD_CMD_THREADS_GET,
@@ -65,6 +71,8 @@
NFSD_CMD_VERSION_GET,
NFSD_CMD_LISTENER_SET,
NFSD_CMD_LISTENER_GET,
+ NFSD_CMD_POOL_MODE_SET,
+ NFSD_CMD_POOL_MODE_GET,
__NFSD_CMD_MAX,
NFSD_CMD_MAX = (__NFSD_CMD_MAX - 1)
};
diff --git a/libc/kernel/uapi/linux/nl80211.h b/libc/kernel/uapi/linux/nl80211.h
index 98180c2..1bad2f2 100644
--- a/libc/kernel/uapi/linux/nl80211.h
+++ b/libc/kernel/uapi/linux/nl80211.h
@@ -528,6 +528,8 @@
NL80211_ATTR_MLO_TTLM_DLINK,
NL80211_ATTR_MLO_TTLM_ULINK,
NL80211_ATTR_ASSOC_SPP_AMSDU,
+ NL80211_ATTR_WIPHY_RADIOS,
+ NL80211_ATTR_WIPHY_INTERFACE_COMBINATIONS,
__NL80211_ATTR_AFTER_LAST,
NUM_NL80211_ATTR = __NL80211_ATTR_AFTER_LAST,
NL80211_ATTR_MAX = __NL80211_ATTR_AFTER_LAST - 1
@@ -878,6 +880,7 @@
NL80211_FREQUENCY_ATTR_NO_6GHZ_VLP_CLIENT,
NL80211_FREQUENCY_ATTR_NO_6GHZ_AFC_CLIENT,
NL80211_FREQUENCY_ATTR_CAN_MONITOR,
+ NL80211_FREQUENCY_ATTR_ALLOW_6GHZ_VLP_AP,
__NL80211_FREQUENCY_ATTR_AFTER_LAST,
NL80211_FREQUENCY_ATTR_MAX = __NL80211_FREQUENCY_ATTR_AFTER_LAST - 1
};
@@ -955,6 +958,7 @@
NL80211_RRF_DFS_CONCURRENT = 1 << 21,
NL80211_RRF_NO_6GHZ_VLP_CLIENT = 1 << 22,
NL80211_RRF_NO_6GHZ_AFC_CLIENT = 1 << 23,
+ NL80211_RRF_ALLOW_6GHZ_VLP_AP = 1 << 24,
};
#define NL80211_RRF_PASSIVE_SCAN NL80211_RRF_NO_IR
#define NL80211_RRF_NO_IBSS NL80211_RRF_NO_IR
@@ -1973,4 +1977,19 @@
NL80211_AP_SETTINGS_EXTERNAL_AUTH_SUPPORT = 1 << 0,
NL80211_AP_SETTINGS_SA_QUERY_OFFLOAD_SUPPORT = 1 << 1,
};
+enum nl80211_wiphy_radio_attrs {
+ __NL80211_WIPHY_RADIO_ATTR_INVALID,
+ NL80211_WIPHY_RADIO_ATTR_INDEX,
+ NL80211_WIPHY_RADIO_ATTR_FREQ_RANGE,
+ NL80211_WIPHY_RADIO_ATTR_INTERFACE_COMBINATION,
+ __NL80211_WIPHY_RADIO_ATTR_LAST,
+ NL80211_WIPHY_RADIO_ATTR_MAX = __NL80211_WIPHY_RADIO_ATTR_LAST - 1,
+};
+enum nl80211_wiphy_radio_freq_range {
+ __NL80211_WIPHY_RADIO_FREQ_ATTR_INVALID,
+ NL80211_WIPHY_RADIO_FREQ_ATTR_START,
+ NL80211_WIPHY_RADIO_FREQ_ATTR_END,
+ __NL80211_WIPHY_RADIO_FREQ_ATTR_LAST,
+ NL80211_WIPHY_RADIO_FREQ_ATTR_MAX = __NL80211_WIPHY_RADIO_FREQ_ATTR_LAST - 1,
+};
#endif
diff --git a/libc/kernel/uapi/linux/nsfs.h b/libc/kernel/uapi/linux/nsfs.h
index 61a5797..c8f2208 100644
--- a/libc/kernel/uapi/linux/nsfs.h
+++ b/libc/kernel/uapi/linux/nsfs.h
@@ -7,9 +7,15 @@
#ifndef __LINUX_NSFS_H
#define __LINUX_NSFS_H
#include <linux/ioctl.h>
+#include <linux/types.h>
#define NSIO 0xb7
#define NS_GET_USERNS _IO(NSIO, 0x1)
#define NS_GET_PARENT _IO(NSIO, 0x2)
#define NS_GET_NSTYPE _IO(NSIO, 0x3)
#define NS_GET_OWNER_UID _IO(NSIO, 0x4)
+#define NS_GET_MNTNS_ID _IOR(NSIO, 0x5, __u64)
+#define NS_GET_PID_FROM_PIDNS _IOR(NSIO, 0x6, int)
+#define NS_GET_TGID_FROM_PIDNS _IOR(NSIO, 0x7, int)
+#define NS_GET_PID_IN_PIDNS _IOR(NSIO, 0x8, int)
+#define NS_GET_TGID_IN_PIDNS _IOR(NSIO, 0x9, int)
#endif
diff --git a/libc/kernel/uapi/linux/openvswitch.h b/libc/kernel/uapi/linux/openvswitch.h
index d45f4fa..98c8037 100644
--- a/libc/kernel/uapi/linux/openvswitch.h
+++ b/libc/kernel/uapi/linux/openvswitch.h
@@ -429,6 +429,13 @@
__OVS_CHECK_PKT_LEN_ATTR_MAX,
};
#define OVS_CHECK_PKT_LEN_ATTR_MAX (__OVS_CHECK_PKT_LEN_ATTR_MAX - 1)
+#define OVS_PSAMPLE_COOKIE_MAX_SIZE 16
+enum ovs_psample_attr {
+ OVS_PSAMPLE_ATTR_GROUP = 1,
+ OVS_PSAMPLE_ATTR_COOKIE,
+ __OVS_PSAMPLE_ATTR_MAX
+};
+#define OVS_PSAMPLE_ATTR_MAX (__OVS_PSAMPLE_ATTR_MAX - 1)
enum ovs_action_attr {
OVS_ACTION_ATTR_UNSPEC,
OVS_ACTION_ATTR_OUTPUT,
@@ -455,6 +462,7 @@
OVS_ACTION_ATTR_ADD_MPLS,
OVS_ACTION_ATTR_DEC_TTL,
OVS_ACTION_ATTR_DROP,
+ OVS_ACTION_ATTR_PSAMPLE,
__OVS_ACTION_ATTR_MAX,
};
#define OVS_ACTION_ATTR_MAX (__OVS_ACTION_ATTR_MAX - 1)
diff --git a/libc/kernel/uapi/linux/perf_event.h b/libc/kernel/uapi/linux/perf_event.h
index 16a1a2e..ec9b856 100644
--- a/libc/kernel/uapi/linux/perf_event.h
+++ b/libc/kernel/uapi/linux/perf_event.h
@@ -460,6 +460,8 @@
#define PERF_MEM_LVLNUM_L2 0x02
#define PERF_MEM_LVLNUM_L3 0x03
#define PERF_MEM_LVLNUM_L4 0x04
+#define PERF_MEM_LVLNUM_L2_MHB 0x05
+#define PERF_MEM_LVLNUM_MSC 0x06
#define PERF_MEM_LVLNUM_UNC 0x08
#define PERF_MEM_LVLNUM_CXL 0x09
#define PERF_MEM_LVLNUM_IO 0x0a
diff --git a/libc/kernel/uapi/linux/pidfd.h b/libc/kernel/uapi/linux/pidfd.h
index 082b4a0..9068727 100644
--- a/libc/kernel/uapi/linux/pidfd.h
+++ b/libc/kernel/uapi/linux/pidfd.h
@@ -8,9 +8,21 @@
#define _UAPI_LINUX_PIDFD_H
#include <linux/types.h>
#include <linux/fcntl.h>
+#include <linux/ioctl.h>
#define PIDFD_NONBLOCK O_NONBLOCK
#define PIDFD_THREAD O_EXCL
#define PIDFD_SIGNAL_THREAD (1UL << 0)
#define PIDFD_SIGNAL_THREAD_GROUP (1UL << 1)
#define PIDFD_SIGNAL_PROCESS_GROUP (1UL << 2)
+#define PIDFS_IOCTL_MAGIC 0xFF
+#define PIDFD_GET_CGROUP_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 1)
+#define PIDFD_GET_IPC_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 2)
+#define PIDFD_GET_MNT_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 3)
+#define PIDFD_GET_NET_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 4)
+#define PIDFD_GET_PID_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 5)
+#define PIDFD_GET_PID_FOR_CHILDREN_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 6)
+#define PIDFD_GET_TIME_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 7)
+#define PIDFD_GET_TIME_FOR_CHILDREN_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 8)
+#define PIDFD_GET_USER_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 9)
+#define PIDFD_GET_UTS_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 10)
#endif
diff --git a/libc/kernel/uapi/linux/pkt_cls.h b/libc/kernel/uapi/linux/pkt_cls.h
index 6b5143c..bdca553 100644
--- a/libc/kernel/uapi/linux/pkt_cls.h
+++ b/libc/kernel/uapi/linux/pkt_cls.h
@@ -419,6 +419,8 @@
TCA_FLOWER_KEY_CFM,
TCA_FLOWER_KEY_SPI,
TCA_FLOWER_KEY_SPI_MASK,
+ TCA_FLOWER_KEY_ENC_FLAGS,
+ TCA_FLOWER_KEY_ENC_FLAGS_MASK,
__TCA_FLOWER_MAX,
};
#define TCA_FLOWER_MAX (__TCA_FLOWER_MAX - 1)
@@ -497,7 +499,13 @@
enum {
TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT = (1 << 0),
TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST = (1 << 1),
+ TCA_FLOWER_KEY_FLAGS_TUNNEL_CSUM = (1 << 2),
+ TCA_FLOWER_KEY_FLAGS_TUNNEL_DONT_FRAGMENT = (1 << 3),
+ TCA_FLOWER_KEY_FLAGS_TUNNEL_OAM = (1 << 4),
+ TCA_FLOWER_KEY_FLAGS_TUNNEL_CRIT_OPT = (1 << 5),
+ __TCA_FLOWER_KEY_FLAGS_MAX,
};
+#define TCA_FLOWER_KEY_FLAGS_MAX (__TCA_FLOWER_KEY_FLAGS_MAX - 1)
enum {
TCA_FLOWER_KEY_CFM_OPT_UNSPEC,
TCA_FLOWER_KEY_CFM_MD_LEVEL,
diff --git a/libc/kernel/uapi/linux/psample.h b/libc/kernel/uapi/linux/psample.h
index c82e76e..f9f979c 100644
--- a/libc/kernel/uapi/linux/psample.h
+++ b/libc/kernel/uapi/linux/psample.h
@@ -22,6 +22,8 @@
PSAMPLE_ATTR_LATENCY,
PSAMPLE_ATTR_TIMESTAMP,
PSAMPLE_ATTR_PROTO,
+ PSAMPLE_ATTR_USER_COOKIE,
+ PSAMPLE_ATTR_SAMPLE_PROBABILITY,
__PSAMPLE_ATTR_MAX
};
enum psample_command {
diff --git a/libc/kernel/uapi/linux/psp-sev.h b/libc/kernel/uapi/linux/psp-sev.h
index 82fcbf1..7274081 100644
--- a/libc/kernel/uapi/linux/psp-sev.h
+++ b/libc/kernel/uapi/linux/psp-sev.h
@@ -20,6 +20,7 @@
SNP_PLATFORM_STATUS,
SNP_COMMIT,
SNP_SET_CONFIG,
+ SNP_VLEK_LOAD,
SEV_MAX,
};
typedef enum {
@@ -28,6 +29,7 @@
SEV_RET_INVALID_PLATFORM_STATE,
SEV_RET_INVALID_GUEST_STATE,
SEV_RET_INAVLID_CONFIG,
+ SEV_RET_INVALID_CONFIG = SEV_RET_INAVLID_CONFIG,
SEV_RET_INVALID_LEN,
SEV_RET_ALREADY_OWNED,
SEV_RET_INVALID_CERTIFICATE,
@@ -113,6 +115,15 @@
__u32 rsvd : 30;
__u8 rsvd1[52];
} __attribute__((__packed__));
+struct sev_user_data_snp_vlek_load {
+ __u32 len;
+ __u8 vlek_wrapped_version;
+ __u8 rsvd[3];
+ __u64 vlek_wrapped_address;
+} __attribute__((__packed__));
+struct sev_user_data_snp_wrapped_vlek_hashstick {
+ __u8 data[432];
+} __attribute__((__packed__));
struct sev_issue_cmd {
__u32 cmd;
__u64 data;
diff --git a/libc/kernel/uapi/linux/random.h b/libc/kernel/uapi/linux/random.h
index d1fd998..64f62d9 100644
--- a/libc/kernel/uapi/linux/random.h
+++ b/libc/kernel/uapi/linux/random.h
@@ -24,4 +24,10 @@
#define GRND_NONBLOCK 0x0001
#define GRND_RANDOM 0x0002
#define GRND_INSECURE 0x0004
+struct vgetrandom_opaque_params {
+ __u32 size_of_opaque_state;
+ __u32 mmap_prot;
+ __u32 mmap_flags;
+ __u32 reserved[13];
+};
#endif
diff --git a/libc/kernel/uapi/linux/sev-guest.h b/libc/kernel/uapi/linux/sev-guest.h
index a822bed..a722641 100644
--- a/libc/kernel/uapi/linux/sev-guest.h
+++ b/libc/kernel/uapi/linux/sev-guest.h
@@ -51,6 +51,8 @@
#define SNP_GUEST_FW_ERR_MASK GENMASK_ULL(31, 0)
#define SNP_GUEST_VMM_ERR_SHIFT 32
#define SNP_GUEST_VMM_ERR(x) (((u64) x) << SNP_GUEST_VMM_ERR_SHIFT)
+#define SNP_GUEST_FW_ERR(x) ((x) & SNP_GUEST_FW_ERR_MASK)
+#define SNP_GUEST_ERR(vmm_err,fw_err) (SNP_GUEST_VMM_ERR(vmm_err) | SNP_GUEST_FW_ERR(fw_err))
#define SNP_GUEST_VMM_ERR_INVALID_LEN 1
#define SNP_GUEST_VMM_ERR_BUSY 2
#endif
diff --git a/libc/kernel/uapi/linux/stat.h b/libc/kernel/uapi/linux/stat.h
index ff98fd2..aae9ed4 100644
--- a/libc/kernel/uapi/linux/stat.h
+++ b/libc/kernel/uapi/linux/stat.h
@@ -69,7 +69,11 @@
__u32 stx_dio_mem_align;
__u32 stx_dio_offset_align;
__u64 stx_subvol;
- __u64 __spare3[11];
+ __u32 stx_atomic_write_unit_min;
+ __u32 stx_atomic_write_unit_max;
+ __u32 stx_atomic_write_segments_max;
+ __u32 __spare1[1];
+ __u64 __spare3[9];
};
#define STATX_TYPE 0x00000001U
#define STATX_MODE 0x00000002U
@@ -88,6 +92,7 @@
#define STATX_DIOALIGN 0x00002000U
#define STATX_MNT_ID_UNIQUE 0x00004000U
#define STATX_SUBVOL 0x00008000U
+#define STATX_WRITE_ATOMIC 0x00010000U
#define STATX__RESERVED 0x80000000U
#define STATX_ALL 0x00000fffU
#define STATX_ATTR_COMPRESSED 0x00000004
@@ -99,4 +104,5 @@
#define STATX_ATTR_MOUNT_ROOT 0x00002000
#define STATX_ATTR_VERITY 0x00100000
#define STATX_ATTR_DAX 0x00200000
+#define STATX_ATTR_WRITE_ATOMIC 0x00400000
#endif
diff --git a/libc/kernel/uapi/linux/tcp_metrics.h b/libc/kernel/uapi/linux/tcp_metrics.h
index 931f50c..46ca141 100644
--- a/libc/kernel/uapi/linux/tcp_metrics.h
+++ b/libc/kernel/uapi/linux/tcp_metrics.h
@@ -4,8 +4,8 @@
* See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
* for more information.
*/
-#ifndef _LINUX_TCP_METRICS_H
-#define _LINUX_TCP_METRICS_H
+#ifndef _UAPI_LINUX_TCP_METRICS_H
+#define _UAPI_LINUX_TCP_METRICS_H
#include <linux/types.h>
#define TCP_METRICS_GENL_NAME "tcp_metrics"
#define TCP_METRICS_GENL_VERSION 0x1
@@ -21,6 +21,17 @@
};
#define TCP_METRIC_MAX (__TCP_METRIC_MAX - 1)
enum {
+ TCP_METRICS_A_METRICS_RTT = 1,
+ TCP_METRICS_A_METRICS_RTTVAR,
+ TCP_METRICS_A_METRICS_SSTHRESH,
+ TCP_METRICS_A_METRICS_CWND,
+ TCP_METRICS_A_METRICS_REODERING,
+ TCP_METRICS_A_METRICS_RTT_US,
+ TCP_METRICS_A_METRICS_RTTVAR_US,
+ __TCP_METRICS_A_METRICS_MAX
+};
+#define TCP_METRICS_A_METRICS_MAX (__TCP_METRICS_A_METRICS_MAX - 1)
+enum {
TCP_METRICS_ATTR_UNSPEC,
TCP_METRICS_ATTR_ADDR_IPV4,
TCP_METRICS_ATTR_ADDR_IPV6,
diff --git a/libc/kernel/uapi/linux/um_timetravel.h b/libc/kernel/uapi/linux/um_timetravel.h
index 8706017..49eeb28 100644
--- a/libc/kernel/uapi/linux/um_timetravel.h
+++ b/libc/kernel/uapi/linux/um_timetravel.h
@@ -12,6 +12,15 @@
__u32 seq;
__u64 time;
};
+#define UM_TIMETRAVEL_MAX_FDS 2
+enum um_timetravel_shared_mem_fds {
+ UM_TIMETRAVEL_SHARED_MEMFD,
+ UM_TIMETRAVEL_SHARED_LOGFD,
+ UM_TIMETRAVEL_SHARED_MAX_FDS,
+};
+enum um_timetravel_start_ack {
+ UM_TIMETRAVEL_START_ACK_ID = 0xffff,
+};
enum um_timetravel_ops {
UM_TIMETRAVEL_ACK = 0,
UM_TIMETRAVEL_START = 1,
@@ -22,5 +31,36 @@
UM_TIMETRAVEL_RUN = 6,
UM_TIMETRAVEL_FREE_UNTIL = 7,
UM_TIMETRAVEL_GET_TOD = 8,
+ UM_TIMETRAVEL_BROADCAST = 9,
+};
+#define UM_TIMETRAVEL_SCHEDSHM_VERSION 2
+enum um_timetravel_schedshm_cap {
+ UM_TIMETRAVEL_SCHEDSHM_CAP_TIME_SHARE = 0x1,
+};
+enum um_timetravel_schedshm_flags {
+ UM_TIMETRAVEL_SCHEDSHM_FLAGS_REQ_RUN = 0x1,
+};
+union um_timetravel_schedshm_client {
+ struct {
+ __u32 capa;
+ __u32 flags;
+ __u64 req_time;
+ __u64 name;
+ };
+ char reserve[128];
+};
+struct um_timetravel_schedshm {
+ union {
+ struct {
+ __u32 version;
+ __u32 len;
+ __u64 free_until;
+ __u64 current_time;
+ __u16 running_id;
+ __u16 max_clients;
+ };
+ char hdr[4096];
+ };
+ union um_timetravel_schedshm_client clients[];
};
#endif
diff --git a/libc/kernel/uapi/linux/v4l2-controls.h b/libc/kernel/uapi/linux/v4l2-controls.h
index 23158dc..f6ef26c 100644
--- a/libc/kernel/uapi/linux/v4l2-controls.h
+++ b/libc/kernel/uapi/linux/v4l2-controls.h
@@ -714,6 +714,7 @@
V4L2_MPEG_VIDEO_AV1_LEVEL_7_2 = 22,
V4L2_MPEG_VIDEO_AV1_LEVEL_7_3 = 23
};
+#define V4L2_CID_MPEG_VIDEO_AVERAGE_QP (V4L2_CID_CODEC_BASE + 657)
#define V4L2_CID_CODEC_CX2341X_BASE (V4L2_CTRL_CLASS_CODEC | 0x1000)
#define V4L2_CID_MPEG_CX2341X_VIDEO_SPATIAL_FILTER_MODE (V4L2_CID_CODEC_CX2341X_BASE + 0)
enum v4l2_mpeg_cx2341x_video_spatial_filter_mode {
diff --git a/libc/kernel/uapi/linux/version.h b/libc/kernel/uapi/linux/version.h
index 7faa30f..0cc45cf 100644
--- a/libc/kernel/uapi/linux/version.h
+++ b/libc/kernel/uapi/linux/version.h
@@ -4,8 +4,8 @@
* See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
* for more information.
*/
-#define LINUX_VERSION_CODE 395776
+#define LINUX_VERSION_CODE 396032
#define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
#define LINUX_VERSION_MAJOR 6
-#define LINUX_VERSION_PATCHLEVEL 10
+#define LINUX_VERSION_PATCHLEVEL 11
#define LINUX_VERSION_SUBLEVEL 0
diff --git a/libc/kernel/uapi/linux/videodev2.h b/libc/kernel/uapi/linux/videodev2.h
index 3905949..fb69a8a 100644
--- a/libc/kernel/uapi/linux/videodev2.h
+++ b/libc/kernel/uapi/linux/videodev2.h
@@ -236,6 +236,8 @@
#define V4L2_PIX_FMT_RGBA1010102 v4l2_fourcc('R', 'A', '3', '0')
#define V4L2_PIX_FMT_ARGB2101010 v4l2_fourcc('A', 'R', '3', '0')
#define V4L2_PIX_FMT_BGR48_12 v4l2_fourcc('B', '3', '1', '2')
+#define V4L2_PIX_FMT_BGR48 v4l2_fourcc('B', 'G', 'R', '6')
+#define V4L2_PIX_FMT_RGB48 v4l2_fourcc('R', 'G', 'B', '6')
#define V4L2_PIX_FMT_ABGR64_12 v4l2_fourcc('B', '4', '1', '2')
#define V4L2_PIX_FMT_GREY v4l2_fourcc('G', 'R', 'E', 'Y')
#define V4L2_PIX_FMT_Y4 v4l2_fourcc('Y', '0', '4', ' ')
@@ -425,6 +427,16 @@
#define V4L2_PIX_FMT_IPU3_SGBRG10 v4l2_fourcc('i', 'p', '3', 'g')
#define V4L2_PIX_FMT_IPU3_SGRBG10 v4l2_fourcc('i', 'p', '3', 'G')
#define V4L2_PIX_FMT_IPU3_SRGGB10 v4l2_fourcc('i', 'p', '3', 'r')
+#define V4L2_PIX_FMT_PISP_COMP1_RGGB v4l2_fourcc('P', 'C', '1', 'R')
+#define V4L2_PIX_FMT_PISP_COMP1_GRBG v4l2_fourcc('P', 'C', '1', 'G')
+#define V4L2_PIX_FMT_PISP_COMP1_GBRG v4l2_fourcc('P', 'C', '1', 'g')
+#define V4L2_PIX_FMT_PISP_COMP1_BGGR v4l2_fourcc('P', 'C', '1', 'B')
+#define V4L2_PIX_FMT_PISP_COMP1_MONO v4l2_fourcc('P', 'C', '1', 'M')
+#define V4L2_PIX_FMT_PISP_COMP2_RGGB v4l2_fourcc('P', 'C', '2', 'R')
+#define V4L2_PIX_FMT_PISP_COMP2_GRBG v4l2_fourcc('P', 'C', '2', 'G')
+#define V4L2_PIX_FMT_PISP_COMP2_GBRG v4l2_fourcc('P', 'C', '2', 'g')
+#define V4L2_PIX_FMT_PISP_COMP2_BGGR v4l2_fourcc('P', 'C', '2', 'B')
+#define V4L2_PIX_FMT_PISP_COMP2_MONO v4l2_fourcc('P', 'C', '2', 'M')
#define V4L2_SDR_FMT_CU8 v4l2_fourcc('C', 'U', '0', '8')
#define V4L2_SDR_FMT_CU16LE v4l2_fourcc('C', 'U', '1', '6')
#define V4L2_SDR_FMT_CS8 v4l2_fourcc('C', 'S', '0', '8')
@@ -444,6 +456,7 @@
#define V4L2_META_FMT_VIVID v4l2_fourcc('V', 'I', 'V', 'D')
#define V4L2_META_FMT_RK_ISP1_PARAMS v4l2_fourcc('R', 'K', '1', 'P')
#define V4L2_META_FMT_RK_ISP1_STAT_3A v4l2_fourcc('R', 'K', '1', 'S')
+#define V4L2_META_FMT_RPI_BE_CFG v4l2_fourcc('R', 'P', 'B', 'C')
#define V4L2_PIX_FMT_PRIV_MAGIC 0xfeedcafe
#define V4L2_PIX_FMT_FLAG_PREMUL_ALPHA 0x00000001
#define V4L2_PIX_FMT_FLAG_SET_CSC 0x00000002
diff --git a/libc/kernel/uapi/linux/xfrm.h b/libc/kernel/uapi/linux/xfrm.h
index b8e79e8..9509efe 100644
--- a/libc/kernel/uapi/linux/xfrm.h
+++ b/libc/kernel/uapi/linux/xfrm.h
@@ -260,6 +260,7 @@
XFRMA_IF_ID,
XFRMA_MTIMER_THRESH,
XFRMA_SA_DIR,
+ XFRMA_NAT_KEEPALIVE_INTERVAL,
__XFRMA_MAX
#define XFRMA_OUTPUT_MARK XFRMA_SET_MARK
#define XFRMA_MAX (__XFRMA_MAX - 1)
diff --git a/libc/kernel/uapi/linux/zorro_ids.h b/libc/kernel/uapi/linux/zorro_ids.h
index f47c899..05239cd 100644
--- a/libc/kernel/uapi/linux/zorro_ids.h
+++ b/libc/kernel/uapi/linux/zorro_ids.h
@@ -354,6 +354,8 @@
#define ZORRO_MANUF_VMC 0x1389
#define ZORRO_PROD_VMC_ISDN_BLASTER_Z2 ZORRO_ID(VMC, 0x01, 0)
#define ZORRO_PROD_VMC_HYPERCOM_4 ZORRO_ID(VMC, 0x02, 0)
+#define ZORRO_MANUF_CSLAB 0x1400
+#define ZORRO_PROD_CSLAB_WARP_1260 ZORRO_ID(CSLAB, 0x65, 0)
#define ZORRO_MANUF_INFORMATION 0x157C
#define ZORRO_PROD_INFORMATION_ISDN_ENGINE_I ZORRO_ID(INFORMATION, 0x64, 0)
#define ZORRO_MANUF_VORTEX 0x2017
diff --git a/libc/kernel/uapi/misc/mrvl_cn10k_dpi.h b/libc/kernel/uapi/misc/mrvl_cn10k_dpi.h
new file mode 100644
index 0000000..7d8671f
--- /dev/null
+++ b/libc/kernel/uapi/misc/mrvl_cn10k_dpi.h
@@ -0,0 +1,26 @@
+/*
+ * This file is auto-generated. Modifications will be lost.
+ *
+ * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
+ * for more information.
+ */
+#ifndef __MRVL_CN10K_DPI_H__
+#define __MRVL_CN10K_DPI_H__
+#include <linux/types.h>
+#define DPI_MAX_ENGINES 6
+struct dpi_mps_mrrs_cfg {
+ __u16 max_read_req_sz;
+ __u16 max_payload_sz;
+ __u16 port;
+ __u16 reserved;
+};
+struct dpi_engine_cfg {
+ __u64 fifo_mask;
+ __u16 molr[DPI_MAX_ENGINES];
+ __u16 update_molr;
+ __u16 reserved;
+};
+#define DPI_MAGIC_NUM 0xB8
+#define DPI_MPS_MRRS_CFG _IOW(DPI_MAGIC_NUM, 1, struct dpi_mps_mrrs_cfg)
+#define DPI_ENGINE_CFG _IOW(DPI_MAGIC_NUM, 2, struct dpi_engine_cfg)
+#endif
diff --git a/libc/kernel/uapi/rdma/bnxt_re-abi.h b/libc/kernel/uapi/rdma/bnxt_re-abi.h
index 3dceafd..50f8b8a 100644
--- a/libc/kernel/uapi/rdma/bnxt_re-abi.h
+++ b/libc/kernel/uapi/rdma/bnxt_re-abi.h
@@ -18,7 +18,7 @@
BNXT_RE_UCNTX_CMASK_WC_DPI_ENABLED = 0x04ULL,
BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED = 0x08ULL,
BNXT_RE_UCNTX_CMASK_POW2_DISABLED = 0x10ULL,
- BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40,
+ BNXT_RE_UCNTX_CMASK_MSN_TABLE_ENABLED = 0x40,
};
enum bnxt_re_wqe_mode {
BNXT_QPLIB_WQE_MODE_STATIC = 0x00,
diff --git a/libc/kernel/uapi/rdma/ib_user_ioctl_cmds.h b/libc/kernel/uapi/rdma/ib_user_ioctl_cmds.h
index d774fba..5f5c426 100644
--- a/libc/kernel/uapi/rdma/ib_user_ioctl_cmds.h
+++ b/libc/kernel/uapi/rdma/ib_user_ioctl_cmds.h
@@ -8,8 +8,6 @@
#define IB_USER_IOCTL_CMDS_H
#define UVERBS_ID_NS_MASK 0xF000
#define UVERBS_ID_NS_SHIFT 12
-#define UVERBS_UDATA_DRIVER_DATA_NS 1
-#define UVERBS_UDATA_DRIVER_DATA_FLAG (1UL << UVERBS_ID_NS_SHIFT)
enum uverbs_default_objects {
UVERBS_OBJECT_DEVICE,
UVERBS_OBJECT_PD,
@@ -30,8 +28,10 @@
UVERBS_OBJECT_ASYNC_EVENT,
};
enum {
- UVERBS_ATTR_UHW_IN = UVERBS_UDATA_DRIVER_DATA_FLAG,
+ UVERBS_ID_DRIVER_NS = 1UL << UVERBS_ID_NS_SHIFT,
+ UVERBS_ATTR_UHW_IN = UVERBS_ID_DRIVER_NS,
UVERBS_ATTR_UHW_OUT,
+ UVERBS_ID_DRIVER_NS_WITH_UHW,
};
enum uverbs_methods_device {
UVERBS_METHOD_INVOKE_WRITE,
diff --git a/libc/kernel/uapi/rdma/mana-abi.h b/libc/kernel/uapi/rdma/mana-abi.h
index 7347175..02bb061 100644
--- a/libc/kernel/uapi/rdma/mana-abi.h
+++ b/libc/kernel/uapi/rdma/mana-abi.h
@@ -33,6 +33,13 @@
__u32 tx_vp_offset;
__u32 reserved;
};
+struct mana_ib_create_rc_qp {
+ __aligned_u64 queue_buf[4];
+ __u32 queue_size[4];
+};
+struct mana_ib_create_rc_qp_resp {
+ __u32 queue_id[4];
+};
struct mana_ib_create_wq {
__aligned_u64 wq_buf_addr;
__u32 wq_buf_size;
diff --git a/libc/kernel/uapi/rdma/mlx5_user_ioctl_cmds.h b/libc/kernel/uapi/rdma/mlx5_user_ioctl_cmds.h
index 69d5250..ebafb00 100644
--- a/libc/kernel/uapi/rdma/mlx5_user_ioctl_cmds.h
+++ b/libc/kernel/uapi/rdma/mlx5_user_ioctl_cmds.h
@@ -201,6 +201,9 @@
enum mlx5_ib_device_query_context_attrs {
MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX = (1U << UVERBS_ID_NS_SHIFT),
};
+enum mlx5_ib_create_cq_attrs {
+ MLX5_IB_ATTR_CREATE_CQ_UAR_INDEX = UVERBS_ID_DRIVER_NS_WITH_UHW,
+};
#define MLX5_IB_DW_MATCH_PARAM 0xA0
struct mlx5_ib_match_params {
__u32 match_params[MLX5_IB_DW_MATCH_PARAM];
diff --git a/libc/kernel/uapi/rdma/rdma_netlink.h b/libc/kernel/uapi/rdma/rdma_netlink.h
index 912a3c0..ac027ac 100644
--- a/libc/kernel/uapi/rdma/rdma_netlink.h
+++ b/libc/kernel/uapi/rdma/rdma_netlink.h
@@ -199,6 +199,8 @@
RDMA_NLDEV_CMD_RES_SRQ_GET,
RDMA_NLDEV_CMD_STAT_GET_STATUS,
RDMA_NLDEV_CMD_RES_SRQ_GET_RAW,
+ RDMA_NLDEV_CMD_NEWDEV,
+ RDMA_NLDEV_CMD_DELDEV,
RDMA_NLDEV_NUM_OPS
};
enum rdma_nldev_print_type {
@@ -306,6 +308,9 @@
RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE,
RDMA_NLDEV_ATTR_DRIVER_DETAILS,
RDMA_NLDEV_ATTR_RES_SUBTYPE,
+ RDMA_NLDEV_ATTR_DEV_TYPE,
+ RDMA_NLDEV_ATTR_PARENT_NAME,
+ RDMA_NLDEV_ATTR_NAME_ASSIGN_TYPE,
RDMA_NLDEV_ATTR_MAX
};
enum rdma_nl_counter_mode {
@@ -318,4 +323,11 @@
RDMA_COUNTER_MASK_QP_TYPE = 1,
RDMA_COUNTER_MASK_PID = 1 << 1,
};
+enum rdma_nl_dev_type {
+ RDMA_DEVICE_TYPE_SMI = 1,
+};
+enum rdma_nl_name_assign_type {
+ RDMA_NAME_ASSIGN_TYPE_UNKNOWN = 0,
+ RDMA_NAME_ASSIGN_TYPE_USER = 1,
+};
#endif
diff --git a/libc/kernel/uapi/scsi/scsi_bsg_mpi3mr.h b/libc/kernel/uapi/scsi/scsi_bsg_mpi3mr.h
index d98e8fa..fe3094e 100644
--- a/libc/kernel/uapi/scsi/scsi_bsg_mpi3mr.h
+++ b/libc/kernel/uapi/scsi/scsi_bsg_mpi3mr.h
@@ -144,7 +144,7 @@
};
struct mpi3mr_bsg_in_hdb_status {
__u8 num_hdb_types;
- __u8 rsvd1;
+ __u8 element_trigger_format;
__u16 rsvd2;
__u32 rsvd3;
struct mpi3mr_hdb_entry entry[1];
diff --git a/libc/kernel/uapi/sound/asequencer.h b/libc/kernel/uapi/sound/asequencer.h
index 7918528..a3826a5 100644
--- a/libc/kernel/uapi/sound/asequencer.h
+++ b/libc/kernel/uapi/sound/asequencer.h
@@ -7,7 +7,7 @@
#ifndef _UAPI__SOUND_ASEQUENCER_H
#define _UAPI__SOUND_ASEQUENCER_H
#include <sound/asound.h>
-#define SNDRV_SEQ_VERSION SNDRV_PROTOCOL_VERSION(1, 0, 3)
+#define SNDRV_SEQ_VERSION SNDRV_PROTOCOL_VERSION(1, 0, 4)
#define SNDRV_SEQ_EVENT_SYSTEM 0
#define SNDRV_SEQ_EVENT_RESULT 1
#define SNDRV_SEQ_EVENT_NOTE 5
@@ -343,7 +343,8 @@
int ppq;
unsigned int skew_value;
unsigned int skew_base;
- char reserved[24];
+ unsigned short tempo_base;
+ char reserved[22];
};
#define SNDRV_SEQ_TIMER_ALSA 0
#define SNDRV_SEQ_TIMER_MIDI_CLOCK 1
diff --git a/libc/kernel/uapi/sound/asound.h b/libc/kernel/uapi/sound/asound.h
index b608ed5..cfe9f66 100644
--- a/libc/kernel/uapi/sound/asound.h
+++ b/libc/kernel/uapi/sound/asound.h
@@ -92,7 +92,7 @@
#define SNDRV_HWDEP_IOCTL_INFO _IOR('H', 0x01, struct snd_hwdep_info)
#define SNDRV_HWDEP_IOCTL_DSP_STATUS _IOR('H', 0x02, struct snd_hwdep_dsp_status)
#define SNDRV_HWDEP_IOCTL_DSP_LOAD _IOW('H', 0x03, struct snd_hwdep_dsp_image)
-#define SNDRV_PCM_VERSION SNDRV_PROTOCOL_VERSION(2, 0, 17)
+#define SNDRV_PCM_VERSION SNDRV_PROTOCOL_VERSION(2, 0, 18)
typedef unsigned long snd_pcm_uframes_t;
typedef signed long snd_pcm_sframes_t;
enum {
@@ -263,7 +263,7 @@
unsigned char id[16];
unsigned short id16[8];
unsigned int id32[4];
-};
+} __attribute__((deprecated));
struct snd_pcm_info {
unsigned int device;
unsigned int subdevice;
@@ -276,7 +276,7 @@
int dev_subclass;
unsigned int subdevices_count;
unsigned int subdevices_avail;
- union snd_pcm_sync_id sync;
+ unsigned char pad1[16];
unsigned char reserved[64];
};
typedef int snd_pcm_hw_param_t;
@@ -324,7 +324,8 @@
unsigned int rate_num;
unsigned int rate_den;
snd_pcm_uframes_t fifo_size;
- unsigned char reserved[64];
+ unsigned char sync[16];
+ unsigned char reserved[48];
};
enum {
SNDRV_PCM_TSTAMP_NONE = 0,
diff --git a/libc/kernel/uapi/sound/sof/abi.h b/libc/kernel/uapi/sound/sof/abi.h
index fe1fe47..ed66131 100644
--- a/libc/kernel/uapi/sound/sof/abi.h
+++ b/libc/kernel/uapi/sound/sof/abi.h
@@ -9,7 +9,7 @@
#include <linux/types.h>
#define SOF_ABI_MAJOR 3
#define SOF_ABI_MINOR 23
-#define SOF_ABI_PATCH 0
+#define SOF_ABI_PATCH 1
#define SOF_ABI_MAJOR_SHIFT 24
#define SOF_ABI_MAJOR_MASK 0xff
#define SOF_ABI_MINOR_SHIFT 12
diff --git a/libc/libc.map.txt b/libc/libc.map.txt
index 0d2e42f..d8636a2 100644
--- a/libc/libc.map.txt
+++ b/libc/libc.map.txt
@@ -591,7 +591,6 @@
killpg;
klogctl;
labs;
- lchmod; # introduced=36
lchown;
lcong48; # introduced=23
ldexp;
@@ -671,7 +670,6 @@
mprotect;
mrand48;
mremap;
- mseal; # introduced=36
msync;
munlock;
munlockall;
@@ -846,7 +844,6 @@
pwrite;
pwrite64;
qsort;
- qsort_r; # introduced=36
quick_exit;
raise;
rand;
@@ -955,7 +952,6 @@
setvbuf;
setxattr;
shutdown;
- sig2str; # introduced=36
sigaction;
sigaddset;
sigaltstack;
@@ -998,7 +994,6 @@
stdout; # var introduced=23
stpcpy;
stpncpy;
- str2sig; # introduced=36
strcasecmp;
strcasecmp_l; # introduced=23
strcasestr;
@@ -1613,6 +1608,15 @@
__system_properties_zygote_reload; # apex
} LIBC_U;
+LIBC_36 { # introduced=36
+ global:
+ lchmod;
+ mseal;
+ qsort_r;
+ sig2str;
+ str2sig;
+} LIBC_V;
+
LIBC_PRIVATE {
global:
__accept4; # arm x86
diff --git a/libc/private/bionic_asm_arm.h b/libc/private/bionic_asm_arm.h
index d8381d3..9ca5f38 100644
--- a/libc/private/bionic_asm_arm.h
+++ b/libc/private/bionic_asm_arm.h
@@ -37,7 +37,7 @@
#pragma once
-#define __bionic_asm_align 0
+#define __bionic_asm_align 64
#undef __bionic_asm_custom_entry
#undef __bionic_asm_custom_end
diff --git a/libc/private/bionic_asm_arm64.h b/libc/private/bionic_asm_arm64.h
index ffc7181..1e907a1 100644
--- a/libc/private/bionic_asm_arm64.h
+++ b/libc/private/bionic_asm_arm64.h
@@ -37,7 +37,7 @@
#pragma once
-#define __bionic_asm_align 16
+#define __bionic_asm_align 64
#undef __bionic_asm_function_type
#define __bionic_asm_function_type %function
diff --git a/libc/private/bionic_globals.h b/libc/private/bionic_globals.h
index a1bebda..cd6dca9 100644
--- a/libc/private/bionic_globals.h
+++ b/libc/private/bionic_globals.h
@@ -157,6 +157,10 @@
};
__LIBC_HIDDEN__ libc_shared_globals* __libc_shared_globals();
+__LIBC_HIDDEN__ bool __libc_mte_enabled();
+__LIBC_HIDDEN__ void __libc_init_mte(const memtag_dynamic_entries_t*, const void*, size_t,
+ uintptr_t);
+__LIBC_HIDDEN__ void __libc_init_mte_stack(void*);
__LIBC_HIDDEN__ void __libc_init_fdsan();
__LIBC_HIDDEN__ void __libc_init_fdtrack();
__LIBC_HIDDEN__ void __libc_init_profiling_handlers();
diff --git a/libc/stdio/stdio.cpp b/libc/stdio/stdio.cpp
index 37b9665..a5f2f81 100644
--- a/libc/stdio/stdio.cpp
+++ b/libc/stdio/stdio.cpp
@@ -58,8 +58,6 @@
#include "private/bionic_fortify.h"
#include "private/thread_private.h"
-#include "private/bsd_sys_param.h" // For ALIGN/ALIGNBYTES.
-
#define NDYNAMIC 10 /* add ten more whenever necessary */
#define PRINTF_IMPL(expr) \
@@ -135,12 +133,14 @@
};
static glue* moreglue(int n) {
- char* data = new char[sizeof(glue) + ALIGNBYTES + n * sizeof(FILE) + n * sizeof(__sfileext)];
+ char* data = new char[sizeof(glue) +
+ alignof(FILE) + n * sizeof(FILE) +
+ alignof(__sfileext) + n * sizeof(__sfileext)];
if (data == nullptr) return nullptr;
glue* g = reinterpret_cast<glue*>(data);
- FILE* p = reinterpret_cast<FILE*>(ALIGN(data + sizeof(*g)));
- __sfileext* pext = reinterpret_cast<__sfileext*>(ALIGN(data + sizeof(*g)) + n * sizeof(FILE));
+ FILE* p = reinterpret_cast<FILE*>(__builtin_align_up(g + 1, alignof(FILE)));
+ __sfileext* pext = reinterpret_cast<__sfileext*>(__builtin_align_up(p + n, alignof(__sfileext)));
g->next = nullptr;
g->niobs = n;
g->iobs = p;
diff --git a/libc/upstream-openbsd/android/include/openbsd-compat.h b/libc/upstream-openbsd/android/include/openbsd-compat.h
index cbc52b5..ac6840a 100644
--- a/libc/upstream-openbsd/android/include/openbsd-compat.h
+++ b/libc/upstream-openbsd/android/include/openbsd-compat.h
@@ -25,8 +25,6 @@
#include <sys/random.h> // For getentropy.
-#include "private/bsd_sys_param.h"
-
#define __BEGIN_HIDDEN_DECLS _Pragma("GCC visibility push(hidden)")
#define __END_HIDDEN_DECLS _Pragma("GCC visibility pop")
diff --git a/linker/Android.bp b/linker/Android.bp
index 563cf3d..a06ca29 100644
--- a/linker/Android.bp
+++ b/linker/Android.bp
@@ -184,6 +184,7 @@
"linker_mapped_file_fragment.cpp",
"linker_note_gnu_property.cpp",
"linker_phdr.cpp",
+ "linker_phdr_16kib_compat.cpp",
"linker_relocate.cpp",
"linker_sdk_versions.cpp",
"linker_soinfo.cpp",
diff --git a/linker/linker.cpp b/linker/linker.cpp
index bcc2500..227b599 100644
--- a/linker/linker.cpp
+++ b/linker/linker.cpp
@@ -640,6 +640,11 @@
si_->set_gap_start(elf_reader.gap_start());
si_->set_gap_size(elf_reader.gap_size());
si_->set_should_pad_segments(elf_reader.should_pad_segments());
+ si_->set_should_use_16kib_app_compat(elf_reader.should_use_16kib_app_compat());
+ if (si_->should_use_16kib_app_compat()) {
+ si_->set_compat_relro_start(elf_reader.compat_relro_start());
+ si_->set_compat_relro_size(elf_reader.compat_relro_size());
+ }
return true;
}
@@ -1701,7 +1706,7 @@
}
// si->memtag_stack() needs to be called after si->prelink_image() which populates
// the dynamic section.
- if (si->has_min_version(7) && si->memtag_stack()) {
+ if (si->memtag_stack()) {
any_memtag_stack = true;
LD_LOG(kLogDlopen,
"... load_library requesting stack MTE for: realpath=\"%s\", soname=\"%s\"",
@@ -1715,7 +1720,7 @@
} else {
// find_library is used by the initial linking step, so we communicate that we
// want memtag_stack enabled to __libc_init_mte.
- __libc_shared_globals()->initial_memtag_stack = true;
+ __libc_shared_globals()->initial_memtag_stack_abi = true;
}
}
@@ -3361,7 +3366,8 @@
"\"%s\" has text relocations",
get_realpath());
add_dlwarning(get_realpath(), "text relocations");
- if (phdr_table_unprotect_segments(phdr, phnum, load_bias, should_pad_segments_) < 0) {
+ if (phdr_table_unprotect_segments(phdr, phnum, load_bias, should_pad_segments_,
+ should_use_16kib_app_compat_) < 0) {
DL_ERR("can't unprotect loadable segments for \"%s\": %m", get_realpath());
return false;
}
@@ -3377,7 +3383,8 @@
#if !defined(__LP64__)
if (has_text_relocations) {
// All relocations are done, we can protect our segments back to read-only.
- if (phdr_table_protect_segments(phdr, phnum, load_bias, should_pad_segments_) < 0) {
+ if (phdr_table_protect_segments(phdr, phnum, load_bias, should_pad_segments_,
+ should_use_16kib_app_compat_) < 0) {
DL_ERR("can't protect segments for \"%s\": %m", get_realpath());
return false;
}
@@ -3412,9 +3419,18 @@
}
bool soinfo::protect_relro() {
- if (phdr_table_protect_gnu_relro(phdr, phnum, load_bias, should_pad_segments_) < 0) {
- DL_ERR("can't enable GNU RELRO protection for \"%s\": %m", get_realpath());
- return false;
+ if (should_use_16kib_app_compat_) {
+ if (phdr_table_protect_gnu_relro_16kib_compat(compat_relro_start_, compat_relro_size_) < 0) {
+ DL_ERR("can't enable COMPAT GNU RELRO protection for \"%s\": %s", get_realpath(),
+ strerror(errno));
+ return false;
+ }
+ } else {
+ if (phdr_table_protect_gnu_relro(phdr, phnum, load_bias, should_pad_segments_,
+ should_use_16kib_app_compat_) < 0) {
+ DL_ERR("can't enable GNU RELRO protection for \"%s\": %m", get_realpath());
+ return false;
+ }
}
return true;
}
diff --git a/linker/linker_main.cpp b/linker/linker_main.cpp
index 6ccd75b..69355d3 100644
--- a/linker/linker_main.cpp
+++ b/linker/linker_main.cpp
@@ -71,7 +71,9 @@
static void set_bss_vma_name(soinfo* si);
void __libc_init_mte(const memtag_dynamic_entries_t* memtag_dynamic_entries, const void* phdr_start,
- size_t phdr_count, uintptr_t load_bias, void* stack_top);
+ size_t phdr_count, uintptr_t load_bias);
+
+void __libc_init_mte_stack(void* stack_top);
static void __linker_cannot_link(const char* argv0) {
__linker_error("CANNOT LINK EXECUTABLE \"%s\": %s", argv0, linker_get_error_buffer());
@@ -370,8 +372,9 @@
// Apply appropriate protections here if it is needed.
auto note_gnu_property = GnuPropertySection(somain);
if (note_gnu_property.IsBTICompatible() &&
- (phdr_table_protect_segments(somain->phdr, somain->phnum, somain->load_bias,
- somain->should_pad_segments(), ¬e_gnu_property) < 0)) {
+ (phdr_table_protect_segments(
+ somain->phdr, somain->phnum, somain->load_bias, somain->should_pad_segments(),
+ somain->should_use_16kib_app_compat(), ¬e_gnu_property) < 0)) {
__linker_error("error: can't protect segments for \"%s\": %m", exe_info.path.c_str());
}
}
@@ -464,8 +467,8 @@
#if defined(__aarch64__)
// This has to happen after the find_libraries, which will have collected any possible
// libraries that request memtag_stack in the dynamic section.
- __libc_init_mte(somain->memtag_dynamic_entries(), somain->phdr, somain->phnum, somain->load_bias,
- args.argv);
+ __libc_init_mte(somain->memtag_dynamic_entries(), somain->phdr, somain->phnum, somain->load_bias);
+ __libc_init_mte_stack(args.argv);
#endif
linker_finalize_static_tls();
@@ -722,6 +725,8 @@
tmp_linker_so.set_linker_flag();
if (!tmp_linker_so.prelink_image()) __linker_cannot_link(args.argv[0]);
+ // There is special logic in soinfo::relocate to avoid duplicating the
+ // relocations we did in relocate_linker().
if (!tmp_linker_so.link_image(SymbolLookupList(&tmp_linker_so), &tmp_linker_so, nullptr, nullptr)) __linker_cannot_link(args.argv[0]);
return __linker_init_post_relocation(args, tmp_linker_so);
diff --git a/linker/linker_phdr.cpp b/linker/linker_phdr.cpp
index 48206be..7691031 100644
--- a/linker/linker_phdr.cpp
+++ b/linker/linker_phdr.cpp
@@ -47,6 +47,7 @@
#include "private/elf_note.h"
#include <android-base/file.h>
+#include <android-base/properties.h>
static int GetTargetElfMachine() {
#if defined(__arm__)
@@ -139,11 +140,6 @@
**/
-#define MAYBE_MAP_FLAG(x, from, to) (((x) & (from)) ? (to) : 0)
-#define PFLAGS_TO_PROT(x) (MAYBE_MAP_FLAG((x), PF_X, PROT_EXEC) | \
- MAYBE_MAP_FLAG((x), PF_R, PROT_READ) | \
- MAYBE_MAP_FLAG((x), PF_W, PROT_WRITE))
-
static const size_t kPageSize = page_size();
/*
@@ -182,6 +178,14 @@
did_read_ = true;
}
+ if (kPageSize == 0x4000 && phdr_table_get_minimum_alignment(phdr_table_, phdr_num_) == 0x1000) {
+ // This prop needs to be read on 16KiB devices for each ELF where min_palign is 4KiB.
+ // It cannot be cached since the developer may toggle app compat on/off.
+ // This check will be removed once app compat is made the default on 16KiB devices.
+ should_use_16kib_app_compat_ =
+ ::android::base::GetBoolProperty("bionic.linker.16kb.app_compat.enabled", false);
+ }
+
return did_read_;
}
@@ -197,8 +201,9 @@
#if defined(__aarch64__)
// For Armv8.5-A loaded executable segments may require PROT_BTI.
if (note_gnu_property_.IsBTICompatible()) {
- did_load_ = (phdr_table_protect_segments(phdr_table_, phdr_num_, load_bias_,
- should_pad_segments_, ¬e_gnu_property_) == 0);
+ did_load_ =
+ (phdr_table_protect_segments(phdr_table_, phdr_num_, load_bias_, should_pad_segments_,
+ should_use_16kib_app_compat_, ¬e_gnu_property_) == 0);
}
#endif
}
@@ -690,6 +695,13 @@
return false;
}
+ if (should_use_16kib_app_compat_) {
+ // Reserve additional space for aligning the permission boundary in compat loading
+ // Up to kPageSize-kCompatPageSize additional space is needed, but reservation
+ // is done with mmap which gives kPageSize multiple-sized reservations.
+ load_size_ += kPageSize;
+ }
+
uint8_t* addr = reinterpret_cast<uint8_t*>(min_vaddr);
void* start;
@@ -725,6 +737,13 @@
load_start_ = start;
load_bias_ = reinterpret_cast<uint8_t*>(start) - addr;
+
+ if (should_use_16kib_app_compat_) {
+ // In compat mode make the initial mapping RW since the ELF contents will be read
+ // into it; instead of mapped over it.
+ mprotect(reinterpret_cast<void*>(start), load_size_, PROT_READ | PROT_WRITE);
+ }
+
return true;
}
@@ -808,8 +827,15 @@
}
static inline void _extend_load_segment_vma(const ElfW(Phdr)* phdr_table, size_t phdr_count,
- size_t phdr_idx, ElfW(Addr)* p_memsz,
- ElfW(Addr)* p_filesz, bool should_pad_segments) {
+ size_t phdr_idx, ElfW(Addr)* p_memsz,
+ ElfW(Addr)* p_filesz, bool should_pad_segments,
+ bool should_use_16kib_app_compat) {
+ // NOTE: Segment extension is only applicable where the ELF's max-page-size > runtime page size;
+ // to save kernel VMA slab memory. 16KiB compat mode is the exact opposite scenario.
+ if (should_use_16kib_app_compat) {
+ return;
+ }
+
const ElfW(Phdr)* phdr = &phdr_table[phdr_idx];
const ElfW(Phdr)* next = nullptr;
size_t next_idx = phdr_idx + 1;
@@ -879,6 +905,13 @@
}
void ElfReader::ZeroFillSegment(const ElfW(Phdr)* phdr) {
+ // NOTE: In 16KiB app compat mode, the ELF mapping is anonymous, meaning that
+ // RW segments are COW-ed from the kernel's zero page. So there is no need to
+ // explicitly zero-fill until the last page's limit.
+ if (should_use_16kib_app_compat_) {
+ return;
+ }
+
ElfW(Addr) seg_start = phdr->p_vaddr + load_bias_;
uint64_t unextended_seg_file_end = seg_start + phdr->p_filesz;
@@ -898,6 +931,12 @@
}
void ElfReader::DropPaddingPages(const ElfW(Phdr)* phdr, uint64_t seg_file_end) {
+ // NOTE: Padding pages are only applicable where the ELF's max-page-size > runtime page size;
+ // 16KiB compat mode is the exact opposite scenario.
+ if (should_use_16kib_app_compat_) {
+ return;
+ }
+
ElfW(Addr) seg_start = phdr->p_vaddr + load_bias_;
uint64_t unextended_seg_file_end = seg_start + phdr->p_filesz;
@@ -926,6 +965,12 @@
bool ElfReader::MapBssSection(const ElfW(Phdr)* phdr, ElfW(Addr) seg_page_end,
ElfW(Addr) seg_file_end) {
+ // NOTE: We do not need to handle .bss in 16KiB compat mode since the mapping
+ // reservation is anonymous and RW to begin with.
+ if (should_use_16kib_app_compat_) {
+ return true;
+ }
+
// seg_file_end is now the first page address after the file content.
seg_file_end = page_end(seg_file_end);
@@ -952,15 +997,27 @@
}
bool ElfReader::LoadSegments() {
+ // NOTE: The compat(legacy) page size (4096) must be used when aligning
+ // the 4KiB segments for loading in compat mode. The larger 16KiB page size
+ // will lead to overwriting adjacent segments since the ELF's segment(s)
+ // are not 16KiB aligned.
+ size_t seg_align = should_use_16kib_app_compat_ ? kCompatPageSize : kPageSize;
+
size_t min_palign = phdr_table_get_minimum_alignment(phdr_table_, phdr_num_);
- // Only enforce this on 16 KB systems. Apps may rely on undefined behavior
- // here on 4 KB systems, which is the norm before this change is introduced.
- if (kPageSize >= 16384 && min_palign < kPageSize) {
+ // Only enforce this on 16 KB systems with app compat disabled.
+ // Apps may rely on undefined behavior here on 4 KB systems,
+ // which is the norm before this change is introduced
+ if (kPageSize >= 16384 && min_palign < kPageSize && !should_use_16kib_app_compat_) {
DL_ERR("\"%s\" program alignment (%zu) cannot be smaller than system page size (%zu)",
name_.c_str(), min_palign, kPageSize);
return false;
}
+ if (!Setup16KiBAppCompat()) {
+ DL_ERR("\"%s\" failed to setup 16KiB App Compat", name_.c_str());
+ return false;
+ }
+
for (size_t i = 0; i < phdr_num_; ++i) {
const ElfW(Phdr)* phdr = &phdr_table_[i];
@@ -970,14 +1027,14 @@
ElfW(Addr) p_memsz = phdr->p_memsz;
ElfW(Addr) p_filesz = phdr->p_filesz;
- _extend_load_segment_vma(phdr_table_, phdr_num_, i, &p_memsz, &p_filesz, should_pad_segments_);
+ _extend_load_segment_vma(phdr_table_, phdr_num_, i, &p_memsz, &p_filesz, should_pad_segments_,
+ should_use_16kib_app_compat_);
// Segment addresses in memory.
ElfW(Addr) seg_start = phdr->p_vaddr + load_bias_;
ElfW(Addr) seg_end = seg_start + p_memsz;
- ElfW(Addr) seg_page_start = page_start(seg_start);
- ElfW(Addr) seg_page_end = page_end(seg_end);
+ ElfW(Addr) seg_page_end = align_up(seg_end, seg_align);
ElfW(Addr) seg_file_end = seg_start + p_filesz;
@@ -985,7 +1042,7 @@
ElfW(Addr) file_start = phdr->p_offset;
ElfW(Addr) file_end = file_start + p_filesz;
- ElfW(Addr) file_page_start = page_start(file_start);
+ ElfW(Addr) file_page_start = align_down(file_start, seg_align);
ElfW(Addr) file_length = file_end - file_page_start;
if (file_size_ <= 0) {
@@ -1018,8 +1075,14 @@
}
// Pass the file_length, since it may have been extended by _extend_load_segment_vma().
- if (!MapSegment(i, file_length)) {
- return false;
+ if (should_use_16kib_app_compat_) {
+ if (!CompatMapSegment(i, file_length)) {
+ return false;
+ }
+ } else {
+ if (!MapSegment(i, file_length)) {
+ return false;
+ }
}
}
@@ -1040,7 +1103,7 @@
*/
static int _phdr_table_set_load_prot(const ElfW(Phdr)* phdr_table, size_t phdr_count,
ElfW(Addr) load_bias, int extra_prot_flags,
- bool should_pad_segments) {
+ bool should_pad_segments, bool should_use_16kib_app_compat) {
for (size_t i = 0; i < phdr_count; ++i) {
const ElfW(Phdr)* phdr = &phdr_table[i];
@@ -1050,7 +1113,8 @@
ElfW(Addr) p_memsz = phdr->p_memsz;
ElfW(Addr) p_filesz = phdr->p_filesz;
- _extend_load_segment_vma(phdr_table, phdr_count, i, &p_memsz, &p_filesz, should_pad_segments);
+ _extend_load_segment_vma(phdr_table, phdr_count, i, &p_memsz, &p_filesz, should_pad_segments,
+ should_use_16kib_app_compat);
ElfW(Addr) seg_page_start = page_start(phdr->p_vaddr + load_bias);
ElfW(Addr) seg_page_end = page_end(phdr->p_vaddr + p_memsz + load_bias);
@@ -1089,12 +1153,14 @@
* phdr_count -> number of entries in tables
* load_bias -> load bias
* should_pad_segments -> Are segments extended to avoid gaps in the memory map
+ * should_use_16kib_app_compat -> Is the ELF being loaded in 16KiB app compat mode.
* prop -> GnuPropertySection or nullptr
* Return:
* 0 on success, -1 on failure (error code in errno).
*/
int phdr_table_protect_segments(const ElfW(Phdr)* phdr_table, size_t phdr_count,
ElfW(Addr) load_bias, bool should_pad_segments,
+ bool should_use_16kib_app_compat,
const GnuPropertySection* prop __unused) {
int prot = 0;
#if defined(__aarch64__)
@@ -1102,7 +1168,8 @@
prot |= PROT_BTI;
}
#endif
- return _phdr_table_set_load_prot(phdr_table, phdr_count, load_bias, prot, should_pad_segments);
+ return _phdr_table_set_load_prot(phdr_table, phdr_count, load_bias, prot, should_pad_segments,
+ should_use_16kib_app_compat);
}
/* Change the protection of all loaded segments in memory to writable.
@@ -1119,20 +1186,22 @@
* phdr_count -> number of entries in tables
* load_bias -> load bias
* should_pad_segments -> Are segments extended to avoid gaps in the memory map
+ * should_use_16kib_app_compat -> Is the ELF being loaded in 16KiB app compat mode.
* Return:
* 0 on success, -1 on failure (error code in errno).
*/
-int phdr_table_unprotect_segments(const ElfW(Phdr)* phdr_table,
- size_t phdr_count, ElfW(Addr) load_bias,
- bool should_pad_segments) {
+int phdr_table_unprotect_segments(const ElfW(Phdr)* phdr_table, size_t phdr_count,
+ ElfW(Addr) load_bias, bool should_pad_segments,
+ bool should_use_16kib_app_compat) {
return _phdr_table_set_load_prot(phdr_table, phdr_count, load_bias, PROT_WRITE,
- should_pad_segments);
+ should_pad_segments, should_use_16kib_app_compat);
}
static inline void _extend_gnu_relro_prot_end(const ElfW(Phdr)* relro_phdr,
const ElfW(Phdr)* phdr_table, size_t phdr_count,
ElfW(Addr) load_bias, ElfW(Addr)* seg_page_end,
- bool should_pad_segments) {
+ bool should_pad_segments,
+ bool should_use_16kib_app_compat) {
// Find the index and phdr of the LOAD containing the GNU_RELRO segment
for (size_t index = 0; index < phdr_count; ++index) {
const ElfW(Phdr)* phdr = &phdr_table[index];
@@ -1180,7 +1249,7 @@
// mprotect will only RO protect a part of the extended RW LOAD segment, which
// will leave an extra split RW VMA (the gap).
_extend_load_segment_vma(phdr_table, phdr_count, index, &p_memsz, &p_filesz,
- should_pad_segments);
+ should_pad_segments, should_use_16kib_app_compat);
*seg_page_end = page_end(phdr->p_vaddr + p_memsz + load_bias);
return;
@@ -1193,7 +1262,8 @@
*/
static int _phdr_table_set_gnu_relro_prot(const ElfW(Phdr)* phdr_table, size_t phdr_count,
ElfW(Addr) load_bias, int prot_flags,
- bool should_pad_segments) {
+ bool should_pad_segments,
+ bool should_use_16kib_app_compat) {
const ElfW(Phdr)* phdr = phdr_table;
const ElfW(Phdr)* phdr_limit = phdr + phdr_count;
@@ -1221,7 +1291,7 @@
ElfW(Addr) seg_page_start = page_start(phdr->p_vaddr) + load_bias;
ElfW(Addr) seg_page_end = page_end(phdr->p_vaddr + phdr->p_memsz) + load_bias;
_extend_gnu_relro_prot_end(phdr, phdr_table, phdr_count, load_bias, &seg_page_end,
- should_pad_segments);
+ should_pad_segments, should_use_16kib_app_compat);
int ret = mprotect(reinterpret_cast<void*>(seg_page_start),
seg_page_end - seg_page_start,
@@ -1247,13 +1317,29 @@
* phdr_count -> number of entries in tables
* load_bias -> load bias
* should_pad_segments -> Were segments extended to avoid gaps in the memory map
+ * should_use_16kib_app_compat -> Is the ELF being loaded in 16KiB app compat mode.
* Return:
* 0 on success, -1 on failure (error code in errno).
*/
int phdr_table_protect_gnu_relro(const ElfW(Phdr)* phdr_table, size_t phdr_count,
- ElfW(Addr) load_bias, bool should_pad_segments) {
+ ElfW(Addr) load_bias, bool should_pad_segments,
+ bool should_use_16kib_app_compat) {
return _phdr_table_set_gnu_relro_prot(phdr_table, phdr_count, load_bias, PROT_READ,
- should_pad_segments);
+ should_pad_segments, should_use_16kib_app_compat);
+}
+
+/*
+ * Apply RX protection to the compat relro region of the ELF being loaded in
+ * 16KiB compat mode.
+ *
+ * Input:
+ * start -> start address of the compat relro region.
+ * size -> size of the compat relro region in bytes.
+ * Return:
+ * 0 on success, -1 on failure (error code in errno).
+ */
+int phdr_table_protect_gnu_relro_16kib_compat(ElfW(Addr) start, ElfW(Addr) size) {
+ return mprotect(reinterpret_cast<void*>(start), size, PROT_READ | PROT_EXEC);
}
/* Serialize the GNU relro segments to the given file descriptor. This can be
diff --git a/linker/linker_phdr.h b/linker/linker_phdr.h
index 1d6bbe3..2f159f3 100644
--- a/linker/linker_phdr.h
+++ b/linker/linker_phdr.h
@@ -39,6 +39,13 @@
#include "linker_mapped_file_fragment.h"
#include "linker_note_gnu_property.h"
+#define MAYBE_MAP_FLAG(x, from, to) (((x) & (from)) ? (to) : 0)
+#define PFLAGS_TO_PROT(x) (MAYBE_MAP_FLAG((x), PF_X, PROT_EXEC) | \
+ MAYBE_MAP_FLAG((x), PF_R, PROT_READ) | \
+ MAYBE_MAP_FLAG((x), PF_W, PROT_WRITE))
+
+static constexpr size_t kCompatPageSize = 0x1000;
+
class ElfReader {
public:
ElfReader();
@@ -59,6 +66,9 @@
bool is_mapped_by_caller() const { return mapped_by_caller_; }
ElfW(Addr) entry_point() const { return header_.e_entry + load_bias_; }
bool should_pad_segments() const { return should_pad_segments_; }
+ bool should_use_16kib_app_compat() const { return should_use_16kib_app_compat_; }
+ ElfW(Addr) compat_relro_start() const { return compat_relro_start_; }
+ ElfW(Addr) compat_relro_size() const { return compat_relro_size_; }
private:
[[nodiscard]] bool ReadElfHeader();
@@ -69,10 +79,14 @@
[[nodiscard]] bool ReadPadSegmentNote();
[[nodiscard]] bool ReserveAddressSpace(address_space_params* address_space);
[[nodiscard]] bool MapSegment(size_t seg_idx, size_t len);
+ [[nodiscard]] bool CompatMapSegment(size_t seg_idx, size_t len);
void ZeroFillSegment(const ElfW(Phdr)* phdr);
void DropPaddingPages(const ElfW(Phdr)* phdr, uint64_t seg_file_end);
[[nodiscard]] bool MapBssSection(const ElfW(Phdr)* phdr, ElfW(Addr) seg_page_end,
ElfW(Addr) seg_file_end);
+ [[nodiscard]] bool IsEligibleFor16KiBAppCompat(ElfW(Addr)* vaddr);
+ [[nodiscard]] bool HasAtMostOneRelroSegment(const ElfW(Phdr)** relro_phdr);
+ [[nodiscard]] bool Setup16KiBAppCompat();
[[nodiscard]] bool LoadSegments();
[[nodiscard]] bool FindPhdr();
[[nodiscard]] bool FindGnuPropertySection();
@@ -123,6 +137,13 @@
// Pad gaps between segments when memory mapping?
bool should_pad_segments_ = false;
+ // Use app compat mode when loading 4KiB max-page-size ELFs on 16KiB page-size devices?
+ bool should_use_16kib_app_compat_ = false;
+
+ // RELRO region for 16KiB compat loading
+ ElfW(Addr) compat_relro_start_ = 0;
+ ElfW(Addr) compat_relro_size_ = 0;
+
// Only used by AArch64 at the moment.
GnuPropertySection note_gnu_property_ __unused;
};
@@ -135,13 +156,18 @@
int phdr_table_protect_segments(const ElfW(Phdr)* phdr_table, size_t phdr_count,
ElfW(Addr) load_bias, bool should_pad_segments,
+ bool should_use_16kib_app_compat,
const GnuPropertySection* prop = nullptr);
int phdr_table_unprotect_segments(const ElfW(Phdr)* phdr_table, size_t phdr_count,
- ElfW(Addr) load_bias, bool should_pad_segments);
+ ElfW(Addr) load_bias, bool should_pad_segments,
+ bool should_use_16kib_app_compat);
int phdr_table_protect_gnu_relro(const ElfW(Phdr)* phdr_table, size_t phdr_count,
- ElfW(Addr) load_bias, bool should_pad_segments);
+ ElfW(Addr) load_bias, bool should_pad_segments,
+ bool should_use_16kib_app_compat);
+
+int phdr_table_protect_gnu_relro_16kib_compat(ElfW(Addr) start, ElfW(Addr) size);
int phdr_table_serialize_gnu_relro(const ElfW(Phdr)* phdr_table, size_t phdr_count,
ElfW(Addr) load_bias, int fd, size_t* file_offset);
diff --git a/linker/linker_phdr_16kib_compat.cpp b/linker/linker_phdr_16kib_compat.cpp
new file mode 100644
index 0000000..a4d8459
--- /dev/null
+++ b/linker/linker_phdr_16kib_compat.cpp
@@ -0,0 +1,237 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "linker_phdr.h"
+
+#include <linux/prctl.h>
+#include <sys/mman.h>
+#include <sys/prctl.h>
+#include <unistd.h>
+
+#include "linker_debug.h"
+#include "linker_dlwarning.h"
+#include "linker_globals.h"
+
+#include "platform/bionic/macros.h"
+#include "platform/bionic/page.h"
+
+#include <string>
+
+static inline bool segment_contains_prefix(const ElfW(Phdr)* segment, const ElfW(Phdr)* prefix) {
+ return segment && prefix && segment->p_vaddr == prefix->p_vaddr;
+}
+
+/*
+ * Returns true if the ELF contains at most 1 RELRO segment; and populates @relro_phdr
+ * with the relro phdr or nullptr if none.
+ *
+ * Returns false if more than 1 RELRO segments are found.
+ */
+bool ElfReader::HasAtMostOneRelroSegment(const ElfW(Phdr)** relro_phdr) {
+ const ElfW(Phdr)* relro = nullptr;
+ for (size_t i = 0; i < phdr_num_; ++i) {
+ const ElfW(Phdr)* phdr = &phdr_table_[i];
+
+ if (phdr->p_type != PT_GNU_RELRO) {
+ continue;
+ }
+
+ if (relro == nullptr) {
+ relro = phdr;
+ } else {
+ return false;
+ }
+ }
+
+ *relro_phdr = relro;
+
+ return true;
+}
+
+/*
+ * In 16KiB compatibility mode ELFs with the following segment layout
+ * can be loaded successfully:
+ *
+ * ┌────────────┬─────────────────────────┬────────────┐
+ * │ │ │ │
+ * │ (RO|RX)* │ (RW - RELRO prefix)? │ (RW)* │
+ * │ │ │ │
+ * └────────────┴─────────────────────────┴────────────┘
+ *
+ * In other words, compatible layouts have:
+ * - zero or more RO or RX segments;
+ * - followed by zero or one RELRO prefix;
+ * - followed by zero or more RW segments (this can include the RW
+ * suffix from the segment containing the RELRO prefix, if any)
+ *
+ * In 16KiB compat mode, after relocation, the ELF is layout in virtual
+ * memory is as shown below:
+ * ┌──────────────────────────────────────┬────────────┐
+ * │ │ │
+ * │ (RX)? │ (RW)? │
+ * │ │ │
+ * └──────────────────────────────────────┴────────────┘
+ *
+ * In compat mode:
+ * - the RO and RX segments along with the RELRO prefix are protected
+ * as RX;
+ * - and the RW segments along with RW suffix from the relro segment,
+ * if any; are RW protected.
+ *
+ * This allows for the single RX|RW permission boundary to be aligned with
+ * a 16KiB page boundary; since a single page cannot share multiple
+ * permissions.
+ *
+ * IsEligibleFor16KiBAppCompat() identifies compatible ELFs and populates @vaddr
+ * with the boundary between RX|RW portions.
+ *
+ * Returns true if the ELF can be loaded in compat mode, else false.
+ */
+bool ElfReader::IsEligibleFor16KiBAppCompat(ElfW(Addr)* vaddr) {
+ const ElfW(Phdr)* relro_phdr = nullptr;
+ if (!HasAtMostOneRelroSegment(&relro_phdr)) {
+ DL_WARN("\"%s\": Compat loading failed: Multiple RELRO segments found", name_.c_str());
+ return false;
+ }
+
+ const ElfW(Phdr)* last_rw = nullptr;
+ const ElfW(Phdr)* first_rw = nullptr;
+
+ for (size_t i = 0; i < phdr_num_; ++i) {
+ const ElfW(Phdr)* curr = &phdr_table_[i];
+ const ElfW(Phdr)* prev = (i > 0) ? &phdr_table_[i - 1] : nullptr;
+
+ if (curr->p_type != PT_LOAD) {
+ continue;
+ }
+
+ int prot = PFLAGS_TO_PROT(curr->p_flags);
+
+ if ((prot & PROT_WRITE) && (prot & PROT_READ)) {
+ if (!first_rw) {
+ first_rw = curr;
+ }
+
+ if (last_rw && last_rw != prev) {
+ DL_WARN("\"%s\": Compat loading failed: ELF contains multiple non-adjacent RW segments",
+ name_.c_str());
+ return false;
+ }
+
+ last_rw = curr;
+ }
+ }
+
+ if (!relro_phdr) {
+ *vaddr = align_down(first_rw->p_vaddr, kCompatPageSize);
+ return true;
+ }
+
+ // The RELRO segment is present, it must be the prefix of the first RW segment.
+ if (!segment_contains_prefix(first_rw, relro_phdr)) {
+ DL_WARN("\"%s\": Compat loading failed: RELRO is not in the first RW segment",
+ name_.c_str());
+ return false;
+ }
+
+ uint64_t end;
+ if (__builtin_add_overflow(relro_phdr->p_vaddr, relro_phdr->p_memsz, &end)) {
+ DL_WARN("\"%s\": Compat loading failed: relro vaddr + memsz overflowed", name_.c_str());
+ return false;
+ }
+
+ *vaddr = align_up(end, kCompatPageSize);
+ return true;
+}
+
+/*
+ * Returns the offset/shift needed to align @vaddr to a page boundary.
+ */
+static inline ElfW(Addr) perm_boundary_offset(const ElfW(Addr) addr) {
+ ElfW(Addr) offset = page_offset(addr);
+
+ return offset ? page_size() - offset : 0;
+}
+
+bool ElfReader::Setup16KiBAppCompat() {
+ if (!should_use_16kib_app_compat_) {
+ return true;
+ }
+
+ ElfW(Addr) rx_rw_boundary; // Permission bounadry for compat mode
+ if (!IsEligibleFor16KiBAppCompat(&rx_rw_boundary)) {
+ return false;
+ }
+
+ // Adjust the load_bias to position the RX|RW boundary on a page boundary
+ load_bias_ += perm_boundary_offset(rx_rw_boundary);
+
+ // RW region (.data, .bss ...)
+ ElfW(Addr) rw_start = load_bias_ + rx_rw_boundary;
+ ElfW(Addr) rw_size = load_size_ - (rw_start - reinterpret_cast<ElfW(Addr)>(load_start_));
+
+ CHECK(rw_start % getpagesize() == 0);
+ CHECK(rw_size % getpagesize() == 0);
+
+ // Compat RELRO (RX) region (.text, .data.relro, ...)
+ compat_relro_start_ = reinterpret_cast<ElfW(Addr)>(load_start_);
+ compat_relro_size_ = load_size_ - rw_size;
+
+ // Label the ELF VMA, since compat mode uses anonymous mappings.
+ std::string compat_name = name_ + " (compat loaded)";
+ prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, load_start_, load_size_, compat_name.c_str());
+
+ return true;
+}
+
+bool ElfReader::CompatMapSegment(size_t seg_idx, size_t len) {
+ const ElfW(Phdr)* phdr = &phdr_table_[seg_idx];
+
+ // NOTE: The compat(legacy) page size (4096) must be used when aligning
+ // the 4KiB segments for loading (reading). The larger 16KiB page size
+ // will lead to overwriting adjacent segments since the ELF's segment(s)
+ // are not 16KiB aligned.
+
+ void* start = reinterpret_cast<void*>(align_down(phdr->p_vaddr + load_bias_, kCompatPageSize));
+
+ // The ELF could be being loaded directly from a zipped APK,
+ // the zip offset must be added to find the segment offset.
+ const ElfW(Addr) offset = file_offset_ + align_down(phdr->p_offset, kCompatPageSize);
+
+ CHECK(should_use_16kib_app_compat_);
+
+ // Since the 4KiB max-page-size ELF is not properly aligned, loading it by
+ // directly mmapping the ELF file is not feasible.
+ // Instead, read the ELF contents into the anonymous RW mapping.
+ if (TEMP_FAILURE_RETRY(pread64(fd_, start, len, offset)) == -1) {
+ DL_ERR("Compat loading: \"%s\" failed to read LOAD segment %zu: %m", name_.c_str(), seg_idx);
+ return false;
+ }
+
+ return true;
+}
diff --git a/linker/linker_relocate.cpp b/linker/linker_relocate.cpp
index bcb1efc..0470f87 100644
--- a/linker/linker_relocate.cpp
+++ b/linker/linker_relocate.cpp
@@ -188,8 +188,8 @@
auto protect_segments = [&]() {
// Make .text executable.
if (phdr_table_protect_segments(relocator.si->phdr, relocator.si->phnum,
- relocator.si->load_bias,
- relocator.si->should_pad_segments()) < 0) {
+ relocator.si->load_bias, relocator.si->should_pad_segments(),
+ relocator.si->should_use_16kib_app_compat()) < 0) {
DL_ERR("can't protect segments for \"%s\": %m", relocator.si->get_realpath());
return false;
}
@@ -198,8 +198,8 @@
auto unprotect_segments = [&]() {
// Make .text writable.
if (phdr_table_unprotect_segments(relocator.si->phdr, relocator.si->phnum,
- relocator.si->load_bias,
- relocator.si->should_pad_segments()) < 0) {
+ relocator.si->load_bias, relocator.si->should_pad_segments(),
+ relocator.si->should_use_16kib_app_compat()) < 0) {
DL_ERR("can't unprotect loadable segments for \"%s\": %m",
relocator.si->get_realpath());
return false;
diff --git a/linker/linker_soinfo.h b/linker/linker_soinfo.h
index 9a13af2..9bec2aa 100644
--- a/linker/linker_soinfo.h
+++ b/linker/linker_soinfo.h
@@ -293,6 +293,9 @@
#if defined(__work_around_b_24465209__)
return (flags_ & FLAG_NEW_SOINFO) != 0 && version_ >= min_version;
#else
+ // If you make this return non-true in the case where
+ // __work_around_b_24465209__ is not defined, you will have to change
+ // memtag_dynamic_entries.
return true;
#endif
}
@@ -354,21 +357,55 @@
size_t get_gap_size() const;
const memtag_dynamic_entries_t* memtag_dynamic_entries() const {
- CHECK(has_min_version(7));
+#ifdef __aarch64__
+#ifdef __work_around_b_24465209__
+#error "Assuming aarch64 does not use versioned soinfo."
+#endif
return &memtag_dynamic_entries_;
+#endif
+ return nullptr;
}
- void* memtag_globals() const { return memtag_dynamic_entries()->memtag_globals; }
- size_t memtag_globalssz() const { return memtag_dynamic_entries()->memtag_globalssz; }
- bool has_memtag_mode() const { return memtag_dynamic_entries()->has_memtag_mode; }
- unsigned memtag_mode() const { return memtag_dynamic_entries()->memtag_mode; }
- bool memtag_heap() const { return memtag_dynamic_entries()->memtag_heap; }
- bool memtag_stack() const { return memtag_dynamic_entries()->memtag_stack; }
+ void* memtag_globals() const {
+ const memtag_dynamic_entries_t* entries = memtag_dynamic_entries();
+ return entries ? entries->memtag_globals : nullptr;
+ }
+ size_t memtag_globalssz() const {
+ const memtag_dynamic_entries_t* entries = memtag_dynamic_entries();
+ return entries ? entries->memtag_globalssz : 0U;
+ }
+ bool has_memtag_mode() const {
+ const memtag_dynamic_entries_t* entries = memtag_dynamic_entries();
+ return entries ? entries->has_memtag_mode : false;
+ }
+ unsigned memtag_mode() const {
+ const memtag_dynamic_entries_t* entries = memtag_dynamic_entries();
+ return entries ? entries->memtag_mode : 0U;
+ }
+ bool memtag_heap() const {
+ const memtag_dynamic_entries_t* entries = memtag_dynamic_entries();
+ return entries ? entries->memtag_heap : false;
+ }
+ bool memtag_stack() const {
+ const memtag_dynamic_entries_t* entries = memtag_dynamic_entries();
+ return entries ? entries->memtag_stack : false;
+ }
void set_should_pad_segments(bool should_pad_segments) {
should_pad_segments_ = should_pad_segments;
}
bool should_pad_segments() const { return should_pad_segments_; }
+ void set_should_use_16kib_app_compat(bool should_use_16kib_app_compat) {
+ should_use_16kib_app_compat_ = should_use_16kib_app_compat;
+ }
+ bool should_use_16kib_app_compat() const { return should_use_16kib_app_compat_; }
+
+ void set_compat_relro_start(ElfW(Addr) start) { compat_relro_start_ = start; }
+ ElfW(Addr) compat_relro_start() const { return compat_relro_start_; }
+
+ void set_compat_relro_size(ElfW(Addr) size) { compat_relro_size_ = size; }
+ ElfW(Addr) compat_relro_size() const { return compat_relro_start_; }
+
private:
bool is_image_linked() const;
void set_image_linked();
@@ -450,11 +487,18 @@
ElfW(Addr) gap_start_;
size_t gap_size_;
- // version >= 7
+ // __aarch64__ only, which does not use versioning.
memtag_dynamic_entries_t memtag_dynamic_entries_;
// Pad gaps between segments when memory mapping?
bool should_pad_segments_ = false;
+
+ // Use app compat mode when loading 4KiB max-page-size ELFs on 16KiB page-size devices?
+ bool should_use_16kib_app_compat_ = false;
+
+ // RELRO region for 16KiB compat loading
+ ElfW(Addr) compat_relro_start_ = 0;
+ ElfW(Addr) compat_relro_size_ = 0;
};
// This function is used by dlvsym() to calculate hash of sym_ver
diff --git a/tests/Android.bp b/tests/Android.bp
index d1ca78f..b537483 100644
--- a/tests/Android.bp
+++ b/tests/Android.bp
@@ -389,6 +389,7 @@
"bug_26110743_test.cpp",
"byteswap_test.cpp",
"complex_test.cpp",
+ "cpu_target_features_test.cpp",
"ctype_test.cpp",
"dirent_test.cpp",
"elf_test.cpp",
@@ -745,6 +746,40 @@
},
}
+cc_defaults {
+ name: "bionic_fortify_c_tests_defaults",
+ defaults: [
+ "bionic_clang_fortify_tests_w_flags",
+ "bionic_tests_defaults",
+ ],
+ cflags: [
+ "-U_FORTIFY_SOURCE",
+ // -fbuiltin is required here to counteract -fno-builtin from
+ // `bionic_tests_defaults`. With `-fno-builtin`, Clang won't
+ // const-evaluate calls to `strlen`, which is tested for here.
+ "-fbuiltin",
+ ],
+ srcs: [
+ "clang_fortify_c_only_tests.c",
+ ],
+ tidy: false,
+ shared: {
+ enabled: false,
+ },
+}
+
+cc_test_library {
+ name: "libfortify1-c-tests-clang",
+ defaults: ["bionic_fortify_c_tests_defaults"],
+ cflags: ["-D_FORTIFY_SOURCE=1"],
+}
+
+cc_test_library {
+ name: "libfortify2-c-tests-clang",
+ defaults: ["bionic_fortify_c_tests_defaults"],
+ cflags: ["-D_FORTIFY_SOURCE=2"],
+}
+
// -----------------------------------------------------------------------------
// Library of all tests (excluding the dynamic linker tests).
// -----------------------------------------------------------------------------
@@ -756,8 +791,10 @@
"libBionicStandardTests",
"libBionicElfTlsTests",
"libBionicFramePointerTests",
+ "libfortify1-c-tests-clang",
"libfortify1-tests-clang",
"libfortify1-new-tests-clang",
+ "libfortify2-c-tests-clang",
"libfortify2-tests-clang",
"libfortify2-new-tests-clang",
],
@@ -778,6 +815,7 @@
"dlfcn_test.cpp",
"execinfo_test.cpp",
"link_test.cpp",
+ "page_size_16kib_compat_test.cpp",
"pthread_dlfcn_test.cpp",
],
static_libs: [
@@ -786,6 +824,7 @@
],
include_dirs: [
"bionic/libc",
+ "bionic/tests/libs",
],
shared: {
enabled: false,
@@ -933,6 +972,7 @@
"libtest_dt_runpath_d",
"libtest_dt_runpath_x",
"libtest_dt_runpath_y",
+ "libtest_elf_max_page_size_4kib",
"libtest_elftls_dynamic",
"libtest_elftls_dynamic_filler_1",
"libtest_elftls_dynamic_filler_2",
@@ -1177,6 +1217,7 @@
shared_libs: [
"libbase",
+ "liblog",
],
target: {
diff --git a/libc/private/bsd_sys_param.h b/tests/DoNotOptimize.h
similarity index 63%
rename from libc/private/bsd_sys_param.h
rename to tests/DoNotOptimize.h
index ab54aa0..711d339 100644
--- a/libc/private/bsd_sys_param.h
+++ b/tests/DoNotOptimize.h
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2014 The Android Open Source Project
+ * Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,8 +16,12 @@
#pragma once
-#include <inttypes.h>
-
-/* OpenBSD has these in <sys/param.h>, but "ALIGN" isn't something we want to reserve. */
-#define ALIGNBYTES (sizeof(uintptr_t) - 1)
-#define ALIGN(p) ((__BIONIC_CAST(reinterpret_cast, uintptr_t, p) + ALIGNBYTES) & ~ALIGNBYTES)
+// From <benchmark/benchmark.h>.
+template <class Tp>
+static inline void DoNotOptimize(Tp const& value) {
+ asm volatile("" : : "r,m"(value) : "memory");
+}
+template <class Tp>
+static inline void DoNotOptimize(Tp& value) {
+ asm volatile("" : "+r,m"(value) : : "memory");
+}
diff --git a/tests/clang_fortify_c_only_tests.c b/tests/clang_fortify_c_only_tests.c
new file mode 100644
index 0000000..3bec848
--- /dev/null
+++ b/tests/clang_fortify_c_only_tests.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <string.h>
+
+// This is a test specifically of bionic's FORTIFY machinery. Other stdlibs need not apply.
+#ifdef __BIONIC__
+
+// Ensure that strlen can be evaluated at compile-time. Clang doesn't support
+// this in C++, but does in C.
+_Static_assert(strlen("foo") == 3, "");
+
+#endif // __BIONIC__
diff --git a/tests/clang_fortify_tests.cpp b/tests/clang_fortify_tests.cpp
index f08fd1f..da7926d 100644
--- a/tests/clang_fortify_tests.cpp
+++ b/tests/clang_fortify_tests.cpp
@@ -89,6 +89,10 @@
#include <unistd.h>
#include <wchar.h>
+#include <array>
+
+#include "DoNotOptimize.h"
+
#ifndef COMPILATION_TESTS
#include <android-base/silent_death_test.h>
#include <gtest/gtest.h>
@@ -133,6 +137,24 @@
const static int kBogusFD = -1;
+FORTIFY_TEST(strlen) {
+ auto run_strlen_with_contents = [&](std::array<char, 3> contents) {
+ // A lot of cruft is necessary to make this test DTRT. LLVM and Clang love to fold/optimize
+ // strlen calls, and that's the opposite of what we want to happen.
+
+ // Loop to convince LLVM that `contents` can never be known (since `xor volatile_value` can flip
+ // any bit in each elem of `contents`).
+ volatile char always_zero = 0;
+ for (char& c : contents) {
+ c ^= always_zero;
+ }
+ DoNotOptimize(strlen(&contents.front()));
+ };
+
+ EXPECT_NO_DEATH(run_strlen_with_contents({'f', 'o', '\0'}));
+ EXPECT_FORTIFY_DEATH(run_strlen_with_contents({'f', 'o', 'o'}));
+}
+
FORTIFY_TEST(string) {
char small_buffer[8] = {};
diff --git a/tests/complex_test.cpp b/tests/complex_test.cpp
index 8fdb2b2..ed0109a 100644
--- a/tests/complex_test.cpp
+++ b/tests/complex_test.cpp
@@ -20,6 +20,9 @@
#if !defined(__INTRODUCED_IN)
#define __INTRODUCED_IN(x)
#endif
+#if !defined(__BIONIC_AVAILABILITY_GUARD)
+#define __BIONIC_AVAILABILITY_GUARD(x) 1
+#endif
// libc++ actively gets in the way of including <complex.h> from C++, so we
// have to be naughty.
diff --git a/tests/cpu_target_features_test.cpp b/tests/cpu_target_features_test.cpp
new file mode 100644
index 0000000..d773772
--- /dev/null
+++ b/tests/cpu_target_features_test.cpp
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <gtest/gtest.h>
+#include <stdlib.h>
+
+#include "utils.h"
+
+TEST(cpu_target_features, has_expected_x86_compiler_values) {
+#if defined(__x86_64__) || defined(__i386__)
+ ExecTestHelper eth;
+ char* const argv[] = {nullptr};
+ const auto invocation = [&] { execvp("cpu-target-features", argv); };
+ eth.Run(invocation, 0, "(^|\n)__AES__=1($|\n)");
+ eth.Run(invocation, 0, "(^|\n)__CRC32__=1($|\n)");
+#else
+ GTEST_SKIP() << "Not targeting an x86 architecture.";
+#endif
+}
+
+TEST(cpu_target_features, has_expected_aarch64_compiler_values) {
+#if defined(__aarch64__)
+ ExecTestHelper eth;
+ char* const argv[] = {nullptr};
+ const auto invocation = [&] { execvp("cpu-target-features", argv); };
+ eth.Run(invocation, 0, "(^|\n)__ARM_FEATURE_AES=1($|\n)");
+ eth.Run(invocation, 0, "(^|\n)__ARM_FEATURE_CRC32=1($|\n)");
+#else
+ GTEST_SKIP() << "Not targeting an aarch64 architecture.";
+#endif
+}
diff --git a/tests/fenv_test.cpp b/tests/fenv_test.cpp
index 9cf9d98..bbf339f 100644
--- a/tests/fenv_test.cpp
+++ b/tests/fenv_test.cpp
@@ -16,6 +16,7 @@
#include <gtest/gtest.h>
+#include "DoNotOptimize.h"
#include "utils.h"
#include <fenv.h>
diff --git a/tests/headers/posix/stdatomic_h.c b/tests/headers/posix/stdatomic_h.c
new file mode 100644
index 0000000..05be859
--- /dev/null
+++ b/tests/headers/posix/stdatomic_h.c
@@ -0,0 +1,175 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <stdatomic.h>
+
+#include "header_checks.h"
+
+static void stdatomic_h() {
+ TYPE(atomic_flag);
+ TYPE(atomic_bool);
+ TYPE(atomic_char);
+ TYPE(atomic_schar);
+ TYPE(atomic_uchar);
+ TYPE(atomic_short);
+ TYPE(atomic_ushort);
+ TYPE(atomic_int);
+ TYPE(atomic_uint);
+ TYPE(atomic_long);
+ TYPE(atomic_ulong);
+ TYPE(atomic_llong);
+ TYPE(atomic_ullong);
+#if !defined(__GLIBC__)
+ TYPE(atomic_char16_t);
+ TYPE(atomic_char32_t);
+#endif
+ TYPE(atomic_wchar_t);
+ TYPE(atomic_int_least8_t);
+ TYPE(atomic_uint_least8_t);
+ TYPE(atomic_int_least16_t);
+ TYPE(atomic_uint_least16_t);
+ TYPE(atomic_int_least32_t);
+ TYPE(atomic_uint_least32_t);
+ TYPE(atomic_int_least64_t);
+ TYPE(atomic_uint_least64_t);
+ TYPE(atomic_int_fast8_t);
+ TYPE(atomic_uint_fast8_t);
+ TYPE(atomic_int_fast16_t);
+ TYPE(atomic_uint_fast16_t);
+ TYPE(atomic_int_fast32_t);
+ TYPE(atomic_uint_fast32_t);
+ TYPE(atomic_int_fast64_t);
+ TYPE(atomic_uint_fast64_t);
+ TYPE(atomic_intptr_t);
+ TYPE(atomic_uintptr_t);
+ TYPE(atomic_size_t);
+ TYPE(atomic_ptrdiff_t);
+ TYPE(atomic_intmax_t);
+ TYPE(atomic_uintmax_t);
+
+ memory_order m1 = memory_order_relaxed;
+ memory_order m2 = memory_order_consume;
+ memory_order m3 = memory_order_acquire;
+ memory_order m4 = memory_order_release;
+ memory_order m5 = memory_order_acq_rel;
+ memory_order m6 = memory_order_seq_cst;
+
+ MACRO(ATOMIC_BOOL_LOCK_FREE);
+ MACRO(ATOMIC_CHAR_LOCK_FREE);
+ MACRO(ATOMIC_CHAR16_T_LOCK_FREE);
+ MACRO(ATOMIC_CHAR32_T_LOCK_FREE);
+ MACRO(ATOMIC_WCHAR_T_LOCK_FREE);
+ MACRO(ATOMIC_SHORT_LOCK_FREE);
+ MACRO(ATOMIC_INT_LOCK_FREE);
+ MACRO(ATOMIC_LONG_LOCK_FREE);
+ MACRO(ATOMIC_LLONG_LOCK_FREE);
+ MACRO(ATOMIC_POINTER_LOCK_FREE);
+
+ atomic_flag f = ATOMIC_FLAG_INIT;
+ atomic_int i = ATOMIC_VAR_INIT(123);
+
+ // TODO: remove this #if after the next toolchain update (http://b/374104004).
+#if !defined(__GLIBC__)
+ i = kill_dependency(i);
+#endif
+
+#if !defined(atomic_compare_exchange_strong)
+#error atomic_compare_exchange_strong
+#endif
+#if !defined(atomic_compare_exchange_strong_explicit)
+#error atomic_compare_exchange_strong_explicit
+#endif
+#if !defined(atomic_compare_exchange_weak)
+#error atomic_compare_exchange_weak
+#endif
+#if !defined(atomic_compare_exchange_weak_explicit)
+#error atomic_compare_exchange_weak_explicit
+#endif
+#if !defined(atomic_exchange)
+#error atomic_exchange
+#endif
+#if !defined(atomic_exchange_explicit)
+#error atomic_exchange_explicit
+#endif
+#if !defined(atomic_fetch_add)
+#error atomic_fetch_add
+#endif
+#if !defined(atomic_fetch_add_explicit)
+#error atomic_fetch_add_explicit
+#endif
+#if !defined(atomic_fetch_and)
+#error atomic_fetch_and
+#endif
+#if !defined(atomic_fetch_and_explicit)
+#error atomic_fetch_and_explicit
+#endif
+#if !defined(atomic_fetch_or)
+#error atomic_fetch_or
+#endif
+#if !defined(atomic_fetch_or_explicit)
+#error atomic_fetch_or_explicit
+#endif
+#if !defined(atomic_fetch_sub)
+#error atomic_fetch_sub
+#endif
+#if !defined(atomic_fetch_sub_explicit)
+#error atomic_fetch_sub_explicit
+#endif
+#if !defined(atomic_fetch_xor)
+#error atomic_fetch_xor
+#endif
+#if !defined(atomic_fetch_xor_explicit)
+#error atomic_fetch_xor_explicit
+#endif
+#if !defined(atomic_init)
+#error atomic_init
+#endif
+#if !defined(atomic_is_lock_free)
+#error atomic_is_lock_free
+#endif
+#if !defined(atomic_load)
+#error atomic_load
+#endif
+#if !defined(atomic_load_explicit)
+#error atomic_load_explicit
+#endif
+#if !defined(atomic_store)
+#error atomic_store
+#endif
+#if !defined(atomic_store_explicit)
+#error atomic_store_explicit
+#endif
+
+ FUNCTION(atomic_flag_clear, void (*f)(volatile atomic_flag*));
+ FUNCTION(atomic_flag_clear_explicit, void (*f)(volatile atomic_flag*, memory_order));
+ FUNCTION(atomic_flag_test_and_set, bool (*f)(volatile atomic_flag*));
+ FUNCTION(atomic_flag_test_and_set_explicit, bool (*f)(volatile atomic_flag*, memory_order));
+
+ FUNCTION(atomic_signal_fence, void (*f)(memory_order));
+ FUNCTION(atomic_thread_fence, void (*f)(memory_order));
+}
diff --git a/tests/libs/Android.bp b/tests/libs/Android.bp
index 35f0f0c..d13ee60 100644
--- a/tests/libs/Android.bp
+++ b/tests/libs/Android.bp
@@ -48,6 +48,16 @@
}
// -----------------------------------------------------------------------------
+// Test library ELFs for linker page size related tests
+// -----------------------------------------------------------------------------
+cc_test_library {
+ name: "libtest_elf_max_page_size_4kib",
+ defaults: ["bionic_testlib_defaults"],
+ srcs: ["elf_max_page_size.c"],
+ ldflags: ["-z max-page-size=0x1000"],
+}
+
+// -----------------------------------------------------------------------------
// Libraries and helper binaries for ELF TLS
// -----------------------------------------------------------------------------
cc_test_library {
diff --git a/tests/libs/elf_max_page_size.c b/tests/libs/elf_max_page_size.c
new file mode 100644
index 0000000..24c7e89
--- /dev/null
+++ b/tests/libs/elf_max_page_size.c
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "elf_max_page_size.h"
+
+const int ro0 = RO0;
+const int ro1 = RO1;
+int rw0 = RW0;
+
+/* Force some padding alignment */
+int rw1 __attribute__((aligned(0x10000))) = RW1;
+
+int bss0, bss1;
+
+int* const prw0 = &rw0;
+
+int loader_test_func(void) {
+ rw0 += RW0_INCREMENT;
+ rw1 += RW1_INCREMENT;
+
+ bss0 += BSS0_INCREMENT;
+ bss1 += BSS1_INCREMENT;
+
+ return ro0 + ro1 + rw0 + rw1 + bss0 + bss1 + *prw0;
+}
diff --git a/tests/libs/elf_max_page_size.h b/tests/libs/elf_max_page_size.h
new file mode 100644
index 0000000..846a8b6
--- /dev/null
+++ b/tests/libs/elf_max_page_size.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#define RO0 23
+#define RO1 234
+#define RW0 2345
+#define RW1 23456
+#define BSS0 0
+#define BSS1 0
+
+#define RW0_INCREMENT 12
+#define RW1_INCREMENT 123
+#define BSS0_INCREMENT 1234
+#define BSS1_INCREMENT 12345
+
+#define TEST_RESULT_BASE (RO0 + RO1 + RW0 + RW1 + BSS0 + BSS1 + RW0)
+#define TEST_RESULT_INCREMENT \
+ (RW0_INCREMENT + RW1_INCREMENT + BSS0_INCREMENT + BSS1_INCREMENT + RW0_INCREMENT)
+
+typedef int (*loader_test_func_t)(void);
diff --git a/tests/malloc_stress_test.cpp b/tests/malloc_stress_test.cpp
index b5b06ab..00f5919 100644
--- a/tests/malloc_stress_test.cpp
+++ b/tests/malloc_stress_test.cpp
@@ -27,10 +27,58 @@
#include <thread>
#include <vector>
-#include <android-base/strings.h>
#if defined(__BIONIC__)
#include <meminfo/procmeminfo.h>
#include <procinfo/process_map.h>
+
+#include <log/log.h>
+#include <log/log_read.h>
+#endif
+
+#if defined(__BIONIC__)
+static void PrintLogStats(uint64_t& last_time) {
+ logger_list* logger =
+ android_logger_list_open(android_name_to_log_id("main"), ANDROID_LOG_NONBLOCK, 0, getpid());
+ if (logger == nullptr) {
+ printf("Failed to open log for main\n");
+ return;
+ }
+
+ uint64_t last_message_time = last_time;
+ while (true) {
+ log_msg entry;
+ ssize_t retval = android_logger_list_read(logger, &entry);
+ if (retval == 0) {
+ break;
+ }
+ if (retval < 0) {
+ if (retval == -EINTR) {
+ continue;
+ }
+ // EAGAIN means there is nothing left to read when ANDROID_LOG_NONBLOCK is set.
+ if (retval != -EAGAIN) {
+ printf("Failed to read log entry: %s\n", strerrordesc_np(retval));
+ }
+ break;
+ }
+ if (entry.msg() == nullptr) {
+ continue;
+ }
+ // Only print allocator tagged log entries.
+ std::string_view tag(entry.msg() + 1);
+ if (tag != "scudo" && tag != "jemalloc") {
+ continue;
+ }
+ if (entry.nsec() > last_time) {
+ printf(" %s\n", &tag.back() + 2);
+ // Only update the last time outside this loop just in case two or more
+ // messages have the same timestamp.
+ last_message_time = entry.nsec();
+ }
+ }
+ android_logger_list_close(logger);
+ last_time = last_message_time;
+}
#endif
TEST(malloc_stress, multiple_threads_forever) {
@@ -45,6 +93,8 @@
#endif
uint64_t mallinfo_min = UINT64_MAX;
uint64_t mallinfo_max = 0;
+
+ uint64_t last_message_time = 0;
for (size_t i = 0; ; i++) {
printf("Pass %zu\n", i);
@@ -74,8 +124,8 @@
uint64_t rss_bytes = 0;
uint64_t vss_bytes = 0;
for (auto& vma : maps) {
- if (vma.name == "[anon:libc_malloc]" || android::base::StartsWith(vma.name, "[anon:scudo:") ||
- android::base::StartsWith(vma.name, "[anon:GWP-ASan")) {
+ if (vma.name == "[anon:libc_malloc]" || vma.name.starts_with("[anon:scudo:") ||
+ vma.name.starts_with("[anon:GWP-ASan")) {
android::meminfo::Vma update_vma(vma);
ASSERT_TRUE(proc_mem.FillInVmaStats(update_vma));
rss_bytes += update_vma.usage.rss;
@@ -112,5 +162,15 @@
printf("Allocated memory %zu %0.2fMB\n", mallinfo_bytes, mallinfo_bytes / (1024.0 * 1024.0));
printf(" Min %" PRIu64 " %0.2fMB\n", mallinfo_min, mallinfo_min / (1024.0 * 1024.0));
printf(" Max %" PRIu64 " %0.2fMB\n", mallinfo_max, mallinfo_max / (1024.0 * 1024.0));
+
+#if defined(__BIONIC__)
+ if (((i + 1) % 100) == 0) {
+ // Send native allocator stats to the log
+ mallopt(M_LOG_STATS, 0);
+
+ printf("Log stats:\n");
+ PrintLogStats(last_message_time);
+ }
+#endif
}
}
diff --git a/tests/malloc_test.cpp b/tests/malloc_test.cpp
index 813f348..3f1ba79 100644
--- a/tests/malloc_test.cpp
+++ b/tests/malloc_test.cpp
@@ -47,6 +47,7 @@
#include <android-base/file.h>
#include <android-base/test_utils.h>
+#include "DoNotOptimize.h"
#include "utils.h"
#if defined(__BIONIC__)
diff --git a/tests/page_size_16kib_compat_test.cpp b/tests/page_size_16kib_compat_test.cpp
new file mode 100644
index 0000000..9aecfba
--- /dev/null
+++ b/tests/page_size_16kib_compat_test.cpp
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "page_size_compat_helpers.h"
+
+#include <android-base/properties.h>
+
+TEST(PageSize16KiBCompatTest, ElfAlignment4KiB_LoadElf) {
+ if (getpagesize() != 0x4000) {
+ GTEST_SKIP() << "This test is only applicable to 16kB page-size devices";
+ }
+
+ bool app_compat_enabled =
+ android::base::GetBoolProperty("bionic.linker.16kb.app_compat.enabled", false);
+ std::string lib = GetTestLibRoot() + "/libtest_elf_max_page_size_4kib.so";
+ void* handle = nullptr;
+
+ OpenTestLibrary(lib, !app_compat_enabled, &handle);
+
+ if (app_compat_enabled) CallTestFunction(handle);
+}
diff --git a/tests/page_size_compat_helpers.h b/tests/page_size_compat_helpers.h
new file mode 100644
index 0000000..2f0f1d0
--- /dev/null
+++ b/tests/page_size_compat_helpers.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "elf_max_page_size.h"
+#include "gtest_globals.h"
+
+#include <android-base/stringprintf.h>
+
+#include <string>
+
+#include <dlfcn.h>
+#include <gtest/gtest.h>
+#include <unistd.h>
+
+static inline void OpenTestLibrary(std::string lib, bool expect_fail, void** handle) {
+ void* _handle = dlopen(lib.c_str(), RTLD_NODELETE);
+ const char* dlopen_error = dlerror();
+
+ if (expect_fail) {
+ ASSERT_EQ(_handle, nullptr);
+
+ const std::string expected_error = android::base::StringPrintf(
+ "dlopen failed: \"%s\" program alignment (%d) cannot be smaller than system page size (%d)",
+ lib.c_str(), 4096, getpagesize());
+
+ ASSERT_EQ(expected_error, dlopen_error);
+ } else {
+ ASSERT_NE(_handle, nullptr) << "Failed to dlopen shared library \"" << lib
+ << "\": " << dlopen_error;
+ }
+
+ *handle = _handle;
+}
+
+static inline void CallTestFunction(void* handle) {
+ loader_test_func_t loader_test_func = (loader_test_func_t)dlsym(handle, "loader_test_func");
+ const char* dlsym_error = dlerror();
+
+ ASSERT_EQ(dlsym_error, nullptr) << "Failed to locate symbol \"loader_test_func\": "
+ << dlsym_error;
+
+ int res = loader_test_func();
+ ASSERT_EQ(res, TEST_RESULT_BASE + TEST_RESULT_INCREMENT);
+
+ // Call loader_test_func() twice to ensure we can modify writeable data and bss data
+ res = loader_test_func();
+ ASSERT_EQ(res, TEST_RESULT_BASE + (2 * TEST_RESULT_INCREMENT));
+}
diff --git a/tests/prebuilt-elf-files/arm64/libtest_invalid-empty_shdr_table.so b/tests/prebuilt-elf-files/arm64/libtest_invalid-empty_shdr_table.so
index c8b5430..21a8f26 100755
--- a/tests/prebuilt-elf-files/arm64/libtest_invalid-empty_shdr_table.so
+++ b/tests/prebuilt-elf-files/arm64/libtest_invalid-empty_shdr_table.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/arm64/libtest_invalid-local-tls.so b/tests/prebuilt-elf-files/arm64/libtest_invalid-local-tls.so
index 20c5765..c902bbe 100755
--- a/tests/prebuilt-elf-files/arm64/libtest_invalid-local-tls.so
+++ b/tests/prebuilt-elf-files/arm64/libtest_invalid-local-tls.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/arm64/libtest_invalid-unaligned_shdr_offset.so b/tests/prebuilt-elf-files/arm64/libtest_invalid-unaligned_shdr_offset.so
index 6e5a6e3..fb86bca 100755
--- a/tests/prebuilt-elf-files/arm64/libtest_invalid-unaligned_shdr_offset.so
+++ b/tests/prebuilt-elf-files/arm64/libtest_invalid-unaligned_shdr_offset.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/arm64/libtest_invalid-zero_shdr_table_content.so b/tests/prebuilt-elf-files/arm64/libtest_invalid-zero_shdr_table_content.so
index 14b80b5..0416db2 100755
--- a/tests/prebuilt-elf-files/arm64/libtest_invalid-zero_shdr_table_content.so
+++ b/tests/prebuilt-elf-files/arm64/libtest_invalid-zero_shdr_table_content.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/arm64/libtest_invalid-zero_shdr_table_offset.so b/tests/prebuilt-elf-files/arm64/libtest_invalid-zero_shdr_table_offset.so
index 0aaca72..90892a6 100755
--- a/tests/prebuilt-elf-files/arm64/libtest_invalid-zero_shdr_table_offset.so
+++ b/tests/prebuilt-elf-files/arm64/libtest_invalid-zero_shdr_table_offset.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/gen-libtest_invalid-local-tls.sh b/tests/prebuilt-elf-files/gen-libtest_invalid-local-tls.sh
index 0f3e736..98a2b00 100755
--- a/tests/prebuilt-elf-files/gen-libtest_invalid-local-tls.sh
+++ b/tests/prebuilt-elf-files/gen-libtest_invalid-local-tls.sh
@@ -19,12 +19,18 @@
build() {
arch=$1
target=$2
+
+ if [[ "$arch" == "arm64" || "$arch" == "x86_64" ]]; then
+ alignment="-Wl,-z,max-page-size=16384"
+ fi
+
$NDK21E/toolchains/llvm/prebuilt/linux-x86_64/bin/clang -O2 --target=$target \
-fpic -shared -o $arch/libtest_invalid-local-tls.so -fno-emulated-tls \
- -fuse-ld=gold test.c test2.c
+ $alignment -fuse-ld=gold test.c test2.c
}
build arm armv7a-linux-androideabi29
build arm64 aarch64-linux-android29
build x86 i686-linux-android29
build x86_64 x86_64-linux-android29
+
diff --git a/tests/prebuilt-elf-files/x86_64/libtest_invalid-empty_shdr_table.so b/tests/prebuilt-elf-files/x86_64/libtest_invalid-empty_shdr_table.so
index af1538d..00fefd4 100755
--- a/tests/prebuilt-elf-files/x86_64/libtest_invalid-empty_shdr_table.so
+++ b/tests/prebuilt-elf-files/x86_64/libtest_invalid-empty_shdr_table.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/x86_64/libtest_invalid-local-tls.so b/tests/prebuilt-elf-files/x86_64/libtest_invalid-local-tls.so
index 5b689ba..31d2b37 100755
--- a/tests/prebuilt-elf-files/x86_64/libtest_invalid-local-tls.so
+++ b/tests/prebuilt-elf-files/x86_64/libtest_invalid-local-tls.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/x86_64/libtest_invalid-unaligned_shdr_offset.so b/tests/prebuilt-elf-files/x86_64/libtest_invalid-unaligned_shdr_offset.so
index 87631af..f9c310f 100755
--- a/tests/prebuilt-elf-files/x86_64/libtest_invalid-unaligned_shdr_offset.so
+++ b/tests/prebuilt-elf-files/x86_64/libtest_invalid-unaligned_shdr_offset.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/x86_64/libtest_invalid-zero_shdr_table_content.so b/tests/prebuilt-elf-files/x86_64/libtest_invalid-zero_shdr_table_content.so
index 27d1138..3d1f5d3 100755
--- a/tests/prebuilt-elf-files/x86_64/libtest_invalid-zero_shdr_table_content.so
+++ b/tests/prebuilt-elf-files/x86_64/libtest_invalid-zero_shdr_table_content.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/x86_64/libtest_invalid-zero_shdr_table_offset.so b/tests/prebuilt-elf-files/x86_64/libtest_invalid-zero_shdr_table_offset.so
index 3e2c1d1..aeea1d2 100755
--- a/tests/prebuilt-elf-files/x86_64/libtest_invalid-zero_shdr_table_offset.so
+++ b/tests/prebuilt-elf-files/x86_64/libtest_invalid-zero_shdr_table_offset.so
Binary files differ
diff --git a/tests/stdatomic_test.cpp b/tests/stdatomic_test.cpp
index f5c6bb1..1c51b11 100644
--- a/tests/stdatomic_test.cpp
+++ b/tests/stdatomic_test.cpp
@@ -37,7 +37,7 @@
}
TEST(stdatomic, init) {
- atomic_int v = ATOMIC_VAR_INIT(123);
+ atomic_int v = 123;
ASSERT_EQ(123, atomic_load(&v));
atomic_init(&v, 456);
@@ -145,35 +145,35 @@
}
TEST(stdatomic, atomic_fetch_add) {
- atomic_int i = ATOMIC_VAR_INIT(123);
+ atomic_int i = 123;
ASSERT_EQ(123, atomic_fetch_add(&i, 1));
ASSERT_EQ(124, atomic_fetch_add_explicit(&i, 1, memory_order_relaxed));
ASSERT_EQ(125, atomic_load(&i));
}
TEST(stdatomic, atomic_fetch_sub) {
- atomic_int i = ATOMIC_VAR_INIT(123);
+ atomic_int i = 123;
ASSERT_EQ(123, atomic_fetch_sub(&i, 1));
ASSERT_EQ(122, atomic_fetch_sub_explicit(&i, 1, memory_order_relaxed));
ASSERT_EQ(121, atomic_load(&i));
}
TEST(stdatomic, atomic_fetch_or) {
- atomic_int i = ATOMIC_VAR_INIT(0x100);
+ atomic_int i = 0x100;
ASSERT_EQ(0x100, atomic_fetch_or(&i, 0x020));
ASSERT_EQ(0x120, atomic_fetch_or_explicit(&i, 0x003, memory_order_relaxed));
ASSERT_EQ(0x123, atomic_load(&i));
}
TEST(stdatomic, atomic_fetch_xor) {
- atomic_int i = ATOMIC_VAR_INIT(0x100);
+ atomic_int i = 0x100;
ASSERT_EQ(0x100, atomic_fetch_xor(&i, 0x120));
ASSERT_EQ(0x020, atomic_fetch_xor_explicit(&i, 0x103, memory_order_relaxed));
ASSERT_EQ(0x123, atomic_load(&i));
}
TEST(stdatomic, atomic_fetch_and) {
- atomic_int i = ATOMIC_VAR_INIT(0x123);
+ atomic_int i = 0x123;
ASSERT_EQ(0x123, atomic_fetch_and(&i, 0x00f));
ASSERT_EQ(0x003, atomic_fetch_and_explicit(&i, 0x2, memory_order_relaxed));
ASSERT_EQ(0x002, atomic_load(&i));
diff --git a/tests/string_test.cpp b/tests/string_test.cpp
index 6e1fcfc..502405f 100644
--- a/tests/string_test.cpp
+++ b/tests/string_test.cpp
@@ -740,15 +740,11 @@
// Set the second half of ptr to the expected pattern in ptr2.
memset(state.ptr + state.MAX_LEN, '\1', state.MAX_LEN);
memcpy(state.ptr + state.MAX_LEN, state.ptr1, copy_len);
- size_t expected_end;
if (copy_len > ptr1_len) {
memset(state.ptr + state.MAX_LEN + ptr1_len, '\0', copy_len - ptr1_len);
- expected_end = ptr1_len;
- } else {
- expected_end = copy_len;
}
- ASSERT_EQ(state.ptr2 + expected_end, stpncpy(state.ptr2, state.ptr1, copy_len));
+ ASSERT_EQ(state.ptr2, strncpy(state.ptr2, state.ptr1, copy_len));
// Verify ptr1 was not modified.
ASSERT_EQ(0, memcmp(state.ptr1, state.ptr, state.MAX_LEN));
diff --git a/tests/unistd_test.cpp b/tests/unistd_test.cpp
index 78b55c1..9ad3b6d 100644
--- a/tests/unistd_test.cpp
+++ b/tests/unistd_test.cpp
@@ -16,6 +16,7 @@
#include <gtest/gtest.h>
+#include "DoNotOptimize.h"
#include "SignalUtils.h"
#include "utils.h"
diff --git a/tests/utils.h b/tests/utils.h
index 3c83b73..4740e59 100644
--- a/tests/utils.h
+++ b/tests/utils.h
@@ -295,16 +295,6 @@
size_t start_count_ = CountOpenFds();
};
-// From <benchmark/benchmark.h>.
-template <class Tp>
-static inline void DoNotOptimize(Tp const& value) {
- asm volatile("" : : "r,m"(value) : "memory");
-}
-template <class Tp>
-static inline void DoNotOptimize(Tp& value) {
- asm volatile("" : "+r,m"(value) : : "memory");
-}
-
static inline bool running_with_mte() {
#ifdef __aarch64__
int level = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);