diff --git a/libc/Android.bp b/libc/Android.bp
index 1bf7ac3..ce60cef 100644
--- a/libc/Android.bp
+++ b/libc/Android.bp
@@ -418,20 +418,6 @@
         "upstream-freebsd/lib/libc/string/wmemmove.c",
         "upstream-freebsd/lib/libc/string/wmemset.c",
     ],
-    arch: {
-        x86: {
-            exclude_srcs: [
-                "upstream-freebsd/lib/libc/string/wcschr.c",
-                "upstream-freebsd/lib/libc/string/wcscmp.c",
-                "upstream-freebsd/lib/libc/string/wcslen.c",
-                "upstream-freebsd/lib/libc/string/wcsrchr.c",
-                "upstream-freebsd/lib/libc/string/wmemcmp.c",
-                "upstream-freebsd/lib/libc/string/wcscat.c",
-                "upstream-freebsd/lib/libc/string/wcscpy.c",
-                "upstream-freebsd/lib/libc/string/wmemcmp.c",
-            ],
-        },
-    },
 
     cflags: [
         "-Wno-sign-compare",
@@ -1181,12 +1167,9 @@
                 "arch-x86/bionic/vfork.S",
                 "arch-x86/bionic/__x86.get_pc_thunk.S",
 
-                "arch-x86/generic/string/memcmp.S",
-
                 "arch-x86/string/sse2-memchr-atom.S",
                 "arch-x86/string/sse2-memmove-slm.S",
                 "arch-x86/string/sse2-memrchr-atom.S",
-                "arch-x86/string/sse2-memset-atom.S",
                 "arch-x86/string/sse2-memset-slm.S",
                 "arch-x86/string/sse2-stpcpy-slm.S",
                 "arch-x86/string/sse2-stpncpy-slm.S",
@@ -1196,28 +1179,16 @@
                 "arch-x86/string/sse2-strncpy-slm.S",
                 "arch-x86/string/sse2-strnlen-atom.S",
                 "arch-x86/string/sse2-strrchr-atom.S",
-                "arch-x86/string/sse2-wcschr-atom.S",
-                "arch-x86/string/sse2-wcsrchr-atom.S",
-                "arch-x86/string/sse2-wcslen-atom.S",
-                "arch-x86/string/sse2-wcscmp-atom.S",
-                "arch-x86/string/sse2-strlen-atom.S",
 
                 "arch-x86/string/ssse3-memcmp-atom.S",
-                "arch-x86/string/ssse3-memmove-atom.S",
                 "arch-x86/string/ssse3-strcat-atom.S",
                 "arch-x86/string/ssse3-strcmp-atom.S",
-                "arch-x86/string/ssse3-strcpy-atom.S",
                 "arch-x86/string/ssse3-strlcat-atom.S",
                 "arch-x86/string/ssse3-strlcpy-atom.S",
                 "arch-x86/string/ssse3-strncat-atom.S",
                 "arch-x86/string/ssse3-strncmp-atom.S",
-                "arch-x86/string/ssse3-strncpy-atom.S",
-                "arch-x86/string/ssse3-wcscat-atom.S",
-                "arch-x86/string/ssse3-wcscpy-atom.S",
-                "arch-x86/string/ssse3-wmemcmp-atom.S",
 
                 "arch-x86/string/sse4-memcmp-slm.S",
-                "arch-x86/string/sse4-wmemcmp-slm.S",
 
                 "bionic/strchrnul.cpp",
             ],
@@ -2026,15 +1997,6 @@
 cc_defaults {
     name: "crt_so_defaults",
     defaults: ["crt_defaults"],
-
-    arch: {
-        x86: {
-            cflags: ["-fPIC"],
-        },
-        x86_64: {
-            cflags: ["-fPIC"],
-        },
-    },
     stl: "none",
 }
 
diff --git a/libc/NOTICE b/libc/NOTICE
index 1a84d3c..1fa036a 100644
--- a/libc/NOTICE
+++ b/libc/NOTICE
@@ -3862,36 +3862,6 @@
 
 -------------------------------------------------------------------
 
-Copyright (c) 2011 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
--------------------------------------------------------------------
-
 Copyright (c) 2011 Martin Pieuchot <mpi@openbsd.org>
 
 Permission to use, copy, modify, and distribute this software for any
@@ -3937,36 +3907,6 @@
 
 -------------------------------------------------------------------
 
-Copyright (c) 2011, 2012, 2013 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
--------------------------------------------------------------------
-
 Copyright (c) 2011, Intel Corporation
 All rights reserved.
 
diff --git a/libc/arch-x86/dynamic_function_dispatch.cpp b/libc/arch-x86/dynamic_function_dispatch.cpp
index e6cc5fb..240fcdf 100644
--- a/libc/arch-x86/dynamic_function_dispatch.cpp
+++ b/libc/arch-x86/dynamic_function_dispatch.cpp
@@ -33,65 +33,9 @@
 
 DEFINE_IFUNC_FOR(memcmp) {
   __builtin_cpu_init();
-  if (__builtin_cpu_is("atom")) RETURN_FUNC(memcmp_func_t, memcmp_atom);
   if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(memcmp_func_t, memcmp_sse4);
-  RETURN_FUNC(memcmp_func_t, memcmp_generic);
+  RETURN_FUNC(memcmp_func_t, memcmp_atom);
 }
 MEMCMP_SHIM()
 
-DEFINE_IFUNC_FOR(memset) {
-  __builtin_cpu_init();
-  if (__builtin_cpu_is("atom")) RETURN_FUNC(memset_func_t, memset_atom);
-  RETURN_FUNC(memset_func_t, memset_generic);
-}
-MEMSET_SHIM()
-
-DEFINE_IFUNC_FOR(__memset_chk) {
-  __builtin_cpu_init();
-  if (__builtin_cpu_is("atom")) RETURN_FUNC(__memset_chk_func_t, __memset_chk_atom);
-  RETURN_FUNC(__memset_chk_func_t, __memset_chk_generic);
-}
-__MEMSET_CHK_SHIM()
-
-DEFINE_IFUNC_FOR(memmove) {
-  __builtin_cpu_init();
-  if (__builtin_cpu_is("atom")) RETURN_FUNC(memmove_func_t, memmove_atom);
-  RETURN_FUNC(memmove_func_t, memmove_generic);
-}
-MEMMOVE_SHIM()
-
-DEFINE_IFUNC_FOR(memcpy) { return memmove_resolver(); }
-MEMCPY_SHIM()
-
-DEFINE_IFUNC_FOR(strcpy) {
-  __builtin_cpu_init();
-  if (__builtin_cpu_is("atom")) RETURN_FUNC(strcpy_func_t, strcpy_atom);
-  RETURN_FUNC(strcpy_func_t, strcpy_generic);
-}
-STRCPY_SHIM()
-
-DEFINE_IFUNC_FOR(strncpy) {
-  __builtin_cpu_init();
-  if (__builtin_cpu_is("atom")) RETURN_FUNC(strncpy_func_t, strncpy_atom);
-  RETURN_FUNC(strncpy_func_t, strncpy_generic);
-}
-STRNCPY_SHIM()
-
-DEFINE_IFUNC_FOR(strlen) {
-  __builtin_cpu_init();
-  if (__builtin_cpu_is("atom")) RETURN_FUNC(strlen_func_t, strlen_atom);
-  RETURN_FUNC(strlen_func_t, strlen_generic);
-}
-STRLEN_SHIM()
-
-typedef int wmemcmp_func_t(const wchar_t*, const wchar_t*, size_t);
-DEFINE_IFUNC_FOR(wmemcmp) {
-  __builtin_cpu_init();
-  if (__builtin_cpu_supports("sse4.1")) RETURN_FUNC(wmemcmp_func_t, wmemcmp_sse4);
-  RETURN_FUNC(wmemcmp_func_t, wmemcmp_atom);
-}
-DEFINE_STATIC_SHIM(int wmemcmp(const wchar_t* lhs, const wchar_t* rhs, size_t n) {
-  FORWARD(wmemcmp)(lhs, rhs, n);
-})
-
 }  // extern "C"
diff --git a/libc/arch-x86/generic/string/memcmp.S b/libc/arch-x86/generic/string/memcmp.S
deleted file mode 100644
index 1d327c7..0000000
--- a/libc/arch-x86/generic/string/memcmp.S
+++ /dev/null
@@ -1,44 +0,0 @@
-/*	$OpenBSD: memcmp.S,v 1.4 2005/08/07 11:30:38 espie Exp $ */
-/*
- * Written by J.T. Conklin <jtc@netbsd.org>.
- * Public domain.
- */
-
-#include <private/bionic_asm.h>
-
-ENTRY(memcmp_generic)
-	pushl	%edi
-	pushl	%esi
-	movl	12(%esp),%edi
-	movl	16(%esp),%esi
-	cld				/* set compare direction forward */
-
-	movl	20(%esp),%ecx		/* compare by words */
-	shrl	$2,%ecx
-	repe
-	cmpsl
-	jne	L5			/* do we match so far? */
-
-	movl	20(%esp),%ecx		/* compare remainder by bytes */
-	andl	$3,%ecx
-	repe
-	cmpsb
-	jne	L6			/* do we match? */
-
-	xorl	%eax,%eax		/* we match, return zero	*/
-	popl	%esi
-	popl	%edi
-	ret
-
-L5:	movl	$4,%ecx			/* We know that one of the next	*/
-	subl	%ecx,%edi		/* four pairs of bytes do not	*/
-	subl	%ecx,%esi		/* match.			*/
-	repe
-	cmpsb
-L6:	movzbl  -1(%edi),%eax		/* Perform unsigned comparison	*/
-	movzbl	-1(%esi),%edx
-	subl	%edx,%eax
-	popl	%esi
-	popl	%edi
-	ret
-END(memcmp_generic)
diff --git a/libc/arch-x86/string/sse2-memmove-slm.S b/libc/arch-x86/string/sse2-memmove-slm.S
index 7f42374..2ed4e7b 100644
--- a/libc/arch-x86/string/sse2-memmove-slm.S
+++ b/libc/arch-x86/string/sse2-memmove-slm.S
@@ -31,7 +31,7 @@
 #define FOR_SILVERMONT
 
 #ifndef MEMMOVE
-# define MEMMOVE	memmove_generic
+# define MEMMOVE	memmove
 #endif
 
 #ifndef L
@@ -551,3 +551,9 @@
 	jmp	L(mm_recalc_len)
 
 END (MEMMOVE)
+
+// N.B., `private/bionic_asm.h` provides ALIAS_SYMBOL, but that file provides
+// conflicting definitions for some macros in this file. Since ALIAS_SYMBOL is
+// small, inline it here.
+.globl memcpy;
+.equ memcpy, MEMMOVE
diff --git a/libc/arch-x86/string/sse2-memset-atom.S b/libc/arch-x86/string/sse2-memset-atom.S
deleted file mode 100644
index e43ead0..0000000
--- a/libc/arch-x86/string/sse2-memset-atom.S
+++ /dev/null
@@ -1,841 +0,0 @@
-/*
-Copyright (c) 2010, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#include <private/bionic_asm.h>
-
-#define FOR_ATOM
-
-#ifndef L
-# define L(label)	.L##label
-#endif
-
-#ifndef ALIGN
-# define ALIGN(n)	.p2align n
-#endif
-
-#define CFI_PUSH(REG)						\
-  .cfi_adjust_cfa_offset 4;					\
-  .cfi_rel_offset REG, 0
-
-#define CFI_POP(REG)						\
-  .cfi_adjust_cfa_offset -4;					\
-  .cfi_restore REG
-
-#define PUSH(REG)	pushl REG; CFI_PUSH(REG)
-#define POP(REG)	popl REG; CFI_POP(REG)
-
-#define PARMS 8  /* Preserve EBX. */
-#define DST PARMS
-#define CHR (DST+4)
-#define LEN (CHR+4)
-#define CHK_DST_LEN (LEN+4)
-#define SETRTNVAL	movl DST(%esp), %eax
-
-#define ENTRANCE	PUSH(%ebx);
-#define RETURN_END	POP(%ebx); ret
-#define RETURN		RETURN_END; CFI_PUSH(%ebx)
-#define JMPTBL(I, B)	I - B
-
-#define SETUP_PIC_REG(x)	call	__x86.get_pc_thunk.x
-
-/* Load an entry in a jump table into EBX and branch to it.  TABLE is a
-   jump table with relative offsets.   */
-# define BRANCH_TO_JMPTBL_ENTRY(TABLE)				\
-    /* We first load PC into EBX.  */				\
-    call	__x86.get_pc_thunk.bx;				\
-    /* Get the address of the jump table.  */			\
-    add		$(TABLE - .), %ebx;				\
-    /* Get the entry and convert the relative offset to the	\
-       absolute address.  */					\
-    add		(%ebx,%ecx,4), %ebx;				\
-    add		%ecx, %edx;					\
-    /* We loaded the jump table and adjusted EDX. Go.  */	\
-    jmp		*%ebx
-
-ENTRY(__memset_chk_atom)
-  ENTRANCE
-
-  movl LEN(%esp), %ecx
-  cmpl CHK_DST_LEN(%esp), %ecx
-  jna L(memset_length_loaded)
-
-  POP(%ebx) // Undo ENTRANCE without returning.
-  jmp __memset_chk_fail
-END(__memset_chk_atom)
-
-	.section .text.sse2,"ax",@progbits
-	ALIGN(4)
-ENTRY(memset_atom)
-	ENTRANCE
-
-	movl	LEN(%esp), %ecx
-L(memset_length_loaded):
-	movzbl	CHR(%esp), %eax
-	movb	%al, %ah
-	/* Fill the whole EAX with pattern.  */
-	movl	%eax, %edx
-	shl	$16, %eax
-	or	%edx, %eax
-	movl	DST(%esp), %edx
-	cmp	$32, %ecx
-	jae	L(32bytesormore)
-
-L(write_less32bytes):
-	BRANCH_TO_JMPTBL_ENTRY(L(table_less_32bytes))
-
-
-	.pushsection .rodata.sse2,"a",@progbits
-	ALIGN(2)
-L(table_less_32bytes):
-	.int	JMPTBL(L(write_0bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_1bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_2bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_3bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_4bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_5bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_6bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_7bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_8bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_9bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_10bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_11bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_12bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_13bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_14bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_15bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_16bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_17bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_18bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_19bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_20bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_21bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_22bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_23bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_24bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_25bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_26bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_27bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_28bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_29bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_30bytes), L(table_less_32bytes))
-	.int	JMPTBL(L(write_31bytes), L(table_less_32bytes))
-	.popsection
-
-	ALIGN(4)
-L(write_28bytes):
-	movl	%eax, -28(%edx)
-L(write_24bytes):
-	movl	%eax, -24(%edx)
-L(write_20bytes):
-	movl	%eax, -20(%edx)
-L(write_16bytes):
-	movl	%eax, -16(%edx)
-L(write_12bytes):
-	movl	%eax, -12(%edx)
-L(write_8bytes):
-	movl	%eax, -8(%edx)
-L(write_4bytes):
-	movl	%eax, -4(%edx)
-L(write_0bytes):
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(write_29bytes):
-	movl	%eax, -29(%edx)
-L(write_25bytes):
-	movl	%eax, -25(%edx)
-L(write_21bytes):
-	movl	%eax, -21(%edx)
-L(write_17bytes):
-	movl	%eax, -17(%edx)
-L(write_13bytes):
-	movl	%eax, -13(%edx)
-L(write_9bytes):
-	movl	%eax, -9(%edx)
-L(write_5bytes):
-	movl	%eax, -5(%edx)
-L(write_1bytes):
-	movb	%al, -1(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(write_30bytes):
-	movl	%eax, -30(%edx)
-L(write_26bytes):
-	movl	%eax, -26(%edx)
-L(write_22bytes):
-	movl	%eax, -22(%edx)
-L(write_18bytes):
-	movl	%eax, -18(%edx)
-L(write_14bytes):
-	movl	%eax, -14(%edx)
-L(write_10bytes):
-	movl	%eax, -10(%edx)
-L(write_6bytes):
-	movl	%eax, -6(%edx)
-L(write_2bytes):
-	movw	%ax, -2(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(write_31bytes):
-	movl	%eax, -31(%edx)
-L(write_27bytes):
-	movl	%eax, -27(%edx)
-L(write_23bytes):
-	movl	%eax, -23(%edx)
-L(write_19bytes):
-	movl	%eax, -19(%edx)
-L(write_15bytes):
-	movl	%eax, -15(%edx)
-L(write_11bytes):
-	movl	%eax, -11(%edx)
-L(write_7bytes):
-	movl	%eax, -7(%edx)
-L(write_3bytes):
-	movw	%ax, -3(%edx)
-	movb	%al, -1(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-/* ECX > 32 and EDX is 4 byte aligned.  */
-L(32bytesormore):
-	/* Fill xmm0 with the pattern.  */
-	movd	%eax, %xmm0
-	pshufd	$0, %xmm0, %xmm0
-	testl	$0xf, %edx
-	jz	L(aligned_16)
-/* ECX > 32 and EDX is not 16 byte aligned.  */
-L(not_aligned_16):
-	movdqu	%xmm0, (%edx)
-	movl	%edx, %eax
-	and	$-16, %edx
-	add	$16, %edx
-	sub	%edx, %eax
-	add	%eax, %ecx
-	movd	%xmm0, %eax
-
-	ALIGN(4)
-L(aligned_16):
-	cmp	$128, %ecx
-	jae	L(128bytesormore)
-
-L(aligned_16_less128bytes):
-	BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
-
-	ALIGN(4)
-L(128bytesormore):
-	PUSH(%ebx)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	mov	__x86_shared_cache_size@GOTOFF(%ebx), %ebx
-	cmp	%ebx, %ecx
-	jae	L(128bytesormore_nt_start)
-
-
-	POP(%ebx)
-# define RESTORE_EBX_STATE CFI_PUSH(%ebx)
-	PUSH(%ebx)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size@GOTOFF(%ebx), %ecx
-	POP(%ebx)
-
-	jae	L(128bytes_L2_normal)
-	subl	$128, %ecx
-L(128bytesormore_normal):
-	sub	$128, %ecx
-	movdqa	%xmm0, (%edx)
-	movdqa	%xmm0, 0x10(%edx)
-	movdqa	%xmm0, 0x20(%edx)
-	movdqa	%xmm0, 0x30(%edx)
-	movdqa	%xmm0, 0x40(%edx)
-	movdqa	%xmm0, 0x50(%edx)
-	movdqa	%xmm0, 0x60(%edx)
-	movdqa	%xmm0, 0x70(%edx)
-	lea	128(%edx), %edx
-	jb	L(128bytesless_normal)
-
-
-	sub	$128, %ecx
-	movdqa	%xmm0, (%edx)
-	movdqa	%xmm0, 0x10(%edx)
-	movdqa	%xmm0, 0x20(%edx)
-	movdqa	%xmm0, 0x30(%edx)
-	movdqa	%xmm0, 0x40(%edx)
-	movdqa	%xmm0, 0x50(%edx)
-	movdqa	%xmm0, 0x60(%edx)
-	movdqa	%xmm0, 0x70(%edx)
-	lea	128(%edx), %edx
-	jae	L(128bytesormore_normal)
-
-L(128bytesless_normal):
-	add	$128, %ecx
-	BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
-
-	ALIGN(4)
-L(128bytes_L2_normal):
-	prefetcht0	0x380(%edx)
-	prefetcht0	0x3c0(%edx)
-	sub	$128, %ecx
-	movdqa	%xmm0, (%edx)
-	movaps	%xmm0, 0x10(%edx)
-	movaps	%xmm0, 0x20(%edx)
-	movaps	%xmm0, 0x30(%edx)
-	movaps	%xmm0, 0x40(%edx)
-	movaps	%xmm0, 0x50(%edx)
-	movaps	%xmm0, 0x60(%edx)
-	movaps	%xmm0, 0x70(%edx)
-	add	$128, %edx
-	cmp	$128, %ecx
-	jae	L(128bytes_L2_normal)
-
-L(128bytesless_L2_normal):
-	BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
-
-	RESTORE_EBX_STATE
-L(128bytesormore_nt_start):
-	sub	%ebx, %ecx
-	mov	%ebx, %eax
-	and	$0x7f, %eax
-	add	%eax, %ecx
-	movd	%xmm0, %eax
-	ALIGN(4)
-L(128bytesormore_shared_cache_loop):
-	prefetcht0	0x3c0(%edx)
-	prefetcht0	0x380(%edx)
-	sub	$0x80, %ebx
-	movdqa	%xmm0, (%edx)
-	movdqa	%xmm0, 0x10(%edx)
-	movdqa	%xmm0, 0x20(%edx)
-	movdqa	%xmm0, 0x30(%edx)
-	movdqa	%xmm0, 0x40(%edx)
-	movdqa	%xmm0, 0x50(%edx)
-	movdqa	%xmm0, 0x60(%edx)
-	movdqa	%xmm0, 0x70(%edx)
-	add	$0x80, %edx
-	cmp	$0x80, %ebx
-	jae	L(128bytesormore_shared_cache_loop)
-	cmp	$0x80, %ecx
-	jb	L(shared_cache_loop_end)
-	ALIGN(4)
-L(128bytesormore_nt):
-	sub	$0x80, %ecx
-	movntdq	%xmm0, (%edx)
-	movntdq	%xmm0, 0x10(%edx)
-	movntdq	%xmm0, 0x20(%edx)
-	movntdq	%xmm0, 0x30(%edx)
-	movntdq	%xmm0, 0x40(%edx)
-	movntdq	%xmm0, 0x50(%edx)
-	movntdq	%xmm0, 0x60(%edx)
-	movntdq	%xmm0, 0x70(%edx)
-	add	$0x80, %edx
-	cmp	$0x80, %ecx
-	jae	L(128bytesormore_nt)
-	sfence
-L(shared_cache_loop_end):
-	POP(%ebx)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
-
-
-	.pushsection .rodata.sse2,"a",@progbits
-	ALIGN(2)
-L(table_16_128bytes):
-	.int	JMPTBL(L(aligned_16_0bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_1bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_2bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_3bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_4bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_5bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_6bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_7bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_8bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_9bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_10bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_11bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_12bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_13bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_14bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_15bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_16bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_17bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_18bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_19bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_20bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_21bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_22bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_23bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_24bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_25bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_26bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_27bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_28bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_29bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_30bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_31bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_32bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_33bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_34bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_35bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_36bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_37bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_38bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_39bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_40bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_41bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_42bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_43bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_44bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_45bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_46bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_47bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_48bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_49bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_50bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_51bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_52bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_53bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_54bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_55bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_56bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_57bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_58bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_59bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_60bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_61bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_62bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_63bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_64bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_65bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_66bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_67bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_68bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_69bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_70bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_71bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_72bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_73bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_74bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_75bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_76bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_77bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_78bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_79bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_80bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_81bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_82bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_83bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_84bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_85bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_86bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_87bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_88bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_89bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_90bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_91bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_92bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_93bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_94bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_95bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_96bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_97bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_98bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_99bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_100bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_101bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_102bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_103bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_104bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_105bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_106bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_107bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_108bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_109bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_110bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_111bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_112bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_113bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_114bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_115bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_116bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_117bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_118bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_119bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_120bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_121bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_122bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_123bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_124bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_125bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_126bytes), L(table_16_128bytes))
-	.int	JMPTBL(L(aligned_16_127bytes), L(table_16_128bytes))
-	.popsection
-
-	ALIGN(4)
-L(aligned_16_112bytes):
-	movdqa	%xmm0, -112(%edx)
-L(aligned_16_96bytes):
-	movdqa	%xmm0, -96(%edx)
-L(aligned_16_80bytes):
-	movdqa	%xmm0, -80(%edx)
-L(aligned_16_64bytes):
-	movdqa	%xmm0, -64(%edx)
-L(aligned_16_48bytes):
-	movdqa	%xmm0, -48(%edx)
-L(aligned_16_32bytes):
-	movdqa	%xmm0, -32(%edx)
-L(aligned_16_16bytes):
-	movdqa	%xmm0, -16(%edx)
-L(aligned_16_0bytes):
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_113bytes):
-	movdqa	%xmm0, -113(%edx)
-L(aligned_16_97bytes):
-	movdqa	%xmm0, -97(%edx)
-L(aligned_16_81bytes):
-	movdqa	%xmm0, -81(%edx)
-L(aligned_16_65bytes):
-	movdqa	%xmm0, -65(%edx)
-L(aligned_16_49bytes):
-	movdqa	%xmm0, -49(%edx)
-L(aligned_16_33bytes):
-	movdqa	%xmm0, -33(%edx)
-L(aligned_16_17bytes):
-	movdqa	%xmm0, -17(%edx)
-L(aligned_16_1bytes):
-	movb	%al, -1(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_114bytes):
-	movdqa	%xmm0, -114(%edx)
-L(aligned_16_98bytes):
-	movdqa	%xmm0, -98(%edx)
-L(aligned_16_82bytes):
-	movdqa	%xmm0, -82(%edx)
-L(aligned_16_66bytes):
-	movdqa	%xmm0, -66(%edx)
-L(aligned_16_50bytes):
-	movdqa	%xmm0, -50(%edx)
-L(aligned_16_34bytes):
-	movdqa	%xmm0, -34(%edx)
-L(aligned_16_18bytes):
-	movdqa	%xmm0, -18(%edx)
-L(aligned_16_2bytes):
-	movw	%ax, -2(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_115bytes):
-	movdqa	%xmm0, -115(%edx)
-L(aligned_16_99bytes):
-	movdqa	%xmm0, -99(%edx)
-L(aligned_16_83bytes):
-	movdqa	%xmm0, -83(%edx)
-L(aligned_16_67bytes):
-	movdqa	%xmm0, -67(%edx)
-L(aligned_16_51bytes):
-	movdqa	%xmm0, -51(%edx)
-L(aligned_16_35bytes):
-	movdqa	%xmm0, -35(%edx)
-L(aligned_16_19bytes):
-	movdqa	%xmm0, -19(%edx)
-L(aligned_16_3bytes):
-	movw	%ax, -3(%edx)
-	movb	%al, -1(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_116bytes):
-	movdqa	%xmm0, -116(%edx)
-L(aligned_16_100bytes):
-	movdqa	%xmm0, -100(%edx)
-L(aligned_16_84bytes):
-	movdqa	%xmm0, -84(%edx)
-L(aligned_16_68bytes):
-	movdqa	%xmm0, -68(%edx)
-L(aligned_16_52bytes):
-	movdqa	%xmm0, -52(%edx)
-L(aligned_16_36bytes):
-	movdqa	%xmm0, -36(%edx)
-L(aligned_16_20bytes):
-	movdqa	%xmm0, -20(%edx)
-L(aligned_16_4bytes):
-	movl	%eax, -4(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_117bytes):
-	movdqa	%xmm0, -117(%edx)
-L(aligned_16_101bytes):
-	movdqa	%xmm0, -101(%edx)
-L(aligned_16_85bytes):
-	movdqa	%xmm0, -85(%edx)
-L(aligned_16_69bytes):
-	movdqa	%xmm0, -69(%edx)
-L(aligned_16_53bytes):
-	movdqa	%xmm0, -53(%edx)
-L(aligned_16_37bytes):
-	movdqa	%xmm0, -37(%edx)
-L(aligned_16_21bytes):
-	movdqa	%xmm0, -21(%edx)
-L(aligned_16_5bytes):
-	movl	%eax, -5(%edx)
-	movb	%al, -1(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_118bytes):
-	movdqa	%xmm0, -118(%edx)
-L(aligned_16_102bytes):
-	movdqa	%xmm0, -102(%edx)
-L(aligned_16_86bytes):
-	movdqa	%xmm0, -86(%edx)
-L(aligned_16_70bytes):
-	movdqa	%xmm0, -70(%edx)
-L(aligned_16_54bytes):
-	movdqa	%xmm0, -54(%edx)
-L(aligned_16_38bytes):
-	movdqa	%xmm0, -38(%edx)
-L(aligned_16_22bytes):
-	movdqa	%xmm0, -22(%edx)
-L(aligned_16_6bytes):
-	movl	%eax, -6(%edx)
-	movw	%ax, -2(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_119bytes):
-	movdqa	%xmm0, -119(%edx)
-L(aligned_16_103bytes):
-	movdqa	%xmm0, -103(%edx)
-L(aligned_16_87bytes):
-	movdqa	%xmm0, -87(%edx)
-L(aligned_16_71bytes):
-	movdqa	%xmm0, -71(%edx)
-L(aligned_16_55bytes):
-	movdqa	%xmm0, -55(%edx)
-L(aligned_16_39bytes):
-	movdqa	%xmm0, -39(%edx)
-L(aligned_16_23bytes):
-	movdqa	%xmm0, -23(%edx)
-L(aligned_16_7bytes):
-	movl	%eax, -7(%edx)
-	movw	%ax, -3(%edx)
-	movb	%al, -1(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_120bytes):
-	movdqa	%xmm0, -120(%edx)
-L(aligned_16_104bytes):
-	movdqa	%xmm0, -104(%edx)
-L(aligned_16_88bytes):
-	movdqa	%xmm0, -88(%edx)
-L(aligned_16_72bytes):
-	movdqa	%xmm0, -72(%edx)
-L(aligned_16_56bytes):
-	movdqa	%xmm0, -56(%edx)
-L(aligned_16_40bytes):
-	movdqa	%xmm0, -40(%edx)
-L(aligned_16_24bytes):
-	movdqa	%xmm0, -24(%edx)
-L(aligned_16_8bytes):
-	movq	%xmm0, -8(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_121bytes):
-	movdqa	%xmm0, -121(%edx)
-L(aligned_16_105bytes):
-	movdqa	%xmm0, -105(%edx)
-L(aligned_16_89bytes):
-	movdqa	%xmm0, -89(%edx)
-L(aligned_16_73bytes):
-	movdqa	%xmm0, -73(%edx)
-L(aligned_16_57bytes):
-	movdqa	%xmm0, -57(%edx)
-L(aligned_16_41bytes):
-	movdqa	%xmm0, -41(%edx)
-L(aligned_16_25bytes):
-	movdqa	%xmm0, -25(%edx)
-L(aligned_16_9bytes):
-	movq	%xmm0, -9(%edx)
-	movb	%al, -1(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_122bytes):
-	movdqa	%xmm0, -122(%edx)
-L(aligned_16_106bytes):
-	movdqa	%xmm0, -106(%edx)
-L(aligned_16_90bytes):
-	movdqa	%xmm0, -90(%edx)
-L(aligned_16_74bytes):
-	movdqa	%xmm0, -74(%edx)
-L(aligned_16_58bytes):
-	movdqa	%xmm0, -58(%edx)
-L(aligned_16_42bytes):
-	movdqa	%xmm0, -42(%edx)
-L(aligned_16_26bytes):
-	movdqa	%xmm0, -26(%edx)
-L(aligned_16_10bytes):
-	movq	%xmm0, -10(%edx)
-	movw	%ax, -2(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_123bytes):
-	movdqa	%xmm0, -123(%edx)
-L(aligned_16_107bytes):
-	movdqa	%xmm0, -107(%edx)
-L(aligned_16_91bytes):
-	movdqa	%xmm0, -91(%edx)
-L(aligned_16_75bytes):
-	movdqa	%xmm0, -75(%edx)
-L(aligned_16_59bytes):
-	movdqa	%xmm0, -59(%edx)
-L(aligned_16_43bytes):
-	movdqa	%xmm0, -43(%edx)
-L(aligned_16_27bytes):
-	movdqa	%xmm0, -27(%edx)
-L(aligned_16_11bytes):
-	movq	%xmm0, -11(%edx)
-	movw	%ax, -3(%edx)
-	movb	%al, -1(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_124bytes):
-	movdqa	%xmm0, -124(%edx)
-L(aligned_16_108bytes):
-	movdqa	%xmm0, -108(%edx)
-L(aligned_16_92bytes):
-	movdqa	%xmm0, -92(%edx)
-L(aligned_16_76bytes):
-	movdqa	%xmm0, -76(%edx)
-L(aligned_16_60bytes):
-	movdqa	%xmm0, -60(%edx)
-L(aligned_16_44bytes):
-	movdqa	%xmm0, -44(%edx)
-L(aligned_16_28bytes):
-	movdqa	%xmm0, -28(%edx)
-L(aligned_16_12bytes):
-	movq	%xmm0, -12(%edx)
-	movl	%eax, -4(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_125bytes):
-	movdqa	%xmm0, -125(%edx)
-L(aligned_16_109bytes):
-	movdqa	%xmm0, -109(%edx)
-L(aligned_16_93bytes):
-	movdqa	%xmm0, -93(%edx)
-L(aligned_16_77bytes):
-	movdqa	%xmm0, -77(%edx)
-L(aligned_16_61bytes):
-	movdqa	%xmm0, -61(%edx)
-L(aligned_16_45bytes):
-	movdqa	%xmm0, -45(%edx)
-L(aligned_16_29bytes):
-	movdqa	%xmm0, -29(%edx)
-L(aligned_16_13bytes):
-	movq	%xmm0, -13(%edx)
-	movl	%eax, -5(%edx)
-	movb	%al, -1(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_126bytes):
-	movdqa	%xmm0, -126(%edx)
-L(aligned_16_110bytes):
-	movdqa	%xmm0, -110(%edx)
-L(aligned_16_94bytes):
-	movdqa	%xmm0, -94(%edx)
-L(aligned_16_78bytes):
-	movdqa	%xmm0, -78(%edx)
-L(aligned_16_62bytes):
-	movdqa	%xmm0, -62(%edx)
-L(aligned_16_46bytes):
-	movdqa	%xmm0, -46(%edx)
-L(aligned_16_30bytes):
-	movdqa	%xmm0, -30(%edx)
-L(aligned_16_14bytes):
-	movq	%xmm0, -14(%edx)
-	movl	%eax, -6(%edx)
-	movw	%ax, -2(%edx)
-	SETRTNVAL
-	RETURN
-
-	ALIGN(4)
-L(aligned_16_127bytes):
-	movdqa	%xmm0, -127(%edx)
-L(aligned_16_111bytes):
-	movdqa	%xmm0, -111(%edx)
-L(aligned_16_95bytes):
-	movdqa	%xmm0, -95(%edx)
-L(aligned_16_79bytes):
-	movdqa	%xmm0, -79(%edx)
-L(aligned_16_63bytes):
-	movdqa	%xmm0, -63(%edx)
-L(aligned_16_47bytes):
-	movdqa	%xmm0, -47(%edx)
-L(aligned_16_31bytes):
-	movdqa	%xmm0, -31(%edx)
-L(aligned_16_15bytes):
-	movq	%xmm0, -15(%edx)
-	movl	%eax, -7(%edx)
-	movw	%ax, -3(%edx)
-	movb	%al, -1(%edx)
-	SETRTNVAL
-	RETURN_END
-
-END(memset_atom)
diff --git a/libc/arch-x86/string/sse2-memset-slm.S b/libc/arch-x86/string/sse2-memset-slm.S
index e4c8fa1..ec2ee52 100644
--- a/libc/arch-x86/string/sse2-memset-slm.S
+++ b/libc/arch-x86/string/sse2-memset-slm.S
@@ -79,7 +79,7 @@
     /* We loaded the jump table and adjusted EDX. Go.  */	\
     jmp		*%ebx
 
-ENTRY(__memset_chk_generic)
+ENTRY(__memset_chk)
   ENTRANCE
 
   movl LEN(%esp), %ecx
@@ -88,11 +88,11 @@
 
   POP(%ebx) // Undo ENTRANCE without returning.
   jmp __memset_chk_fail
-END(__memset_chk_generic)
+END(__memset_chk)
 
 	.section .text.sse2,"ax",@progbits
 	ALIGN(4)
-ENTRY(memset_generic)
+ENTRY(memset)
 	ENTRANCE
 
 	movl	LEN(%esp), %ecx
@@ -755,4 +755,4 @@
 	SETRTNVAL
 	RETURN_END
 
-END(memset_generic)
+END(memset)
diff --git a/libc/arch-x86/string/sse2-stpcpy-slm.S b/libc/arch-x86/string/sse2-stpcpy-slm.S
old mode 100755
new mode 100644
diff --git a/libc/arch-x86/string/sse2-strcpy-slm.S b/libc/arch-x86/string/sse2-strcpy-slm.S
old mode 100755
new mode 100644
index 22ceeab..b5d84b5
--- a/libc/arch-x86/string/sse2-strcpy-slm.S
+++ b/libc/arch-x86/string/sse2-strcpy-slm.S
@@ -79,7 +79,7 @@
 #define POP(REG) popl REG; CFI_POP (REG)
 
 #ifndef STRCPY
-# define STRCPY  strcpy_generic
+# define STRCPY  strcpy
 #endif
 
 #ifdef USE_AS_STPNCPY
diff --git a/libc/arch-x86/string/sse2-strlen-slm.S b/libc/arch-x86/string/sse2-strlen-slm.S
old mode 100755
new mode 100644
index b805ad6..27cc025
--- a/libc/arch-x86/string/sse2-strlen-slm.S
+++ b/libc/arch-x86/string/sse2-strlen-slm.S
@@ -29,7 +29,7 @@
 */
 
 #ifndef STRLEN
-# define STRLEN strlen_generic
+# define STRLEN strlen
 #endif
 
 #ifndef L
diff --git a/libc/arch-x86/string/sse2-strncpy-slm.S b/libc/arch-x86/string/sse2-strncpy-slm.S
old mode 100755
new mode 100644
index aff7fb9..591419f
--- a/libc/arch-x86/string/sse2-strncpy-slm.S
+++ b/libc/arch-x86/string/sse2-strncpy-slm.S
@@ -29,5 +29,5 @@
 */
 
 #define USE_AS_STRNCPY
-#define STRCPY strncpy_generic
+#define STRCPY strncpy
 #include "sse2-strcpy-slm.S"
diff --git a/libc/arch-x86/string/sse2-wcschr-atom.S b/libc/arch-x86/string/sse2-wcschr-atom.S
deleted file mode 100644
index 729302b..0000000
--- a/libc/arch-x86/string/sse2-wcschr-atom.S
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
-Copyright (c) 2011 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef L
-# define L(label)	.L##label
-#endif
-
-#ifndef cfi_startproc
-# define cfi_startproc	.cfi_startproc
-#endif
-
-#ifndef cfi_endproc
-# define cfi_endproc	.cfi_endproc
-#endif
-
-#ifndef cfi_rel_offset
-# define cfi_rel_offset(reg, off)	.cfi_rel_offset reg, off
-#endif
-
-#ifndef cfi_restore
-# define cfi_restore(reg)	.cfi_restore reg
-#endif
-
-#ifndef cfi_adjust_cfa_offset
-# define cfi_adjust_cfa_offset(off)	.cfi_adjust_cfa_offset off
-#endif
-
-#ifndef ENTRY
-# define ENTRY(name)	\
-	.type name,  @function;	\
-	.globl name;	\
-	.p2align 4;	\
-name:	\
-	cfi_startproc
-#endif
-
-#ifndef END
-# define END(name)	\
-	cfi_endproc;	\
-	.size name,	.-name
-#endif
-
-#define CFI_PUSH(REG)	\
-	cfi_adjust_cfa_offset (4);	\
-	cfi_rel_offset (REG, 0)
-
-#define CFI_POP(REG)	\
-	cfi_adjust_cfa_offset (-4);	\
-	cfi_restore (REG)
-
-#define PUSH(REG)	pushl REG; CFI_PUSH (REG)
-#define POP(REG)	popl REG; CFI_POP (REG)
-
-#define PARMS	4
-
-
-#define STR1  PARMS
-#define STR2  STR1+4
-
-	.text
-ENTRY (wcschr)
-
-	mov	STR1(%esp), %ecx
-	movd	STR2(%esp), %xmm1
-
-	mov	%ecx, %eax
-	punpckldq %xmm1, %xmm1
-	pxor	%xmm2, %xmm2
-	punpckldq %xmm1, %xmm1
-
-	and	$63, %eax
-	cmp	$48, %eax
-	ja	L(cross_cache)
-
-	movdqu	(%ecx), %xmm0
-	pcmpeqd	%xmm0, %xmm2
-	pcmpeqd	%xmm1, %xmm0
-	pmovmskb %xmm2, %edx
-	pmovmskb %xmm0, %eax
-	or	%eax, %edx
-	jnz	L(matches)
-	and	$-16, %ecx
-	jmp	L(loop)
-
-	.p2align 4
-L(cross_cache):
-	PUSH	(%edi)
-	mov	%ecx, %edi
-	mov	%eax, %ecx
-	and	$-16, %edi
-	and	$15, %ecx
-	movdqa	(%edi), %xmm0
-	pcmpeqd	%xmm0, %xmm2
-	pcmpeqd	%xmm1, %xmm0
-	pmovmskb %xmm2, %edx
-	pmovmskb %xmm0, %eax
-
-	sarl	%cl, %edx
-	sarl	%cl, %eax
-	test	%eax, %eax
-	jz	L(unaligned_no_match)
-
-	add	%edi, %ecx
-	POP	(%edi)
-
-	test	%edx, %edx
-	jz	L(match_case1)
-	test	%al, %al
-	jz	L(match_higth_case2)
-	test	$15, %al
-	jnz	L(match_case2_4)
-	test	$15, %dl
-	jnz	L(return_null)
-	lea	4(%ecx), %eax
-	ret
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(unaligned_no_match):
-	mov	%edi, %ecx
-	POP	(%edi)
-
-	test	%edx, %edx
-	jnz	L(return_null)
-
-	pxor	%xmm2, %xmm2
-
-/* Loop start on aligned string.  */
-	.p2align 4
-L(loop):
-	add	$16, %ecx
-	movdqa	(%ecx), %xmm0
-	pcmpeqd	%xmm0, %xmm2
-	pcmpeqd	%xmm1, %xmm0
-	pmovmskb %xmm2, %edx
-	pmovmskb %xmm0, %eax
-	or	%eax, %edx
-	jnz	L(matches)
-	add	$16, %ecx
-
-	movdqa	(%ecx), %xmm0
-	pcmpeqd	%xmm0, %xmm2
-	pcmpeqd	%xmm1, %xmm0
-	pmovmskb %xmm2, %edx
-	pmovmskb %xmm0, %eax
-	or	%eax, %edx
-	jnz	L(matches)
-	add	$16, %ecx
-
-	movdqa	(%ecx), %xmm0
-	pcmpeqd	%xmm0, %xmm2
-	pcmpeqd	%xmm1, %xmm0
-	pmovmskb %xmm2, %edx
-	pmovmskb %xmm0, %eax
-	or	%eax, %edx
-	jnz	L(matches)
-	add	$16, %ecx
-
-	movdqa	(%ecx), %xmm0
-	pcmpeqd	%xmm0, %xmm2
-	pcmpeqd	%xmm1, %xmm0
-	pmovmskb %xmm2, %edx
-	pmovmskb %xmm0, %eax
-	or	%eax, %edx
-	jz	L(loop)
-
-	.p2align 4
-L(matches):
-	pmovmskb %xmm2, %edx
-	test	%eax, %eax
-	jz	L(return_null)
-	test	%edx, %edx
-	jz	L(match_case1)
-
-	.p2align 4
-L(match_case2):
-	test	%al, %al
-	jz	L(match_higth_case2)
-	test	$15, %al
-	jnz	L(match_case2_4)
-	test	$15, %dl
-	jnz	L(return_null)
-	lea	4(%ecx), %eax
-	ret
-
-	.p2align 4
-L(match_case2_4):
-	mov	%ecx, %eax
-	ret
-
-	.p2align 4
-L(match_higth_case2):
-	test	%dl, %dl
-	jnz	L(return_null)
-	test	$15, %ah
-	jnz	L(match_case2_12)
-	test	$15, %dh
-	jnz	L(return_null)
-	lea	12(%ecx), %eax
-	ret
-
-	.p2align 4
-L(match_case2_12):
-	lea	8(%ecx), %eax
-	ret
-
-	.p2align 4
-L(match_case1):
-	test	%al, %al
-	jz	L(match_higth_case1)
-
-	test	$0x01, %al
-	jnz	L(exit0)
-	lea	4(%ecx), %eax
-	ret
-
-	.p2align 4
-L(match_higth_case1):
-	test	$0x01, %ah
-	jnz	L(exit3)
-	lea	12(%ecx), %eax
-	ret
-
-	.p2align 4
-L(exit0):
-	mov	%ecx, %eax
-	ret
-
-	.p2align 4
-L(exit3):
-	lea	8(%ecx), %eax
-	ret
-
-	.p2align 4
-L(return_null):
-	xor	%eax, %eax
-	ret
-
-END (wcschr)
diff --git a/libc/arch-x86/string/sse2-wcscmp-atom.S b/libc/arch-x86/string/sse2-wcscmp-atom.S
deleted file mode 100644
index 8867d28..0000000
--- a/libc/arch-x86/string/sse2-wcscmp-atom.S
+++ /dev/null
@@ -1,1062 +0,0 @@
-/*
-Copyright (c) 2011 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef L
-# define L(label)	.L##label
-#endif
-
-#ifndef cfi_startproc
-# define cfi_startproc	.cfi_startproc
-#endif
-
-#ifndef cfi_endproc
-# define cfi_endproc	.cfi_endproc
-#endif
-
-#ifndef cfi_rel_offset
-# define cfi_rel_offset(reg, off)	.cfi_rel_offset reg, off
-#endif
-
-#ifndef cfi_restore
-# define cfi_restore(reg)	.cfi_restore reg
-#endif
-
-#ifndef cfi_adjust_cfa_offset
-# define cfi_adjust_cfa_offset(off)	.cfi_adjust_cfa_offset off
-#endif
-
-#ifndef ENTRY
-# define ENTRY(name)	\
-	.type name, @function;	\
-	.globl name;	\
-	.p2align 4;	\
-name:	\
-	cfi_startproc
-#endif
-
-#ifndef END
-# define END(name)	\
-	cfi_endproc;	\
-	.size name, .-name
-#endif
-
-#define CFI_PUSH(REG)	\
-	cfi_adjust_cfa_offset (4);	\
-	cfi_rel_offset (REG, 0)
-
-#define CFI_POP(REG)	\
-	cfi_adjust_cfa_offset (-4);	\
-	cfi_restore (REG)
-
-#define PUSH(REG) pushl REG; CFI_PUSH (REG)
-#define POP(REG) popl REG; CFI_POP (REG)
-
-#define ENTRANCE PUSH(%esi); PUSH(%edi)
-#define RETURN  POP(%edi); POP(%esi); ret; CFI_PUSH(%esi); CFI_PUSH(%edi);
-#define PARMS  4
-#define STR1  PARMS
-#define STR2  STR1+4
-
-	.text
-ENTRY (wcscmp)
-/*
-	* This implementation uses SSE to compare up to 16 bytes at a time.
-*/
-	mov	STR1(%esp), %edx
-	mov	STR2(%esp), %eax
-
-	mov	(%eax), %ecx
-	cmp	%ecx, (%edx)
-	jne	L(neq)
-	test	%ecx, %ecx
-	jz	L(eq)
-
-	mov	4(%eax), %ecx
-	cmp	%ecx, 4(%edx)
-	jne	L(neq)
-	test	%ecx, %ecx
-	jz	L(eq)
-
-	mov	8(%eax), %ecx
-	cmp	%ecx, 8(%edx)
-	jne	L(neq)
-	test	%ecx, %ecx
-	jz	L(eq)
-
-	mov	12(%eax), %ecx
-	cmp	%ecx, 12(%edx)
-	jne	L(neq)
-	test	%ecx, %ecx
-	jz	L(eq)
-
-	ENTRANCE
-	add	$16, %eax
-	add	$16, %edx
-
-	mov	%eax, %esi
-	mov	%edx, %edi
-	pxor	%xmm0, %xmm0		/* clear %xmm0 for null char checks */
-	mov	%al, %ch
-	mov	%dl, %cl
-	and	$63, %eax		/* esi alignment in cache line */
-	and	$63, %edx		/* edi alignment in cache line */
-	and	$15, %cl
-	jz	L(continue_00)
-	cmp	$16, %edx
-	jb	L(continue_0)
-	cmp	$32, %edx
-	jb	L(continue_16)
-	cmp	$48, %edx
-	jb	L(continue_32)
-
-L(continue_48):
-	and	$15, %ch
-	jz	L(continue_48_00)
-	cmp	$16, %eax
-	jb	L(continue_0_48)
-	cmp	$32, %eax
-	jb	L(continue_16_48)
-	cmp	$48, %eax
-	jb	L(continue_32_48)
-
-	.p2align 4
-L(continue_48_48):
-	mov	(%esi), %ecx
-	cmp	%ecx, (%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	4(%esi), %ecx
-	cmp	%ecx, 4(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	8(%esi), %ecx
-	cmp	%ecx, 8(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	12(%esi), %ecx
-	cmp	%ecx, 12(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	movdqu	16(%edi), %xmm1
-	movdqu	16(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	movdqu	32(%edi), %xmm1
-	movdqu	32(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_32)
-
-	movdqu	48(%edi), %xmm1
-	movdqu	48(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_48)
-
-	add	$64, %esi
-	add	$64, %edi
-	jmp	L(continue_48_48)
-
-L(continue_0):
-	and	$15, %ch
-	jz	L(continue_0_00)
-	cmp	$16, %eax
-	jb	L(continue_0_0)
-	cmp	$32, %eax
-	jb	L(continue_0_16)
-	cmp	$48, %eax
-	jb	L(continue_0_32)
-
-	.p2align 4
-L(continue_0_48):
-	mov	(%esi), %ecx
-	cmp	%ecx, (%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	4(%esi), %ecx
-	cmp	%ecx, 4(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	8(%esi), %ecx
-	cmp	%ecx, 8(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	12(%esi), %ecx
-	cmp	%ecx, 12(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	movdqu	16(%edi), %xmm1
-	movdqu	16(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	movdqu	32(%edi), %xmm1
-	movdqu	32(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_32)
-
-	mov	48(%esi), %ecx
-	cmp	%ecx, 48(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	52(%esi), %ecx
-	cmp	%ecx, 52(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	56(%esi), %ecx
-	cmp	%ecx, 56(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	60(%esi), %ecx
-	cmp	%ecx, 60(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	add	$64, %esi
-	add	$64, %edi
-	jmp	L(continue_0_48)
-
-	.p2align 4
-L(continue_00):
-	and	$15, %ch
-	jz	L(continue_00_00)
-	cmp	$16, %eax
-	jb	L(continue_00_0)
-	cmp	$32, %eax
-	jb	L(continue_00_16)
-	cmp	$48, %eax
-	jb	L(continue_00_32)
-
-	.p2align 4
-L(continue_00_48):
-	pcmpeqd	(%edi), %xmm0
-	mov	(%edi), %eax
-	pmovmskb %xmm0, %ecx
-	test	%ecx, %ecx
-	jnz	L(less4_double_words1)
-
-	cmp	(%esi), %eax
-	jne	L(nequal)
-
-	mov	4(%edi), %eax
-	cmp	4(%esi), %eax
-	jne	L(nequal)
-
-	mov	8(%edi), %eax
-	cmp	8(%esi), %eax
-	jne	L(nequal)
-
-	mov	12(%edi), %eax
-	cmp	12(%esi), %eax
-	jne	L(nequal)
-
-	movdqu	16(%esi), %xmm2
-	pcmpeqd	%xmm2, %xmm0		/* Any null double_word? */
-	pcmpeqd	16(%edi), %xmm2		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm2		/* packed sub of comparison results*/
-	pmovmskb %xmm2, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	movdqu	32(%esi), %xmm2
-	pcmpeqd	%xmm2, %xmm0		/* Any null double_word? */
-	pcmpeqd	32(%edi), %xmm2		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm2		/* packed sub of comparison results*/
-	pmovmskb %xmm2, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_32)
-
-	movdqu	48(%esi), %xmm2
-	pcmpeqd	%xmm2, %xmm0		/* Any null double_word? */
-	pcmpeqd	48(%edi), %xmm2		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm2		/* packed sub of comparison results*/
-	pmovmskb %xmm2, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_48)
-
-	add	$64, %esi
-	add	$64, %edi
-	jmp	L(continue_00_48)
-
-	.p2align 4
-L(continue_32):
-	and	$15, %ch
-	jz	L(continue_32_00)
-	cmp	$16, %eax
-	jb	L(continue_0_32)
-	cmp	$32, %eax
-	jb	L(continue_16_32)
-	cmp	$48, %eax
-	jb	L(continue_32_32)
-
-	.p2align 4
-L(continue_32_48):
-	mov	(%esi), %ecx
-	cmp	%ecx, (%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	4(%esi), %ecx
-	cmp	%ecx, 4(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	8(%esi), %ecx
-	cmp	%ecx, 8(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	12(%esi), %ecx
-	cmp	%ecx, 12(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	16(%esi), %ecx
-	cmp	%ecx, 16(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	20(%esi), %ecx
-	cmp	%ecx, 20(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	24(%esi), %ecx
-	cmp	%ecx, 24(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	28(%esi), %ecx
-	cmp	%ecx, 28(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	movdqu	32(%edi), %xmm1
-	movdqu	32(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_32)
-
-	movdqu	48(%edi), %xmm1
-	movdqu	48(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_48)
-
-	add	$64, %esi
-	add	$64, %edi
-	jmp	L(continue_32_48)
-
-	.p2align 4
-L(continue_16):
-	and	$15, %ch
-	jz	L(continue_16_00)
-	cmp	$16, %eax
-	jb	L(continue_0_16)
-	cmp	$32, %eax
-	jb	L(continue_16_16)
-	cmp	$48, %eax
-	jb	L(continue_16_32)
-
-	.p2align 4
-L(continue_16_48):
-	mov	(%esi), %ecx
-	cmp	%ecx, (%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	4(%esi), %ecx
-	cmp	%ecx, 4(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	8(%esi), %ecx
-	cmp	%ecx, 8(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	12(%esi), %ecx
-	cmp	%ecx, 12(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	movdqu	16(%edi), %xmm1
-	movdqu	16(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	mov	32(%esi), %ecx
-	cmp	%ecx, 32(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	36(%esi), %ecx
-	cmp	%ecx, 36(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	40(%esi), %ecx
-	cmp	%ecx, 40(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	44(%esi), %ecx
-	cmp	%ecx, 44(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	movdqu	48(%edi), %xmm1
-	movdqu	48(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_48)
-
-	add	$64, %esi
-	add	$64, %edi
-	jmp	L(continue_16_48)
-
-	.p2align 4
-L(continue_00_00):
-	movdqa	(%edi), %xmm1
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	(%esi), %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	movdqa	16(%edi), %xmm3
-	pcmpeqd	%xmm3, %xmm0		/* Any null double_word? */
-	pcmpeqd	16(%esi), %xmm3		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm3		/* packed sub of comparison results*/
-	pmovmskb %xmm3, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	movdqa	32(%edi), %xmm5
-	pcmpeqd	%xmm5, %xmm0		/* Any null double_word? */
-	pcmpeqd	32(%esi), %xmm5		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm5		/* packed sub of comparison results*/
-	pmovmskb %xmm5, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_32)
-
-	movdqa	48(%edi), %xmm1
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	48(%esi), %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_48)
-
-	add	$64, %esi
-	add	$64, %edi
-	jmp	L(continue_00_00)
-
-	.p2align 4
-L(continue_00_32):
-	movdqu	(%esi), %xmm2
-	pcmpeqd	%xmm2, %xmm0		/* Any null double_word? */
-	pcmpeqd	(%edi), %xmm2		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm2		/* packed sub of comparison results*/
-	pmovmskb %xmm2, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	add	$16, %esi
-	add	$16, %edi
-	jmp	L(continue_00_48)
-
-	.p2align 4
-L(continue_00_16):
-	movdqu	(%esi), %xmm2
-	pcmpeqd	%xmm2, %xmm0		/* Any null double_word? */
-	pcmpeqd	(%edi), %xmm2		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm2		/* packed sub of comparison results*/
-	pmovmskb %xmm2, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	movdqu	16(%esi), %xmm2
-	pcmpeqd	%xmm2, %xmm0		/* Any null double_word? */
-	pcmpeqd	16(%edi), %xmm2		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm2		/* packed sub of comparison results*/
-	pmovmskb %xmm2, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	add	$32, %esi
-	add	$32, %edi
-	jmp	L(continue_00_48)
-
-	.p2align 4
-L(continue_00_0):
-	movdqu	(%esi), %xmm2
-	pcmpeqd	%xmm2, %xmm0		/* Any null double_word? */
-	pcmpeqd	(%edi), %xmm2		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm2		/* packed sub of comparison results*/
-	pmovmskb %xmm2, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	movdqu	16(%esi), %xmm2
-	pcmpeqd	%xmm2, %xmm0		/* Any null double_word? */
-	pcmpeqd	16(%edi), %xmm2		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm2		/* packed sub of comparison results*/
-	pmovmskb %xmm2, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	movdqu	32(%esi), %xmm2
-	pcmpeqd	%xmm2, %xmm0		/* Any null double_word? */
-	pcmpeqd	32(%edi), %xmm2		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm2		/* packed sub of comparison results*/
-	pmovmskb %xmm2, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_32)
-
-	add	$48, %esi
-	add	$48, %edi
-	jmp	L(continue_00_48)
-
-	.p2align 4
-L(continue_48_00):
-	pcmpeqd	(%esi), %xmm0
-	mov	(%edi), %eax
-	pmovmskb %xmm0, %ecx
-	test	%ecx, %ecx
-	jnz	L(less4_double_words1)
-
-	cmp	(%esi), %eax
-	jne	L(nequal)
-
-	mov	4(%edi), %eax
-	cmp	4(%esi), %eax
-	jne	L(nequal)
-
-	mov	8(%edi), %eax
-	cmp	8(%esi), %eax
-	jne	L(nequal)
-
-	mov	12(%edi), %eax
-	cmp	12(%esi), %eax
-	jne	L(nequal)
-
-	movdqu	16(%edi), %xmm1
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	16(%esi), %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	movdqu	32(%edi), %xmm1
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	32(%esi), %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_32)
-
-	movdqu	48(%edi), %xmm1
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	48(%esi), %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_48)
-
-	add	$64, %esi
-	add	$64, %edi
-	jmp	L(continue_48_00)
-
-	.p2align 4
-L(continue_32_00):
-	movdqu	(%edi), %xmm1
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	(%esi), %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	add	$16, %esi
-	add	$16, %edi
-	jmp	L(continue_48_00)
-
-	.p2align 4
-L(continue_16_00):
-	movdqu	(%edi), %xmm1
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	(%esi), %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	movdqu	16(%edi), %xmm1
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	16(%esi), %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	add	$32, %esi
-	add	$32, %edi
-	jmp	L(continue_48_00)
-
-	.p2align 4
-L(continue_0_00):
-	movdqu	(%edi), %xmm1
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	(%esi), %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	movdqu	16(%edi), %xmm1
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	16(%esi), %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	movdqu	32(%edi), %xmm1
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	32(%esi), %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_32)
-
-	add	$48, %esi
-	add	$48, %edi
-	jmp	L(continue_48_00)
-
-	.p2align 4
-L(continue_32_32):
-	movdqu	(%edi), %xmm1
-	movdqu	(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	add	$16, %esi
-	add	$16, %edi
-	jmp	L(continue_48_48)
-
-	.p2align 4
-L(continue_16_16):
-	movdqu	(%edi), %xmm1
-	movdqu	(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	movdqu	16(%edi), %xmm3
-	movdqu	16(%esi), %xmm4
-	pcmpeqd	%xmm3, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm4, %xmm3		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm3		/* packed sub of comparison results*/
-	pmovmskb %xmm3, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	add	$32, %esi
-	add	$32, %edi
-	jmp	L(continue_48_48)
-
-	.p2align 4
-L(continue_0_0):
-	movdqu	(%edi), %xmm1
-	movdqu	(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	movdqu	16(%edi), %xmm3
-	movdqu	16(%esi), %xmm4
-	pcmpeqd	%xmm3, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm4, %xmm3		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm3		/* packed sub of comparison results*/
-	pmovmskb %xmm3, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	movdqu	32(%edi), %xmm1
-	movdqu	32(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_32)
-
-	add	$48, %esi
-	add	$48, %edi
-	jmp	L(continue_48_48)
-
-	.p2align 4
-L(continue_0_16):
-	movdqu	(%edi), %xmm1
-	movdqu	(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	movdqu	16(%edi), %xmm1
-	movdqu	16(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words_16)
-
-	add	$32, %esi
-	add	$32, %edi
-	jmp	L(continue_32_48)
-
-	.p2align 4
-L(continue_0_32):
-	movdqu	(%edi), %xmm1
-	movdqu	(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	add	$16, %esi
-	add	$16, %edi
-	jmp	L(continue_16_48)
-
-	.p2align 4
-L(continue_16_32):
-	movdqu	(%edi), %xmm1
-	movdqu	(%esi), %xmm2
-	pcmpeqd	%xmm1, %xmm0		/* Any null double_word? */
-	pcmpeqd	%xmm2, %xmm1		/* compare first 4 double_words for equality */
-	psubb	%xmm0, %xmm1		/* packed sub of comparison results*/
-	pmovmskb %xmm1, %edx
-	sub	$0xffff, %edx		/* if first 4 double_words are same, edx == 0xffff */
-	jnz	L(less4_double_words)
-
-	add	$16, %esi
-	add	$16, %edi
-	jmp	L(continue_32_48)
-
-	.p2align 4
-L(less4_double_words1):
-	cmp	(%esi), %eax
-	jne	L(nequal)
-	test	%eax, %eax
-	jz	L(equal)
-
-	mov	4(%esi), %ecx
-	cmp	%ecx, 4(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	8(%esi), %ecx
-	cmp	%ecx, 8(%edi)
-	jne	L(nequal)
-	test	%ecx, %ecx
-	jz	L(equal)
-
-	mov	12(%esi), %ecx
-	cmp	%ecx, 12(%edi)
-	jne	L(nequal)
-	xor	%eax, %eax
-	RETURN
-
-	.p2align 4
-L(less4_double_words):
-	xor	%eax, %eax
-	test	%dl, %dl
-	jz	L(next_two_double_words)
-	and	$15, %dl
-	jz	L(second_double_word)
-	mov	(%esi), %ecx
-	cmp	%ecx, (%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(second_double_word):
-	mov	4(%esi), %ecx
-	cmp	%ecx, 4(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(next_two_double_words):
-	and	$15, %dh
-	jz	L(fourth_double_word)
-	mov	8(%esi), %ecx
-	cmp	%ecx, 8(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(fourth_double_word):
-	mov	12(%esi), %ecx
-	cmp	%ecx, 12(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(less4_double_words_16):
-	xor	%eax, %eax
-	test	%dl, %dl
-	jz	L(next_two_double_words_16)
-	and	$15, %dl
-	jz	L(second_double_word_16)
-	mov	16(%esi), %ecx
-	cmp	%ecx, 16(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(second_double_word_16):
-	mov	20(%esi), %ecx
-	cmp	%ecx, 20(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(next_two_double_words_16):
-	and	$15, %dh
-	jz	L(fourth_double_word_16)
-	mov	24(%esi), %ecx
-	cmp	%ecx, 24(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(fourth_double_word_16):
-	mov	28(%esi), %ecx
-	cmp	%ecx, 28(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(less4_double_words_32):
-	xor	%eax, %eax
-	test	%dl, %dl
-	jz	L(next_two_double_words_32)
-	and	$15, %dl
-	jz	L(second_double_word_32)
-	mov	32(%esi), %ecx
-	cmp	%ecx, 32(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(second_double_word_32):
-	mov	36(%esi), %ecx
-	cmp	%ecx, 36(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(next_two_double_words_32):
-	and	$15, %dh
-	jz	L(fourth_double_word_32)
-	mov	40(%esi), %ecx
-	cmp	%ecx, 40(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(fourth_double_word_32):
-	mov	44(%esi), %ecx
-	cmp	%ecx, 44(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(less4_double_words_48):
-	xor	%eax, %eax
-	test	%dl, %dl
-	jz	L(next_two_double_words_48)
-	and	$15, %dl
-	jz	L(second_double_word_48)
-	mov	48(%esi), %ecx
-	cmp	%ecx, 48(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(second_double_word_48):
-	mov	52(%esi), %ecx
-	cmp	%ecx, 52(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(next_two_double_words_48):
-	and	$15, %dh
-	jz	L(fourth_double_word_48)
-	mov	56(%esi), %ecx
-	cmp	%ecx, 56(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(fourth_double_word_48):
-	mov	60(%esi), %ecx
-	cmp	%ecx, 60(%edi)
-	jne	L(nequal)
-	RETURN
-
-	.p2align 4
-L(nequal):
-	mov	$1, %eax
-	jg	L(return)
-	neg	%eax
-	RETURN
-
-	.p2align 4
-L(return):
-	RETURN
-
-	.p2align 4
-L(equal):
-	xorl	%eax, %eax
-	RETURN
-
-	CFI_POP (%edi)
-	CFI_POP (%esi)
-
-	.p2align 4
-L(neq):
-	mov	$1, %eax
-	jg	L(neq_bigger)
-	neg	%eax
-
-L(neq_bigger):
-	ret
-
-	.p2align 4
-L(eq):
-	xorl	%eax, %eax
-	ret
-
-END (wcscmp)
-
diff --git a/libc/arch-x86/string/sse2-wcslen-atom.S b/libc/arch-x86/string/sse2-wcslen-atom.S
deleted file mode 100644
index 2f10db4..0000000
--- a/libc/arch-x86/string/sse2-wcslen-atom.S
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
-Copyright (c) 2011 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef USE_AS_WCSCAT
-
-# ifndef L
-#  define L(label)	.L##label
-# endif
-
-# ifndef cfi_startproc
-#  define cfi_startproc	.cfi_startproc
-# endif
-
-# ifndef cfi_endproc
-#  define cfi_endproc	.cfi_endproc
-# endif
-
-# ifndef ENTRY
-#  define ENTRY(name)	\
-	.type name,  @function;	\
-	.globl name;	\
-	.p2align 4;	\
-name:	\
-	cfi_startproc
-# endif
-
-# ifndef END
-#  define END(name)	\
-	cfi_endproc;	\
-	.size name, .-name
-# endif
-
-# define PARMS	4
-# define STR	PARMS
-# define RETURN ret
-
-	.text
-ENTRY (wcslen)
-	mov	STR(%esp), %edx
-#endif
-	cmpl	$0, (%edx)
-	jz	L(exit_tail0)
-	cmpl	$0, 4(%edx)
-	jz	L(exit_tail1)
-	cmpl	$0, 8(%edx)
-	jz	L(exit_tail2)
-	cmpl	$0, 12(%edx)
-	jz	L(exit_tail3)
-	cmpl	$0, 16(%edx)
-	jz	L(exit_tail4)
-	cmpl	$0, 20(%edx)
-	jz	L(exit_tail5)
-	cmpl	$0, 24(%edx)
-	jz	L(exit_tail6)
-	cmpl	$0, 28(%edx)
-	jz	L(exit_tail7)
-
-	pxor	%xmm0, %xmm0
-
-	lea	32(%edx), %eax
-	lea	-16(%eax), %ecx
-	and	$-16, %eax
-
-	pcmpeqd	(%eax), %xmm0
-	pmovmskb %xmm0, %edx
-	pxor	%xmm1, %xmm1
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm1
-	pmovmskb %xmm1, %edx
-	pxor	%xmm2, %xmm2
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm2
-	pmovmskb %xmm2, %edx
-	pxor	%xmm3, %xmm3
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm3
-	pmovmskb %xmm3, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm0
-	pmovmskb %xmm0, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm1
-	pmovmskb %xmm1, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm2
-	pmovmskb %xmm2, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm3
-	pmovmskb %xmm3, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm0
-	pmovmskb %xmm0, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm1
-	pmovmskb %xmm1, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm2
-	pmovmskb %xmm2, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm3
-	pmovmskb %xmm3, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm0
-	pmovmskb %xmm0, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm1
-	pmovmskb %xmm1, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm2
-	pmovmskb %xmm2, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	(%eax), %xmm3
-	pmovmskb %xmm3, %edx
-	lea	16(%eax), %eax
-	test	%edx, %edx
-	jnz	L(exit)
-
-	and	$-0x40, %eax
-
-	.p2align 4
-L(aligned_64_loop):
-	movaps	(%eax), %xmm0
-	movaps	16(%eax), %xmm1
-	movaps	32(%eax), %xmm2
-	movaps	48(%eax), %xmm6
-
-	pminub	%xmm1, %xmm0
-	pminub	%xmm6, %xmm2
-	pminub	%xmm0, %xmm2
-	pcmpeqd	%xmm3, %xmm2
-	pmovmskb %xmm2, %edx
-	lea	64(%eax), %eax
-	test	%edx, %edx
-	jz	L(aligned_64_loop)
-
-	pcmpeqd	-64(%eax), %xmm3
-	pmovmskb %xmm3, %edx
-	lea	48(%ecx), %ecx
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	%xmm1, %xmm3
-	pmovmskb %xmm3, %edx
-	lea	-16(%ecx), %ecx
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	-32(%eax), %xmm3
-	pmovmskb %xmm3, %edx
-	lea	-16(%ecx), %ecx
-	test	%edx, %edx
-	jnz	L(exit)
-
-	pcmpeqd	%xmm6, %xmm3
-	pmovmskb %xmm3, %edx
-	lea	-16(%ecx), %ecx
-	test	%edx, %edx
-	jnz	L(exit)
-
-	jmp	L(aligned_64_loop)
-
-	.p2align 4
-L(exit):
-	sub	%ecx, %eax
-	shr	$2, %eax
-	test	%dl, %dl
-	jz	L(exit_high)
-
-	mov	%dl, %cl
-	and	$15, %cl
-	jz	L(exit_1)
-	RETURN
-
-	.p2align 4
-L(exit_high):
-	mov	%dh, %ch
-	and	$15, %ch
-	jz	L(exit_3)
-	add	$2, %eax
-	RETURN
-
-	.p2align 4
-L(exit_1):
-	add	$1, %eax
-	RETURN
-
-	.p2align 4
-L(exit_3):
-	add	$3, %eax
-	RETURN
-
-	.p2align 4
-L(exit_tail0):
-	xor	%eax, %eax
-	RETURN
-
-	.p2align 4
-L(exit_tail1):
-	mov	$1, %eax
-	RETURN
-
-	.p2align 4
-L(exit_tail2):
-	mov	$2, %eax
-	RETURN
-
-	.p2align 4
-L(exit_tail3):
-	mov	$3, %eax
-	RETURN
-
-	.p2align 4
-L(exit_tail4):
-	mov	$4, %eax
-	RETURN
-
-	.p2align 4
-L(exit_tail5):
-	mov	$5, %eax
-	RETURN
-
-	.p2align 4
-L(exit_tail6):
-	mov	$6, %eax
-	RETURN
-
-	.p2align 4
-L(exit_tail7):
-	mov	$7, %eax
-#ifndef USE_AS_WCSCAT
-	RETURN
-
-END (wcslen)
-#endif
diff --git a/libc/arch-x86/string/sse2-wcsrchr-atom.S b/libc/arch-x86/string/sse2-wcsrchr-atom.S
deleted file mode 100644
index 1a55df2..0000000
--- a/libc/arch-x86/string/sse2-wcsrchr-atom.S
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
-Copyright (c) 2011 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef L
-# define L(label)	.L##label
-#endif
-
-#ifndef cfi_startproc
-# define cfi_startproc	.cfi_startproc
-#endif
-
-#ifndef cfi_endproc
-# define cfi_endproc	.cfi_endproc
-#endif
-
-#ifndef cfi_rel_offset
-# define cfi_rel_offset(reg, off)	.cfi_rel_offset reg, off
-#endif
-
-#ifndef cfi_restore
-# define cfi_restore(reg)	.cfi_restore reg
-#endif
-
-#ifndef cfi_adjust_cfa_offset
-# define cfi_adjust_cfa_offset(off)	.cfi_adjust_cfa_offset off
-#endif
-
-#ifndef ENTRY
-# define ENTRY(name)	\
-	.type name, @function;	\
-	.globl name;	\
-	.p2align 4;	\
-name:	\
-	cfi_startproc
-#endif
-
-#ifndef END
-# define END(name)	\
-	cfi_endproc;	\
-	.size name, .-name
-#endif
-
-#define CFI_PUSH(REG)	\
-	cfi_adjust_cfa_offset (4);	\
-	cfi_rel_offset (REG, 0)
-
-#define CFI_POP(REG)	\
-	cfi_adjust_cfa_offset (-4);	\
-	cfi_restore (REG)
-
-#define PUSH(REG)	pushl REG;	CFI_PUSH (REG)
-#define POP(REG)	popl REG;	CFI_POP (REG)
-
-#define PARMS  8
-#define ENTRANCE PUSH(%edi);
-#define RETURN  POP(%edi);	ret;	CFI_PUSH(%edi);
-
-#define STR1  PARMS
-#define STR2  STR1+4
-
-	.text
-ENTRY (wcsrchr)
-
-	ENTRANCE
-	mov	STR1(%esp), %ecx
-	movd	STR2(%esp), %xmm1
-
-	mov	%ecx, %edi
-	punpckldq %xmm1, %xmm1
-	pxor	%xmm2, %xmm2
-	punpckldq %xmm1, %xmm1
-
-/* ECX has OFFSET. */
-	and	$63, %ecx
-	cmp	$48, %ecx
-	ja	L(crosscache)
-
-/* unaligned string. */
-	movdqu	(%edi), %xmm0
-	pcmpeqd	%xmm0, %xmm2
-	pcmpeqd	%xmm1, %xmm0
-/* Find where NULL is.  */
-	pmovmskb %xmm2, %ecx
-/* Check if there is a match.  */
-	pmovmskb %xmm0, %eax
-	add	$16, %edi
-
-	test	%eax, %eax
-	jnz	L(unaligned_match1)
-
-	test	%ecx, %ecx
-	jnz	L(return_null)
-
-	and	$-16, %edi
-
-	PUSH	(%esi)
-
-	xor	%edx, %edx
-	jmp	L(loop)
-
-	CFI_POP	(%esi)
-
-	.p2align 4
-L(unaligned_match1):
-	test	%ecx, %ecx
-	jnz	L(prolog_find_zero_1)
-
-	PUSH	(%esi)
-
-/* Save current match */
-	mov	%eax, %edx
-	mov	%edi, %esi
-	and	$-16, %edi
-	jmp	L(loop)
-
-	CFI_POP	(%esi)
-
-	.p2align 4
-L(crosscache):
-/* Hancle unaligned string.  */
-	and	$15, %ecx
-	and	$-16, %edi
-	pxor	%xmm3, %xmm3
-	movdqa	(%edi), %xmm0
-	pcmpeqd	%xmm0, %xmm3
-	pcmpeqd	%xmm1, %xmm0
-/* Find where NULL is.  */
-	pmovmskb %xmm3, %edx
-/* Check if there is a match.  */
-	pmovmskb %xmm0, %eax
-/* Remove the leading bytes.  */
-	shr	%cl, %edx
-	shr	%cl, %eax
-	add	$16, %edi
-
-	test	%eax, %eax
-	jnz	L(unaligned_match)
-
-	test	%edx, %edx
-	jnz	L(return_null)
-
-	PUSH	(%esi)
-
-	xor	%edx, %edx
-	jmp	L(loop)
-
-	CFI_POP	(%esi)
-
-	.p2align 4
-L(unaligned_match):
-	test	%edx, %edx
-	jnz	L(prolog_find_zero)
-
-	PUSH	(%esi)
-
-	mov	%eax, %edx
-	lea	(%edi, %ecx), %esi
-
-/* Loop start on aligned string.  */
-	.p2align 4
-L(loop):
-	movdqa	(%edi), %xmm0
-	pcmpeqd	%xmm0, %xmm2
-	add	$16, %edi
-	pcmpeqd	%xmm1, %xmm0
-	pmovmskb %xmm2, %ecx
-	pmovmskb %xmm0, %eax
-	or	%eax, %ecx
-	jnz	L(matches)
-
-	movdqa	(%edi), %xmm3
-	pcmpeqd	%xmm3, %xmm2
-	add	$16, %edi
-	pcmpeqd	%xmm1, %xmm3
-	pmovmskb %xmm2, %ecx
-	pmovmskb %xmm3, %eax
-	or	%eax, %ecx
-	jnz	L(matches)
-
-	movdqa	(%edi), %xmm4
-	pcmpeqd	%xmm4, %xmm2
-	add	$16, %edi
-	pcmpeqd	%xmm1, %xmm4
-	pmovmskb %xmm2, %ecx
-	pmovmskb %xmm4, %eax
-	or	%eax, %ecx
-	jnz	L(matches)
-
-	movdqa	(%edi), %xmm5
-	pcmpeqd	%xmm5, %xmm2
-	add	$16, %edi
-	pcmpeqd	%xmm1, %xmm5
-	pmovmskb %xmm2, %ecx
-	pmovmskb %xmm5, %eax
-	or	%eax, %ecx
-	jz	L(loop)
-
-	.p2align 4
-L(matches):
-	test	%eax, %eax
-	jnz	L(match)
-L(return_value):
-	test	%edx, %edx
-	jz	L(return_null_1)
-	mov	%edx, %eax
-	mov	%esi, %edi
-
-	POP	(%esi)
-
-	test	%ah, %ah
-	jnz	L(match_third_or_fourth_wchar)
-	test	$15 << 4, %al
-	jnz	L(match_second_wchar)
-	lea	-16(%edi), %eax
-	RETURN
-
-	CFI_PUSH	(%esi)
-
-	.p2align 4
-L(return_null_1):
-	POP	(%esi)
-
-	xor	%eax, %eax
-	RETURN
-
-	CFI_PUSH	(%esi)
-
-	.p2align 4
-L(match):
-	pmovmskb %xmm2, %ecx
-	test	%ecx, %ecx
-	jnz	L(find_zero)
-/* save match info */
-	mov	%eax, %edx
-	mov	%edi, %esi
-	jmp	L(loop)
-
-	.p2align 4
-L(find_zero):
-	test	%cl, %cl
-	jz	L(find_zero_in_third_or_fourth_wchar)
-	test	$15, %cl
-	jz	L(find_zero_in_second_wchar)
-	and	$1, %eax
-	jz	L(return_value)
-
-	POP	(%esi)
-
-	lea	-16(%edi), %eax
-	RETURN
-
-	CFI_PUSH	(%esi)
-
-	.p2align 4
-L(find_zero_in_second_wchar):
-	and	$(1 << 5) - 1, %eax
-	jz	L(return_value)
-
-	POP	(%esi)
-
-	test	$15 << 4, %al
-	jnz	L(match_second_wchar)
-	lea	-16(%edi), %eax
-	RETURN
-
-	CFI_PUSH	(%esi)
-
-	.p2align 4
-L(find_zero_in_third_or_fourth_wchar):
-	test	$15, %ch
-	jz	L(find_zero_in_fourth_wchar)
-	and	$(1 << 9) - 1, %eax
-	jz	L(return_value)
-
-	POP	(%esi)
-
-	test	%ah, %ah
-	jnz	L(match_third_wchar)
-	test	$15 << 4, %al
-	jnz	L(match_second_wchar)
-	lea	-16(%edi), %eax
-	RETURN
-
-	CFI_PUSH	(%esi)
-
-	.p2align 4
-L(find_zero_in_fourth_wchar):
-
-	POP	(%esi)
-
-	test	%ah, %ah
-	jnz	L(match_third_or_fourth_wchar)
-	test	$15 << 4, %al
-	jnz	L(match_second_wchar)
-	lea	-16(%edi), %eax
-	RETURN
-
-	CFI_PUSH	(%esi)
-
-	.p2align 4
-L(match_second_wchar):
-	lea	-12(%edi), %eax
-	RETURN
-
-	.p2align 4
-L(match_third_or_fourth_wchar):
-	test	$15 << 4, %ah
-	jnz	L(match_fourth_wchar)
-	lea	-8(%edi), %eax
-	RETURN
-
-	.p2align 4
-L(match_third_wchar):
-	lea	-8(%edi), %eax
-	RETURN
-
-	.p2align 4
-L(match_fourth_wchar):
-	lea	-4(%edi), %eax
-	RETURN
-
-	.p2align 4
-L(return_null):
-	xor	%eax, %eax
-	RETURN
-
-	.p2align 4
-L(prolog_find_zero):
-	add	%ecx, %edi
-	mov     %edx, %ecx
-L(prolog_find_zero_1):
-	test	%cl, %cl
-	jz	L(prolog_find_zero_in_third_or_fourth_wchar)
-	test	$15, %cl
-	jz	L(prolog_find_zero_in_second_wchar)
-	and	$1, %eax
-	jz	L(return_null)
-
-	lea	-16(%edi), %eax
-	RETURN
-
-	.p2align 4
-L(prolog_find_zero_in_second_wchar):
-	and	$(1 << 5) - 1, %eax
-	jz	L(return_null)
-
-	test	$15 << 4, %al
-	jnz	L(match_second_wchar)
-	lea	-16(%edi), %eax
-	RETURN
-
-	.p2align 4
-L(prolog_find_zero_in_third_or_fourth_wchar):
-	test	$15, %ch
-	jz	L(prolog_find_zero_in_fourth_wchar)
-	and	$(1 << 9) - 1, %eax
-	jz	L(return_null)
-
-	test	%ah, %ah
-	jnz	L(match_third_wchar)
-	test	$15 << 4, %al
-	jnz	L(match_second_wchar)
-	lea	-16(%edi), %eax
-	RETURN
-
-	.p2align 4
-L(prolog_find_zero_in_fourth_wchar):
-	test	%ah, %ah
-	jnz	L(match_third_or_fourth_wchar)
-	test	$15 << 4, %al
-	jnz	L(match_second_wchar)
-	lea	-16(%edi), %eax
-	RETURN
-
-END (wcsrchr)
diff --git a/libc/arch-x86/string/sse4-memcmp-slm.S b/libc/arch-x86/string/sse4-memcmp-slm.S
old mode 100755
new mode 100644
diff --git a/libc/arch-x86/string/sse4-wmemcmp-slm.S b/libc/arch-x86/string/sse4-wmemcmp-slm.S
deleted file mode 100755
index 2bf92f5..0000000
--- a/libc/arch-x86/string/sse4-wmemcmp-slm.S
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-Copyright (c) 2014, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#define USE_AS_WMEMCMP
-#define MEMCMP wmemcmp_sse4
-#include "sse4-memcmp-slm.S"
diff --git a/libc/arch-x86/string/ssse3-memcpy-atom.S b/libc/arch-x86/string/ssse3-memcpy-atom.S
deleted file mode 100644
index 83e1985..0000000
--- a/libc/arch-x86/string/ssse3-memcpy-atom.S
+++ /dev/null
@@ -1,3124 +0,0 @@
-/*
-Copyright (c) 2010, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#define FOR_ATOM
-
-#ifndef MEMCPY
-# define MEMCPY	memcpy_atom
-#endif
-
-#ifndef L
-# define L(label)	.L##label
-#endif
-
-#ifndef cfi_startproc
-# define cfi_startproc	.cfi_startproc
-#endif
-
-#ifndef cfi_endproc
-# define cfi_endproc	.cfi_endproc
-#endif
-
-#ifndef cfi_rel_offset
-# define cfi_rel_offset(reg, off)	.cfi_rel_offset reg, off
-#endif
-
-#ifndef cfi_restore
-# define cfi_restore(reg)	.cfi_restore reg
-#endif
-
-#ifndef cfi_adjust_cfa_offset
-# define cfi_adjust_cfa_offset(off)	.cfi_adjust_cfa_offset off
-#endif
-
-#ifndef ENTRY
-# define ENTRY(name)		\
-	.type name,  @function;		\
-	.globl name;		\
-	.p2align 4;		\
-name:		\
-	cfi_startproc
-#endif
-
-#ifndef END
-# define END(name)		\
-	cfi_endproc;		\
-	.size name, .-name
-#endif
-
-#define DEST		PARMS
-#define SRC		DEST+4
-#define LEN		SRC+4
-
-#define CFI_PUSH(REG)		\
-  cfi_adjust_cfa_offset (4);		\
-  cfi_rel_offset (REG, 0)
-
-#define CFI_POP(REG)		\
-  cfi_adjust_cfa_offset (-4);		\
-  cfi_restore (REG)
-
-#define PUSH(REG)	pushl REG; CFI_PUSH (REG)
-#define POP(REG)	popl REG; CFI_POP (REG)
-
-#if (defined SHARED || defined __PIC__)
-# define PARMS		8		/* Preserve EBX.  */
-# define ENTRANCE	PUSH (%ebx);
-# define RETURN_END	POP (%ebx); ret
-# define RETURN		RETURN_END; CFI_PUSH (%ebx)
-# define JMPTBL(I, B)	I - B
-
-# define SETUP_PIC_REG(x)	call	__x86.get_pc_thunk.x
-
-/* Load an entry in a jump table into EBX and branch to it.  TABLE is a
-	jump table with relative offsets.  INDEX is a register contains the
-	index into the jump table.   SCALE is the scale of INDEX. */
-
-# define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE)		\
-    /* We first load PC into EBX.  */		\
-	SETUP_PIC_REG(bx);		\
-    /* Get the address of the jump table.  */		\
-	addl	$(TABLE - .), %ebx;		\
-    /* Get the entry and convert the relative offset to the		\
-	absolute	address.  */		\
-	addl	(%ebx, INDEX, SCALE), %ebx;		\
-    /* We loaded the jump table.  Go.  */		\
-	jmp	*%ebx
-#else
-
-# define PARMS		4
-# define ENTRANCE
-# define RETURN_END	ret
-# define RETURN		RETURN_END
-# define JMPTBL(I, B)	I
-
-/* Branch to an entry in a jump table.  TABLE is a jump table with
-	absolute offsets.  INDEX is a register contains the index into the
-	jump table.  SCALE is the scale of INDEX. */
-
-# define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE)		\
-	jmp	*TABLE(, INDEX, SCALE)
-#endif
-
-	.section .text.ssse3,"ax",@progbits
-ENTRY (MEMCPY)
-	ENTRANCE
-	movl	LEN(%esp), %ecx
-	movl	SRC(%esp), %eax
-	movl	DEST(%esp), %edx
-
-#ifdef USE_AS_MEMMOVE
-	cmp	%eax, %edx
-	jb	L(copy_forward)
-	je	L(fwd_write_0bytes)
-	cmp	$32, %ecx
-	jae	L(memmove_bwd)
-	jmp	L(bk_write_less32bytes_2)
-
-	.p2align 4
-L(memmove_bwd):
-	add	%ecx, %eax
-	cmp	%eax, %edx
-	movl	SRC(%esp), %eax
-	jb	L(copy_backward)
-
-L(copy_forward):
-#endif
-	cmp	$48, %ecx
-	jae	L(48bytesormore)
-
-L(fwd_write_less32bytes):
-#ifndef USE_AS_MEMMOVE
-	cmp	%dl, %al
-	jb	L(bk_write)
-#endif
-	add	%ecx, %edx
-	add	%ecx, %eax
-	BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
-#ifndef USE_AS_MEMMOVE
-	.p2align 4
-L(bk_write):
-	BRANCH_TO_JMPTBL_ENTRY (L(table_48_bytes_bwd), %ecx, 4)
-#endif
-
-	.p2align 4
-L(48bytesormore):
-#ifndef USE_AS_MEMMOVE
-	movlpd	(%eax), %xmm0
-	movlpd	8(%eax), %xmm1
-	movlpd	%xmm0, (%edx)
-	movlpd	%xmm1, 8(%edx)
-#else
-	movdqu	(%eax), %xmm0
-#endif
-	PUSH (%edi)
-	movl	%edx, %edi
-	and	$-16, %edx
-	add	$16, %edx
-	sub	%edx, %edi
-	add	%edi, %ecx
-	sub	%edi, %eax
-
-#ifdef SHARED_CACHE_SIZE_HALF
-	cmp	$SHARED_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_shared_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_shared_cache_size_half, %ecx
-# endif
-#endif
-
-	mov	%eax, %edi
-	jae	L(large_page)
-	and	$0xf, %edi
-	jz	L(shl_0)
-	BRANCH_TO_JMPTBL_ENTRY (L(shl_table), %edi, 4)
-
-	.p2align 4
-L(shl_0):
-#ifdef USE_AS_MEMMOVE
-	movl	DEST+4(%esp), %edi
-	movdqu	%xmm0, (%edi)
-#endif
-	xor	%edi, %edi
-	cmp	$127, %ecx
-	ja	L(shl_0_gobble)
-	lea	-32(%ecx), %ecx
-
-	.p2align 4
-L(shl_0_loop):
-	movdqa	(%eax, %edi), %xmm0
-	movdqa	16(%eax, %edi), %xmm1
-	sub	$32, %ecx
-	movdqa	%xmm0, (%edx, %edi)
-	movdqa	%xmm1, 16(%edx, %edi)
-	lea	32(%edi), %edi
-	jb	L(shl_0_end)
-
-	movdqa	(%eax, %edi), %xmm0
-	movdqa	16(%eax, %edi), %xmm1
-	sub	$32, %ecx
-	movdqa	%xmm0, (%edx, %edi)
-	movdqa	%xmm1, 16(%edx, %edi)
-	lea	32(%edi), %edi
-	jb	L(shl_0_end)
-
-	movdqa	(%eax, %edi), %xmm0
-	movdqa	16(%eax, %edi), %xmm1
-	sub	$32, %ecx
-	movdqa	%xmm0, (%edx, %edi)
-	movdqa	%xmm1, 16(%edx, %edi)
-	lea	32(%edi), %edi
-	jb	L(shl_0_end)
-
-	movdqa	(%eax, %edi), %xmm0
-	movdqa	16(%eax, %edi), %xmm1
-	sub	$32, %ecx
-	movdqa	%xmm0, (%edx, %edi)
-	movdqa	%xmm1, 16(%edx, %edi)
-	lea	32(%edi), %edi
-
-L(shl_0_end):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	add	%edi, %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd_align), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_0_gobble):
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	POP	(%edi)
-	lea	-128(%ecx), %ecx
-	jae	L(shl_0_gobble_mem_loop)
-
-	.p2align 4
-L(shl_0_gobble_cache_loop):
-	movdqa	(%eax), %xmm0
-	movdqa	0x10(%eax), %xmm1
-	movdqa	0x20(%eax), %xmm2
-	movdqa	0x30(%eax), %xmm3
-	movdqa	0x40(%eax), %xmm4
-	movdqa	0x50(%eax), %xmm5
-	movdqa	0x60(%eax), %xmm6
-	movdqa	0x70(%eax), %xmm7
-	lea	0x80(%eax), %eax
-	sub	$128, %ecx
-	movdqa	%xmm0, (%edx)
-	movdqa	%xmm1, 0x10(%edx)
-	movdqa	%xmm2, 0x20(%edx)
-	movdqa	%xmm3, 0x30(%edx)
-	movdqa	%xmm4, 0x40(%edx)
-	movdqa	%xmm5, 0x50(%edx)
-	movdqa	%xmm6, 0x60(%edx)
-	movdqa	%xmm7, 0x70(%edx)
-	lea	0x80(%edx), %edx
-
-	jae	L(shl_0_gobble_cache_loop)
-	cmp	$-0x40, %ecx
-	lea	0x80(%ecx), %ecx
-	jl	L(shl_0_cache_less_64bytes)
-
-	movdqa	(%eax), %xmm0
-	sub	$0x40, %ecx
-	movdqa	0x10(%eax), %xmm1
-	movdqa	%xmm0, (%edx)
-	movdqa	%xmm1, 0x10(%edx)
-	movdqa	0x20(%eax), %xmm0
-	movdqa	0x30(%eax), %xmm1
-	add	$0x40, %eax
-	movdqa	%xmm0, 0x20(%edx)
-	movdqa	%xmm1, 0x30(%edx)
-	add	$0x40, %edx
-
-L(shl_0_cache_less_64bytes):
-	cmp	$0x20, %ecx
-	jb	L(shl_0_cache_less_32bytes)
-	movdqa	(%eax), %xmm0
-	sub	$0x20, %ecx
-	movdqa	0x10(%eax), %xmm1
-	add	$0x20, %eax
-	movdqa	%xmm0, (%edx)
-	movdqa	%xmm1, 0x10(%edx)
-	add	$0x20, %edx
-
-L(shl_0_cache_less_32bytes):
-	cmp	$0x10, %ecx
-	jb	L(shl_0_cache_less_16bytes)
-	sub	$0x10, %ecx
-	movdqa	(%eax), %xmm0
-	add	$0x10, %eax
-	movdqa	%xmm0, (%edx)
-	add	$0x10, %edx
-
-L(shl_0_cache_less_16bytes):
-	add	%ecx, %edx
-	add	%ecx, %eax
-	BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
-
-	.p2align 4
-L(shl_0_gobble_mem_loop):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x280(%eax)
-	prefetcht0 0x1c0(%edx)
-
-	movdqa	(%eax), %xmm0
-	movdqa	0x10(%eax), %xmm1
-	movdqa	0x20(%eax), %xmm2
-	movdqa	0x30(%eax), %xmm3
-	movdqa	0x40(%eax), %xmm4
-	movdqa	0x50(%eax), %xmm5
-	movdqa	0x60(%eax), %xmm6
-	movdqa	0x70(%eax), %xmm7
-	lea	0x80(%eax), %eax
-	sub	$0x80, %ecx
-	movdqa	%xmm0, (%edx)
-	movdqa	%xmm1, 0x10(%edx)
-	movdqa	%xmm2, 0x20(%edx)
-	movdqa	%xmm3, 0x30(%edx)
-	movdqa	%xmm4, 0x40(%edx)
-	movdqa	%xmm5, 0x50(%edx)
-	movdqa	%xmm6, 0x60(%edx)
-	movdqa	%xmm7, 0x70(%edx)
-	lea	0x80(%edx), %edx
-
-	jae	L(shl_0_gobble_mem_loop)
-	cmp	$-0x40, %ecx
-	lea	0x80(%ecx), %ecx
-	jl	L(shl_0_mem_less_64bytes)
-
-	movdqa	(%eax), %xmm0
-	sub	$0x40, %ecx
-	movdqa	0x10(%eax), %xmm1
-
-	movdqa	%xmm0, (%edx)
-	movdqa	%xmm1, 0x10(%edx)
-
-	movdqa	0x20(%eax), %xmm0
-	movdqa	0x30(%eax), %xmm1
-	add	$0x40, %eax
-
-	movdqa	%xmm0, 0x20(%edx)
-	movdqa	%xmm1, 0x30(%edx)
-	add	$0x40, %edx
-
-L(shl_0_mem_less_64bytes):
-	cmp	$0x20, %ecx
-	jb	L(shl_0_mem_less_32bytes)
-	movdqa	(%eax), %xmm0
-	sub	$0x20, %ecx
-	movdqa	0x10(%eax), %xmm1
-	add	$0x20, %eax
-	movdqa	%xmm0, (%edx)
-	movdqa	%xmm1, 0x10(%edx)
-	add	$0x20, %edx
-
-L(shl_0_mem_less_32bytes):
-	cmp	$0x10, %ecx
-	jb	L(shl_0_mem_less_16bytes)
-	sub	$0x10, %ecx
-	movdqa	(%eax), %xmm0
-	add	$0x10, %eax
-	movdqa	%xmm0, (%edx)
-	add	$0x10, %edx
-
-L(shl_0_mem_less_16bytes):
-	add	%ecx, %edx
-	add	%ecx, %eax
-	BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd_align), %ecx, 4)
-
-	.p2align 4
-L(shl_1):
-#ifndef USE_AS_MEMMOVE
-	movaps	-1(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-1(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_1_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl1LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	15(%eax), %xmm2
-	movaps	31(%eax), %xmm3
-	movaps	47(%eax), %xmm4
-	movaps	63(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$1, %xmm4, %xmm5
-	palignr	$1, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$1, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$1, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl1LoopStart)
-
-L(Shl1LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	15(%eax), %xmm2
-	movaps	31(%eax), %xmm3
-	palignr	$1, %xmm2, %xmm3
-	palignr	$1, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_1_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-1(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_1_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$1, %xmm2, %xmm3
-	palignr	$1, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jb	L(sh_1_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$1, %xmm2, %xmm3
-	palignr	$1, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jae	L(sh_1_no_prefetch_loop)
-
-L(sh_1_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	1(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_2):
-#ifndef USE_AS_MEMMOVE
-	movaps	-2(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-2(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_2_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl2LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	14(%eax), %xmm2
-	movaps	30(%eax), %xmm3
-	movaps	46(%eax), %xmm4
-	movaps	62(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$2, %xmm4, %xmm5
-	palignr	$2, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$2, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$2, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl2LoopStart)
-
-L(Shl2LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	14(%eax), %xmm2
-	movaps	30(%eax), %xmm3
-	palignr	$2, %xmm2, %xmm3
-	palignr	$2, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_2_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-2(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_2_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$2, %xmm2, %xmm3
-	palignr	$2, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jb	L(sh_2_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$2, %xmm2, %xmm3
-	palignr	$2, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jae	L(sh_2_no_prefetch_loop)
-
-L(sh_2_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	2(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_3):
-#ifndef USE_AS_MEMMOVE
-	movaps	-3(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-3(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_3_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl3LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	13(%eax), %xmm2
-	movaps	29(%eax), %xmm3
-	movaps	45(%eax), %xmm4
-	movaps	61(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$3, %xmm4, %xmm5
-	palignr	$3, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$3, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$3, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl3LoopStart)
-
-L(Shl3LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	13(%eax), %xmm2
-	movaps	29(%eax), %xmm3
-	palignr	$3, %xmm2, %xmm3
-	palignr	$3, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_3_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-3(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_3_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$3, %xmm2, %xmm3
-	palignr	$3, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-
-	jb	L(sh_3_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$3, %xmm2, %xmm3
-	palignr	$3, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-
-	jae	L(sh_3_no_prefetch_loop)
-
-L(sh_3_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	3(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_4):
-#ifndef USE_AS_MEMMOVE
-	movaps	-4(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-4(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_4_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl4LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	12(%eax), %xmm2
-	movaps	28(%eax), %xmm3
-	movaps	44(%eax), %xmm4
-	movaps	60(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$4, %xmm4, %xmm5
-	palignr	$4, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$4, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$4, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl4LoopStart)
-
-L(Shl4LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	12(%eax), %xmm2
-	movaps	28(%eax), %xmm3
-	palignr	$4, %xmm2, %xmm3
-	palignr	$4, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_4_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-4(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_4_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$4, %xmm2, %xmm3
-	palignr	$4, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-
-	jb	L(sh_4_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$4, %xmm2, %xmm3
-	palignr	$4, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-
-	jae	L(sh_4_no_prefetch_loop)
-
-L(sh_4_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	4(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_5):
-#ifndef USE_AS_MEMMOVE
-	movaps	-5(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-5(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_5_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl5LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	11(%eax), %xmm2
-	movaps	27(%eax), %xmm3
-	movaps	43(%eax), %xmm4
-	movaps	59(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$5, %xmm4, %xmm5
-	palignr	$5, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$5, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$5, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl5LoopStart)
-
-L(Shl5LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	11(%eax), %xmm2
-	movaps	27(%eax), %xmm3
-	palignr	$5, %xmm2, %xmm3
-	palignr	$5, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_5_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-5(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_5_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$5, %xmm2, %xmm3
-	palignr	$5, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-
-	jb	L(sh_5_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$5, %xmm2, %xmm3
-	palignr	$5, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-
-	jae	L(sh_5_no_prefetch_loop)
-
-L(sh_5_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	5(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_6):
-#ifndef USE_AS_MEMMOVE
-	movaps	-6(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-6(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_6_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl6LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	10(%eax), %xmm2
-	movaps	26(%eax), %xmm3
-	movaps	42(%eax), %xmm4
-	movaps	58(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$6, %xmm4, %xmm5
-	palignr	$6, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$6, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$6, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl6LoopStart)
-
-L(Shl6LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	10(%eax), %xmm2
-	movaps	26(%eax), %xmm3
-	palignr	$6, %xmm2, %xmm3
-	palignr	$6, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_6_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-6(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_6_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$6, %xmm2, %xmm3
-	palignr	$6, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-
-	jb	L(sh_6_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$6, %xmm2, %xmm3
-	palignr	$6, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-
-	jae	L(sh_6_no_prefetch_loop)
-
-L(sh_6_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	6(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_7):
-#ifndef USE_AS_MEMMOVE
-	movaps	-7(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-7(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_7_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl7LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	9(%eax), %xmm2
-	movaps	25(%eax), %xmm3
-	movaps	41(%eax), %xmm4
-	movaps	57(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$7, %xmm4, %xmm5
-	palignr	$7, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$7, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$7, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl7LoopStart)
-
-L(Shl7LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	9(%eax), %xmm2
-	movaps	25(%eax), %xmm3
-	palignr	$7, %xmm2, %xmm3
-	palignr	$7, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_7_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-7(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_7_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$7, %xmm2, %xmm3
-	palignr	$7, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jb	L(sh_7_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$7, %xmm2, %xmm3
-	palignr	$7, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jae	L(sh_7_no_prefetch_loop)
-
-L(sh_7_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	7(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_8):
-#ifndef USE_AS_MEMMOVE
-	movaps	-8(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-8(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_8_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl8LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	8(%eax), %xmm2
-	movaps	24(%eax), %xmm3
-	movaps	40(%eax), %xmm4
-	movaps	56(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$8, %xmm4, %xmm5
-	palignr	$8, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$8, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$8, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl8LoopStart)
-
-L(LoopLeave8):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	8(%eax), %xmm2
-	movaps	24(%eax), %xmm3
-	palignr	$8, %xmm2, %xmm3
-	palignr	$8, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_8_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-8(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_8_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$8, %xmm2, %xmm3
-	palignr	$8, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jb	L(sh_8_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$8, %xmm2, %xmm3
-	palignr	$8, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jae	L(sh_8_no_prefetch_loop)
-
-L(sh_8_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	8(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_9):
-#ifndef USE_AS_MEMMOVE
-	movaps	-9(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-9(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_9_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl9LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	7(%eax), %xmm2
-	movaps	23(%eax), %xmm3
-	movaps	39(%eax), %xmm4
-	movaps	55(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$9, %xmm4, %xmm5
-	palignr	$9, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$9, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$9, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl9LoopStart)
-
-L(Shl9LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	7(%eax), %xmm2
-	movaps	23(%eax), %xmm3
-	palignr	$9, %xmm2, %xmm3
-	palignr	$9, %xmm1, %xmm2
-
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_9_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-9(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_9_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$9, %xmm2, %xmm3
-	palignr	$9, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jb	L(sh_9_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$9, %xmm2, %xmm3
-	palignr	$9, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jae	L(sh_9_no_prefetch_loop)
-
-L(sh_9_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	9(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_10):
-#ifndef USE_AS_MEMMOVE
-	movaps	-10(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-10(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_10_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl10LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	6(%eax), %xmm2
-	movaps	22(%eax), %xmm3
-	movaps	38(%eax), %xmm4
-	movaps	54(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$10, %xmm4, %xmm5
-	palignr	$10, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$10, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$10, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl10LoopStart)
-
-L(Shl10LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	6(%eax), %xmm2
-	movaps	22(%eax), %xmm3
-	palignr	$10, %xmm2, %xmm3
-	palignr	$10, %xmm1, %xmm2
-
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_10_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-10(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_10_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$10, %xmm2, %xmm3
-	palignr	$10, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jb	L(sh_10_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$10, %xmm2, %xmm3
-	palignr	$10, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jae	L(sh_10_no_prefetch_loop)
-
-L(sh_10_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	10(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_11):
-#ifndef USE_AS_MEMMOVE
-	movaps	-11(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-11(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_11_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl11LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	5(%eax), %xmm2
-	movaps	21(%eax), %xmm3
-	movaps	37(%eax), %xmm4
-	movaps	53(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$11, %xmm4, %xmm5
-	palignr	$11, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$11, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$11, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl11LoopStart)
-
-L(Shl11LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	5(%eax), %xmm2
-	movaps	21(%eax), %xmm3
-	palignr	$11, %xmm2, %xmm3
-	palignr	$11, %xmm1, %xmm2
-
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_11_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-11(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_11_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$11, %xmm2, %xmm3
-	palignr	$11, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jb	L(sh_11_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$11, %xmm2, %xmm3
-	palignr	$11, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jae	L(sh_11_no_prefetch_loop)
-
-L(sh_11_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	11(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_12):
-#ifndef USE_AS_MEMMOVE
-	movaps	-12(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-12(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_12_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl12LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	4(%eax), %xmm2
-	movaps	20(%eax), %xmm3
-	movaps	36(%eax), %xmm4
-	movaps	52(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$12, %xmm4, %xmm5
-	palignr	$12, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$12, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$12, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl12LoopStart)
-
-L(Shl12LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	4(%eax), %xmm2
-	movaps	20(%eax), %xmm3
-	palignr	$12, %xmm2, %xmm3
-	palignr	$12, %xmm1, %xmm2
-
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_12_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-12(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_12_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$12, %xmm2, %xmm3
-	palignr	$12, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jb	L(sh_12_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$12, %xmm2, %xmm3
-	palignr	$12, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jae	L(sh_12_no_prefetch_loop)
-
-L(sh_12_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	12(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_13):
-#ifndef USE_AS_MEMMOVE
-	movaps	-13(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-13(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_13_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl13LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	3(%eax), %xmm2
-	movaps	19(%eax), %xmm3
-	movaps	35(%eax), %xmm4
-	movaps	51(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$13, %xmm4, %xmm5
-	palignr	$13, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$13, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$13, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl13LoopStart)
-
-L(Shl13LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	3(%eax), %xmm2
-	movaps	19(%eax), %xmm3
-	palignr	$13, %xmm2, %xmm3
-	palignr	$13, %xmm1, %xmm2
-
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_13_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-13(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_13_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$13, %xmm2, %xmm3
-	palignr	$13, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jb	L(sh_13_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$13, %xmm2, %xmm3
-	palignr	$13, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jae	L(sh_13_no_prefetch_loop)
-
-L(sh_13_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	13(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_14):
-#ifndef USE_AS_MEMMOVE
-	movaps	-14(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-14(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_14_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl14LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	2(%eax), %xmm2
-	movaps	18(%eax), %xmm3
-	movaps	34(%eax), %xmm4
-	movaps	50(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$14, %xmm4, %xmm5
-	palignr	$14, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$14, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$14, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl14LoopStart)
-
-L(Shl14LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	2(%eax), %xmm2
-	movaps	18(%eax), %xmm3
-	palignr	$14, %xmm2, %xmm3
-	palignr	$14, %xmm1, %xmm2
-
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_14_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-14(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_14_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$14, %xmm2, %xmm3
-	palignr	$14, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jb	L(sh_14_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$14, %xmm2, %xmm3
-	palignr	$14, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jae	L(sh_14_no_prefetch_loop)
-
-L(sh_14_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	14(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_15):
-#ifndef USE_AS_MEMMOVE
-	movaps	-15(%eax), %xmm1
-#else
-	movl	DEST+4(%esp), %edi
-	movaps	-15(%eax), %xmm1
-	movdqu	%xmm0, (%edi)
-#endif
-#ifdef DATA_CACHE_SIZE_HALF
-	cmp	$DATA_CACHE_SIZE_HALF, %ecx
-#else
-# if (defined SHARED || defined __PIC__)
-	SETUP_PIC_REG(bx)
-	add	$_GLOBAL_OFFSET_TABLE_, %ebx
-	cmp	__x86_data_cache_size_half@GOTOFF(%ebx), %ecx
-# else
-	cmp	__x86_data_cache_size_half, %ecx
-# endif
-#endif
-	jb L(sh_15_no_prefetch)
-
-	lea	-64(%ecx), %ecx
-
-	.p2align 4
-L(Shl15LoopStart):
-	prefetcht0 0x1c0(%eax)
-	prefetcht0 0x1c0(%edx)
-	movaps	1(%eax), %xmm2
-	movaps	17(%eax), %xmm3
-	movaps	33(%eax), %xmm4
-	movaps	49(%eax), %xmm5
-	movaps	%xmm5, %xmm7
-	palignr	$15, %xmm4, %xmm5
-	palignr	$15, %xmm3, %xmm4
-	movaps	%xmm5, 48(%edx)
-	palignr	$15, %xmm2, %xmm3
-	lea	64(%eax), %eax
-	palignr	$15, %xmm1, %xmm2
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm7, %xmm1
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	sub	$64, %ecx
-	ja	L(Shl15LoopStart)
-
-L(Shl15LoopLeave):
-	add	$32, %ecx
-	jle	L(shl_end_0)
-
-	movaps	1(%eax), %xmm2
-	movaps	17(%eax), %xmm3
-	palignr	$15, %xmm2, %xmm3
-	palignr	$15, %xmm1, %xmm2
-
-	movaps	%xmm2, (%edx)
-	movaps	%xmm3, 16(%edx)
-	lea	32(%edx, %ecx), %edx
-	lea	32(%eax, %ecx), %eax
-	POP (%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(sh_15_no_prefetch):
-	lea	-32(%ecx), %ecx
-	lea	-15(%eax), %eax
-	xor	%edi, %edi
-
-	.p2align 4
-L(sh_15_no_prefetch_loop):
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm4
-	palignr	$15, %xmm2, %xmm3
-	palignr	$15, %xmm1, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jb	L(sh_15_end_no_prefetch_loop)
-
-	movdqa	16(%eax, %edi), %xmm2
-	sub	$32, %ecx
-	movdqa	32(%eax, %edi), %xmm3
-	movdqa	%xmm3, %xmm1
-	palignr	$15, %xmm2, %xmm3
-	palignr	$15, %xmm4, %xmm2
-	lea	32(%edi), %edi
-	movdqa	%xmm2, -32(%edx, %edi)
-	movdqa	%xmm3, -16(%edx, %edi)
-	jae	L(sh_15_no_prefetch_loop)
-
-L(sh_15_end_no_prefetch_loop):
-	lea	32(%ecx), %ecx
-	add	%ecx, %edi
-	add	%edi, %edx
-	lea	15(%edi, %eax), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(shl_end_0):
-	lea	32(%ecx), %ecx
-	lea	(%edx, %ecx), %edx
-	lea	(%eax, %ecx), %eax
-	POP	(%edi)
-	BRANCH_TO_JMPTBL_ENTRY(L(table_48bytes_fwd), %ecx, 4)
-
-	.p2align 4
-L(fwd_write_44bytes):
-	movq	-44(%eax), %xmm0
-	movq	%xmm0, -44(%edx)
-L(fwd_write_36bytes):
-	movq	-36(%eax), %xmm0
-	movq	%xmm0, -36(%edx)
-L(fwd_write_28bytes):
-	movq	-28(%eax), %xmm0
-	movq	%xmm0, -28(%edx)
-L(fwd_write_20bytes):
-	movq	-20(%eax), %xmm0
-	movq	%xmm0, -20(%edx)
-L(fwd_write_12bytes):
-	movq	-12(%eax), %xmm0
-	movq	%xmm0, -12(%edx)
-L(fwd_write_4bytes):
-	movl	-4(%eax), %ecx
-	movl	%ecx, -4(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_40bytes):
-	movq	-40(%eax), %xmm0
-	movq	%xmm0, -40(%edx)
-L(fwd_write_32bytes):
-	movq	-32(%eax), %xmm0
-	movq	%xmm0, -32(%edx)
-L(fwd_write_24bytes):
-	movq	-24(%eax), %xmm0
-	movq	%xmm0, -24(%edx)
-L(fwd_write_16bytes):
-	movq	-16(%eax), %xmm0
-	movq	%xmm0, -16(%edx)
-L(fwd_write_8bytes):
-	movq	-8(%eax), %xmm0
-	movq	%xmm0, -8(%edx)
-L(fwd_write_0bytes):
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_5bytes):
-	movl	-5(%eax), %ecx
-	movl	-4(%eax), %eax
-	movl	%ecx, -5(%edx)
-	movl	%eax, -4(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_45bytes):
-	movq	-45(%eax), %xmm0
-	movq	%xmm0, -45(%edx)
-L(fwd_write_37bytes):
-	movq	-37(%eax), %xmm0
-	movq	%xmm0, -37(%edx)
-L(fwd_write_29bytes):
-	movq	-29(%eax), %xmm0
-	movq	%xmm0, -29(%edx)
-L(fwd_write_21bytes):
-	movq	-21(%eax), %xmm0
-	movq	%xmm0, -21(%edx)
-L(fwd_write_13bytes):
-	movq	-13(%eax), %xmm0
-	movq	%xmm0, -13(%edx)
-	movl	-5(%eax), %ecx
-	movl	%ecx, -5(%edx)
-	movzbl	-1(%eax), %ecx
-	movb	%cl, -1(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_41bytes):
-	movq	-41(%eax), %xmm0
-	movq	%xmm0, -41(%edx)
-L(fwd_write_33bytes):
-	movq	-33(%eax), %xmm0
-	movq	%xmm0, -33(%edx)
-L(fwd_write_25bytes):
-	movq	-25(%eax), %xmm0
-	movq	%xmm0, -25(%edx)
-L(fwd_write_17bytes):
-	movq	-17(%eax), %xmm0
-	movq	%xmm0, -17(%edx)
-L(fwd_write_9bytes):
-	movq	-9(%eax), %xmm0
-	movq	%xmm0, -9(%edx)
-L(fwd_write_1bytes):
-	movzbl	-1(%eax), %ecx
-	movb	%cl, -1(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_46bytes):
-	movq	-46(%eax), %xmm0
-	movq	%xmm0, -46(%edx)
-L(fwd_write_38bytes):
-	movq	-38(%eax), %xmm0
-	movq	%xmm0, -38(%edx)
-L(fwd_write_30bytes):
-	movq	-30(%eax), %xmm0
-	movq	%xmm0, -30(%edx)
-L(fwd_write_22bytes):
-	movq	-22(%eax), %xmm0
-	movq	%xmm0, -22(%edx)
-L(fwd_write_14bytes):
-	movq	-14(%eax), %xmm0
-	movq	%xmm0, -14(%edx)
-L(fwd_write_6bytes):
-	movl	-6(%eax), %ecx
-	movl	%ecx, -6(%edx)
-	movzwl	-2(%eax), %ecx
-	movw	%cx, -2(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_42bytes):
-	movq	-42(%eax), %xmm0
-	movq	%xmm0, -42(%edx)
-L(fwd_write_34bytes):
-	movq	-34(%eax), %xmm0
-	movq	%xmm0, -34(%edx)
-L(fwd_write_26bytes):
-	movq	-26(%eax), %xmm0
-	movq	%xmm0, -26(%edx)
-L(fwd_write_18bytes):
-	movq	-18(%eax), %xmm0
-	movq	%xmm0, -18(%edx)
-L(fwd_write_10bytes):
-	movq	-10(%eax), %xmm0
-	movq	%xmm0, -10(%edx)
-L(fwd_write_2bytes):
-	movzwl	-2(%eax), %ecx
-	movw	%cx, -2(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_47bytes):
-	movq	-47(%eax), %xmm0
-	movq	%xmm0, -47(%edx)
-L(fwd_write_39bytes):
-	movq	-39(%eax), %xmm0
-	movq	%xmm0, -39(%edx)
-L(fwd_write_31bytes):
-	movq	-31(%eax), %xmm0
-	movq	%xmm0, -31(%edx)
-L(fwd_write_23bytes):
-	movq	-23(%eax), %xmm0
-	movq	%xmm0, -23(%edx)
-L(fwd_write_15bytes):
-	movq	-15(%eax), %xmm0
-	movq	%xmm0, -15(%edx)
-L(fwd_write_7bytes):
-	movl	-7(%eax), %ecx
-	movl	%ecx, -7(%edx)
-	movzwl	-3(%eax), %ecx
-	movzbl	-1(%eax), %eax
-	movw	%cx, -3(%edx)
-	movb	%al, -1(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_43bytes):
-	movq	-43(%eax), %xmm0
-	movq	%xmm0, -43(%edx)
-L(fwd_write_35bytes):
-	movq	-35(%eax), %xmm0
-	movq	%xmm0, -35(%edx)
-L(fwd_write_27bytes):
-	movq	-27(%eax), %xmm0
-	movq	%xmm0, -27(%edx)
-L(fwd_write_19bytes):
-	movq	-19(%eax), %xmm0
-	movq	%xmm0, -19(%edx)
-L(fwd_write_11bytes):
-	movq	-11(%eax), %xmm0
-	movq	%xmm0, -11(%edx)
-L(fwd_write_3bytes):
-	movzwl	-3(%eax), %ecx
-	movzbl	-1(%eax), %eax
-	movw	%cx, -3(%edx)
-	movb	%al, -1(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_40bytes_align):
-	movdqa	-40(%eax), %xmm0
-	movdqa	%xmm0, -40(%edx)
-L(fwd_write_24bytes_align):
-	movdqa	-24(%eax), %xmm0
-	movdqa	%xmm0, -24(%edx)
-L(fwd_write_8bytes_align):
-	movq	-8(%eax), %xmm0
-	movq	%xmm0, -8(%edx)
-L(fwd_write_0bytes_align):
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_32bytes_align):
-	movdqa	-32(%eax), %xmm0
-	movdqa	%xmm0, -32(%edx)
-L(fwd_write_16bytes_align):
-	movdqa	-16(%eax), %xmm0
-	movdqa	%xmm0, -16(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_5bytes_align):
-	movl	-5(%eax), %ecx
-	movl	-4(%eax), %eax
-	movl	%ecx, -5(%edx)
-	movl	%eax, -4(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_45bytes_align):
-	movdqa	-45(%eax), %xmm0
-	movdqa	%xmm0, -45(%edx)
-L(fwd_write_29bytes_align):
-	movdqa	-29(%eax), %xmm0
-	movdqa	%xmm0, -29(%edx)
-L(fwd_write_13bytes_align):
-	movq	-13(%eax), %xmm0
-	movq	%xmm0, -13(%edx)
-	movl	-5(%eax), %ecx
-	movl	%ecx, -5(%edx)
-	movzbl	-1(%eax), %ecx
-	movb	%cl, -1(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_37bytes_align):
-	movdqa	-37(%eax), %xmm0
-	movdqa	%xmm0, -37(%edx)
-L(fwd_write_21bytes_align):
-	movdqa	-21(%eax), %xmm0
-	movdqa	%xmm0, -21(%edx)
-	movl	-5(%eax), %ecx
-	movl	%ecx, -5(%edx)
-	movzbl	-1(%eax), %ecx
-	movb	%cl, -1(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_41bytes_align):
-	movdqa	-41(%eax), %xmm0
-	movdqa	%xmm0, -41(%edx)
-L(fwd_write_25bytes_align):
-	movdqa	-25(%eax), %xmm0
-	movdqa	%xmm0, -25(%edx)
-L(fwd_write_9bytes_align):
-	movq	-9(%eax), %xmm0
-	movq	%xmm0, -9(%edx)
-L(fwd_write_1bytes_align):
-	movzbl	-1(%eax), %ecx
-	movb	%cl, -1(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_33bytes_align):
-	movdqa	-33(%eax), %xmm0
-	movdqa	%xmm0, -33(%edx)
-L(fwd_write_17bytes_align):
-	movdqa	-17(%eax), %xmm0
-	movdqa	%xmm0, -17(%edx)
-	movzbl	-1(%eax), %ecx
-	movb	%cl, -1(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_46bytes_align):
-	movdqa	-46(%eax), %xmm0
-	movdqa	%xmm0, -46(%edx)
-L(fwd_write_30bytes_align):
-	movdqa	-30(%eax), %xmm0
-	movdqa	%xmm0, -30(%edx)
-L(fwd_write_14bytes_align):
-	movq	-14(%eax), %xmm0
-	movq	%xmm0, -14(%edx)
-L(fwd_write_6bytes_align):
-	movl	-6(%eax), %ecx
-	movl	%ecx, -6(%edx)
-	movzwl	-2(%eax), %ecx
-	movw	%cx, -2(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_38bytes_align):
-	movdqa	-38(%eax), %xmm0
-	movdqa	%xmm0, -38(%edx)
-L(fwd_write_22bytes_align):
-	movdqa	-22(%eax), %xmm0
-	movdqa	%xmm0, -22(%edx)
-	movl	-6(%eax), %ecx
-	movl	%ecx, -6(%edx)
-	movzwl	-2(%eax), %ecx
-	movw	%cx, -2(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_42bytes_align):
-	movdqa	-42(%eax), %xmm0
-	movdqa	%xmm0, -42(%edx)
-L(fwd_write_26bytes_align):
-	movdqa	-26(%eax), %xmm0
-	movdqa	%xmm0, -26(%edx)
-L(fwd_write_10bytes_align):
-	movq	-10(%eax), %xmm0
-	movq	%xmm0, -10(%edx)
-L(fwd_write_2bytes_align):
-	movzwl	-2(%eax), %ecx
-	movw	%cx, -2(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_34bytes_align):
-	movdqa	-34(%eax), %xmm0
-	movdqa	%xmm0, -34(%edx)
-L(fwd_write_18bytes_align):
-	movdqa	-18(%eax), %xmm0
-	movdqa	%xmm0, -18(%edx)
-	movzwl	-2(%eax), %ecx
-	movw	%cx, -2(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_47bytes_align):
-	movdqa	-47(%eax), %xmm0
-	movdqa	%xmm0, -47(%edx)
-L(fwd_write_31bytes_align):
-	movdqa	-31(%eax), %xmm0
-	movdqa	%xmm0, -31(%edx)
-L(fwd_write_15bytes_align):
-	movq	-15(%eax), %xmm0
-	movq	%xmm0, -15(%edx)
-L(fwd_write_7bytes_align):
-	movl	-7(%eax), %ecx
-	movl	%ecx, -7(%edx)
-	movzwl	-3(%eax), %ecx
-	movzbl	-1(%eax), %eax
-	movw	%cx, -3(%edx)
-	movb	%al, -1(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_39bytes_align):
-	movdqa	-39(%eax), %xmm0
-	movdqa	%xmm0, -39(%edx)
-L(fwd_write_23bytes_align):
-	movdqa	-23(%eax), %xmm0
-	movdqa	%xmm0, -23(%edx)
-	movl	-7(%eax), %ecx
-	movl	%ecx, -7(%edx)
-	movzwl	-3(%eax), %ecx
-	movzbl	-1(%eax), %eax
-	movw	%cx, -3(%edx)
-	movb	%al, -1(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_43bytes_align):
-	movdqa	-43(%eax), %xmm0
-	movdqa	%xmm0, -43(%edx)
-L(fwd_write_27bytes_align):
-	movdqa	-27(%eax), %xmm0
-	movdqa	%xmm0, -27(%edx)
-L(fwd_write_11bytes_align):
-	movq	-11(%eax), %xmm0
-	movq	%xmm0, -11(%edx)
-L(fwd_write_3bytes_align):
-	movzwl	-3(%eax), %ecx
-	movzbl	-1(%eax), %eax
-	movw	%cx, -3(%edx)
-	movb	%al, -1(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_35bytes_align):
-	movdqa	-35(%eax), %xmm0
-	movdqa	%xmm0, -35(%edx)
-L(fwd_write_19bytes_align):
-	movdqa	-19(%eax), %xmm0
-	movdqa	%xmm0, -19(%edx)
-	movzwl	-3(%eax), %ecx
-	movzbl	-1(%eax), %eax
-	movw	%cx, -3(%edx)
-	movb	%al, -1(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_44bytes_align):
-	movdqa	-44(%eax), %xmm0
-	movdqa	%xmm0, -44(%edx)
-L(fwd_write_28bytes_align):
-	movdqa	-28(%eax), %xmm0
-	movdqa	%xmm0, -28(%edx)
-L(fwd_write_12bytes_align):
-	movq	-12(%eax), %xmm0
-	movq	%xmm0, -12(%edx)
-L(fwd_write_4bytes_align):
-	movl	-4(%eax), %ecx
-	movl	%ecx, -4(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(fwd_write_36bytes_align):
-	movdqa	-36(%eax), %xmm0
-	movdqa	%xmm0, -36(%edx)
-L(fwd_write_20bytes_align):
-	movdqa	-20(%eax), %xmm0
-	movdqa	%xmm0, -20(%edx)
-	movl	-4(%eax), %ecx
-	movl	%ecx, -4(%edx)
-#ifdef USE_AS_MEMPCPY
-	movl	%edx, %eax
-#else
-	movl	DEST(%esp), %eax
-#endif
-	RETURN_END
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(large_page):
-	movdqu	(%eax), %xmm1
-#ifdef USE_AS_MEMMOVE
-	movl	DEST+4(%esp), %edi
-	movdqu	%xmm0, (%edi)
-#endif
-	lea	16(%eax), %eax
-	movntdq	%xmm1, (%edx)
-	lea	16(%edx), %edx
-	lea	-0x90(%ecx), %ecx
-	POP (%edi)
-
-	.p2align 4
-L(large_page_loop):
-	movdqu	(%eax), %xmm0
-	movdqu	0x10(%eax), %xmm1
-	movdqu	0x20(%eax), %xmm2
-	movdqu	0x30(%eax), %xmm3
-	movdqu	0x40(%eax), %xmm4
-	movdqu	0x50(%eax), %xmm5
-	movdqu	0x60(%eax), %xmm6
-	movdqu	0x70(%eax), %xmm7
-	lea	0x80(%eax), %eax
-
-	sub	$0x80, %ecx
-	movntdq	%xmm0, (%edx)
-	movntdq	%xmm1, 0x10(%edx)
-	movntdq	%xmm2, 0x20(%edx)
-	movntdq	%xmm3, 0x30(%edx)
-	movntdq	%xmm4, 0x40(%edx)
-	movntdq	%xmm5, 0x50(%edx)
-	movntdq	%xmm6, 0x60(%edx)
-	movntdq	%xmm7, 0x70(%edx)
-	lea	0x80(%edx), %edx
-	jae	L(large_page_loop)
-	cmp	$-0x40, %ecx
-	lea	0x80(%ecx), %ecx
-	jl	L(large_page_less_64bytes)
-
-	movdqu	(%eax), %xmm0
-	movdqu	0x10(%eax), %xmm1
-	movdqu	0x20(%eax), %xmm2
-	movdqu	0x30(%eax), %xmm3
-	lea	0x40(%eax), %eax
-
-	movntdq	%xmm0, (%edx)
-	movntdq	%xmm1, 0x10(%edx)
-	movntdq	%xmm2, 0x20(%edx)
-	movntdq	%xmm3, 0x30(%edx)
-	lea	0x40(%edx), %edx
-	sub	$0x40, %ecx
-L(large_page_less_64bytes):
-	cmp	$32, %ecx
-	jb	L(large_page_less_32bytes)
-	movdqu	(%eax), %xmm0
-	movdqu	0x10(%eax), %xmm1
-	lea	0x20(%eax), %eax
-	movntdq	%xmm0, (%edx)
-	movntdq	%xmm1, 0x10(%edx)
-	lea	0x20(%edx), %edx
-	sub	$0x20, %ecx
-L(large_page_less_32bytes):
-	add	%ecx, %edx
-	add	%ecx, %eax
-	sfence
-	BRANCH_TO_JMPTBL_ENTRY (L(table_48bytes_fwd), %ecx, 4)
-
-	.p2align 4
-L(bk_write_44bytes):
-	movq	36(%eax), %xmm0
-	movq	%xmm0, 36(%edx)
-L(bk_write_36bytes):
-	movq	28(%eax), %xmm0
-	movq	%xmm0, 28(%edx)
-L(bk_write_28bytes):
-	movq	20(%eax), %xmm0
-	movq	%xmm0, 20(%edx)
-L(bk_write_20bytes):
-	movq	12(%eax), %xmm0
-	movq	%xmm0, 12(%edx)
-L(bk_write_12bytes):
-	movq	4(%eax), %xmm0
-	movq	%xmm0, 4(%edx)
-L(bk_write_4bytes):
-	movl	(%eax), %ecx
-	movl	%ecx, (%edx)
-L(bk_write_0bytes):
-	movl	DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
-	movl	LEN(%esp), %ecx
-	add	%ecx, %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(bk_write_40bytes):
-	movq	32(%eax), %xmm0
-	movq	%xmm0, 32(%edx)
-L(bk_write_32bytes):
-	movq	24(%eax), %xmm0
-	movq	%xmm0, 24(%edx)
-L(bk_write_24bytes):
-	movq	16(%eax), %xmm0
-	movq	%xmm0, 16(%edx)
-L(bk_write_16bytes):
-	movq	8(%eax), %xmm0
-	movq	%xmm0, 8(%edx)
-L(bk_write_8bytes):
-	movq	(%eax), %xmm0
-	movq	%xmm0, (%edx)
-	movl	DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
-	movl	LEN(%esp), %ecx
-	add	%ecx, %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(bk_write_45bytes):
-	movq	37(%eax), %xmm0
-	movq	%xmm0, 37(%edx)
-L(bk_write_37bytes):
-	movq	29(%eax), %xmm0
-	movq	%xmm0, 29(%edx)
-L(bk_write_29bytes):
-	movq	21(%eax), %xmm0
-	movq	%xmm0, 21(%edx)
-L(bk_write_21bytes):
-	movq	13(%eax), %xmm0
-	movq	%xmm0, 13(%edx)
-L(bk_write_13bytes):
-	movq	5(%eax), %xmm0
-	movq	%xmm0, 5(%edx)
-L(bk_write_5bytes):
-	movl	1(%eax), %ecx
-	movl	%ecx, 1(%edx)
-L(bk_write_1bytes):
-	movzbl	(%eax), %ecx
-	movb	%cl, (%edx)
-	movl	DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
-	movl	LEN(%esp), %ecx
-	add	%ecx, %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(bk_write_41bytes):
-	movq	33(%eax), %xmm0
-	movq	%xmm0, 33(%edx)
-L(bk_write_33bytes):
-	movq	25(%eax), %xmm0
-	movq	%xmm0, 25(%edx)
-L(bk_write_25bytes):
-	movq	17(%eax), %xmm0
-	movq	%xmm0, 17(%edx)
-L(bk_write_17bytes):
-	movq	9(%eax), %xmm0
-	movq	%xmm0, 9(%edx)
-L(bk_write_9bytes):
-	movq	1(%eax), %xmm0
-	movq	%xmm0, 1(%edx)
-	movzbl	(%eax), %ecx
-	movb	%cl, (%edx)
-	movl	DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
-	movl	LEN(%esp), %ecx
-	add	%ecx, %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(bk_write_46bytes):
-	movq	38(%eax), %xmm0
-	movq	%xmm0, 38(%edx)
-L(bk_write_38bytes):
-	movq	30(%eax), %xmm0
-	movq	%xmm0, 30(%edx)
-L(bk_write_30bytes):
-	movq	22(%eax), %xmm0
-	movq	%xmm0, 22(%edx)
-L(bk_write_22bytes):
-	movq	14(%eax), %xmm0
-	movq	%xmm0, 14(%edx)
-L(bk_write_14bytes):
-	movq	6(%eax), %xmm0
-	movq	%xmm0, 6(%edx)
-L(bk_write_6bytes):
-	movl	2(%eax), %ecx
-	movl	%ecx, 2(%edx)
-	movzwl	(%eax), %ecx
-	movw	%cx, (%edx)
-	movl	DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
-	movl	LEN(%esp), %ecx
-	add	%ecx, %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(bk_write_42bytes):
-	movq	34(%eax), %xmm0
-	movq	%xmm0, 34(%edx)
-L(bk_write_34bytes):
-	movq	26(%eax), %xmm0
-	movq	%xmm0, 26(%edx)
-L(bk_write_26bytes):
-	movq	18(%eax), %xmm0
-	movq	%xmm0, 18(%edx)
-L(bk_write_18bytes):
-	movq	10(%eax), %xmm0
-	movq	%xmm0, 10(%edx)
-L(bk_write_10bytes):
-	movq	2(%eax), %xmm0
-	movq	%xmm0, 2(%edx)
-L(bk_write_2bytes):
-	movzwl	(%eax), %ecx
-	movw	%cx, (%edx)
-	movl	DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
-	movl	LEN(%esp), %ecx
-	add	%ecx, %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(bk_write_47bytes):
-	movq	39(%eax), %xmm0
-	movq	%xmm0, 39(%edx)
-L(bk_write_39bytes):
-	movq	31(%eax), %xmm0
-	movq	%xmm0, 31(%edx)
-L(bk_write_31bytes):
-	movq	23(%eax), %xmm0
-	movq	%xmm0, 23(%edx)
-L(bk_write_23bytes):
-	movq	15(%eax), %xmm0
-	movq	%xmm0, 15(%edx)
-L(bk_write_15bytes):
-	movq	7(%eax), %xmm0
-	movq	%xmm0, 7(%edx)
-L(bk_write_7bytes):
-	movl	3(%eax), %ecx
-	movl	%ecx, 3(%edx)
-	movzwl	1(%eax), %ecx
-	movw	%cx, 1(%edx)
-	movzbl	(%eax), %eax
-	movb	%al, (%edx)
-	movl	DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
-	movl	LEN(%esp), %ecx
-	add	%ecx, %eax
-#endif
-	RETURN
-
-	.p2align 4
-L(bk_write_43bytes):
-	movq	35(%eax), %xmm0
-	movq	%xmm0, 35(%edx)
-L(bk_write_35bytes):
-	movq	27(%eax), %xmm0
-	movq	%xmm0, 27(%edx)
-L(bk_write_27bytes):
-	movq	19(%eax), %xmm0
-	movq	%xmm0, 19(%edx)
-L(bk_write_19bytes):
-	movq	11(%eax), %xmm0
-	movq	%xmm0, 11(%edx)
-L(bk_write_11bytes):
-	movq	3(%eax), %xmm0
-	movq	%xmm0, 3(%edx)
-L(bk_write_3bytes):
-	movzwl	1(%eax), %ecx
-	movw	%cx, 1(%edx)
-	movzbl	(%eax), %eax
-	movb	%al, (%edx)
-	movl	DEST(%esp), %eax
-#ifdef USE_AS_MEMPCPY
-	movl	LEN(%esp), %ecx
-	add	%ecx, %eax
-#endif
-	RETURN_END
-
-
-	.pushsection .rodata.ssse3,"a",@progbits
-	.p2align 2
-L(table_48bytes_fwd):
-	.int	JMPTBL (L(fwd_write_0bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_1bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_2bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_3bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_4bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_5bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_6bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_7bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_8bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_9bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_10bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_11bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_12bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_13bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_14bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_15bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_16bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_17bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_18bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_19bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_20bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_21bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_22bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_23bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_24bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_25bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_26bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_27bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_28bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_29bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_30bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_31bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_32bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_33bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_34bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_35bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_36bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_37bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_38bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_39bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_40bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_41bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_42bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_43bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_44bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_45bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_46bytes), L(table_48bytes_fwd))
-	.int	JMPTBL (L(fwd_write_47bytes), L(table_48bytes_fwd))
-
-	.p2align 2
-L(table_48bytes_fwd_align):
-	.int	JMPTBL (L(fwd_write_0bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_1bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_2bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_3bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_4bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_5bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_6bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_7bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_8bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_9bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_10bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_11bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_12bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_13bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_14bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_15bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_16bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_17bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_18bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_19bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_20bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_21bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_22bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_23bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_24bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_25bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_26bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_27bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_28bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_29bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_30bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_31bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_32bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_33bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_34bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_35bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_36bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_37bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_38bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_39bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_40bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_41bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_42bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_43bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_44bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_45bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_46bytes_align), L(table_48bytes_fwd_align))
-	.int	JMPTBL (L(fwd_write_47bytes_align), L(table_48bytes_fwd_align))
-
-	.p2align 2
-L(shl_table):
-	.int	JMPTBL (L(shl_0), L(shl_table))
-	.int	JMPTBL (L(shl_1), L(shl_table))
-	.int	JMPTBL (L(shl_2), L(shl_table))
-	.int	JMPTBL (L(shl_3), L(shl_table))
-	.int	JMPTBL (L(shl_4), L(shl_table))
-	.int	JMPTBL (L(shl_5), L(shl_table))
-	.int	JMPTBL (L(shl_6), L(shl_table))
-	.int	JMPTBL (L(shl_7), L(shl_table))
-	.int	JMPTBL (L(shl_8), L(shl_table))
-	.int	JMPTBL (L(shl_9), L(shl_table))
-	.int	JMPTBL (L(shl_10), L(shl_table))
-	.int	JMPTBL (L(shl_11), L(shl_table))
-	.int	JMPTBL (L(shl_12), L(shl_table))
-	.int	JMPTBL (L(shl_13), L(shl_table))
-	.int	JMPTBL (L(shl_14), L(shl_table))
-	.int	JMPTBL (L(shl_15), L(shl_table))
-
-	.p2align 2
-L(table_48_bytes_bwd):
-	.int	JMPTBL (L(bk_write_0bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_1bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_2bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_3bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_4bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_5bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_6bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_7bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_8bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_9bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_10bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_11bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_12bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_13bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_14bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_15bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_16bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_17bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_18bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_19bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_20bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_21bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_22bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_23bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_24bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_25bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_26bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_27bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_28bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_29bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_30bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_31bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_32bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_33bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_34bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_35bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_36bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_37bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_38bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_39bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_40bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_41bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_42bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_43bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_44bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_45bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_46bytes), L(table_48_bytes_bwd))
-	.int	JMPTBL (L(bk_write_47bytes), L(table_48_bytes_bwd))
-
-	.popsection
-
-#ifdef USE_AS_MEMMOVE
-	.p2align 4
-L(copy_backward):
-	PUSH (%edi)
-	movl	%eax, %edi
-	lea	(%ecx,%edx,1),%edx
-	lea	(%ecx,%edi,1),%edi
-	testl	$0x3, %edx
-	jnz	L(bk_align)
-
-L(bk_aligned_4):
-	cmp	$64, %ecx
-	jae	L(bk_write_more64bytes)
-
-L(bk_write_64bytesless):
-	cmp	$32, %ecx
-	jb	L(bk_write_less32bytes)
-
-L(bk_write_more32bytes):
-	/* Copy 32 bytes at a time.  */
-	sub	$32, %ecx
-	movq	-8(%edi), %xmm0
-	movq	%xmm0, -8(%edx)
-	movq	-16(%edi), %xmm0
-	movq	%xmm0, -16(%edx)
-	movq	-24(%edi), %xmm0
-	movq	%xmm0, -24(%edx)
-	movq	-32(%edi), %xmm0
-	movq	%xmm0, -32(%edx)
-	sub	$32, %edx
-	sub	$32, %edi
-
-L(bk_write_less32bytes):
-	movl	%edi, %eax
-	sub	%ecx, %edx
-	sub	%ecx, %eax
-	POP (%edi)
-L(bk_write_less32bytes_2):
-	BRANCH_TO_JMPTBL_ENTRY (L(table_48_bytes_bwd), %ecx, 4)
-
-	CFI_PUSH (%edi)
-
-	.p2align 4
-L(bk_align):
-	cmp	$8, %ecx
-	jbe	L(bk_write_less32bytes)
-	testl	$1, %edx
-	/* We get here only if (EDX & 3 ) != 0 so if (EDX & 1) ==0,
-	then	(EDX & 2) must be != 0.  */
-	jz	L(bk_got2)
-	sub	$1, %edi
-	sub	$1, %ecx
-	sub	$1, %edx
-	movzbl	(%edi), %eax
-	movb	%al, (%edx)
-
-	testl	$2, %edx
-	jz	L(bk_aligned_4)
-
-L(bk_got2):
-	sub	$2, %edi
-	sub	$2, %ecx
-	sub	$2, %edx
-	movzwl	(%edi), %eax
-	movw	%ax, (%edx)
-	jmp	L(bk_aligned_4)
-
-	.p2align 4
-L(bk_write_more64bytes):
-	/* Check alignment of last byte.  */
-	testl	$15, %edx
-	jz	L(bk_ssse3_cpy_pre)
-
-/* EDX is aligned 4 bytes, but not 16 bytes.  */
-L(bk_ssse3_align):
-	sub	$4, %edi
-	sub	$4, %ecx
-	sub	$4, %edx
-	movl	(%edi), %eax
-	movl	%eax, (%edx)
-
-	testl	$15, %edx
-	jz	L(bk_ssse3_cpy_pre)
-
-	sub	$4, %edi
-	sub	$4, %ecx
-	sub	$4, %edx
-	movl	(%edi), %eax
-	movl	%eax, (%edx)
-
-	testl	$15, %edx
-	jz	L(bk_ssse3_cpy_pre)
-
-	sub	$4, %edi
-	sub	$4, %ecx
-	sub	$4, %edx
-	movl	(%edi), %eax
-	movl	%eax, (%edx)
-
-L(bk_ssse3_cpy_pre):
-	cmp	$64, %ecx
-	jb	L(bk_write_more32bytes)
-
-	.p2align 4
-L(bk_ssse3_cpy):
-	sub	$64, %edi
-	sub	$64, %ecx
-	sub	$64, %edx
-	movdqu	0x30(%edi), %xmm3
-	movdqa	%xmm3, 0x30(%edx)
-	movdqu	0x20(%edi), %xmm2
-	movdqa	%xmm2, 0x20(%edx)
-	movdqu	0x10(%edi), %xmm1
-	movdqa	%xmm1, 0x10(%edx)
-	movdqu	(%edi), %xmm0
-	movdqa	%xmm0, (%edx)
-	cmp	$64, %ecx
-	jae	L(bk_ssse3_cpy)
-	jmp	L(bk_write_64bytesless)
-
-#endif
-
-END (MEMCPY)
diff --git a/libc/arch-x86/string/ssse3-memmove-atom.S b/libc/arch-x86/string/ssse3-memmove-atom.S
deleted file mode 100644
index 3572eac..0000000
--- a/libc/arch-x86/string/ssse3-memmove-atom.S
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
-Copyright (c) 2010, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-
-#define MEMCPY memmove_atom
-#define USE_AS_MEMMOVE
-#include "ssse3-memcpy-atom.S"
diff --git a/libc/arch-x86/string/ssse3-strncpy-atom.S b/libc/arch-x86/string/ssse3-strncpy-atom.S
deleted file mode 100644
index 0c27ffe..0000000
--- a/libc/arch-x86/string/ssse3-strncpy-atom.S
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
-Copyright (c) 2011, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#define USE_AS_STRNCPY
-#define STRCPY strncpy_atom
-#include "ssse3-strcpy-atom.S"
diff --git a/libc/arch-x86/string/ssse3-wcscat-atom.S b/libc/arch-x86/string/ssse3-wcscat-atom.S
deleted file mode 100644
index 8a389a3..0000000
--- a/libc/arch-x86/string/ssse3-wcscat-atom.S
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
-Copyright (c) 2011 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef L
-# define L(label)	.L##label
-#endif
-
-#ifndef cfi_startproc
-# define cfi_startproc                  .cfi_startproc
-#endif
-
-#ifndef cfi_endproc
-# define cfi_endproc                    .cfi_endproc
-#endif
-
-#ifndef cfi_rel_offset
-# define cfi_rel_offset(reg, off)	.cfi_rel_offset reg, off
-#endif
-
-#ifndef cfi_restore
-# define cfi_restore(reg)	.cfi_restore reg
-#endif
-
-#ifndef cfi_adjust_cfa_offset
-# define cfi_adjust_cfa_offset(off)	.cfi_adjust_cfa_offset off
-#endif
-
-#ifndef ENTRY
-# define ENTRY(name)	\
-	.type name,  @function;	\
-	.globl name;	\
-	.p2align 4;	\
-name:	\
-	cfi_startproc
-#endif
-
-#ifndef END
-# define END(name)	\
-	cfi_endproc;	\
-	.size name, .-name
-#endif
-
-#define CFI_PUSH(REG)	\
-	cfi_adjust_cfa_offset (4);	\
-	cfi_rel_offset (REG, 0)
-
-#define CFI_POP(REG)	\
-	cfi_adjust_cfa_offset (-4);	\
-	cfi_restore (REG)
-
-#define PUSH(REG)	pushl REG;	CFI_PUSH (REG)
-#define POP(REG)	popl REG;	CFI_POP (REG)
-
-#define PARMS  4
-#define STR1  PARMS+4
-#define STR2  STR1+4
-
-#define USE_AS_WCSCAT
-
-.text
-ENTRY (wcscat)
-	PUSH    (%edi)
-	mov	STR1(%esp), %edi
-	mov	%edi, %edx
-
-#define RETURN  jmp L(WcscpyAtom)
-#include "sse2-wcslen-atom.S"
-
-L(WcscpyAtom):
-	shl	$2, %eax
-	mov	STR2(%esp), %ecx
-	lea	(%edi, %eax), %edx
-
-	cmpl	$0, (%ecx)
-	jz	L(Exit4)
-	cmpl	$0, 4(%ecx)
-	jz	L(Exit8)
-	cmpl	$0, 8(%ecx)
-	jz	L(Exit12)
-	cmpl	$0, 12(%ecx)
-	jz	L(Exit16)
-
-#undef RETURN
-#define RETURN  POP(%edi);	ret;	CFI_PUSH(%edi)
-#include "ssse3-wcscpy-atom.S"
-
-END (wcscat)
diff --git a/libc/arch-x86/string/ssse3-wcscpy-atom.S b/libc/arch-x86/string/ssse3-wcscpy-atom.S
deleted file mode 100644
index 27cb61e..0000000
--- a/libc/arch-x86/string/ssse3-wcscpy-atom.S
+++ /dev/null
@@ -1,652 +0,0 @@
-/*
-Copyright (c) 2011, Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef USE_AS_WCSCAT
-
-# ifndef L
-#  define L(label)	.L##label
-# endif
-
-# ifndef cfi_startproc
-#  define cfi_startproc	.cfi_startproc
-# endif
-
-# ifndef cfi_endproc
-#  define cfi_endproc	.cfi_endproc
-# endif
-
-# ifndef cfi_rel_offset
-#  define cfi_rel_offset(reg, off)	.cfi_rel_offset reg, off
-# endif
-
-# ifndef cfi_restore
-#  define cfi_restore(reg)	.cfi_restore reg
-# endif
-
-# ifndef cfi_adjust_cfa_offset
-#  define cfi_adjust_cfa_offset(off)	.cfi_adjust_cfa_offset off
-# endif
-
-# ifndef ENTRY
-#  define ENTRY(name)	\
-	.type name, @function;	\
-	.globl name;	\
-	.p2align 4;	\
-name:	\
-	cfi_startproc
-# endif
-
-# ifndef END
-#  define END(name)	\
-	cfi_endproc;	\
-	.size name, .-name
-# endif
-
-# define CFI_PUSH(REG)	\
-	cfi_adjust_cfa_offset (4);	\
-	cfi_rel_offset (REG, 0)
-
-# define CFI_POP(REG)	\
-	cfi_adjust_cfa_offset (-4);	\
-	cfi_restore (REG)
-
-# define PUSH(REG)	pushl REG; CFI_PUSH (REG)
-# define POP(REG)	popl REG; CFI_POP (REG)
-
-# define PARMS	4
-# define RETURN	POP (%edi); ret; CFI_PUSH (%edi)
-
-# define STR1	PARMS
-# define STR2	STR1+4
-# define LEN	STR2+4
-
-.text
-ENTRY (wcscpy)
-	mov	STR1(%esp), %edx
-	mov	STR2(%esp), %ecx
-
-	cmpl	$0, (%ecx)
-	jz	L(ExitTail4)
-	cmpl	$0, 4(%ecx)
-	jz	L(ExitTail8)
-	cmpl	$0, 8(%ecx)
-	jz	L(ExitTail12)
-	cmpl	$0, 12(%ecx)
-	jz	L(ExitTail16)
-
-	PUSH	(%edi)
-	mov	%edx, %edi
-#endif
-	PUSH	(%esi)
-	lea	16(%ecx), %esi
-
-	and	$-16, %esi
-
-	pxor	%xmm0, %xmm0
-	pcmpeqd	(%esi), %xmm0
-	movdqu	(%ecx), %xmm1
-	movdqu	%xmm1, (%edx)
-
-	pmovmskb %xmm0, %eax
-	sub	%ecx, %esi
-
-	test	%eax, %eax
-	jnz	L(CopyFrom1To16Bytes)
-
-	mov	%edx, %eax
-	lea	16(%edx), %edx
-	and	$-16, %edx
-	sub	%edx, %eax
-
-	sub	%eax, %ecx
-	mov	%ecx, %eax
-	and	$0xf, %eax
-	mov	$0, %esi
-
-	jz	L(Align16Both)
-	cmp	$4, %eax
-	je	L(Shl4)
-	cmp	$8, %eax
-	je	L(Shl8)
-	jmp	L(Shl12)
-
-L(Align16Both):
-	movaps	(%ecx), %xmm1
-	movaps	16(%ecx), %xmm2
-	movaps	%xmm1, (%edx)
-	pcmpeqd	%xmm2, %xmm0
-	pmovmskb %xmm0, %eax
-	lea	16(%esi), %esi
-
-	test	%eax, %eax
-	jnz	L(CopyFrom1To16Bytes)
-
-	movaps	16(%ecx, %esi), %xmm3
-	movaps	%xmm2, (%edx, %esi)
-	pcmpeqd	%xmm3, %xmm0
-	pmovmskb %xmm0, %eax
-	lea	16(%esi), %esi
-
-	test	%eax, %eax
-	jnz	L(CopyFrom1To16Bytes)
-
-	movaps	16(%ecx, %esi), %xmm4
-	movaps	%xmm3, (%edx, %esi)
-	pcmpeqd	%xmm4, %xmm0
-	pmovmskb %xmm0, %eax
-	lea	16(%esi), %esi
-
-	test	%eax, %eax
-	jnz	L(CopyFrom1To16Bytes)
-
-	movaps	16(%ecx, %esi), %xmm1
-	movaps	%xmm4, (%edx, %esi)
-	pcmpeqd	%xmm1, %xmm0
-	pmovmskb %xmm0, %eax
-	lea	16(%esi), %esi
-
-	test	%eax, %eax
-	jnz	L(CopyFrom1To16Bytes)
-
-	movaps	16(%ecx, %esi), %xmm2
-	movaps	%xmm1, (%edx, %esi)
-	pcmpeqd	%xmm2, %xmm0
-	pmovmskb %xmm0, %eax
-	lea	16(%esi), %esi
-
-	test	%eax, %eax
-	jnz	L(CopyFrom1To16Bytes)
-
-	movaps	16(%ecx, %esi), %xmm3
-	movaps	%xmm2, (%edx, %esi)
-	pcmpeqd	%xmm3, %xmm0
-	pmovmskb %xmm0, %eax
-	lea	16(%esi), %esi
-
-	test	%eax, %eax
-	jnz	L(CopyFrom1To16Bytes)
-
-	movaps	%xmm3, (%edx, %esi)
-	mov	%ecx, %eax
-	lea	16(%ecx, %esi), %ecx
-	and	$-0x40, %ecx
-	sub	%ecx, %eax
-	sub	%eax, %edx
-
-	mov	$-0x40, %esi
-
-L(Aligned64Loop):
-	movaps	(%ecx), %xmm2
-	movaps	32(%ecx), %xmm3
-	movaps	%xmm2, %xmm4
-	movaps	16(%ecx), %xmm5
-	movaps	%xmm3, %xmm6
-	movaps	48(%ecx), %xmm7
-	pminub	%xmm5, %xmm2
-	pminub	%xmm7, %xmm3
-	pminub	%xmm2, %xmm3
-	lea	64(%edx), %edx
-	pcmpeqd	%xmm0, %xmm3
-	lea	64(%ecx), %ecx
-	pmovmskb %xmm3, %eax
-
-	test	%eax, %eax
-	jnz	L(Aligned64Leave)
-	movaps	%xmm4, -64(%edx)
-	movaps	%xmm5, -48(%edx)
-	movaps	%xmm6, -32(%edx)
-	movaps	%xmm7, -16(%edx)
-	jmp	L(Aligned64Loop)
-
-L(Aligned64Leave):
-	pcmpeqd	%xmm4, %xmm0
-	pmovmskb %xmm0, %eax
-	test	%eax, %eax
-	jnz	L(CopyFrom1To16Bytes)
-
-	pcmpeqd	%xmm5, %xmm0
-	pmovmskb %xmm0, %eax
-	movaps	%xmm4, -64(%edx)
-	lea	16(%esi), %esi
-	test	%eax, %eax
-	jnz	L(CopyFrom1To16Bytes)
-
-	pcmpeqd	%xmm6, %xmm0
-	pmovmskb %xmm0, %eax
-	movaps	%xmm5, -48(%edx)
-	lea	16(%esi), %esi
-	test	%eax, %eax
-	jnz	L(CopyFrom1To16Bytes)
-
-	movaps	%xmm6, -32(%edx)
-	pcmpeqd	%xmm7, %xmm0
-	pmovmskb %xmm0, %eax
-	lea	16(%esi), %esi
-	test	%eax, %eax
-	jnz	L(CopyFrom1To16Bytes)
-
-	mov	$-0x40, %esi
-	movaps	%xmm7, -16(%edx)
-	jmp	L(Aligned64Loop)
-
-	.p2align 4
-L(Shl4):
-	movaps	-4(%ecx), %xmm1
-	movaps	12(%ecx), %xmm2
-L(Shl4Start):
-	pcmpeqd	%xmm2, %xmm0
-	pmovmskb %xmm0, %eax
-	movaps	%xmm2, %xmm3
-
-	test	%eax, %eax
-	jnz	L(Shl4LoopExit)
-
-	palignr	$4, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	28(%ecx), %xmm2
-
-	pcmpeqd	%xmm2, %xmm0
-	lea	16(%edx), %edx
-	pmovmskb %xmm0, %eax
-	lea	16(%ecx), %ecx
-	movaps	%xmm2, %xmm1
-
-	test	%eax, %eax
-	jnz	L(Shl4LoopExit)
-
-	palignr	$4, %xmm3, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	28(%ecx), %xmm2
-
-	pcmpeqd	%xmm2, %xmm0
-	lea	16(%edx), %edx
-	pmovmskb %xmm0, %eax
-	lea	16(%ecx), %ecx
-	movaps	%xmm2, %xmm3
-
-	test	%eax, %eax
-	jnz	L(Shl4LoopExit)
-
-	palignr	$4, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	28(%ecx), %xmm2
-
-	pcmpeqd	%xmm2, %xmm0
-	lea	16(%edx), %edx
-	pmovmskb %xmm0, %eax
-	lea	16(%ecx), %ecx
-
-	test	%eax, %eax
-	jnz	L(Shl4LoopExit)
-
-	palignr	$4, %xmm3, %xmm2
-	movaps	%xmm2, (%edx)
-	lea	28(%ecx), %ecx
-	lea	16(%edx), %edx
-
-	mov	%ecx, %eax
-	and	$-0x40, %ecx
-	sub	%ecx, %eax
-	lea	-12(%ecx), %ecx
-	sub	%eax, %edx
-
-	movaps	-4(%ecx), %xmm1
-
-L(Shl4LoopStart):
-	movaps	12(%ecx), %xmm2
-	movaps	28(%ecx), %xmm3
-	movaps	%xmm3, %xmm6
-	movaps	44(%ecx), %xmm4
-	movaps	%xmm4, %xmm7
-	movaps	60(%ecx), %xmm5
-	pminub	%xmm2, %xmm6
-	pminub	%xmm5, %xmm7
-	pminub	%xmm6, %xmm7
-	pcmpeqd	%xmm0, %xmm7
-	pmovmskb %xmm7, %eax
-	movaps	%xmm5, %xmm7
-	palignr	$4, %xmm4, %xmm5
-	palignr	$4, %xmm3, %xmm4
-	test	%eax, %eax
-	jnz	L(Shl4Start)
-
-	palignr	$4, %xmm2, %xmm3
-	lea	64(%ecx), %ecx
-	palignr	$4, %xmm1, %xmm2
-	movaps	%xmm7, %xmm1
-	movaps	%xmm5, 48(%edx)
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	jmp	L(Shl4LoopStart)
-
-L(Shl4LoopExit):
-	movlpd	(%ecx), %xmm0
-	movl	8(%ecx), %esi
-	movlpd	%xmm0, (%edx)
-	movl	%esi, 8(%edx)
-	POP	(%esi)
-	add	$12, %edx
-	add	$12, %ecx
-	test	%al, %al
-	jz	L(ExitHigh)
-	test	$0x01, %al
-	jnz	L(Exit4)
-	movlpd	(%ecx), %xmm0
-	movlpd	%xmm0, (%edx)
-	movl	%edi, %eax
-	RETURN
-
-	CFI_PUSH	(%esi)
-
-	.p2align 4
-L(Shl8):
-	movaps	-8(%ecx), %xmm1
-	movaps	8(%ecx), %xmm2
-L(Shl8Start):
-	pcmpeqd	%xmm2, %xmm0
-	pmovmskb %xmm0, %eax
-	movaps	%xmm2, %xmm3
-
-	test	%eax, %eax
-	jnz	L(Shl8LoopExit)
-
-	palignr	$8, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	24(%ecx), %xmm2
-
-	pcmpeqd	%xmm2, %xmm0
-	lea	16(%edx), %edx
-	pmovmskb %xmm0, %eax
-	lea	16(%ecx), %ecx
-	movaps	%xmm2, %xmm1
-
-	test	%eax, %eax
-	jnz	L(Shl8LoopExit)
-
-	palignr	$8, %xmm3, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	24(%ecx), %xmm2
-
-	pcmpeqd	%xmm2, %xmm0
-	lea	16(%edx), %edx
-	pmovmskb %xmm0, %eax
-	lea	16(%ecx), %ecx
-	movaps	%xmm2, %xmm3
-
-	test	%eax, %eax
-	jnz	L(Shl8LoopExit)
-
-	palignr	$8, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	24(%ecx), %xmm2
-
-	pcmpeqd	%xmm2, %xmm0
-	lea	16(%edx), %edx
-	pmovmskb %xmm0, %eax
-	lea	16(%ecx), %ecx
-
-	test	%eax, %eax
-	jnz	L(Shl8LoopExit)
-
-	palignr	$8, %xmm3, %xmm2
-	movaps	%xmm2, (%edx)
-	lea	24(%ecx), %ecx
-	lea	16(%edx), %edx
-
-	mov	%ecx, %eax
-	and	$-0x40, %ecx
-	sub	%ecx, %eax
-	lea	-8(%ecx), %ecx
-	sub	%eax, %edx
-
-	movaps	-8(%ecx), %xmm1
-
-L(Shl8LoopStart):
-	movaps	8(%ecx), %xmm2
-	movaps	24(%ecx), %xmm3
-	movaps	%xmm3, %xmm6
-	movaps	40(%ecx), %xmm4
-	movaps	%xmm4, %xmm7
-	movaps	56(%ecx), %xmm5
-	pminub	%xmm2, %xmm6
-	pminub	%xmm5, %xmm7
-	pminub	%xmm6, %xmm7
-	pcmpeqd	%xmm0, %xmm7
-	pmovmskb %xmm7, %eax
-	movaps	%xmm5, %xmm7
-	palignr	$8, %xmm4, %xmm5
-	palignr	$8, %xmm3, %xmm4
-	test	%eax, %eax
-	jnz	L(Shl8Start)
-
-	palignr	$8, %xmm2, %xmm3
-	lea	64(%ecx), %ecx
-	palignr	$8, %xmm1, %xmm2
-	movaps	%xmm7, %xmm1
-	movaps	%xmm5, 48(%edx)
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	jmp	L(Shl8LoopStart)
-
-L(Shl8LoopExit):
-	movlpd	(%ecx), %xmm0
-	movlpd	%xmm0, (%edx)
-	POP	(%esi)
-	add	$8, %edx
-	add	$8, %ecx
-	test	%al, %al
-	jz	L(ExitHigh)
-	test	$0x01, %al
-	jnz	L(Exit4)
-	movlpd	(%ecx), %xmm0
-	movlpd	%xmm0, (%edx)
-	movl	%edi, %eax
-	RETURN
-
-	CFI_PUSH	(%esi)
-
-	.p2align 4
-L(Shl12):
-	movaps	-12(%ecx), %xmm1
-	movaps	4(%ecx), %xmm2
-L(Shl12Start):
-	pcmpeqd	%xmm2, %xmm0
-	pmovmskb %xmm0, %eax
-	movaps	%xmm2, %xmm3
-
-	test	%eax, %eax
-	jnz	L(Shl12LoopExit)
-
-	palignr	$12, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	20(%ecx), %xmm2
-
-	pcmpeqd	%xmm2, %xmm0
-	lea	16(%edx), %edx
-	pmovmskb %xmm0, %eax
-	lea	16(%ecx), %ecx
-	movaps	%xmm2, %xmm1
-
-	test	%eax, %eax
-	jnz	L(Shl12LoopExit)
-
-	palignr	$12, %xmm3, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	20(%ecx), %xmm2
-
-	pcmpeqd	%xmm2, %xmm0
-	lea	16(%edx), %edx
-	pmovmskb %xmm0, %eax
-	lea	16(%ecx), %ecx
-	movaps	%xmm2, %xmm3
-
-	test	%eax, %eax
-	jnz	L(Shl12LoopExit)
-
-	palignr	$12, %xmm1, %xmm2
-	movaps	%xmm2, (%edx)
-	movaps	20(%ecx), %xmm2
-
-	pcmpeqd	%xmm2, %xmm0
-	lea	16(%edx), %edx
-	pmovmskb %xmm0, %eax
-	lea	16(%ecx), %ecx
-
-	test	%eax, %eax
-	jnz	L(Shl12LoopExit)
-
-	palignr	$12, %xmm3, %xmm2
-	movaps	%xmm2, (%edx)
-	lea	20(%ecx), %ecx
-	lea	16(%edx), %edx
-
-	mov	%ecx, %eax
-	and	$-0x40, %ecx
-	sub	%ecx, %eax
-	lea	-4(%ecx), %ecx
-	sub	%eax, %edx
-
-	movaps	-12(%ecx), %xmm1
-
-L(Shl12LoopStart):
-	movaps	4(%ecx), %xmm2
-	movaps	20(%ecx), %xmm3
-	movaps	%xmm3, %xmm6
-	movaps	36(%ecx), %xmm4
-	movaps	%xmm4, %xmm7
-	movaps	52(%ecx), %xmm5
-	pminub	%xmm2, %xmm6
-	pminub	%xmm5, %xmm7
-	pminub	%xmm6, %xmm7
-	pcmpeqd	%xmm0, %xmm7
-	pmovmskb %xmm7, %eax
-	movaps	%xmm5, %xmm7
-	palignr	$12, %xmm4, %xmm5
-	palignr	$12, %xmm3, %xmm4
-	test	%eax, %eax
-	jnz	L(Shl12Start)
-
-	palignr	$12, %xmm2, %xmm3
-	lea	64(%ecx), %ecx
-	palignr	$12, %xmm1, %xmm2
-	movaps	%xmm7, %xmm1
-	movaps	%xmm5, 48(%edx)
-	movaps	%xmm4, 32(%edx)
-	movaps	%xmm3, 16(%edx)
-	movaps	%xmm2, (%edx)
-	lea	64(%edx), %edx
-	jmp	L(Shl12LoopStart)
-
-L(Shl12LoopExit):
-	movl	(%ecx), %esi
-	movl	%esi, (%edx)
-	mov	$4, %esi
-
-	.p2align 4
-L(CopyFrom1To16Bytes):
-	add	%esi, %edx
-	add	%esi, %ecx
-
-	POP	(%esi)
-	test	%al, %al
-	jz	L(ExitHigh)
-	test	$0x01, %al
-	jnz	L(Exit4)
-L(Exit8):
-	movlpd	(%ecx), %xmm0
-	movlpd	%xmm0, (%edx)
-	movl	%edi, %eax
-	RETURN
-
-	.p2align 4
-L(ExitHigh):
-	test	$0x01, %ah
-	jnz	L(Exit12)
-L(Exit16):
-	movdqu	(%ecx), %xmm0
-	movdqu	%xmm0, (%edx)
-	movl	%edi, %eax
-	RETURN
-
-	.p2align 4
-L(Exit4):
-	movl	(%ecx), %eax
-	movl	%eax, (%edx)
-	movl	%edi, %eax
-	RETURN
-
-	.p2align 4
-L(Exit12):
-	movlpd	(%ecx), %xmm0
-	movlpd	%xmm0, (%edx)
-	movl	8(%ecx), %eax
-	movl	%eax, 8(%edx)
-	movl	%edi, %eax
-	RETURN
-
-CFI_POP	(%edi)
-
-	.p2align 4
-L(ExitTail4):
-	movl	(%ecx), %eax
-	movl	%eax, (%edx)
-	movl	%edx, %eax
-	ret
-
-	.p2align 4
-L(ExitTail8):
-	movlpd	(%ecx), %xmm0
-	movlpd	%xmm0, (%edx)
-	movl	%edx, %eax
-	ret
-
-	.p2align 4
-L(ExitTail12):
-	movlpd	(%ecx), %xmm0
-	movlpd	%xmm0, (%edx)
-	movl	8(%ecx), %eax
-	movl	%eax, 8(%edx)
-	movl	%edx, %eax
-	ret
-
-	.p2align 4
-L(ExitTail16):
-	movdqu	(%ecx), %xmm0
-	movdqu	%xmm0, (%edx)
-	movl	%edx, %eax
-	ret
-
-#ifndef USE_AS_WCSCAT
-END (wcscpy)
-#endif
diff --git a/libc/arch-x86/string/ssse3-wmemcmp-atom.S b/libc/arch-x86/string/ssse3-wmemcmp-atom.S
deleted file mode 100644
index a81b78b..0000000
--- a/libc/arch-x86/string/ssse3-wmemcmp-atom.S
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
-Copyright (c) 2011, 2012, 2013 Intel Corporation
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice,
-    * this list of conditions and the following disclaimer.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-    * this list of conditions and the following disclaimer in the documentation
-    * and/or other materials provided with the distribution.
-
-    * Neither the name of Intel Corporation nor the names of its contributors
-    * may be used to endorse or promote products derived from this software
-    * without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#define MEMCMP  wmemcmp_atom
-
-#define USE_WCHAR
-#define USE_AS_WMEMCMP 1
-#include "ssse3-memcmp-atom.S"
diff --git a/libc/include/android/versioning.h b/libc/include/android/versioning.h
index fe9264d..4411aa1 100644
--- a/libc/include/android/versioning.h
+++ b/libc/include/android/versioning.h
@@ -60,8 +60,8 @@
 #define __INTRODUCED_IN_64(api_level) __BIONIC_AVAILABILITY(introduced=api_level)
 #endif
 
-// Vendor modules do not follow SDK versioning. Ignore NDK guards for vendor modules.
-#if defined(__ANDROID_VENDOR__)
+// Vendor and product modules do not follow SDK versioning. Ignore NDK guards for these modules.
+#if defined(__ANDROID_VNDK__)
 #undef __BIONIC_AVAILABILITY
 #define __BIONIC_AVAILABILITY(api_level, ...)
-#endif // defined(__ANDROID_VENDOR__)
+#endif // defined(__ANDROID_VNDK__)
diff --git a/libc/include/bits/fortify/string.h b/libc/include/bits/fortify/string.h
index 4d32b04..041967b 100644
--- a/libc/include/bits/fortify/string.h
+++ b/libc/include/bits/fortify/string.h
@@ -220,8 +220,13 @@
 }
 
 #if __BIONIC_FORTIFY_RUNTIME_CHECKS_ENABLED
-__BIONIC_FORTIFY_INLINE
-size_t strlen(const char* _Nonnull const s __pass_object_size0) __overloadable {
+/*
+ * Clang, when parsing C, can fold strlen to a constant without LLVM's help.
+ * This doesn't apply to overloads of strlen, so write this differently. We
+ * can't use `__pass_object_size0` here, but that's fine: it doesn't help much
+ * on __always_inline functions.
+ */
+extern __always_inline __inline__ __attribute__((gnu_inline)) size_t strlen(const char* _Nonnull s) {
     return __strlen_chk(s, __bos0(s));
 }
 #endif
diff --git a/libc/include/bits/glibc-syscalls.h b/libc/include/bits/glibc-syscalls.h
index 7b171e8..8c5a91d 100644
--- a/libc/include/bits/glibc-syscalls.h
+++ b/libc/include/bits/glibc-syscalls.h
@@ -36,9 +36,6 @@
 #if defined(__NR_arch_prctl)
   #define SYS_arch_prctl __NR_arch_prctl
 #endif
-#if defined(__NR_arch_specific_syscall)
-  #define SYS_arch_specific_syscall __NR_arch_specific_syscall
-#endif
 #if defined(__NR_arm_fadvise64_64)
   #define SYS_arm_fadvise64_64 __NR_arm_fadvise64_64
 #endif
@@ -1272,9 +1269,6 @@
 #if defined(__NR_syscall)
   #define SYS_syscall __NR_syscall
 #endif
-#if defined(__NR_syscalls)
-  #define SYS_syscalls __NR_syscalls
-#endif
 #if defined(__NR_sysfs)
   #define SYS_sysfs __NR_sysfs
 #endif
@@ -1371,6 +1365,9 @@
 #if defined(__NR_unshare)
   #define SYS_unshare __NR_unshare
 #endif
+#if defined(__NR_uretprobe)
+  #define SYS_uretprobe __NR_uretprobe
+#endif
 #if defined(__NR_uselib)
   #define SYS_uselib __NR_uselib
 #endif
diff --git a/libc/include/unistd.h b/libc/include/unistd.h
index e1c268f..9bc01f0 100644
--- a/libc/include/unistd.h
+++ b/libc/include/unistd.h
@@ -257,8 +257,29 @@
 int linkat(int __old_dir_fd, const char* _Nonnull __old_path, int __new_dir_fd, const char* _Nonnull __new_path, int __flags);
 int unlink(const char* _Nonnull __path);
 int unlinkat(int __dirfd, const char* _Nonnull __path, int __flags);
+
+/**
+ * [chdir(2)](https://man7.org/linux/man-pages/man2/chdir.2.html) changes
+ * the current working directory to the given path.
+ *
+ * This function affects all threads in the process, so is generally a bad idea
+ * on Android where most code will be running in a multi-threaded context.
+ *
+ * Returns 0 on success, and returns -1 and sets `errno` on failure.
+ */
 int chdir(const char* _Nonnull __path);
+
+/**
+ * [fchdir(2)](https://man7.org/linux/man-pages/man2/chdir.2.html) changes
+ * the current working directory to the given fd.
+ *
+ * This function affects all threads in the process, so is generally a bad idea
+ * on Android where most code will be running in a multi-threaded context.
+ *
+ * Returns 0 on success, and returns -1 and sets `errno` on failure.
+ */
 int fchdir(int __fd);
+
 int rmdir(const char* _Nonnull __path);
 int pipe(int __fds[_Nonnull 2]);
 #if defined(__USE_GNU)
diff --git a/libc/kernel/android/scsi/scsi/scsi_proto.h b/libc/kernel/android/scsi/scsi/scsi_proto.h
index d873fad..754e12a 100644
--- a/libc/kernel/android/scsi/scsi/scsi_proto.h
+++ b/libc/kernel/android/scsi/scsi/scsi_proto.h
@@ -109,6 +109,7 @@
 #define WRITE_SAME_16 0x93
 #define ZBC_OUT 0x94
 #define ZBC_IN 0x95
+#define WRITE_ATOMIC_16 0x9c
 #define SERVICE_ACTION_BIDIRECTIONAL 0x9d
 #define SERVICE_ACTION_IN_16 0x9e
 #define SERVICE_ACTION_OUT_16 0x9f
diff --git a/libc/kernel/tools/cpp.py b/libc/kernel/tools/cpp.py
index 08b786a..df50806 100755
--- a/libc/kernel/tools/cpp.py
+++ b/libc/kernel/tools/cpp.py
@@ -1430,7 +1430,7 @@
                         state = VAR_DECL
                     elif state == NORMAL and token_id in ['struct', 'typedef',
                                                           'enum', 'union',
-                                                          '__extension__']:
+                                                          '__extension__', '=']:
                         state = OTHER_DECL
                         state_token = token_id
                     elif block.tokens[i].kind == TokenKind.IDENTIFIER:
@@ -2057,7 +2057,7 @@
   struct timeval val2;
 };
 """
-        self.assertEqual(self.parse(text, {"remove": True}), expected)
+        self.assertEqual(self.parse(text, {"remove": None}), expected)
 
     def test_remove_struct_from_end(self):
         text = """\
@@ -2076,7 +2076,7 @@
   struct timeval val2;
 };
 """
-        self.assertEqual(self.parse(text, {"remove": True}), expected)
+        self.assertEqual(self.parse(text, {"remove": None}), expected)
 
     def test_remove_minimal_struct(self):
         text = """\
@@ -2084,7 +2084,7 @@
 };
 """
         expected = "";
-        self.assertEqual(self.parse(text, {"remove": True}), expected)
+        self.assertEqual(self.parse(text, {"remove": None}), expected)
 
     def test_remove_struct_with_struct_fields(self):
         text = """\
@@ -2104,7 +2104,7 @@
   struct remove val2;
 };
 """
-        self.assertEqual(self.parse(text, {"remove": True}), expected)
+        self.assertEqual(self.parse(text, {"remove": None}), expected)
 
     def test_remove_consecutive_structs(self):
         text = """\
@@ -2136,7 +2136,7 @@
   struct timeval val2;
 };
 """
-        self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
+        self.assertEqual(self.parse(text, {"remove1": None, "remove2": None}), expected)
 
     def test_remove_multiple_structs(self):
         text = """\
@@ -2169,7 +2169,7 @@
   int val;
 };
 """
-        self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
+        self.assertEqual(self.parse(text, {"remove1": None, "remove2": None}), expected)
 
     def test_remove_struct_with_inline_structs(self):
         text = """\
@@ -2194,7 +2194,7 @@
   struct timeval val2;
 };
 """
-        self.assertEqual(self.parse(text, {"remove": True}), expected)
+        self.assertEqual(self.parse(text, {"remove": None}), expected)
 
     def test_remove_struct_across_blocks(self):
         text = """\
@@ -2219,7 +2219,7 @@
   struct timeval val2;
 };
 """
-        self.assertEqual(self.parse(text, {"remove": True}), expected)
+        self.assertEqual(self.parse(text, {"remove": None}), expected)
 
     def test_remove_struct_across_blocks_multiple_structs(self):
         text = """\
@@ -2246,7 +2246,7 @@
   struct timeval val2;
 };
 """
-        self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
+        self.assertEqual(self.parse(text, {"remove1": None, "remove2": None}), expected)
 
     def test_remove_multiple_struct_and_add_includes(self):
         text = """\
@@ -2263,7 +2263,7 @@
 #include <bits/remove1.h>
 #include <bits/remove2.h>
 """
-        self.assertEqual(self.parse(text, {"remove1": False, "remove2": False}), expected)
+        self.assertEqual(self.parse(text, {"remove1": "bits/remove1.h", "remove2": "bits/remove2.h"}), expected)
 
 
 class FullPathTest(unittest.TestCase):
@@ -2580,6 +2580,71 @@
   struct timeval timeval;
   struct itimerval itimerval;
 };
+#include <linux/time.h>
+"""
+        self.assertEqual(self.parse(text), expected)
+
+    def test_var_definition(self):
+        # If we're definining the whole thing, it's probably worth keeping.
+        text = """\
+static const char *kString = "hello world";
+static const int kInteger = 42;
+"""
+        expected = """\
+static const char * kString = "hello world";
+static const int kInteger = 42;
+"""
+        self.assertEqual(self.parse(text), expected)
+
+    def test_struct_array_definition(self):
+        text = """\
+struct descriptor {
+  int args;
+  int size;
+};
+static const struct descriptor[] = {
+  {0, 0},
+  {1, 12},
+  {0, 42},
+};
+"""
+        expected = """\
+struct descriptor {
+  int args;
+  int size;
+};
+static const struct descriptor[] = {
+ {
+    0, 0
+  }
+ , {
+    1, 12
+  }
+ , {
+    0, 42
+  }
+ ,
+};
+"""
+        self.assertEqual(self.parse(text), expected)
+
+    def test_array_definition(self):
+        text = """\
+static const char *arr[] = {
+  "foo",
+  "bar",
+  "baz",
+};
+
+static int another_arr[5] = { 1, 2, 3, 4, 5};
+"""
+        expected = """\
+static const char * arr[] = {
+  "foo", "bar", "baz",
+};
+static int another_arr[5] = {
+  1, 2, 3, 4, 5
+};
 """
         self.assertEqual(self.parse(text), expected)
 
diff --git a/libc/kernel/tools/update_all.py b/libc/kernel/tools/update_all.py
index ae89a80..331a957 100755
--- a/libc/kernel/tools/update_all.py
+++ b/libc/kernel/tools/update_all.py
@@ -88,15 +88,8 @@
     # Collect the set of all syscalls for all architectures.
     syscalls = set()
     pattern = re.compile(r'^\s*#\s*define\s*__NR_([a-z_]\S+)')
-    for unistd_h in ['kernel/uapi/asm-generic/unistd.h',
-                     'kernel/uapi/asm-arm/asm/unistd.h',
-                     'kernel/uapi/asm-arm/asm/unistd-eabi.h',
-                     'kernel/uapi/asm-arm/asm/unistd-oabi.h',
-                     'kernel/uapi/asm-riscv/asm/unistd.h',
-                     'kernel/uapi/asm-x86/asm/unistd_32.h',
-                     'kernel/uapi/asm-x86/asm/unistd_64.h',
-                     'kernel/uapi/asm-x86/asm/unistd_x32.h']:
-        for line in open(os.path.join(libc_root, unistd_h)):
+    for unistd_h in glob.glob('%s/kernel/uapi/asm-*/asm/unistd*.h' % libc_root):
+        for line in open(unistd_h):
             m = re.search(pattern, line)
             if m:
                 nr_name = m.group(1)
diff --git a/libc/kernel/uapi/asm-arm64/asm/unistd.h b/libc/kernel/uapi/asm-arm64/asm/unistd.h
index 7457ebc..178578f 100644
--- a/libc/kernel/uapi/asm-arm64/asm/unistd.h
+++ b/libc/kernel/uapi/asm-arm64/asm/unistd.h
@@ -4,10 +4,4 @@
  * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
  * for more information.
  */
-#define __ARCH_WANT_RENAMEAT
-#define __ARCH_WANT_NEW_STAT
-#define __ARCH_WANT_SET_GET_RLIMIT
-#define __ARCH_WANT_TIME32_SYSCALLS
-#define __ARCH_WANT_SYS_CLONE3
-#define __ARCH_WANT_MEMFD_SECRET
-#include <asm-generic/unistd.h>
+#include <asm/unistd_64.h>
diff --git a/libc/kernel/uapi/asm-arm64/asm/unistd_64.h b/libc/kernel/uapi/asm-arm64/asm/unistd_64.h
new file mode 100644
index 0000000..0a0a1c0
--- /dev/null
+++ b/libc/kernel/uapi/asm-arm64/asm/unistd_64.h
@@ -0,0 +1,327 @@
+/*
+ * This file is auto-generated. Modifications will be lost.
+ *
+ * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
+ * for more information.
+ */
+#ifndef _UAPI_ASM_UNISTD_64_H
+#define _UAPI_ASM_UNISTD_64_H
+#define __NR_io_setup 0
+#define __NR_io_destroy 1
+#define __NR_io_submit 2
+#define __NR_io_cancel 3
+#define __NR_io_getevents 4
+#define __NR_setxattr 5
+#define __NR_lsetxattr 6
+#define __NR_fsetxattr 7
+#define __NR_getxattr 8
+#define __NR_lgetxattr 9
+#define __NR_fgetxattr 10
+#define __NR_listxattr 11
+#define __NR_llistxattr 12
+#define __NR_flistxattr 13
+#define __NR_removexattr 14
+#define __NR_lremovexattr 15
+#define __NR_fremovexattr 16
+#define __NR_getcwd 17
+#define __NR_lookup_dcookie 18
+#define __NR_eventfd2 19
+#define __NR_epoll_create1 20
+#define __NR_epoll_ctl 21
+#define __NR_epoll_pwait 22
+#define __NR_dup 23
+#define __NR_dup3 24
+#define __NR_fcntl 25
+#define __NR_inotify_init1 26
+#define __NR_inotify_add_watch 27
+#define __NR_inotify_rm_watch 28
+#define __NR_ioctl 29
+#define __NR_ioprio_set 30
+#define __NR_ioprio_get 31
+#define __NR_flock 32
+#define __NR_mknodat 33
+#define __NR_mkdirat 34
+#define __NR_unlinkat 35
+#define __NR_symlinkat 36
+#define __NR_linkat 37
+#define __NR_renameat 38
+#define __NR_umount2 39
+#define __NR_mount 40
+#define __NR_pivot_root 41
+#define __NR_nfsservctl 42
+#define __NR_statfs 43
+#define __NR_fstatfs 44
+#define __NR_truncate 45
+#define __NR_ftruncate 46
+#define __NR_fallocate 47
+#define __NR_faccessat 48
+#define __NR_chdir 49
+#define __NR_fchdir 50
+#define __NR_chroot 51
+#define __NR_fchmod 52
+#define __NR_fchmodat 53
+#define __NR_fchownat 54
+#define __NR_fchown 55
+#define __NR_openat 56
+#define __NR_close 57
+#define __NR_vhangup 58
+#define __NR_pipe2 59
+#define __NR_quotactl 60
+#define __NR_getdents64 61
+#define __NR_lseek 62
+#define __NR_read 63
+#define __NR_write 64
+#define __NR_readv 65
+#define __NR_writev 66
+#define __NR_pread64 67
+#define __NR_pwrite64 68
+#define __NR_preadv 69
+#define __NR_pwritev 70
+#define __NR_sendfile 71
+#define __NR_pselect6 72
+#define __NR_ppoll 73
+#define __NR_signalfd4 74
+#define __NR_vmsplice 75
+#define __NR_splice 76
+#define __NR_tee 77
+#define __NR_readlinkat 78
+#define __NR_newfstatat 79
+#define __NR_fstat 80
+#define __NR_sync 81
+#define __NR_fsync 82
+#define __NR_fdatasync 83
+#define __NR_sync_file_range 84
+#define __NR_timerfd_create 85
+#define __NR_timerfd_settime 86
+#define __NR_timerfd_gettime 87
+#define __NR_utimensat 88
+#define __NR_acct 89
+#define __NR_capget 90
+#define __NR_capset 91
+#define __NR_personality 92
+#define __NR_exit 93
+#define __NR_exit_group 94
+#define __NR_waitid 95
+#define __NR_set_tid_address 96
+#define __NR_unshare 97
+#define __NR_futex 98
+#define __NR_set_robust_list 99
+#define __NR_get_robust_list 100
+#define __NR_nanosleep 101
+#define __NR_getitimer 102
+#define __NR_setitimer 103
+#define __NR_kexec_load 104
+#define __NR_init_module 105
+#define __NR_delete_module 106
+#define __NR_timer_create 107
+#define __NR_timer_gettime 108
+#define __NR_timer_getoverrun 109
+#define __NR_timer_settime 110
+#define __NR_timer_delete 111
+#define __NR_clock_settime 112
+#define __NR_clock_gettime 113
+#define __NR_clock_getres 114
+#define __NR_clock_nanosleep 115
+#define __NR_syslog 116
+#define __NR_ptrace 117
+#define __NR_sched_setparam 118
+#define __NR_sched_setscheduler 119
+#define __NR_sched_getscheduler 120
+#define __NR_sched_getparam 121
+#define __NR_sched_setaffinity 122
+#define __NR_sched_getaffinity 123
+#define __NR_sched_yield 124
+#define __NR_sched_get_priority_max 125
+#define __NR_sched_get_priority_min 126
+#define __NR_sched_rr_get_interval 127
+#define __NR_restart_syscall 128
+#define __NR_kill 129
+#define __NR_tkill 130
+#define __NR_tgkill 131
+#define __NR_sigaltstack 132
+#define __NR_rt_sigsuspend 133
+#define __NR_rt_sigaction 134
+#define __NR_rt_sigprocmask 135
+#define __NR_rt_sigpending 136
+#define __NR_rt_sigtimedwait 137
+#define __NR_rt_sigqueueinfo 138
+#define __NR_rt_sigreturn 139
+#define __NR_setpriority 140
+#define __NR_getpriority 141
+#define __NR_reboot 142
+#define __NR_setregid 143
+#define __NR_setgid 144
+#define __NR_setreuid 145
+#define __NR_setuid 146
+#define __NR_setresuid 147
+#define __NR_getresuid 148
+#define __NR_setresgid 149
+#define __NR_getresgid 150
+#define __NR_setfsuid 151
+#define __NR_setfsgid 152
+#define __NR_times 153
+#define __NR_setpgid 154
+#define __NR_getpgid 155
+#define __NR_getsid 156
+#define __NR_setsid 157
+#define __NR_getgroups 158
+#define __NR_setgroups 159
+#define __NR_uname 160
+#define __NR_sethostname 161
+#define __NR_setdomainname 162
+#define __NR_getrlimit 163
+#define __NR_setrlimit 164
+#define __NR_getrusage 165
+#define __NR_umask 166
+#define __NR_prctl 167
+#define __NR_getcpu 168
+#define __NR_gettimeofday 169
+#define __NR_settimeofday 170
+#define __NR_adjtimex 171
+#define __NR_getpid 172
+#define __NR_getppid 173
+#define __NR_getuid 174
+#define __NR_geteuid 175
+#define __NR_getgid 176
+#define __NR_getegid 177
+#define __NR_gettid 178
+#define __NR_sysinfo 179
+#define __NR_mq_open 180
+#define __NR_mq_unlink 181
+#define __NR_mq_timedsend 182
+#define __NR_mq_timedreceive 183
+#define __NR_mq_notify 184
+#define __NR_mq_getsetattr 185
+#define __NR_msgget 186
+#define __NR_msgctl 187
+#define __NR_msgrcv 188
+#define __NR_msgsnd 189
+#define __NR_semget 190
+#define __NR_semctl 191
+#define __NR_semtimedop 192
+#define __NR_semop 193
+#define __NR_shmget 194
+#define __NR_shmctl 195
+#define __NR_shmat 196
+#define __NR_shmdt 197
+#define __NR_socket 198
+#define __NR_socketpair 199
+#define __NR_bind 200
+#define __NR_listen 201
+#define __NR_accept 202
+#define __NR_connect 203
+#define __NR_getsockname 204
+#define __NR_getpeername 205
+#define __NR_sendto 206
+#define __NR_recvfrom 207
+#define __NR_setsockopt 208
+#define __NR_getsockopt 209
+#define __NR_shutdown 210
+#define __NR_sendmsg 211
+#define __NR_recvmsg 212
+#define __NR_readahead 213
+#define __NR_brk 214
+#define __NR_munmap 215
+#define __NR_mremap 216
+#define __NR_add_key 217
+#define __NR_request_key 218
+#define __NR_keyctl 219
+#define __NR_clone 220
+#define __NR_execve 221
+#define __NR_mmap 222
+#define __NR_fadvise64 223
+#define __NR_swapon 224
+#define __NR_swapoff 225
+#define __NR_mprotect 226
+#define __NR_msync 227
+#define __NR_mlock 228
+#define __NR_munlock 229
+#define __NR_mlockall 230
+#define __NR_munlockall 231
+#define __NR_mincore 232
+#define __NR_madvise 233
+#define __NR_remap_file_pages 234
+#define __NR_mbind 235
+#define __NR_get_mempolicy 236
+#define __NR_set_mempolicy 237
+#define __NR_migrate_pages 238
+#define __NR_move_pages 239
+#define __NR_rt_tgsigqueueinfo 240
+#define __NR_perf_event_open 241
+#define __NR_accept4 242
+#define __NR_recvmmsg 243
+#define __NR_wait4 260
+#define __NR_prlimit64 261
+#define __NR_fanotify_init 262
+#define __NR_fanotify_mark 263
+#define __NR_name_to_handle_at 264
+#define __NR_open_by_handle_at 265
+#define __NR_clock_adjtime 266
+#define __NR_syncfs 267
+#define __NR_setns 268
+#define __NR_sendmmsg 269
+#define __NR_process_vm_readv 270
+#define __NR_process_vm_writev 271
+#define __NR_kcmp 272
+#define __NR_finit_module 273
+#define __NR_sched_setattr 274
+#define __NR_sched_getattr 275
+#define __NR_renameat2 276
+#define __NR_seccomp 277
+#define __NR_getrandom 278
+#define __NR_memfd_create 279
+#define __NR_bpf 280
+#define __NR_execveat 281
+#define __NR_userfaultfd 282
+#define __NR_membarrier 283
+#define __NR_mlock2 284
+#define __NR_copy_file_range 285
+#define __NR_preadv2 286
+#define __NR_pwritev2 287
+#define __NR_pkey_mprotect 288
+#define __NR_pkey_alloc 289
+#define __NR_pkey_free 290
+#define __NR_statx 291
+#define __NR_io_pgetevents 292
+#define __NR_rseq 293
+#define __NR_kexec_file_load 294
+#define __NR_pidfd_send_signal 424
+#define __NR_io_uring_setup 425
+#define __NR_io_uring_enter 426
+#define __NR_io_uring_register 427
+#define __NR_open_tree 428
+#define __NR_move_mount 429
+#define __NR_fsopen 430
+#define __NR_fsconfig 431
+#define __NR_fsmount 432
+#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_close_range 436
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
+#define __NR_process_madvise 440
+#define __NR_epoll_pwait2 441
+#define __NR_mount_setattr 442
+#define __NR_quotactl_fd 443
+#define __NR_landlock_create_ruleset 444
+#define __NR_landlock_add_rule 445
+#define __NR_landlock_restrict_self 446
+#define __NR_memfd_secret 447
+#define __NR_process_mrelease 448
+#define __NR_futex_waitv 449
+#define __NR_set_mempolicy_home_node 450
+#define __NR_cachestat 451
+#define __NR_fchmodat2 452
+#define __NR_map_shadow_stack 453
+#define __NR_futex_wake 454
+#define __NR_futex_wait 455
+#define __NR_futex_requeue 456
+#define __NR_statmount 457
+#define __NR_listmount 458
+#define __NR_lsm_get_self_attr 459
+#define __NR_lsm_set_self_attr 460
+#define __NR_lsm_list_modules 461
+#define __NR_mseal 462
+#endif
diff --git a/libc/kernel/uapi/asm-generic/unistd.h b/libc/kernel/uapi/asm-generic/unistd.h
index 7eaa89a..652e7a2 100644
--- a/libc/kernel/uapi/asm-generic/unistd.h
+++ b/libc/kernel/uapi/asm-generic/unistd.h
@@ -381,9 +381,7 @@
 #define __NR_fsmount 432
 #define __NR_fspick 433
 #define __NR_pidfd_open 434
-#ifdef __ARCH_WANT_SYS_CLONE3
 #define __NR_clone3 435
-#endif
 #define __NR_close_range 436
 #define __NR_openat2 437
 #define __NR_pidfd_getfd 438
diff --git a/libc/kernel/uapi/asm-riscv/asm/hwprobe.h b/libc/kernel/uapi/asm-riscv/asm/hwprobe.h
index 3f30c88..2e5f9a4 100644
--- a/libc/kernel/uapi/asm-riscv/asm/hwprobe.h
+++ b/libc/kernel/uapi/asm-riscv/asm/hwprobe.h
@@ -54,6 +54,18 @@
 #define RISCV_HWPROBE_EXT_ZACAS (1ULL << 34)
 #define RISCV_HWPROBE_EXT_ZICOND (1ULL << 35)
 #define RISCV_HWPROBE_EXT_ZIHINTPAUSE (1ULL << 36)
+#define RISCV_HWPROBE_EXT_ZVE32X (1ULL << 37)
+#define RISCV_HWPROBE_EXT_ZVE32F (1ULL << 38)
+#define RISCV_HWPROBE_EXT_ZVE64X (1ULL << 39)
+#define RISCV_HWPROBE_EXT_ZVE64F (1ULL << 40)
+#define RISCV_HWPROBE_EXT_ZVE64D (1ULL << 41)
+#define RISCV_HWPROBE_EXT_ZIMOP (1ULL << 42)
+#define RISCV_HWPROBE_EXT_ZCA (1ULL << 43)
+#define RISCV_HWPROBE_EXT_ZCB (1ULL << 44)
+#define RISCV_HWPROBE_EXT_ZCD (1ULL << 45)
+#define RISCV_HWPROBE_EXT_ZCF (1ULL << 46)
+#define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47)
+#define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48)
 #define RISCV_HWPROBE_KEY_CPUPERF_0 5
 #define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0)
 #define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0)
@@ -62,5 +74,13 @@
 #define RISCV_HWPROBE_MISALIGNED_UNSUPPORTED (4 << 0)
 #define RISCV_HWPROBE_MISALIGNED_MASK (7 << 0)
 #define RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE 6
+#define RISCV_HWPROBE_KEY_HIGHEST_VIRT_ADDRESS 7
+#define RISCV_HWPROBE_KEY_TIME_CSR_FREQ 8
+#define RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF 9
+#define RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN 0
+#define RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED 1
+#define RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW 2
+#define RISCV_HWPROBE_MISALIGNED_SCALAR_FAST 3
+#define RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED 4
 #define RISCV_HWPROBE_WHICH_CPUS (1 << 0)
 #endif
diff --git a/libc/kernel/uapi/asm-riscv/asm/kvm.h b/libc/kernel/uapi/asm-riscv/asm/kvm.h
index 12d8f61..51f4977 100644
--- a/libc/kernel/uapi/asm-riscv/asm/kvm.h
+++ b/libc/kernel/uapi/asm-riscv/asm/kvm.h
@@ -128,6 +128,13 @@
   KVM_RISCV_ISA_EXT_ZTSO,
   KVM_RISCV_ISA_EXT_ZACAS,
   KVM_RISCV_ISA_EXT_SSCOFPMF,
+  KVM_RISCV_ISA_EXT_ZIMOP,
+  KVM_RISCV_ISA_EXT_ZCA,
+  KVM_RISCV_ISA_EXT_ZCB,
+  KVM_RISCV_ISA_EXT_ZCD,
+  KVM_RISCV_ISA_EXT_ZCF,
+  KVM_RISCV_ISA_EXT_ZCMOP,
+  KVM_RISCV_ISA_EXT_ZAWRS,
   KVM_RISCV_ISA_EXT_MAX,
 };
 enum KVM_RISCV_SBI_EXT_ID {
diff --git a/libc/kernel/uapi/asm-riscv/asm/unistd.h b/libc/kernel/uapi/asm-riscv/asm/unistd.h
index 0c58887..f395f71 100644
--- a/libc/kernel/uapi/asm-riscv/asm/unistd.h
+++ b/libc/kernel/uapi/asm-riscv/asm/unistd.h
@@ -4,18 +4,9 @@
  * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
  * for more information.
  */
-#if defined(__LP64__) && !defined(__SYSCALL_COMPAT)
-#define __ARCH_WANT_NEW_STAT
-#define __ARCH_WANT_SET_GET_RLIMIT
+#include <asm/bitsperlong.h>
+#if __BITS_PER_LONG == 64
+#include <asm/unistd_64.h>
+#else
+#include <asm/unistd_32.h>
 #endif
-#define __ARCH_WANT_SYS_CLONE3
-#define __ARCH_WANT_MEMFD_SECRET
-#include <asm-generic/unistd.h>
-#ifndef __NR_riscv_flush_icache
-#define __NR_riscv_flush_icache (__NR_arch_specific_syscall + 15)
-#endif
-__SYSCALL(__NR_riscv_flush_icache, sys_riscv_flush_icache)
-#ifndef __NR_riscv_hwprobe
-#define __NR_riscv_hwprobe (__NR_arch_specific_syscall + 14)
-#endif
-__SYSCALL(__NR_riscv_hwprobe, sys_riscv_hwprobe)
diff --git a/libc/kernel/uapi/asm-riscv/asm/unistd_32.h b/libc/kernel/uapi/asm-riscv/asm/unistd_32.h
new file mode 100644
index 0000000..864a556
--- /dev/null
+++ b/libc/kernel/uapi/asm-riscv/asm/unistd_32.h
@@ -0,0 +1,318 @@
+/*
+ * This file is auto-generated. Modifications will be lost.
+ *
+ * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
+ * for more information.
+ */
+#ifndef _UAPI_ASM_UNISTD_32_H
+#define _UAPI_ASM_UNISTD_32_H
+#define __NR_io_setup 0
+#define __NR_io_destroy 1
+#define __NR_io_submit 2
+#define __NR_io_cancel 3
+#define __NR_setxattr 5
+#define __NR_lsetxattr 6
+#define __NR_fsetxattr 7
+#define __NR_getxattr 8
+#define __NR_lgetxattr 9
+#define __NR_fgetxattr 10
+#define __NR_listxattr 11
+#define __NR_llistxattr 12
+#define __NR_flistxattr 13
+#define __NR_removexattr 14
+#define __NR_lremovexattr 15
+#define __NR_fremovexattr 16
+#define __NR_getcwd 17
+#define __NR_lookup_dcookie 18
+#define __NR_eventfd2 19
+#define __NR_epoll_create1 20
+#define __NR_epoll_ctl 21
+#define __NR_epoll_pwait 22
+#define __NR_dup 23
+#define __NR_dup3 24
+#define __NR_fcntl64 25
+#define __NR_inotify_init1 26
+#define __NR_inotify_add_watch 27
+#define __NR_inotify_rm_watch 28
+#define __NR_ioctl 29
+#define __NR_ioprio_set 30
+#define __NR_ioprio_get 31
+#define __NR_flock 32
+#define __NR_mknodat 33
+#define __NR_mkdirat 34
+#define __NR_unlinkat 35
+#define __NR_symlinkat 36
+#define __NR_linkat 37
+#define __NR_umount2 39
+#define __NR_mount 40
+#define __NR_pivot_root 41
+#define __NR_nfsservctl 42
+#define __NR_statfs64 43
+#define __NR_fstatfs64 44
+#define __NR_truncate64 45
+#define __NR_ftruncate64 46
+#define __NR_fallocate 47
+#define __NR_faccessat 48
+#define __NR_chdir 49
+#define __NR_fchdir 50
+#define __NR_chroot 51
+#define __NR_fchmod 52
+#define __NR_fchmodat 53
+#define __NR_fchownat 54
+#define __NR_fchown 55
+#define __NR_openat 56
+#define __NR_close 57
+#define __NR_vhangup 58
+#define __NR_pipe2 59
+#define __NR_quotactl 60
+#define __NR_getdents64 61
+#define __NR_llseek 62
+#define __NR_read 63
+#define __NR_write 64
+#define __NR_readv 65
+#define __NR_writev 66
+#define __NR_pread64 67
+#define __NR_pwrite64 68
+#define __NR_preadv 69
+#define __NR_pwritev 70
+#define __NR_sendfile64 71
+#define __NR_signalfd4 74
+#define __NR_vmsplice 75
+#define __NR_splice 76
+#define __NR_tee 77
+#define __NR_readlinkat 78
+#define __NR_sync 81
+#define __NR_fsync 82
+#define __NR_fdatasync 83
+#define __NR_sync_file_range 84
+#define __NR_timerfd_create 85
+#define __NR_acct 89
+#define __NR_capget 90
+#define __NR_capset 91
+#define __NR_personality 92
+#define __NR_exit 93
+#define __NR_exit_group 94
+#define __NR_waitid 95
+#define __NR_set_tid_address 96
+#define __NR_unshare 97
+#define __NR_set_robust_list 99
+#define __NR_get_robust_list 100
+#define __NR_getitimer 102
+#define __NR_setitimer 103
+#define __NR_kexec_load 104
+#define __NR_init_module 105
+#define __NR_delete_module 106
+#define __NR_timer_create 107
+#define __NR_timer_getoverrun 109
+#define __NR_timer_delete 111
+#define __NR_syslog 116
+#define __NR_ptrace 117
+#define __NR_sched_setparam 118
+#define __NR_sched_setscheduler 119
+#define __NR_sched_getscheduler 120
+#define __NR_sched_getparam 121
+#define __NR_sched_setaffinity 122
+#define __NR_sched_getaffinity 123
+#define __NR_sched_yield 124
+#define __NR_sched_get_priority_max 125
+#define __NR_sched_get_priority_min 126
+#define __NR_restart_syscall 128
+#define __NR_kill 129
+#define __NR_tkill 130
+#define __NR_tgkill 131
+#define __NR_sigaltstack 132
+#define __NR_rt_sigsuspend 133
+#define __NR_rt_sigaction 134
+#define __NR_rt_sigprocmask 135
+#define __NR_rt_sigpending 136
+#define __NR_rt_sigqueueinfo 138
+#define __NR_rt_sigreturn 139
+#define __NR_setpriority 140
+#define __NR_getpriority 141
+#define __NR_reboot 142
+#define __NR_setregid 143
+#define __NR_setgid 144
+#define __NR_setreuid 145
+#define __NR_setuid 146
+#define __NR_setresuid 147
+#define __NR_getresuid 148
+#define __NR_setresgid 149
+#define __NR_getresgid 150
+#define __NR_setfsuid 151
+#define __NR_setfsgid 152
+#define __NR_times 153
+#define __NR_setpgid 154
+#define __NR_getpgid 155
+#define __NR_getsid 156
+#define __NR_setsid 157
+#define __NR_getgroups 158
+#define __NR_setgroups 159
+#define __NR_uname 160
+#define __NR_sethostname 161
+#define __NR_setdomainname 162
+#define __NR_getrusage 165
+#define __NR_umask 166
+#define __NR_prctl 167
+#define __NR_getcpu 168
+#define __NR_getpid 172
+#define __NR_getppid 173
+#define __NR_getuid 174
+#define __NR_geteuid 175
+#define __NR_getgid 176
+#define __NR_getegid 177
+#define __NR_gettid 178
+#define __NR_sysinfo 179
+#define __NR_mq_open 180
+#define __NR_mq_unlink 181
+#define __NR_mq_notify 184
+#define __NR_mq_getsetattr 185
+#define __NR_msgget 186
+#define __NR_msgctl 187
+#define __NR_msgrcv 188
+#define __NR_msgsnd 189
+#define __NR_semget 190
+#define __NR_semctl 191
+#define __NR_semop 193
+#define __NR_shmget 194
+#define __NR_shmctl 195
+#define __NR_shmat 196
+#define __NR_shmdt 197
+#define __NR_socket 198
+#define __NR_socketpair 199
+#define __NR_bind 200
+#define __NR_listen 201
+#define __NR_accept 202
+#define __NR_connect 203
+#define __NR_getsockname 204
+#define __NR_getpeername 205
+#define __NR_sendto 206
+#define __NR_recvfrom 207
+#define __NR_setsockopt 208
+#define __NR_getsockopt 209
+#define __NR_shutdown 210
+#define __NR_sendmsg 211
+#define __NR_recvmsg 212
+#define __NR_readahead 213
+#define __NR_brk 214
+#define __NR_munmap 215
+#define __NR_mremap 216
+#define __NR_add_key 217
+#define __NR_request_key 218
+#define __NR_keyctl 219
+#define __NR_clone 220
+#define __NR_execve 221
+#define __NR_mmap2 222
+#define __NR_fadvise64_64 223
+#define __NR_swapon 224
+#define __NR_swapoff 225
+#define __NR_mprotect 226
+#define __NR_msync 227
+#define __NR_mlock 228
+#define __NR_munlock 229
+#define __NR_mlockall 230
+#define __NR_munlockall 231
+#define __NR_mincore 232
+#define __NR_madvise 233
+#define __NR_remap_file_pages 234
+#define __NR_mbind 235
+#define __NR_get_mempolicy 236
+#define __NR_set_mempolicy 237
+#define __NR_migrate_pages 238
+#define __NR_move_pages 239
+#define __NR_rt_tgsigqueueinfo 240
+#define __NR_perf_event_open 241
+#define __NR_accept4 242
+#define __NR_riscv_hwprobe 258
+#define __NR_riscv_flush_icache 259
+#define __NR_prlimit64 261
+#define __NR_fanotify_init 262
+#define __NR_fanotify_mark 263
+#define __NR_name_to_handle_at 264
+#define __NR_open_by_handle_at 265
+#define __NR_syncfs 267
+#define __NR_setns 268
+#define __NR_sendmmsg 269
+#define __NR_process_vm_readv 270
+#define __NR_process_vm_writev 271
+#define __NR_kcmp 272
+#define __NR_finit_module 273
+#define __NR_sched_setattr 274
+#define __NR_sched_getattr 275
+#define __NR_renameat2 276
+#define __NR_seccomp 277
+#define __NR_getrandom 278
+#define __NR_memfd_create 279
+#define __NR_bpf 280
+#define __NR_execveat 281
+#define __NR_userfaultfd 282
+#define __NR_membarrier 283
+#define __NR_mlock2 284
+#define __NR_copy_file_range 285
+#define __NR_preadv2 286
+#define __NR_pwritev2 287
+#define __NR_pkey_mprotect 288
+#define __NR_pkey_alloc 289
+#define __NR_pkey_free 290
+#define __NR_statx 291
+#define __NR_rseq 293
+#define __NR_kexec_file_load 294
+#define __NR_clock_gettime64 403
+#define __NR_clock_settime64 404
+#define __NR_clock_adjtime64 405
+#define __NR_clock_getres_time64 406
+#define __NR_clock_nanosleep_time64 407
+#define __NR_timer_gettime64 408
+#define __NR_timer_settime64 409
+#define __NR_timerfd_gettime64 410
+#define __NR_timerfd_settime64 411
+#define __NR_utimensat_time64 412
+#define __NR_pselect6_time64 413
+#define __NR_ppoll_time64 414
+#define __NR_io_pgetevents_time64 416
+#define __NR_recvmmsg_time64 417
+#define __NR_mq_timedsend_time64 418
+#define __NR_mq_timedreceive_time64 419
+#define __NR_semtimedop_time64 420
+#define __NR_rt_sigtimedwait_time64 421
+#define __NR_futex_time64 422
+#define __NR_sched_rr_get_interval_time64 423
+#define __NR_pidfd_send_signal 424
+#define __NR_io_uring_setup 425
+#define __NR_io_uring_enter 426
+#define __NR_io_uring_register 427
+#define __NR_open_tree 428
+#define __NR_move_mount 429
+#define __NR_fsopen 430
+#define __NR_fsconfig 431
+#define __NR_fsmount 432
+#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_close_range 436
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
+#define __NR_process_madvise 440
+#define __NR_epoll_pwait2 441
+#define __NR_mount_setattr 442
+#define __NR_quotactl_fd 443
+#define __NR_landlock_create_ruleset 444
+#define __NR_landlock_add_rule 445
+#define __NR_landlock_restrict_self 446
+#define __NR_memfd_secret 447
+#define __NR_process_mrelease 448
+#define __NR_futex_waitv 449
+#define __NR_set_mempolicy_home_node 450
+#define __NR_cachestat 451
+#define __NR_fchmodat2 452
+#define __NR_map_shadow_stack 453
+#define __NR_futex_wake 454
+#define __NR_futex_wait 455
+#define __NR_futex_requeue 456
+#define __NR_statmount 457
+#define __NR_listmount 458
+#define __NR_lsm_get_self_attr 459
+#define __NR_lsm_set_self_attr 460
+#define __NR_lsm_list_modules 461
+#define __NR_mseal 462
+#endif
diff --git a/libc/kernel/uapi/asm-riscv/asm/unistd_64.h b/libc/kernel/uapi/asm-riscv/asm/unistd_64.h
new file mode 100644
index 0000000..f15b65b
--- /dev/null
+++ b/libc/kernel/uapi/asm-riscv/asm/unistd_64.h
@@ -0,0 +1,328 @@
+/*
+ * This file is auto-generated. Modifications will be lost.
+ *
+ * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
+ * for more information.
+ */
+#ifndef _UAPI_ASM_UNISTD_64_H
+#define _UAPI_ASM_UNISTD_64_H
+#define __NR_io_setup 0
+#define __NR_io_destroy 1
+#define __NR_io_submit 2
+#define __NR_io_cancel 3
+#define __NR_io_getevents 4
+#define __NR_setxattr 5
+#define __NR_lsetxattr 6
+#define __NR_fsetxattr 7
+#define __NR_getxattr 8
+#define __NR_lgetxattr 9
+#define __NR_fgetxattr 10
+#define __NR_listxattr 11
+#define __NR_llistxattr 12
+#define __NR_flistxattr 13
+#define __NR_removexattr 14
+#define __NR_lremovexattr 15
+#define __NR_fremovexattr 16
+#define __NR_getcwd 17
+#define __NR_lookup_dcookie 18
+#define __NR_eventfd2 19
+#define __NR_epoll_create1 20
+#define __NR_epoll_ctl 21
+#define __NR_epoll_pwait 22
+#define __NR_dup 23
+#define __NR_dup3 24
+#define __NR_fcntl 25
+#define __NR_inotify_init1 26
+#define __NR_inotify_add_watch 27
+#define __NR_inotify_rm_watch 28
+#define __NR_ioctl 29
+#define __NR_ioprio_set 30
+#define __NR_ioprio_get 31
+#define __NR_flock 32
+#define __NR_mknodat 33
+#define __NR_mkdirat 34
+#define __NR_unlinkat 35
+#define __NR_symlinkat 36
+#define __NR_linkat 37
+#define __NR_umount2 39
+#define __NR_mount 40
+#define __NR_pivot_root 41
+#define __NR_nfsservctl 42
+#define __NR_statfs 43
+#define __NR_fstatfs 44
+#define __NR_truncate 45
+#define __NR_ftruncate 46
+#define __NR_fallocate 47
+#define __NR_faccessat 48
+#define __NR_chdir 49
+#define __NR_fchdir 50
+#define __NR_chroot 51
+#define __NR_fchmod 52
+#define __NR_fchmodat 53
+#define __NR_fchownat 54
+#define __NR_fchown 55
+#define __NR_openat 56
+#define __NR_close 57
+#define __NR_vhangup 58
+#define __NR_pipe2 59
+#define __NR_quotactl 60
+#define __NR_getdents64 61
+#define __NR_lseek 62
+#define __NR_read 63
+#define __NR_write 64
+#define __NR_readv 65
+#define __NR_writev 66
+#define __NR_pread64 67
+#define __NR_pwrite64 68
+#define __NR_preadv 69
+#define __NR_pwritev 70
+#define __NR_sendfile 71
+#define __NR_pselect6 72
+#define __NR_ppoll 73
+#define __NR_signalfd4 74
+#define __NR_vmsplice 75
+#define __NR_splice 76
+#define __NR_tee 77
+#define __NR_readlinkat 78
+#define __NR_newfstatat 79
+#define __NR_fstat 80
+#define __NR_sync 81
+#define __NR_fsync 82
+#define __NR_fdatasync 83
+#define __NR_sync_file_range 84
+#define __NR_timerfd_create 85
+#define __NR_timerfd_settime 86
+#define __NR_timerfd_gettime 87
+#define __NR_utimensat 88
+#define __NR_acct 89
+#define __NR_capget 90
+#define __NR_capset 91
+#define __NR_personality 92
+#define __NR_exit 93
+#define __NR_exit_group 94
+#define __NR_waitid 95
+#define __NR_set_tid_address 96
+#define __NR_unshare 97
+#define __NR_futex 98
+#define __NR_set_robust_list 99
+#define __NR_get_robust_list 100
+#define __NR_nanosleep 101
+#define __NR_getitimer 102
+#define __NR_setitimer 103
+#define __NR_kexec_load 104
+#define __NR_init_module 105
+#define __NR_delete_module 106
+#define __NR_timer_create 107
+#define __NR_timer_gettime 108
+#define __NR_timer_getoverrun 109
+#define __NR_timer_settime 110
+#define __NR_timer_delete 111
+#define __NR_clock_settime 112
+#define __NR_clock_gettime 113
+#define __NR_clock_getres 114
+#define __NR_clock_nanosleep 115
+#define __NR_syslog 116
+#define __NR_ptrace 117
+#define __NR_sched_setparam 118
+#define __NR_sched_setscheduler 119
+#define __NR_sched_getscheduler 120
+#define __NR_sched_getparam 121
+#define __NR_sched_setaffinity 122
+#define __NR_sched_getaffinity 123
+#define __NR_sched_yield 124
+#define __NR_sched_get_priority_max 125
+#define __NR_sched_get_priority_min 126
+#define __NR_sched_rr_get_interval 127
+#define __NR_restart_syscall 128
+#define __NR_kill 129
+#define __NR_tkill 130
+#define __NR_tgkill 131
+#define __NR_sigaltstack 132
+#define __NR_rt_sigsuspend 133
+#define __NR_rt_sigaction 134
+#define __NR_rt_sigprocmask 135
+#define __NR_rt_sigpending 136
+#define __NR_rt_sigtimedwait 137
+#define __NR_rt_sigqueueinfo 138
+#define __NR_rt_sigreturn 139
+#define __NR_setpriority 140
+#define __NR_getpriority 141
+#define __NR_reboot 142
+#define __NR_setregid 143
+#define __NR_setgid 144
+#define __NR_setreuid 145
+#define __NR_setuid 146
+#define __NR_setresuid 147
+#define __NR_getresuid 148
+#define __NR_setresgid 149
+#define __NR_getresgid 150
+#define __NR_setfsuid 151
+#define __NR_setfsgid 152
+#define __NR_times 153
+#define __NR_setpgid 154
+#define __NR_getpgid 155
+#define __NR_getsid 156
+#define __NR_setsid 157
+#define __NR_getgroups 158
+#define __NR_setgroups 159
+#define __NR_uname 160
+#define __NR_sethostname 161
+#define __NR_setdomainname 162
+#define __NR_getrlimit 163
+#define __NR_setrlimit 164
+#define __NR_getrusage 165
+#define __NR_umask 166
+#define __NR_prctl 167
+#define __NR_getcpu 168
+#define __NR_gettimeofday 169
+#define __NR_settimeofday 170
+#define __NR_adjtimex 171
+#define __NR_getpid 172
+#define __NR_getppid 173
+#define __NR_getuid 174
+#define __NR_geteuid 175
+#define __NR_getgid 176
+#define __NR_getegid 177
+#define __NR_gettid 178
+#define __NR_sysinfo 179
+#define __NR_mq_open 180
+#define __NR_mq_unlink 181
+#define __NR_mq_timedsend 182
+#define __NR_mq_timedreceive 183
+#define __NR_mq_notify 184
+#define __NR_mq_getsetattr 185
+#define __NR_msgget 186
+#define __NR_msgctl 187
+#define __NR_msgrcv 188
+#define __NR_msgsnd 189
+#define __NR_semget 190
+#define __NR_semctl 191
+#define __NR_semtimedop 192
+#define __NR_semop 193
+#define __NR_shmget 194
+#define __NR_shmctl 195
+#define __NR_shmat 196
+#define __NR_shmdt 197
+#define __NR_socket 198
+#define __NR_socketpair 199
+#define __NR_bind 200
+#define __NR_listen 201
+#define __NR_accept 202
+#define __NR_connect 203
+#define __NR_getsockname 204
+#define __NR_getpeername 205
+#define __NR_sendto 206
+#define __NR_recvfrom 207
+#define __NR_setsockopt 208
+#define __NR_getsockopt 209
+#define __NR_shutdown 210
+#define __NR_sendmsg 211
+#define __NR_recvmsg 212
+#define __NR_readahead 213
+#define __NR_brk 214
+#define __NR_munmap 215
+#define __NR_mremap 216
+#define __NR_add_key 217
+#define __NR_request_key 218
+#define __NR_keyctl 219
+#define __NR_clone 220
+#define __NR_execve 221
+#define __NR_mmap 222
+#define __NR_fadvise64 223
+#define __NR_swapon 224
+#define __NR_swapoff 225
+#define __NR_mprotect 226
+#define __NR_msync 227
+#define __NR_mlock 228
+#define __NR_munlock 229
+#define __NR_mlockall 230
+#define __NR_munlockall 231
+#define __NR_mincore 232
+#define __NR_madvise 233
+#define __NR_remap_file_pages 234
+#define __NR_mbind 235
+#define __NR_get_mempolicy 236
+#define __NR_set_mempolicy 237
+#define __NR_migrate_pages 238
+#define __NR_move_pages 239
+#define __NR_rt_tgsigqueueinfo 240
+#define __NR_perf_event_open 241
+#define __NR_accept4 242
+#define __NR_recvmmsg 243
+#define __NR_riscv_hwprobe 258
+#define __NR_riscv_flush_icache 259
+#define __NR_wait4 260
+#define __NR_prlimit64 261
+#define __NR_fanotify_init 262
+#define __NR_fanotify_mark 263
+#define __NR_name_to_handle_at 264
+#define __NR_open_by_handle_at 265
+#define __NR_clock_adjtime 266
+#define __NR_syncfs 267
+#define __NR_setns 268
+#define __NR_sendmmsg 269
+#define __NR_process_vm_readv 270
+#define __NR_process_vm_writev 271
+#define __NR_kcmp 272
+#define __NR_finit_module 273
+#define __NR_sched_setattr 274
+#define __NR_sched_getattr 275
+#define __NR_renameat2 276
+#define __NR_seccomp 277
+#define __NR_getrandom 278
+#define __NR_memfd_create 279
+#define __NR_bpf 280
+#define __NR_execveat 281
+#define __NR_userfaultfd 282
+#define __NR_membarrier 283
+#define __NR_mlock2 284
+#define __NR_copy_file_range 285
+#define __NR_preadv2 286
+#define __NR_pwritev2 287
+#define __NR_pkey_mprotect 288
+#define __NR_pkey_alloc 289
+#define __NR_pkey_free 290
+#define __NR_statx 291
+#define __NR_io_pgetevents 292
+#define __NR_rseq 293
+#define __NR_kexec_file_load 294
+#define __NR_pidfd_send_signal 424
+#define __NR_io_uring_setup 425
+#define __NR_io_uring_enter 426
+#define __NR_io_uring_register 427
+#define __NR_open_tree 428
+#define __NR_move_mount 429
+#define __NR_fsopen 430
+#define __NR_fsconfig 431
+#define __NR_fsmount 432
+#define __NR_fspick 433
+#define __NR_pidfd_open 434
+#define __NR_clone3 435
+#define __NR_close_range 436
+#define __NR_openat2 437
+#define __NR_pidfd_getfd 438
+#define __NR_faccessat2 439
+#define __NR_process_madvise 440
+#define __NR_epoll_pwait2 441
+#define __NR_mount_setattr 442
+#define __NR_quotactl_fd 443
+#define __NR_landlock_create_ruleset 444
+#define __NR_landlock_add_rule 445
+#define __NR_landlock_restrict_self 446
+#define __NR_memfd_secret 447
+#define __NR_process_mrelease 448
+#define __NR_futex_waitv 449
+#define __NR_set_mempolicy_home_node 450
+#define __NR_cachestat 451
+#define __NR_fchmodat2 452
+#define __NR_map_shadow_stack 453
+#define __NR_futex_wake 454
+#define __NR_futex_wait 455
+#define __NR_futex_requeue 456
+#define __NR_statmount 457
+#define __NR_listmount 458
+#define __NR_lsm_get_self_attr 459
+#define __NR_lsm_set_self_attr 460
+#define __NR_lsm_list_modules 461
+#define __NR_mseal 462
+#endif
diff --git a/libc/kernel/uapi/asm-x86/asm/kvm.h b/libc/kernel/uapi/asm-x86/asm/kvm.h
index 17b1c5d..cd647b6 100644
--- a/libc/kernel/uapi/asm-x86/asm/kvm.h
+++ b/libc/kernel/uapi/asm-x86/asm/kvm.h
@@ -94,6 +94,7 @@
 #define KVM_NR_IRQCHIPS 3
 #define KVM_RUN_X86_SMM (1 << 0)
 #define KVM_RUN_X86_BUS_LOCK (1 << 1)
+#define KVM_RUN_X86_GUEST_MODE (1 << 2)
 struct kvm_regs {
   __u64 rax, rbx, rcx, rdx;
   __u64 rsi, rdi, rsp, rbp;
@@ -532,6 +533,9 @@
   KVM_SEV_GET_ATTESTATION_REPORT,
   KVM_SEV_SEND_CANCEL,
   KVM_SEV_INIT2,
+  KVM_SEV_SNP_LAUNCH_START = 100,
+  KVM_SEV_SNP_LAUNCH_UPDATE,
+  KVM_SEV_SNP_LAUNCH_FINISH,
   KVM_SEV_NR_MAX,
 };
 struct kvm_sev_cmd {
@@ -644,6 +648,42 @@
   __u32 trans_len;
   __u32 pad2;
 };
+struct kvm_sev_snp_launch_start {
+  __u64 policy;
+  __u8 gosvw[16];
+  __u16 flags;
+  __u8 pad0[6];
+  __u64 pad1[4];
+};
+#define KVM_SEV_SNP_PAGE_TYPE_NORMAL 0x1
+#define KVM_SEV_SNP_PAGE_TYPE_ZERO 0x3
+#define KVM_SEV_SNP_PAGE_TYPE_UNMEASURED 0x4
+#define KVM_SEV_SNP_PAGE_TYPE_SECRETS 0x5
+#define KVM_SEV_SNP_PAGE_TYPE_CPUID 0x6
+struct kvm_sev_snp_launch_update {
+  __u64 gfn_start;
+  __u64 uaddr;
+  __u64 len;
+  __u8 type;
+  __u8 pad0;
+  __u16 flags;
+  __u32 pad1;
+  __u64 pad2[4];
+};
+#define KVM_SEV_SNP_ID_BLOCK_SIZE 96
+#define KVM_SEV_SNP_ID_AUTH_SIZE 4096
+#define KVM_SEV_SNP_FINISH_DATA_SIZE 32
+struct kvm_sev_snp_launch_finish {
+  __u64 id_block_uaddr;
+  __u64 id_auth_uaddr;
+  __u8 id_block_en;
+  __u8 auth_key_en;
+  __u8 vcek_disabled;
+  __u8 host_data[KVM_SEV_SNP_FINISH_DATA_SIZE];
+  __u8 pad0[3];
+  __u16 flags;
+  __u64 pad1[4];
+};
 #define KVM_X2APIC_API_USE_32BIT_IDS (1ULL << 0)
 #define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK (1ULL << 1)
 struct kvm_hyperv_eventfd {
@@ -667,4 +707,5 @@
 #define KVM_X86_SW_PROTECTED_VM 1
 #define KVM_X86_SEV_VM 2
 #define KVM_X86_SEV_ES_VM 3
+#define KVM_X86_SNP_VM 4
 #endif
diff --git a/libc/kernel/uapi/asm-x86/asm/svm.h b/libc/kernel/uapi/asm-x86/asm/svm.h
index ffbf0b3..4f165fa 100644
--- a/libc/kernel/uapi/asm-x86/asm/svm.h
+++ b/libc/kernel/uapi/asm-x86/asm/svm.h
@@ -117,6 +117,7 @@
 #define SVM_VMGEXIT_AP_CREATE_ON_INIT 0
 #define SVM_VMGEXIT_AP_CREATE 1
 #define SVM_VMGEXIT_AP_DESTROY 2
+#define SVM_VMGEXIT_SNP_RUN_VMPL 0x80000018
 #define SVM_VMGEXIT_HV_FEATURES 0x8000fffd
 #define SVM_VMGEXIT_TERM_REQUEST 0x8000fffe
 #define SVM_VMGEXIT_TERM_REASON(reason_set,reason_code) (((((u64) reason_set) & 0xf)) | ((((u64) reason_code) & 0xff) << 4))
diff --git a/libc/kernel/uapi/asm-x86/asm/unistd_64.h b/libc/kernel/uapi/asm-x86/asm/unistd_64.h
index 5dd666c..d5408a3 100644
--- a/libc/kernel/uapi/asm-x86/asm/unistd_64.h
+++ b/libc/kernel/uapi/asm-x86/asm/unistd_64.h
@@ -341,6 +341,7 @@
 #define __NR_statx 332
 #define __NR_io_pgetevents 333
 #define __NR_rseq 334
+#define __NR_uretprobe 335
 #define __NR_pidfd_send_signal 424
 #define __NR_io_uring_setup 425
 #define __NR_io_uring_enter 426
diff --git a/libc/kernel/uapi/asm-x86/asm/unistd_x32.h b/libc/kernel/uapi/asm-x86/asm/unistd_x32.h
index a2ff6f4..fdcf7e6 100644
--- a/libc/kernel/uapi/asm-x86/asm/unistd_x32.h
+++ b/libc/kernel/uapi/asm-x86/asm/unistd_x32.h
@@ -294,6 +294,7 @@
 #define __NR_statx (__X32_SYSCALL_BIT + 332)
 #define __NR_io_pgetevents (__X32_SYSCALL_BIT + 333)
 #define __NR_rseq (__X32_SYSCALL_BIT + 334)
+#define __NR_uretprobe (__X32_SYSCALL_BIT + 335)
 #define __NR_pidfd_send_signal (__X32_SYSCALL_BIT + 424)
 #define __NR_io_uring_setup (__X32_SYSCALL_BIT + 425)
 #define __NR_io_uring_enter (__X32_SYSCALL_BIT + 426)
diff --git a/libc/kernel/uapi/drm/amdgpu_drm.h b/libc/kernel/uapi/drm/amdgpu_drm.h
index 0ad0bc2..7bbd5de 100644
--- a/libc/kernel/uapi/drm/amdgpu_drm.h
+++ b/libc/kernel/uapi/drm/amdgpu_drm.h
@@ -65,6 +65,7 @@
 #define AMDGPU_GEM_CREATE_COHERENT (1 << 13)
 #define AMDGPU_GEM_CREATE_UNCACHED (1 << 14)
 #define AMDGPU_GEM_CREATE_EXT_COHERENT (1 << 15)
+#define AMDGPU_GEM_CREATE_GFX12_DCC (1 << 16)
 struct drm_amdgpu_gem_create_in {
   __u64 bo_size;
   __u64 alignment;
@@ -216,6 +217,14 @@
 #define AMDGPU_TILING_DCC_INDEPENDENT_128B_MASK 0x1
 #define AMDGPU_TILING_SCANOUT_SHIFT 63
 #define AMDGPU_TILING_SCANOUT_MASK 0x1
+#define AMDGPU_TILING_GFX12_SWIZZLE_MODE_SHIFT 0
+#define AMDGPU_TILING_GFX12_SWIZZLE_MODE_MASK 0x7
+#define AMDGPU_TILING_GFX12_DCC_MAX_COMPRESSED_BLOCK_SHIFT 3
+#define AMDGPU_TILING_GFX12_DCC_MAX_COMPRESSED_BLOCK_MASK 0x3
+#define AMDGPU_TILING_GFX12_DCC_NUMBER_TYPE_SHIFT 5
+#define AMDGPU_TILING_GFX12_DCC_NUMBER_TYPE_MASK 0x7
+#define AMDGPU_TILING_GFX12_DCC_DATA_FORMAT_SHIFT 8
+#define AMDGPU_TILING_GFX12_DCC_DATA_FORMAT_MASK 0x3f
 #define AMDGPU_TILING_SET(field,value) (((__u64) (value) & AMDGPU_TILING_ ##field ##_MASK) << AMDGPU_TILING_ ##field ##_SHIFT)
 #define AMDGPU_TILING_GET(value,field) (((__u64) (value) >> AMDGPU_TILING_ ##field ##_SHIFT) & AMDGPU_TILING_ ##field ##_MASK)
 #define AMDGPU_GEM_METADATA_OP_SET_METADATA 1
@@ -744,6 +753,10 @@
 #define AMDGPU_FAMILY_GC_10_3_6 149
 #define AMDGPU_FAMILY_GC_10_3_7 151
 #define AMDGPU_FAMILY_GC_11_5_0 150
+#define AMDGPU_FAMILY_GC_12_0_0 152
+struct drm_color_ctm_3x4 {
+  __u64 matrix[12];
+};
 #ifdef __cplusplus
 }
 #endif
diff --git a/libc/kernel/uapi/drm/drm_fourcc.h b/libc/kernel/uapi/drm/drm_fourcc.h
index 6fd2eb8..4902d6c 100644
--- a/libc/kernel/uapi/drm/drm_fourcc.h
+++ b/libc/kernel/uapi/drm/drm_fourcc.h
@@ -255,12 +255,17 @@
 #define AMD_FMT_MOD_TILE_VER_GFX10 2
 #define AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS 3
 #define AMD_FMT_MOD_TILE_VER_GFX11 4
+#define AMD_FMT_MOD_TILE_VER_GFX12 5
 #define AMD_FMT_MOD_TILE_GFX9_64K_S 9
 #define AMD_FMT_MOD_TILE_GFX9_64K_D 10
 #define AMD_FMT_MOD_TILE_GFX9_64K_S_X 25
 #define AMD_FMT_MOD_TILE_GFX9_64K_D_X 26
 #define AMD_FMT_MOD_TILE_GFX9_64K_R_X 27
 #define AMD_FMT_MOD_TILE_GFX11_256K_R_X 31
+#define AMD_FMT_MOD_TILE_GFX12_256B_2D 1
+#define AMD_FMT_MOD_TILE_GFX12_4K_2D 2
+#define AMD_FMT_MOD_TILE_GFX12_64K_2D 3
+#define AMD_FMT_MOD_TILE_GFX12_256K_2D 4
 #define AMD_FMT_MOD_DCC_BLOCK_64B 0
 #define AMD_FMT_MOD_DCC_BLOCK_128B 1
 #define AMD_FMT_MOD_DCC_BLOCK_256B 2
diff --git a/libc/kernel/uapi/drm/drm_mode.h b/libc/kernel/uapi/drm/drm_mode.h
index 8fccdaf..06c91c5 100644
--- a/libc/kernel/uapi/drm/drm_mode.h
+++ b/libc/kernel/uapi/drm/drm_mode.h
@@ -357,9 +357,6 @@
 struct drm_color_ctm {
   __u64 matrix[9];
 };
-struct drm_color_ctm_3x4 {
-  __u64 matrix[12];
-};
 struct drm_color_lut {
   __u16 red;
   __u16 green;
diff --git a/libc/kernel/uapi/drm/i915_drm.h b/libc/kernel/uapi/drm/i915_drm.h
index 13eda7c..b43d8df 100644
--- a/libc/kernel/uapi/drm/i915_drm.h
+++ b/libc/kernel/uapi/drm/i915_drm.h
@@ -745,6 +745,7 @@
 #define I915_CONTEXT_PARAM_RINGSIZE 0xc
 #define I915_CONTEXT_PARAM_PROTECTED_CONTENT 0xd
 #define I915_CONTEXT_PARAM_LOW_LATENCY 0xe
+#define I915_CONTEXT_PARAM_CONTEXT_IMAGE 0xf
   __u64 value;
 };
 struct drm_i915_gem_context_param_sseu {
@@ -799,6 +800,14 @@
 } __attribute__((packed));
 #define I915_DEFINE_CONTEXT_PARAM_ENGINES(name__,N__) struct { __u64 extensions; struct i915_engine_class_instance engines[N__]; \
 } __attribute__((packed)) name__
+struct i915_gem_context_param_context_image {
+  struct i915_engine_class_instance engine;
+  __u32 flags;
+#define I915_CONTEXT_IMAGE_FLAG_ENGINE_INDEX (1u << 0)
+  __u32 size;
+  __u32 mbz;
+  __u64 image;
+} __attribute__((packed));
 struct drm_i915_gem_context_create_ext_setparam {
   struct i915_user_extension base;
   struct drm_i915_gem_context_param param;
diff --git a/libc/kernel/uapi/drm/ivpu_accel.h b/libc/kernel/uapi/drm/ivpu_accel.h
index fcbf6f7..960bd43 100644
--- a/libc/kernel/uapi/drm/ivpu_accel.h
+++ b/libc/kernel/uapi/drm/ivpu_accel.h
@@ -18,12 +18,20 @@
 #define DRM_IVPU_BO_INFO 0x03
 #define DRM_IVPU_SUBMIT 0x05
 #define DRM_IVPU_BO_WAIT 0x06
+#define DRM_IVPU_METRIC_STREAMER_START 0x07
+#define DRM_IVPU_METRIC_STREAMER_STOP 0x08
+#define DRM_IVPU_METRIC_STREAMER_GET_DATA 0x09
+#define DRM_IVPU_METRIC_STREAMER_GET_INFO 0x0a
 #define DRM_IOCTL_IVPU_GET_PARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_GET_PARAM, struct drm_ivpu_param)
 #define DRM_IOCTL_IVPU_SET_PARAM DRM_IOW(DRM_COMMAND_BASE + DRM_IVPU_SET_PARAM, struct drm_ivpu_param)
 #define DRM_IOCTL_IVPU_BO_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_BO_CREATE, struct drm_ivpu_bo_create)
 #define DRM_IOCTL_IVPU_BO_INFO DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_BO_INFO, struct drm_ivpu_bo_info)
 #define DRM_IOCTL_IVPU_SUBMIT DRM_IOW(DRM_COMMAND_BASE + DRM_IVPU_SUBMIT, struct drm_ivpu_submit)
 #define DRM_IOCTL_IVPU_BO_WAIT DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_BO_WAIT, struct drm_ivpu_bo_wait)
+#define DRM_IOCTL_IVPU_METRIC_STREAMER_START DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_METRIC_STREAMER_START, struct drm_ivpu_metric_streamer_start)
+#define DRM_IOCTL_IVPU_METRIC_STREAMER_STOP DRM_IOW(DRM_COMMAND_BASE + DRM_IVPU_METRIC_STREAMER_STOP, struct drm_ivpu_metric_streamer_stop)
+#define DRM_IOCTL_IVPU_METRIC_STREAMER_GET_DATA DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_METRIC_STREAMER_GET_DATA, struct drm_ivpu_metric_streamer_get_data)
+#define DRM_IOCTL_IVPU_METRIC_STREAMER_GET_INFO DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_METRIC_STREAMER_GET_INFO, struct drm_ivpu_metric_streamer_get_data)
 #define DRM_IVPU_PARAM_DEVICE_ID 0
 #define DRM_IVPU_PARAM_DEVICE_REVISION 1
 #define DRM_IVPU_PARAM_PLATFORM_TYPE 2
@@ -96,6 +104,22 @@
   __u32 job_status;
   __u32 pad;
 };
+struct drm_ivpu_metric_streamer_start {
+  __u64 metric_group_mask;
+  __u64 sampling_period_ns;
+  __u32 read_period_samples;
+  __u32 sample_size;
+  __u32 max_data_size;
+};
+struct drm_ivpu_metric_streamer_get_data {
+  __u64 metric_group_mask;
+  __u64 buffer_ptr;
+  __u64 buffer_size;
+  __u64 data_size;
+};
+struct drm_ivpu_metric_streamer_stop {
+  __u64 metric_group_mask;
+};
 #ifdef __cplusplus
 }
 #endif
diff --git a/libc/kernel/uapi/drm/msm_drm.h b/libc/kernel/uapi/drm/msm_drm.h
index 4d83744..7ec5ed2 100644
--- a/libc/kernel/uapi/drm/msm_drm.h
+++ b/libc/kernel/uapi/drm/msm_drm.h
@@ -37,6 +37,7 @@
 #define MSM_PARAM_VA_START 0x0e
 #define MSM_PARAM_VA_SIZE 0x0f
 #define MSM_PARAM_HIGHEST_BANK_BIT 0x10
+#define MSM_PARAM_RAYTRACING 0x11
 #define MSM_PARAM_NR_RINGS MSM_PARAM_PRIORITIES
 struct drm_msm_param {
   __u32 pipe;
diff --git a/libc/kernel/uapi/drm/v3d_drm.h b/libc/kernel/uapi/drm/v3d_drm.h
index 4000fd3..b7aca21 100644
--- a/libc/kernel/uapi/drm/v3d_drm.h
+++ b/libc/kernel/uapi/drm/v3d_drm.h
@@ -22,6 +22,7 @@
 #define DRM_V3D_PERFMON_DESTROY 0x09
 #define DRM_V3D_PERFMON_GET_VALUES 0x0a
 #define DRM_V3D_SUBMIT_CPU 0x0b
+#define DRM_V3D_PERFMON_GET_COUNTER 0x0c
 #define DRM_IOCTL_V3D_SUBMIT_CL DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_SUBMIT_CL, struct drm_v3d_submit_cl)
 #define DRM_IOCTL_V3D_WAIT_BO DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_WAIT_BO, struct drm_v3d_wait_bo)
 #define DRM_IOCTL_V3D_CREATE_BO DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_CREATE_BO, struct drm_v3d_create_bo)
@@ -34,6 +35,7 @@
 #define DRM_IOCTL_V3D_PERFMON_DESTROY DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_PERFMON_DESTROY, struct drm_v3d_perfmon_destroy)
 #define DRM_IOCTL_V3D_PERFMON_GET_VALUES DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_PERFMON_GET_VALUES, struct drm_v3d_perfmon_get_values)
 #define DRM_IOCTL_V3D_SUBMIT_CPU DRM_IOW(DRM_COMMAND_BASE + DRM_V3D_SUBMIT_CPU, struct drm_v3d_submit_cpu)
+#define DRM_IOCTL_V3D_PERFMON_GET_COUNTER DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_PERFMON_GET_COUNTER, struct drm_v3d_perfmon_get_counter)
 #define DRM_V3D_SUBMIT_CL_FLUSH_CACHE 0x01
 #define DRM_V3D_SUBMIT_EXTENSION 0x02
 struct drm_v3d_extension {
@@ -119,6 +121,7 @@
   DRM_V3D_PARAM_SUPPORTS_PERFMON,
   DRM_V3D_PARAM_SUPPORTS_MULTISYNC_EXT,
   DRM_V3D_PARAM_SUPPORTS_CPU_QUEUE,
+  DRM_V3D_PARAM_MAX_PERF_COUNTERS,
 };
 struct drm_v3d_get_param {
   __u32 param;
@@ -324,6 +327,16 @@
   __u32 pad;
   __u64 values_ptr;
 };
+#define DRM_V3D_PERFCNT_MAX_NAME 64
+#define DRM_V3D_PERFCNT_MAX_CATEGORY 32
+#define DRM_V3D_PERFCNT_MAX_DESCRIPTION 256
+struct drm_v3d_perfmon_get_counter {
+  __u8 counter;
+  __u8 name[DRM_V3D_PERFCNT_MAX_NAME];
+  __u8 category[DRM_V3D_PERFCNT_MAX_CATEGORY];
+  __u8 description[DRM_V3D_PERFCNT_MAX_DESCRIPTION];
+  __u8 reserved[7];
+};
 #ifdef __cplusplus
 }
 #endif
diff --git a/libc/kernel/uapi/drm/xe_drm.h b/libc/kernel/uapi/drm/xe_drm.h
index d1b6dad..a034b29 100644
--- a/libc/kernel/uapi/drm/xe_drm.h
+++ b/libc/kernel/uapi/drm/xe_drm.h
@@ -21,6 +21,7 @@
 #define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x08
 #define DRM_XE_EXEC 0x09
 #define DRM_XE_WAIT_USER_FENCE 0x0a
+#define DRM_XE_OBSERVATION 0x0b
 #define DRM_IOCTL_XE_DEVICE_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_DEVICE_QUERY, struct drm_xe_device_query)
 #define DRM_IOCTL_XE_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_CREATE, struct drm_xe_gem_create)
 #define DRM_IOCTL_XE_GEM_MMAP_OFFSET DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_MMAP_OFFSET, struct drm_xe_gem_mmap_offset)
@@ -32,6 +33,7 @@
 #define DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_GET_PROPERTY, struct drm_xe_exec_queue_get_property)
 #define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec)
 #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence)
+#define DRM_IOCTL_XE_OBSERVATION DRM_IOW(DRM_COMMAND_BASE + DRM_XE_OBSERVATION, struct drm_xe_observation_param)
 struct drm_xe_user_extension {
   __u64 next_extension;
   __u32 name;
@@ -120,6 +122,7 @@
   __u16 gt_id;
 #define DRM_XE_TOPO_DSS_GEOMETRY 1
 #define DRM_XE_TOPO_DSS_COMPUTE 2
+#define DRM_XE_TOPO_L3_BANK 3
 #define DRM_XE_TOPO_EU_PER_DSS 4
   __u16 type;
   __u32 num_bytes;
@@ -155,6 +158,7 @@
 #define DRM_XE_DEVICE_QUERY_GT_TOPOLOGY 5
 #define DRM_XE_DEVICE_QUERY_ENGINE_CYCLES 6
 #define DRM_XE_DEVICE_QUERY_UC_FW_VERSION 7
+#define DRM_XE_DEVICE_QUERY_OA_UNITS 8
   __u32 query;
   __u32 size;
   __u64 data;
@@ -309,6 +313,92 @@
   __u32 pad2;
   __u64 reserved[2];
 };
+enum drm_xe_observation_type {
+  DRM_XE_OBSERVATION_TYPE_OA,
+};
+enum drm_xe_observation_op {
+  DRM_XE_OBSERVATION_OP_STREAM_OPEN,
+  DRM_XE_OBSERVATION_OP_ADD_CONFIG,
+  DRM_XE_OBSERVATION_OP_REMOVE_CONFIG,
+};
+struct drm_xe_observation_param {
+  __u64 extensions;
+  __u64 observation_type;
+  __u64 observation_op;
+  __u64 param;
+};
+enum drm_xe_observation_ioctls {
+  DRM_XE_OBSERVATION_IOCTL_ENABLE = _IO('i', 0x0),
+  DRM_XE_OBSERVATION_IOCTL_DISABLE = _IO('i', 0x1),
+  DRM_XE_OBSERVATION_IOCTL_CONFIG = _IO('i', 0x2),
+  DRM_XE_OBSERVATION_IOCTL_STATUS = _IO('i', 0x3),
+  DRM_XE_OBSERVATION_IOCTL_INFO = _IO('i', 0x4),
+};
+enum drm_xe_oa_unit_type {
+  DRM_XE_OA_UNIT_TYPE_OAG,
+  DRM_XE_OA_UNIT_TYPE_OAM,
+};
+struct drm_xe_oa_unit {
+  __u64 extensions;
+  __u32 oa_unit_id;
+  __u32 oa_unit_type;
+  __u64 capabilities;
+#define DRM_XE_OA_CAPS_BASE (1 << 0)
+  __u64 oa_timestamp_freq;
+  __u64 reserved[4];
+  __u64 num_engines;
+  struct drm_xe_engine_class_instance eci[];
+};
+struct drm_xe_query_oa_units {
+  __u64 extensions;
+  __u32 num_oa_units;
+  __u32 pad;
+  __u64 oa_units[];
+};
+enum drm_xe_oa_format_type {
+  DRM_XE_OA_FMT_TYPE_OAG,
+  DRM_XE_OA_FMT_TYPE_OAR,
+  DRM_XE_OA_FMT_TYPE_OAM,
+  DRM_XE_OA_FMT_TYPE_OAC,
+  DRM_XE_OA_FMT_TYPE_OAM_MPEC,
+  DRM_XE_OA_FMT_TYPE_PEC,
+};
+enum drm_xe_oa_property_id {
+#define DRM_XE_OA_EXTENSION_SET_PROPERTY 0
+  DRM_XE_OA_PROPERTY_OA_UNIT_ID = 1,
+  DRM_XE_OA_PROPERTY_SAMPLE_OA,
+  DRM_XE_OA_PROPERTY_OA_METRIC_SET,
+  DRM_XE_OA_PROPERTY_OA_FORMAT,
+#define DRM_XE_OA_FORMAT_MASK_FMT_TYPE (0xffu << 0)
+#define DRM_XE_OA_FORMAT_MASK_COUNTER_SEL (0xffu << 8)
+#define DRM_XE_OA_FORMAT_MASK_COUNTER_SIZE (0xffu << 16)
+#define DRM_XE_OA_FORMAT_MASK_BC_REPORT (0xffu << 24)
+  DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT,
+  DRM_XE_OA_PROPERTY_OA_DISABLED,
+  DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID,
+  DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE,
+  DRM_XE_OA_PROPERTY_NO_PREEMPT,
+};
+struct drm_xe_oa_config {
+  __u64 extensions;
+  char uuid[36];
+  __u32 n_regs;
+  __u64 regs_ptr;
+};
+struct drm_xe_oa_stream_status {
+  __u64 extensions;
+  __u64 oa_status;
+#define DRM_XE_OASTATUS_MMIO_TRG_Q_FULL (1 << 3)
+#define DRM_XE_OASTATUS_COUNTER_OVERFLOW (1 << 2)
+#define DRM_XE_OASTATUS_BUFFER_OVERFLOW (1 << 1)
+#define DRM_XE_OASTATUS_REPORT_LOST (1 << 0)
+  __u64 reserved[3];
+};
+struct drm_xe_oa_stream_info {
+  __u64 extensions;
+  __u64 oa_buf_size;
+  __u64 reserved[3];
+};
 #ifdef __cplusplus
 }
 #endif
diff --git a/libc/kernel/uapi/linux/bpf.h b/libc/kernel/uapi/linux/bpf.h
index c732920..8d64816 100644
--- a/libc/kernel/uapi/linux/bpf.h
+++ b/libc/kernel/uapi/linux/bpf.h
@@ -367,6 +367,7 @@
 #define BPF_F_QUERY_EFFECTIVE (1U << 0)
 #define BPF_F_TEST_RUN_ON_CPU (1U << 0)
 #define BPF_F_TEST_XDP_LIVE_FRAMES (1U << 1)
+#define BPF_F_TEST_SKB_CHECKSUM_COMPLETE (1U << 2)
 enum bpf_stats_type {
   BPF_STATS_RUN_TIME = 0,
 };
@@ -773,8 +774,11 @@
 #define __bpf_md_ptr(type,name) union { type name; __u64 : 64; \
 } __attribute__((aligned(8)))
 enum {
-  BPF_SKB_TSTAMP_UNSPEC,
-  BPF_SKB_TSTAMP_DELIVERY_MONO,
+  BPF_SKB_TSTAMP_UNSPEC = 0,
+  BPF_SKB_TSTAMP_DELIVERY_MONO = 1,
+  BPF_SKB_CLOCK_REALTIME = 0,
+  BPF_SKB_CLOCK_MONOTONIC = 1,
+  BPF_SKB_CLOCK_TAI = 2,
 };
 struct __sk_buff {
   __u32 len;
diff --git a/libc/kernel/uapi/linux/btrfs_tree.h b/libc/kernel/uapi/linux/btrfs_tree.h
index ea33eee..88f44d9 100644
--- a/libc/kernel/uapi/linux/btrfs_tree.h
+++ b/libc/kernel/uapi/linux/btrfs_tree.h
@@ -279,18 +279,8 @@
   __le64 devid;
   __le64 physical;
 } __attribute__((__packed__));
-#define BTRFS_STRIPE_RAID0 1
-#define BTRFS_STRIPE_RAID1 2
-#define BTRFS_STRIPE_DUP 3
-#define BTRFS_STRIPE_RAID10 4
-#define BTRFS_STRIPE_RAID5 5
-#define BTRFS_STRIPE_RAID6 6
-#define BTRFS_STRIPE_RAID1C3 7
-#define BTRFS_STRIPE_RAID1C4 8
 struct btrfs_stripe_extent {
-  __u8 encoding;
-  __u8 reserved[7];
-  struct btrfs_raid_stride strides[];
+  __DECLARE_FLEX_ARRAY(struct btrfs_raid_stride, strides);
 } __attribute__((__packed__));
 #define BTRFS_HEADER_FLAG_WRITTEN (1ULL << 0)
 #define BTRFS_HEADER_FLAG_RELOC (1ULL << 1)
@@ -300,6 +290,9 @@
 #define BTRFS_SUPER_FLAG_METADUMP_V2 (1ULL << 34)
 #define BTRFS_SUPER_FLAG_CHANGING_FSID (1ULL << 35)
 #define BTRFS_SUPER_FLAG_CHANGING_FSID_V2 (1ULL << 36)
+#define BTRFS_SUPER_FLAG_CHANGING_BG_TREE (1ULL << 38)
+#define BTRFS_SUPER_FLAG_CHANGING_DATA_CSUM (1ULL << 39)
+#define BTRFS_SUPER_FLAG_CHANGING_META_CSUM (1ULL << 40)
 struct btrfs_extent_item {
   __le64 refs;
   __le64 generation;
diff --git a/libc/kernel/uapi/linux/dlm.h b/libc/kernel/uapi/linux/dlm.h
index 3c73908..1344f9b 100644
--- a/libc/kernel/uapi/linux/dlm.h
+++ b/libc/kernel/uapi/linux/dlm.h
@@ -20,4 +20,5 @@
 };
 #define DLM_LSFL_TIMEWARN 0x00000002
 #define DLM_LSFL_NEWEXCL 0x00000008
+#define __DLM_LSFL_RESERVED0 0x00000010
 #endif
diff --git a/libc/kernel/uapi/linux/dma-heap.h b/libc/kernel/uapi/linux/dma-heap.h
index 467e336..ac86b59 100644
--- a/libc/kernel/uapi/linux/dma-heap.h
+++ b/libc/kernel/uapi/linux/dma-heap.h
@@ -9,7 +9,7 @@
 #include <linux/ioctl.h>
 #include <linux/types.h>
 #define DMA_HEAP_VALID_FD_FLAGS (O_CLOEXEC | O_ACCMODE)
-#define DMA_HEAP_VALID_HEAP_FLAGS (0)
+#define DMA_HEAP_VALID_HEAP_FLAGS (0ULL)
 struct dma_heap_allocation_data {
   __u64 len;
   __u32 fd;
diff --git a/libc/kernel/uapi/linux/ethtool.h b/libc/kernel/uapi/linux/ethtool.h
index 334c788..e213ba1 100644
--- a/libc/kernel/uapi/linux/ethtool.h
+++ b/libc/kernel/uapi/linux/ethtool.h
@@ -270,6 +270,56 @@
   ETHTOOL_MODULE_POWER_MODE_LOW = 1,
   ETHTOOL_MODULE_POWER_MODE_HIGH,
 };
+enum ethtool_c33_pse_ext_state {
+  ETHTOOL_C33_PSE_EXT_STATE_ERROR_CONDITION = 1,
+  ETHTOOL_C33_PSE_EXT_STATE_MR_MPS_VALID,
+  ETHTOOL_C33_PSE_EXT_STATE_MR_PSE_ENABLE,
+  ETHTOOL_C33_PSE_EXT_STATE_OPTION_DETECT_TED,
+  ETHTOOL_C33_PSE_EXT_STATE_OPTION_VPORT_LIM,
+  ETHTOOL_C33_PSE_EXT_STATE_OVLD_DETECTED,
+  ETHTOOL_C33_PSE_EXT_STATE_PD_DLL_POWER_TYPE,
+  ETHTOOL_C33_PSE_EXT_STATE_POWER_NOT_AVAILABLE,
+  ETHTOOL_C33_PSE_EXT_STATE_SHORT_DETECTED,
+};
+enum ethtool_c33_pse_ext_substate_mr_mps_valid {
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_MR_MPS_VALID_DETECTED_UNDERLOAD = 1,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_MR_MPS_VALID_CONNECTION_OPEN,
+};
+enum ethtool_c33_pse_ext_substate_error_condition {
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_NON_EXISTING_PORT = 1,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_UNDEFINED_PORT,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_INTERNAL_HW_FAULT,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_COMM_ERROR_AFTER_FORCE_ON,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_UNKNOWN_PORT_STATUS,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_HOST_CRASH_TURN_OFF,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_HOST_CRASH_FORCE_SHUTDOWN,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_CONFIG_CHANGE,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_ERROR_CONDITION_DETECTED_OVER_TEMP,
+};
+enum ethtool_c33_pse_ext_substate_mr_pse_enable {
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_MR_PSE_ENABLE_DISABLE_PIN_ACTIVE = 1,
+};
+enum ethtool_c33_pse_ext_substate_option_detect_ted {
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_OPTION_DETECT_TED_DET_IN_PROCESS = 1,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_OPTION_DETECT_TED_CONNECTION_CHECK_ERROR,
+};
+enum ethtool_c33_pse_ext_substate_option_vport_lim {
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_OPTION_VPORT_LIM_HIGH_VOLTAGE = 1,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_OPTION_VPORT_LIM_LOW_VOLTAGE,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_OPTION_VPORT_LIM_VOLTAGE_INJECTION,
+};
+enum ethtool_c33_pse_ext_substate_ovld_detected {
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_OVLD_DETECTED_OVERLOAD = 1,
+};
+enum ethtool_c33_pse_ext_substate_power_not_available {
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_POWER_NOT_AVAILABLE_BUDGET_EXCEEDED = 1,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_POWER_NOT_AVAILABLE_PORT_PW_LIMIT_EXCEEDS_CONTROLLER_BUDGET,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_POWER_NOT_AVAILABLE_PD_REQUEST_EXCEEDS_PORT_LIMIT,
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_POWER_NOT_AVAILABLE_HW_PW_LIMIT,
+};
+enum ethtool_c33_pse_ext_substate_short_detected {
+  ETHTOOL_C33_PSE_EXT_SUBSTATE_SHORT_DETECTED_SHORT_CONDITION = 1,
+};
 enum ethtool_pse_types {
   ETHTOOL_PSE_UNKNOWN = 1 << 0,
   ETHTOOL_PSE_PODL = 1 << 1,
@@ -311,6 +361,12 @@
   ETHTOOL_MM_VERIFY_STATUS_FAILED,
   ETHTOOL_MM_VERIFY_STATUS_DISABLED,
 };
+enum ethtool_module_fw_flash_status {
+  ETHTOOL_MODULE_FW_FLASH_STATUS_STARTED = 1,
+  ETHTOOL_MODULE_FW_FLASH_STATUS_IN_PROGRESS,
+  ETHTOOL_MODULE_FW_FLASH_STATUS_COMPLETED,
+  ETHTOOL_MODULE_FW_FLASH_STATUS_ERROR,
+};
 struct ethtool_gstrings {
   __u32 cmd;
   __u32 string_set;
@@ -749,6 +805,7 @@
   ETHTOOL_LINK_MODE_10baseT1S_Full_BIT = 99,
   ETHTOOL_LINK_MODE_10baseT1S_Half_BIT = 100,
   ETHTOOL_LINK_MODE_10baseT1S_P2MP_Half_BIT = 101,
+  ETHTOOL_LINK_MODE_10baseT1BRR_Full_BIT = 102,
   __ETHTOOL_LINK_MODE_MASK_NBITS
 };
 #define __ETHTOOL_LINK_MODE_LEGACY_MASK(base_name) (1UL << (ETHTOOL_LINK_MODE_ ##base_name ##_BIT))
diff --git a/libc/kernel/uapi/linux/ethtool_netlink.h b/libc/kernel/uapi/linux/ethtool_netlink.h
index 6757ef5..ac6391a 100644
--- a/libc/kernel/uapi/linux/ethtool_netlink.h
+++ b/libc/kernel/uapi/linux/ethtool_netlink.h
@@ -52,6 +52,7 @@
   ETHTOOL_MSG_PLCA_GET_STATUS,
   ETHTOOL_MSG_MM_GET,
   ETHTOOL_MSG_MM_SET,
+  ETHTOOL_MSG_MODULE_FW_FLASH_ACT,
   __ETHTOOL_MSG_USER_CNT,
   ETHTOOL_MSG_USER_MAX = __ETHTOOL_MSG_USER_CNT - 1
 };
@@ -100,6 +101,7 @@
   ETHTOOL_MSG_PLCA_NTF,
   ETHTOOL_MSG_MM_GET_REPLY,
   ETHTOOL_MSG_MM_NTF,
+  ETHTOOL_MSG_MODULE_FW_FLASH_NTF,
   __ETHTOOL_MSG_KERNEL_CNT,
   ETHTOOL_MSG_KERNEL_MAX = __ETHTOOL_MSG_KERNEL_CNT - 1
 };
@@ -316,10 +318,26 @@
   ETHTOOL_A_COALESCE_TX_AGGR_MAX_BYTES,
   ETHTOOL_A_COALESCE_TX_AGGR_MAX_FRAMES,
   ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS,
+  ETHTOOL_A_COALESCE_RX_PROFILE,
+  ETHTOOL_A_COALESCE_TX_PROFILE,
   __ETHTOOL_A_COALESCE_CNT,
   ETHTOOL_A_COALESCE_MAX = (__ETHTOOL_A_COALESCE_CNT - 1)
 };
 enum {
+  ETHTOOL_A_PROFILE_UNSPEC,
+  ETHTOOL_A_PROFILE_IRQ_MODERATION,
+  __ETHTOOL_A_PROFILE_CNT,
+  ETHTOOL_A_PROFILE_MAX = (__ETHTOOL_A_PROFILE_CNT - 1)
+};
+enum {
+  ETHTOOL_A_IRQ_MODERATION_UNSPEC,
+  ETHTOOL_A_IRQ_MODERATION_USEC,
+  ETHTOOL_A_IRQ_MODERATION_PKTS,
+  ETHTOOL_A_IRQ_MODERATION_COMPS,
+  __ETHTOOL_A_IRQ_MODERATION_CNT,
+  ETHTOOL_A_IRQ_MODERATION_MAX = (__ETHTOOL_A_IRQ_MODERATION_CNT - 1)
+};
+enum {
   ETHTOOL_A_PAUSE_UNSPEC,
   ETHTOOL_A_PAUSE_HEADER,
   ETHTOOL_A_PAUSE_AUTONEG,
@@ -635,6 +653,11 @@
   ETHTOOL_A_MODULE_MAX = (__ETHTOOL_A_MODULE_CNT - 1)
 };
 enum {
+  ETHTOOL_A_C33_PSE_PW_LIMIT_UNSPEC,
+  ETHTOOL_A_C33_PSE_PW_LIMIT_MIN,
+  ETHTOOL_A_C33_PSE_PW_LIMIT_MAX,
+};
+enum {
   ETHTOOL_A_PSE_UNSPEC,
   ETHTOOL_A_PSE_HEADER,
   ETHTOOL_A_PODL_PSE_ADMIN_STATE,
@@ -643,6 +666,12 @@
   ETHTOOL_A_C33_PSE_ADMIN_STATE,
   ETHTOOL_A_C33_PSE_ADMIN_CONTROL,
   ETHTOOL_A_C33_PSE_PW_D_STATUS,
+  ETHTOOL_A_C33_PSE_PW_CLASS,
+  ETHTOOL_A_C33_PSE_ACTUAL_PW,
+  ETHTOOL_A_C33_PSE_EXT_STATE,
+  ETHTOOL_A_C33_PSE_EXT_SUBSTATE,
+  ETHTOOL_A_C33_PSE_AVAIL_PW_LIMIT,
+  ETHTOOL_A_C33_PSE_PW_LIMIT_RANGES,
   __ETHTOOL_A_PSE_CNT,
   ETHTOOL_A_PSE_MAX = (__ETHTOOL_A_PSE_CNT - 1)
 };
@@ -699,6 +728,18 @@
   __ETHTOOL_A_MM_CNT,
   ETHTOOL_A_MM_MAX = (__ETHTOOL_A_MM_CNT - 1)
 };
+enum {
+  ETHTOOL_A_MODULE_FW_FLASH_UNSPEC,
+  ETHTOOL_A_MODULE_FW_FLASH_HEADER,
+  ETHTOOL_A_MODULE_FW_FLASH_FILE_NAME,
+  ETHTOOL_A_MODULE_FW_FLASH_PASSWORD,
+  ETHTOOL_A_MODULE_FW_FLASH_STATUS,
+  ETHTOOL_A_MODULE_FW_FLASH_STATUS_MSG,
+  ETHTOOL_A_MODULE_FW_FLASH_DONE,
+  ETHTOOL_A_MODULE_FW_FLASH_TOTAL,
+  __ETHTOOL_A_MODULE_FW_FLASH_CNT,
+  ETHTOOL_A_MODULE_FW_FLASH_MAX = (__ETHTOOL_A_MODULE_FW_FLASH_CNT - 1)
+};
 #define ETHTOOL_GENL_NAME "ethtool"
 #define ETHTOOL_GENL_VERSION 1
 #define ETHTOOL_MCGRP_MONITOR_NAME "monitor"
diff --git a/libc/kernel/uapi/linux/fs.h b/libc/kernel/uapi/linux/fs.h
index 38a2d9e..adab56f 100644
--- a/libc/kernel/uapi/linux/fs.h
+++ b/libc/kernel/uapi/linux/fs.h
@@ -194,8 +194,10 @@
 #define RWF_NOWAIT (( __kernel_rwf_t) 0x00000008)
 #define RWF_APPEND (( __kernel_rwf_t) 0x00000010)
 #define RWF_NOAPPEND (( __kernel_rwf_t) 0x00000020)
-#define RWF_SUPPORTED (RWF_HIPRI | RWF_DSYNC | RWF_SYNC | RWF_NOWAIT | RWF_APPEND | RWF_NOAPPEND)
-#define PAGEMAP_SCAN _IOWR('f', 16, struct pm_scan_arg)
+#define RWF_ATOMIC (( __kernel_rwf_t) 0x00000040)
+#define RWF_SUPPORTED (RWF_HIPRI | RWF_DSYNC | RWF_SYNC | RWF_NOWAIT | RWF_APPEND | RWF_NOAPPEND | RWF_ATOMIC)
+#define PROCFS_IOCTL_MAGIC 'f'
+#define PAGEMAP_SCAN _IOWR(PROCFS_IOCTL_MAGIC, 16, struct pm_scan_arg)
 #define PAGE_IS_WPALLOWED (1 << 0)
 #define PAGE_IS_WRITTEN (1 << 1)
 #define PAGE_IS_FILE (1 << 2)
@@ -225,4 +227,30 @@
   __u64 category_anyof_mask;
   __u64 return_mask;
 };
+#define PROCMAP_QUERY _IOWR(PROCFS_IOCTL_MAGIC, 17, struct procmap_query)
+enum procmap_query_flags {
+  PROCMAP_QUERY_VMA_READABLE = 0x01,
+  PROCMAP_QUERY_VMA_WRITABLE = 0x02,
+  PROCMAP_QUERY_VMA_EXECUTABLE = 0x04,
+  PROCMAP_QUERY_VMA_SHARED = 0x08,
+  PROCMAP_QUERY_COVERING_OR_NEXT_VMA = 0x10,
+  PROCMAP_QUERY_FILE_BACKED_VMA = 0x20,
+};
+struct procmap_query {
+  __u64 size;
+  __u64 query_flags;
+  __u64 query_addr;
+  __u64 vma_start;
+  __u64 vma_end;
+  __u64 vma_flags;
+  __u64 vma_page_size;
+  __u64 vma_offset;
+  __u64 inode;
+  __u32 dev_major;
+  __u32 dev_minor;
+  __u32 vma_name_size;
+  __u32 build_id_size;
+  __u64 vma_name_addr;
+  __u64 build_id_addr;
+};
 #endif
diff --git a/libc/kernel/uapi/linux/if_xdp.h b/libc/kernel/uapi/linux/if_xdp.h
index b7eec87..7201e06 100644
--- a/libc/kernel/uapi/linux/if_xdp.h
+++ b/libc/kernel/uapi/linux/if_xdp.h
@@ -14,6 +14,7 @@
 #define XDP_USE_SG (1 << 4)
 #define XDP_UMEM_UNALIGNED_CHUNK_FLAG (1 << 0)
 #define XDP_UMEM_TX_SW_CSUM (1 << 1)
+#define XDP_UMEM_TX_METADATA_LEN (1 << 2)
 struct sockaddr_xdp {
   __u16 sxdp_family;
   __u16 sxdp_flags;
diff --git a/libc/kernel/uapi/linux/iio/buffer.h b/libc/kernel/uapi/linux/iio/buffer.h
index 45c6f65..7e03a8c 100644
--- a/libc/kernel/uapi/linux/iio/buffer.h
+++ b/libc/kernel/uapi/linux/iio/buffer.h
@@ -6,5 +6,16 @@
  */
 #ifndef _UAPI_IIO_BUFFER_H_
 #define _UAPI_IIO_BUFFER_H_
+#include <linux/types.h>
+#define IIO_BUFFER_DMABUF_CYCLIC (1 << 0)
+#define IIO_BUFFER_DMABUF_SUPPORTED_FLAGS 0x00000001
+struct iio_dmabuf {
+  __u32 fd;
+  __u32 flags;
+  __u64 bytes_used;
+};
 #define IIO_BUFFER_GET_FD_IOCTL _IOWR('i', 0x91, int)
+#define IIO_BUFFER_DMABUF_ATTACH_IOCTL _IOW('i', 0x92, int)
+#define IIO_BUFFER_DMABUF_DETACH_IOCTL _IOW('i', 0x93, int)
+#define IIO_BUFFER_DMABUF_ENQUEUE_IOCTL _IOW('i', 0x94, struct iio_dmabuf)
 #endif
diff --git a/libc/kernel/uapi/linux/in.h b/libc/kernel/uapi/linux/in.h
index 44efdd8..97bf493 100644
--- a/libc/kernel/uapi/linux/in.h
+++ b/libc/kernel/uapi/linux/in.h
@@ -69,6 +69,8 @@
 #define IPPROTO_ETHERNET IPPROTO_ETHERNET
   IPPROTO_RAW = 255,
 #define IPPROTO_RAW IPPROTO_RAW
+  IPPROTO_SMC = 256,
+#define IPPROTO_SMC IPPROTO_SMC
   IPPROTO_MPTCP = 262,
 #define IPPROTO_MPTCP IPPROTO_MPTCP
   IPPROTO_MAX
diff --git a/libc/kernel/uapi/linux/io_uring.h b/libc/kernel/uapi/linux/io_uring.h
index 5505963..6b4f2ea 100644
--- a/libc/kernel/uapi/linux/io_uring.h
+++ b/libc/kernel/uapi/linux/io_uring.h
@@ -176,6 +176,8 @@
   IORING_OP_FUTEX_WAITV,
   IORING_OP_FIXED_FD_INSTALL,
   IORING_OP_FTRUNCATE,
+  IORING_OP_BIND,
+  IORING_OP_LISTEN,
   IORING_OP_LAST,
 };
 #define IORING_URING_CMD_FIXED (1U << 0)
diff --git a/libc/kernel/uapi/linux/iommufd.h b/libc/kernel/uapi/linux/iommufd.h
index 2570628..6f663b4 100644
--- a/libc/kernel/uapi/linux/iommufd.h
+++ b/libc/kernel/uapi/linux/iommufd.h
@@ -12,19 +12,20 @@
 enum {
   IOMMUFD_CMD_BASE = 0x80,
   IOMMUFD_CMD_DESTROY = IOMMUFD_CMD_BASE,
-  IOMMUFD_CMD_IOAS_ALLOC,
-  IOMMUFD_CMD_IOAS_ALLOW_IOVAS,
-  IOMMUFD_CMD_IOAS_COPY,
-  IOMMUFD_CMD_IOAS_IOVA_RANGES,
-  IOMMUFD_CMD_IOAS_MAP,
-  IOMMUFD_CMD_IOAS_UNMAP,
-  IOMMUFD_CMD_OPTION,
-  IOMMUFD_CMD_VFIO_IOAS,
-  IOMMUFD_CMD_HWPT_ALLOC,
-  IOMMUFD_CMD_GET_HW_INFO,
-  IOMMUFD_CMD_HWPT_SET_DIRTY_TRACKING,
-  IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP,
-  IOMMUFD_CMD_HWPT_INVALIDATE,
+  IOMMUFD_CMD_IOAS_ALLOC = 0x81,
+  IOMMUFD_CMD_IOAS_ALLOW_IOVAS = 0x82,
+  IOMMUFD_CMD_IOAS_COPY = 0x83,
+  IOMMUFD_CMD_IOAS_IOVA_RANGES = 0x84,
+  IOMMUFD_CMD_IOAS_MAP = 0x85,
+  IOMMUFD_CMD_IOAS_UNMAP = 0x86,
+  IOMMUFD_CMD_OPTION = 0x87,
+  IOMMUFD_CMD_VFIO_IOAS = 0x88,
+  IOMMUFD_CMD_HWPT_ALLOC = 0x89,
+  IOMMUFD_CMD_GET_HW_INFO = 0x8a,
+  IOMMUFD_CMD_HWPT_SET_DIRTY_TRACKING = 0x8b,
+  IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP = 0x8c,
+  IOMMUFD_CMD_HWPT_INVALIDATE = 0x8d,
+  IOMMUFD_CMD_FAULT_QUEUE_ALLOC = 0x8e,
 };
 struct iommu_destroy {
   __u32 size;
@@ -122,6 +123,7 @@
 enum iommufd_hwpt_alloc_flags {
   IOMMU_HWPT_ALLOC_NEST_PARENT = 1 << 0,
   IOMMU_HWPT_ALLOC_DIRTY_TRACKING = 1 << 1,
+  IOMMU_HWPT_FAULT_ID_VALID = 1 << 2,
 };
 enum iommu_hwpt_vtd_s1_flags {
   IOMMU_VTD_S1_SRE = 1 << 0,
@@ -135,8 +137,8 @@
   __u32 __reserved;
 };
 enum iommu_hwpt_data_type {
-  IOMMU_HWPT_DATA_NONE,
-  IOMMU_HWPT_DATA_VTD_S1,
+  IOMMU_HWPT_DATA_NONE = 0,
+  IOMMU_HWPT_DATA_VTD_S1 = 1,
 };
 struct iommu_hwpt_alloc {
   __u32 size;
@@ -148,6 +150,8 @@
   __u32 data_type;
   __u32 data_len;
   __aligned_u64 data_uptr;
+  __u32 fault_id;
+  __u32 __reserved2;
 };
 #define IOMMU_HWPT_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_ALLOC)
 enum iommu_hw_info_vtd_flags {
@@ -160,8 +164,8 @@
   __aligned_u64 ecap_reg;
 };
 enum iommu_hw_info_type {
-  IOMMU_HW_INFO_TYPE_NONE,
-  IOMMU_HW_INFO_TYPE_INTEL_VTD,
+  IOMMU_HW_INFO_TYPE_NONE = 0,
+  IOMMU_HW_INFO_TYPE_INTEL_VTD = 1,
 };
 enum iommufd_hw_capabilities {
   IOMMU_HW_CAP_DIRTY_TRACKING = 1 << 0,
@@ -202,7 +206,7 @@
 };
 #define IOMMU_HWPT_GET_DIRTY_BITMAP _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP)
 enum iommu_hwpt_invalidate_data_type {
-  IOMMU_HWPT_INVALIDATE_DATA_VTD_S1,
+  IOMMU_HWPT_INVALIDATE_DATA_VTD_S1 = 0,
 };
 enum iommu_hwpt_vtd_s1_invalidate_flags {
   IOMMU_VTD_INV_FLAGS_LEAF = 1 << 0,
@@ -223,4 +227,39 @@
   __u32 __reserved;
 };
 #define IOMMU_HWPT_INVALIDATE _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HWPT_INVALIDATE)
+enum iommu_hwpt_pgfault_flags {
+  IOMMU_PGFAULT_FLAGS_PASID_VALID = (1 << 0),
+  IOMMU_PGFAULT_FLAGS_LAST_PAGE = (1 << 1),
+};
+enum iommu_hwpt_pgfault_perm {
+  IOMMU_PGFAULT_PERM_READ = (1 << 0),
+  IOMMU_PGFAULT_PERM_WRITE = (1 << 1),
+  IOMMU_PGFAULT_PERM_EXEC = (1 << 2),
+  IOMMU_PGFAULT_PERM_PRIV = (1 << 3),
+};
+struct iommu_hwpt_pgfault {
+  __u32 flags;
+  __u32 dev_id;
+  __u32 pasid;
+  __u32 grpid;
+  __u32 perm;
+  __u64 addr;
+  __u32 length;
+  __u32 cookie;
+};
+enum iommufd_page_response_code {
+  IOMMUFD_PAGE_RESP_SUCCESS = 0,
+  IOMMUFD_PAGE_RESP_INVALID = 1,
+};
+struct iommu_hwpt_page_response {
+  __u32 cookie;
+  __u32 code;
+};
+struct iommu_fault_alloc {
+  __u32 size;
+  __u32 flags;
+  __u32 out_fault_id;
+  __u32 out_fault_fd;
+};
+#define IOMMU_FAULT_QUEUE_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_FAULT_QUEUE_ALLOC)
 #endif
diff --git a/libc/kernel/uapi/linux/kfd_ioctl.h b/libc/kernel/uapi/linux/kfd_ioctl.h
index 62c9872..193dd8e 100644
--- a/libc/kernel/uapi/linux/kfd_ioctl.h
+++ b/libc/kernel/uapi/linux/kfd_ioctl.h
@@ -9,7 +9,7 @@
 #include <drm/drm.h>
 #include <linux/ioctl.h>
 #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 15
+#define KFD_IOCTL_MINOR_VERSION 16
 struct kfd_ioctl_get_version_args {
   __u32 major_version;
   __u32 minor_version;
@@ -269,6 +269,7 @@
 #define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26)
 #define KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED (1 << 25)
 #define KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT (1 << 24)
+#define KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS (1 << 23)
 struct kfd_ioctl_alloc_memory_of_gpu_args {
   __u64 va_addr;
   __u64 size;
@@ -465,6 +466,7 @@
 };
 enum kfd_dbg_trap_flags {
   KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP = 1,
+  KFD_DBG_TRAP_FLAG_SINGLE_ALU_OP = 2,
 };
 enum kfd_dbg_trap_exception_code {
   EC_NONE = 0,
diff --git a/libc/kernel/uapi/linux/kfd_sysfs.h b/libc/kernel/uapi/linux/kfd_sysfs.h
index e538cf2..7771582 100644
--- a/libc/kernel/uapi/linux/kfd_sysfs.h
+++ b/libc/kernel/uapi/linux/kfd_sysfs.h
@@ -35,7 +35,8 @@
 #define HSA_CAP_SVMAPI_SUPPORTED 0x08000000
 #define HSA_CAP_FLAGS_COHERENTHOSTACCESS 0x10000000
 #define HSA_CAP_TRAP_DEBUG_FIRMWARE_SUPPORTED 0x20000000
-#define HSA_CAP_RESERVED 0xe00f8000
+#define HSA_CAP_TRAP_DEBUG_PRECISE_ALU_OPERATIONS_SUPPORTED 0x40000000
+#define HSA_CAP_RESERVED 0x800f8000
 #define HSA_DBG_WATCH_ADDR_MASK_LO_BIT_MASK 0x0000000f
 #define HSA_DBG_WATCH_ADDR_MASK_LO_BIT_SHIFT 0
 #define HSA_DBG_WATCH_ADDR_MASK_HI_BIT_MASK 0x000003f0
diff --git a/libc/kernel/uapi/linux/kvm.h b/libc/kernel/uapi/linux/kvm.h
index ffaf5e6..297a09d 100644
--- a/libc/kernel/uapi/linux/kvm.h
+++ b/libc/kernel/uapi/linux/kvm.h
@@ -149,9 +149,10 @@
 #define KVM_INTERNAL_ERROR_DELIVERY_EV 3
 #define KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON 4
 #define KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES (1ULL << 0)
+#define HINT_UNSAFE_IN_KVM(_symbol) _symbol
 struct kvm_run {
   __u8 request_interrupt_window;
-  __u8 immediate_exit;
+  __u8 HINT_UNSAFE_IN_KVM(immediate_exit);
   __u8 padding1[6];
   __u32 exit_reason;
   __u8 ready_for_interrupt_injection;
@@ -719,6 +720,9 @@
 #define KVM_CAP_MEMORY_ATTRIBUTES 233
 #define KVM_CAP_GUEST_MEMFD 234
 #define KVM_CAP_VM_TYPES 235
+#define KVM_CAP_PRE_FAULT_MEMORY 236
+#define KVM_CAP_X86_APIC_BUS_CYCLES_NS 237
+#define KVM_CAP_X86_GUEST_MODE 238
 struct kvm_irq_routing_irqchip {
   __u32 irqchip;
   __u32 pin;
@@ -1109,4 +1113,11 @@
   __u64 flags;
   __u64 reserved[6];
 };
+#define KVM_PRE_FAULT_MEMORY _IOWR(KVMIO, 0xd5, struct kvm_pre_fault_memory)
+struct kvm_pre_fault_memory {
+  __u64 gpa;
+  __u64 size;
+  __u64 flags;
+  __u64 padding[5];
+};
 #endif
diff --git a/libc/kernel/uapi/linux/media/raspberrypi/pisp_be_config.h b/libc/kernel/uapi/linux/media/raspberrypi/pisp_be_config.h
new file mode 100644
index 0000000..2e981ad
--- /dev/null
+++ b/libc/kernel/uapi/linux/media/raspberrypi/pisp_be_config.h
@@ -0,0 +1,418 @@
+/*
+ * This file is auto-generated. Modifications will be lost.
+ *
+ * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
+ * for more information.
+ */
+#ifndef _UAPI_PISP_BE_CONFIG_H_
+#define _UAPI_PISP_BE_CONFIG_H_
+#include <linux/types.h>
+#include "pisp_common.h"
+#define PISP_BACK_END_INPUT_ALIGN 4u
+#define PISP_BACK_END_COMPRESSED_ALIGN 8u
+#define PISP_BACK_END_OUTPUT_MIN_ALIGN 16u
+#define PISP_BACK_END_OUTPUT_MAX_ALIGN 64u
+#define PISP_BACK_END_MIN_TILE_WIDTH 16u
+#define PISP_BACK_END_MIN_TILE_HEIGHT 16u
+#define PISP_BACK_END_NUM_OUTPUTS 2
+#define PISP_BACK_END_HOG_OUTPUT 1
+#define PISP_BACK_END_NUM_TILES 64
+enum pisp_be_bayer_enable {
+  PISP_BE_BAYER_ENABLE_INPUT = 0x000001,
+  PISP_BE_BAYER_ENABLE_DECOMPRESS = 0x000002,
+  PISP_BE_BAYER_ENABLE_DPC = 0x000004,
+  PISP_BE_BAYER_ENABLE_GEQ = 0x000008,
+  PISP_BE_BAYER_ENABLE_TDN_INPUT = 0x000010,
+  PISP_BE_BAYER_ENABLE_TDN_DECOMPRESS = 0x000020,
+  PISP_BE_BAYER_ENABLE_TDN = 0x000040,
+  PISP_BE_BAYER_ENABLE_TDN_COMPRESS = 0x000080,
+  PISP_BE_BAYER_ENABLE_TDN_OUTPUT = 0x000100,
+  PISP_BE_BAYER_ENABLE_SDN = 0x000200,
+  PISP_BE_BAYER_ENABLE_BLC = 0x000400,
+  PISP_BE_BAYER_ENABLE_STITCH_INPUT = 0x000800,
+  PISP_BE_BAYER_ENABLE_STITCH_DECOMPRESS = 0x001000,
+  PISP_BE_BAYER_ENABLE_STITCH = 0x002000,
+  PISP_BE_BAYER_ENABLE_STITCH_COMPRESS = 0x004000,
+  PISP_BE_BAYER_ENABLE_STITCH_OUTPUT = 0x008000,
+  PISP_BE_BAYER_ENABLE_WBG = 0x010000,
+  PISP_BE_BAYER_ENABLE_CDN = 0x020000,
+  PISP_BE_BAYER_ENABLE_LSC = 0x040000,
+  PISP_BE_BAYER_ENABLE_TONEMAP = 0x080000,
+  PISP_BE_BAYER_ENABLE_CAC = 0x100000,
+  PISP_BE_BAYER_ENABLE_DEBIN = 0x200000,
+  PISP_BE_BAYER_ENABLE_DEMOSAIC = 0x400000,
+};
+enum pisp_be_rgb_enable {
+  PISP_BE_RGB_ENABLE_INPUT = 0x000001,
+  PISP_BE_RGB_ENABLE_CCM = 0x000002,
+  PISP_BE_RGB_ENABLE_SAT_CONTROL = 0x000004,
+  PISP_BE_RGB_ENABLE_YCBCR = 0x000008,
+  PISP_BE_RGB_ENABLE_FALSE_COLOUR = 0x000010,
+  PISP_BE_RGB_ENABLE_SHARPEN = 0x000020,
+  PISP_BE_RGB_ENABLE_YCBCR_INVERSE = 0x000080,
+  PISP_BE_RGB_ENABLE_GAMMA = 0x000100,
+  PISP_BE_RGB_ENABLE_CSC0 = 0x000200,
+  PISP_BE_RGB_ENABLE_CSC1 = 0x000400,
+  PISP_BE_RGB_ENABLE_DOWNSCALE0 = 0x001000,
+  PISP_BE_RGB_ENABLE_DOWNSCALE1 = 0x002000,
+  PISP_BE_RGB_ENABLE_RESAMPLE0 = 0x008000,
+  PISP_BE_RGB_ENABLE_RESAMPLE1 = 0x010000,
+  PISP_BE_RGB_ENABLE_OUTPUT0 = 0x040000,
+  PISP_BE_RGB_ENABLE_OUTPUT1 = 0x080000,
+  PISP_BE_RGB_ENABLE_HOG = 0x200000
+};
+#define PISP_BE_RGB_ENABLE_CSC(i) (PISP_BE_RGB_ENABLE_CSC0 << (i))
+#define PISP_BE_RGB_ENABLE_DOWNSCALE(i) (PISP_BE_RGB_ENABLE_DOWNSCALE0 << (i))
+#define PISP_BE_RGB_ENABLE_RESAMPLE(i) (PISP_BE_RGB_ENABLE_RESAMPLE0 << (i))
+#define PISP_BE_RGB_ENABLE_OUTPUT(i) (PISP_BE_RGB_ENABLE_OUTPUT0 << (i))
+enum pisp_be_dirty {
+  PISP_BE_DIRTY_GLOBAL = 0x0001,
+  PISP_BE_DIRTY_SH_FC_COMBINE = 0x0002,
+  PISP_BE_DIRTY_CROP = 0x0004
+};
+struct pisp_be_global_config {
+  __u32 bayer_enables;
+  __u32 rgb_enables;
+  __u8 bayer_order;
+  __u8 pad[3];
+} __attribute__((packed));
+struct pisp_be_input_buffer_config {
+  __u32 addr[3][2];
+} __attribute__((packed));
+struct pisp_be_dpc_config {
+  __u8 coeff_level;
+  __u8 coeff_range;
+  __u8 pad;
+#define PISP_BE_DPC_FLAG_FOLDBACK 1
+  __u8 flags;
+} __attribute__((packed));
+struct pisp_be_geq_config {
+  __u16 offset;
+#define PISP_BE_GEQ_SHARPER (1U << 15)
+#define PISP_BE_GEQ_SLOPE ((1 << 10) - 1)
+  __u16 slope_sharper;
+  __u16 min;
+  __u16 max;
+} __attribute__((packed));
+struct pisp_be_tdn_input_buffer_config {
+  __u32 addr[2];
+} __attribute__((packed));
+struct pisp_be_tdn_config {
+  __u16 black_level;
+  __u16 ratio;
+  __u16 noise_constant;
+  __u16 noise_slope;
+  __u16 threshold;
+  __u8 reset;
+  __u8 pad;
+} __attribute__((packed));
+struct pisp_be_tdn_output_buffer_config {
+  __u32 addr[2];
+} __attribute__((packed));
+struct pisp_be_sdn_config {
+  __u16 black_level;
+  __u8 leakage;
+  __u8 pad;
+  __u16 noise_constant;
+  __u16 noise_slope;
+  __u16 noise_constant2;
+  __u16 noise_slope2;
+} __attribute__((packed));
+struct pisp_be_stitch_input_buffer_config {
+  __u32 addr[2];
+} __attribute__((packed));
+#define PISP_BE_STITCH_STREAMING_LONG 0x8000
+#define PISP_BE_STITCH_EXPOSURE_RATIO_MASK 0x7fff
+struct pisp_be_stitch_config {
+  __u16 threshold_lo;
+  __u8 threshold_diff_power;
+  __u8 pad;
+  __u16 exposure_ratio;
+  __u8 motion_threshold_256;
+  __u8 motion_threshold_recip;
+} __attribute__((packed));
+struct pisp_be_stitch_output_buffer_config {
+  __u32 addr[2];
+} __attribute__((packed));
+struct pisp_be_cdn_config {
+  __u16 thresh;
+  __u8 iir_strength;
+  __u8 g_adjust;
+} __attribute__((packed));
+#define PISP_BE_LSC_LOG_GRID_SIZE 5
+#define PISP_BE_LSC_GRID_SIZE (1 << PISP_BE_LSC_LOG_GRID_SIZE)
+#define PISP_BE_LSC_STEP_PRECISION 18
+struct pisp_be_lsc_config {
+  __u16 grid_step_x;
+  __u16 grid_step_y;
+#define PISP_BE_LSC_LUT_SIZE (PISP_BE_LSC_GRID_SIZE + 1)
+  __u32 lut_packed[PISP_BE_LSC_LUT_SIZE][PISP_BE_LSC_LUT_SIZE];
+} __attribute__((packed));
+struct pisp_be_lsc_extra {
+  __u16 offset_x;
+  __u16 offset_y;
+} __attribute__((packed));
+#define PISP_BE_CAC_LOG_GRID_SIZE 3
+#define PISP_BE_CAC_GRID_SIZE (1 << PISP_BE_CAC_LOG_GRID_SIZE)
+#define PISP_BE_CAC_STEP_PRECISION 20
+struct pisp_be_cac_config {
+  __u16 grid_step_x;
+  __u16 grid_step_y;
+#define PISP_BE_CAC_LUT_SIZE (PISP_BE_CAC_GRID_SIZE + 1)
+  __s8 lut[PISP_BE_CAC_LUT_SIZE][PISP_BE_CAC_LUT_SIZE][2][2];
+} __attribute__((packed));
+struct pisp_be_cac_extra {
+  __u16 offset_x;
+  __u16 offset_y;
+} __attribute__((packed));
+#define PISP_BE_DEBIN_NUM_COEFFS 4
+struct pisp_be_debin_config {
+  __s8 coeffs[PISP_BE_DEBIN_NUM_COEFFS];
+  __s8 h_enable;
+  __s8 v_enable;
+  __s8 pad[2];
+} __attribute__((packed));
+#define PISP_BE_TONEMAP_LUT_SIZE 64
+struct pisp_be_tonemap_config {
+  __u16 detail_constant;
+  __u16 detail_slope;
+  __u16 iir_strength;
+  __u16 strength;
+  __u32 lut[PISP_BE_TONEMAP_LUT_SIZE];
+} __attribute__((packed));
+struct pisp_be_demosaic_config {
+  __u8 sharper;
+  __u8 fc_mode;
+  __u8 pad[2];
+} __attribute__((packed));
+struct pisp_be_ccm_config {
+  __s16 coeffs[9];
+  __u8 pad[2];
+  __s32 offsets[3];
+} __attribute__((packed));
+struct pisp_be_sat_control_config {
+  __u8 shift_r;
+  __u8 shift_g;
+  __u8 shift_b;
+  __u8 pad;
+} __attribute__((packed));
+struct pisp_be_false_colour_config {
+  __u8 distance;
+  __u8 pad[3];
+} __attribute__((packed));
+#define PISP_BE_SHARPEN_SIZE 5
+#define PISP_BE_SHARPEN_FUNC_NUM_POINTS 9
+struct pisp_be_sharpen_config {
+  __s8 kernel0[PISP_BE_SHARPEN_SIZE * PISP_BE_SHARPEN_SIZE];
+  __s8 pad0[3];
+  __s8 kernel1[PISP_BE_SHARPEN_SIZE * PISP_BE_SHARPEN_SIZE];
+  __s8 pad1[3];
+  __s8 kernel2[PISP_BE_SHARPEN_SIZE * PISP_BE_SHARPEN_SIZE];
+  __s8 pad2[3];
+  __s8 kernel3[PISP_BE_SHARPEN_SIZE * PISP_BE_SHARPEN_SIZE];
+  __s8 pad3[3];
+  __s8 kernel4[PISP_BE_SHARPEN_SIZE * PISP_BE_SHARPEN_SIZE];
+  __s8 pad4[3];
+  __u16 threshold_offset0;
+  __u16 threshold_slope0;
+  __u16 scale0;
+  __u16 pad5;
+  __u16 threshold_offset1;
+  __u16 threshold_slope1;
+  __u16 scale1;
+  __u16 pad6;
+  __u16 threshold_offset2;
+  __u16 threshold_slope2;
+  __u16 scale2;
+  __u16 pad7;
+  __u16 threshold_offset3;
+  __u16 threshold_slope3;
+  __u16 scale3;
+  __u16 pad8;
+  __u16 threshold_offset4;
+  __u16 threshold_slope4;
+  __u16 scale4;
+  __u16 pad9;
+  __u16 positive_strength;
+  __u16 positive_pre_limit;
+  __u16 positive_func[PISP_BE_SHARPEN_FUNC_NUM_POINTS];
+  __u16 positive_limit;
+  __u16 negative_strength;
+  __u16 negative_pre_limit;
+  __u16 negative_func[PISP_BE_SHARPEN_FUNC_NUM_POINTS];
+  __u16 negative_limit;
+  __u8 enables;
+  __u8 white;
+  __u8 black;
+  __u8 grey;
+} __attribute__((packed));
+struct pisp_be_sh_fc_combine_config {
+  __u8 y_factor;
+  __u8 c1_factor;
+  __u8 c2_factor;
+  __u8 pad;
+} __attribute__((packed));
+#define PISP_BE_GAMMA_LUT_SIZE 64
+struct pisp_be_gamma_config {
+  __u32 lut[PISP_BE_GAMMA_LUT_SIZE];
+} __attribute__((packed));
+struct pisp_be_crop_config {
+  __u16 offset_x, offset_y;
+  __u16 width, height;
+} __attribute__((packed));
+#define PISP_BE_RESAMPLE_FILTER_SIZE 96
+struct pisp_be_resample_config {
+  __u16 scale_factor_h, scale_factor_v;
+  __s16 coef[PISP_BE_RESAMPLE_FILTER_SIZE];
+} __attribute__((packed));
+struct pisp_be_resample_extra {
+  __u16 scaled_width;
+  __u16 scaled_height;
+  __s16 initial_phase_h[3];
+  __s16 initial_phase_v[3];
+} __attribute__((packed));
+struct pisp_be_downscale_config {
+  __u16 scale_factor_h;
+  __u16 scale_factor_v;
+  __u16 scale_recip_h;
+  __u16 scale_recip_v;
+} __attribute__((packed));
+struct pisp_be_downscale_extra {
+  __u16 scaled_width;
+  __u16 scaled_height;
+} __attribute__((packed));
+struct pisp_be_hog_config {
+  __u8 compute_signed;
+  __u8 channel_mix[3];
+  __u32 stride;
+} __attribute__((packed));
+struct pisp_be_axi_config {
+  __u8 r_qos;
+  __u8 r_cache_prot;
+  __u8 w_qos;
+  __u8 w_cache_prot;
+} __attribute__((packed));
+enum pisp_be_transform {
+  PISP_BE_TRANSFORM_NONE = 0x0,
+  PISP_BE_TRANSFORM_HFLIP = 0x1,
+  PISP_BE_TRANSFORM_VFLIP = 0x2,
+  PISP_BE_TRANSFORM_ROT180 = (PISP_BE_TRANSFORM_HFLIP | PISP_BE_TRANSFORM_VFLIP)
+};
+struct pisp_be_output_format_config {
+  struct pisp_image_format_config image;
+  __u8 transform;
+  __u8 pad[3];
+  __u16 lo;
+  __u16 hi;
+  __u16 lo2;
+  __u16 hi2;
+} __attribute__((packed));
+struct pisp_be_output_buffer_config {
+  __u32 addr[3][2];
+} __attribute__((packed));
+struct pisp_be_hog_buffer_config {
+  __u32 addr[2];
+} __attribute__((packed));
+struct pisp_be_config {
+  struct pisp_be_input_buffer_config input_buffer;
+  struct pisp_be_tdn_input_buffer_config tdn_input_buffer;
+  struct pisp_be_stitch_input_buffer_config stitch_input_buffer;
+  struct pisp_be_tdn_output_buffer_config tdn_output_buffer;
+  struct pisp_be_stitch_output_buffer_config stitch_output_buffer;
+  struct pisp_be_output_buffer_config output_buffer[PISP_BACK_END_NUM_OUTPUTS];
+  struct pisp_be_hog_buffer_config hog_buffer;
+  struct pisp_be_global_config global;
+  struct pisp_image_format_config input_format;
+  struct pisp_decompress_config decompress;
+  struct pisp_be_dpc_config dpc;
+  struct pisp_be_geq_config geq;
+  struct pisp_image_format_config tdn_input_format;
+  struct pisp_decompress_config tdn_decompress;
+  struct pisp_be_tdn_config tdn;
+  struct pisp_compress_config tdn_compress;
+  struct pisp_image_format_config tdn_output_format;
+  struct pisp_be_sdn_config sdn;
+  struct pisp_bla_config blc;
+  struct pisp_compress_config stitch_compress;
+  struct pisp_image_format_config stitch_output_format;
+  struct pisp_image_format_config stitch_input_format;
+  struct pisp_decompress_config stitch_decompress;
+  struct pisp_be_stitch_config stitch;
+  struct pisp_be_lsc_config lsc;
+  struct pisp_wbg_config wbg;
+  struct pisp_be_cdn_config cdn;
+  struct pisp_be_cac_config cac;
+  struct pisp_be_debin_config debin;
+  struct pisp_be_tonemap_config tonemap;
+  struct pisp_be_demosaic_config demosaic;
+  struct pisp_be_ccm_config ccm;
+  struct pisp_be_sat_control_config sat_control;
+  struct pisp_be_ccm_config ycbcr;
+  struct pisp_be_sharpen_config sharpen;
+  struct pisp_be_false_colour_config false_colour;
+  struct pisp_be_sh_fc_combine_config sh_fc_combine;
+  struct pisp_be_ccm_config ycbcr_inverse;
+  struct pisp_be_gamma_config gamma;
+  struct pisp_be_ccm_config csc[PISP_BACK_END_NUM_OUTPUTS];
+  struct pisp_be_downscale_config downscale[PISP_BACK_END_NUM_OUTPUTS];
+  struct pisp_be_resample_config resample[PISP_BACK_END_NUM_OUTPUTS];
+  struct pisp_be_output_format_config output_format[PISP_BACK_END_NUM_OUTPUTS];
+  struct pisp_be_hog_config hog;
+  struct pisp_be_axi_config axi;
+  struct pisp_be_lsc_extra lsc_extra;
+  struct pisp_be_cac_extra cac_extra;
+  struct pisp_be_downscale_extra downscale_extra[PISP_BACK_END_NUM_OUTPUTS];
+  struct pisp_be_resample_extra resample_extra[PISP_BACK_END_NUM_OUTPUTS];
+  struct pisp_be_crop_config crop;
+  struct pisp_image_format_config hog_format;
+  __u32 dirty_flags_bayer;
+  __u32 dirty_flags_rgb;
+  __u32 dirty_flags_extra;
+} __attribute__((packed));
+enum pisp_tile_edge {
+  PISP_LEFT_EDGE = (1 << 0),
+  PISP_RIGHT_EDGE = (1 << 1),
+  PISP_TOP_EDGE = (1 << 2),
+  PISP_BOTTOM_EDGE = (1 << 3)
+};
+struct pisp_tile {
+  __u8 edge;
+  __u8 pad0[3];
+  __u32 input_addr_offset;
+  __u32 input_addr_offset2;
+  __u16 input_offset_x;
+  __u16 input_offset_y;
+  __u16 input_width;
+  __u16 input_height;
+  __u32 tdn_input_addr_offset;
+  __u32 tdn_output_addr_offset;
+  __u32 stitch_input_addr_offset;
+  __u32 stitch_output_addr_offset;
+  __u32 lsc_grid_offset_x;
+  __u32 lsc_grid_offset_y;
+  __u32 cac_grid_offset_x;
+  __u32 cac_grid_offset_y;
+  __u16 crop_x_start[PISP_BACK_END_NUM_OUTPUTS];
+  __u16 crop_x_end[PISP_BACK_END_NUM_OUTPUTS];
+  __u16 crop_y_start[PISP_BACK_END_NUM_OUTPUTS];
+  __u16 crop_y_end[PISP_BACK_END_NUM_OUTPUTS];
+  __u16 downscale_phase_x[3 * PISP_BACK_END_NUM_OUTPUTS];
+  __u16 downscale_phase_y[3 * PISP_BACK_END_NUM_OUTPUTS];
+  __u16 resample_in_width[PISP_BACK_END_NUM_OUTPUTS];
+  __u16 resample_in_height[PISP_BACK_END_NUM_OUTPUTS];
+  __u16 resample_phase_x[3 * PISP_BACK_END_NUM_OUTPUTS];
+  __u16 resample_phase_y[3 * PISP_BACK_END_NUM_OUTPUTS];
+  __u16 output_offset_x[PISP_BACK_END_NUM_OUTPUTS];
+  __u16 output_offset_y[PISP_BACK_END_NUM_OUTPUTS];
+  __u16 output_width[PISP_BACK_END_NUM_OUTPUTS];
+  __u16 output_height[PISP_BACK_END_NUM_OUTPUTS];
+  __u32 output_addr_offset[PISP_BACK_END_NUM_OUTPUTS];
+  __u32 output_addr_offset2[PISP_BACK_END_NUM_OUTPUTS];
+  __u32 output_hog_addr_offset;
+} __attribute__((packed));
+struct pisp_be_tiles_config {
+  struct pisp_be_config config;
+  struct pisp_tile tiles[PISP_BACK_END_NUM_TILES];
+  __u32 num_tiles;
+} __attribute__((packed));
+#endif
diff --git a/libc/kernel/uapi/linux/media/raspberrypi/pisp_common.h b/libc/kernel/uapi/linux/media/raspberrypi/pisp_common.h
new file mode 100644
index 0000000..0e0b23f
--- /dev/null
+++ b/libc/kernel/uapi/linux/media/raspberrypi/pisp_common.h
@@ -0,0 +1,119 @@
+/*
+ * This file is auto-generated. Modifications will be lost.
+ *
+ * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
+ * for more information.
+ */
+#ifndef _UAPI_PISP_COMMON_H_
+#define _UAPI_PISP_COMMON_H_
+#include <linux/types.h>
+struct pisp_image_format_config {
+  __u16 width;
+  __u16 height;
+  __u32 format;
+  __s32 stride;
+  __s32 stride2;
+} __attribute__((packed));
+enum pisp_bayer_order {
+  PISP_BAYER_ORDER_RGGB = 0,
+  PISP_BAYER_ORDER_GBRG = 1,
+  PISP_BAYER_ORDER_BGGR = 2,
+  PISP_BAYER_ORDER_GRBG = 3,
+  PISP_BAYER_ORDER_GREYSCALE = 128
+};
+enum pisp_image_format {
+  PISP_IMAGE_FORMAT_BPS_8 = 0x00000000,
+  PISP_IMAGE_FORMAT_BPS_10 = 0x00000001,
+  PISP_IMAGE_FORMAT_BPS_12 = 0x00000002,
+  PISP_IMAGE_FORMAT_BPS_16 = 0x00000003,
+  PISP_IMAGE_FORMAT_BPS_MASK = 0x00000003,
+  PISP_IMAGE_FORMAT_PLANARITY_INTERLEAVED = 0x00000000,
+  PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR = 0x00000010,
+  PISP_IMAGE_FORMAT_PLANARITY_PLANAR = 0x00000020,
+  PISP_IMAGE_FORMAT_PLANARITY_MASK = 0x00000030,
+  PISP_IMAGE_FORMAT_SAMPLING_444 = 0x00000000,
+  PISP_IMAGE_FORMAT_SAMPLING_422 = 0x00000100,
+  PISP_IMAGE_FORMAT_SAMPLING_420 = 0x00000200,
+  PISP_IMAGE_FORMAT_SAMPLING_MASK = 0x00000300,
+  PISP_IMAGE_FORMAT_ORDER_NORMAL = 0x00000000,
+  PISP_IMAGE_FORMAT_ORDER_SWAPPED = 0x00001000,
+  PISP_IMAGE_FORMAT_SHIFT_0 = 0x00000000,
+  PISP_IMAGE_FORMAT_SHIFT_1 = 0x00010000,
+  PISP_IMAGE_FORMAT_SHIFT_2 = 0x00020000,
+  PISP_IMAGE_FORMAT_SHIFT_3 = 0x00030000,
+  PISP_IMAGE_FORMAT_SHIFT_4 = 0x00040000,
+  PISP_IMAGE_FORMAT_SHIFT_5 = 0x00050000,
+  PISP_IMAGE_FORMAT_SHIFT_6 = 0x00060000,
+  PISP_IMAGE_FORMAT_SHIFT_7 = 0x00070000,
+  PISP_IMAGE_FORMAT_SHIFT_8 = 0x00080000,
+  PISP_IMAGE_FORMAT_SHIFT_MASK = 0x000f0000,
+  PISP_IMAGE_FORMAT_BPP_32 = 0x00100000,
+  PISP_IMAGE_FORMAT_UNCOMPRESSED = 0x00000000,
+  PISP_IMAGE_FORMAT_COMPRESSION_MODE_1 = 0x01000000,
+  PISP_IMAGE_FORMAT_COMPRESSION_MODE_2 = 0x02000000,
+  PISP_IMAGE_FORMAT_COMPRESSION_MODE_3 = 0x03000000,
+  PISP_IMAGE_FORMAT_COMPRESSION_MASK = 0x03000000,
+  PISP_IMAGE_FORMAT_HOG_SIGNED = 0x04000000,
+  PISP_IMAGE_FORMAT_HOG_UNSIGNED = 0x08000000,
+  PISP_IMAGE_FORMAT_INTEGRAL_IMAGE = 0x10000000,
+  PISP_IMAGE_FORMAT_WALLPAPER_ROLL = 0x20000000,
+  PISP_IMAGE_FORMAT_THREE_CHANNEL = 0x40000000,
+  PISP_IMAGE_FORMAT_SINGLE_16 = PISP_IMAGE_FORMAT_BPS_16,
+  PISP_IMAGE_FORMAT_THREE_16 = PISP_IMAGE_FORMAT_BPS_16 | PISP_IMAGE_FORMAT_THREE_CHANNEL
+};
+#define PISP_IMAGE_FORMAT_BPS_8(fmt) (((fmt) & PISP_IMAGE_FORMAT_BPS_MASK) == PISP_IMAGE_FORMAT_BPS_8)
+#define PISP_IMAGE_FORMAT_BPS_10(fmt) (((fmt) & PISP_IMAGE_FORMAT_BPS_MASK) == PISP_IMAGE_FORMAT_BPS_10)
+#define PISP_IMAGE_FORMAT_BPS_12(fmt) (((fmt) & PISP_IMAGE_FORMAT_BPS_MASK) == PISP_IMAGE_FORMAT_BPS_12)
+#define PISP_IMAGE_FORMAT_BPS_16(fmt) (((fmt) & PISP_IMAGE_FORMAT_BPS_MASK) == PISP_IMAGE_FORMAT_BPS_16)
+#define PISP_IMAGE_FORMAT_BPS(fmt) (((fmt) & PISP_IMAGE_FORMAT_BPS_MASK) ? 8 + (2 << (((fmt) & PISP_IMAGE_FORMAT_BPS_MASK) - 1)) : 8)
+#define PISP_IMAGE_FORMAT_SHIFT(fmt) (((fmt) & PISP_IMAGE_FORMAT_SHIFT_MASK) / PISP_IMAGE_FORMAT_SHIFT_1)
+#define PISP_IMAGE_FORMAT_THREE_CHANNEL(fmt) ((fmt) & PISP_IMAGE_FORMAT_THREE_CHANNEL)
+#define PISP_IMAGE_FORMAT_SINGLE_CHANNEL(fmt) (! ((fmt) & PISP_IMAGE_FORMAT_THREE_CHANNEL))
+#define PISP_IMAGE_FORMAT_COMPRESSED(fmt) (((fmt) & PISP_IMAGE_FORMAT_COMPRESSION_MASK) != PISP_IMAGE_FORMAT_UNCOMPRESSED)
+#define PISP_IMAGE_FORMAT_SAMPLING_444(fmt) (((fmt) & PISP_IMAGE_FORMAT_SAMPLING_MASK) == PISP_IMAGE_FORMAT_SAMPLING_444)
+#define PISP_IMAGE_FORMAT_SAMPLING_422(fmt) (((fmt) & PISP_IMAGE_FORMAT_SAMPLING_MASK) == PISP_IMAGE_FORMAT_SAMPLING_422)
+#define PISP_IMAGE_FORMAT_SAMPLING_420(fmt) (((fmt) & PISP_IMAGE_FORMAT_SAMPLING_MASK) == PISP_IMAGE_FORMAT_SAMPLING_420)
+#define PISP_IMAGE_FORMAT_ORDER_NORMAL(fmt) (! ((fmt) & PISP_IMAGE_FORMAT_ORDER_SWAPPED))
+#define PISP_IMAGE_FORMAT_ORDER_SWAPPED(fmt) ((fmt) & PISP_IMAGE_FORMAT_ORDER_SWAPPED)
+#define PISP_IMAGE_FORMAT_INTERLEAVED(fmt) (((fmt) & PISP_IMAGE_FORMAT_PLANARITY_MASK) == PISP_IMAGE_FORMAT_PLANARITY_INTERLEAVED)
+#define PISP_IMAGE_FORMAT_SEMIPLANAR(fmt) (((fmt) & PISP_IMAGE_FORMAT_PLANARITY_MASK) == PISP_IMAGE_FORMAT_PLANARITY_SEMI_PLANAR)
+#define PISP_IMAGE_FORMAT_PLANAR(fmt) (((fmt) & PISP_IMAGE_FORMAT_PLANARITY_MASK) == PISP_IMAGE_FORMAT_PLANARITY_PLANAR)
+#define PISP_IMAGE_FORMAT_WALLPAPER(fmt) ((fmt) & PISP_IMAGE_FORMAT_WALLPAPER_ROLL)
+#define PISP_IMAGE_FORMAT_BPP_32(fmt) ((fmt) & PISP_IMAGE_FORMAT_BPP_32)
+#define PISP_IMAGE_FORMAT_HOG(fmt) ((fmt) & (PISP_IMAGE_FORMAT_HOG_SIGNED | PISP_IMAGE_FORMAT_HOG_UNSIGNED))
+#define PISP_WALLPAPER_WIDTH 128
+struct pisp_bla_config {
+  __u16 black_level_r;
+  __u16 black_level_gr;
+  __u16 black_level_gb;
+  __u16 black_level_b;
+  __u16 output_black_level;
+  __u8 pad[2];
+} __attribute__((packed));
+struct pisp_wbg_config {
+  __u16 gain_r;
+  __u16 gain_g;
+  __u16 gain_b;
+  __u8 pad[2];
+} __attribute__((packed));
+struct pisp_compress_config {
+  __u16 offset;
+  __u8 pad;
+  __u8 mode;
+} __attribute__((packed));
+struct pisp_decompress_config {
+  __u16 offset;
+  __u8 pad;
+  __u8 mode;
+} __attribute__((packed));
+enum pisp_axi_flags {
+  PISP_AXI_FLAG_ALIGN = 128,
+  PISP_AXI_FLAG_PAD = 64,
+  PISP_AXI_FLAG_PANIC = 32,
+};
+struct pisp_axi_config {
+  __u8 maxlen_flags;
+  __u8 cache_prot;
+  __u16 qos;
+} __attribute__((packed));
+#endif
diff --git a/libc/kernel/uapi/linux/mman.h b/libc/kernel/uapi/linux/mman.h
index cf1e978..f50b51c 100644
--- a/libc/kernel/uapi/linux/mman.h
+++ b/libc/kernel/uapi/linux/mman.h
@@ -18,6 +18,7 @@
 #define MAP_SHARED 0x01
 #define MAP_PRIVATE 0x02
 #define MAP_SHARED_VALIDATE 0x03
+#define MAP_DROPPABLE 0x08
 #define MAP_HUGE_SHIFT HUGETLB_FLAG_ENCODE_SHIFT
 #define MAP_HUGE_MASK HUGETLB_FLAG_ENCODE_MASK
 #define MAP_HUGE_16KB HUGETLB_FLAG_ENCODE_16KB
diff --git a/libc/kernel/uapi/linux/mount.h b/libc/kernel/uapi/linux/mount.h
index 1166a7a..c4278b5 100644
--- a/libc/kernel/uapi/linux/mount.h
+++ b/libc/kernel/uapi/linux/mount.h
@@ -90,7 +90,7 @@
 #define MOUNT_ATTR_SIZE_VER0 32
 struct statmount {
   __u32 size;
-  __u32 __spare1;
+  __u32 mnt_opts;
   __u64 mask;
   __u32 sb_dev_major;
   __u32 sb_dev_minor;
@@ -108,7 +108,8 @@
   __u64 propagate_from;
   __u32 mnt_root;
   __u32 mnt_point;
-  __u64 __spare2[50];
+  __u64 mnt_ns_id;
+  __u64 __spare2[49];
   char str[];
 };
 struct mnt_id_req {
@@ -116,13 +117,18 @@
   __u32 spare;
   __u64 mnt_id;
   __u64 param;
+  __u64 mnt_ns_id;
 };
 #define MNT_ID_REQ_SIZE_VER0 24
+#define MNT_ID_REQ_SIZE_VER1 32
 #define STATMOUNT_SB_BASIC 0x00000001U
 #define STATMOUNT_MNT_BASIC 0x00000002U
 #define STATMOUNT_PROPAGATE_FROM 0x00000004U
 #define STATMOUNT_MNT_ROOT 0x00000008U
 #define STATMOUNT_MNT_POINT 0x00000010U
 #define STATMOUNT_FS_TYPE 0x00000020U
+#define STATMOUNT_MNT_NS_ID 0x00000040U
+#define STATMOUNT_MNT_OPTS 0x00000080U
 #define LSMT_ROOT 0xffffffffffffffff
+#define LISTMOUNT_REVERSE (1 << 0)
 #endif
diff --git a/libc/kernel/uapi/linux/netfilter/nf_tables.h b/libc/kernel/uapi/linux/netfilter/nf_tables.h
index 7922147..bfc6e25 100644
--- a/libc/kernel/uapi/linux/netfilter/nf_tables.h
+++ b/libc/kernel/uapi/linux/netfilter/nf_tables.h
@@ -695,7 +695,7 @@
   __NFTA_SECMARK_MAX,
 };
 #define NFTA_SECMARK_MAX (__NFTA_SECMARK_MAX - 1)
-#define NFT_SECMARK_CTX_MAXLEN 256
+#define NFT_SECMARK_CTX_MAXLEN 4096
 enum nft_reject_types {
   NFT_REJECT_ICMP_UNREACH,
   NFT_REJECT_TCP_RST,
diff --git a/libc/kernel/uapi/linux/nfs4.h b/libc/kernel/uapi/linux/nfs4.h
index 21f1103..6512901 100644
--- a/libc/kernel/uapi/linux/nfs4.h
+++ b/libc/kernel/uapi/linux/nfs4.h
@@ -34,6 +34,7 @@
 #define NFS4_OPEN_RESULT_CONFIRM 0x0002
 #define NFS4_OPEN_RESULT_LOCKTYPE_POSIX 0x0004
 #define NFS4_OPEN_RESULT_PRESERVE_UNLINKED 0x0008
+#define NFS4_OPEN_RESULT_NO_OPEN_STATEID 0x0010
 #define NFS4_OPEN_RESULT_MAY_NOTIFY_LOCK 0x0020
 #define NFS4_SHARE_ACCESS_MASK 0x000F
 #define NFS4_SHARE_ACCESS_READ 0x0001
@@ -52,6 +53,8 @@
 #define NFS4_SHARE_WHEN_MASK 0xF0000
 #define NFS4_SHARE_SIGNAL_DELEG_WHEN_RESRC_AVAIL 0x10000
 #define NFS4_SHARE_PUSH_DELEG_WHEN_UNCONTENDED 0x20000
+#define NFS4_SHARE_WANT_DELEG_TIMESTAMPS 0x100000
+#define NFS4_SHARE_WANT_OPEN_XOR_DELEGATION 0x200000
 #define NFS4_CDFC4_FORE 0x1
 #define NFS4_CDFC4_BACK 0x2
 #define NFS4_CDFC4_BOTH 0x3
diff --git a/libc/kernel/uapi/linux/nfsd_netlink.h b/libc/kernel/uapi/linux/nfsd_netlink.h
index 45cb504..bd3f02c 100644
--- a/libc/kernel/uapi/linux/nfsd_netlink.h
+++ b/libc/kernel/uapi/linux/nfsd_netlink.h
@@ -58,6 +58,12 @@
   NFSD_A_SERVER_SOCK_MAX = (__NFSD_A_SERVER_SOCK_MAX - 1)
 };
 enum {
+  NFSD_A_POOL_MODE_MODE = 1,
+  NFSD_A_POOL_MODE_NPOOLS,
+  __NFSD_A_POOL_MODE_MAX,
+  NFSD_A_POOL_MODE_MAX = (__NFSD_A_POOL_MODE_MAX - 1)
+};
+enum {
   NFSD_CMD_RPC_STATUS_GET = 1,
   NFSD_CMD_THREADS_SET,
   NFSD_CMD_THREADS_GET,
@@ -65,6 +71,8 @@
   NFSD_CMD_VERSION_GET,
   NFSD_CMD_LISTENER_SET,
   NFSD_CMD_LISTENER_GET,
+  NFSD_CMD_POOL_MODE_SET,
+  NFSD_CMD_POOL_MODE_GET,
   __NFSD_CMD_MAX,
   NFSD_CMD_MAX = (__NFSD_CMD_MAX - 1)
 };
diff --git a/libc/kernel/uapi/linux/nl80211.h b/libc/kernel/uapi/linux/nl80211.h
index 98180c2..1bad2f2 100644
--- a/libc/kernel/uapi/linux/nl80211.h
+++ b/libc/kernel/uapi/linux/nl80211.h
@@ -528,6 +528,8 @@
   NL80211_ATTR_MLO_TTLM_DLINK,
   NL80211_ATTR_MLO_TTLM_ULINK,
   NL80211_ATTR_ASSOC_SPP_AMSDU,
+  NL80211_ATTR_WIPHY_RADIOS,
+  NL80211_ATTR_WIPHY_INTERFACE_COMBINATIONS,
   __NL80211_ATTR_AFTER_LAST,
   NUM_NL80211_ATTR = __NL80211_ATTR_AFTER_LAST,
   NL80211_ATTR_MAX = __NL80211_ATTR_AFTER_LAST - 1
@@ -878,6 +880,7 @@
   NL80211_FREQUENCY_ATTR_NO_6GHZ_VLP_CLIENT,
   NL80211_FREQUENCY_ATTR_NO_6GHZ_AFC_CLIENT,
   NL80211_FREQUENCY_ATTR_CAN_MONITOR,
+  NL80211_FREQUENCY_ATTR_ALLOW_6GHZ_VLP_AP,
   __NL80211_FREQUENCY_ATTR_AFTER_LAST,
   NL80211_FREQUENCY_ATTR_MAX = __NL80211_FREQUENCY_ATTR_AFTER_LAST - 1
 };
@@ -955,6 +958,7 @@
   NL80211_RRF_DFS_CONCURRENT = 1 << 21,
   NL80211_RRF_NO_6GHZ_VLP_CLIENT = 1 << 22,
   NL80211_RRF_NO_6GHZ_AFC_CLIENT = 1 << 23,
+  NL80211_RRF_ALLOW_6GHZ_VLP_AP = 1 << 24,
 };
 #define NL80211_RRF_PASSIVE_SCAN NL80211_RRF_NO_IR
 #define NL80211_RRF_NO_IBSS NL80211_RRF_NO_IR
@@ -1973,4 +1977,19 @@
   NL80211_AP_SETTINGS_EXTERNAL_AUTH_SUPPORT = 1 << 0,
   NL80211_AP_SETTINGS_SA_QUERY_OFFLOAD_SUPPORT = 1 << 1,
 };
+enum nl80211_wiphy_radio_attrs {
+  __NL80211_WIPHY_RADIO_ATTR_INVALID,
+  NL80211_WIPHY_RADIO_ATTR_INDEX,
+  NL80211_WIPHY_RADIO_ATTR_FREQ_RANGE,
+  NL80211_WIPHY_RADIO_ATTR_INTERFACE_COMBINATION,
+  __NL80211_WIPHY_RADIO_ATTR_LAST,
+  NL80211_WIPHY_RADIO_ATTR_MAX = __NL80211_WIPHY_RADIO_ATTR_LAST - 1,
+};
+enum nl80211_wiphy_radio_freq_range {
+  __NL80211_WIPHY_RADIO_FREQ_ATTR_INVALID,
+  NL80211_WIPHY_RADIO_FREQ_ATTR_START,
+  NL80211_WIPHY_RADIO_FREQ_ATTR_END,
+  __NL80211_WIPHY_RADIO_FREQ_ATTR_LAST,
+  NL80211_WIPHY_RADIO_FREQ_ATTR_MAX = __NL80211_WIPHY_RADIO_FREQ_ATTR_LAST - 1,
+};
 #endif
diff --git a/libc/kernel/uapi/linux/nsfs.h b/libc/kernel/uapi/linux/nsfs.h
index 61a5797..c8f2208 100644
--- a/libc/kernel/uapi/linux/nsfs.h
+++ b/libc/kernel/uapi/linux/nsfs.h
@@ -7,9 +7,15 @@
 #ifndef __LINUX_NSFS_H
 #define __LINUX_NSFS_H
 #include <linux/ioctl.h>
+#include <linux/types.h>
 #define NSIO 0xb7
 #define NS_GET_USERNS _IO(NSIO, 0x1)
 #define NS_GET_PARENT _IO(NSIO, 0x2)
 #define NS_GET_NSTYPE _IO(NSIO, 0x3)
 #define NS_GET_OWNER_UID _IO(NSIO, 0x4)
+#define NS_GET_MNTNS_ID _IOR(NSIO, 0x5, __u64)
+#define NS_GET_PID_FROM_PIDNS _IOR(NSIO, 0x6, int)
+#define NS_GET_TGID_FROM_PIDNS _IOR(NSIO, 0x7, int)
+#define NS_GET_PID_IN_PIDNS _IOR(NSIO, 0x8, int)
+#define NS_GET_TGID_IN_PIDNS _IOR(NSIO, 0x9, int)
 #endif
diff --git a/libc/kernel/uapi/linux/openvswitch.h b/libc/kernel/uapi/linux/openvswitch.h
index d45f4fa..98c8037 100644
--- a/libc/kernel/uapi/linux/openvswitch.h
+++ b/libc/kernel/uapi/linux/openvswitch.h
@@ -429,6 +429,13 @@
   __OVS_CHECK_PKT_LEN_ATTR_MAX,
 };
 #define OVS_CHECK_PKT_LEN_ATTR_MAX (__OVS_CHECK_PKT_LEN_ATTR_MAX - 1)
+#define OVS_PSAMPLE_COOKIE_MAX_SIZE 16
+enum ovs_psample_attr {
+  OVS_PSAMPLE_ATTR_GROUP = 1,
+  OVS_PSAMPLE_ATTR_COOKIE,
+  __OVS_PSAMPLE_ATTR_MAX
+};
+#define OVS_PSAMPLE_ATTR_MAX (__OVS_PSAMPLE_ATTR_MAX - 1)
 enum ovs_action_attr {
   OVS_ACTION_ATTR_UNSPEC,
   OVS_ACTION_ATTR_OUTPUT,
@@ -455,6 +462,7 @@
   OVS_ACTION_ATTR_ADD_MPLS,
   OVS_ACTION_ATTR_DEC_TTL,
   OVS_ACTION_ATTR_DROP,
+  OVS_ACTION_ATTR_PSAMPLE,
   __OVS_ACTION_ATTR_MAX,
 };
 #define OVS_ACTION_ATTR_MAX (__OVS_ACTION_ATTR_MAX - 1)
diff --git a/libc/kernel/uapi/linux/perf_event.h b/libc/kernel/uapi/linux/perf_event.h
index 16a1a2e..ec9b856 100644
--- a/libc/kernel/uapi/linux/perf_event.h
+++ b/libc/kernel/uapi/linux/perf_event.h
@@ -460,6 +460,8 @@
 #define PERF_MEM_LVLNUM_L2 0x02
 #define PERF_MEM_LVLNUM_L3 0x03
 #define PERF_MEM_LVLNUM_L4 0x04
+#define PERF_MEM_LVLNUM_L2_MHB 0x05
+#define PERF_MEM_LVLNUM_MSC 0x06
 #define PERF_MEM_LVLNUM_UNC 0x08
 #define PERF_MEM_LVLNUM_CXL 0x09
 #define PERF_MEM_LVLNUM_IO 0x0a
diff --git a/libc/kernel/uapi/linux/pidfd.h b/libc/kernel/uapi/linux/pidfd.h
index 082b4a0..9068727 100644
--- a/libc/kernel/uapi/linux/pidfd.h
+++ b/libc/kernel/uapi/linux/pidfd.h
@@ -8,9 +8,21 @@
 #define _UAPI_LINUX_PIDFD_H
 #include <linux/types.h>
 #include <linux/fcntl.h>
+#include <linux/ioctl.h>
 #define PIDFD_NONBLOCK O_NONBLOCK
 #define PIDFD_THREAD O_EXCL
 #define PIDFD_SIGNAL_THREAD (1UL << 0)
 #define PIDFD_SIGNAL_THREAD_GROUP (1UL << 1)
 #define PIDFD_SIGNAL_PROCESS_GROUP (1UL << 2)
+#define PIDFS_IOCTL_MAGIC 0xFF
+#define PIDFD_GET_CGROUP_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 1)
+#define PIDFD_GET_IPC_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 2)
+#define PIDFD_GET_MNT_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 3)
+#define PIDFD_GET_NET_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 4)
+#define PIDFD_GET_PID_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 5)
+#define PIDFD_GET_PID_FOR_CHILDREN_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 6)
+#define PIDFD_GET_TIME_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 7)
+#define PIDFD_GET_TIME_FOR_CHILDREN_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 8)
+#define PIDFD_GET_USER_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 9)
+#define PIDFD_GET_UTS_NAMESPACE _IO(PIDFS_IOCTL_MAGIC, 10)
 #endif
diff --git a/libc/kernel/uapi/linux/pkt_cls.h b/libc/kernel/uapi/linux/pkt_cls.h
index 6b5143c..bdca553 100644
--- a/libc/kernel/uapi/linux/pkt_cls.h
+++ b/libc/kernel/uapi/linux/pkt_cls.h
@@ -419,6 +419,8 @@
   TCA_FLOWER_KEY_CFM,
   TCA_FLOWER_KEY_SPI,
   TCA_FLOWER_KEY_SPI_MASK,
+  TCA_FLOWER_KEY_ENC_FLAGS,
+  TCA_FLOWER_KEY_ENC_FLAGS_MASK,
   __TCA_FLOWER_MAX,
 };
 #define TCA_FLOWER_MAX (__TCA_FLOWER_MAX - 1)
@@ -497,7 +499,13 @@
 enum {
   TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT = (1 << 0),
   TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST = (1 << 1),
+  TCA_FLOWER_KEY_FLAGS_TUNNEL_CSUM = (1 << 2),
+  TCA_FLOWER_KEY_FLAGS_TUNNEL_DONT_FRAGMENT = (1 << 3),
+  TCA_FLOWER_KEY_FLAGS_TUNNEL_OAM = (1 << 4),
+  TCA_FLOWER_KEY_FLAGS_TUNNEL_CRIT_OPT = (1 << 5),
+  __TCA_FLOWER_KEY_FLAGS_MAX,
 };
+#define TCA_FLOWER_KEY_FLAGS_MAX (__TCA_FLOWER_KEY_FLAGS_MAX - 1)
 enum {
   TCA_FLOWER_KEY_CFM_OPT_UNSPEC,
   TCA_FLOWER_KEY_CFM_MD_LEVEL,
diff --git a/libc/kernel/uapi/linux/psample.h b/libc/kernel/uapi/linux/psample.h
index c82e76e..f9f979c 100644
--- a/libc/kernel/uapi/linux/psample.h
+++ b/libc/kernel/uapi/linux/psample.h
@@ -22,6 +22,8 @@
   PSAMPLE_ATTR_LATENCY,
   PSAMPLE_ATTR_TIMESTAMP,
   PSAMPLE_ATTR_PROTO,
+  PSAMPLE_ATTR_USER_COOKIE,
+  PSAMPLE_ATTR_SAMPLE_PROBABILITY,
   __PSAMPLE_ATTR_MAX
 };
 enum psample_command {
diff --git a/libc/kernel/uapi/linux/psp-sev.h b/libc/kernel/uapi/linux/psp-sev.h
index 82fcbf1..7274081 100644
--- a/libc/kernel/uapi/linux/psp-sev.h
+++ b/libc/kernel/uapi/linux/psp-sev.h
@@ -20,6 +20,7 @@
   SNP_PLATFORM_STATUS,
   SNP_COMMIT,
   SNP_SET_CONFIG,
+  SNP_VLEK_LOAD,
   SEV_MAX,
 };
 typedef enum {
@@ -28,6 +29,7 @@
   SEV_RET_INVALID_PLATFORM_STATE,
   SEV_RET_INVALID_GUEST_STATE,
   SEV_RET_INAVLID_CONFIG,
+  SEV_RET_INVALID_CONFIG = SEV_RET_INAVLID_CONFIG,
   SEV_RET_INVALID_LEN,
   SEV_RET_ALREADY_OWNED,
   SEV_RET_INVALID_CERTIFICATE,
@@ -113,6 +115,15 @@
   __u32 rsvd : 30;
   __u8 rsvd1[52];
 } __attribute__((__packed__));
+struct sev_user_data_snp_vlek_load {
+  __u32 len;
+  __u8 vlek_wrapped_version;
+  __u8 rsvd[3];
+  __u64 vlek_wrapped_address;
+} __attribute__((__packed__));
+struct sev_user_data_snp_wrapped_vlek_hashstick {
+  __u8 data[432];
+} __attribute__((__packed__));
 struct sev_issue_cmd {
   __u32 cmd;
   __u64 data;
diff --git a/libc/kernel/uapi/linux/random.h b/libc/kernel/uapi/linux/random.h
index d1fd998..64f62d9 100644
--- a/libc/kernel/uapi/linux/random.h
+++ b/libc/kernel/uapi/linux/random.h
@@ -24,4 +24,10 @@
 #define GRND_NONBLOCK 0x0001
 #define GRND_RANDOM 0x0002
 #define GRND_INSECURE 0x0004
+struct vgetrandom_opaque_params {
+  __u32 size_of_opaque_state;
+  __u32 mmap_prot;
+  __u32 mmap_flags;
+  __u32 reserved[13];
+};
 #endif
diff --git a/libc/kernel/uapi/linux/sev-guest.h b/libc/kernel/uapi/linux/sev-guest.h
index a822bed..a722641 100644
--- a/libc/kernel/uapi/linux/sev-guest.h
+++ b/libc/kernel/uapi/linux/sev-guest.h
@@ -51,6 +51,8 @@
 #define SNP_GUEST_FW_ERR_MASK GENMASK_ULL(31, 0)
 #define SNP_GUEST_VMM_ERR_SHIFT 32
 #define SNP_GUEST_VMM_ERR(x) (((u64) x) << SNP_GUEST_VMM_ERR_SHIFT)
+#define SNP_GUEST_FW_ERR(x) ((x) & SNP_GUEST_FW_ERR_MASK)
+#define SNP_GUEST_ERR(vmm_err,fw_err) (SNP_GUEST_VMM_ERR(vmm_err) | SNP_GUEST_FW_ERR(fw_err))
 #define SNP_GUEST_VMM_ERR_INVALID_LEN 1
 #define SNP_GUEST_VMM_ERR_BUSY 2
 #endif
diff --git a/libc/kernel/uapi/linux/stat.h b/libc/kernel/uapi/linux/stat.h
index ff98fd2..aae9ed4 100644
--- a/libc/kernel/uapi/linux/stat.h
+++ b/libc/kernel/uapi/linux/stat.h
@@ -69,7 +69,11 @@
   __u32 stx_dio_mem_align;
   __u32 stx_dio_offset_align;
   __u64 stx_subvol;
-  __u64 __spare3[11];
+  __u32 stx_atomic_write_unit_min;
+  __u32 stx_atomic_write_unit_max;
+  __u32 stx_atomic_write_segments_max;
+  __u32 __spare1[1];
+  __u64 __spare3[9];
 };
 #define STATX_TYPE 0x00000001U
 #define STATX_MODE 0x00000002U
@@ -88,6 +92,7 @@
 #define STATX_DIOALIGN 0x00002000U
 #define STATX_MNT_ID_UNIQUE 0x00004000U
 #define STATX_SUBVOL 0x00008000U
+#define STATX_WRITE_ATOMIC 0x00010000U
 #define STATX__RESERVED 0x80000000U
 #define STATX_ALL 0x00000fffU
 #define STATX_ATTR_COMPRESSED 0x00000004
@@ -99,4 +104,5 @@
 #define STATX_ATTR_MOUNT_ROOT 0x00002000
 #define STATX_ATTR_VERITY 0x00100000
 #define STATX_ATTR_DAX 0x00200000
+#define STATX_ATTR_WRITE_ATOMIC 0x00400000
 #endif
diff --git a/libc/kernel/uapi/linux/tcp_metrics.h b/libc/kernel/uapi/linux/tcp_metrics.h
index 931f50c..46ca141 100644
--- a/libc/kernel/uapi/linux/tcp_metrics.h
+++ b/libc/kernel/uapi/linux/tcp_metrics.h
@@ -4,8 +4,8 @@
  * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
  * for more information.
  */
-#ifndef _LINUX_TCP_METRICS_H
-#define _LINUX_TCP_METRICS_H
+#ifndef _UAPI_LINUX_TCP_METRICS_H
+#define _UAPI_LINUX_TCP_METRICS_H
 #include <linux/types.h>
 #define TCP_METRICS_GENL_NAME "tcp_metrics"
 #define TCP_METRICS_GENL_VERSION 0x1
@@ -21,6 +21,17 @@
 };
 #define TCP_METRIC_MAX (__TCP_METRIC_MAX - 1)
 enum {
+  TCP_METRICS_A_METRICS_RTT = 1,
+  TCP_METRICS_A_METRICS_RTTVAR,
+  TCP_METRICS_A_METRICS_SSTHRESH,
+  TCP_METRICS_A_METRICS_CWND,
+  TCP_METRICS_A_METRICS_REODERING,
+  TCP_METRICS_A_METRICS_RTT_US,
+  TCP_METRICS_A_METRICS_RTTVAR_US,
+  __TCP_METRICS_A_METRICS_MAX
+};
+#define TCP_METRICS_A_METRICS_MAX (__TCP_METRICS_A_METRICS_MAX - 1)
+enum {
   TCP_METRICS_ATTR_UNSPEC,
   TCP_METRICS_ATTR_ADDR_IPV4,
   TCP_METRICS_ATTR_ADDR_IPV6,
diff --git a/libc/kernel/uapi/linux/um_timetravel.h b/libc/kernel/uapi/linux/um_timetravel.h
index 8706017..49eeb28 100644
--- a/libc/kernel/uapi/linux/um_timetravel.h
+++ b/libc/kernel/uapi/linux/um_timetravel.h
@@ -12,6 +12,15 @@
   __u32 seq;
   __u64 time;
 };
+#define UM_TIMETRAVEL_MAX_FDS 2
+enum um_timetravel_shared_mem_fds {
+  UM_TIMETRAVEL_SHARED_MEMFD,
+  UM_TIMETRAVEL_SHARED_LOGFD,
+  UM_TIMETRAVEL_SHARED_MAX_FDS,
+};
+enum um_timetravel_start_ack {
+  UM_TIMETRAVEL_START_ACK_ID = 0xffff,
+};
 enum um_timetravel_ops {
   UM_TIMETRAVEL_ACK = 0,
   UM_TIMETRAVEL_START = 1,
@@ -22,5 +31,36 @@
   UM_TIMETRAVEL_RUN = 6,
   UM_TIMETRAVEL_FREE_UNTIL = 7,
   UM_TIMETRAVEL_GET_TOD = 8,
+  UM_TIMETRAVEL_BROADCAST = 9,
+};
+#define UM_TIMETRAVEL_SCHEDSHM_VERSION 2
+enum um_timetravel_schedshm_cap {
+  UM_TIMETRAVEL_SCHEDSHM_CAP_TIME_SHARE = 0x1,
+};
+enum um_timetravel_schedshm_flags {
+  UM_TIMETRAVEL_SCHEDSHM_FLAGS_REQ_RUN = 0x1,
+};
+union um_timetravel_schedshm_client {
+  struct {
+    __u32 capa;
+    __u32 flags;
+    __u64 req_time;
+    __u64 name;
+  };
+  char reserve[128];
+};
+struct um_timetravel_schedshm {
+  union {
+    struct {
+      __u32 version;
+      __u32 len;
+      __u64 free_until;
+      __u64 current_time;
+      __u16 running_id;
+      __u16 max_clients;
+    };
+    char hdr[4096];
+  };
+  union um_timetravel_schedshm_client clients[];
 };
 #endif
diff --git a/libc/kernel/uapi/linux/v4l2-controls.h b/libc/kernel/uapi/linux/v4l2-controls.h
index 23158dc..f6ef26c 100644
--- a/libc/kernel/uapi/linux/v4l2-controls.h
+++ b/libc/kernel/uapi/linux/v4l2-controls.h
@@ -714,6 +714,7 @@
   V4L2_MPEG_VIDEO_AV1_LEVEL_7_2 = 22,
   V4L2_MPEG_VIDEO_AV1_LEVEL_7_3 = 23
 };
+#define V4L2_CID_MPEG_VIDEO_AVERAGE_QP (V4L2_CID_CODEC_BASE + 657)
 #define V4L2_CID_CODEC_CX2341X_BASE (V4L2_CTRL_CLASS_CODEC | 0x1000)
 #define V4L2_CID_MPEG_CX2341X_VIDEO_SPATIAL_FILTER_MODE (V4L2_CID_CODEC_CX2341X_BASE + 0)
 enum v4l2_mpeg_cx2341x_video_spatial_filter_mode {
diff --git a/libc/kernel/uapi/linux/version.h b/libc/kernel/uapi/linux/version.h
index 7faa30f..0cc45cf 100644
--- a/libc/kernel/uapi/linux/version.h
+++ b/libc/kernel/uapi/linux/version.h
@@ -4,8 +4,8 @@
  * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
  * for more information.
  */
-#define LINUX_VERSION_CODE 395776
+#define LINUX_VERSION_CODE 396032
 #define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
 #define LINUX_VERSION_MAJOR 6
-#define LINUX_VERSION_PATCHLEVEL 10
+#define LINUX_VERSION_PATCHLEVEL 11
 #define LINUX_VERSION_SUBLEVEL 0
diff --git a/libc/kernel/uapi/linux/videodev2.h b/libc/kernel/uapi/linux/videodev2.h
index 3905949..fb69a8a 100644
--- a/libc/kernel/uapi/linux/videodev2.h
+++ b/libc/kernel/uapi/linux/videodev2.h
@@ -236,6 +236,8 @@
 #define V4L2_PIX_FMT_RGBA1010102 v4l2_fourcc('R', 'A', '3', '0')
 #define V4L2_PIX_FMT_ARGB2101010 v4l2_fourcc('A', 'R', '3', '0')
 #define V4L2_PIX_FMT_BGR48_12 v4l2_fourcc('B', '3', '1', '2')
+#define V4L2_PIX_FMT_BGR48 v4l2_fourcc('B', 'G', 'R', '6')
+#define V4L2_PIX_FMT_RGB48 v4l2_fourcc('R', 'G', 'B', '6')
 #define V4L2_PIX_FMT_ABGR64_12 v4l2_fourcc('B', '4', '1', '2')
 #define V4L2_PIX_FMT_GREY v4l2_fourcc('G', 'R', 'E', 'Y')
 #define V4L2_PIX_FMT_Y4 v4l2_fourcc('Y', '0', '4', ' ')
@@ -425,6 +427,16 @@
 #define V4L2_PIX_FMT_IPU3_SGBRG10 v4l2_fourcc('i', 'p', '3', 'g')
 #define V4L2_PIX_FMT_IPU3_SGRBG10 v4l2_fourcc('i', 'p', '3', 'G')
 #define V4L2_PIX_FMT_IPU3_SRGGB10 v4l2_fourcc('i', 'p', '3', 'r')
+#define V4L2_PIX_FMT_PISP_COMP1_RGGB v4l2_fourcc('P', 'C', '1', 'R')
+#define V4L2_PIX_FMT_PISP_COMP1_GRBG v4l2_fourcc('P', 'C', '1', 'G')
+#define V4L2_PIX_FMT_PISP_COMP1_GBRG v4l2_fourcc('P', 'C', '1', 'g')
+#define V4L2_PIX_FMT_PISP_COMP1_BGGR v4l2_fourcc('P', 'C', '1', 'B')
+#define V4L2_PIX_FMT_PISP_COMP1_MONO v4l2_fourcc('P', 'C', '1', 'M')
+#define V4L2_PIX_FMT_PISP_COMP2_RGGB v4l2_fourcc('P', 'C', '2', 'R')
+#define V4L2_PIX_FMT_PISP_COMP2_GRBG v4l2_fourcc('P', 'C', '2', 'G')
+#define V4L2_PIX_FMT_PISP_COMP2_GBRG v4l2_fourcc('P', 'C', '2', 'g')
+#define V4L2_PIX_FMT_PISP_COMP2_BGGR v4l2_fourcc('P', 'C', '2', 'B')
+#define V4L2_PIX_FMT_PISP_COMP2_MONO v4l2_fourcc('P', 'C', '2', 'M')
 #define V4L2_SDR_FMT_CU8 v4l2_fourcc('C', 'U', '0', '8')
 #define V4L2_SDR_FMT_CU16LE v4l2_fourcc('C', 'U', '1', '6')
 #define V4L2_SDR_FMT_CS8 v4l2_fourcc('C', 'S', '0', '8')
@@ -444,6 +456,7 @@
 #define V4L2_META_FMT_VIVID v4l2_fourcc('V', 'I', 'V', 'D')
 #define V4L2_META_FMT_RK_ISP1_PARAMS v4l2_fourcc('R', 'K', '1', 'P')
 #define V4L2_META_FMT_RK_ISP1_STAT_3A v4l2_fourcc('R', 'K', '1', 'S')
+#define V4L2_META_FMT_RPI_BE_CFG v4l2_fourcc('R', 'P', 'B', 'C')
 #define V4L2_PIX_FMT_PRIV_MAGIC 0xfeedcafe
 #define V4L2_PIX_FMT_FLAG_PREMUL_ALPHA 0x00000001
 #define V4L2_PIX_FMT_FLAG_SET_CSC 0x00000002
diff --git a/libc/kernel/uapi/linux/xfrm.h b/libc/kernel/uapi/linux/xfrm.h
index b8e79e8..9509efe 100644
--- a/libc/kernel/uapi/linux/xfrm.h
+++ b/libc/kernel/uapi/linux/xfrm.h
@@ -260,6 +260,7 @@
   XFRMA_IF_ID,
   XFRMA_MTIMER_THRESH,
   XFRMA_SA_DIR,
+  XFRMA_NAT_KEEPALIVE_INTERVAL,
   __XFRMA_MAX
 #define XFRMA_OUTPUT_MARK XFRMA_SET_MARK
 #define XFRMA_MAX (__XFRMA_MAX - 1)
diff --git a/libc/kernel/uapi/linux/zorro_ids.h b/libc/kernel/uapi/linux/zorro_ids.h
index f47c899..05239cd 100644
--- a/libc/kernel/uapi/linux/zorro_ids.h
+++ b/libc/kernel/uapi/linux/zorro_ids.h
@@ -354,6 +354,8 @@
 #define ZORRO_MANUF_VMC 0x1389
 #define ZORRO_PROD_VMC_ISDN_BLASTER_Z2 ZORRO_ID(VMC, 0x01, 0)
 #define ZORRO_PROD_VMC_HYPERCOM_4 ZORRO_ID(VMC, 0x02, 0)
+#define ZORRO_MANUF_CSLAB 0x1400
+#define ZORRO_PROD_CSLAB_WARP_1260 ZORRO_ID(CSLAB, 0x65, 0)
 #define ZORRO_MANUF_INFORMATION 0x157C
 #define ZORRO_PROD_INFORMATION_ISDN_ENGINE_I ZORRO_ID(INFORMATION, 0x64, 0)
 #define ZORRO_MANUF_VORTEX 0x2017
diff --git a/libc/kernel/uapi/misc/mrvl_cn10k_dpi.h b/libc/kernel/uapi/misc/mrvl_cn10k_dpi.h
new file mode 100644
index 0000000..7d8671f
--- /dev/null
+++ b/libc/kernel/uapi/misc/mrvl_cn10k_dpi.h
@@ -0,0 +1,26 @@
+/*
+ * This file is auto-generated. Modifications will be lost.
+ *
+ * See https://android.googlesource.com/platform/bionic/+/master/libc/kernel/
+ * for more information.
+ */
+#ifndef __MRVL_CN10K_DPI_H__
+#define __MRVL_CN10K_DPI_H__
+#include <linux/types.h>
+#define DPI_MAX_ENGINES 6
+struct dpi_mps_mrrs_cfg {
+  __u16 max_read_req_sz;
+  __u16 max_payload_sz;
+  __u16 port;
+  __u16 reserved;
+};
+struct dpi_engine_cfg {
+  __u64 fifo_mask;
+  __u16 molr[DPI_MAX_ENGINES];
+  __u16 update_molr;
+  __u16 reserved;
+};
+#define DPI_MAGIC_NUM 0xB8
+#define DPI_MPS_MRRS_CFG _IOW(DPI_MAGIC_NUM, 1, struct dpi_mps_mrrs_cfg)
+#define DPI_ENGINE_CFG _IOW(DPI_MAGIC_NUM, 2, struct dpi_engine_cfg)
+#endif
diff --git a/libc/kernel/uapi/rdma/bnxt_re-abi.h b/libc/kernel/uapi/rdma/bnxt_re-abi.h
index 3dceafd..50f8b8a 100644
--- a/libc/kernel/uapi/rdma/bnxt_re-abi.h
+++ b/libc/kernel/uapi/rdma/bnxt_re-abi.h
@@ -18,7 +18,7 @@
   BNXT_RE_UCNTX_CMASK_WC_DPI_ENABLED = 0x04ULL,
   BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED = 0x08ULL,
   BNXT_RE_UCNTX_CMASK_POW2_DISABLED = 0x10ULL,
-  BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40,
+  BNXT_RE_UCNTX_CMASK_MSN_TABLE_ENABLED = 0x40,
 };
 enum bnxt_re_wqe_mode {
   BNXT_QPLIB_WQE_MODE_STATIC = 0x00,
diff --git a/libc/kernel/uapi/rdma/ib_user_ioctl_cmds.h b/libc/kernel/uapi/rdma/ib_user_ioctl_cmds.h
index d774fba..5f5c426 100644
--- a/libc/kernel/uapi/rdma/ib_user_ioctl_cmds.h
+++ b/libc/kernel/uapi/rdma/ib_user_ioctl_cmds.h
@@ -8,8 +8,6 @@
 #define IB_USER_IOCTL_CMDS_H
 #define UVERBS_ID_NS_MASK 0xF000
 #define UVERBS_ID_NS_SHIFT 12
-#define UVERBS_UDATA_DRIVER_DATA_NS 1
-#define UVERBS_UDATA_DRIVER_DATA_FLAG (1UL << UVERBS_ID_NS_SHIFT)
 enum uverbs_default_objects {
   UVERBS_OBJECT_DEVICE,
   UVERBS_OBJECT_PD,
@@ -30,8 +28,10 @@
   UVERBS_OBJECT_ASYNC_EVENT,
 };
 enum {
-  UVERBS_ATTR_UHW_IN = UVERBS_UDATA_DRIVER_DATA_FLAG,
+  UVERBS_ID_DRIVER_NS = 1UL << UVERBS_ID_NS_SHIFT,
+  UVERBS_ATTR_UHW_IN = UVERBS_ID_DRIVER_NS,
   UVERBS_ATTR_UHW_OUT,
+  UVERBS_ID_DRIVER_NS_WITH_UHW,
 };
 enum uverbs_methods_device {
   UVERBS_METHOD_INVOKE_WRITE,
diff --git a/libc/kernel/uapi/rdma/mana-abi.h b/libc/kernel/uapi/rdma/mana-abi.h
index 7347175..02bb061 100644
--- a/libc/kernel/uapi/rdma/mana-abi.h
+++ b/libc/kernel/uapi/rdma/mana-abi.h
@@ -33,6 +33,13 @@
   __u32 tx_vp_offset;
   __u32 reserved;
 };
+struct mana_ib_create_rc_qp {
+  __aligned_u64 queue_buf[4];
+  __u32 queue_size[4];
+};
+struct mana_ib_create_rc_qp_resp {
+  __u32 queue_id[4];
+};
 struct mana_ib_create_wq {
   __aligned_u64 wq_buf_addr;
   __u32 wq_buf_size;
diff --git a/libc/kernel/uapi/rdma/mlx5_user_ioctl_cmds.h b/libc/kernel/uapi/rdma/mlx5_user_ioctl_cmds.h
index 69d5250..ebafb00 100644
--- a/libc/kernel/uapi/rdma/mlx5_user_ioctl_cmds.h
+++ b/libc/kernel/uapi/rdma/mlx5_user_ioctl_cmds.h
@@ -201,6 +201,9 @@
 enum mlx5_ib_device_query_context_attrs {
   MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX = (1U << UVERBS_ID_NS_SHIFT),
 };
+enum mlx5_ib_create_cq_attrs {
+  MLX5_IB_ATTR_CREATE_CQ_UAR_INDEX = UVERBS_ID_DRIVER_NS_WITH_UHW,
+};
 #define MLX5_IB_DW_MATCH_PARAM 0xA0
 struct mlx5_ib_match_params {
   __u32 match_params[MLX5_IB_DW_MATCH_PARAM];
diff --git a/libc/kernel/uapi/rdma/rdma_netlink.h b/libc/kernel/uapi/rdma/rdma_netlink.h
index 912a3c0..ac027ac 100644
--- a/libc/kernel/uapi/rdma/rdma_netlink.h
+++ b/libc/kernel/uapi/rdma/rdma_netlink.h
@@ -199,6 +199,8 @@
   RDMA_NLDEV_CMD_RES_SRQ_GET,
   RDMA_NLDEV_CMD_STAT_GET_STATUS,
   RDMA_NLDEV_CMD_RES_SRQ_GET_RAW,
+  RDMA_NLDEV_CMD_NEWDEV,
+  RDMA_NLDEV_CMD_DELDEV,
   RDMA_NLDEV_NUM_OPS
 };
 enum rdma_nldev_print_type {
@@ -306,6 +308,9 @@
   RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE,
   RDMA_NLDEV_ATTR_DRIVER_DETAILS,
   RDMA_NLDEV_ATTR_RES_SUBTYPE,
+  RDMA_NLDEV_ATTR_DEV_TYPE,
+  RDMA_NLDEV_ATTR_PARENT_NAME,
+  RDMA_NLDEV_ATTR_NAME_ASSIGN_TYPE,
   RDMA_NLDEV_ATTR_MAX
 };
 enum rdma_nl_counter_mode {
@@ -318,4 +323,11 @@
   RDMA_COUNTER_MASK_QP_TYPE = 1,
   RDMA_COUNTER_MASK_PID = 1 << 1,
 };
+enum rdma_nl_dev_type {
+  RDMA_DEVICE_TYPE_SMI = 1,
+};
+enum rdma_nl_name_assign_type {
+  RDMA_NAME_ASSIGN_TYPE_UNKNOWN = 0,
+  RDMA_NAME_ASSIGN_TYPE_USER = 1,
+};
 #endif
diff --git a/libc/kernel/uapi/scsi/scsi_bsg_mpi3mr.h b/libc/kernel/uapi/scsi/scsi_bsg_mpi3mr.h
index d98e8fa..fe3094e 100644
--- a/libc/kernel/uapi/scsi/scsi_bsg_mpi3mr.h
+++ b/libc/kernel/uapi/scsi/scsi_bsg_mpi3mr.h
@@ -144,7 +144,7 @@
 };
 struct mpi3mr_bsg_in_hdb_status {
   __u8 num_hdb_types;
-  __u8 rsvd1;
+  __u8 element_trigger_format;
   __u16 rsvd2;
   __u32 rsvd3;
   struct mpi3mr_hdb_entry entry[1];
diff --git a/libc/kernel/uapi/sound/asequencer.h b/libc/kernel/uapi/sound/asequencer.h
index 7918528..a3826a5 100644
--- a/libc/kernel/uapi/sound/asequencer.h
+++ b/libc/kernel/uapi/sound/asequencer.h
@@ -7,7 +7,7 @@
 #ifndef _UAPI__SOUND_ASEQUENCER_H
 #define _UAPI__SOUND_ASEQUENCER_H
 #include <sound/asound.h>
-#define SNDRV_SEQ_VERSION SNDRV_PROTOCOL_VERSION(1, 0, 3)
+#define SNDRV_SEQ_VERSION SNDRV_PROTOCOL_VERSION(1, 0, 4)
 #define SNDRV_SEQ_EVENT_SYSTEM 0
 #define SNDRV_SEQ_EVENT_RESULT 1
 #define SNDRV_SEQ_EVENT_NOTE 5
@@ -343,7 +343,8 @@
   int ppq;
   unsigned int skew_value;
   unsigned int skew_base;
-  char reserved[24];
+  unsigned short tempo_base;
+  char reserved[22];
 };
 #define SNDRV_SEQ_TIMER_ALSA 0
 #define SNDRV_SEQ_TIMER_MIDI_CLOCK 1
diff --git a/libc/kernel/uapi/sound/asound.h b/libc/kernel/uapi/sound/asound.h
index b608ed5..cfe9f66 100644
--- a/libc/kernel/uapi/sound/asound.h
+++ b/libc/kernel/uapi/sound/asound.h
@@ -92,7 +92,7 @@
 #define SNDRV_HWDEP_IOCTL_INFO _IOR('H', 0x01, struct snd_hwdep_info)
 #define SNDRV_HWDEP_IOCTL_DSP_STATUS _IOR('H', 0x02, struct snd_hwdep_dsp_status)
 #define SNDRV_HWDEP_IOCTL_DSP_LOAD _IOW('H', 0x03, struct snd_hwdep_dsp_image)
-#define SNDRV_PCM_VERSION SNDRV_PROTOCOL_VERSION(2, 0, 17)
+#define SNDRV_PCM_VERSION SNDRV_PROTOCOL_VERSION(2, 0, 18)
 typedef unsigned long snd_pcm_uframes_t;
 typedef signed long snd_pcm_sframes_t;
 enum {
@@ -263,7 +263,7 @@
   unsigned char id[16];
   unsigned short id16[8];
   unsigned int id32[4];
-};
+} __attribute__((deprecated));
 struct snd_pcm_info {
   unsigned int device;
   unsigned int subdevice;
@@ -276,7 +276,7 @@
   int dev_subclass;
   unsigned int subdevices_count;
   unsigned int subdevices_avail;
-  union snd_pcm_sync_id sync;
+  unsigned char pad1[16];
   unsigned char reserved[64];
 };
 typedef int snd_pcm_hw_param_t;
@@ -324,7 +324,8 @@
   unsigned int rate_num;
   unsigned int rate_den;
   snd_pcm_uframes_t fifo_size;
-  unsigned char reserved[64];
+  unsigned char sync[16];
+  unsigned char reserved[48];
 };
 enum {
   SNDRV_PCM_TSTAMP_NONE = 0,
diff --git a/libc/kernel/uapi/sound/sof/abi.h b/libc/kernel/uapi/sound/sof/abi.h
index fe1fe47..ed66131 100644
--- a/libc/kernel/uapi/sound/sof/abi.h
+++ b/libc/kernel/uapi/sound/sof/abi.h
@@ -9,7 +9,7 @@
 #include <linux/types.h>
 #define SOF_ABI_MAJOR 3
 #define SOF_ABI_MINOR 23
-#define SOF_ABI_PATCH 0
+#define SOF_ABI_PATCH 1
 #define SOF_ABI_MAJOR_SHIFT 24
 #define SOF_ABI_MAJOR_MASK 0xff
 #define SOF_ABI_MINOR_SHIFT 12
diff --git a/linker/linker_main.cpp b/linker/linker_main.cpp
index 6ccd75b..74cd517 100644
--- a/linker/linker_main.cpp
+++ b/linker/linker_main.cpp
@@ -722,6 +722,8 @@
   tmp_linker_so.set_linker_flag();
 
   if (!tmp_linker_so.prelink_image()) __linker_cannot_link(args.argv[0]);
+  // There is special logic in soinfo::relocate to avoid duplicating the
+  // relocations we did in relocate_linker().
   if (!tmp_linker_so.link_image(SymbolLookupList(&tmp_linker_so), &tmp_linker_so, nullptr, nullptr)) __linker_cannot_link(args.argv[0]);
 
   return __linker_init_post_relocation(args, tmp_linker_so);
diff --git a/linker/linker_phdr.cpp b/linker/linker_phdr.cpp
index 48206be..b7db4cd 100644
--- a/linker/linker_phdr.cpp
+++ b/linker/linker_phdr.cpp
@@ -976,7 +976,6 @@
     ElfW(Addr) seg_start = phdr->p_vaddr + load_bias_;
     ElfW(Addr) seg_end = seg_start + p_memsz;
 
-    ElfW(Addr) seg_page_start = page_start(seg_start);
     ElfW(Addr) seg_page_end = page_end(seg_end);
 
     ElfW(Addr) seg_file_end = seg_start + p_filesz;
diff --git a/tests/Android.bp b/tests/Android.bp
index 6aecb40..0bd4a32 100644
--- a/tests/Android.bp
+++ b/tests/Android.bp
@@ -746,6 +746,40 @@
     },
 }
 
+cc_defaults {
+    name: "bionic_fortify_c_tests_defaults",
+    defaults: [
+        "bionic_clang_fortify_tests_w_flags",
+        "bionic_tests_defaults",
+    ],
+    cflags: [
+        "-U_FORTIFY_SOURCE",
+        // -fbuiltin is required here to counteract -fno-builtin from
+        // `bionic_tests_defaults`. With `-fno-builtin`, Clang won't
+        // const-evaluate calls to `strlen`, which is tested for here.
+        "-fbuiltin",
+    ],
+    srcs: [
+        "clang_fortify_c_only_tests.c",
+    ],
+    tidy: false,
+    shared: {
+        enabled: false,
+    },
+}
+
+cc_test_library {
+    name: "libfortify1-c-tests-clang",
+    defaults: ["bionic_fortify_c_tests_defaults"],
+    cflags: ["-D_FORTIFY_SOURCE=1"],
+}
+
+cc_test_library {
+    name: "libfortify2-c-tests-clang",
+    defaults: ["bionic_fortify_c_tests_defaults"],
+    cflags: ["-D_FORTIFY_SOURCE=2"],
+}
+
 // -----------------------------------------------------------------------------
 // Library of all tests (excluding the dynamic linker tests).
 // -----------------------------------------------------------------------------
@@ -757,8 +791,10 @@
         "libBionicStandardTests",
         "libBionicElfTlsTests",
         "libBionicFramePointerTests",
+        "libfortify1-c-tests-clang",
         "libfortify1-tests-clang",
         "libfortify1-new-tests-clang",
+        "libfortify2-c-tests-clang",
         "libfortify2-tests-clang",
         "libfortify2-new-tests-clang",
     ],
diff --git a/tests/DoNotOptimize.h b/tests/DoNotOptimize.h
new file mode 100644
index 0000000..711d339
--- /dev/null
+++ b/tests/DoNotOptimize.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+// From <benchmark/benchmark.h>.
+template <class Tp>
+static inline void DoNotOptimize(Tp const& value) {
+  asm volatile("" : : "r,m"(value) : "memory");
+}
+template <class Tp>
+static inline void DoNotOptimize(Tp& value) {
+  asm volatile("" : "+r,m"(value) : : "memory");
+}
diff --git a/tests/clang_fortify_c_only_tests.c b/tests/clang_fortify_c_only_tests.c
new file mode 100644
index 0000000..3bec848
--- /dev/null
+++ b/tests/clang_fortify_c_only_tests.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <string.h>
+
+// This is a test specifically of bionic's FORTIFY machinery. Other stdlibs need not apply.
+#ifdef __BIONIC__
+
+// Ensure that strlen can be evaluated at compile-time. Clang doesn't support
+// this in C++, but does in C.
+_Static_assert(strlen("foo") == 3, "");
+
+#endif  // __BIONIC__
diff --git a/tests/clang_fortify_tests.cpp b/tests/clang_fortify_tests.cpp
index f08fd1f..da7926d 100644
--- a/tests/clang_fortify_tests.cpp
+++ b/tests/clang_fortify_tests.cpp
@@ -89,6 +89,10 @@
 #include <unistd.h>
 #include <wchar.h>
 
+#include <array>
+
+#include "DoNotOptimize.h"
+
 #ifndef COMPILATION_TESTS
 #include <android-base/silent_death_test.h>
 #include <gtest/gtest.h>
@@ -133,6 +137,24 @@
 
 const static int kBogusFD = -1;
 
+FORTIFY_TEST(strlen) {
+  auto run_strlen_with_contents = [&](std::array<char, 3> contents) {
+    // A lot of cruft is necessary to make this test DTRT. LLVM and Clang love to fold/optimize
+    // strlen calls, and that's the opposite of what we want to happen.
+
+    // Loop to convince LLVM that `contents` can never be known (since `xor volatile_value` can flip
+    // any bit in each elem of `contents`).
+    volatile char always_zero = 0;
+    for (char& c : contents) {
+      c ^= always_zero;
+    }
+    DoNotOptimize(strlen(&contents.front()));
+  };
+
+  EXPECT_NO_DEATH(run_strlen_with_contents({'f', 'o', '\0'}));
+  EXPECT_FORTIFY_DEATH(run_strlen_with_contents({'f', 'o', 'o'}));
+}
+
 FORTIFY_TEST(string) {
   char small_buffer[8] = {};
 
diff --git a/tests/cpu_target_features_test.cpp b/tests/cpu_target_features_test.cpp
index 3458bca..d773772 100644
--- a/tests/cpu_target_features_test.cpp
+++ b/tests/cpu_target_features_test.cpp
@@ -54,15 +54,3 @@
   GTEST_SKIP() << "Not targeting an aarch64 architecture.";
 #endif
 }
-
-TEST(cpu_target_features, has_expected_arm_compiler_values) {
-#if defined(__arm__)
-  ExecTestHelper eth;
-  char* const argv[] = {nullptr};
-  const auto invocation = [&] { execvp("cpu-target-features", argv); };
-  eth.Run(invocation, 0, "(^|\n)__ARM_FEATURE_AES=1($|\n)");
-  eth.Run(invocation, 0, "(^|\n)__ARM_FEATURE_CRC32=1($|\n)");
-#else
-  GTEST_SKIP() << "Not targeting an arm architecture.";
-#endif
-}
diff --git a/tests/fenv_test.cpp b/tests/fenv_test.cpp
index 9cf9d98..bbf339f 100644
--- a/tests/fenv_test.cpp
+++ b/tests/fenv_test.cpp
@@ -16,6 +16,7 @@
 
 #include <gtest/gtest.h>
 
+#include "DoNotOptimize.h"
 #include "utils.h"
 
 #include <fenv.h>
diff --git a/tests/malloc_test.cpp b/tests/malloc_test.cpp
index 813f348..3f1ba79 100644
--- a/tests/malloc_test.cpp
+++ b/tests/malloc_test.cpp
@@ -47,6 +47,7 @@
 #include <android-base/file.h>
 #include <android-base/test_utils.h>
 
+#include "DoNotOptimize.h"
 #include "utils.h"
 
 #if defined(__BIONIC__)
diff --git a/tests/prebuilt-elf-files/arm64/libtest_invalid-empty_shdr_table.so b/tests/prebuilt-elf-files/arm64/libtest_invalid-empty_shdr_table.so
index c8b5430..21a8f26 100755
--- a/tests/prebuilt-elf-files/arm64/libtest_invalid-empty_shdr_table.so
+++ b/tests/prebuilt-elf-files/arm64/libtest_invalid-empty_shdr_table.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/arm64/libtest_invalid-local-tls.so b/tests/prebuilt-elf-files/arm64/libtest_invalid-local-tls.so
index 20c5765..c902bbe 100755
--- a/tests/prebuilt-elf-files/arm64/libtest_invalid-local-tls.so
+++ b/tests/prebuilt-elf-files/arm64/libtest_invalid-local-tls.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/arm64/libtest_invalid-unaligned_shdr_offset.so b/tests/prebuilt-elf-files/arm64/libtest_invalid-unaligned_shdr_offset.so
index 6e5a6e3..fb86bca 100755
--- a/tests/prebuilt-elf-files/arm64/libtest_invalid-unaligned_shdr_offset.so
+++ b/tests/prebuilt-elf-files/arm64/libtest_invalid-unaligned_shdr_offset.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/gen-libtest_invalid-local-tls.sh b/tests/prebuilt-elf-files/gen-libtest_invalid-local-tls.sh
index 0f3e736..98a2b00 100755
--- a/tests/prebuilt-elf-files/gen-libtest_invalid-local-tls.sh
+++ b/tests/prebuilt-elf-files/gen-libtest_invalid-local-tls.sh
@@ -19,12 +19,18 @@
 build() {
   arch=$1
   target=$2
+
+  if [[ "$arch" == "arm64" || "$arch" == "x86_64" ]]; then
+    alignment="-Wl,-z,max-page-size=16384"
+  fi
+
   $NDK21E/toolchains/llvm/prebuilt/linux-x86_64/bin/clang -O2 --target=$target \
       -fpic -shared -o $arch/libtest_invalid-local-tls.so -fno-emulated-tls \
-      -fuse-ld=gold test.c test2.c
+      $alignment -fuse-ld=gold test.c test2.c
 }
 
 build arm armv7a-linux-androideabi29
 build arm64 aarch64-linux-android29
 build x86 i686-linux-android29
 build x86_64 x86_64-linux-android29
+
diff --git a/tests/prebuilt-elf-files/x86_64/libtest_invalid-empty_shdr_table.so b/tests/prebuilt-elf-files/x86_64/libtest_invalid-empty_shdr_table.so
index af1538d..00fefd4 100755
--- a/tests/prebuilt-elf-files/x86_64/libtest_invalid-empty_shdr_table.so
+++ b/tests/prebuilt-elf-files/x86_64/libtest_invalid-empty_shdr_table.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/x86_64/libtest_invalid-local-tls.so b/tests/prebuilt-elf-files/x86_64/libtest_invalid-local-tls.so
index 5b689ba..31d2b37 100755
--- a/tests/prebuilt-elf-files/x86_64/libtest_invalid-local-tls.so
+++ b/tests/prebuilt-elf-files/x86_64/libtest_invalid-local-tls.so
Binary files differ
diff --git a/tests/prebuilt-elf-files/x86_64/libtest_invalid-unaligned_shdr_offset.so b/tests/prebuilt-elf-files/x86_64/libtest_invalid-unaligned_shdr_offset.so
index 87631af..f9c310f 100755
--- a/tests/prebuilt-elf-files/x86_64/libtest_invalid-unaligned_shdr_offset.so
+++ b/tests/prebuilt-elf-files/x86_64/libtest_invalid-unaligned_shdr_offset.so
Binary files differ
diff --git a/tests/string_test.cpp b/tests/string_test.cpp
index 6e1fcfc..289a483 100644
--- a/tests/string_test.cpp
+++ b/tests/string_test.cpp
@@ -748,7 +748,7 @@
       expected_end = copy_len;
     }
 
-    ASSERT_EQ(state.ptr2 + expected_end, stpncpy(state.ptr2, state.ptr1, copy_len));
+    ASSERT_EQ(state.ptr2, strncpy(state.ptr2, state.ptr1, copy_len));
 
     // Verify ptr1 was not modified.
     ASSERT_EQ(0, memcmp(state.ptr1, state.ptr, state.MAX_LEN));
diff --git a/tests/unistd_test.cpp b/tests/unistd_test.cpp
index 78b55c1..9ad3b6d 100644
--- a/tests/unistd_test.cpp
+++ b/tests/unistd_test.cpp
@@ -16,6 +16,7 @@
 
 #include <gtest/gtest.h>
 
+#include "DoNotOptimize.h"
 #include "SignalUtils.h"
 #include "utils.h"
 
diff --git a/tests/utils.h b/tests/utils.h
index 3c83b73..4740e59 100644
--- a/tests/utils.h
+++ b/tests/utils.h
@@ -295,16 +295,6 @@
   size_t start_count_ = CountOpenFds();
 };
 
-// From <benchmark/benchmark.h>.
-template <class Tp>
-static inline void DoNotOptimize(Tp const& value) {
-  asm volatile("" : : "r,m"(value) : "memory");
-}
-template <class Tp>
-static inline void DoNotOptimize(Tp& value) {
-  asm volatile("" : "+r,m"(value) : : "memory");
-}
-
 static inline bool running_with_mte() {
 #ifdef __aarch64__
   int level = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);