libc: ARM64: update memset/strlen/memcpy/memmove to newlib/cortex-strings
* Bionic benchmarks results at the bottom
* This is a squash of the following commits:
libc: ARM64: optimize memset.
This is an optimized memset for AArch64. Memset is split into 4 main
cases: small sets of up to 16 bytes, medium of 16..96 bytes which are
fully unrolled. Large memsets of more than 96 bytes align the
destination and use an unrolled loop processing 64 bytes per
iteration. Memsets of zero of more than 256 use the dc zva
instruction, and there are faster versions for the common ZVA sizes 64
or 128. STP of Q registers is used to reduce codesize without loss of
performance.
Change-Id: I0c5b5ec5ab8a1fd0f23eee8fbacada0be08e841f
libc: ARM64: improve performance in strlen
Change-Id: Ic20f93a0052a49bd76cd6795f51e8606ccfbf11c
libc: ARM64: Optimize memcpy.
This is an optimized memcpy for AArch64. Copies are split into 3 main
cases: small copies of up to 16 bytes, medium copies of 17..96 bytes
which are fully unrolled. Large copies of more than 96 bytes align
the destination and use an unrolled loop processing 64 bytes per
iteration. In order to share code with memmove, small and medium
copies read all data before writing, allowing any kind of overlap. On
a random copy test memcpy is 40.8% faster on A57 and 28.4% on A53.
Change-Id: Ibb9483e45bbc0e8ca3d5ce98a31c55dfd8a5ac28
libc: AArch64: Tune memcpy
* Further tuning for performance.
Change-Id: Id08eaab885f9743fa7575077924a947c1b88e4ff
libc: ARM64: optimize memmove for Cortex-A53
* Sadly does not work on Denver or Kryo, so can't go to generic
This is an optimized memmove for AArch64. All copies of up to 96
bytes and all backward copies are done by the new memcpy. The only
remaining case is large forward copies which are done in the same way
as the memcpy loop, but copying from the end rather than the start.
Tested on the Nextbit Robin with MSM8992 (Snapdragon 808):
Before
BM_string_memcmp/8 1000k 27 0.286 GiB/s
BM_string_memcmp/64 50M 20 3.053 GiB/s
BM_string_memcmp/512 20M 126 4.060 GiB/s
BM_string_memcmp/1024 10M 234 4.372 GiB/s
BM_string_memcmp/8Ki 1000k 1726 4.745 GiB/s
BM_string_memcmp/16Ki 500k 3711 4.415 GiB/s
BM_string_memcmp/32Ki 200k 8276 3.959 GiB/s
BM_string_memcmp/64Ki 100k 16351 4.008 GiB/s
BM_string_memcpy/8 1000k 13 0.612 GiB/s
BM_string_memcpy/64 1000k 8 7.187 GiB/s
BM_string_memcpy/512 50M 38 13.311 GiB/s
BM_string_memcpy/1024 20M 86 11.858 GiB/s
BM_string_memcpy/8Ki 5M 620 13.203 GiB/s
BM_string_memcpy/16Ki 1000k 1265 12.950 GiB/s
BM_string_memcpy/32Ki 500k 2977 11.004 GiB/s
BM_string_memcpy/64Ki 500k 8003 8.188 GiB/s
BM_string_memmove/8 1000k 11 0.684 GiB/s
BM_string_memmove/64 1000k 16 3.855 GiB/s
BM_string_memmove/512 50M 57 8.915 GiB/s
BM_string_memmove/1024 20M 117 8.720 GiB/s
BM_string_memmove/8Ki 2M 853 9.594 GiB/s
BM_string_memmove/16Ki 1000k 1731 9.462 GiB/s
BM_string_memmove/32Ki 500k 3566 9.189 GiB/s
BM_string_memmove/64Ki 500k 7708 8.501 GiB/s
BM_string_memset/8 1000k 16 0.487 GiB/s
BM_string_memset/64 1000k 16 3.995 GiB/s
BM_string_memset/512 50M 37 13.489 GiB/s
BM_string_memset/1024 50M 58 17.405 GiB/s
BM_string_memset/8Ki 5M 451 18.160 GiB/s
BM_string_memset/16Ki 2M 883 18.554 GiB/s
BM_string_memset/32Ki 1000k 2181 15.022 GiB/s
BM_string_memset/64Ki 500k 4563 14.362 GiB/s
BM_string_strlen/8 1000k 8 0.965 GiB/s
BM_string_strlen/64 1000k 16 3.855 GiB/s
BM_string_strlen/512 20M 92 5.540 GiB/s
BM_string_strlen/1024 10M 167 6.111 GiB/s
BM_string_strlen/8Ki 1000k 1237 6.620 GiB/s
BM_string_strlen/16Ki 1000k 2765 5.923 GiB/s
BM_string_strlen/32Ki 500k 6135 5.341 GiB/s
BM_string_strlen/64Ki 200k 13168 4.977 GiB/s
After
BM_string_memcmp/8 1000k 21 0.369 GiB/s
BM_string_memcmp/64 1000k 28 2.272 GiB/s
BM_string_memcmp/512 20M 128 3.983 GiB/s
BM_string_memcmp/1024 10M 234 4.375 GiB/s
BM_string_memcmp/8Ki 1000k 1732 4.728 GiB/s
BM_string_memcmp/16Ki 500k 3485 4.701 GiB/s
BM_string_memcmp/32Ki 500k 7031 4.660 GiB/s
BM_string_memcmp/64Ki 200k 14296 4.584 GiB/s
BM_string_memcpy/8 1000k 5 1.458 GiB/s
BM_string_memcpy/64 1000k 7 8.952 GiB/s
BM_string_memcpy/512 50M 36 13.907 GiB/s
BM_string_memcpy/1024 20M 80 12.750 GiB/s
BM_string_memcpy/8Ki 5M 572 14.307 GiB/s
BM_string_memcpy/16Ki 1000k 1165 14.053 GiB/s
BM_string_memcpy/32Ki 500k 3141 10.430 GiB/s
BM_string_memcpy/64Ki 500k 7008 9.351 GiB/s
BM_string_memmove/8 50M 7 1.074 GiB/s
BM_string_memmove/64 1000k 9 6.593 GiB/s
BM_string_memmove/512 50M 37 13.502 GiB/s
BM_string_memmove/1024 20M 80 12.656 GiB/s
BM_string_memmove/8Ki 5M 573 14.281 GiB/s
BM_string_memmove/16Ki 1000k 1168 14.018 GiB/s
BM_string_memmove/32Ki 1000k 2825 11.599 GiB/s
BM_string_memmove/64Ki 500k 6548 10.008 GiB/s
BM_string_memset/8 1000k 7 1.038 GiB/s
BM_string_memset/64 1000k 8 7.151 GiB/s
BM_string_memset/512 1000k 29 17.272 GiB/s
BM_string_memset/1024 50M 53 18.969 GiB/s
BM_string_memset/8Ki 5M 424 19.300 GiB/s
BM_string_memset/16Ki 2M 846 19.350 GiB/s
BM_string_memset/32Ki 1000k 2028 16.156 GiB/s
BM_string_memset/64Ki 500k 4514 14.517 GiB/s
BM_string_strlen/8 1000k 7 1.120 GiB/s
BM_string_strlen/64 1000k 16 3.918 GiB/s
BM_string_strlen/512 50M 64 7.894 GiB/s
BM_string_strlen/1024 20M 104 9.815 GiB/s
BM_string_strlen/8Ki 5M 664 12.337 GiB/s
BM_string_strlen/16Ki 1000k 1291 12.682 GiB/s
BM_string_strlen/32Ki 1000k 2940 11.143 GiB/s
BM_string_strlen/64Ki 500k 6440 10.175 GiB/s
Change-Id: I635bd2798a755256f748b2af19b1a56fb85a40c6
diff --git a/libc/arch-arm64/generic/bionic/strlen.S b/libc/arch-arm64/generic/bionic/strlen.S
index 3bd9809..6e540fc 100644
--- a/libc/arch-arm64/generic/bionic/strlen.S
+++ b/libc/arch-arm64/generic/bionic/strlen.S
@@ -1,16 +1,16 @@
-/* Copyright (c) 2014, Linaro Limited
+/* Copyright (c) 2013-2015, Linaro Limited
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
+ notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
* Neither the name of the Linaro nor the
- names of its contributors may be used to endorse or promote products
- derived from this software without specific prior written permission.
+ names of its contributors may be used to endorse or promote products
+ derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
@@ -22,16 +22,19 @@
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */
/* Assumptions:
*
- * ARMv8-a, AArch64
+ * ARMv8-a, AArch64, unaligned accesses, min page size 4k.
*/
#include <private/bionic_asm.h>
+/* To test the page crossing code path more thoroughly, compile with
+ -DTEST_PAGE_CROSS - this will force all calls through the slower
+ entry path. This option is not intended for production use. */
+
/* Arguments and results. */
#define srcin x0
#define len x0
@@ -40,87 +43,185 @@
#define src x1
#define data1 x2
#define data2 x3
-#define data2a x4
-#define has_nul1 x5
-#define has_nul2 x6
-#define tmp1 x7
-#define tmp2 x8
-#define tmp3 x9
-#define tmp4 x10
-#define zeroones x11
-#define pos x12
+#define has_nul1 x4
+#define has_nul2 x5
+#define tmp1 x4
+#define tmp2 x5
+#define tmp3 x6
+#define tmp4 x7
+#define zeroones x8
+
+#define L(l) .L ## l
+
+ /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
+ (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
+ can be done in parallel across the entire word. A faster check
+ (X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives
+ false hits for characters 129..255. */
#define REP8_01 0x0101010101010101
#define REP8_7f 0x7f7f7f7f7f7f7f7f
#define REP8_80 0x8080808080808080
- /* Start of critial section -- keep to one 64Byte cache line. */
-ENTRY(strlen)
- mov zeroones, #REP8_01
- bic src, srcin, #15
- ands tmp1, srcin, #15
- b.ne .Lmisaligned
- /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
- (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
- can be done in parallel across the entire word. */
- /* The inner loop deals with two Dwords at a time. This has a
- slightly higher start-up cost, but we should win quite quickly,
- especially on cores with a high number of issue slots per
- cycle, as we get much better parallelism out of the operations. */
-.Lloop:
- ldp data1, data2, [src], #16
-.Lrealigned:
- sub tmp1, data1, zeroones
- orr tmp2, data1, #REP8_7f
- sub tmp3, data2, zeroones
- orr tmp4, data2, #REP8_7f
- bic has_nul1, tmp1, tmp2
- bics has_nul2, tmp3, tmp4
- ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
- b.eq .Lloop
- /* End of critical section -- keep to one 64Byte cache line. */
-
- sub len, src, srcin
- cbz has_nul1, .Lnul_in_data2
-#ifdef __AARCH64EB__
- mov data2, data1
+#ifdef TEST_PAGE_CROSS
+# define MIN_PAGE_SIZE 15
+#else
+# define MIN_PAGE_SIZE 4096
#endif
- sub len, len, #8
- mov has_nul2, has_nul1
-.Lnul_in_data2:
+
+ /* Since strings are short on average, we check the first 16 bytes
+ of the string for a NUL character. In order to do an unaligned ldp
+ safely we have to do a page cross check first. If there is a NUL
+ byte we calculate the length from the 2 8-byte words using
+ conditional select to reduce branch mispredictions (it is unlikely
+ strlen will be repeatedly called on strings with the same length).
+
+ If the string is longer than 16 bytes, we align src so don't need
+ further page cross checks, and process 32 bytes per iteration
+ using the fast NUL check. If we encounter non-ASCII characters,
+ fallback to a second loop using the full NUL check.
+
+ If the page cross check fails, we read 16 bytes from an aligned
+ address, remove any characters before the string, and continue
+ in the main loop using aligned loads. Since strings crossing a
+ page in the first 16 bytes are rare (probability of
+ 16/MIN_PAGE_SIZE ~= 0.4%), this case does not need to be optimized.
+
+ AArch64 systems have a minimum page size of 4k. We don't bother
+ checking for larger page sizes - the cost of setting up the correct
+ page size is just not worth the extra gain from a small reduction in
+ the cases taking the slow path. Note that we only care about
+ whether the first fetch, which may be misaligned, crosses a page
+ boundary. */
+
+ENTRY(strlen)
+ and tmp1, srcin, MIN_PAGE_SIZE - 1
+ mov zeroones, REP8_01
+ cmp tmp1, MIN_PAGE_SIZE - 16
+ b.gt L(page_cross)
+ ldp data1, data2, [srcin]
#ifdef __AARCH64EB__
/* For big-endian, carry propagation (if the final byte in the
- string is 0x01) means we cannot use has_nul directly. The
- easiest way to get the correct byte is to byte-swap the data
- and calculate the syndrome a second time. */
+ string is 0x01) means we cannot use has_nul1/2 directly.
+ Since we expect strings to be small and early-exit,
+ byte-swap the data now so has_null1/2 will be correct. */
+ rev data1, data1
rev data2, data2
- sub tmp1, data2, zeroones
- orr tmp2, data2, #REP8_7f
- bic has_nul2, tmp1, tmp2
#endif
- sub len, len, #8
- rev has_nul2, has_nul2
- clz pos, has_nul2
- add len, len, pos, lsr #3 /* Bits to bytes. */
+ sub tmp1, data1, zeroones
+ orr tmp2, data1, REP8_7f
+ sub tmp3, data2, zeroones
+ orr tmp4, data2, REP8_7f
+ bics has_nul1, tmp1, tmp2
+ bic has_nul2, tmp3, tmp4
+ ccmp has_nul2, 0, 0, eq
+ beq L(main_loop_entry)
+
+ /* Enter with C = has_nul1 == 0. */
+ csel has_nul1, has_nul1, has_nul2, cc
+ mov len, 8
+ rev has_nul1, has_nul1
+ clz tmp1, has_nul1
+ csel len, xzr, len, cc
+ add len, len, tmp1, lsr 3
ret
-.Lmisaligned:
- cmp tmp1, #8
- neg tmp1, tmp1
- ldp data1, data2, [src], #16
- lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */
- mov tmp2, #~0
+ /* The inner loop processes 32 bytes per iteration and uses the fast
+ NUL check. If we encounter non-ASCII characters, use a second
+ loop with the accurate NUL check. */
+ .p2align 4
+L(main_loop_entry):
+ bic src, srcin, 15
+ sub src, src, 16
+L(main_loop):
+ ldp data1, data2, [src, 32]!
+.Lpage_cross_entry:
+ sub tmp1, data1, zeroones
+ sub tmp3, data2, zeroones
+ orr tmp2, tmp1, tmp3
+ tst tmp2, zeroones, lsl 7
+ bne 1f
+ ldp data1, data2, [src, 16]
+ sub tmp1, data1, zeroones
+ sub tmp3, data2, zeroones
+ orr tmp2, tmp1, tmp3
+ tst tmp2, zeroones, lsl 7
+ beq L(main_loop)
+ add src, src, 16
+1:
+ /* The fast check failed, so do the slower, accurate NUL check. */
+ orr tmp2, data1, REP8_7f
+ orr tmp4, data2, REP8_7f
+ bics has_nul1, tmp1, tmp2
+ bic has_nul2, tmp3, tmp4
+ ccmp has_nul2, 0, 0, eq
+ beq L(nonascii_loop)
+
+ /* Enter with C = has_nul1 == 0. */
+L(tail):
#ifdef __AARCH64EB__
- /* Big-endian. Early bytes are at MSB. */
- lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
+ /* For big-endian, carry propagation (if the final byte in the
+ string is 0x01) means we cannot use has_nul1/2 directly. The
+ easiest way to get the correct byte is to byte-swap the data
+ and calculate the syndrome a second time. */
+ csel data1, data1, data2, cc
+ rev data1, data1
+ sub tmp1, data1, zeroones
+ orr tmp2, data1, REP8_7f
+ bic has_nul1, tmp1, tmp2
+#else
+ csel has_nul1, has_nul1, has_nul2, cc
+#endif
+ sub len, src, srcin
+ rev has_nul1, has_nul1
+ add tmp2, len, 8
+ clz tmp1, has_nul1
+ csel len, len, tmp2, cc
+ add len, len, tmp1, lsr 3
+ ret
+
+L(nonascii_loop):
+ ldp data1, data2, [src, 16]!
+ sub tmp1, data1, zeroones
+ orr tmp2, data1, REP8_7f
+ sub tmp3, data2, zeroones
+ orr tmp4, data2, REP8_7f
+ bics has_nul1, tmp1, tmp2
+ bic has_nul2, tmp3, tmp4
+ ccmp has_nul2, 0, 0, eq
+ bne L(tail)
+ ldp data1, data2, [src, 16]!
+ sub tmp1, data1, zeroones
+ orr tmp2, data1, REP8_7f
+ sub tmp3, data2, zeroones
+ orr tmp4, data2, REP8_7f
+ bics has_nul1, tmp1, tmp2
+ bic has_nul2, tmp3, tmp4
+ ccmp has_nul2, 0, 0, eq
+ beq L(nonascii_loop)
+ b L(tail)
+
+ /* Load 16 bytes from [srcin & ~15] and force the bytes that precede
+ srcin to 0x7f, so we ignore any NUL bytes before the string.
+ Then continue in the aligned loop. */
+L(page_cross):
+ bic src, srcin, 15
+ ldp data1, data2, [src]
+ lsl tmp1, srcin, 3
+ mov tmp4, -1
+#ifdef __AARCH64EB__
+ /* Big-endian. Early bytes are at MSB. */
+ lsr tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */
#else
/* Little-endian. Early bytes are at LSB. */
- lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
+ lsl tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */
#endif
- orr data1, data1, tmp2
- orr data2a, data2, tmp2
- csinv data1, data1, xzr, le
- csel data2, data2, data2a, le
- b .Lrealigned
+ orr tmp1, tmp1, REP8_80
+ orn data1, data1, tmp1
+ orn tmp2, data2, tmp1
+ tst srcin, 8
+ csel data1, data1, tmp4, eq
+ csel data2, data2, tmp2, eq
+ b L(page_cross_entry)
END(strlen)