Merge "Use builtins for fma/fmax/fmin/round on arm/arm64."
diff --git a/benchmarks/tests/interface_test.cpp b/benchmarks/tests/interface_test.cpp
index b8279b6..6987320 100644
--- a/benchmarks/tests/interface_test.cpp
+++ b/benchmarks/tests/interface_test.cpp
@@ -71,7 +71,8 @@
char path[PATH_MAX];
ssize_t path_len = readlink("/proc/self/exe", path, sizeof(path));
ASSERT_TRUE(path_len >= 0);
- *exe_name = std::string(std::regex_replace(path, std::regex("-tests"), ""));
+ *exe_name = std::string(std::regex_replace(path, std::regex("nativetest"), "benchmarktest"));
+ *exe_name = std::regex_replace(*exe_name, std::regex("-tests"), "");
}
void SystemTests::Exec(std::vector<const char*> args) {
@@ -289,6 +290,10 @@
"BM_stdio_fopen_fgetc_fclose_no_locking/1024/iterations:1\n"
"BM_stdio_fopen_getline_fclose_locking/iterations:1\n"
"BM_stdio_fopen_getline_fclose_no_locking/iterations:1\n"
+ "BM_stdio_printf_literal/iterations:1\n"
+ "BM_stdio_printf_s/iterations:1\n"
+ "BM_stdio_printf_d/iterations:1\n"
+ "BM_stdio_printf_1$s/iterations:1\n"
"BM_string_memcmp/8/0/0/iterations:1\n"
"BM_string_memcmp/64/0/0/iterations:1\n"
"BM_string_memcmp/512/0/0/iterations:1\n"
diff --git a/libc/NOTICE b/libc/NOTICE
index cbcafc8..db4b5a5 100644
--- a/libc/NOTICE
+++ b/libc/NOTICE
@@ -5643,6 +5643,34 @@
-------------------------------------------------------------------
+Copyright (c) 2017 ARM Ltd
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+3. The name of the company may not be used to endorse or promote
+ products derived from this software without specific prior written
+ permission.
+
+THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+-------------------------------------------------------------------
+
Copyright (c) 2017 Imagination Technologies.
All rights reserved.
diff --git a/libc/arch-arm64/generic/bionic/memcmp.S b/libc/arch-arm64/generic/bionic/memcmp.S
index 3d08ecd..3a138bf 100644
--- a/libc/arch-arm64/generic/bionic/memcmp.S
+++ b/libc/arch-arm64/generic/bionic/memcmp.S
@@ -1,33 +1,34 @@
-/* Copyright (c) 2014, Linaro Limited
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- * Neither the name of the Linaro nor the
- names of its contributors may be used to endorse or promote products
- derived from this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
+/*
+ * Copyright (c) 2017 ARM Ltd
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. The name of the company may not be used to endorse or promote
+ * products derived from this software without specific prior written
+ * permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
/* Assumptions:
*
- * ARMv8-a, AArch64
+ * ARMv8-a, AArch64, unaligned accesses.
*/
#include <private/bionic_asm.h>
@@ -36,120 +37,93 @@
#define src1 x0
#define src2 x1
#define limit x2
-#define result x0
+#define result w0
/* Internal variables. */
#define data1 x3
#define data1w w3
#define data2 x4
#define data2w w4
-#define has_nul x5
-#define diff x6
-#define endloop x7
-#define tmp1 x8
-#define tmp2 x9
-#define tmp3 x10
-#define pos x11
-#define limit_wd x12
-#define mask x13
+#define tmp1 x5
+/* Small inputs of less than 8 bytes are handled separately. This allows the
+ main code to be sped up using unaligned loads since there are now at least
+ 8 bytes to be compared. If the first 8 bytes are equal, align src1.
+ This ensures each iteration does at most one unaligned access even if both
+ src1 and src2 are unaligned, and mutually aligned inputs behave as if
+ aligned. After the main loop, process the last 8 bytes using unaligned
+ accesses. */
+
+.p2align 6
ENTRY(memcmp)
- cbz limit, .Lret0
- eor tmp1, src1, src2
- tst tmp1, #7
- b.ne .Lmisaligned8
- ands tmp1, src1, #7
- b.ne .Lmutual_align
- add limit_wd, limit, #7
- lsr limit_wd, limit_wd, #3
- /* Start of performance-critical section -- one 64B cache line. */
-.Lloop_aligned:
- ldr data1, [src1], #8
- ldr data2, [src2], #8
-.Lstart_realigned:
- subs limit_wd, limit_wd, #1
- eor diff, data1, data2 /* Non-zero if differences found. */
- csinv endloop, diff, xzr, ne /* Last Dword or differences. */
- cbz endloop, .Lloop_aligned
- /* End of performance-critical section -- one 64B cache line. */
+ subs limit, limit, 8
+ b.lo .Lless8
- /* Not reached the limit, must have found a diff. */
- cbnz limit_wd, .Lnot_limit
+ /* Limit >= 8, so check first 8 bytes using unaligned loads. */
+ ldr data1, [src1], 8
+ ldr data2, [src2], 8
+ and tmp1, src1, 7
+ add limit, limit, tmp1
+ cmp data1, data2
+ bne .Lreturn
- /* Limit % 8 == 0 => all bytes significant. */
- ands limit, limit, #7
- b.eq .Lnot_limit
+ /* Align src1 and adjust src2 with bytes not yet done. */
+ sub src1, src1, tmp1
+ sub src2, src2, tmp1
- lsl limit, limit, #3 /* Bits -> bytes. */
- mov mask, #~0
-#ifdef __AARCH64EB__
- lsr mask, mask, limit
-#else
- lsl mask, mask, limit
-#endif
- bic data1, data1, mask
- bic data2, data2, mask
+ subs limit, limit, 8
+ b.ls .Llast_bytes
- orr diff, diff, mask
-.Lnot_limit:
+ /* Loop performing 8 bytes per iteration using aligned src1.
+ Limit is pre-decremented by 8 and must be larger than zero.
+ Exit if <= 8 bytes left to do or if the data is not equal. */
+ .p2align 4
+.Lloop8:
+ ldr data1, [src1], 8
+ ldr data2, [src2], 8
+ subs limit, limit, 8
+ ccmp data1, data2, 0, hi /* NZCV = 0b0000. */
+ b.eq .Lloop8
-#ifndef __AARCH64EB__
- rev diff, diff
+ cmp data1, data2
+ bne .Lreturn
+
+ /* Compare last 1-8 bytes using unaligned access. */
+.Llast_bytes:
+ ldr data1, [src1, limit]
+ ldr data2, [src2, limit]
+
+ /* Compare data bytes and set return value to 0, -1 or 1. */
+.Lreturn:
+#ifndef __AARCH64EB__
rev data1, data1
rev data2, data2
#endif
- /* The MS-non-zero bit of DIFF marks either the first bit
- that is different, or the end of the significant data.
- Shifting left now will bring the critical information into the
- top bits. */
- clz pos, diff
- lsl data1, data1, pos
- lsl data2, data2, pos
- /* But we need to zero-extend (char is unsigned) the value and then
- perform a signed 32-bit subtraction. */
- lsr data1, data1, #56
- sub result, data1, data2, lsr #56
- ret
+ cmp data1, data2
+.Lret_eq:
+ cset result, ne
+ cneg result, result, lo
+ ret
-.Lmutual_align:
- /* Sources are mutually aligned, but are not currently at an
- alignment boundary. Round down the addresses and then mask off
- the bytes that precede the start point. */
- bic src1, src1, #7
- bic src2, src2, #7
- add limit, limit, tmp1 /* Adjust the limit for the extra. */
- lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */
- ldr data1, [src1], #8
- neg tmp1, tmp1 /* Bits to alignment -64. */
- ldr data2, [src2], #8
- mov tmp2, #~0
-#ifdef __AARCH64EB__
- /* Big-endian. Early bytes are at MSB. */
- lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
-#else
- /* Little-endian. Early bytes are at LSB. */
- lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
-#endif
- add limit_wd, limit, #7
- orr data1, data1, tmp2
- orr data2, data2, tmp2
- lsr limit_wd, limit_wd, #3
- b .Lstart_realigned
-
-.Lret0:
- mov result, #0
- ret
-
- .p2align 6
-.Lmisaligned8:
- sub limit, limit, #1
-1:
- /* Perhaps we can do better than this. */
- ldrb data1w, [src1], #1
- ldrb data2w, [src2], #1
- subs limit, limit, #1
- ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */
- b.eq 1b
- sub result, data1, data2
+ .p2align 4
+ /* Compare up to 8 bytes. Limit is [-8..-1]. */
+.Lless8:
+ adds limit, limit, 4
+ b.lo .Lless4
+ ldr data1w, [src1], 4
+ ldr data2w, [src2], 4
+ cmp data1w, data2w
+ b.ne .Lreturn
+ sub limit, limit, 4
+.Lless4:
+ adds limit, limit, 4
+ beq .Lret_eq
+.Lbyte_loop:
+ ldrb data1w, [src1], 1
+ ldrb data2w, [src2], 1
+ subs limit, limit, 1
+ ccmp data1w, data2w, 0, ne /* NZCV = 0b0000. */
+ b.eq .Lbyte_loop
+ sub result, data1w, data2w
ret
END(memcmp)
diff --git a/libc/stdio/vfprintf.cpp b/libc/stdio/vfprintf.cpp
index 366b196..fe2e338 100644
--- a/libc/stdio/vfprintf.cpp
+++ b/libc/stdio/vfprintf.cpp
@@ -32,6 +32,7 @@
*/
#define CHAR_TYPE char
+#define FUNCTION_NAME __vfprintf
#include <sys/mman.h>
#include <sys/types.h>
@@ -108,8 +109,7 @@
* temporary buffer. We only work on write-only files; this avoids
* worries about ungetc buffers and so forth.
*/
-static int __sbprintf(FILE* fp, const char* fmt, va_list ap) {
- int ret;
+static int __sbprintf(FILE* fp, const CHAR_TYPE* fmt, va_list ap) {
FILE fake;
struct __sfileext fakeext;
unsigned char buf[BUFSIZ];
@@ -127,7 +127,7 @@
fake._lbfsize = 0; /* not actually used, but Just In Case */
/* do the work, then copy any error status */
- ret = __vfprintf(&fake, fmt, ap);
+ int ret = FUNCTION_NAME(&fake, fmt, ap);
if (ret >= 0 && __sflush(&fake)) ret = EOF;
if (fake._flags & __SERR) fp->_flags |= __SERR;
return (ret);
@@ -250,17 +250,16 @@
#define CHARINT 0x0800 /* 8 bit integer */
#define MAXINT 0x1000 /* largest integer size (intmax_t) */
-int __vfprintf(FILE* fp, const char* fmt0, __va_list ap) {
+int FUNCTION_NAME(FILE* fp, const CHAR_TYPE* fmt0, __va_list ap) {
int ch; /* character from fmt */
int n, n2; /* handy integers (short term usage) */
- char* cp; /* handy char pointer (short term usage) */
+ CHAR_TYPE* cp; /* handy char pointer (short term usage) */
struct __siov* iovp; /* for PRINT macro */
+ CHAR_TYPE sign; /* sign prefix (' ', '+', '-', or \0) */
int flags; /* flags as above */
int ret; /* return value accumulator */
int width; /* width from format (%8d), or 0 */
int prec; /* precision from format; <0 for N/A */
- char sign; /* sign prefix (' ', '+', '-', or \0) */
- mbstate_t ps;
/*
* We can decompose the printed representation of floating
* point numbers into several parts, some of which may be empty:
@@ -287,7 +286,7 @@
int expsize; /* character count for expstr */
int lead; /* sig figs before decimal or group sep */
int ndig; /* actual number of digits returned by dtoa */
- char expstr[MAXEXPDIG + 2]; /* buffer for exponent string: e+ZZZ */
+ CHAR_TYPE expstr[MAXEXPDIG + 2]; /* buffer for exponent string: e+ZZZ */
char* dtoaresult = NULL;
uintmax_t _umax; /* integer arguments %[diouxX] */
@@ -299,14 +298,14 @@
#define NIOV 8
struct __suio uio; /* output information: summary */
struct __siov iov[NIOV]; /* ... and individual io vectors */
- char buf[BUF]; /* buffer with space for digits of uintmax_t */
- char ox[2]; /* space for 0x; ox[1] is either x, X, or \0 */
+ CHAR_TYPE buf[BUF]; /* buffer with space for digits of uintmax_t */
+ CHAR_TYPE ox[2]; /* space for 0x; ox[1] is either x, X, or \0 */
union arg* argtable; /* args, built due to positional arg */
union arg statargtable[STATIC_ARG_TBL_SIZE];
size_t argtablesiz;
int nextarg; /* 1-based argument index */
va_list orgap; /* original argument pointer */
- char* convbuf; /* buffer for wide to multi-byte conversion */
+ CHAR_TYPE* convbuf; /* buffer for wide/multibyte conversion */
/*
* Choose PADSIZE to trade efficiency vs. size. If larger printf
@@ -466,7 +465,6 @@
ret = 0;
convbuf = NULL;
- memset(&ps, 0, sizeof(ps));
/*
* Scan the format for conversions (`%' character).
*/
@@ -1275,10 +1273,7 @@
flags |= LONGINT;
/*FALLTHROUGH*/
case 's':
- if (flags & LONGINT)
- ADDTYPE(TP_WCHAR);
- else
- ADDTYPE(TP_CHAR);
+ ADDTYPE((flags & LONGINT) ? TP_WCHAR : TP_CHAR);
break;
case 'U':
flags |= LONGINT;
diff --git a/libc/stdio/vfwprintf.cpp b/libc/stdio/vfwprintf.cpp
index 7235969..e8f1056 100644
--- a/libc/stdio/vfwprintf.cpp
+++ b/libc/stdio/vfwprintf.cpp
@@ -32,6 +32,7 @@
*/
#define CHAR_TYPE wchar_t
+#define FUNCTION_NAME __vfwprintf
#include <sys/mman.h>
#include <sys/types.h>
@@ -91,8 +92,7 @@
* temporary buffer. We only work on write-only files; this avoids
* worries about ungetc buffers and so forth.
*/
-static int __sbprintf(FILE* fp, const wchar_t* fmt, va_list ap) {
- int ret;
+static int __sbprintf(FILE* fp, const CHAR_TYPE* fmt, va_list ap) {
FILE fake;
struct __sfileext fakeext;
unsigned char buf[BUFSIZ];
@@ -110,7 +110,7 @@
fake._lbfsize = 0; /* not actually used, but Just In Case */
/* do the work, then copy any error status */
- ret = __vfwprintf(&fake, fmt, ap);
+ int ret = FUNCTION_NAME(&fake, fmt, ap);
if (ret >= 0 && __sflush(&fake)) ret = EOF;
if (fake._flags & __SERR) fp->_flags |= __SERR;
return (ret);
@@ -276,15 +276,15 @@
#define CHARINT 0x0800 /* 8 bit integer */
#define MAXINT 0x1000 /* largest integer size (intmax_t) */
-int __vfwprintf(FILE* __restrict fp, const wchar_t* __restrict fmt0, __va_list ap) {
+int FUNCTION_NAME(FILE* fp, const CHAR_TYPE* fmt0, __va_list ap) {
wchar_t ch; /* character from fmt */
int n, n2, n3; /* handy integers (short term usage) */
- wchar_t* cp; /* handy char pointer (short term usage) */
+ CHAR_TYPE* cp; /* handy char pointer (short term usage) */
+ CHAR_TYPE sign; /* sign prefix (' ', '+', '-', or \0) */
int flags; /* flags as above */
int ret; /* return value accumulator */
int width; /* width from format (%8d), or 0 */
int prec; /* precision from format; <0 for N/A */
- wchar_t sign; /* sign prefix (' ', '+', '-', or \0) */
/*
* We can decompose the printed representation of floating
* point numbers into several parts, some of which may be empty:
@@ -311,7 +311,7 @@
int expsize; /* character count for expstr */
int lead; /* sig figs before decimal or group sep */
int ndig; /* actual number of digits returned by dtoa */
- wchar_t expstr[MAXEXPDIG + 2]; /* buffer for exponent string: e+ZZZ */
+ CHAR_TYPE expstr[MAXEXPDIG + 2]; /* buffer for exponent string: e+ZZZ */
char* dtoaresult = NULL;
uintmax_t _umax; /* integer arguments %[diouxX] */
@@ -320,14 +320,14 @@
int realsz; /* field size expanded by dprec */
int size; /* size of converted field or string */
const char* xdigs; /* digits for %[xX] conversion */
- wchar_t buf[BUF]; /* buffer with space for digits of uintmax_t */
- wchar_t ox[2]; /* space for 0x; ox[1] is either x, X, or \0 */
+ CHAR_TYPE buf[BUF]; /* buffer with space for digits of uintmax_t */
+ CHAR_TYPE ox[2]; /* space for 0x; ox[1] is either x, X, or \0 */
union arg* argtable; /* args, built due to positional arg */
union arg statargtable[STATIC_ARG_TBL_SIZE];
size_t argtablesiz;
int nextarg; /* 1-based argument index */
va_list orgap; /* original argument pointer */
- wchar_t* convbuf; /* buffer for multibyte to wide conversion */
+ CHAR_TYPE* convbuf; /* buffer for wide/multibyte conversion */
/*
* Choose PADSIZE to trade efficiency vs. size. If larger printf
@@ -1271,10 +1271,7 @@
flags |= LONGINT;
/*FALLTHROUGH*/
case 's':
- if (flags & LONGINT)
- ADDTYPE(TP_CHAR);
- else
- ADDTYPE(TP_WCHAR);
+ ADDTYPE((flags & LONGINT) ? TP_WCHAR : TP_CHAR);
break;
case 'U':
flags |= LONGINT;
diff --git a/tests/libs/Android.bp b/tests/libs/Android.bp
index 5f27387..8d0271a 100644
--- a/tests/libs/Android.bp
+++ b/tests/libs/Android.bp
@@ -446,7 +446,7 @@
name: "libtestshared",
defaults: ["bionic_testlib_defaults"],
srcs: ["empty.cpp"],
- relative_install_path: "/inaccessible_libs",
+ relative_install_path: "bionic-loader-test-libs/inaccessible_libs",
}
// -----------------------------------------------------------------------------
@@ -629,7 +629,7 @@
defaults: ["bionic_testlib_defaults"],
srcs: ["ld_config_test_helper_lib1.cpp"],
shared_libs: ["ld_config_test_helper_lib2"],
- relative_install_path: "/ns2",
+ relative_install_path: "bionic-loader-test-libs/ns2",
}
cc_test_library {
@@ -637,7 +637,7 @@
host_supported: false,
defaults: ["bionic_testlib_defaults"],
srcs: ["ld_config_test_helper_lib2.cpp"],
- relative_install_path: "/ns2",
+ relative_install_path: "bionic-loader-test-libs/ns2",
}
cc_test_library {