Merge "Introduce filegroups for libc sources"
diff --git a/Android.bp b/Android.bp
deleted file mode 100644
index b44c296..0000000
--- a/Android.bp
+++ /dev/null
@@ -1 +0,0 @@
-subdirs = ["*"]
diff --git a/libc/Android.bp b/libc/Android.bp
index 87793a5..0098d07 100644
--- a/libc/Android.bp
+++ b/libc/Android.bp
@@ -81,12 +81,6 @@
xom: false,
}
-// ANDROIDMK TRANSLATION ERROR: unsupported directive
-// ifeq ($(strip $(DEBUG_BIONIC_LIBC)),true)
-//libc_common_cflags += ["-DDEBUG"]
-// ANDROIDMK TRANSLATION ERROR: unsupported directive
-// endif
-
// ========================================================
// libc_stack_protector.a - stack protector code
// ========================================================
@@ -714,8 +708,8 @@
"arch-arm/cortex-a53/bionic/__strcat_chk.S",
"arch-arm/cortex-a53/bionic/__strcpy_chk.S",
- "arch-arm/denver/bionic/__strcat_chk.S",
- "arch-arm/denver/bionic/__strcpy_chk.S",
+ "arch-arm/cortex-a55/bionic/__strcat_chk.S",
+ "arch-arm/cortex-a55/bionic/__strcpy_chk.S",
],
},
arm64: {
@@ -786,11 +780,9 @@
"arch-arm/cortex-a15/bionic/strlen.S",
"arch-arm/cortex-a7/bionic/memcpy.S",
- "arch-arm/cortex-a7/bionic/memmove.S",
"arch-arm/cortex-a7/bionic/memset.S",
"arch-arm/cortex-a9/bionic/memcpy.S",
- "arch-arm/cortex-a9/bionic/memmove.S",
"arch-arm/cortex-a9/bionic/memset.S",
"arch-arm/cortex-a9/bionic/stpcpy.S",
"arch-arm/cortex-a9/bionic/strcat.S",
@@ -799,19 +791,14 @@
"arch-arm/cortex-a9/bionic/strlen.S",
"arch-arm/krait/bionic/memcpy.S",
- "arch-arm/krait/bionic/memmove.S",
"arch-arm/krait/bionic/memset.S",
"arch-arm/krait/bionic/strcmp.S",
"arch-arm/cortex-a53/bionic/memcpy.S",
- "arch-arm/cortex-a53/bionic/memmove.S",
- "arch-arm/denver/bionic/memcpy.S",
- "arch-arm/denver/bionic/memmove.S",
- "arch-arm/denver/bionic/memset.S",
+ "arch-arm/cortex-a55/bionic/memcpy.S",
"arch-arm/kryo/bionic/memcpy.S",
- "arch-arm/kryo/bionic/memmove.S",
],
},
arm64: {
@@ -1640,9 +1627,15 @@
],
name: "libstdc++",
static_ndk_lib: true,
- system_shared_libs: ["libc"],
static_libs: ["libasync_safe"],
+ static: {
+ system_shared_libs: [],
+ },
+ shared: {
+ system_shared_libs: ["libc"],
+ },
+
//TODO (dimitry): This is to work around b/24465209. Remove after root cause is fixed
arch: {
arm: {
diff --git a/libc/NOTICE b/libc/NOTICE
index 314b936..40a5704 100644
--- a/libc/NOTICE
+++ b/libc/NOTICE
@@ -808,35 +808,6 @@
-------------------------------------------------------------------
-Copyright (C) 2013 The Android Open Source Project
-Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in
- the documentation and/or other materials provided with the
- distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
-OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
-OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGE.
-
--------------------------------------------------------------------
-
Copyright (C) 2014 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/libc/arch-arm/cortex-a15/bionic/memmove.S b/libc/arch-arm/cortex-a15/bionic/memmove.S
index 2b12de0..92eba8d 100644
--- a/libc/arch-arm/cortex-a15/bionic/memmove.S
+++ b/libc/arch-arm/cortex-a15/bionic/memmove.S
@@ -1,6 +1,7 @@
/*
- * Copyright (C) 2018 The Android Open Source Project
+ * Copyright (C) 2013 The Android Open Source Project
* All rights reserved.
+ * Copyright (c) 2013-2014 NVIDIA Corporation. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -26,7 +27,254 @@
* SUCH DAMAGE.
*/
-#define MEMMOVE memmove_a15
-#define MEMCPY __memcpy_a15
+#include <private/bionic_asm.h>
-#include <arch-arm/denver/bionic/memmove.S>
+ .text
+ .syntax unified
+ .fpu neon
+
+#define CACHE_LINE_SIZE (64)
+#define MEMCPY_BLOCK_SIZE_SMALL (32768)
+#define MEMCPY_BLOCK_SIZE_MID (1048576)
+#define PREFETCH_DISTANCE_NEAR (CACHE_LINE_SIZE*4)
+#define PREFETCH_DISTANCE_MID (CACHE_LINE_SIZE*4)
+#define PREFETCH_DISTANCE_FAR (CACHE_LINE_SIZE*16)
+
+ENTRY(memmove_a15)
+ cmp r2, #0
+ cmpne r0, r1
+ bxeq lr
+ subs r3, r0, r1
+ bls .L_jump_to_memcpy
+ cmp r2, r3
+ bhi .L_reversed_memcpy
+
+.L_jump_to_memcpy:
+ b __memcpy
+
+.L_reversed_memcpy:
+ push {r0, lr}
+ .cfi_def_cfa_offset 8
+ .cfi_rel_offset r0, 0
+ .cfi_rel_offset lr, 4
+
+ add r0, r0, r2
+ add r1, r1, r2
+
+ /* preload next cache line */
+ pld [r1, #-CACHE_LINE_SIZE]
+ pld [r1, #-CACHE_LINE_SIZE*2]
+
+.L_reversed_memcpy_align_dest:
+ /* Deal with very small blocks (< 32bytes) asap */
+ cmp r2, #32
+ blo .L_reversed_memcpy_lt_32bytes
+ /* no need to align if len < 128 bytes */
+ cmp r2, #128
+ blo .L_reversed_memcpy_lt_128bytes
+ /* align destination to 64 bytes (1 cache line) */
+ ands r3, r0, #0x3f
+ beq .L_reversed_memcpy_dispatch
+ sub r2, r2, r3
+0: /* copy 1 byte */
+ movs ip, r3, lsl #31
+ ldrbmi ip, [r1, #-1]!
+ strbmi ip, [r0, #-1]!
+1: /* copy 2 bytes */
+ ldrbcs ip, [r1, #-1]!
+ strbcs ip, [r0, #-1]!
+ ldrbcs ip, [r1, #-1]!
+ strbcs ip, [r0, #-1]!
+2: /* copy 4 bytes */
+ movs ip, r3, lsl #29
+ bpl 3f
+ sub r1, r1, #4
+ sub r0, r0, #4
+ vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]
+ vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]
+3: /* copy 8 bytes */
+ bcc 4f
+ sub r1, r1, #8
+ sub r0, r0, #8
+ vld1.8 {d0}, [r1]
+ vst1.8 {d0}, [r0, :64]
+4: /* copy 16 bytes */
+ movs ip, r3, lsl #27
+ bpl 5f
+ sub r1, r1, #16
+ sub r0, r0, #16
+ vld1.8 {q0}, [r1]
+ vst1.8 {q0}, [r0, :128]
+5: /* copy 32 bytes */
+ bcc .L_reversed_memcpy_dispatch
+ sub r1, r1, #32
+ sub r0, r0, #32
+ vld1.8 {q0, q1}, [r1]
+ vst1.8 {q0, q1}, [r0, :256]
+
+.L_reversed_memcpy_dispatch:
+ /* preload more cache lines */
+ pld [r1, #-CACHE_LINE_SIZE*3]
+ pld [r1, #-CACHE_LINE_SIZE*4]
+
+ cmp r2, #MEMCPY_BLOCK_SIZE_SMALL
+ blo .L_reversed_memcpy_neon_pld_near
+ cmp r2, #MEMCPY_BLOCK_SIZE_MID
+ blo .L_reversed_memcpy_neon_pld_mid
+ b .L_reversed_memcpy_neon_pld_far
+
+.L_reversed_memcpy_neon_pld_near:
+ /* less than 128 bytes? */
+ subs r2, r2, #128
+ blo 1f
+ sub r1, r1, #32
+ sub r0, r0, #32
+ mov r3, #-32
+ .align 4
+0:
+ /* copy 128 bytes in each loop */
+ subs r2, r2, #128
+
+ /* preload to cache */
+ pld [r1, #-(PREFETCH_DISTANCE_NEAR+CACHE_LINE_SIZE*2)+32]
+ /* copy a cache line */
+ vld1.8 {q0, q1}, [r1], r3
+ vst1.8 {q0, q1}, [r0, :256], r3
+ vld1.8 {q0, q1}, [r1], r3
+ vst1.8 {q0, q1}, [r0, :256], r3
+
+ /* preload to cache */
+ pld [r1, #-(PREFETCH_DISTANCE_NEAR+CACHE_LINE_SIZE*2)+32]
+ /* copy a cache line */
+ vld1.8 {q0, q1}, [r1], r3
+ vst1.8 {q0, q1}, [r0, :256], r3
+ vld1.8 {q0, q1}, [r1], r3
+ vst1.8 {q0, q1}, [r0, :256], r3
+
+ bhs 0b
+ add r1, r1, #32
+ add r0, r0, #32
+1:
+ adds r2, r2, #128
+ bne .L_reversed_memcpy_lt_128bytes
+ pop {r0, pc}
+
+.L_reversed_memcpy_neon_pld_mid:
+ subs r2, r2, #128
+ sub r1, r1, #32
+ sub r0, r0, #32
+ mov r3, #-32
+ .align 4
+0:
+ /* copy 128 bytes in each loop */
+ subs r2, r2, #128
+
+ /* preload to cache */
+ pld [r1, #-(PREFETCH_DISTANCE_MID+CACHE_LINE_SIZE)+32]
+ /* copy a cache line */
+ vld1.8 {q0, q1}, [r1], r3
+ vst1.8 {q0, q1}, [r0, :256], r3
+ vld1.8 {q0, q1}, [r1], r3
+ vst1.8 {q0, q1}, [r0, :256], r3
+
+ /* preload to cache */
+ pld [r1, #-(PREFETCH_DISTANCE_MID+CACHE_LINE_SIZE)+32]
+ /* copy a cache line */
+ vld1.8 {q0, q1}, [r1], r3
+ vst1.8 {q0, q1}, [r0, :256], r3
+ vld1.8 {q0, q1}, [r1], r3
+ vst1.8 {q0, q1}, [r0, :256], r3
+
+ bhs 0b
+ add r1, r1, #32
+ add r0, r0, #32
+1:
+ adds r2, r2, #128
+ bne .L_reversed_memcpy_lt_128bytes
+ pop {r0, pc}
+
+.L_reversed_memcpy_neon_pld_far:
+ sub r2, r2, #128
+ sub r0, r0, #128
+ sub r1, r1, #128
+ .align 4
+0:
+ /* copy 128 bytes in each loop */
+ subs r2, r2, #128
+
+ /* preload to cache */
+ pld [r1, #-(PREFETCH_DISTANCE_FAR+CACHE_LINE_SIZE*2)+128]
+ pld [r1, #-(PREFETCH_DISTANCE_FAR+CACHE_LINE_SIZE)+128]
+ /* read */
+ vld1.8 {q0, q1}, [r1]!
+ vld1.8 {q2, q3}, [r1]!
+ vld1.8 {q8, q9}, [r1]!
+ vld1.8 {q10, q11}, [r1]!
+ /* write */
+ vst1.8 {q0, q1}, [r0, :256]!
+ vst1.8 {q2, q3}, [r0, :256]!
+ vst1.8 {q8, q9}, [r0, :256]!
+ vst1.8 {q10, q11}, [r0, :256]!
+
+ sub r0, r0, #256
+ sub r1, r1, #256
+ bhs 0b
+ add r0, r0, #128
+ add r1, r1, #128
+1:
+ adds r2, r2, #128
+ bne .L_reversed_memcpy_lt_128bytes
+ pop {r0, pc}
+
+.L_reversed_memcpy_lt_128bytes:
+6: /* copy 64 bytes */
+ movs ip, r2, lsl #26
+ bcc 5f
+ sub r1, r1, #32
+ sub r0, r0, #32
+ vld1.8 {q0, q1}, [r1]
+ vst1.8 {q0, q1}, [r0]
+ sub r1, r1, #32
+ sub r0, r0, #32
+ vld1.8 {q0, q1}, [r1]
+ vst1.8 {q0, q1}, [r0]
+5: /* copy 32 bytes */
+ bpl 4f
+ sub r1, r1, #32
+ sub r0, r0, #32
+ vld1.8 {q0, q1}, [r1]
+ vst1.8 {q0, q1}, [r0]
+.L_reversed_memcpy_lt_32bytes:
+4: /* copy 16 bytes */
+ movs ip, r2, lsl #28
+ bcc 3f
+ sub r1, r1, #16
+ sub r0, r0, #16
+ vld1.8 {q0}, [r1]
+ vst1.8 {q0}, [r0]
+3: /* copy 8 bytes */
+ bpl 2f
+ sub r1, r1, #8
+ sub r0, r0, #8
+ vld1.8 {d0}, [r1]
+ vst1.8 {d0}, [r0]
+2: /* copy 4 bytes */
+ ands ip, r2, #0x4
+ beq 1f
+ sub r1, r1, #4
+ sub r0, r0, #4
+ vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]
+ vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]
+1: /* copy 2 bytes */
+ movs ip, r2, lsl #31
+ ldrbcs ip, [r1, #-1]!
+ strbcs ip, [r0, #-1]!
+ ldrbcs ip, [r1, #-1]!
+ strbcs ip, [r0, #-1]!
+0: /* copy 1 byte */
+ ldrbmi ip, [r1, #-1]!
+ strbmi ip, [r0, #-1]!
+
+ pop {r0, pc}
+
+END(memmove_a15)
diff --git a/libc/arch-arm/cortex-a53/bionic/memmove.S b/libc/arch-arm/cortex-a53/bionic/memmove.S
deleted file mode 100644
index 741acb3..0000000
--- a/libc/arch-arm/cortex-a53/bionic/memmove.S
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#define MEMMOVE memmove_a53
-#define MEMCPY __memcpy_a53
-
-#include <arch-arm/denver/bionic/memmove.S>
diff --git a/libc/arch-arm/denver/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a55/bionic/__strcat_chk.S
similarity index 96%
rename from libc/arch-arm/denver/bionic/__strcat_chk.S
rename to libc/arch-arm/cortex-a55/bionic/__strcat_chk.S
index 1899908..eec1005 100644
--- a/libc/arch-arm/denver/bionic/__strcat_chk.S
+++ b/libc/arch-arm/cortex-a55/bionic/__strcat_chk.S
@@ -40,7 +40,7 @@
// Get the length of src string, then get the source of the dst string.
// Check that the two lengths together don't exceed the threshold, then
// do a memcpy of the data.
-ENTRY(__strcat_chk_denver)
+ENTRY(__strcat_chk_a55)
pld [r0, #0]
push {r0, lr}
.cfi_def_cfa_offset 8
@@ -190,9 +190,9 @@
mov r2, r4
add r0, r0, r3
pop {r4, r5}
-END(__strcat_chk_denver)
+END(__strcat_chk_a55)
-#define MEMCPY_BASE __strcat_chk_denver_memcpy_base
-#define MEMCPY_BASE_ALIGNED __strcat_chk_denver_memcpy_base_aligned
+#define MEMCPY_BASE __strcat_chk_a55_memcpy_base
+#define MEMCPY_BASE_ALIGNED __strcat_chk_a55_memcpy_base_aligned
#include "memcpy_base.S"
diff --git a/libc/arch-arm/denver/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a55/bionic/__strcpy_chk.S
similarity index 95%
rename from libc/arch-arm/denver/bionic/__strcpy_chk.S
rename to libc/arch-arm/cortex-a55/bionic/__strcpy_chk.S
index 9910c76..dfcc589 100644
--- a/libc/arch-arm/denver/bionic/__strcpy_chk.S
+++ b/libc/arch-arm/cortex-a55/bionic/__strcpy_chk.S
@@ -39,7 +39,7 @@
// Get the length of the source string first, then do a memcpy of the data
// instead of a strcpy.
-ENTRY(__strcpy_chk_denver)
+ENTRY(__strcpy_chk_a55)
pld [r0, #0]
push {r0, lr}
.cfi_def_cfa_offset 8
@@ -161,8 +161,8 @@
bhi __strcpy_chk_fail
// Fall through into the memcpy_base function.
-END(__strcpy_chk_denver)
+END(__strcpy_chk_a55)
-#define MEMCPY_BASE __strcpy_chk_denver_memcpy_base
-#define MEMCPY_BASE_ALIGNED __strcpy_chk_denver_memcpy_base_aligned
+#define MEMCPY_BASE __strcpy_chk_a55_memcpy_base
+#define MEMCPY_BASE_ALIGNED __strcpy_chk_a55_memcpy_base_aligned
#include "memcpy_base.S"
diff --git a/libc/arch-arm/denver/bionic/memcpy.S b/libc/arch-arm/cortex-a55/bionic/memcpy.S
similarity index 95%
rename from libc/arch-arm/denver/bionic/memcpy.S
rename to libc/arch-arm/cortex-a55/bionic/memcpy.S
index 5edee1e..ac9675f 100644
--- a/libc/arch-arm/denver/bionic/memcpy.S
+++ b/libc/arch-arm/cortex-a55/bionic/memcpy.S
@@ -65,14 +65,14 @@
// arch. The code generated is exactly the same.
.arch armv7-a
-ENTRY(__memcpy_denver)
+ENTRY(__memcpy_a55)
pld [r1, #64]
push {r0, lr}
.cfi_def_cfa_offset 8
.cfi_rel_offset r0, 0
.cfi_rel_offset lr, 4
-END(__memcpy_denver)
+END(__memcpy_a55)
-#define MEMCPY_BASE __memcpy_base_denver
-#define MEMCPY_BASE_ALIGNED __memcpy_base_aligned_denver
+#define MEMCPY_BASE __memcpy_base_a55
+#define MEMCPY_BASE_ALIGNED __memcpy_base_aligned_a55
#include "memcpy_base.S"
diff --git a/libc/arch-arm/denver/bionic/memcpy_base.S b/libc/arch-arm/cortex-a55/bionic/memcpy_base.S
similarity index 100%
rename from libc/arch-arm/denver/bionic/memcpy_base.S
rename to libc/arch-arm/cortex-a55/bionic/memcpy_base.S
diff --git a/libc/arch-arm/cortex-a7/bionic/memmove.S b/libc/arch-arm/cortex-a7/bionic/memmove.S
deleted file mode 100644
index 17ead5a..0000000
--- a/libc/arch-arm/cortex-a7/bionic/memmove.S
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#define MEMMOVE memmove_a7
-#define MEMCPY __memcpy_a7
-
-#include <arch-arm/denver/bionic/memmove.S>
diff --git a/libc/arch-arm/cortex-a9/bionic/memmove.S b/libc/arch-arm/cortex-a9/bionic/memmove.S
deleted file mode 100644
index a457633..0000000
--- a/libc/arch-arm/cortex-a9/bionic/memmove.S
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#define MEMMOVE memmove_a9
-#define MEMCPY __memcpy_a9
-
-#include <arch-arm/denver/bionic/memmove.S>
diff --git a/libc/arch-arm/denver/bionic/memmove.S b/libc/arch-arm/denver/bionic/memmove.S
deleted file mode 100644
index 13c90ef..0000000
--- a/libc/arch-arm/denver/bionic/memmove.S
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- * All rights reserved.
- * Copyright (c) 2013-2014 NVIDIA Corporation. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <private/bionic_asm.h>
-
-#ifndef MEMMOVE
-# define MEMMOVE memmove_denver
-#endif
-
-#ifndef MEMCPY
-# define MEMCPY __memcpy_denver
-#endif
-
- .text
- .syntax unified
- .fpu neon
-
-#define CACHE_LINE_SIZE (64)
-#define MEMCPY_BLOCK_SIZE_SMALL (32768)
-#define MEMCPY_BLOCK_SIZE_MID (1048576)
-#define PREFETCH_DISTANCE_NEAR (CACHE_LINE_SIZE*4)
-#define PREFETCH_DISTANCE_MID (CACHE_LINE_SIZE*4)
-#define PREFETCH_DISTANCE_FAR (CACHE_LINE_SIZE*16)
-
-ENTRY(MEMMOVE)
- cmp r2, #0
- cmpne r0, r1
- bxeq lr
- subs r3, r0, r1
- bls .L_jump_to_memcpy
- cmp r2, r3
- bhi .L_reversed_memcpy
-
-.L_jump_to_memcpy:
- b MEMCPY
-
-.L_reversed_memcpy:
- push {r0, lr}
- .cfi_def_cfa_offset 8
- .cfi_rel_offset r0, 0
- .cfi_rel_offset lr, 4
-
- add r0, r0, r2
- add r1, r1, r2
-
- /* preload next cache line */
- pld [r1, #-CACHE_LINE_SIZE]
- pld [r1, #-CACHE_LINE_SIZE*2]
-
-.L_reversed_memcpy_align_dest:
- /* Deal with very small blocks (< 32bytes) asap */
- cmp r2, #32
- blo .L_reversed_memcpy_lt_32bytes
- /* no need to align if len < 128 bytes */
- cmp r2, #128
- blo .L_reversed_memcpy_lt_128bytes
- /* align destination to 64 bytes (1 cache line) */
- ands r3, r0, #0x3f
- beq .L_reversed_memcpy_dispatch
- sub r2, r2, r3
-0: /* copy 1 byte */
- movs ip, r3, lsl #31
- ldrbmi ip, [r1, #-1]!
- strbmi ip, [r0, #-1]!
-1: /* copy 2 bytes */
- ldrbcs ip, [r1, #-1]!
- strbcs ip, [r0, #-1]!
- ldrbcs ip, [r1, #-1]!
- strbcs ip, [r0, #-1]!
-2: /* copy 4 bytes */
- movs ip, r3, lsl #29
- bpl 3f
- sub r1, r1, #4
- sub r0, r0, #4
- vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]
- vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]
-3: /* copy 8 bytes */
- bcc 4f
- sub r1, r1, #8
- sub r0, r0, #8
- vld1.8 {d0}, [r1]
- vst1.8 {d0}, [r0, :64]
-4: /* copy 16 bytes */
- movs ip, r3, lsl #27
- bpl 5f
- sub r1, r1, #16
- sub r0, r0, #16
- vld1.8 {q0}, [r1]
- vst1.8 {q0}, [r0, :128]
-5: /* copy 32 bytes */
- bcc .L_reversed_memcpy_dispatch
- sub r1, r1, #32
- sub r0, r0, #32
- vld1.8 {q0, q1}, [r1]
- vst1.8 {q0, q1}, [r0, :256]
-
-.L_reversed_memcpy_dispatch:
- /* preload more cache lines */
- pld [r1, #-CACHE_LINE_SIZE*3]
- pld [r1, #-CACHE_LINE_SIZE*4]
-
- cmp r2, #MEMCPY_BLOCK_SIZE_SMALL
- blo .L_reversed_memcpy_neon_pld_near
- cmp r2, #MEMCPY_BLOCK_SIZE_MID
- blo .L_reversed_memcpy_neon_pld_mid
- b .L_reversed_memcpy_neon_pld_far
-
-.L_reversed_memcpy_neon_pld_near:
- /* less than 128 bytes? */
- subs r2, r2, #128
- blo 1f
- sub r1, r1, #32
- sub r0, r0, #32
- mov r3, #-32
- .align 4
-0:
- /* copy 128 bytes in each loop */
- subs r2, r2, #128
-
- /* preload to cache */
- pld [r1, #-(PREFETCH_DISTANCE_NEAR+CACHE_LINE_SIZE*2)+32]
- /* copy a cache line */
- vld1.8 {q0, q1}, [r1], r3
- vst1.8 {q0, q1}, [r0, :256], r3
- vld1.8 {q0, q1}, [r1], r3
- vst1.8 {q0, q1}, [r0, :256], r3
-
- /* preload to cache */
- pld [r1, #-(PREFETCH_DISTANCE_NEAR+CACHE_LINE_SIZE*2)+32]
- /* copy a cache line */
- vld1.8 {q0, q1}, [r1], r3
- vst1.8 {q0, q1}, [r0, :256], r3
- vld1.8 {q0, q1}, [r1], r3
- vst1.8 {q0, q1}, [r0, :256], r3
-
- bhs 0b
- add r1, r1, #32
- add r0, r0, #32
-1:
- adds r2, r2, #128
- bne .L_reversed_memcpy_lt_128bytes
- pop {r0, pc}
-
-.L_reversed_memcpy_neon_pld_mid:
- subs r2, r2, #128
- sub r1, r1, #32
- sub r0, r0, #32
- mov r3, #-32
- .align 4
-0:
- /* copy 128 bytes in each loop */
- subs r2, r2, #128
-
- /* preload to cache */
- pld [r1, #-(PREFETCH_DISTANCE_MID+CACHE_LINE_SIZE)+32]
- /* copy a cache line */
- vld1.8 {q0, q1}, [r1], r3
- vst1.8 {q0, q1}, [r0, :256], r3
- vld1.8 {q0, q1}, [r1], r3
- vst1.8 {q0, q1}, [r0, :256], r3
-
- /* preload to cache */
- pld [r1, #-(PREFETCH_DISTANCE_MID+CACHE_LINE_SIZE)+32]
- /* copy a cache line */
- vld1.8 {q0, q1}, [r1], r3
- vst1.8 {q0, q1}, [r0, :256], r3
- vld1.8 {q0, q1}, [r1], r3
- vst1.8 {q0, q1}, [r0, :256], r3
-
- bhs 0b
- add r1, r1, #32
- add r0, r0, #32
-1:
- adds r2, r2, #128
- bne .L_reversed_memcpy_lt_128bytes
- pop {r0, pc}
-
-.L_reversed_memcpy_neon_pld_far:
- sub r2, r2, #128
- sub r0, r0, #128
- sub r1, r1, #128
- .align 4
-0:
- /* copy 128 bytes in each loop */
- subs r2, r2, #128
-
- /* preload to cache */
- pld [r1, #-(PREFETCH_DISTANCE_FAR+CACHE_LINE_SIZE*2)+128]
- pld [r1, #-(PREFETCH_DISTANCE_FAR+CACHE_LINE_SIZE)+128]
- /* read */
- vld1.8 {q0, q1}, [r1]!
- vld1.8 {q2, q3}, [r1]!
- vld1.8 {q8, q9}, [r1]!
- vld1.8 {q10, q11}, [r1]!
- /* write */
- vst1.8 {q0, q1}, [r0, :256]!
- vst1.8 {q2, q3}, [r0, :256]!
- vst1.8 {q8, q9}, [r0, :256]!
- vst1.8 {q10, q11}, [r0, :256]!
-
- sub r0, r0, #256
- sub r1, r1, #256
- bhs 0b
- add r0, r0, #128
- add r1, r1, #128
-1:
- adds r2, r2, #128
- bne .L_reversed_memcpy_lt_128bytes
- pop {r0, pc}
-
-.L_reversed_memcpy_lt_128bytes:
-6: /* copy 64 bytes */
- movs ip, r2, lsl #26
- bcc 5f
- sub r1, r1, #32
- sub r0, r0, #32
- vld1.8 {q0, q1}, [r1]
- vst1.8 {q0, q1}, [r0]
- sub r1, r1, #32
- sub r0, r0, #32
- vld1.8 {q0, q1}, [r1]
- vst1.8 {q0, q1}, [r0]
-5: /* copy 32 bytes */
- bpl 4f
- sub r1, r1, #32
- sub r0, r0, #32
- vld1.8 {q0, q1}, [r1]
- vst1.8 {q0, q1}, [r0]
-.L_reversed_memcpy_lt_32bytes:
-4: /* copy 16 bytes */
- movs ip, r2, lsl #28
- bcc 3f
- sub r1, r1, #16
- sub r0, r0, #16
- vld1.8 {q0}, [r1]
- vst1.8 {q0}, [r0]
-3: /* copy 8 bytes */
- bpl 2f
- sub r1, r1, #8
- sub r0, r0, #8
- vld1.8 {d0}, [r1]
- vst1.8 {d0}, [r0]
-2: /* copy 4 bytes */
- ands ip, r2, #0x4
- beq 1f
- sub r1, r1, #4
- sub r0, r0, #4
- vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]
- vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]
-1: /* copy 2 bytes */
- movs ip, r2, lsl #31
- ldrbcs ip, [r1, #-1]!
- strbcs ip, [r0, #-1]!
- ldrbcs ip, [r1, #-1]!
- strbcs ip, [r0, #-1]!
-0: /* copy 1 byte */
- ldrbmi ip, [r1, #-1]!
- strbmi ip, [r0, #-1]!
-
- pop {r0, pc}
-
-END(MEMMOVE)
diff --git a/libc/arch-arm/denver/bionic/memset.S b/libc/arch-arm/denver/bionic/memset.S
deleted file mode 100644
index 198ecf3..0000000
--- a/libc/arch-arm/denver/bionic/memset.S
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <private/bionic_asm.h>
-
- /*
- * Optimized memset() for ARM.
- *
- * memset() returns its first argument.
- */
-
- .cpu cortex-a15
- .fpu neon
- .syntax unified
-
-ENTRY(__memset_chk_denver)
- cmp r2, r3
- bls memset
-
- // Preserve lr for backtrace.
- push {lr}
- .cfi_def_cfa_offset 4
- .cfi_rel_offset lr, 0
-
- bl __memset_chk_fail
-END(__memset_chk_denver)
-
-ENTRY(memset_denver)
- pldw [r0]
- mov r3, r0
-
- // Duplicate the low byte of r1
- mov r1, r1, lsl #24
- orr r1, r1, r1, lsr #8
- orr r1, r1, r1, lsr #16
-
- cmp r2, #16
- blo .L_less_than_16
-
- // This section handles regions 16 bytes or larger
- //
- // Use aligned vst1.8 and vstm when possible. Register values will be:
- // ip is scratch
- // q0, q1, and r1 contain the memset value
- // r2 is the number of bytes to set
- // r3 is the advancing destination pointer
- vdup.32 q0, r1
-
- ands ip, r3, 0xF
- beq .L_memset_aligned
-
- // Align dest pointer to 16-byte boundary.
- pldw [r0, #64]
- rsb ip, ip, #16
-
- // Pre-adjust the byte count to reflect post-aligment value. Expecting
- // 8-byte alignment to be rather common so we special case that one.
- sub r2, r2, ip
-
- /* set 1 byte */
- tst ip, #1
- it ne
- strbne r1, [r3], #1
- /* set 2 bytes */
- tst ip, #2
- it ne
- strhne r1, [r3], #2
- /* set 4 bytes */
- movs ip, ip, lsl #29
- it mi
- strmi r1, [r3], #4
- /* set 8 bytes */
- itt cs
- strcs r1, [r3], #4
- strcs r1, [r3], #4
-
-.L_memset_aligned:
- // Destination is now 16-byte aligned. Determine how to handle
- // remaining bytes.
- vmov q1, q0
- cmp r2, #128
- blo .L_less_than_128
-
- // We need to set a larger block of memory. Use four Q regs to
- // set a full cache line in one instruction. Pre-decrement
- // r2 to simplify end-of-loop detection
- vmov q2, q0
- vmov q3, q0
- pldw [r0, #128]
- sub r2, r2, #128
- .align 4
-.L_memset_loop_128:
- pldw [r3, #192]
- vstm r3!, {q0, q1, q2, q3}
- vstm r3!, {q0, q1, q2, q3}
- subs r2, r2, #128
- bhs .L_memset_loop_128
-
- // Un-bias r2 so it contains the number of bytes left. Early
- // exit if we are done.
- adds r2, r2, #128
- beq 2f
-
- .align 4
-.L_less_than_128:
- // set 64 bytes
- movs ip, r2, lsl #26
- bcc 1f
- vst1.8 {q0, q1}, [r3, :128]!
- vst1.8 {q0, q1}, [r3, :128]!
- beq 2f
-1:
- // set 32 bytes
- bpl 1f
- vst1.8 {q0, q1}, [r3, :128]!
-1:
- // set 16 bytes
- movs ip, r2, lsl #28
- bcc 1f
- vst1.8 {q0}, [r3, :128]!
- beq 2f
-1:
- // set 8 bytes
- bpl 1f
- vst1.8 {d0}, [r3, :64]!
-1:
- // set 4 bytes
- tst r2, #4
- it ne
- strne r1, [r3], #4
-1:
- // set 2 bytes
- movs ip, r2, lsl #31
- it cs
- strhcs r1, [r3], #2
- // set 1 byte
- it mi
- strbmi r1, [r3]
-2:
- bx lr
-
-.L_less_than_16:
- // Store up to 15 bytes without worrying about byte alignment
- movs ip, r2, lsl #29
- bcc 1f
- str r1, [r3], #4
- str r1, [r3], #4
- beq 2f
-1:
- it mi
- strmi r1, [r3], #4
- movs ip, r2, lsl #31
- it mi
- strbmi r1, [r3], #1
- itt cs
- strbcs r1, [r3], #1
- strbcs r1, [r3]
-2:
- bx lr
-END(memset_denver)
diff --git a/libc/arch-arm/dynamic_function_dispatch.cpp b/libc/arch-arm/dynamic_function_dispatch.cpp
index 822581c..640f330 100644
--- a/libc/arch-arm/dynamic_function_dispatch.cpp
+++ b/libc/arch-arm/dynamic_function_dispatch.cpp
@@ -38,7 +38,6 @@
kCortexA9,
kCortexA53,
kCortexA55,
- kDenver,
kKrait,
kKryo,
};
@@ -51,6 +50,7 @@
static constexpr CpuVariantNames cpu_variant_names[] = {
{"cortex-a76", kCortexA55},
+ {"kryo385", kCortexA55},
{"cortex-a75", kCortexA55},
{"kryo", kKryo},
{"cortex-a73", kCortexA55},
@@ -59,7 +59,6 @@
{"krait", kKrait},
{"cortex-a9", kCortexA9},
{"cortex-a7", kCortexA7},
- {"denver", kDenver},
// kUnknown indicates the end of this array.
{"", kUnknown},
};
@@ -157,23 +156,7 @@
typedef void* memmove_func(void* __dst, const void* __src, size_t __n);
DEFINE_IFUNC(memmove) {
- switch(get_cpu_variant()) {
- case kCortexA7:
- RETURN_FUNC(memmove_func, memmove_a7);
- case kCortexA9:
- RETURN_FUNC(memmove_func, memmove_a9);
- case kKrait:
- RETURN_FUNC(memmove_func, memmove_krait);
- case kCortexA53:
- RETURN_FUNC(memmove_func, memmove_a53);
- case kCortexA55:
- case kDenver:
- RETURN_FUNC(memmove_func, memmove_denver);
- case kKryo:
- RETURN_FUNC(memmove_func, memmove_kryo);
- default:
- RETURN_FUNC(memmove_func, memmove_a15);
- }
+ RETURN_FUNC(memmove_func, memmove_a15);
}
typedef void* memcpy_func(void*, const void*, size_t);
@@ -181,6 +164,26 @@
return memmove_resolver();
}
+typedef void* __memcpy_func(void*, const void*, size_t);
+DEFINE_IFUNC(__memcpy) {
+ switch(get_cpu_variant()) {
+ case kCortexA7:
+ RETURN_FUNC(__memcpy_func, __memcpy_a7);
+ case kCortexA9:
+ RETURN_FUNC(__memcpy_func, __memcpy_a9);
+ case kKrait:
+ RETURN_FUNC(__memcpy_func, __memcpy_krait);
+ case kCortexA53:
+ RETURN_FUNC(__memcpy_func, __memcpy_a53);
+ case kCortexA55:
+ RETURN_FUNC(__memcpy_func, __memcpy_a55);
+ case kKryo:
+ RETURN_FUNC(__memcpy_func, __memcpy_kryo);
+ default:
+ RETURN_FUNC(__memcpy_func, __memcpy_a15);
+ }
+}
+
typedef void* __memset_chk_func(void* s, int c, size_t n, size_t n2);
DEFINE_IFUNC(__memset_chk) {
switch(get_cpu_variant()) {
@@ -193,8 +196,6 @@
RETURN_FUNC(__memset_chk_func, __memset_chk_a9);
case kKrait:
RETURN_FUNC(__memset_chk_func, __memset_chk_krait);
- case kDenver:
- RETURN_FUNC(__memset_chk_func, __memset_chk_denver);
default:
RETURN_FUNC(__memset_chk_func, __memset_chk_a15);
}
@@ -212,8 +213,6 @@
RETURN_FUNC(memset_func, memset_a9);
case kKrait:
RETURN_FUNC(memset_func, memset_krait);
- case kDenver:
- RETURN_FUNC(memset_func, memset_denver);
default:
RETURN_FUNC(memset_func, memset_a15);
}
@@ -242,8 +241,7 @@
case kCortexA53:
RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a53);
case kCortexA55:
- case kDenver:
- RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_denver);
+ RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a55);
default:
RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a15);
}
@@ -282,8 +280,7 @@
case kCortexA53:
RETURN_FUNC(__strcat_chk_func, __strcat_chk_a53);
case kCortexA55:
- case kDenver:
- RETURN_FUNC(__strcat_chk_func, __strcat_chk_denver);
+ RETURN_FUNC(__strcat_chk_func, __strcat_chk_a55);
default:
RETURN_FUNC(__strcat_chk_func, __strcat_chk_a15);
}
diff --git a/libc/arch-arm/krait/bionic/memmove.S b/libc/arch-arm/krait/bionic/memmove.S
deleted file mode 100644
index af85ea6..0000000
--- a/libc/arch-arm/krait/bionic/memmove.S
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#define MEMMOVE memmove_krait
-#define MEMCPY __memcpy_krait
-
-#include <arch-arm/denver/bionic/memmove.S>
diff --git a/libc/arch-arm/kryo/bionic/memmove.S b/libc/arch-arm/kryo/bionic/memmove.S
deleted file mode 100644
index d0fdd8f..0000000
--- a/libc/arch-arm/kryo/bionic/memmove.S
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#define MEMMOVE memmove_kryo
-#define MEMCPY __memcpy_kryo
-
-#include <arch-arm/denver/bionic/memmove.S>
diff --git a/libc/arch-x86/bionic/__libc_init_sysinfo.cpp b/libc/arch-x86/bionic/__libc_init_sysinfo.cpp
index 1bb7ab0..849d253 100644
--- a/libc/arch-x86/bionic/__libc_init_sysinfo.cpp
+++ b/libc/arch-x86/bionic/__libc_init_sysinfo.cpp
@@ -26,14 +26,15 @@
* SUCH DAMAGE.
*/
-#include "private/KernelArgumentBlock.h"
+#include "private/bionic_auxv.h"
#include "private/bionic_globals.h"
// This file is compiled without stack protection, because it runs before TLS
// has been set up.
-__LIBC_HIDDEN__ void __libc_init_sysinfo(KernelArgumentBlock& args) {
- __libc_sysinfo = reinterpret_cast<void*>(args.getauxval(AT_SYSINFO));
+__LIBC_HIDDEN__ void __libc_init_sysinfo() {
+ bool dummy;
+ __libc_sysinfo = reinterpret_cast<void*>(__bionic_getauxval(AT_SYSINFO, dummy));
}
// TODO: lose this function and just access __libc_sysinfo directly.
diff --git a/libc/bionic/__libc_init_main_thread.cpp b/libc/bionic/__libc_init_main_thread.cpp
index be9d32e..5abdc07 100644
--- a/libc/bionic/__libc_init_main_thread.cpp
+++ b/libc/bionic/__libc_init_main_thread.cpp
@@ -30,12 +30,12 @@
#include "private/KernelArgumentBlock.h"
#include "private/bionic_arc4random.h"
-#include "private/bionic_auxv.h"
#include "private/bionic_defs.h"
#include "private/bionic_globals.h"
#include "private/bionic_ssp.h"
#include "pthread_internal.h"
+extern "C" pid_t __getpid();
extern "C" int __set_tid_address(int* tid_address);
// Declared in "private/bionic_ssp.h".
@@ -61,23 +61,36 @@
// -fno-stack-protector because it's responsible for setting up the main
// thread's TLS (which stack protector relies on).
+// Do enough setup to:
+// - Let the dynamic linker invoke system calls (and access errno)
+// - Ensure that TLS access functions (__get_{tls,thread}) never return NULL
+// - Allow the stack protector to work (with a zero cookie)
+// Avoid doing much more because, when this code is called within the dynamic
+// linker, the linker binary hasn't been relocated yet, so certain kinds of code
+// are hazardous, such as accessing non-hidden global variables.
__BIONIC_WEAK_FOR_NATIVE_BRIDGE
-void __libc_init_main_thread(KernelArgumentBlock& args) {
- __libc_auxv = args.auxv;
+void __libc_init_main_thread_early(KernelArgumentBlock& args) {
+ __libc_shared_globals()->auxv = args.auxv;
#if defined(__i386__)
- __libc_init_sysinfo(args);
+ __libc_init_sysinfo();
#endif
-
- // The -fstack-protector implementation uses TLS, so make sure that's
- // set up before we call any function that might get a stack check inserted.
- // TLS also needs to be set up before errno (and therefore syscalls) can be used.
__set_tls(main_thread.tls);
- if (!__init_tls(&main_thread)) async_safe_fatal("failed to initialize TLS: %s", strerror(errno));
+ __init_tls(&main_thread);
+ main_thread.tid = __getpid();
+ main_thread.set_cached_pid(main_thread.tid);
+}
+
+// Finish initializing the main thread.
+__BIONIC_WEAK_FOR_NATIVE_BRIDGE
+void __libc_init_main_thread_late() {
+ main_thread.bionic_tls = __allocate_bionic_tls();
+ if (main_thread.bionic_tls == nullptr) {
+ // Avoid strerror because it might need bionic_tls.
+ async_safe_fatal("failed to allocate bionic_tls: error %d", errno);
+ }
// Tell the kernel to clear our tid field when we exit, so we're like any other pthread.
- // As a side-effect, this tells us our pid (which is the same as the main thread's tid).
- main_thread.tid = __set_tid_address(&main_thread.tid);
- main_thread.set_cached_pid(main_thread.tid);
+ __set_tid_address(&main_thread.tid);
// We don't want to free the main thread's stack even when the main thread exits
// because things like environment variables with global scope live on it.
@@ -96,8 +109,8 @@
// The TLS stack guard is set from the global, so ensure that we've initialized the global
// before we initialize the TLS. Dynamic executables will initialize their copy of the global
// stack protector from the one in the main thread's TLS.
- __libc_safe_arc4random_buf(&__stack_chk_guard, sizeof(__stack_chk_guard), args);
- __init_thread_stack_guard(&main_thread);
+ __libc_safe_arc4random_buf(&__stack_chk_guard, sizeof(__stack_chk_guard));
+ __init_tls_stack_guard(&main_thread);
__init_thread(&main_thread);
diff --git a/libc/bionic/bionic_arc4random.cpp b/libc/bionic/bionic_arc4random.cpp
index fa2617f..74ac43c 100644
--- a/libc/bionic/bionic_arc4random.cpp
+++ b/libc/bionic/bionic_arc4random.cpp
@@ -35,9 +35,7 @@
#include <async_safe/log.h>
-#include "private/KernelArgumentBlock.h"
-
-void __libc_safe_arc4random_buf(void* buf, size_t n, KernelArgumentBlock& args) {
+void __libc_safe_arc4random_buf(void* buf, size_t n) {
// Only call arc4random_buf once we have `/dev/urandom` because getentropy(3)
// will fall back to using `/dev/urandom` if getrandom(2) fails, and abort if
// if can't use `/dev/urandom`.
@@ -53,7 +51,7 @@
16 - at_random_bytes_consumed, n);
}
- memcpy(buf, reinterpret_cast<char*>(args.getauxval(AT_RANDOM)) + at_random_bytes_consumed, n);
+ memcpy(buf, reinterpret_cast<char*>(getauxval(AT_RANDOM)) + at_random_bytes_consumed, n);
at_random_bytes_consumed += n;
return;
}
diff --git a/libc/bionic/getauxval.cpp b/libc/bionic/getauxval.cpp
index 607e89c..c8f867b 100644
--- a/libc/bionic/getauxval.cpp
+++ b/libc/bionic/getauxval.cpp
@@ -30,17 +30,27 @@
#include <sys/cdefs.h>
#include <sys/auxv.h>
#include <private/bionic_auxv.h>
+#include <private/bionic_globals.h>
#include <elf.h>
#include <errno.h>
-__LIBC_HIDDEN__ ElfW(auxv_t)* __libc_auxv = nullptr;
-
-extern "C" unsigned long int getauxval(unsigned long int type) {
- for (ElfW(auxv_t)* v = __libc_auxv; v->a_type != AT_NULL; ++v) {
+// This function needs to be safe to call before TLS is set up, so it can't
+// access errno or the stack protector.
+__attribute__((no_stack_protector))
+__LIBC_HIDDEN__ unsigned long __bionic_getauxval(unsigned long type, bool& exists) {
+ for (ElfW(auxv_t)* v = __libc_shared_globals()->auxv; v->a_type != AT_NULL; ++v) {
if (v->a_type == type) {
+ exists = true;
return v->a_un.a_val;
}
}
- errno = ENOENT;
+ exists = false;
return 0;
}
+
+extern "C" unsigned long getauxval(unsigned long type) {
+ bool exists;
+ unsigned long result = __bionic_getauxval(type, exists);
+ if (!exists) errno = ENOENT;
+ return result;
+}
diff --git a/libc/bionic/libc_init_common.cpp b/libc/bionic/libc_init_common.cpp
index 4702e1c..f1fbfa9 100644
--- a/libc/bionic/libc_init_common.cpp
+++ b/libc/bionic/libc_init_common.cpp
@@ -43,9 +43,7 @@
#include <async_safe/log.h>
-#include "private/KernelArgumentBlock.h"
#include "private/WriteProtected.h"
-#include "private/bionic_auxv.h"
#include "private/bionic_defs.h"
#include "private/bionic_globals.h"
#include "private/bionic_tls.h"
@@ -59,14 +57,14 @@
// Not public, but well-known in the BSDs.
const char* __progname;
-void __libc_init_globals(KernelArgumentBlock& args) {
+void __libc_init_globals() {
// Initialize libc globals that are needed in both the linker and in libc.
// In dynamic binaries, this is run at least twice for different copies of the
// globals, once for the linker's copy and once for the one in libc.so.
__libc_globals.initialize();
- __libc_globals.mutate([&args](libc_globals* globals) {
- __libc_init_vdso(globals, args);
- __libc_init_setjmp_cookie(globals, args);
+ __libc_globals.mutate([](libc_globals* globals) {
+ __libc_init_vdso(globals);
+ __libc_init_setjmp_cookie(globals);
});
}
diff --git a/libc/bionic/libc_init_common.h b/libc/bionic/libc_init_common.h
index 84b59ca..73f5817 100644
--- a/libc/bionic/libc_init_common.h
+++ b/libc/bionic/libc_init_common.h
@@ -50,9 +50,7 @@
#if defined(__cplusplus)
-class KernelArgumentBlock;
-
-__LIBC_HIDDEN__ void __libc_init_globals(KernelArgumentBlock& args);
+__LIBC_HIDDEN__ void __libc_init_globals();
__LIBC_HIDDEN__ void __libc_init_common();
diff --git a/libc/bionic/libc_init_dynamic.cpp b/libc/bionic/libc_init_dynamic.cpp
index 25d462f..af1b847 100644
--- a/libc/bionic/libc_init_dynamic.cpp
+++ b/libc/bionic/libc_init_dynamic.cpp
@@ -51,7 +51,6 @@
#include <elf.h>
#include "libc_init_common.h"
-#include "private/bionic_auxv.h"
#include "private/bionic_globals.h"
#include "private/bionic_macros.h"
#include "private/bionic_ssp.h"
@@ -78,13 +77,12 @@
// after __stack_chk_guard is initialized and therefore can safely have a stack
// protector.
__attribute__((noinline))
-static void __libc_preinit_impl(KernelArgumentBlock& args) {
- __libc_auxv = args.auxv;
+static void __libc_preinit_impl() {
#if defined(__i386__)
- __libc_init_sysinfo(args);
+ __libc_init_sysinfo();
#endif
- __libc_init_globals(args);
+ __libc_init_globals();
__libc_init_common();
// Hooks for various libraries to let them know that we're starting up.
@@ -100,17 +98,11 @@
// to run before any others (such as the jemalloc constructor), and lower
// is better (http://b/68046352).
__attribute__((constructor(1))) static void __libc_preinit() {
- // Read the kernel argument block pointer from TLS, then clear the slot so no
- // other initializer sees its value.
- void** tls = __get_tls();
- KernelArgumentBlock* args = static_cast<KernelArgumentBlock*>(tls[TLS_SLOT_BIONIC_PREINIT]);
- tls[TLS_SLOT_BIONIC_PREINIT] = nullptr;
-
// The linker has initialized its copy of the global stack_chk_guard, and filled in the main
// thread's TLS slot with that value. Initialize the local global stack guard with its value.
- __stack_chk_guard = reinterpret_cast<uintptr_t>(tls[TLS_SLOT_STACK_GUARD]);
+ __stack_chk_guard = reinterpret_cast<uintptr_t>(__get_tls()[TLS_SLOT_STACK_GUARD]);
- __libc_preinit_impl(*args);
+ __libc_preinit_impl();
}
// This function is called from the executable's _start entry point
diff --git a/libc/bionic/libc_init_static.cpp b/libc/bionic/libc_init_static.cpp
index 51fbe07..289b4a3 100644
--- a/libc/bionic/libc_init_static.cpp
+++ b/libc/bionic/libc_init_static.cpp
@@ -95,14 +95,12 @@
structors_array_t const * const structors) {
BIONIC_STOP_UNWIND;
+ // Initialize TLS early so system calls and errno work.
KernelArgumentBlock args(raw_args);
+ __libc_init_main_thread_early(args);
+ __libc_init_main_thread_late();
+ __libc_init_globals();
__libc_shared_globals()->init_progname = args.argv[0];
-
- // Initializing the globals requires TLS to be available for errno.
- __libc_init_main_thread(args);
-
- __libc_init_globals(args);
-
__libc_init_AT_SECURE(args.envp);
__libc_init_common();
diff --git a/libc/bionic/pthread_create.cpp b/libc/bionic/pthread_create.cpp
index 6f632e8..8e8d180 100644
--- a/libc/bionic/pthread_create.cpp
+++ b/libc/bionic/pthread_create.cpp
@@ -53,39 +53,43 @@
#endif
// This code is used both by each new pthread and the code that initializes the main thread.
-bool __init_tls(pthread_internal_t* thread) {
+__attribute__((no_stack_protector))
+void __init_tls(pthread_internal_t* thread) {
// Slot 0 must point to itself. The x86 Linux kernel reads the TLS from %fs:0.
thread->tls[TLS_SLOT_SELF] = thread->tls;
thread->tls[TLS_SLOT_THREAD_ID] = thread;
+}
+__attribute__((no_stack_protector))
+void __init_tls_stack_guard(pthread_internal_t* thread) {
+ // GCC looks in the TLS for the stack guard on x86, so copy it there from our global.
+ thread->tls[TLS_SLOT_STACK_GUARD] = reinterpret_cast<void*>(__stack_chk_guard);
+}
+
+bionic_tls* __allocate_bionic_tls() {
// Add a guard before and after.
size_t allocation_size = BIONIC_TLS_SIZE + (2 * PTHREAD_GUARD_SIZE);
void* allocation = mmap(nullptr, allocation_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (allocation == MAP_FAILED) {
async_safe_format_log(ANDROID_LOG_WARN, "libc",
"pthread_create failed: couldn't allocate TLS: %s", strerror(errno));
- return false;
+ return nullptr;
}
prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, allocation, allocation_size, "bionic TLS guard");
// Carve out the writable TLS section.
- thread->bionic_tls = reinterpret_cast<bionic_tls*>(static_cast<char*>(allocation) +
+ bionic_tls* result = reinterpret_cast<bionic_tls*>(static_cast<char*>(allocation) +
PTHREAD_GUARD_SIZE);
- if (mprotect(thread->bionic_tls, BIONIC_TLS_SIZE, PROT_READ | PROT_WRITE) != 0) {
+ if (mprotect(result, BIONIC_TLS_SIZE, PROT_READ | PROT_WRITE) != 0) {
async_safe_format_log(ANDROID_LOG_WARN, "libc",
"pthread_create failed: couldn't mprotect TLS: %s", strerror(errno));
munmap(allocation, allocation_size);
- return false;
+ return nullptr;
}
- prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, thread->bionic_tls, BIONIC_TLS_SIZE, "bionic TLS");
- return true;
-}
-
-void __init_thread_stack_guard(pthread_internal_t* thread) {
- // GCC looks in the TLS for the stack guard on x86, so copy it there from our global.
- thread->tls[TLS_SLOT_STACK_GUARD] = reinterpret_cast<void*>(__stack_chk_guard);
+ prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, result, BIONIC_TLS_SIZE, "bionic TLS");
+ return result;
}
static void __init_alternate_signal_stack(pthread_internal_t* thread) {
@@ -255,11 +259,15 @@
thread->mmap_size = mmap_size;
thread->attr = *attr;
- if (!__init_tls(thread)) {
+
+ thread->bionic_tls = __allocate_bionic_tls();
+ if (thread->bionic_tls == nullptr) {
if (thread->mmap_size != 0) munmap(thread->attr.stack_base, thread->mmap_size);
return EAGAIN;
}
- __init_thread_stack_guard(thread);
+
+ __init_tls(thread);
+ __init_tls_stack_guard(thread);
*threadp = thread;
*child_stack = stack_top;
diff --git a/libc/bionic/pthread_internal.h b/libc/bionic/pthread_internal.h
index 81b885a..5a5318d 100644
--- a/libc/bionic/pthread_internal.h
+++ b/libc/bionic/pthread_internal.h
@@ -152,10 +152,11 @@
pthread_key_data_t key_data[BIONIC_PTHREAD_KEY_COUNT];
};
-__LIBC_HIDDEN__ int __init_thread(pthread_internal_t* thread);
-__LIBC_HIDDEN__ bool __init_tls(pthread_internal_t* thread);
-__LIBC_HIDDEN__ void __init_thread_stack_guard(pthread_internal_t* thread);
+__LIBC_HIDDEN__ void __init_tls(pthread_internal_t* thread);
+__LIBC_HIDDEN__ void __init_tls_stack_guard(pthread_internal_t* thread);
+__LIBC_HIDDEN__ bionic_tls* __allocate_bionic_tls();
__LIBC_HIDDEN__ void __init_additional_stacks(pthread_internal_t*);
+__LIBC_HIDDEN__ int __init_thread(pthread_internal_t* thread);
__LIBC_HIDDEN__ pthread_t __pthread_internal_add(pthread_internal_t* thread);
__LIBC_HIDDEN__ pthread_internal_t* __pthread_internal_find(pthread_t pthread_id);
diff --git a/libc/bionic/setjmp_cookie.cpp b/libc/bionic/setjmp_cookie.cpp
index 41a439f..e2a3fc0 100644
--- a/libc/bionic/setjmp_cookie.cpp
+++ b/libc/bionic/setjmp_cookie.cpp
@@ -38,11 +38,10 @@
#include "private/bionic_arc4random.h"
#include "private/bionic_globals.h"
-#include "private/KernelArgumentBlock.h"
-void __libc_init_setjmp_cookie(libc_globals* globals, KernelArgumentBlock& args) {
+void __libc_init_setjmp_cookie(libc_globals* globals) {
long value;
- __libc_safe_arc4random_buf(&value, sizeof(value), args);
+ __libc_safe_arc4random_buf(&value, sizeof(value));
// Mask off the last bit to store the signal flag.
globals->setjmp_cookie = value & ~1;
diff --git a/libc/bionic/vdso.cpp b/libc/bionic/vdso.cpp
index c926a58..dbca9c0 100644
--- a/libc/bionic/vdso.cpp
+++ b/libc/bionic/vdso.cpp
@@ -20,11 +20,11 @@
#include <limits.h>
#include <link.h>
#include <string.h>
+#include <sys/auxv.h>
#include <sys/cdefs.h>
#include <sys/time.h>
#include <time.h>
#include <unistd.h>
-#include "private/KernelArgumentBlock.h"
static inline int vdso_return(int result) {
if (__predict_true(result == 0)) return 0;
@@ -73,7 +73,7 @@
return tv.tv_sec;
}
-void __libc_init_vdso(libc_globals* globals, KernelArgumentBlock& args) {
+void __libc_init_vdso(libc_globals* globals) {
auto&& vdso = globals->vdso;
vdso[VDSO_CLOCK_GETTIME] = { VDSO_CLOCK_GETTIME_SYMBOL, nullptr };
vdso[VDSO_CLOCK_GETRES] = { VDSO_CLOCK_GETRES_SYMBOL, nullptr };
@@ -81,7 +81,7 @@
vdso[VDSO_TIME] = { VDSO_TIME_SYMBOL, nullptr };
// Do we have a vdso?
- uintptr_t vdso_ehdr_addr = args.getauxval(AT_SYSINFO_EHDR);
+ uintptr_t vdso_ehdr_addr = getauxval(AT_SYSINFO_EHDR);
ElfW(Ehdr)* vdso_ehdr = reinterpret_cast<ElfW(Ehdr)*>(vdso_ehdr_addr);
if (vdso_ehdr == nullptr) {
return;
diff --git a/libc/private/bionic_arc4random.h b/libc/private/bionic_arc4random.h
index 0e9376e..cdc9b6d 100644
--- a/libc/private/bionic_arc4random.h
+++ b/libc/private/bionic_arc4random.h
@@ -31,13 +31,11 @@
#include <stddef.h>
-#include "private/KernelArgumentBlock.h"
-
// arc4random(3) aborts if it's unable to fetch entropy, which is always
// the case for init on devices. GCE kernels have a workaround to ensure
// sufficient entropy during early boot, but no device kernels do. This
// wrapper falls back to AT_RANDOM if the kernel doesn't have enough
// entropy for getrandom(2) or /dev/urandom.
-void __libc_safe_arc4random_buf(void* buf, size_t n, KernelArgumentBlock& args);
+void __libc_safe_arc4random_buf(void* buf, size_t n);
#endif
diff --git a/libc/private/bionic_auxv.h b/libc/private/bionic_auxv.h
index 53fcc49..8e33c1c 100644
--- a/libc/private/bionic_auxv.h
+++ b/libc/private/bionic_auxv.h
@@ -25,17 +25,9 @@
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
-#ifndef _PRIVATE_BIONIC_AUXV_H_
-#define _PRIVATE_BIONIC_AUXV_H_
-#include <elf.h>
-#include <link.h>
+#pragma once
+
#include <sys/cdefs.h>
-__BEGIN_DECLS
-
-extern ElfW(auxv_t)* __libc_auxv;
-
-__END_DECLS
-
-#endif /* _PRIVATE_BIONIC_AUXV_H_ */
+__LIBC_HIDDEN__ unsigned long __bionic_getauxval(unsigned long type, bool& exists);
diff --git a/libc/private/bionic_globals.h b/libc/private/bionic_globals.h
index 906d569..ba510b1 100644
--- a/libc/private/bionic_globals.h
+++ b/libc/private/bionic_globals.h
@@ -30,6 +30,7 @@
#define _PRIVATE_BIONIC_GLOBALS_H
#include <sys/cdefs.h>
+#include <link.h>
#include <pthread.h>
#include "private/bionic_fdsan.h"
@@ -56,6 +57,8 @@
// the program it's loading. Typically 0, sometimes 1.
int initial_linker_arg_count;
+ ElfW(auxv_t)* auxv;
+
pthread_mutex_t abort_msg_lock;
abort_msg_t* abort_msg;
@@ -67,14 +70,13 @@
__LIBC_HIDDEN__ libc_shared_globals* __libc_shared_globals();
__LIBC_HIDDEN__ void __libc_init_fdsan();
-class KernelArgumentBlock;
__LIBC_HIDDEN__ void __libc_init_malloc(libc_globals* globals);
-__LIBC_HIDDEN__ void __libc_init_setjmp_cookie(libc_globals* globals, KernelArgumentBlock& args);
-__LIBC_HIDDEN__ void __libc_init_vdso(libc_globals* globals, KernelArgumentBlock& args);
+__LIBC_HIDDEN__ void __libc_init_setjmp_cookie(libc_globals* globals);
+__LIBC_HIDDEN__ void __libc_init_vdso(libc_globals* globals);
#if defined(__i386__)
__LIBC_HIDDEN__ extern void* __libc_sysinfo;
-__LIBC_HIDDEN__ void __libc_init_sysinfo(KernelArgumentBlock& args);
+__LIBC_HIDDEN__ void __libc_init_sysinfo();
#endif
#endif
diff --git a/libc/private/bionic_tls.h b/libc/private/bionic_tls.h
index de086f2..80dc9bc 100644
--- a/libc/private/bionic_tls.h
+++ b/libc/private/bionic_tls.h
@@ -63,12 +63,6 @@
TLS_SLOT_OPENGL_API = 3,
TLS_SLOT_OPENGL = 4,
- // This slot is only used to pass information from the dynamic linker to
- // libc.so when the C library is loaded in to memory. The C runtime init
- // function will then clear it. Since its use is extremely temporary,
- // we reuse an existing location that isn't needed during libc startup.
- TLS_SLOT_BIONIC_PREINIT = TLS_SLOT_OPENGL_API,
-
TLS_SLOT_STACK_GUARD = 5, // GCC requires this specific slot for x86.
TLS_SLOT_DLERROR,
@@ -132,7 +126,8 @@
#if defined(__cplusplus)
class KernelArgumentBlock;
-extern void __libc_init_main_thread(KernelArgumentBlock&);
+extern void __libc_init_main_thread_early(KernelArgumentBlock& args);
+extern void __libc_init_main_thread_late();
#endif
#endif /* __BIONIC_PRIVATE_BIONIC_TLS_H_ */
diff --git a/libdl/Android.bp b/libdl/Android.bp
index 13c8714..656d3ab 100644
--- a/libdl/Android.bp
+++ b/libdl/Android.bp
@@ -22,6 +22,7 @@
include_dirs: ["bionic/libc"],
stl: "none",
+ system_shared_libs: [],
sanitize: {
never: true,
diff --git a/linker/Android.bp b/linker/Android.bp
index a9907ca..0e3484a 100644
--- a/linker/Android.bp
+++ b/linker/Android.bp
@@ -283,6 +283,7 @@
suffix: "64",
},
},
+ system_shared_libs: [],
target: {
android: {
static_libs: ["libdebuggerd_handler_fallback"],
diff --git a/linker/linker_main.cpp b/linker/linker_main.cpp
index aa12b6e..3318c2c 100644
--- a/linker/linker_main.cpp
+++ b/linker/linker_main.cpp
@@ -148,8 +148,8 @@
// An empty list of soinfos
static soinfo_list_t g_empty_list;
-static void add_vdso(KernelArgumentBlock& args) {
- ElfW(Ehdr)* ehdr_vdso = reinterpret_cast<ElfW(Ehdr)*>(args.getauxval(AT_SYSINFO_EHDR));
+static void add_vdso() {
+ ElfW(Ehdr)* ehdr_vdso = reinterpret_cast<ElfW(Ehdr)*>(getauxval(AT_SYSINFO_EHDR));
if (ehdr_vdso == nullptr) {
return;
}
@@ -191,7 +191,7 @@
ElfW(Addr) entry_point;
};
-static ExecutableInfo get_executable_info(KernelArgumentBlock& args) {
+static ExecutableInfo get_executable_info() {
ExecutableInfo result = {};
if (is_first_stage_init()) {
@@ -222,9 +222,9 @@
result.path = std::string(path, path_len);
}
- result.phdr = reinterpret_cast<const ElfW(Phdr)*>(args.getauxval(AT_PHDR));
- result.phdr_count = args.getauxval(AT_PHNUM);
- result.entry_point = args.getauxval(AT_ENTRY);
+ result.phdr = reinterpret_cast<const ElfW(Phdr)*>(getauxval(AT_PHDR));
+ result.phdr_count = getauxval(AT_PHNUM);
+ result.entry_point = getauxval(AT_ENTRY);
return result;
}
@@ -344,7 +344,7 @@
}
const ExecutableInfo exe_info = exe_to_load ? load_executable(exe_to_load) :
- get_executable_info(args);
+ get_executable_info();
// Assign to a static variable for the sake of the debug map, which needs
// a C-style string to last until the program exits.
@@ -375,7 +375,7 @@
insert_link_map_into_debug_map(&si->link_map_head);
insert_link_map_into_debug_map(&solinker->link_map_head);
- add_vdso(args);
+ add_vdso();
ElfW(Ehdr)* elf_hdr = reinterpret_cast<ElfW(Ehdr)*>(si->base);
@@ -452,10 +452,6 @@
if (!get_cfi_shadow()->InitialLinkDone(solist)) __linker_cannot_link(g_argv[0]);
- // Store a pointer to the kernel argument block in a TLS slot to be
- // picked up by the libc constructor.
- __get_tls()[TLS_SLOT_BIONIC_PREINIT] = &args;
-
si->call_pre_init_constructors();
si->call_constructors();
@@ -559,22 +555,20 @@
* function, or other GOT reference will generate a segfault.
*/
extern "C" ElfW(Addr) __linker_init(void* raw_args) {
+ // Initialize TLS early so system calls and errno work.
KernelArgumentBlock args(raw_args);
-
-#if defined(__i386__)
- __libc_init_sysinfo(args);
-#endif
+ __libc_init_main_thread_early(args);
// When the linker is run by itself (rather than as an interpreter for
// another program), AT_BASE is 0.
- ElfW(Addr) linker_addr = args.getauxval(AT_BASE);
+ ElfW(Addr) linker_addr = getauxval(AT_BASE);
if (linker_addr == 0) {
// Detect an attempt to run the linker on itself (e.g.
// `linker64 /system/bin/linker64`). If the kernel loaded this instance of
// the linker, then AT_ENTRY will refer to &_start. If it doesn't, then
// something else must have loaded this instance of the linker. It's
// simpler if we only allow one copy of the linker to be loaded at a time.
- if (args.getauxval(AT_ENTRY) != reinterpret_cast<uintptr_t>(&_start)) {
+ if (getauxval(AT_ENTRY) != reinterpret_cast<uintptr_t>(&_start)) {
// The first linker already relocated this one and set up TLS, so we don't
// need further libc initialization.
__linker_error("error: linker cannot load itself\n");
@@ -583,7 +577,7 @@
// instance, so use the phdr to find the linker's base address.
ElfW(Addr) load_bias;
get_elf_base_from_phdr(
- reinterpret_cast<ElfW(Phdr)*>(args.getauxval(AT_PHDR)), args.getauxval(AT_PHNUM),
+ reinterpret_cast<ElfW(Phdr)*>(getauxval(AT_PHDR)), getauxval(AT_PHNUM),
&linker_addr, &load_bias);
}
@@ -622,15 +616,15 @@
*/
static ElfW(Addr) __attribute__((noinline))
__linker_init_post_relocation(KernelArgumentBlock& args, soinfo& tmp_linker_so) {
- // Initialize the main thread (including TLS, so system calls really work).
- __libc_init_main_thread(args);
+ // Finish initializing the main thread.
+ __libc_init_main_thread_late();
// We didn't protect the linker's RELRO pages in link_image because we
// couldn't make system calls on x86 at that point, but we can now...
if (!tmp_linker_so.protect_relro()) __linker_cannot_link(args.argv[0]);
// Initialize the linker's static libc's globals
- __libc_init_globals(args);
+ __libc_init_globals();
// Initialize the linker's own global variables
tmp_linker_so.call_constructors();
@@ -640,7 +634,7 @@
// as PT_INTERP, AT_ENTRY will refer to the loaded executable rather than the
// linker's _start.
const char* exe_to_load = nullptr;
- if (args.getauxval(AT_ENTRY) == reinterpret_cast<uintptr_t>(&_start)) {
+ if (getauxval(AT_ENTRY) == reinterpret_cast<uintptr_t>(&_start)) {
if (args.argc <= 1 || !strcmp(args.argv[1], "--help")) {
async_safe_format_fd(STDOUT_FILENO,
"Usage: %s program [arguments...]\n"
diff --git a/tests/dl_test.cpp b/tests/dl_test.cpp
index 57d04e9..aea92b4 100644
--- a/tests/dl_test.cpp
+++ b/tests/dl_test.cpp
@@ -148,7 +148,7 @@
#endif
}
-TEST(dl, xfail_preinit_getauxval) {
+TEST(dl, preinit_getauxval) {
#if defined(__BIONIC__)
std::string helper = GetTestlibRoot() +
"/preinit_getauxval_test_helper/preinit_getauxval_test_helper";
diff --git a/tools/versioner/src/Android.bp b/tools/versioner/src/Android.bp
index 97e85f8..a78ab5d 100644
--- a/tools/versioner/src/Android.bp
+++ b/tools/versioner/src/Android.bp
@@ -3,9 +3,6 @@
defaults: ["llvm-build-host-tools-defaults"],
- // b/117120485 bugprone-exception-escape hangs with DeclarationDatabase.cpp
- tidy_checks: ["-bugprone-exception-escape"],
-
srcs: [
"versioner.cpp",
"Arch.cpp",
diff --git a/tools/versioner/src/Driver.cpp b/tools/versioner/src/Driver.cpp
index 8ba3d42..6062240 100644
--- a/tools/versioner/src/Driver.cpp
+++ b/tools/versioner/src/Driver.cpp
@@ -41,6 +41,7 @@
#include <llvm/ADT/IntrusiveRefCntPtr.h>
#include <llvm/ADT/SmallVector.h>
#include <llvm/ADT/StringRef.h>
+#include <llvm/Option/Option.h>
#include <llvm/Support/VirtualFileSystem.h>
#include "Arch.h"
@@ -168,7 +169,7 @@
}
const driver::Command& driver_cmd = llvm::cast<driver::Command>(*jobs.begin());
- const driver::ArgStringList& cc_args = driver_cmd.getArguments();
+ const llvm::opt::ArgStringList& cc_args = driver_cmd.getArguments();
if (cc_args.size() == 0) {
errx(1, "driver returned empty command for %s", to_string(type).c_str());