Merge "Optimize tolower(3)/toupper(3) from <ctype.h>."
diff --git a/benchmarks/README.md b/benchmarks/README.md
index 2616a51..fe447d1 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -1,48 +1,44 @@
-Bionic Benchmarks
-=================
+# Bionic Benchmarks
 
-Bionic benchmarks is a command line tool for measuring the runtimes of libc functions. It is built
-on top of [Google benchmarks](https://github.com/google/benchmark) with some additions to organize
+[TOC]
+
+## libc benchmarks (bionic-benchmarks)
+
+`bionic-benchmarks` is a command line tool for measuring the runtimes of libc functions. It is built
+on top of [Google Benchmark](https://github.com/google/benchmark) with some additions to organize
 tests into suites.
 
-Running the benchmarks
-----------------------
-
 ### Device benchmarks
 
-    $ mma
-    $ adb remount
-    $ adb sync
+    $ mmma bionic/benchmarks
+    $ adb root
+    $ adb sync data
     $ adb shell /data/benchmarktest/bionic-benchmarks/bionic-benchmarks
     $ adb shell /data/benchmarktest64/bionic-benchmarks/bionic-benchmarks
 
-When operated without specifying an xml file, the default is to run all
-of the benchmarks in alphabetical order.
-
-You can use `--benchmark_filter=getpid` to just run benchmarks with "getpid"
-in their name.
+By default, `bionic-benchmarks` runs all of the benchmarks in alphabetical order. Pass
+`--benchmark_filter=getpid` to run just the benchmarks with "getpid" in their name.
 
 ### Host benchmarks
 
-See the benchmarks/run-on-host.sh script. The host benchmarks can be run
-with 32 bit or 64 bit bionic, or the host glibc.
+See the `benchmarks/run-on-host.sh` script. The host benchmarks can be run with 32-bit or 64-bit
+Bionic, or the host glibc.
 
-## Suites
+### XML suites
 
 Suites are stored in the `suites/` directory and can be chosen with the command line flag
-'--bionic_xml'.
+`--bionic_xml`.
 
-To choose a specific xml file, use the `--bionic_xml=FILE.XML` option. By default, this
-option searches for the xml file in the `suites/` directory. If it doesn't exist
-in that directory then the file will be found as relative to the current
-directory. If the option specifies the full path to an xml file such as
-`/data/nativetest/suites/example.xml`, it will be used as is.
+To choose a specific XML file, use the `--bionic_xml=FILE.XML` option. By default, this option
+searches for the XML file in the `suites/` directory. If it doesn't exist in that directory, then
+the file will be found as relative to the current directory. If the option specifies the full path
+to an XML file such as `/data/nativetest/suites/example.xml`, it will be used as-is.
 
-If no xml file is specified through the command-line option, the default is to use `suites/full.xml`.
-However, for the host bionic benchmarks (bionic-benchmarks-glibc), the default
-is to use `suites/host.xml`.
+If no XML file is specified through the command-line option, the default is to use `suites/full.xml`.
+However, for the host bionic benchmarks (`bionic-benchmarks-glibc`), the default is to use
+`suites/host.xml`.
 
-### Format
+### XML suite format
 
 The format for a benchmark is:
 
@@ -55,19 +51,19 @@
 </fn>
 ```
 
-xml-specified values for iterations and cpu take precedence over those specified via command line
-(via '--bionic_iterations' and '--bionic_cpu', respectively.)
+XML-specified values for iterations and cpu take precedence over those specified via command line
+(via `--bionic_iterations` and `--bionic_cpu`, respectively.)
 
 To make small changes in runs, you can also schedule benchmarks by passing in their name and a
-space-separated list of arguments via the 'bionic_extra' command line flag, e.g.
-'--bionic_extra="BM_string_memcpy AT_COMMON_SIZES"' or '--bionic_extra="BM_string_memcmp 32 8 8"'
+space-separated list of arguments via the `--bionic_extra` command line flag, e.g.
+`--bionic_extra="BM_string_memcpy AT_COMMON_SIZES"` or `--bionic_extra="BM_string_memcmp 32 8 8"`
 
 Note that benchmarks will run normally if extra arguments are passed in, and it will fail
 with a segfault if too few are passed in.
 
 ### Shorthand
 
-For the sake of brevity, multiple runs can be scheduled in one xml element by putting one of the
+For the sake of brevity, multiple runs can be scheduled in one XML element by putting one of the
 following in the args field:
 
     NUM_PROPS
@@ -81,5 +77,112 @@
 
 ### Unit Tests
 
-Bionic benchmarks also has its own set of unit tests, which can be run from the binary in
+`bionic-benchmarks` also has its own set of unit tests, which can be run from the binary in
 `/data/nativetest[64]/bionic-benchmarks-tests`
+
+## Process startup time (bionic-spawn-benchmarks)
+
+The `spawn/` subdirectory has a few benchmarks measuring the time used to start simple programs
+(e.g. Toybox's `true` and `sh -c true`). Run it on a device like so:
+
+    m bionic-spawn-benchmarks
+    adb root
+    adb sync data
+    adb shell /data/benchmarktest/bionic-spawn-benchmarks/bionic-spawn-benchmarks
+    adb shell /data/benchmarktest64/bionic-spawn-benchmarks/bionic-spawn-benchmarks
+
+Google Benchmark reports both a real-time figure ("Time") and a CPU usage figure. For these
+benchmarks, the CPU measurement only counts time spent in the thread calling `posix_spawn`, not that
+spent in the spawned process. The real-time is probably more useful, and it is the figure used to
+determine the iteration count.
+
+Locking the CPU frequency seems to improve the results of these benchmarks significantly, and it
+reduces variability.
+
+## Google Benchmark notes
+
+### Repetitions
+
+Google Benchmark uses two settings to control how many times to run each benchmark, "iterations" and
+"repetitions". By default, the repetition count is one. Google Benchmark runs the benchmark a few
+times to determine a sufficiently-large iteration count.
+
+Google Benchmark can optionally run a benchmark run repeatedly and report statistics (median, mean,
+standard deviation) for the runs. To do so, pass the `--benchmark_repetitions` option, e.g.:
+
+    # ./bionic-benchmarks --benchmark_filter=BM_stdlib_strtoll --benchmark_repetitions=4
+    ...
+    -------------------------------------------------------------------
+    Benchmark                         Time             CPU   Iterations
+    -------------------------------------------------------------------
+    BM_stdlib_strtoll              27.7 ns         27.7 ns     25290525
+    BM_stdlib_strtoll              27.7 ns         27.7 ns     25290525
+    BM_stdlib_strtoll              27.7 ns         27.7 ns     25290525
+    BM_stdlib_strtoll              27.8 ns         27.7 ns     25290525
+    BM_stdlib_strtoll_mean         27.7 ns         27.7 ns            4
+    BM_stdlib_strtoll_median       27.7 ns         27.7 ns            4
+    BM_stdlib_strtoll_stddev      0.023 ns        0.023 ns            4
+
+There are 4 runs, each with 25290525 iterations. Measurements for the individual runs can be
+suppressed if they aren't needed:
+
+    # ./bionic-benchmarks --benchmark_filter=BM_stdlib_strtoll --benchmark_repetitions=4 --benchmark_report_aggregates_only
+    ...
+    -------------------------------------------------------------------
+    Benchmark                         Time             CPU   Iterations
+    -------------------------------------------------------------------
+    BM_stdlib_strtoll_mean         27.8 ns         27.7 ns            4
+    BM_stdlib_strtoll_median       27.7 ns         27.7 ns            4
+    BM_stdlib_strtoll_stddev      0.043 ns        0.043 ns            4
+
+### CPU frequencies
+
+To get consistent results between runs, it can sometimes be helpful to restrict a benchmark to
+specific cores, or to lock cores at specific frequencies. Some phones have a big.LITTLE core setup,
+or at least allow some cores to run at higher frequencies than others.
+
+A core can be selected for `bionic-benchmarks` using the `--bionic_cpu` option or using the
+`taskset` utility. e.g. A Pixel 3 device has 4 Kryo 385 Silver cores followed by 4 Gold cores:
+
+    blueline:/ # /data/benchmarktest64/bionic-benchmarks/bionic-benchmarks --benchmark_filter=BM_stdlib_strtoll --bionic_cpu=0
+    ...
+    ------------------------------------------------------------
+    Benchmark                  Time             CPU   Iterations
+    ------------------------------------------------------------
+    BM_stdlib_strtoll       64.2 ns         63.6 ns     11017493
+
+    blueline:/ # /data/benchmarktest64/bionic-benchmarks/bionic-benchmarks --benchmark_filter=BM_stdlib_strtoll --bionic_cpu=4
+    ...
+    ------------------------------------------------------------
+    Benchmark                  Time             CPU   Iterations
+    ------------------------------------------------------------
+    BM_stdlib_strtoll       21.8 ns         21.7 ns     33167103
+
+A similar result can be achieved using `taskset`. The first parameter is a bitmask of core numbers
+to pass to `sched_setaffinity`:
+
+    blueline:/ # taskset f /data/benchmarktest64/bionic-benchmarks/bionic-benchmarks --benchmark_filter=BM_stdlib_strtoll
+    ...
+    ------------------------------------------------------------
+    Benchmark                  Time             CPU   Iterations
+    ------------------------------------------------------------
+    BM_stdlib_strtoll       64.3 ns         63.6 ns     10998697
+
+    blueline:/ # taskset f0 /data/benchmarktest64/bionic-benchmarks/bionic-benchmarks --benchmark_filter=BM_stdlib_strtoll
+    ...
+    ------------------------------------------------------------
+    Benchmark                  Time             CPU   Iterations
+    ------------------------------------------------------------
+    BM_stdlib_strtoll       21.3 ns         21.2 ns     33094801
+
+To lock the CPU frequency, use the sysfs interface at `/sys/devices/system/cpu/cpu*/cpufreq/`.
+Changing the scaling governor to `performance` suppresses the warning that Google Benchmark
+otherwise prints:
+
+    ***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead.
+
+Some devices have a `perf-setup.sh` script that locks CPU and GPU frequencies. Some TradeFed
+benchmarks appear to be using the script. For more information:
+ * run `get_build_var BOARD_PERFSETUP_SCRIPT`
+ * run `m perf-setup.sh` to install the script into `${OUT}/data/local/tmp/perf-setup.sh`
+ * see: https://android.googlesource.com/platform/platform_testing/+/refs/heads/master/scripts/perf-setup/
diff --git a/benchmarks/spawn/Android.bp b/benchmarks/spawn/Android.bp
new file mode 100644
index 0000000..ecce8e8
--- /dev/null
+++ b/benchmarks/spawn/Android.bp
@@ -0,0 +1,96 @@
+//
+// Copyright (C) 2019 The Android Open Source Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in
+//    the documentation and/or other materials provided with the
+//    distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+// AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+// OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+// SUCH DAMAGE.
+//
+
+cc_benchmark {
+    name: "bionic-spawn-benchmarks",
+    srcs: ["spawn_benchmarks.cpp"],
+    static_libs: ["libbase"],
+
+    // Install these binaries in the same directory as the main benchmark binary.
+    data: [
+        ":bench_noop",
+        ":bench_noop_nostl",
+        ":bench_noop_static",
+    ],
+
+    host_supported: true,
+    target: {
+        darwin: { enabled: false },
+        windows: { enabled: false },
+        linux_glibc_x86: { enabled: false },
+    },
+}
+
+cc_defaults {
+    name: "noop_binary_defaults",
+
+    compile_multilib: "both",
+    multilib: {
+        lib32: { suffix: "32" },
+        lib64: { suffix: "64" },
+    },
+
+    host_supported: true,
+    target: {
+        darwin: { enabled: false },
+        windows: { enabled: false },
+        linux_glibc_x86: { enabled: false },
+    },
+}
+
+cc_binary {
+    defaults: ["noop_binary_defaults"],
+    name: "bench_noop",
+    srcs: ["noop.cpp"],
+
+    // When this binary is installed to host/linux-x86/bin, its runpath is ${ORIGIN}/../lib64, which
+    // is fine for finding host/linux-x86/lib64/libc++.so. When it's installed to
+    // host/linux-x86/benchmarktest64/bionic-spawn-benchmarks, the runpath needs an extra "..".
+    target: {
+        linux_glibc_x86_64: {
+            ldflags: [
+                "-Wl,--rpath,${ORIGIN}/../../lib64",
+            ],
+        },
+    }
+}
+
+cc_binary {
+    defaults: ["noop_binary_defaults"],
+    name: "bench_noop_nostl",
+    srcs: ["noop.cpp"],
+    stl: "none",
+}
+
+cc_binary {
+    defaults: ["noop_binary_defaults"],
+    name: "bench_noop_static",
+    srcs: ["noop.cpp"],
+    static_executable: true,
+    stl: "libc++_static",
+}
diff --git a/benchmarks/spawn/AndroidTest.xml b/benchmarks/spawn/AndroidTest.xml
new file mode 100644
index 0000000..9331675
--- /dev/null
+++ b/benchmarks/spawn/AndroidTest.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2019 The Android Open Source Project
+     All rights reserved.
+
+     Redistribution and use in source and binary forms, with or without
+     modification, are permitted provided that the following conditions
+     are met:
+      * Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+      * Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in
+        the documentation and/or other materials provided with the
+        distribution.
+
+     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+     "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+     LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+     FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+     COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+     INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+     BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+     OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+     AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+     OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+     OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+     SUCH DAMAGE.
+-->
+<configuration description="Runs bionic-spawn-benchmarks.">
+    <option name="test-suite-tag" value="apct" />
+
+    <target_preparer class="com.android.tradefed.targetprep.PushFilePreparer">
+        <option name="cleanup" value="true" />
+        <option name="push" value="bionic-spawn-benchmarks->/data/local/tmp/bionic-spawn-benchmarks" />
+    </target_preparer>
+
+    <!-- TODO(b/120549168): This seems necessary for consistent results on a walleye, but it's not working with atest
+    <target_preparer class="com.android.tradefed.targetprep.PushFilePreparer">
+        <option name="push" value="perf-setup.sh->/data/local/tmp/perf-setup.sh" />
+        <option name="post-push" value="chmod 755 /data/local/tmp/perf-setup.sh;/data/local/tmp/perf-setup.sh" />
+    </target_preparer>
+    -->
+
+    <test class="com.android.tradefed.testtype.GoogleBenchmarkTest" >
+        <option name="native-benchmark-device-path" value="/data/local/tmp" />
+        <option name="benchmark-module-name" value="bionic-spawn-benchmarks" />
+
+        <!-- The GoogleBenchmarkTest class ordinarily expects every file in the benchmark's
+             directory (recursively) to be a google-benchmark binary, so we need this setting to
+             avoid failing on the bench_* noop programs, which don't output benchmark results. -->
+        <option name="file-exclusion-filter-regex" value=".*/bench_[^/]*$"  />
+    </test>
+
+</configuration>
diff --git a/benchmarks/spawn/noop.cpp b/benchmarks/spawn/noop.cpp
new file mode 100644
index 0000000..1f1cac0
--- /dev/null
+++ b/benchmarks/spawn/noop.cpp
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+int main() {
+  return 0;
+}
diff --git a/benchmarks/spawn/spawn_benchmarks.cpp b/benchmarks/spawn/spawn_benchmarks.cpp
new file mode 100644
index 0000000..e8f7c17
--- /dev/null
+++ b/benchmarks/spawn/spawn_benchmarks.cpp
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <errno.h>
+#include <spawn.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <android-base/file.h>
+#include <android-base/stringprintf.h>
+#include <benchmark/benchmark.h>
+
+static std::string test_program(const char* name) {
+#if defined(__LP64__)
+  return android::base::GetExecutableDirectory() + "/" + name + "64";
+#else
+  return android::base::GetExecutableDirectory() + "/" + name + "32";
+#endif
+}
+
+extern char** environ;
+
+static void BM_spawn_test(benchmark::State& state, const char* const* argv) {
+  for (auto _ : state) {
+    pid_t child = 0;
+    if (int spawn_err = posix_spawn(&child, argv[0], nullptr, nullptr, const_cast<char**>(argv),
+                                    environ)) {
+      state.SkipWithError(android::base::StringPrintf(
+          "posix_spawn of %s failed: %s", argv[0], strerror(spawn_err)).c_str());
+      break;
+    }
+
+    int wstatus = 0;
+    const pid_t wait_result = TEMP_FAILURE_RETRY(waitpid(child, &wstatus, 0));
+    if (wait_result != child) {
+      state.SkipWithError(android::base::StringPrintf(
+          "waitpid on pid %d for %s failed: %s",
+          static_cast<int>(child), argv[0], strerror(errno)).c_str());
+      break;
+    }
+    if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 127) {
+      state.SkipWithError(android::base::StringPrintf("could not exec %s", argv[0]).c_str());
+      break;
+    }
+  }
+}
+
+#define SPAWN_BENCHMARK(name, ...)                                                    \
+    BENCHMARK_CAPTURE(BM_spawn_test, name, (const char*[]) { __VA_ARGS__, nullptr })  \
+        ->UseRealTime()                                                               \
+        ->Unit(benchmark::kMicrosecond)                                               \
+
+SPAWN_BENCHMARK(noop, test_program("bench_noop").c_str());
+SPAWN_BENCHMARK(noop_nostl, test_program("bench_noop_nostl").c_str());
+SPAWN_BENCHMARK(noop_static, test_program("bench_noop_static").c_str());
+
+// Android has a /bin -> /system/bin symlink, but use /system/bin explicitly so we can more easily
+// compare Bionic-vs-glibc on a Linux desktop machine.
+#if defined(__GLIBC__)
+
+SPAWN_BENCHMARK(bin_true, "/bin/true");
+SPAWN_BENCHMARK(sh_true, "/bin/sh", "-c", "true");
+
+#elif defined(__ANDROID__)
+
+SPAWN_BENCHMARK(system_bin_true, "/system/bin/true");
+SPAWN_BENCHMARK(vendor_bin_true, "/vendor/bin/true");
+SPAWN_BENCHMARK(system_sh_true, "/system/bin/sh", "-c", "true");
+SPAWN_BENCHMARK(vendor_sh_true, "/vendor/bin/sh", "-c", "true");
+
+#endif
+
+BENCHMARK_MAIN();
diff --git a/libc/Android.bp b/libc/Android.bp
index 5ffb2e1..eee59d4 100644
--- a/libc/Android.bp
+++ b/libc/Android.bp
@@ -1712,32 +1712,19 @@
     cmd: "$(location :bionic-generate-version-script) x86_64 $(in) $(out)",
 }
 
-// Makes bionic_tls.h available for art to use in its implementation of Thread::Current().
-cc_library_headers {
-    name: "bionic_libc_private_headers",
-    visibility: [
-        "//art:__subpackages__",
-    ],
-    host_supported: true,
-    export_include_dirs: [
-        "private",
-    ],
-    sdk_version: "current",
-}
-
 // Headers that only other parts of the platform can include.
 cc_library_headers {
     name: "bionic_libc_platform_headers",
     visibility: [
-        "//bionic/libc/malloc_debug:__subpackages__",
-        "//bionic/libc/malloc_hooks:__subpackages__",
-        "//frameworks/av/media/libmedia:__subpackages__",
-        "//frameworks/av/media/utils:__subpackages__",
-        "//frameworks/base/core/jni:__subpackages__",
-        "//frameworks/base/services/core/jni:__subpackages__",
+        "//art:__subpackages__",
+        "//bionic/libc:__subpackages__",
+        "//frameworks:__subpackages__",
         "//external/perfetto:__subpackages__",
+        "//external/scudo:__subpackages__",
     ],
     host_supported: true,
+    recovery_available: true,
+    native_bridge_supported: true,
     export_include_dirs: [
         "platform",
     ],
@@ -1890,6 +1877,9 @@
         "-Wall",
         "-Werror",
     ],
+    sanitize: {
+        never: true,
+    },
 }
 
 cc_defaults {
diff --git a/libc/arch-arm/bionic/vfork.S b/libc/arch-arm/bionic/vfork.S
index 0b17d64..6855db7 100644
--- a/libc/arch-arm/bionic/vfork.S
+++ b/libc/arch-arm/bionic/vfork.S
@@ -26,8 +26,8 @@
  * SUCH DAMAGE.
  */
 
+#include <platform/bionic/tls_defines.h>
 #include <private/bionic_asm.h>
-#include <private/bionic_asm_tls.h>
 
 ENTRY(vfork)
 __BIONIC_WEAK_ASM_FOR_NATIVE_BRIDGE(vfork)
diff --git a/libc/arch-arm64/bionic/vfork.S b/libc/arch-arm64/bionic/vfork.S
index 6c01572..c307bc3 100644
--- a/libc/arch-arm64/bionic/vfork.S
+++ b/libc/arch-arm64/bionic/vfork.S
@@ -26,8 +26,8 @@
  * SUCH DAMAGE.
  */
 
+#include <platform/bionic/tls_defines.h>
 #include <private/bionic_asm.h>
-#include <private/bionic_asm_tls.h>
 #include <asm/signal.h>
 #include <linux/sched.h>
 
diff --git a/libc/arch-x86/bionic/vfork.S b/libc/arch-x86/bionic/vfork.S
index 24ede3d..663169c 100644
--- a/libc/arch-x86/bionic/vfork.S
+++ b/libc/arch-x86/bionic/vfork.S
@@ -26,8 +26,8 @@
  * SUCH DAMAGE.
  */
 
+#include <platform/bionic/tls_defines.h>
 #include <private/bionic_asm.h>
-#include <private/bionic_asm_tls.h>
 
 // This custom code preserves the return address across the system call.
 
diff --git a/libc/arch-x86_64/bionic/vfork.S b/libc/arch-x86_64/bionic/vfork.S
index e32b517..86c5db2 100644
--- a/libc/arch-x86_64/bionic/vfork.S
+++ b/libc/arch-x86_64/bionic/vfork.S
@@ -26,8 +26,8 @@
  * SUCH DAMAGE.
  */
 
+#include <platform/bionic/tls_defines.h>
 #include <private/bionic_asm.h>
-#include <private/bionic_asm_tls.h>
 
 // This custom code preserves the return address across the system call.
 
diff --git a/libc/bionic/ndk_cruft.cpp b/libc/bionic/ndk_cruft.cpp
index 2c3299f..c816830 100644
--- a/libc/bionic/ndk_cruft.cpp
+++ b/libc/bionic/ndk_cruft.cpp
@@ -72,7 +72,7 @@
 
 // TODO: does anything still need this?
 void** __get_tls() {
-#include "private/__get_tls.h"
+#include "platform/bionic/tls.h"
   return __get_tls();
 }
 
diff --git a/libc/private/__get_tls.h b/libc/platform/bionic/tls.h
similarity index 94%
rename from libc/private/__get_tls.h
rename to libc/platform/bionic/tls.h
index 04c5fdb..ca39020 100644
--- a/libc/private/__get_tls.h
+++ b/libc/platform/bionic/tls.h
@@ -26,8 +26,7 @@
  * SUCH DAMAGE.
  */
 
-#ifndef __BIONIC_PRIVATE_GET_TLS_H_
-#define __BIONIC_PRIVATE_GET_TLS_H_
+#pragma once
 
 #if defined(__aarch64__)
 # define __get_tls() ({ void** __val; __asm__("mrs %0, tpidr_el0" : "=r"(__val)); __val; })
@@ -50,4 +49,4 @@
 #error unsupported architecture
 #endif
 
-#endif /* __BIONIC_PRIVATE_GET_TLS_H_ */
+#include "tls_defines.h"
diff --git a/libc/private/bionic_asm_tls.h b/libc/platform/bionic/tls_defines.h
similarity index 100%
rename from libc/private/bionic_asm_tls.h
rename to libc/platform/bionic/tls_defines.h
diff --git a/libc/private/bionic_tls.h b/libc/private/bionic_tls.h
index 90914c3..9a80140 100644
--- a/libc/private/bionic_tls.h
+++ b/libc/private/bionic_tls.h
@@ -34,9 +34,9 @@
 #include <sys/cdefs.h>
 #include <sys/param.h>
 
-#include "bionic_asm_tls.h"
+#include <platform/bionic/tls.h>
+
 #include "bionic_macros.h"
-#include "__get_tls.h"
 #include "grp_pwd.h"
 
 /** WARNING WARNING WARNING
diff --git a/linker/arch/arm64/tlsdesc_resolver.S b/linker/arch/arm64/tlsdesc_resolver.S
index ef46839..96eff8e 100644
--- a/linker/arch/arm64/tlsdesc_resolver.S
+++ b/linker/arch/arm64/tlsdesc_resolver.S
@@ -26,8 +26,8 @@
  * SUCH DAMAGE.
  */
 
+#include <platform/bionic/tls_defines.h>
 #include <private/bionic_asm.h>
-#include <private/bionic_asm_tls.h>
 
 .globl __tls_get_addr
 
diff --git a/tests/__aeabi_read_tp_test.cpp b/tests/__aeabi_read_tp_test.cpp
index 7209a75..f318c0f 100644
--- a/tests/__aeabi_read_tp_test.cpp
+++ b/tests/__aeabi_read_tp_test.cpp
@@ -28,7 +28,7 @@
 
 #include <gtest/gtest.h>
 
-#include "private/__get_tls.h"
+#include "platform/bionic/tls.h"
 
 #if defined(__arm__)
 extern "C" void* __aeabi_read_tp();
diff --git a/tests/elftls_dl_test.cpp b/tests/elftls_dl_test.cpp
index 012aad7..f31497a 100644
--- a/tests/elftls_dl_test.cpp
+++ b/tests/elftls_dl_test.cpp
@@ -33,7 +33,7 @@
 #include <thread>
 
 #include "gtest_globals.h"
-#include "private/__get_tls.h"
+#include "platform/bionic/tls.h"
 #include "utils.h"
 
 #if defined(__BIONIC__)