libc/arch-arm/dynamic_function_dispatch.cpp

/*
 * Copyright (C) 2018 The Android Open Source Project
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <fcntl.h>
#include <sys/syscall.h>

extern "C" {

enum CpuVariant {
    kUnknown = 0,
    kGeneric,
    kCortexA7,
    kCortexA9,
    kCortexA53,
    kCortexA55,
    kKrait,
    kKryo,
};

static constexpr int MAX_CPU_NAME_LEN = 12;
struct CpuVariantNames {
    alignas(alignof(int)) char name[MAX_CPU_NAME_LEN];
    CpuVariant variant;
};

static constexpr CpuVariantNames cpu_variant_names[] = {
    {"cortex-a76", kCortexA55},
    {"kryo385", kCortexA55},
    {"cortex-a75", kCortexA55},
    {"kryo", kKryo},
    {"cortex-a73", kCortexA55},
    {"cortex-a55", kCortexA55},
    {"cortex-a53", kCortexA53},
    {"krait", kKrait},
    {"cortex-a9", kCortexA9},
    {"cortex-a7", kCortexA7},
    // kUnknown indicates the end of this array.
    {"", kUnknown},
};

static long ifunc_open(const char* pathname) {
    register long r0 __asm__("r0") = AT_FDCWD;
    register long r1 __asm__("r1") = reinterpret_cast<long>(pathname);
    register long r2 __asm__("r2") = O_RDONLY;
    register long r3 __asm__("r3") = 0;
    register long r7 __asm__("r7") = __NR_openat;
    __asm__ volatile("swi #0" : "=r"(r0) : "r"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r7));
    return r0;
}

static ssize_t ifunc_read(int fd, void* buf, size_t count) {
    register long r0 __asm__("r0") = fd;
    register long r1 __asm__("r1") = reinterpret_cast<long>(buf);
    register long r2 __asm__("r2") = count;
    register long r7 __asm__("r7") = __NR_read;
    __asm__ volatile("swi #0" : "=r"(r0) : "r"(r0), "r"(r1), "r"(r2), "r"(r7) : "memory");
    return r0;
}

static int ifunc_close(int fd) {
    register long r0 __asm__("r0") = fd;
    register long r7 __asm__("r7") = __NR_close;
    __asm__ volatile("swi #0" : "=r"(r0) : "r"(r0), "r"(r7));
    return r0;
}

#define DEFINE_IFUNC_WITH_SUFFIX(name, suffix) \
    name##_func name##suffix __attribute__((ifunc(#name "_resolver"))); \
    __attribute__((visibility("hidden"))) \
    name##_func* name##_resolver()

#define DEFINE_IFUNC(name) DEFINE_IFUNC_WITH_SUFFIX(name, )

#define DEFINE_INTERNAL_IFUNC(name) DEFINE_IFUNC_WITH_SUFFIX(name, _internal)

#define DECLARE_FUNC(type, name) \
    __attribute__((visibility("hidden"))) \
    type name

#define RETURN_FUNC(type, name) { \
        DECLARE_FUNC(type, name); \
        return name; \
    }

static bool is_same_name(const char* a, const char* b) {
    static_assert(MAX_CPU_NAME_LEN % sizeof(int) == 0, "");
    const int* ia = reinterpret_cast<const int*>(a);
    const int* ib = reinterpret_cast<const int*>(b);
    for (size_t i = 0; i < MAX_CPU_NAME_LEN / sizeof(int); ++i) {
        if (ia[i] != ib[i]) {
            return false;
        }
    }
    return true;
}

static CpuVariant init_cpu_variant() {
    int fd = ifunc_open("/dev/cpu_variant:arm");
    if (fd < 0) return kGeneric;

    alignas(alignof(int)) char name[MAX_CPU_NAME_LEN] = {};

    int bytes_read, total_read = 0;
    while (total_read < MAX_CPU_NAME_LEN - 1 &&
           (bytes_read = ifunc_read(fd, name + total_read,
                                    MAX_CPU_NAME_LEN - 1 - total_read)) > 0) {
        total_read += bytes_read;
    }
    ifunc_close(fd);

    if (bytes_read != 0) {
        // The file is too big. We haven't reach the end. Or maybe there is an
        // error when reading.
        return kGeneric;
    }
    name[total_read] = 0;

    const CpuVariantNames* cpu_variant = cpu_variant_names;
    while (cpu_variant->variant != kUnknown) {
        if (is_same_name(cpu_variant->name, name)) {
            return cpu_variant->variant;
        }
        cpu_variant++;
    }
    return kGeneric;
}

static CpuVariant get_cpu_variant() {
    static CpuVariant cpu_variant = kUnknown;
    if (cpu_variant == kUnknown) {
        cpu_variant = init_cpu_variant();
    }
    return cpu_variant;
}

typedef void* memmove_func(void* __dst, const void* __src, size_t __n);
DEFINE_IFUNC(memmove) {
    RETURN_FUNC(memmove_func, memmove_a15);
}

typedef void* memcpy_func(void*, const void*, size_t);
DEFINE_IFUNC(memcpy) {
    return memmove_resolver();
}

typedef void* __memcpy_func(void*, const void*, size_t);
DEFINE_IFUNC(__memcpy) {
    switch(get_cpu_variant()) {
        case kCortexA7:
            RETURN_FUNC(__memcpy_func, __memcpy_a7);
        case kCortexA9:
            RETURN_FUNC(__memcpy_func, __memcpy_a9);
        case kKrait:
            RETURN_FUNC(__memcpy_func, __memcpy_krait);
        case kCortexA53:
            RETURN_FUNC(__memcpy_func, __memcpy_a53);
        case kCortexA55:
            RETURN_FUNC(__memcpy_func, __memcpy_a55);
        case kKryo:
            RETURN_FUNC(__memcpy_func, __memcpy_kryo);
        default:
            RETURN_FUNC(__memcpy_func, __memcpy_a15);
    }
}

typedef void* __memset_chk_func(void* s, int c, size_t n, size_t n2);
DEFINE_IFUNC(__memset_chk) {
    switch(get_cpu_variant()) {
        case kCortexA7:
        case kCortexA53:
        case kCortexA55:
        case kKryo:
            RETURN_FUNC(__memset_chk_func, __memset_chk_a7);
        case kCortexA9:
            RETURN_FUNC(__memset_chk_func, __memset_chk_a9);
        case kKrait:
            RETURN_FUNC(__memset_chk_func, __memset_chk_krait);
        default:
            RETURN_FUNC(__memset_chk_func, __memset_chk_a15);
    }
}

typedef void* memset_func(void* __dst, int __ch, size_t __n);
DEFINE_IFUNC(memset) {
    switch(get_cpu_variant()) {
        case kCortexA7:
        case kCortexA53:
        case kCortexA55:
        case kKryo:
             RETURN_FUNC(memset_func, memset_a7);
        case kCortexA9:
             RETURN_FUNC(memset_func, memset_a9);
        case kKrait:
             RETURN_FUNC(memset_func, memset_krait);
        default:
             RETURN_FUNC(memset_func, memset_a15);
    }
}

typedef char* strcpy_func(char* __dst, const char* __src);
DEFINE_IFUNC(strcpy) {
    switch(get_cpu_variant()) {
        case kCortexA9:
            RETURN_FUNC(strcpy_func, strcpy_a9);
        default:
            RETURN_FUNC(strcpy_func, strcpy_a15);
    }
}

typedef char* __strcpy_chk_func(char* dst, const char* src, size_t dst_len);
DEFINE_IFUNC(__strcpy_chk) {
    switch(get_cpu_variant()) {
        case kCortexA7:
            RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a7);
        case kCortexA9:
            RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a9);
        case kKrait:
        case kKryo:
            RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_krait);
        case kCortexA53:
            RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a53);
        case kCortexA55:
            RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a55);
        default:
            RETURN_FUNC(__strcpy_chk_func, __strcpy_chk_a15);
    }
}

typedef char* stpcpy_func(char* __dst, const char* __src);
DEFINE_IFUNC(stpcpy) {
    switch(get_cpu_variant()) {
        case kCortexA9:
            RETURN_FUNC(stpcpy_func, stpcpy_a9);
        default:
            RETURN_FUNC(stpcpy_func, stpcpy_a15);
    }
}

typedef char* strcat_func(char* __dst, const char* __src);
DEFINE_IFUNC(strcat) {
    switch(get_cpu_variant()) {
        case kCortexA9:
            RETURN_FUNC(strcat_func, strcat_a9);
        default:
            RETURN_FUNC(strcat_func, strcat_a15);
    }
}

typedef char* __strcat_chk_func(char* dst, const char* src, size_t dst_buf_size);
DEFINE_IFUNC(__strcat_chk) {
    switch(get_cpu_variant()) {
        case kCortexA7:
            RETURN_FUNC(__strcat_chk_func, __strcat_chk_a7);
        case kCortexA9:
            RETURN_FUNC(__strcat_chk_func, __strcat_chk_a9);
        case kKrait:
        case kKryo:
            RETURN_FUNC(__strcat_chk_func, __strcat_chk_krait);
        case kCortexA53:
            RETURN_FUNC(__strcat_chk_func, __strcat_chk_a53);
        case kCortexA55:
            RETURN_FUNC(__strcat_chk_func, __strcat_chk_a55);
        default:
            RETURN_FUNC(__strcat_chk_func, __strcat_chk_a15);
    }
}

typedef int strcmp_func(const char* __lhs, const char* __rhs);
DEFINE_INTERNAL_IFUNC(strcmp) {
    switch(get_cpu_variant()) {
        case kCortexA9:
            RETURN_FUNC(strcmp_func, strcmp_a9);
        case kCortexA55:
        case kKrait:
        case kKryo:
            RETURN_FUNC(strcmp_func, strcmp_krait);
        default:
            RETURN_FUNC(strcmp_func, strcmp_a15);
    }
}

typedef size_t strlen_func(const char* __s);
DEFINE_INTERNAL_IFUNC(strlen) {
    switch(get_cpu_variant()) {
        case kCortexA9:
            RETURN_FUNC(strlen_func, strlen_a9);
        default:
            RETURN_FUNC(strlen_func, strlen_a15);
    }
}

}  // extern "C"