Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2014 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
Josh Gao | 93c0f5e | 2015-10-06 11:08:13 -0700 | [diff] [blame] | 17 | #include "private/bionic_globals.h" |
| 18 | #include "private/bionic_vdso.h" |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 19 | |
Elliott Hughes | 613f814 | 2015-07-20 22:34:27 +0000 | [diff] [blame] | 20 | #include <limits.h> |
Josh Gao | 93c0f5e | 2015-10-06 11:08:13 -0700 | [diff] [blame] | 21 | #include <link.h> |
| 22 | #include <string.h> |
Ryan Prichard | 746ad15 | 2018-11-22 03:16:06 -0800 | [diff] [blame] | 23 | #include <sys/auxv.h> |
Josh Gao | 93c0f5e | 2015-10-06 11:08:13 -0700 | [diff] [blame] | 24 | #include <sys/cdefs.h> |
Elliott Hughes | 9a7d048 | 2023-07-28 17:18:48 -0700 | [diff] [blame] | 25 | #include <sys/hwprobe.h> |
Josh Gao | 93c0f5e | 2015-10-06 11:08:13 -0700 | [diff] [blame] | 26 | #include <sys/time.h> |
Elliott Hughes | fce8a15 | 2023-08-29 15:38:33 -0700 | [diff] [blame] | 27 | #include <syscall.h> |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 28 | #include <time.h> |
Josh Gao | 93c0f5e | 2015-10-06 11:08:13 -0700 | [diff] [blame] | 29 | #include <unistd.h> |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 30 | |
Elliott Hughes | 9a7d048 | 2023-07-28 17:18:48 -0700 | [diff] [blame] | 31 | extern "C" int __clock_gettime(int, struct timespec*); |
| 32 | extern "C" int __clock_getres(int, struct timespec*); |
| 33 | extern "C" int __gettimeofday(struct timeval*, struct timezone*); |
Elliott Hughes | 9a7d048 | 2023-07-28 17:18:48 -0700 | [diff] [blame] | 34 | |
Elliott Hughes | 9591df5 | 2017-11-27 20:30:25 -0800 | [diff] [blame] | 35 | static inline int vdso_return(int result) { |
| 36 | if (__predict_true(result == 0)) return 0; |
| 37 | |
| 38 | errno = -result; |
| 39 | return -1; |
| 40 | } |
| 41 | |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 42 | int clock_gettime(int clock_id, timespec* tp) { |
Josh Gao | 93c0f5e | 2015-10-06 11:08:13 -0700 | [diff] [blame] | 43 | auto vdso_clock_gettime = reinterpret_cast<decltype(&clock_gettime)>( |
| 44 | __libc_globals->vdso[VDSO_CLOCK_GETTIME].fn); |
| 45 | if (__predict_true(vdso_clock_gettime)) { |
Elliott Hughes | 9591df5 | 2017-11-27 20:30:25 -0800 | [diff] [blame] | 46 | return vdso_return(vdso_clock_gettime(clock_id, tp)); |
Josh Gao | 93c0f5e | 2015-10-06 11:08:13 -0700 | [diff] [blame] | 47 | } |
| 48 | return __clock_gettime(clock_id, tp); |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 49 | } |
| 50 | |
Mark Salyzyn | 79249b0 | 2017-11-07 08:19:20 -0800 | [diff] [blame] | 51 | int clock_getres(int clock_id, timespec* tp) { |
| 52 | auto vdso_clock_getres = reinterpret_cast<decltype(&clock_getres)>( |
| 53 | __libc_globals->vdso[VDSO_CLOCK_GETRES].fn); |
| 54 | if (__predict_true(vdso_clock_getres)) { |
| 55 | return vdso_return(vdso_clock_getres(clock_id, tp)); |
| 56 | } |
| 57 | return __clock_getres(clock_id, tp); |
| 58 | } |
| 59 | |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 60 | int gettimeofday(timeval* tv, struct timezone* tz) { |
Josh Gao | 93c0f5e | 2015-10-06 11:08:13 -0700 | [diff] [blame] | 61 | auto vdso_gettimeofday = reinterpret_cast<decltype(&gettimeofday)>( |
| 62 | __libc_globals->vdso[VDSO_GETTIMEOFDAY].fn); |
| 63 | if (__predict_true(vdso_gettimeofday)) { |
Elliott Hughes | 9591df5 | 2017-11-27 20:30:25 -0800 | [diff] [blame] | 64 | return vdso_return(vdso_gettimeofday(tv, tz)); |
Josh Gao | 93c0f5e | 2015-10-06 11:08:13 -0700 | [diff] [blame] | 65 | } |
| 66 | return __gettimeofday(tv, tz); |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 67 | } |
| 68 | |
Mark Salyzyn | 4473ccd | 2017-12-04 13:51:29 -0800 | [diff] [blame] | 69 | time_t time(time_t* t) { |
Elliott Hughes | 9a7d048 | 2023-07-28 17:18:48 -0700 | [diff] [blame] | 70 | // Only x86/x86-64 actually have time() in the vdso. |
| 71 | #if defined(VDSO_TIME_SYMBOL) |
Elliott Hughes | f1515f6 | 2018-01-12 15:20:28 -0800 | [diff] [blame] | 72 | auto vdso_time = reinterpret_cast<decltype(&time)>(__libc_globals->vdso[VDSO_TIME].fn); |
Mark Salyzyn | 4473ccd | 2017-12-04 13:51:29 -0800 | [diff] [blame] | 73 | if (__predict_true(vdso_time)) { |
| 74 | return vdso_time(t); |
| 75 | } |
Elliott Hughes | 9a7d048 | 2023-07-28 17:18:48 -0700 | [diff] [blame] | 76 | #endif |
Elliott Hughes | f1515f6 | 2018-01-12 15:20:28 -0800 | [diff] [blame] | 77 | |
| 78 | // We can't fallback to the time(2) system call because it doesn't exist for most architectures. |
| 79 | timeval tv; |
| 80 | if (gettimeofday(&tv, nullptr) == -1) return -1; |
| 81 | if (t) *t = tv.tv_sec; |
| 82 | return tv.tv_sec; |
Mark Salyzyn | 4473ccd | 2017-12-04 13:51:29 -0800 | [diff] [blame] | 83 | } |
| 84 | |
Elliott Hughes | 9a7d048 | 2023-07-28 17:18:48 -0700 | [diff] [blame] | 85 | #if defined(__riscv) |
| 86 | int __riscv_hwprobe(struct riscv_hwprobe* _Nonnull pairs, size_t pair_count, size_t cpu_count, |
| 87 | unsigned long* _Nullable cpus, unsigned flags) { |
| 88 | auto vdso_riscv_hwprobe = |
| 89 | reinterpret_cast<decltype(&__riscv_hwprobe)>(__libc_globals->vdso[VDSO_RISCV_HWPROBE].fn); |
| 90 | if (__predict_true(vdso_riscv_hwprobe)) { |
Elliott Hughes | fce8a15 | 2023-08-29 15:38:33 -0700 | [diff] [blame] | 91 | return -vdso_riscv_hwprobe(pairs, pair_count, cpu_count, cpus, flags); |
Elliott Hughes | 9a7d048 | 2023-07-28 17:18:48 -0700 | [diff] [blame] | 92 | } |
Elliott Hughes | fce8a15 | 2023-08-29 15:38:33 -0700 | [diff] [blame] | 93 | // Inline the syscall directly in case someone's calling it from an |
| 94 | // ifunc resolver where we won't be able to set errno on failure. |
| 95 | // (Rather than our usual trick of letting the python-generated |
| 96 | // wrapper set errno but saving/restoring errno in cases where the API |
| 97 | // is to return an error value rather than setting errno.) |
| 98 | register long a0 __asm__("a0") = reinterpret_cast<long>(pairs); |
| 99 | register long a1 __asm__("a1") = pair_count; |
| 100 | register long a2 __asm__("a2") = cpu_count; |
| 101 | register long a3 __asm__("a3") = reinterpret_cast<long>(cpus); |
| 102 | register long a4 __asm__("a4") = flags; |
| 103 | register long a7 __asm__("a7") = __NR_riscv_hwprobe; |
| 104 | __asm__ volatile("ecall" : "=r"(a0) : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a7)); |
| 105 | return -a0; |
Elliott Hughes | 9a7d048 | 2023-07-28 17:18:48 -0700 | [diff] [blame] | 106 | } |
| 107 | #endif |
| 108 | |
Ryan Prichard | 746ad15 | 2018-11-22 03:16:06 -0800 | [diff] [blame] | 109 | void __libc_init_vdso(libc_globals* globals) { |
Josh Gao | 93c0f5e | 2015-10-06 11:08:13 -0700 | [diff] [blame] | 110 | auto&& vdso = globals->vdso; |
Elliott Hughes | 9a7d048 | 2023-07-28 17:18:48 -0700 | [diff] [blame] | 111 | vdso[VDSO_CLOCK_GETTIME] = {VDSO_CLOCK_GETTIME_SYMBOL, nullptr}; |
| 112 | vdso[VDSO_CLOCK_GETRES] = {VDSO_CLOCK_GETRES_SYMBOL, nullptr}; |
| 113 | vdso[VDSO_GETTIMEOFDAY] = {VDSO_GETTIMEOFDAY_SYMBOL, nullptr}; |
| 114 | #if defined(VDSO_TIME_SYMBOL) |
| 115 | vdso[VDSO_TIME] = {VDSO_TIME_SYMBOL, nullptr}; |
| 116 | #endif |
| 117 | #if defined(VDSO_RISCV_HWPROBE_SYMBOL) |
| 118 | vdso[VDSO_RISCV_HWPROBE] = {VDSO_RISCV_HWPROBE_SYMBOL, nullptr}; |
| 119 | #endif |
Josh Gao | 93c0f5e | 2015-10-06 11:08:13 -0700 | [diff] [blame] | 120 | |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 121 | // Do we have a vdso? |
Ryan Prichard | 746ad15 | 2018-11-22 03:16:06 -0800 | [diff] [blame] | 122 | uintptr_t vdso_ehdr_addr = getauxval(AT_SYSINFO_EHDR); |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 123 | ElfW(Ehdr)* vdso_ehdr = reinterpret_cast<ElfW(Ehdr)*>(vdso_ehdr_addr); |
Elliott Hughes | 613f814 | 2015-07-20 22:34:27 +0000 | [diff] [blame] | 124 | if (vdso_ehdr == nullptr) { |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 125 | return; |
| 126 | } |
| 127 | |
| 128 | // How many symbols does it have? |
| 129 | size_t symbol_count = 0; |
| 130 | ElfW(Shdr)* vdso_shdr = reinterpret_cast<ElfW(Shdr)*>(vdso_ehdr_addr + vdso_ehdr->e_shoff); |
| 131 | for (size_t i = 0; i < vdso_ehdr->e_shnum; ++i) { |
| 132 | if (vdso_shdr[i].sh_type == SHT_DYNSYM) { |
| 133 | symbol_count = vdso_shdr[i].sh_size / sizeof(ElfW(Sym)); |
Elliott Hughes | cbcccd5 | 2023-12-12 15:10:40 -0800 | [diff] [blame^] | 134 | break; |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 135 | } |
| 136 | } |
| 137 | if (symbol_count == 0) { |
| 138 | return; |
| 139 | } |
| 140 | |
| 141 | // Where's the dynamic table? |
| 142 | ElfW(Addr) vdso_addr = 0; |
Elliott Hughes | 613f814 | 2015-07-20 22:34:27 +0000 | [diff] [blame] | 143 | ElfW(Dyn)* vdso_dyn = nullptr; |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 144 | ElfW(Phdr)* vdso_phdr = reinterpret_cast<ElfW(Phdr)*>(vdso_ehdr_addr + vdso_ehdr->e_phoff); |
| 145 | for (size_t i = 0; i < vdso_ehdr->e_phnum; ++i) { |
| 146 | if (vdso_phdr[i].p_type == PT_DYNAMIC) { |
| 147 | vdso_dyn = reinterpret_cast<ElfW(Dyn)*>(vdso_ehdr_addr + vdso_phdr[i].p_offset); |
| 148 | } else if (vdso_phdr[i].p_type == PT_LOAD) { |
| 149 | vdso_addr = vdso_ehdr_addr + vdso_phdr[i].p_offset - vdso_phdr[i].p_vaddr; |
| 150 | } |
Elliott Hughes | cbcccd5 | 2023-12-12 15:10:40 -0800 | [diff] [blame^] | 151 | if (vdso_addr && vdso_dyn) break; |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 152 | } |
Elliott Hughes | 613f814 | 2015-07-20 22:34:27 +0000 | [diff] [blame] | 153 | if (vdso_addr == 0 || vdso_dyn == nullptr) { |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 154 | return; |
| 155 | } |
| 156 | |
| 157 | // Where are the string and symbol tables? |
Elliott Hughes | 613f814 | 2015-07-20 22:34:27 +0000 | [diff] [blame] | 158 | const char* strtab = nullptr; |
| 159 | ElfW(Sym)* symtab = nullptr; |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 160 | for (ElfW(Dyn)* d = vdso_dyn; d->d_tag != DT_NULL; ++d) { |
| 161 | if (d->d_tag == DT_STRTAB) { |
| 162 | strtab = reinterpret_cast<const char*>(vdso_addr + d->d_un.d_ptr); |
| 163 | } else if (d->d_tag == DT_SYMTAB) { |
| 164 | symtab = reinterpret_cast<ElfW(Sym)*>(vdso_addr + d->d_un.d_ptr); |
| 165 | } |
Elliott Hughes | cbcccd5 | 2023-12-12 15:10:40 -0800 | [diff] [blame^] | 166 | if (strtab && symtab) break; |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 167 | } |
Elliott Hughes | 613f814 | 2015-07-20 22:34:27 +0000 | [diff] [blame] | 168 | if (strtab == nullptr || symtab == nullptr) { |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 169 | return; |
| 170 | } |
| 171 | |
| 172 | // Are there any symbols we want? |
Elliott Hughes | cbcccd5 | 2023-12-12 15:10:40 -0800 | [diff] [blame^] | 173 | for (size_t i = 0; i < VDSO_END; ++i) { |
| 174 | for (size_t j = 0; j < symbol_count; ++j) { |
| 175 | if (strcmp(vdso[i].name, strtab + symtab[j].st_name) == 0) { |
| 176 | vdso[i].fn = reinterpret_cast<void*>(vdso_addr + symtab[j].st_value); |
| 177 | break; |
Elliott Hughes | 625993d | 2014-07-15 16:53:13 -0700 | [diff] [blame] | 178 | } |
| 179 | } |
| 180 | } |
| 181 | } |