blob: 1e36f5e6209e459c84870ba0772a00895d875e52 [file] [log] [blame]
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001/*
2 * Copyright (C) 2012 The Android Open Source Project
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in
12 * the documentation and/or other materials provided with the
13 * distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
Elliott Hughes650be4e2013-03-05 18:47:58 -080029#include "linker_phdr.h"
30
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +020031#include <errno.h>
Elliott Hughes05fc1d72015-01-28 18:02:33 -080032#include <string.h>
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +020033#include <sys/mman.h>
Elliott Hughes99d54652018-08-22 10:36:23 -070034#include <sys/prctl.h>
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +000035#include <sys/types.h>
36#include <sys/stat.h>
37#include <unistd.h>
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +020038
Elliott Hughes650be4e2013-03-05 18:47:58 -080039#include "linker.h"
Florian Mayer4edc20d2024-10-30 14:24:26 -070040#include "linker_debug.h"
Elliott Hughes4cc5a602016-11-15 16:54:16 -080041#include "linker_dlwarning.h"
Dimitry Ivanov48ec2882016-08-04 11:50:36 -070042#include "linker_globals.h"
Florian Mayer4edc20d2024-10-30 14:24:26 -070043#include "linker_logger.h"
44#include "linker_main.h"
45#include "linker_soinfo.h"
Dmitriy Ivanov3c524812015-11-20 17:28:12 -080046#include "linker_utils.h"
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +020047
Kalesh Singh377f0b92024-01-31 20:23:39 -080048#include "private/bionic_asm_note.h"
Evgenii Stepanov0a3637d2016-07-06 13:20:59 -070049#include "private/CFIShadow.h" // For kLibraryAlignment
Kalesh Singh377f0b92024-01-31 20:23:39 -080050#include "private/elf_note.h"
Dimitry Ivanov1b84afc2016-01-27 18:12:03 -080051
Kalesh Singhc5c1d192024-04-09 16:27:56 -070052#include <android-base/file.h>
Kalesh Singhb23787f2024-09-05 08:22:06 +000053#include <android-base/properties.h>
Kalesh Singhc5c1d192024-04-09 16:27:56 -070054
Elliott Hughesb5140262014-12-02 16:16:29 -080055static int GetTargetElfMachine() {
56#if defined(__arm__)
57 return EM_ARM;
58#elif defined(__aarch64__)
59 return EM_AARCH64;
60#elif defined(__i386__)
61 return EM_386;
Elliott Hughes43462702022-10-10 19:21:44 +000062#elif defined(__riscv)
63 return EM_RISCV;
Elliott Hughesb5140262014-12-02 16:16:29 -080064#elif defined(__x86_64__)
65 return EM_X86_64;
66#endif
67}
68
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +020069/**
70 TECHNICAL NOTE ON ELF LOADING.
71
72 An ELF file's program header table contains one or more PT_LOAD
73 segments, which corresponds to portions of the file that need to
74 be mapped into the process' address space.
75
76 Each loadable segment has the following important properties:
77
78 p_offset -> segment file offset
79 p_filesz -> segment file size
80 p_memsz -> segment memory size (always >= p_filesz)
81 p_vaddr -> segment's virtual address
82 p_flags -> segment flags (e.g. readable, writable, executable)
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -070083 p_align -> segment's in-memory and in-file alignment
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +020084
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -070085 We will ignore the p_paddr field of ElfW(Phdr) for now.
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +020086
87 The loadable segments can be seen as a list of [p_vaddr ... p_vaddr+p_memsz)
88 ranges of virtual addresses. A few rules apply:
89
90 - the virtual address ranges should not overlap.
91
92 - if a segment's p_filesz is smaller than its p_memsz, the extra bytes
93 between them should always be initialized to 0.
94
95 - ranges do not necessarily start or end at page boundaries. Two distinct
96 segments can have their start and end on the same page. In this case, the
97 page inherits the mapping flags of the latter segment.
98
99 Finally, the real load addrs of each segment is not p_vaddr. Instead the
100 loader decides where to load the first segment, then will load all others
101 relative to the first one to respect the initial range layout.
102
103 For example, consider the following list:
104
105 [ offset:0, filesz:0x4000, memsz:0x4000, vaddr:0x30000 ],
106 [ offset:0x4000, filesz:0x2000, memsz:0x8000, vaddr:0x40000 ],
107
108 This corresponds to two segments that cover these virtual address ranges:
109
110 0x30000...0x34000
111 0x40000...0x48000
112
113 If the loader decides to load the first segment at address 0xa0000000
114 then the segments' load address ranges will be:
115
116 0xa0030000...0xa0034000
117 0xa0040000...0xa0048000
118
119 In other words, all segments must be loaded at an address that has the same
120 constant offset from their p_vaddr value. This offset is computed as the
121 difference between the first segment's load address, and its p_vaddr value.
122
123 However, in practice, segments do _not_ start at page boundaries. Since we
124 can only memory-map at page boundaries, this means that the bias is
125 computed as:
126
Peter Collingbournebb11ee62022-05-02 12:26:16 -0700127 load_bias = phdr0_load_address - page_start(phdr0->p_vaddr)
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200128
129 (NOTE: The value must be used as a 32-bit unsigned integer, to deal with
130 possible wrap around UINT32_MAX for possible large p_vaddr values).
131
132 And that the phdr0_load_address must start at a page boundary, with
133 the segment's real content starting at:
134
Peter Collingbournebb11ee62022-05-02 12:26:16 -0700135 phdr0_load_address + page_offset(phdr0->p_vaddr)
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200136
137 Note that ELF requires the following condition to make the mmap()-ing work:
138
Peter Collingbournebb11ee62022-05-02 12:26:16 -0700139 page_offset(phdr0->p_vaddr) == page_offset(phdr0->p_offset)
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200140
141 The load_bias must be added to any p_vaddr value read from the ELF file to
142 determine the corresponding memory address.
143
144 **/
145
Kalesh Singh1dd68582024-02-01 00:14:36 -0800146static const size_t kPageSize = page_size();
147
148/*
149 * Generic PMD size calculation:
150 * - Each page table (PT) is of size 1 page.
151 * - Each page table entry (PTE) is of size 64 bits.
152 * - Each PTE locates one physical page frame (PFN) of size 1 page.
153 * - A PMD entry locates 1 page table (PT)
154 *
155 * PMD size = Num entries in a PT * page_size
156 */
157static const size_t kPmdSize = (kPageSize / sizeof(uint64_t)) * kPageSize;
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -0700158
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700159ElfReader::ElfReader()
160 : did_read_(false), did_load_(false), fd_(-1), file_offset_(0), file_size_(0), phdr_num_(0),
161 phdr_table_(nullptr), shdr_table_(nullptr), shdr_num_(0), dynamic_(nullptr), strtab_(nullptr),
Elliott Hughesd9a97792024-12-10 20:32:44 +0000162 strtab_size_(0), load_start_(nullptr), load_size_(0), load_bias_(0), max_align_(0), min_align_(0),
163 loaded_phdr_(nullptr), mapped_by_caller_(false) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700164}
165
166bool ElfReader::Read(const char* name, int fd, off64_t file_offset, off64_t file_size) {
Jiyong Park02586a22017-05-20 01:01:24 +0900167 if (did_read_) {
168 return true;
169 }
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700170 name_ = name;
171 fd_ = fd;
172 file_offset_ = file_offset;
173 file_size_ = file_size;
174
175 if (ReadElfHeader() &&
176 VerifyElfHeader() &&
177 ReadProgramHeaders() &&
Elliott Hughesd9a97792024-12-10 20:32:44 +0000178 CheckProgramHeaderAlignment() &&
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700179 ReadSectionHeaders() &&
Kalesh Singh377f0b92024-01-31 20:23:39 -0800180 ReadDynamicSection() &&
181 ReadPadSegmentNote()) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700182 did_read_ = true;
183 }
184
Elliott Hughesd9a97792024-12-10 20:32:44 +0000185 if (kPageSize == 16*1024 && min_align_ == 4096) {
Kalesh Singhb23787f2024-09-05 08:22:06 +0000186 // This prop needs to be read on 16KiB devices for each ELF where min_palign is 4KiB.
187 // It cannot be cached since the developer may toggle app compat on/off.
188 // This check will be removed once app compat is made the default on 16KiB devices.
189 should_use_16kib_app_compat_ =
Pawan Wagh8e5de062024-10-17 18:05:19 +0000190 ::android::base::GetBoolProperty("bionic.linker.16kb.app_compat.enabled", false) ||
191 get_16kb_appcompat_mode();
Kalesh Singhb23787f2024-09-05 08:22:06 +0000192 }
193
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700194 return did_read_;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200195}
196
Torne (Richard Coles)efbe9a52018-10-17 15:59:38 -0400197bool ElfReader::Load(address_space_params* address_space) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700198 CHECK(did_read_);
Jiyong Park02586a22017-05-20 01:01:24 +0900199 if (did_load_) {
200 return true;
201 }
huangchaochaobdc37962022-12-27 19:38:41 +0800202 bool reserveSuccess = ReserveAddressSpace(address_space);
203 if (reserveSuccess && LoadSegments() && FindPhdr() &&
Tamas Petz8d55d182020-02-24 14:15:25 +0100204 FindGnuPropertySection()) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700205 did_load_ = true;
Tamas Petz8d55d182020-02-24 14:15:25 +0100206#if defined(__aarch64__)
207 // For Armv8.5-A loaded executable segments may require PROT_BTI.
208 if (note_gnu_property_.IsBTICompatible()) {
Kalesh Singhb23787f2024-09-05 08:22:06 +0000209 did_load_ =
210 (phdr_table_protect_segments(phdr_table_, phdr_num_, load_bias_, should_pad_segments_,
211 should_use_16kib_app_compat_, &note_gnu_property_) == 0);
Tamas Petz8d55d182020-02-24 14:15:25 +0100212 }
213#endif
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700214 }
huangchaochaobdc37962022-12-27 19:38:41 +0800215 if (reserveSuccess && !did_load_) {
216 if (load_start_ != nullptr && load_size_ != 0) {
217 if (!mapped_by_caller_) {
218 munmap(load_start_, load_size_);
219 }
220 }
221 }
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700222
223 return did_load_;
224}
225
226const char* ElfReader::get_string(ElfW(Word) index) const {
227 CHECK(strtab_ != nullptr);
228 CHECK(index < strtab_size_);
229
230 return strtab_ + index;
Elliott Hughes650be4e2013-03-05 18:47:58 -0800231}
232
233bool ElfReader::ReadElfHeader() {
Suren Baghdasaryanc16828b2024-08-01 00:14:15 +0000234 ssize_t rc = TEMP_FAILURE_RETRY(pread64(fd_, &header_, sizeof(header_), file_offset_));
235 if (rc < 0) {
236 DL_ERR("can't read file \"%s\": %s", name_.c_str(), strerror(errno));
237 return false;
238 }
239
240 if (rc != sizeof(header_)) {
Suren Baghdasaryanea5dd952024-07-19 17:12:16 -0700241 DL_ERR("\"%s\" is too small to be an ELF executable: only found %zd bytes", name_.c_str(),
Suren Baghdasaryanc16828b2024-08-01 00:14:15 +0000242 static_cast<size_t>(rc));
Elliott Hughes650be4e2013-03-05 18:47:58 -0800243 return false;
244 }
Elliott Hughes650be4e2013-03-05 18:47:58 -0800245 return true;
246}
247
Elliott Hughes72007ee2017-04-19 17:44:57 -0700248static const char* EM_to_string(int em) {
249 if (em == EM_386) return "EM_386";
250 if (em == EM_AARCH64) return "EM_AARCH64";
251 if (em == EM_ARM) return "EM_ARM";
Ulya Trafimovichb973c752022-11-15 14:39:44 +0000252 if (em == EM_RISCV) return "EM_RISCV";
Elliott Hughes72007ee2017-04-19 17:44:57 -0700253 if (em == EM_X86_64) return "EM_X86_64";
254 return "EM_???";
255}
256
Elliott Hughes650be4e2013-03-05 18:47:58 -0800257bool ElfReader::VerifyElfHeader() {
Elliott Hughes625993d2014-07-15 16:53:13 -0700258 if (memcmp(header_.e_ident, ELFMAG, SELFMAG) != 0) {
Elliott Hughesa8971512018-06-27 14:39:06 -0700259 DL_ERR("\"%s\" has bad ELF magic: %02x%02x%02x%02x", name_.c_str(),
260 header_.e_ident[0], header_.e_ident[1], header_.e_ident[2], header_.e_ident[3]);
Elliott Hughes650be4e2013-03-05 18:47:58 -0800261 return false;
262 }
263
Elliott Hughesc00f2cb2013-10-04 17:01:33 -0700264 // Try to give a clear diagnostic for ELF class mismatches, since they're
265 // an easy mistake to make during the 32-bit/64-bit transition period.
266 int elf_class = header_.e_ident[EI_CLASS];
267#if defined(__LP64__)
268 if (elf_class != ELFCLASS64) {
269 if (elf_class == ELFCLASS32) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700270 DL_ERR("\"%s\" is 32-bit instead of 64-bit", name_.c_str());
Elliott Hughesc00f2cb2013-10-04 17:01:33 -0700271 } else {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700272 DL_ERR("\"%s\" has unknown ELF class: %d", name_.c_str(), elf_class);
Elliott Hughesc00f2cb2013-10-04 17:01:33 -0700273 }
Elliott Hughes650be4e2013-03-05 18:47:58 -0800274 return false;
275 }
Elliott Hughesc00f2cb2013-10-04 17:01:33 -0700276#else
277 if (elf_class != ELFCLASS32) {
278 if (elf_class == ELFCLASS64) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700279 DL_ERR("\"%s\" is 64-bit instead of 32-bit", name_.c_str());
Elliott Hughesc00f2cb2013-10-04 17:01:33 -0700280 } else {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700281 DL_ERR("\"%s\" has unknown ELF class: %d", name_.c_str(), elf_class);
Elliott Hughesc00f2cb2013-10-04 17:01:33 -0700282 }
283 return false;
284 }
285#endif
286
Elliott Hughes650be4e2013-03-05 18:47:58 -0800287 if (header_.e_ident[EI_DATA] != ELFDATA2LSB) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700288 DL_ERR("\"%s\" not little-endian: %d", name_.c_str(), header_.e_ident[EI_DATA]);
Elliott Hughes650be4e2013-03-05 18:47:58 -0800289 return false;
290 }
291
292 if (header_.e_type != ET_DYN) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700293 DL_ERR("\"%s\" has unexpected e_type: %d", name_.c_str(), header_.e_type);
Elliott Hughes650be4e2013-03-05 18:47:58 -0800294 return false;
295 }
296
297 if (header_.e_version != EV_CURRENT) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700298 DL_ERR("\"%s\" has unexpected e_version: %d", name_.c_str(), header_.e_version);
Elliott Hughes650be4e2013-03-05 18:47:58 -0800299 return false;
300 }
301
Elliott Hughesb5140262014-12-02 16:16:29 -0800302 if (header_.e_machine != GetTargetElfMachine()) {
Elliott Hughesd16cfac2018-09-17 15:50:09 -0700303 DL_ERR("\"%s\" is for %s (%d) instead of %s (%d)",
304 name_.c_str(),
305 EM_to_string(header_.e_machine), header_.e_machine,
306 EM_to_string(GetTargetElfMachine()), GetTargetElfMachine());
Elliott Hughes650be4e2013-03-05 18:47:58 -0800307 return false;
308 }
309
Dimitry Ivanovc73eec12016-07-22 12:50:59 -0700310 if (header_.e_shentsize != sizeof(ElfW(Shdr))) {
Elliott Hughes95c6cd72019-12-20 13:26:14 -0800311 if (get_application_target_sdk_version() >= 26) {
Dimitry Ivanov12b91872016-11-16 12:29:37 -0800312 DL_ERR_AND_LOG("\"%s\" has unsupported e_shentsize: 0x%x (expected 0x%zx)",
313 name_.c_str(), header_.e_shentsize, sizeof(ElfW(Shdr)));
314 return false;
315 }
Elliott Hughes95c6cd72019-12-20 13:26:14 -0800316 DL_WARN_documented_change(26,
Elliott Hughes9076b0c2018-02-28 11:29:45 -0800317 "invalid-elf-header_section-headers-enforced-for-api-level-26",
318 "\"%s\" has unsupported e_shentsize 0x%x (expected 0x%zx)",
319 name_.c_str(), header_.e_shentsize, sizeof(ElfW(Shdr)));
Dimitry Ivanov12b91872016-11-16 12:29:37 -0800320 add_dlwarning(name_.c_str(), "has invalid ELF header");
Dimitry Ivanovc73eec12016-07-22 12:50:59 -0700321 }
322
323 if (header_.e_shstrndx == 0) {
Elliott Hughes95c6cd72019-12-20 13:26:14 -0800324 if (get_application_target_sdk_version() >= 26) {
Dimitry Ivanov12b91872016-11-16 12:29:37 -0800325 DL_ERR_AND_LOG("\"%s\" has invalid e_shstrndx", name_.c_str());
326 return false;
327 }
Elliott Hughes95c6cd72019-12-20 13:26:14 -0800328 DL_WARN_documented_change(26,
Elliott Hughes9076b0c2018-02-28 11:29:45 -0800329 "invalid-elf-header_section-headers-enforced-for-api-level-26",
330 "\"%s\" has invalid e_shstrndx", name_.c_str());
Dimitry Ivanov12b91872016-11-16 12:29:37 -0800331 add_dlwarning(name_.c_str(), "has invalid ELF header");
Dimitry Ivanovc73eec12016-07-22 12:50:59 -0700332 }
333
Elliott Hughes650be4e2013-03-05 18:47:58 -0800334 return true;
335}
336
Dimitry Ivanovbd906752016-08-08 17:12:18 -0700337bool ElfReader::CheckFileRange(ElfW(Addr) offset, size_t size, size_t alignment) {
Dmitriy Ivanov3c524812015-11-20 17:28:12 -0800338 off64_t range_start;
339 off64_t range_end;
340
Dimitry Ivanov0c9d30f2016-07-13 17:06:36 -0700341 // Only header can be located at the 0 offset... This function called to
342 // check DYNSYM and DYNAMIC sections and phdr/shdr - none of them can be
Dimitry Ivanovebe5af72016-07-14 11:15:44 -0700343 // at offset 0.
Dimitry Ivanov0c9d30f2016-07-13 17:06:36 -0700344
345 return offset > 0 &&
346 safe_add(&range_start, file_offset_, offset) &&
Dmitriy Ivanov3c524812015-11-20 17:28:12 -0800347 safe_add(&range_end, range_start, size) &&
Dimitry Ivanovbd906752016-08-08 17:12:18 -0700348 (range_start < file_size_) &&
349 (range_end <= file_size_) &&
350 ((offset % alignment) == 0);
Dmitriy Ivanov3c524812015-11-20 17:28:12 -0800351}
352
Elliott Hughes650be4e2013-03-05 18:47:58 -0800353// Loads the program header table from an ELF file into a read-only private
354// anonymous mmap-ed block.
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700355bool ElfReader::ReadProgramHeaders() {
Elliott Hughes650be4e2013-03-05 18:47:58 -0800356 phdr_num_ = header_.e_phnum;
357
358 // Like the kernel, we only accept program header tables that
359 // are smaller than 64KiB.
Elliott Hughes0266ae52014-02-10 17:46:57 -0800360 if (phdr_num_ < 1 || phdr_num_ > 65536/sizeof(ElfW(Phdr))) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700361 DL_ERR("\"%s\" has invalid e_phnum: %zd", name_.c_str(), phdr_num_);
Elliott Hughes650be4e2013-03-05 18:47:58 -0800362 return false;
363 }
364
Dmitriy Ivanov3c524812015-11-20 17:28:12 -0800365 // Boundary checks
366 size_t size = phdr_num_ * sizeof(ElfW(Phdr));
Dimitry Ivanovbd906752016-08-08 17:12:18 -0700367 if (!CheckFileRange(header_.e_phoff, size, alignof(ElfW(Phdr)))) {
368 DL_ERR_AND_LOG("\"%s\" has invalid phdr offset/size: %zu/%zu",
369 name_.c_str(),
370 static_cast<size_t>(header_.e_phoff),
371 size);
Dmitriy Ivanov3c524812015-11-20 17:28:12 -0800372 return false;
373 }
374
Suren Baghdasaryanc16828b2024-08-01 00:14:15 +0000375 if (!phdr_fragment_.Map(fd_, file_offset_, header_.e_phoff, size)) {
Elliott Hughesf5e21d92024-07-26 11:48:19 +0000376 DL_ERR("\"%s\" phdr mmap failed: %m", name_.c_str());
Elliott Hughes650be4e2013-03-05 18:47:58 -0800377 return false;
378 }
379
Suren Baghdasaryanc16828b2024-08-01 00:14:15 +0000380 phdr_table_ = static_cast<ElfW(Phdr)*>(phdr_fragment_.data());
Elliott Hughes650be4e2013-03-05 18:47:58 -0800381 return true;
382}
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200383
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700384bool ElfReader::ReadSectionHeaders() {
385 shdr_num_ = header_.e_shnum;
386
Dmitriy Ivanovb76123f2015-11-20 10:42:02 -0800387 if (shdr_num_ == 0) {
Dimitry Ivanovbd906752016-08-08 17:12:18 -0700388 DL_ERR_AND_LOG("\"%s\" has no section headers", name_.c_str());
Dmitriy Ivanovb76123f2015-11-20 10:42:02 -0800389 return false;
390 }
391
Dmitriy Ivanov3c524812015-11-20 17:28:12 -0800392 size_t size = shdr_num_ * sizeof(ElfW(Shdr));
Dimitry Ivanovbd906752016-08-08 17:12:18 -0700393 if (!CheckFileRange(header_.e_shoff, size, alignof(const ElfW(Shdr)))) {
394 DL_ERR_AND_LOG("\"%s\" has invalid shdr offset/size: %zu/%zu",
395 name_.c_str(),
396 static_cast<size_t>(header_.e_shoff),
397 size);
Dmitriy Ivanov3c524812015-11-20 17:28:12 -0800398 return false;
399 }
400
Suren Baghdasaryanc16828b2024-08-01 00:14:15 +0000401 if (!shdr_fragment_.Map(fd_, file_offset_, header_.e_shoff, size)) {
Elliott Hughesf5e21d92024-07-26 11:48:19 +0000402 DL_ERR("\"%s\" shdr mmap failed: %m", name_.c_str());
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700403 return false;
404 }
405
Suren Baghdasaryanc16828b2024-08-01 00:14:15 +0000406 shdr_table_ = static_cast<const ElfW(Shdr)*>(shdr_fragment_.data());
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700407 return true;
408}
409
410bool ElfReader::ReadDynamicSection() {
411 // 1. Find .dynamic section (in section headers)
412 const ElfW(Shdr)* dynamic_shdr = nullptr;
413 for (size_t i = 0; i < shdr_num_; ++i) {
414 if (shdr_table_[i].sh_type == SHT_DYNAMIC) {
415 dynamic_shdr = &shdr_table_ [i];
416 break;
417 }
418 }
419
420 if (dynamic_shdr == nullptr) {
Dimitry Ivanovbd906752016-08-08 17:12:18 -0700421 DL_ERR_AND_LOG("\"%s\" .dynamic section header was not found", name_.c_str());
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700422 return false;
423 }
424
Dimitry Ivanov0c9d30f2016-07-13 17:06:36 -0700425 // Make sure dynamic_shdr offset and size matches PT_DYNAMIC phdr
426 size_t pt_dynamic_offset = 0;
427 size_t pt_dynamic_filesz = 0;
428 for (size_t i = 0; i < phdr_num_; ++i) {
429 const ElfW(Phdr)* phdr = &phdr_table_[i];
430 if (phdr->p_type == PT_DYNAMIC) {
431 pt_dynamic_offset = phdr->p_offset;
432 pt_dynamic_filesz = phdr->p_filesz;
433 }
434 }
435
436 if (pt_dynamic_offset != dynamic_shdr->sh_offset) {
Elliott Hughes95c6cd72019-12-20 13:26:14 -0800437 if (get_application_target_sdk_version() >= 26) {
Dimitry Ivanove30c17f2016-12-28 16:21:49 -0800438 DL_ERR_AND_LOG("\"%s\" .dynamic section has invalid offset: 0x%zx, "
439 "expected to match PT_DYNAMIC offset: 0x%zx",
440 name_.c_str(),
441 static_cast<size_t>(dynamic_shdr->sh_offset),
442 pt_dynamic_offset);
443 return false;
444 }
Elliott Hughes95c6cd72019-12-20 13:26:14 -0800445 DL_WARN_documented_change(26,
Elliott Hughes9076b0c2018-02-28 11:29:45 -0800446 "invalid-elf-header_section-headers-enforced-for-api-level-26",
447 "\"%s\" .dynamic section has invalid offset: 0x%zx "
448 "(expected to match PT_DYNAMIC offset 0x%zx)",
449 name_.c_str(),
450 static_cast<size_t>(dynamic_shdr->sh_offset),
451 pt_dynamic_offset);
Dimitry Ivanove30c17f2016-12-28 16:21:49 -0800452 add_dlwarning(name_.c_str(), "invalid .dynamic section");
Dimitry Ivanov0c9d30f2016-07-13 17:06:36 -0700453 }
454
455 if (pt_dynamic_filesz != dynamic_shdr->sh_size) {
Elliott Hughes95c6cd72019-12-20 13:26:14 -0800456 if (get_application_target_sdk_version() >= 26) {
Dimitry Ivanove30c17f2016-12-28 16:21:49 -0800457 DL_ERR_AND_LOG("\"%s\" .dynamic section has invalid size: 0x%zx, "
458 "expected to match PT_DYNAMIC filesz: 0x%zx",
459 name_.c_str(),
460 static_cast<size_t>(dynamic_shdr->sh_size),
461 pt_dynamic_filesz);
462 return false;
463 }
Elliott Hughes95c6cd72019-12-20 13:26:14 -0800464 DL_WARN_documented_change(26,
Elliott Hughes9076b0c2018-02-28 11:29:45 -0800465 "invalid-elf-header_section-headers-enforced-for-api-level-26",
466 "\"%s\" .dynamic section has invalid size: 0x%zx "
467 "(expected to match PT_DYNAMIC filesz 0x%zx)",
468 name_.c_str(),
469 static_cast<size_t>(dynamic_shdr->sh_size),
470 pt_dynamic_filesz);
Dimitry Ivanove30c17f2016-12-28 16:21:49 -0800471 add_dlwarning(name_.c_str(), "invalid .dynamic section");
Dimitry Ivanov0c9d30f2016-07-13 17:06:36 -0700472 }
473
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700474 if (dynamic_shdr->sh_link >= shdr_num_) {
Dimitry Ivanovbd906752016-08-08 17:12:18 -0700475 DL_ERR_AND_LOG("\"%s\" .dynamic section has invalid sh_link: %d",
476 name_.c_str(),
477 dynamic_shdr->sh_link);
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700478 return false;
479 }
480
481 const ElfW(Shdr)* strtab_shdr = &shdr_table_[dynamic_shdr->sh_link];
482
483 if (strtab_shdr->sh_type != SHT_STRTAB) {
Dimitry Ivanovbd906752016-08-08 17:12:18 -0700484 DL_ERR_AND_LOG("\"%s\" .dynamic section has invalid link(%d) sh_type: %d (expected SHT_STRTAB)",
485 name_.c_str(), dynamic_shdr->sh_link, strtab_shdr->sh_type);
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700486 return false;
487 }
488
Dimitry Ivanovbd906752016-08-08 17:12:18 -0700489 if (!CheckFileRange(dynamic_shdr->sh_offset, dynamic_shdr->sh_size, alignof(const ElfW(Dyn)))) {
490 DL_ERR_AND_LOG("\"%s\" has invalid offset/size of .dynamic section", name_.c_str());
Dmitriy Ivanov3c524812015-11-20 17:28:12 -0800491 return false;
492 }
493
Suren Baghdasaryanc16828b2024-08-01 00:14:15 +0000494 if (!dynamic_fragment_.Map(fd_, file_offset_, dynamic_shdr->sh_offset, dynamic_shdr->sh_size)) {
Elliott Hughesf5e21d92024-07-26 11:48:19 +0000495 DL_ERR("\"%s\" dynamic section mmap failed: %m", name_.c_str());
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700496 return false;
497 }
498
Suren Baghdasaryanc16828b2024-08-01 00:14:15 +0000499 dynamic_ = static_cast<const ElfW(Dyn)*>(dynamic_fragment_.data());
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700500
Dimitry Ivanovbd906752016-08-08 17:12:18 -0700501 if (!CheckFileRange(strtab_shdr->sh_offset, strtab_shdr->sh_size, alignof(const char))) {
502 DL_ERR_AND_LOG("\"%s\" has invalid offset/size of the .strtab section linked from .dynamic section",
503 name_.c_str());
Dmitriy Ivanov3c524812015-11-20 17:28:12 -0800504 return false;
505 }
506
Suren Baghdasaryanc16828b2024-08-01 00:14:15 +0000507 if (!strtab_fragment_.Map(fd_, file_offset_, strtab_shdr->sh_offset, strtab_shdr->sh_size)) {
Elliott Hughesf5e21d92024-07-26 11:48:19 +0000508 DL_ERR("\"%s\" strtab section mmap failed: %m", name_.c_str());
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700509 return false;
510 }
511
Suren Baghdasaryanc16828b2024-08-01 00:14:15 +0000512 strtab_ = static_cast<const char*>(strtab_fragment_.data());
513 strtab_size_ = strtab_fragment_.size();
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700514 return true;
515}
516
Brian Carlstrome7dffe12013-01-10 16:39:58 -0800517/* Returns the size of the extent of all the possibly non-contiguous
518 * loadable segments in an ELF program header table. This corresponds
519 * to the page-aligned size in bytes that needs to be reserved in the
520 * process' address space. If there are no loadable segments, 0 is
521 * returned.
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200522 *
Dmitriy Ivanov851135b2014-08-29 12:02:36 -0700523 * If out_min_vaddr or out_max_vaddr are not null, they will be
Brian Carlstrome7dffe12013-01-10 16:39:58 -0800524 * set to the minimum and maximum addresses of pages to be reserved,
525 * or 0 if there is nothing to load.
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200526 */
Elliott Hughes0266ae52014-02-10 17:46:57 -0800527size_t phdr_table_get_load_size(const ElfW(Phdr)* phdr_table, size_t phdr_count,
528 ElfW(Addr)* out_min_vaddr,
529 ElfW(Addr)* out_max_vaddr) {
530 ElfW(Addr) min_vaddr = UINTPTR_MAX;
531 ElfW(Addr) max_vaddr = 0;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200532
Elliott Hughes0266ae52014-02-10 17:46:57 -0800533 bool found_pt_load = false;
534 for (size_t i = 0; i < phdr_count; ++i) {
535 const ElfW(Phdr)* phdr = &phdr_table[i];
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200536
Elliott Hughes0266ae52014-02-10 17:46:57 -0800537 if (phdr->p_type != PT_LOAD) {
538 continue;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200539 }
Elliott Hughes0266ae52014-02-10 17:46:57 -0800540 found_pt_load = true;
541
542 if (phdr->p_vaddr < min_vaddr) {
543 min_vaddr = phdr->p_vaddr;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200544 }
545
Elliott Hughes0266ae52014-02-10 17:46:57 -0800546 if (phdr->p_vaddr + phdr->p_memsz > max_vaddr) {
547 max_vaddr = phdr->p_vaddr + phdr->p_memsz;
548 }
549 }
550 if (!found_pt_load) {
551 min_vaddr = 0;
552 }
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200553
Peter Collingbournebb11ee62022-05-02 12:26:16 -0700554 min_vaddr = page_start(min_vaddr);
555 max_vaddr = page_end(max_vaddr);
Elliott Hughes0266ae52014-02-10 17:46:57 -0800556
Dmitriy Ivanov851135b2014-08-29 12:02:36 -0700557 if (out_min_vaddr != nullptr) {
Elliott Hughes0266ae52014-02-10 17:46:57 -0800558 *out_min_vaddr = min_vaddr;
559 }
Dmitriy Ivanov851135b2014-08-29 12:02:36 -0700560 if (out_max_vaddr != nullptr) {
Elliott Hughes0266ae52014-02-10 17:46:57 -0800561 *out_max_vaddr = max_vaddr;
562 }
563 return max_vaddr - min_vaddr;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200564}
565
Elliott Hughesd9a97792024-12-10 20:32:44 +0000566bool ElfReader::CheckProgramHeaderAlignment() {
567 max_align_ = min_align_ = page_size();
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -0700568
Elliott Hughesd9a97792024-12-10 20:32:44 +0000569 for (size_t i = 0; i < phdr_num_; ++i) {
570 const ElfW(Phdr)* phdr = &phdr_table_[i];
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -0700571
572 // p_align must be 0, 1, or a positive, integral power of two.
573 if (phdr->p_type != PT_LOAD || ((phdr->p_align & (phdr->p_align - 1)) != 0)) {
Elliott Hughesd9a97792024-12-10 20:32:44 +0000574 // TODO: reject ELF files with bad p_align values.
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -0700575 continue;
576 }
577
Elliott Hughesd9a97792024-12-10 20:32:44 +0000578 max_align_ = std::max(max_align_, static_cast<size_t>(phdr->p_align));
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -0700579
Elliott Hughesd9a97792024-12-10 20:32:44 +0000580 if (phdr->p_align > 1) {
581 min_align_ = std::min(min_align_, static_cast<size_t>(phdr->p_align));
Steven Morelandfc89c8a2024-08-01 21:20:33 +0000582 }
Steven Morelandfc89c8a2024-08-01 21:20:33 +0000583 }
584
Elliott Hughesd9a97792024-12-10 20:32:44 +0000585 return true;
Steven Morelandfc89c8a2024-08-01 21:20:33 +0000586}
587
Evgenii Stepanovd13e9a62016-07-15 16:31:42 -0700588// Reserve a virtual address range such that if it's limits were extended to the next 2**align
589// boundary, it would not overlap with any existing mappings.
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -0700590static void* ReserveWithAlignmentPadding(size_t size, size_t mapping_align, size_t start_align,
591 void** out_gap_start, size_t* out_gap_size) {
Evgenii Stepanovd13e9a62016-07-15 16:31:42 -0700592 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS;
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -0700593 // Reserve enough space to properly align the library's start address.
594 mapping_align = std::max(mapping_align, start_align);
Peter Collingbournebb11ee62022-05-02 12:26:16 -0700595 if (mapping_align == page_size()) {
Elliott Hughes8178c412018-11-05 13:34:36 -0800596 void* mmap_ptr = mmap(nullptr, size, PROT_NONE, mmap_flags, -1, 0);
Evgenii Stepanovd13e9a62016-07-15 16:31:42 -0700597 if (mmap_ptr == MAP_FAILED) {
598 return nullptr;
599 }
600 return mmap_ptr;
601 }
602
Evgenii Stepanove0848bb2020-07-14 16:44:57 -0700603 // Minimum alignment of shared library gap. For efficiency, this should match the second level
604 // page size of the platform.
605#if defined(__LP64__)
Elliott Hughesc05035e2024-12-10 20:31:58 +0000606 constexpr size_t kGapAlignment = 2 * 1024 * 1024;
Evgenii Stepanove0848bb2020-07-14 16:44:57 -0700607#endif
608 // Maximum gap size, in the units of kGapAlignment.
609 constexpr size_t kMaxGapUnits = 32;
Evgenii Stepanovd13e9a62016-07-15 16:31:42 -0700610 // Allocate enough space so that the end of the desired region aligned up is still inside the
611 // mapping.
Elliott Hughesfed0ce92024-12-11 09:53:34 -0800612 size_t mmap_size = __builtin_align_up(size, mapping_align) + mapping_align - page_size();
Evgenii Stepanovd13e9a62016-07-15 16:31:42 -0700613 uint8_t* mmap_ptr =
614 reinterpret_cast<uint8_t*>(mmap(nullptr, mmap_size, PROT_NONE, mmap_flags, -1, 0));
615 if (mmap_ptr == MAP_FAILED) {
616 return nullptr;
617 }
Evgenii Stepanove0848bb2020-07-14 16:44:57 -0700618 size_t gap_size = 0;
Elliott Hughesfed0ce92024-12-11 09:53:34 -0800619 size_t first_byte = reinterpret_cast<size_t>(__builtin_align_up(mmap_ptr, mapping_align));
620 size_t last_byte = reinterpret_cast<size_t>(__builtin_align_down(mmap_ptr + mmap_size, mapping_align) - 1);
621#if defined(__LP64__)
622 if (first_byte / kGapAlignment != last_byte / kGapAlignment) {
Evgenii Stepanove0848bb2020-07-14 16:44:57 -0700623 // This library crosses a 2MB boundary and will fragment a new huge page.
624 // Lets take advantage of that and insert a random number of inaccessible huge pages before that
625 // to improve address randomization and make it harder to locate this library code by probing.
626 munmap(mmap_ptr, mmap_size);
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -0700627 mapping_align = std::max(mapping_align, kGapAlignment);
Evgenii Stepanove0848bb2020-07-14 16:44:57 -0700628 gap_size =
629 kGapAlignment * (is_first_stage_init() ? 1 : arc4random_uniform(kMaxGapUnits - 1) + 1);
Elliott Hughesfed0ce92024-12-11 09:53:34 -0800630 mmap_size = __builtin_align_up(size + gap_size, mapping_align) + mapping_align - page_size();
Evgenii Stepanove0848bb2020-07-14 16:44:57 -0700631 mmap_ptr = reinterpret_cast<uint8_t*>(mmap(nullptr, mmap_size, PROT_NONE, mmap_flags, -1, 0));
632 if (mmap_ptr == MAP_FAILED) {
633 return nullptr;
634 }
635 }
Elliott Hughesfed0ce92024-12-11 09:53:34 -0800636#endif
Evgenii Stepanove0848bb2020-07-14 16:44:57 -0700637
Elliott Hughesfed0ce92024-12-11 09:53:34 -0800638 uint8_t* gap_end = mmap_ptr + mmap_size;
639#if defined(__LP64__)
Evgenii Stepanove0848bb2020-07-14 16:44:57 -0700640 if (gap_size) {
Elliott Hughesfed0ce92024-12-11 09:53:34 -0800641 gap_end = __builtin_align_down(gap_end, kGapAlignment);
Evgenii Stepanove0848bb2020-07-14 16:44:57 -0700642 }
Elliott Hughesfed0ce92024-12-11 09:53:34 -0800643#endif
644 uint8_t* gap_start = gap_end - gap_size;
Evgenii Stepanovd13e9a62016-07-15 16:31:42 -0700645
Elliott Hughesfed0ce92024-12-11 09:53:34 -0800646 uint8_t* first = __builtin_align_up(mmap_ptr, mapping_align);
647 uint8_t* last = __builtin_align_down(gap_start, mapping_align) - size;
Jiyong Park31cd08f2018-06-01 19:18:56 +0900648
Tom Cherry66bc4282018-11-08 13:40:52 -0800649 // arc4random* is not available in first stage init because /dev/urandom hasn't yet been
Jiyong Park31cd08f2018-06-01 19:18:56 +0900650 // created. Don't randomize then.
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -0700651 size_t n = is_first_stage_init() ? 0 : arc4random_uniform((last - first) / start_align + 1);
652 uint8_t* start = first + n * start_align;
Evgenii Stepanove0848bb2020-07-14 16:44:57 -0700653 // Unmap the extra space around the allocation.
654 // Keep it mapped PROT_NONE on 64-bit targets where address space is plentiful to make it harder
655 // to defeat ASLR by probing for readable memory mappings.
Evgenii Stepanovd13e9a62016-07-15 16:31:42 -0700656 munmap(mmap_ptr, start - mmap_ptr);
Evgenii Stepanove0848bb2020-07-14 16:44:57 -0700657 munmap(start + size, gap_start - (start + size));
658 if (gap_end != mmap_ptr + mmap_size) {
659 munmap(gap_end, mmap_ptr + mmap_size - gap_end);
660 }
661 *out_gap_start = gap_start;
662 *out_gap_size = gap_size;
Evgenii Stepanovd13e9a62016-07-15 16:31:42 -0700663 return start;
664}
665
Elliott Hughes650be4e2013-03-05 18:47:58 -0800666// Reserve a virtual address range big enough to hold all loadable
667// segments of a program header table. This is done by creating a
668// private anonymous mmap() with PROT_NONE.
Torne (Richard Coles)efbe9a52018-10-17 15:59:38 -0400669bool ElfReader::ReserveAddressSpace(address_space_params* address_space) {
Elliott Hughes0266ae52014-02-10 17:46:57 -0800670 ElfW(Addr) min_vaddr;
Brian Carlstrome7dffe12013-01-10 16:39:58 -0800671 load_size_ = phdr_table_get_load_size(phdr_table_, phdr_num_, &min_vaddr);
Elliott Hughes650be4e2013-03-05 18:47:58 -0800672 if (load_size_ == 0) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700673 DL_ERR("\"%s\" has no loadable segments", name_.c_str());
Elliott Hughes650be4e2013-03-05 18:47:58 -0800674 return false;
675 }
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200676
Kalesh Singhce1c3cf2024-09-30 13:26:23 -0700677 if (should_use_16kib_app_compat_) {
678 // Reserve additional space for aligning the permission boundary in compat loading
679 // Up to kPageSize-kCompatPageSize additional space is needed, but reservation
680 // is done with mmap which gives kPageSize multiple-sized reservations.
681 load_size_ += kPageSize;
682 }
683
Brian Carlstrome7dffe12013-01-10 16:39:58 -0800684 uint8_t* addr = reinterpret_cast<uint8_t*>(min_vaddr);
Torne (Richard Coles)12bbb912014-02-06 14:34:21 +0000685 void* start;
Torne (Richard Coles)12bbb912014-02-06 14:34:21 +0000686
Torne (Richard Coles)efbe9a52018-10-17 15:59:38 -0400687 if (load_size_ > address_space->reserved_size) {
688 if (address_space->must_use_address) {
Torne (Richard Coles)12bbb912014-02-06 14:34:21 +0000689 DL_ERR("reserved address space %zd smaller than %zd bytes needed for \"%s\"",
Torne (Richard Coles)efbe9a52018-10-17 15:59:38 -0400690 load_size_ - address_space->reserved_size, load_size_, name_.c_str());
Torne (Richard Coles)12bbb912014-02-06 14:34:21 +0000691 return false;
692 }
Peter Collingbournebb11ee62022-05-02 12:26:16 -0700693 size_t start_alignment = page_size();
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -0700694 if (get_transparent_hugepages_supported() && get_application_target_sdk_version() >= 31) {
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -0700695 // Limit alignment to PMD size as other alignments reduce the number of
696 // bits available for ASLR for no benefit.
Elliott Hughesd9a97792024-12-10 20:32:44 +0000697 start_alignment = max_align_ == kPmdSize ? kPmdSize : page_size();
Collin Fijalkovich47d27aa2021-03-24 10:17:39 -0700698 }
699 start = ReserveWithAlignmentPadding(load_size_, kLibraryAlignment, start_alignment, &gap_start_,
700 &gap_size_);
Evgenii Stepanovd13e9a62016-07-15 16:31:42 -0700701 if (start == nullptr) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -0700702 DL_ERR("couldn't reserve %zd bytes of address space for \"%s\"", load_size_, name_.c_str());
Torne (Richard Coles)12bbb912014-02-06 14:34:21 +0000703 return false;
704 }
705 } else {
Torne (Richard Coles)efbe9a52018-10-17 15:59:38 -0400706 start = address_space->start_addr;
Evgenii Stepanove0848bb2020-07-14 16:44:57 -0700707 gap_start_ = nullptr;
708 gap_size_ = 0;
Dimitry Ivanovf45b0e92016-01-15 11:13:35 -0800709 mapped_by_caller_ = true;
Torne (Richard Coles)efbe9a52018-10-17 15:59:38 -0400710
711 // Update the reserved address space to subtract the space used by this library.
712 address_space->start_addr = reinterpret_cast<uint8_t*>(address_space->start_addr) + load_size_;
713 address_space->reserved_size -= load_size_;
Elliott Hughes650be4e2013-03-05 18:47:58 -0800714 }
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200715
Elliott Hughes650be4e2013-03-05 18:47:58 -0800716 load_start_ = start;
Brian Carlstrome7dffe12013-01-10 16:39:58 -0800717 load_bias_ = reinterpret_cast<uint8_t*>(start) - addr;
Kalesh Singhce1c3cf2024-09-30 13:26:23 -0700718
719 if (should_use_16kib_app_compat_) {
720 // In compat mode make the initial mapping RW since the ELF contents will be read
721 // into it; instead of mapped over it.
722 mprotect(reinterpret_cast<void*>(start), load_size_, PROT_READ | PROT_WRITE);
723 }
724
Elliott Hughes650be4e2013-03-05 18:47:58 -0800725 return true;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +0200726}
727
Kalesh Singhc5c1d192024-04-09 16:27:56 -0700728/*
Elliott Hughes25498992024-12-03 14:50:44 -0500729 * Returns true if the kernel supports page size migration for this process.
Kalesh Singhc5c1d192024-04-09 16:27:56 -0700730 */
731bool page_size_migration_supported() {
Elliott Hughes25498992024-12-03 14:50:44 -0500732#if defined(__LP64__)
Kalesh Singhc5c1d192024-04-09 16:27:56 -0700733 static bool pgsize_migration_enabled = []() {
734 std::string enabled;
735 if (!android::base::ReadFileToString("/sys/kernel/mm/pgsize_migration/enabled", &enabled)) {
736 return false;
737 }
738 return enabled.find("1") != std::string::npos;
739 }();
740 return pgsize_migration_enabled;
Elliott Hughes25498992024-12-03 14:50:44 -0500741#else
742 return false;
743#endif
Kalesh Singhc5c1d192024-04-09 16:27:56 -0700744}
745
Kalesh Singh377f0b92024-01-31 20:23:39 -0800746// Find the ELF note of type NT_ANDROID_TYPE_PAD_SEGMENT and check that the desc value is 1.
747bool ElfReader::ReadPadSegmentNote() {
Kalesh Singhc5c1d192024-04-09 16:27:56 -0700748 if (!page_size_migration_supported()) {
749 // Don't attempt to read the note, since segment extension isn't
750 // supported; but return true so that loading can continue normally.
751 return true;
752 }
753
Kalesh Singh377f0b92024-01-31 20:23:39 -0800754 // The ELF can have multiple PT_NOTE's, check them all
755 for (size_t i = 0; i < phdr_num_; ++i) {
756 const ElfW(Phdr)* phdr = &phdr_table_[i];
757
758 if (phdr->p_type != PT_NOTE) {
759 continue;
760 }
761
Kalesh Singh13fb3cf2024-02-08 14:58:04 -0800762 // Some obfuscated ELFs may contain "empty" PT_NOTE program headers that don't
763 // point to any part of the ELF (p_memsz == 0). Skip these since there is
764 // nothing to decode. See: b/324468126
765 if (phdr->p_memsz == 0) {
766 continue;
767 }
768
Kalesh Singh751bb8a2024-03-29 17:55:37 -0700769 // If the PT_NOTE extends beyond the file. The ELF is doing something
770 // strange -- obfuscation, embedding hidden loaders, ...
771 //
772 // It doesn't contain the pad_segment note. Skip it to avoid SIGBUS
773 // by accesses beyond the file.
774 off64_t note_end_off = file_offset_ + phdr->p_offset + phdr->p_filesz;
775 if (note_end_off > file_size_) {
776 continue;
777 }
778
Kalesh Singh377f0b92024-01-31 20:23:39 -0800779 // note_fragment is scoped to within the loop so that there is
780 // at most 1 PT_NOTE mapped at anytime during this search.
781 MappedFileFragment note_fragment;
Suren Baghdasaryanc16828b2024-08-01 00:14:15 +0000782 if (!note_fragment.Map(fd_, file_offset_, phdr->p_offset, phdr->p_memsz)) {
Kalesh Singh32b6d8c2024-02-13 18:37:12 -0800783 DL_ERR("\"%s\": PT_NOTE mmap(nullptr, %p, PROT_READ, MAP_PRIVATE, %d, %p) failed: %m",
784 name_.c_str(), reinterpret_cast<void*>(phdr->p_memsz), fd_,
785 reinterpret_cast<void*>(page_start(file_offset_ + phdr->p_offset)));
Kalesh Singh13fb3cf2024-02-08 14:58:04 -0800786 return false;
Kalesh Singh377f0b92024-01-31 20:23:39 -0800787 }
788
789 const ElfW(Nhdr)* note_hdr = nullptr;
790 const char* note_desc = nullptr;
791 if (!__get_elf_note(NT_ANDROID_TYPE_PAD_SEGMENT, "Android",
Suren Baghdasaryanc16828b2024-08-01 00:14:15 +0000792 reinterpret_cast<ElfW(Addr)>(note_fragment.data()),
Kalesh Singh377f0b92024-01-31 20:23:39 -0800793 phdr, &note_hdr, &note_desc)) {
794 continue;
795 }
796
797 if (note_hdr->n_descsz != sizeof(ElfW(Word))) {
798 DL_ERR("\"%s\" NT_ANDROID_TYPE_PAD_SEGMENT note has unexpected n_descsz: %u",
799 name_.c_str(), reinterpret_cast<unsigned int>(note_hdr->n_descsz));
800 return false;
801 }
802
803 // 1 == enabled, 0 == disabled
804 should_pad_segments_ = *reinterpret_cast<const ElfW(Word)*>(note_desc) == 1;
805 return true;
806 }
807
808 return true;
809}
810
Kalesh Singh4084b552024-03-13 13:35:49 -0700811static inline void _extend_load_segment_vma(const ElfW(Phdr)* phdr_table, size_t phdr_count,
Kalesh Singhb23787f2024-09-05 08:22:06 +0000812 size_t phdr_idx, ElfW(Addr)* p_memsz,
813 ElfW(Addr)* p_filesz, bool should_pad_segments,
814 bool should_use_16kib_app_compat) {
815 // NOTE: Segment extension is only applicable where the ELF's max-page-size > runtime page size;
816 // to save kernel VMA slab memory. 16KiB compat mode is the exact opposite scenario.
817 if (should_use_16kib_app_compat) {
818 return;
819 }
820
Kalesh Singh4084b552024-03-13 13:35:49 -0700821 const ElfW(Phdr)* phdr = &phdr_table[phdr_idx];
822 const ElfW(Phdr)* next = nullptr;
823 size_t next_idx = phdr_idx + 1;
824
Kalesh Singhe1e74792024-04-09 11:48:52 -0700825 // Don't do segment extension for p_align > 64KiB, such ELFs already existed in the
826 // field e.g. 2MiB p_align for THPs and are relatively small in number.
827 //
828 // The kernel can only represent padding for p_align up to 64KiB. This is because
829 // the kernel uses 4 available bits in the vm_area_struct to represent padding
830 // extent; and so cannot enable mitigations to avoid breaking app compatibility for
831 // p_aligns > 64KiB.
832 //
833 // Don't perform segment extension on these to avoid app compatibility issues.
834 if (phdr->p_align <= kPageSize || phdr->p_align > 64*1024 || !should_pad_segments) {
Kalesh Singh4084b552024-03-13 13:35:49 -0700835 return;
836 }
837
838 if (next_idx < phdr_count && phdr_table[next_idx].p_type == PT_LOAD) {
839 next = &phdr_table[next_idx];
840 }
841
842 // If this is the last LOAD segment, no extension is needed
843 if (!next || *p_memsz != *p_filesz) {
844 return;
845 }
846
847 ElfW(Addr) next_start = page_start(next->p_vaddr);
848 ElfW(Addr) curr_end = page_end(phdr->p_vaddr + *p_memsz);
849
850 // If adjacent segment mappings overlap, no extension is needed.
851 if (curr_end >= next_start) {
852 return;
853 }
854
855 // Extend the LOAD segment mapping to be contiguous with that of
856 // the next LOAD segment.
857 ElfW(Addr) extend = next_start - curr_end;
858 *p_memsz += extend;
859 *p_filesz += extend;
860}
861
Kalesh Singh86e04f62024-09-05 06:24:14 +0000862bool ElfReader::MapSegment(size_t seg_idx, size_t len) {
863 const ElfW(Phdr)* phdr = &phdr_table_[seg_idx];
864
865 void* start = reinterpret_cast<void*>(page_start(phdr->p_vaddr + load_bias_));
866
867 // The ELF could be being loaded directly from a zipped APK,
868 // the zip offset must be added to find the segment offset.
869 const ElfW(Addr) offset = file_offset_ + page_start(phdr->p_offset);
870
871 int prot = PFLAGS_TO_PROT(phdr->p_flags);
872
873 void* seg_addr = mmap64(start, len, prot, MAP_FIXED | MAP_PRIVATE, fd_, offset);
874
875 if (seg_addr == MAP_FAILED) {
876 DL_ERR("couldn't map \"%s\" segment %zd: %m", name_.c_str(), seg_idx);
877 return false;
878 }
879
880 // Mark segments as huge page eligible if they meet the requirements
881 if ((phdr->p_flags & PF_X) && phdr->p_align == kPmdSize &&
882 get_transparent_hugepages_supported()) {
883 madvise(seg_addr, len, MADV_HUGEPAGE);
884 }
885
886 return true;
887}
888
Kalesh Singh37bcaea2024-09-05 06:32:07 +0000889void ElfReader::ZeroFillSegment(const ElfW(Phdr)* phdr) {
Kalesh Singhb23787f2024-09-05 08:22:06 +0000890 // NOTE: In 16KiB app compat mode, the ELF mapping is anonymous, meaning that
891 // RW segments are COW-ed from the kernel's zero page. So there is no need to
892 // explicitly zero-fill until the last page's limit.
893 if (should_use_16kib_app_compat_) {
894 return;
895 }
896
Kalesh Singh37bcaea2024-09-05 06:32:07 +0000897 ElfW(Addr) seg_start = phdr->p_vaddr + load_bias_;
898 uint64_t unextended_seg_file_end = seg_start + phdr->p_filesz;
899
900 // If the segment is writable, and does not end on a page boundary,
901 // zero-fill it until the page limit.
902 //
903 // Do not attempt to zero the extended region past the first partial page,
904 // since doing so may:
905 // 1) Result in a SIGBUS, as the region is not backed by the underlying
906 // file.
907 // 2) Break the COW backing, faulting in new anon pages for a region
908 // that will not be used.
909 if ((phdr->p_flags & PF_W) != 0 && page_offset(unextended_seg_file_end) > 0) {
910 memset(reinterpret_cast<void*>(unextended_seg_file_end), 0,
911 kPageSize - page_offset(unextended_seg_file_end));
912 }
913}
914
Kalesh Singhe0f4a372024-09-05 07:07:21 +0000915void ElfReader::DropPaddingPages(const ElfW(Phdr)* phdr, uint64_t seg_file_end) {
Kalesh Singhb23787f2024-09-05 08:22:06 +0000916 // NOTE: Padding pages are only applicable where the ELF's max-page-size > runtime page size;
917 // 16KiB compat mode is the exact opposite scenario.
918 if (should_use_16kib_app_compat_) {
919 return;
920 }
921
Kalesh Singhe0f4a372024-09-05 07:07:21 +0000922 ElfW(Addr) seg_start = phdr->p_vaddr + load_bias_;
923 uint64_t unextended_seg_file_end = seg_start + phdr->p_filesz;
924
925 uint64_t pad_start = page_end(unextended_seg_file_end);
926 uint64_t pad_end = page_end(seg_file_end);
927 CHECK(pad_start <= pad_end);
928
929 uint64_t pad_len = pad_end - pad_start;
930 if (pad_len == 0 || !page_size_migration_supported()) {
931 return;
932 }
933
934 // Pages may be brought in due to readahead.
935 // Drop the padding (zero) pages, to avoid reclaim work later.
936 //
937 // NOTE: The madvise() here is special, as it also serves to hint to the
938 // kernel the portion of the LOAD segment that is padding.
939 //
940 // See: [1] https://android-review.googlesource.com/c/kernel/common/+/3032411
941 // [2] https://android-review.googlesource.com/c/kernel/common/+/3048835
942 if (madvise(reinterpret_cast<void*>(pad_start), pad_len, MADV_DONTNEED)) {
943 DL_WARN("\"%s\": madvise(0x%" PRIx64 ", 0x%" PRIx64 ", MADV_DONTNEED) failed: %m",
944 name_.c_str(), pad_start, pad_len);
945 }
946}
947
Kalesh Singh138a9552024-09-05 08:05:56 +0000948bool ElfReader::MapBssSection(const ElfW(Phdr)* phdr, ElfW(Addr) seg_page_end,
949 ElfW(Addr) seg_file_end) {
Kalesh Singhb23787f2024-09-05 08:22:06 +0000950 // NOTE: We do not need to handle .bss in 16KiB compat mode since the mapping
951 // reservation is anonymous and RW to begin with.
952 if (should_use_16kib_app_compat_) {
953 return true;
954 }
955
Kalesh Singh138a9552024-09-05 08:05:56 +0000956 // seg_file_end is now the first page address after the file content.
957 seg_file_end = page_end(seg_file_end);
958
959 if (seg_page_end <= seg_file_end) {
960 return true;
961 }
962
963 // If seg_page_end is larger than seg_file_end, we need to zero
964 // anything between them. This is done by using a private anonymous
965 // map for all extra pages
966 size_t zeromap_size = seg_page_end - seg_file_end;
967 void* zeromap =
968 mmap(reinterpret_cast<void*>(seg_file_end), zeromap_size, PFLAGS_TO_PROT(phdr->p_flags),
969 MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
970 if (zeromap == MAP_FAILED) {
971 DL_ERR("couldn't map .bss section for \"%s\": %m", name_.c_str());
972 return false;
973 }
974
975 // Set the VMA name using prctl
976 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, zeromap, zeromap_size, ".bss");
977
978 return true;
979}
980
Elliott Hughes650be4e2013-03-05 18:47:58 -0800981bool ElfReader::LoadSegments() {
Kalesh Singhce1c3cf2024-09-30 13:26:23 -0700982 // NOTE: The compat(legacy) page size (4096) must be used when aligning
983 // the 4KiB segments for loading in compat mode. The larger 16KiB page size
984 // will lead to overwriting adjacent segments since the ELF's segment(s)
985 // are not 16KiB aligned.
986 size_t seg_align = should_use_16kib_app_compat_ ? kCompatPageSize : kPageSize;
Kalesh Singhb23787f2024-09-05 08:22:06 +0000987
Kalesh Singhb23787f2024-09-05 08:22:06 +0000988 // Only enforce this on 16 KB systems with app compat disabled.
989 // Apps may rely on undefined behavior here on 4 KB systems,
990 // which is the norm before this change is introduced
Elliott Hughesd9a97792024-12-10 20:32:44 +0000991 if (kPageSize >= 16384 && min_align_ < kPageSize && !should_use_16kib_app_compat_) {
Steven Morelandfc89c8a2024-08-01 21:20:33 +0000992 DL_ERR("\"%s\" program alignment (%zu) cannot be smaller than system page size (%zu)",
Elliott Hughesd9a97792024-12-10 20:32:44 +0000993 name_.c_str(), min_align_, kPageSize);
Steven Morelandfc89c8a2024-08-01 21:20:33 +0000994 return false;
995 }
996
Kalesh Singhce1c3cf2024-09-30 13:26:23 -0700997 if (!Setup16KiBAppCompat()) {
998 DL_ERR("\"%s\" failed to setup 16KiB App Compat", name_.c_str());
999 return false;
1000 }
1001
Elliott Hughes650be4e2013-03-05 18:47:58 -08001002 for (size_t i = 0; i < phdr_num_; ++i) {
Elliott Hughes0266ae52014-02-10 17:46:57 -08001003 const ElfW(Phdr)* phdr = &phdr_table_[i];
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001004
Elliott Hughes650be4e2013-03-05 18:47:58 -08001005 if (phdr->p_type != PT_LOAD) {
1006 continue;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001007 }
Elliott Hughes650be4e2013-03-05 18:47:58 -08001008
Kalesh Singh4084b552024-03-13 13:35:49 -07001009 ElfW(Addr) p_memsz = phdr->p_memsz;
1010 ElfW(Addr) p_filesz = phdr->p_filesz;
Kalesh Singhb23787f2024-09-05 08:22:06 +00001011 _extend_load_segment_vma(phdr_table_, phdr_num_, i, &p_memsz, &p_filesz, should_pad_segments_,
1012 should_use_16kib_app_compat_);
Kalesh Singh4084b552024-03-13 13:35:49 -07001013
Elliott Hughes650be4e2013-03-05 18:47:58 -08001014 // Segment addresses in memory.
Elliott Hughes0266ae52014-02-10 17:46:57 -08001015 ElfW(Addr) seg_start = phdr->p_vaddr + load_bias_;
Kalesh Singh4084b552024-03-13 13:35:49 -07001016 ElfW(Addr) seg_end = seg_start + p_memsz;
Elliott Hughes650be4e2013-03-05 18:47:58 -08001017
Elliott Hughesfed0ce92024-12-11 09:53:34 -08001018 ElfW(Addr) seg_page_end = __builtin_align_up(seg_end, seg_align);
Elliott Hughes650be4e2013-03-05 18:47:58 -08001019
Kalesh Singh4084b552024-03-13 13:35:49 -07001020 ElfW(Addr) seg_file_end = seg_start + p_filesz;
Elliott Hughes650be4e2013-03-05 18:47:58 -08001021
1022 // File offsets.
Elliott Hughes0266ae52014-02-10 17:46:57 -08001023 ElfW(Addr) file_start = phdr->p_offset;
Kalesh Singh4084b552024-03-13 13:35:49 -07001024 ElfW(Addr) file_end = file_start + p_filesz;
Elliott Hughes650be4e2013-03-05 18:47:58 -08001025
Elliott Hughesfed0ce92024-12-11 09:53:34 -08001026 ElfW(Addr) file_page_start = __builtin_align_down(file_start, seg_align);
Elliott Hughes0266ae52014-02-10 17:46:57 -08001027 ElfW(Addr) file_length = file_end - file_page_start;
Elliott Hughes650be4e2013-03-05 18:47:58 -08001028
Dmitriy Ivanov3f987f52015-06-25 15:51:41 -07001029 if (file_size_ <= 0) {
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -07001030 DL_ERR("\"%s\" invalid file size: %" PRId64, name_.c_str(), file_size_);
Dmitriy Ivanov3f987f52015-06-25 15:51:41 -07001031 return false;
1032 }
1033
Kalesh Singh4084b552024-03-13 13:35:49 -07001034 if (file_start + phdr->p_filesz > static_cast<size_t>(file_size_)) {
Dmitriy Ivanov3f987f52015-06-25 15:51:41 -07001035 DL_ERR("invalid ELF file \"%s\" load segment[%zd]:"
1036 " p_offset (%p) + p_filesz (%p) ( = %p) past end of file (0x%" PRIx64 ")",
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -07001037 name_.c_str(), i, reinterpret_cast<void*>(phdr->p_offset),
Dmitriy Ivanov3f987f52015-06-25 15:51:41 -07001038 reinterpret_cast<void*>(phdr->p_filesz),
Kalesh Singh4084b552024-03-13 13:35:49 -07001039 reinterpret_cast<void*>(file_start + phdr->p_filesz), file_size_);
Dmitriy Ivanov3f987f52015-06-25 15:51:41 -07001040 return false;
1041 }
1042
Brian Carlstrom82dcc792013-05-21 16:49:24 -07001043 if (file_length != 0) {
Dimitry Ivanov9700bab2016-08-10 18:54:06 -07001044 int prot = PFLAGS_TO_PROT(phdr->p_flags);
Dimitry Ivanov9700bab2016-08-10 18:54:06 -07001045 if ((prot & (PROT_EXEC | PROT_WRITE)) == (PROT_EXEC | PROT_WRITE)) {
Elliott Hughes4cc5a602016-11-15 16:54:16 -08001046 // W + E PT_LOAD segments are not allowed in O.
Elliott Hughes95c6cd72019-12-20 13:26:14 -08001047 if (get_application_target_sdk_version() >= 26) {
Elliott Hughes9076b0c2018-02-28 11:29:45 -08001048 DL_ERR_AND_LOG("\"%s\": W+E load segments are not allowed", name_.c_str());
Elliott Hughes4cc5a602016-11-15 16:54:16 -08001049 return false;
1050 }
Elliott Hughes95c6cd72019-12-20 13:26:14 -08001051 DL_WARN_documented_change(26,
Elliott Hughes9076b0c2018-02-28 11:29:45 -08001052 "writable-and-executable-segments-enforced-for-api-level-26",
1053 "\"%s\" has load segments that are both writable and executable",
1054 name_.c_str());
Elliott Hughes4cc5a602016-11-15 16:54:16 -08001055 add_dlwarning(name_.c_str(), "W+E load segments");
Dimitry Ivanov9700bab2016-08-10 18:54:06 -07001056 }
1057
Kalesh Singh86e04f62024-09-05 06:24:14 +00001058 // Pass the file_length, since it may have been extended by _extend_load_segment_vma().
Kalesh Singhce1c3cf2024-09-30 13:26:23 -07001059 if (should_use_16kib_app_compat_) {
1060 if (!CompatMapSegment(i, file_length)) {
1061 return false;
1062 }
1063 } else {
1064 if (!MapSegment(i, file_length)) {
1065 return false;
1066 }
Brian Carlstrom82dcc792013-05-21 16:49:24 -07001067 }
Elliott Hughes650be4e2013-03-05 18:47:58 -08001068 }
1069
Kalesh Singh37bcaea2024-09-05 06:32:07 +00001070 ZeroFillSegment(phdr);
Kalesh Singh1d3ba112024-03-06 17:33:36 -08001071
Kalesh Singhe0f4a372024-09-05 07:07:21 +00001072 DropPaddingPages(phdr, seg_file_end);
Elliott Hughes650be4e2013-03-05 18:47:58 -08001073
Kalesh Singh138a9552024-09-05 08:05:56 +00001074 if (!MapBssSection(phdr, seg_page_end, seg_file_end)) {
1075 return false;
Elliott Hughes650be4e2013-03-05 18:47:58 -08001076 }
1077 }
1078 return true;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001079}
1080
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001081/* Used internally. Used to set the protection bits of all loaded segments
1082 * with optional extra flags (i.e. really PROT_WRITE). Used by
1083 * phdr_table_protect_segments and phdr_table_unprotect_segments.
1084 */
1085static int _phdr_table_set_load_prot(const ElfW(Phdr)* phdr_table, size_t phdr_count,
Kalesh Singh4084b552024-03-13 13:35:49 -07001086 ElfW(Addr) load_bias, int extra_prot_flags,
Kalesh Singhb23787f2024-09-05 08:22:06 +00001087 bool should_pad_segments, bool should_use_16kib_app_compat) {
Kalesh Singh4084b552024-03-13 13:35:49 -07001088 for (size_t i = 0; i < phdr_count; ++i) {
1089 const ElfW(Phdr)* phdr = &phdr_table[i];
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001090
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001091 if (phdr->p_type != PT_LOAD || (phdr->p_flags & PF_W) != 0) {
1092 continue;
1093 }
1094
Kalesh Singh4084b552024-03-13 13:35:49 -07001095 ElfW(Addr) p_memsz = phdr->p_memsz;
1096 ElfW(Addr) p_filesz = phdr->p_filesz;
Kalesh Singhb23787f2024-09-05 08:22:06 +00001097 _extend_load_segment_vma(phdr_table, phdr_count, i, &p_memsz, &p_filesz, should_pad_segments,
1098 should_use_16kib_app_compat);
Kalesh Singh4084b552024-03-13 13:35:49 -07001099
1100 ElfW(Addr) seg_page_start = page_start(phdr->p_vaddr + load_bias);
1101 ElfW(Addr) seg_page_end = page_end(phdr->p_vaddr + p_memsz + load_bias);
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001102
Tamas Petz8d55d182020-02-24 14:15:25 +01001103 int prot = PFLAGS_TO_PROT(phdr->p_flags) | extra_prot_flags;
1104 if ((prot & PROT_WRITE) != 0) {
Nick Kralevich8fdb3412015-04-01 16:57:50 -07001105 // make sure we're never simultaneously writable / executable
1106 prot &= ~PROT_EXEC;
1107 }
Tamas Petz8d55d182020-02-24 14:15:25 +01001108#if defined(__aarch64__)
1109 if ((prot & PROT_EXEC) == 0) {
1110 // Though it is not specified don't add PROT_BTI if segment is not
1111 // executable.
1112 prot &= ~PROT_BTI;
1113 }
1114#endif
Nick Kralevich8fdb3412015-04-01 16:57:50 -07001115
Tamas Petz8d55d182020-02-24 14:15:25 +01001116 int ret =
1117 mprotect(reinterpret_cast<void*>(seg_page_start), seg_page_end - seg_page_start, prot);
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001118 if (ret < 0) {
1119 return -1;
1120 }
1121 }
1122 return 0;
1123}
1124
1125/* Restore the original protection modes for all loadable segments.
1126 * You should only call this after phdr_table_unprotect_segments and
1127 * applying all relocations.
1128 *
Tamas Petz8d55d182020-02-24 14:15:25 +01001129 * AArch64: also called from linker_main and ElfReader::Load to apply
1130 * PROT_BTI for loaded main so and other so-s.
1131 *
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001132 * Input:
1133 * phdr_table -> program header table
1134 * phdr_count -> number of entries in tables
1135 * load_bias -> load bias
Kalesh Singh4084b552024-03-13 13:35:49 -07001136 * should_pad_segments -> Are segments extended to avoid gaps in the memory map
Kalesh Singhb23787f2024-09-05 08:22:06 +00001137 * should_use_16kib_app_compat -> Is the ELF being loaded in 16KiB app compat mode.
Tamas Petz8d55d182020-02-24 14:15:25 +01001138 * prop -> GnuPropertySection or nullptr
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001139 * Return:
Mitch Phillips117e45e2023-10-20 13:32:33 +00001140 * 0 on success, -1 on failure (error code in errno).
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001141 */
Tamas Petz8d55d182020-02-24 14:15:25 +01001142int phdr_table_protect_segments(const ElfW(Phdr)* phdr_table, size_t phdr_count,
Kalesh Singh4084b552024-03-13 13:35:49 -07001143 ElfW(Addr) load_bias, bool should_pad_segments,
Kalesh Singhb23787f2024-09-05 08:22:06 +00001144 bool should_use_16kib_app_compat,
Kalesh Singh4084b552024-03-13 13:35:49 -07001145 const GnuPropertySection* prop __unused) {
Tamas Petz8d55d182020-02-24 14:15:25 +01001146 int prot = 0;
1147#if defined(__aarch64__)
1148 if ((prop != nullptr) && prop->IsBTICompatible()) {
1149 prot |= PROT_BTI;
1150 }
1151#endif
Kalesh Singhb23787f2024-09-05 08:22:06 +00001152 return _phdr_table_set_load_prot(phdr_table, phdr_count, load_bias, prot, should_pad_segments,
1153 should_use_16kib_app_compat);
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001154}
1155
Florian Mayer4edc20d2024-10-30 14:24:26 -07001156static bool segment_needs_memtag_globals_remapping(const ElfW(Phdr) * phdr) {
1157 // For now, MTE globals is only supported on writeable data segments.
1158 return phdr->p_type == PT_LOAD && !(phdr->p_flags & PF_X) && (phdr->p_flags & PF_W);
1159}
1160
1161/* When MTE globals are requested by the binary, and when the hardware supports
1162 * it, remap the executable's PT_LOAD data pages to have PROT_MTE.
1163 *
1164 * Returns 0 on success, -1 on failure (error code in errno).
1165 */
1166int remap_memtag_globals_segments(const ElfW(Phdr) * phdr_table __unused,
1167 size_t phdr_count __unused, ElfW(Addr) load_bias __unused) {
1168#if defined(__aarch64__)
1169 for (const ElfW(Phdr)* phdr = phdr_table; phdr < phdr_table + phdr_count; phdr++) {
1170 if (!segment_needs_memtag_globals_remapping(phdr)) {
1171 continue;
1172 }
1173
1174 uintptr_t seg_page_start = page_start(phdr->p_vaddr) + load_bias;
1175 uintptr_t seg_page_end = page_end(phdr->p_vaddr + phdr->p_memsz) + load_bias;
1176 size_t seg_page_aligned_size = seg_page_end - seg_page_start;
1177
1178 int prot = PFLAGS_TO_PROT(phdr->p_flags);
1179 // For anonymous private mappings, it may be possible to simply mprotect()
1180 // the PROT_MTE flag over the top. For file-based mappings, this will fail,
1181 // and we'll need to fall back. We also allow PROT_WRITE here to allow
1182 // writing memory tags (in `soinfo::tag_globals()`), and set these sections
1183 // back to read-only after tags are applied (similar to RELRO).
1184 prot |= PROT_MTE;
1185 if (mprotect(reinterpret_cast<void*>(seg_page_start), seg_page_aligned_size,
1186 prot | PROT_WRITE) == 0) {
1187 continue;
1188 }
1189
1190 void* mapping_copy = mmap(nullptr, seg_page_aligned_size, PROT_READ | PROT_WRITE,
1191 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1192 linker_memcpy(mapping_copy, reinterpret_cast<void*>(seg_page_start), seg_page_aligned_size);
1193
1194 void* seg_addr = mmap(reinterpret_cast<void*>(seg_page_start), seg_page_aligned_size,
1195 prot | PROT_WRITE, MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1196 if (seg_addr == MAP_FAILED) return -1;
1197
1198 linker_memcpy(seg_addr, mapping_copy, seg_page_aligned_size);
1199 munmap(mapping_copy, seg_page_aligned_size);
1200 }
1201#endif // defined(__aarch64__)
1202 return 0;
1203}
1204
1205void protect_memtag_globals_ro_segments(const ElfW(Phdr) * phdr_table __unused,
1206 size_t phdr_count __unused, ElfW(Addr) load_bias __unused) {
1207#if defined(__aarch64__)
1208 for (const ElfW(Phdr)* phdr = phdr_table; phdr < phdr_table + phdr_count; phdr++) {
1209 int prot = PFLAGS_TO_PROT(phdr->p_flags);
1210 if (!segment_needs_memtag_globals_remapping(phdr) || (prot & PROT_WRITE)) {
1211 continue;
1212 }
1213
1214 prot |= PROT_MTE;
1215
1216 uintptr_t seg_page_start = page_start(phdr->p_vaddr) + load_bias;
1217 uintptr_t seg_page_end = page_end(phdr->p_vaddr + phdr->p_memsz) + load_bias;
1218 size_t seg_page_aligned_size = seg_page_end - seg_page_start;
1219 mprotect(reinterpret_cast<void*>(seg_page_start), seg_page_aligned_size, prot);
1220 }
1221#endif // defined(__aarch64__)
1222}
1223
1224void name_memtag_globals_segments(const ElfW(Phdr) * phdr_table, size_t phdr_count,
1225 ElfW(Addr) load_bias, const char* soname,
1226 std::list<std::string>* vma_names) {
1227 for (const ElfW(Phdr)* phdr = phdr_table; phdr < phdr_table + phdr_count; phdr++) {
1228 if (!segment_needs_memtag_globals_remapping(phdr)) {
1229 continue;
1230 }
1231
1232 uintptr_t seg_page_start = page_start(phdr->p_vaddr) + load_bias;
1233 uintptr_t seg_page_end = page_end(phdr->p_vaddr + phdr->p_memsz) + load_bias;
1234 size_t seg_page_aligned_size = seg_page_end - seg_page_start;
1235
1236 // For file-based mappings that we're now forcing to be anonymous mappings, set the VMA name to
1237 // make debugging easier.
1238 // Once we are targeting only devices that run kernel 5.10 or newer (and thus include
1239 // https://android-review.git.corp.google.com/c/kernel/common/+/1934723 which causes the
1240 // VMA_ANON_NAME to be copied into the kernel), we can get rid of the storage here.
1241 // For now, that is not the case:
1242 // https://source.android.com/docs/core/architecture/kernel/android-common#compatibility-matrix
1243 constexpr int kVmaNameLimit = 80;
1244 std::string& vma_name = vma_names->emplace_back(kVmaNameLimit, '\0');
1245 int full_vma_length =
1246 async_safe_format_buffer(vma_name.data(), kVmaNameLimit, "mt:%s+%" PRIxPTR, soname,
1247 page_start(phdr->p_vaddr)) +
1248 /* include the null terminator */ 1;
1249 // There's an upper limit of 80 characters, including the null terminator, in the anonymous VMA
1250 // name. If we run over that limit, we end up truncating the segment offset and parts of the
1251 // DSO's name, starting on the right hand side of the basename. Because the basename is the most
1252 // important thing, chop off the soname from the left hand side first.
1253 //
1254 // Example (with '#' as the null terminator):
1255 // - "mt:/data/nativetest64/bionic-unit-tests/bionic-loader-test-libs/libdlext_test.so+e000#"
1256 // is a `full_vma_length` == 86.
1257 //
1258 // We need to left-truncate (86 - 80) 6 characters from the soname, plus the
1259 // `vma_truncation_prefix`, so 9 characters total.
1260 if (full_vma_length > kVmaNameLimit) {
1261 const char vma_truncation_prefix[] = "...";
1262 int soname_truncated_bytes =
1263 full_vma_length - kVmaNameLimit + sizeof(vma_truncation_prefix) - 1;
1264 async_safe_format_buffer(vma_name.data(), kVmaNameLimit, "mt:%s%s+%" PRIxPTR,
1265 vma_truncation_prefix, soname + soname_truncated_bytes,
1266 page_start(phdr->p_vaddr));
1267 }
1268 if (prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, reinterpret_cast<void*>(seg_page_start),
1269 seg_page_aligned_size, vma_name.data()) != 0) {
1270 DL_WARN("Failed to rename memtag global segment: %m");
1271 }
1272 }
1273}
1274
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001275/* Change the protection of all loaded segments in memory to writable.
1276 * This is useful before performing relocations. Once completed, you
1277 * will have to call phdr_table_protect_segments to restore the original
1278 * protection flags on all segments.
1279 *
1280 * Note that some writable segments can also have their content turned
1281 * to read-only by calling phdr_table_protect_gnu_relro. This is no
1282 * performed here.
1283 *
1284 * Input:
1285 * phdr_table -> program header table
1286 * phdr_count -> number of entries in tables
1287 * load_bias -> load bias
Kalesh Singh4084b552024-03-13 13:35:49 -07001288 * should_pad_segments -> Are segments extended to avoid gaps in the memory map
Kalesh Singhb23787f2024-09-05 08:22:06 +00001289 * should_use_16kib_app_compat -> Is the ELF being loaded in 16KiB app compat mode.
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001290 * Return:
Mitch Phillips117e45e2023-10-20 13:32:33 +00001291 * 0 on success, -1 on failure (error code in errno).
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001292 */
Kalesh Singhb23787f2024-09-05 08:22:06 +00001293int phdr_table_unprotect_segments(const ElfW(Phdr)* phdr_table, size_t phdr_count,
1294 ElfW(Addr) load_bias, bool should_pad_segments,
1295 bool should_use_16kib_app_compat) {
Kalesh Singh4084b552024-03-13 13:35:49 -07001296 return _phdr_table_set_load_prot(phdr_table, phdr_count, load_bias, PROT_WRITE,
Kalesh Singhb23787f2024-09-05 08:22:06 +00001297 should_pad_segments, should_use_16kib_app_compat);
Dimitry Ivanov56be6ed2015-04-01 21:18:48 +00001298}
1299
Kalesh Singh702d9b02024-03-13 13:38:04 -07001300static inline void _extend_gnu_relro_prot_end(const ElfW(Phdr)* relro_phdr,
1301 const ElfW(Phdr)* phdr_table, size_t phdr_count,
1302 ElfW(Addr) load_bias, ElfW(Addr)* seg_page_end,
Kalesh Singhb23787f2024-09-05 08:22:06 +00001303 bool should_pad_segments,
1304 bool should_use_16kib_app_compat) {
Kalesh Singh702d9b02024-03-13 13:38:04 -07001305 // Find the index and phdr of the LOAD containing the GNU_RELRO segment
1306 for (size_t index = 0; index < phdr_count; ++index) {
1307 const ElfW(Phdr)* phdr = &phdr_table[index];
1308
1309 if (phdr->p_type == PT_LOAD && phdr->p_vaddr == relro_phdr->p_vaddr) {
1310 // If the PT_GNU_RELRO mem size is not at least as large as the corresponding
1311 // LOAD segment mem size, we need to protect only a partial region of the
1312 // LOAD segment and therefore cannot avoid a VMA split.
1313 //
1314 // Note: Don't check the page-aligned mem sizes since the extended protection
1315 // may incorrectly write protect non-relocation data.
1316 //
1317 // Example:
1318 //
1319 // |---- 3K ----|-- 1K --|---- 3K ---- |-- 1K --|
1320 // ----------------------------------------------------------------
1321 // | | | | |
1322 // SEG X | RO | RO | RW | | SEG Y
1323 // | | | | |
1324 // ----------------------------------------------------------------
1325 // | | |
1326 // | | |
1327 // | | |
1328 // relro_vaddr relro_vaddr relro_vaddr
1329 // (load_vaddr) + +
1330 // relro_memsz load_memsz
1331 //
1332 // ----------------------------------------------------------------
1333 // | PAGE | PAGE |
1334 // ----------------------------------------------------------------
1335 // | Potential |
1336 // |----- Extended RO ----|
1337 // | Protection |
1338 //
1339 // If the check below uses page aligned mem sizes it will cause incorrect write
1340 // protection of the 3K RW part of the LOAD segment containing the GNU_RELRO.
1341 if (relro_phdr->p_memsz < phdr->p_memsz) {
1342 return;
1343 }
1344
1345 ElfW(Addr) p_memsz = phdr->p_memsz;
1346 ElfW(Addr) p_filesz = phdr->p_filesz;
1347
1348 // Attempt extending the VMA (mprotect range). Without extending the range,
1349 // mprotect will only RO protect a part of the extended RW LOAD segment, which
1350 // will leave an extra split RW VMA (the gap).
1351 _extend_load_segment_vma(phdr_table, phdr_count, index, &p_memsz, &p_filesz,
Kalesh Singhb23787f2024-09-05 08:22:06 +00001352 should_pad_segments, should_use_16kib_app_compat);
Kalesh Singh702d9b02024-03-13 13:38:04 -07001353
1354 *seg_page_end = page_end(phdr->p_vaddr + p_memsz + load_bias);
1355 return;
1356 }
1357 }
1358}
1359
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001360/* Used internally by phdr_table_protect_gnu_relro and
1361 * phdr_table_unprotect_gnu_relro.
1362 */
Elliott Hughes0266ae52014-02-10 17:46:57 -08001363static int _phdr_table_set_gnu_relro_prot(const ElfW(Phdr)* phdr_table, size_t phdr_count,
Kalesh Singh702d9b02024-03-13 13:38:04 -07001364 ElfW(Addr) load_bias, int prot_flags,
Kalesh Singhb23787f2024-09-05 08:22:06 +00001365 bool should_pad_segments,
1366 bool should_use_16kib_app_compat) {
Elliott Hughes0266ae52014-02-10 17:46:57 -08001367 const ElfW(Phdr)* phdr = phdr_table;
1368 const ElfW(Phdr)* phdr_limit = phdr + phdr_count;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001369
Elliott Hughes0266ae52014-02-10 17:46:57 -08001370 for (phdr = phdr_table; phdr < phdr_limit; phdr++) {
1371 if (phdr->p_type != PT_GNU_RELRO) {
1372 continue;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001373 }
Elliott Hughes0266ae52014-02-10 17:46:57 -08001374
1375 // Tricky: what happens when the relro segment does not start
1376 // or end at page boundaries? We're going to be over-protective
1377 // here and put every page touched by the segment as read-only.
1378
1379 // This seems to match Ian Lance Taylor's description of the
1380 // feature at http://www.airs.com/blog/archives/189.
1381
1382 // Extract:
1383 // Note that the current dynamic linker code will only work
1384 // correctly if the PT_GNU_RELRO segment starts on a page
1385 // boundary. This is because the dynamic linker rounds the
1386 // p_vaddr field down to the previous page boundary. If
1387 // there is anything on the page which should not be read-only,
1388 // the program is likely to fail at runtime. So in effect the
1389 // linker must only emit a PT_GNU_RELRO segment if it ensures
1390 // that it starts on a page boundary.
Zheng Pan9535c322024-02-14 00:04:10 +00001391 ElfW(Addr) seg_page_start = page_start(phdr->p_vaddr) + load_bias;
1392 ElfW(Addr) seg_page_end = page_end(phdr->p_vaddr + phdr->p_memsz) + load_bias;
Kalesh Singh702d9b02024-03-13 13:38:04 -07001393 _extend_gnu_relro_prot_end(phdr, phdr_table, phdr_count, load_bias, &seg_page_end,
Kalesh Singhb23787f2024-09-05 08:22:06 +00001394 should_pad_segments, should_use_16kib_app_compat);
Elliott Hughes0266ae52014-02-10 17:46:57 -08001395
Elliott Hughesfaf05ba2014-02-11 16:59:37 -08001396 int ret = mprotect(reinterpret_cast<void*>(seg_page_start),
Elliott Hughes0266ae52014-02-10 17:46:57 -08001397 seg_page_end - seg_page_start,
1398 prot_flags);
1399 if (ret < 0) {
1400 return -1;
1401 }
1402 }
1403 return 0;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001404}
1405
1406/* Apply GNU relro protection if specified by the program header. This will
1407 * turn some of the pages of a writable PT_LOAD segment to read-only, as
1408 * specified by one or more PT_GNU_RELRO segments. This must be always
1409 * performed after relocations.
1410 *
Ard Biesheuvel12c78bb2012-08-14 12:30:09 +02001411 * The areas typically covered are .got and .data.rel.ro, these are
1412 * read-only from the program's POV, but contain absolute addresses
1413 * that need to be relocated before use.
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001414 *
1415 * Input:
1416 * phdr_table -> program header table
Elliott Hughes105bc262012-08-15 16:56:00 -07001417 * phdr_count -> number of entries in tables
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001418 * load_bias -> load bias
Kalesh Singh702d9b02024-03-13 13:38:04 -07001419 * should_pad_segments -> Were segments extended to avoid gaps in the memory map
Kalesh Singhb23787f2024-09-05 08:22:06 +00001420 * should_use_16kib_app_compat -> Is the ELF being loaded in 16KiB app compat mode.
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001421 * Return:
Mitch Phillips117e45e2023-10-20 13:32:33 +00001422 * 0 on success, -1 on failure (error code in errno).
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001423 */
Kalesh Singh702d9b02024-03-13 13:38:04 -07001424int phdr_table_protect_gnu_relro(const ElfW(Phdr)* phdr_table, size_t phdr_count,
Kalesh Singhb23787f2024-09-05 08:22:06 +00001425 ElfW(Addr) load_bias, bool should_pad_segments,
1426 bool should_use_16kib_app_compat) {
Kalesh Singh702d9b02024-03-13 13:38:04 -07001427 return _phdr_table_set_gnu_relro_prot(phdr_table, phdr_count, load_bias, PROT_READ,
Kalesh Singhb23787f2024-09-05 08:22:06 +00001428 should_pad_segments, should_use_16kib_app_compat);
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001429}
1430
Kalesh Singhce1c3cf2024-09-30 13:26:23 -07001431/*
1432 * Apply RX protection to the compat relro region of the ELF being loaded in
1433 * 16KiB compat mode.
1434 *
1435 * Input:
1436 * start -> start address of the compat relro region.
1437 * size -> size of the compat relro region in bytes.
1438 * Return:
1439 * 0 on success, -1 on failure (error code in errno).
1440 */
1441int phdr_table_protect_gnu_relro_16kib_compat(ElfW(Addr) start, ElfW(Addr) size) {
1442 return mprotect(reinterpret_cast<void*>(start), size, PROT_READ | PROT_EXEC);
1443}
1444
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001445/* Serialize the GNU relro segments to the given file descriptor. This can be
1446 * performed after relocations to allow another process to later share the
1447 * relocated segment, if it was loaded at the same address.
1448 *
1449 * Input:
1450 * phdr_table -> program header table
1451 * phdr_count -> number of entries in tables
1452 * load_bias -> load bias
1453 * fd -> writable file descriptor to use
Torne (Richard Coles)fa9f7f22019-04-02 17:04:42 -04001454 * file_offset -> pointer to offset into file descriptor to use/update
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001455 * Return:
Mitch Phillips117e45e2023-10-20 13:32:33 +00001456 * 0 on success, -1 on failure (error code in errno).
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001457 */
Dmitriy Ivanov20d89cb2015-03-30 18:43:38 -07001458int phdr_table_serialize_gnu_relro(const ElfW(Phdr)* phdr_table,
1459 size_t phdr_count,
1460 ElfW(Addr) load_bias,
Torne (Richard Coles)fa9f7f22019-04-02 17:04:42 -04001461 int fd,
1462 size_t* file_offset) {
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001463 const ElfW(Phdr)* phdr = phdr_table;
1464 const ElfW(Phdr)* phdr_limit = phdr + phdr_count;
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001465
1466 for (phdr = phdr_table; phdr < phdr_limit; phdr++) {
1467 if (phdr->p_type != PT_GNU_RELRO) {
1468 continue;
1469 }
1470
Peter Collingbournebb11ee62022-05-02 12:26:16 -07001471 ElfW(Addr) seg_page_start = page_start(phdr->p_vaddr) + load_bias;
1472 ElfW(Addr) seg_page_end = page_end(phdr->p_vaddr + phdr->p_memsz) + load_bias;
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001473 ssize_t size = seg_page_end - seg_page_start;
1474
1475 ssize_t written = TEMP_FAILURE_RETRY(write(fd, reinterpret_cast<void*>(seg_page_start), size));
1476 if (written != size) {
1477 return -1;
1478 }
1479 void* map = mmap(reinterpret_cast<void*>(seg_page_start), size, PROT_READ,
Torne (Richard Coles)fa9f7f22019-04-02 17:04:42 -04001480 MAP_PRIVATE|MAP_FIXED, fd, *file_offset);
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001481 if (map == MAP_FAILED) {
1482 return -1;
1483 }
Torne (Richard Coles)fa9f7f22019-04-02 17:04:42 -04001484 *file_offset += size;
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001485 }
1486 return 0;
1487}
1488
1489/* Where possible, replace the GNU relro segments with mappings of the given
1490 * file descriptor. This can be performed after relocations to allow a file
1491 * previously created by phdr_table_serialize_gnu_relro in another process to
1492 * replace the dirty relocated pages, saving memory, if it was loaded at the
1493 * same address. We have to compare the data before we map over it, since some
1494 * parts of the relro segment may not be identical due to other libraries in
1495 * the process being loaded at different addresses.
1496 *
1497 * Input:
1498 * phdr_table -> program header table
1499 * phdr_count -> number of entries in tables
1500 * load_bias -> load bias
1501 * fd -> readable file descriptor to use
Torne (Richard Coles)efbe9a52018-10-17 15:59:38 -04001502 * file_offset -> pointer to offset into file descriptor to use/update
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001503 * Return:
Mitch Phillips117e45e2023-10-20 13:32:33 +00001504 * 0 on success, -1 on failure (error code in errno).
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001505 */
Dmitriy Ivanov20d89cb2015-03-30 18:43:38 -07001506int phdr_table_map_gnu_relro(const ElfW(Phdr)* phdr_table,
1507 size_t phdr_count,
1508 ElfW(Addr) load_bias,
Torne (Richard Coles)efbe9a52018-10-17 15:59:38 -04001509 int fd,
1510 size_t* file_offset) {
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001511 // Map the file at a temporary location so we can compare its contents.
1512 struct stat file_stat;
1513 if (TEMP_FAILURE_RETRY(fstat(fd, &file_stat)) != 0) {
1514 return -1;
1515 }
1516 off_t file_size = file_stat.st_size;
Dmitriy Ivanov851135b2014-08-29 12:02:36 -07001517 void* temp_mapping = nullptr;
Torne (Richard Coles)26ec9672014-04-30 15:48:40 +01001518 if (file_size > 0) {
Dmitriy Ivanov851135b2014-08-29 12:02:36 -07001519 temp_mapping = mmap(nullptr, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
Torne (Richard Coles)26ec9672014-04-30 15:48:40 +01001520 if (temp_mapping == MAP_FAILED) {
1521 return -1;
1522 }
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001523 }
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001524
1525 // Iterate over the relro segments and compare/remap the pages.
1526 const ElfW(Phdr)* phdr = phdr_table;
1527 const ElfW(Phdr)* phdr_limit = phdr + phdr_count;
1528
1529 for (phdr = phdr_table; phdr < phdr_limit; phdr++) {
1530 if (phdr->p_type != PT_GNU_RELRO) {
1531 continue;
1532 }
1533
Peter Collingbournebb11ee62022-05-02 12:26:16 -07001534 ElfW(Addr) seg_page_start = page_start(phdr->p_vaddr) + load_bias;
1535 ElfW(Addr) seg_page_end = page_end(phdr->p_vaddr + phdr->p_memsz) + load_bias;
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001536
Torne (Richard Coles)efbe9a52018-10-17 15:59:38 -04001537 char* file_base = static_cast<char*>(temp_mapping) + *file_offset;
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001538 char* mem_base = reinterpret_cast<char*>(seg_page_start);
1539 size_t match_offset = 0;
1540 size_t size = seg_page_end - seg_page_start;
1541
Torne (Richard Coles)efbe9a52018-10-17 15:59:38 -04001542 if (file_size - *file_offset < size) {
Torne (Richard Coles)26ec9672014-04-30 15:48:40 +01001543 // File is too short to compare to this segment. The contents are likely
1544 // different as well (it's probably for a different library version) so
1545 // just don't bother checking.
1546 break;
1547 }
1548
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001549 while (match_offset < size) {
1550 // Skip over dissimilar pages.
1551 while (match_offset < size &&
Peter Collingbournebb11ee62022-05-02 12:26:16 -07001552 memcmp(mem_base + match_offset, file_base + match_offset, page_size()) != 0) {
1553 match_offset += page_size();
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001554 }
1555
1556 // Count similar pages.
1557 size_t mismatch_offset = match_offset;
1558 while (mismatch_offset < size &&
Peter Collingbournebb11ee62022-05-02 12:26:16 -07001559 memcmp(mem_base + mismatch_offset, file_base + mismatch_offset, page_size()) == 0) {
1560 mismatch_offset += page_size();
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001561 }
1562
1563 // Map over similar pages.
1564 if (mismatch_offset > match_offset) {
1565 void* map = mmap(mem_base + match_offset, mismatch_offset - match_offset,
Torne (Richard Coles)efbe9a52018-10-17 15:59:38 -04001566 PROT_READ, MAP_PRIVATE|MAP_FIXED, fd, *file_offset + match_offset);
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001567 if (map == MAP_FAILED) {
1568 munmap(temp_mapping, file_size);
1569 return -1;
1570 }
1571 }
1572
1573 match_offset = mismatch_offset;
1574 }
1575
1576 // Add to the base file offset in case there are multiple relro segments.
Torne (Richard Coles)efbe9a52018-10-17 15:59:38 -04001577 *file_offset += size;
Torne (Richard Coles)183ad9d2014-02-27 13:18:00 +00001578 }
1579 munmap(temp_mapping, file_size);
1580 return 0;
1581}
1582
1583
Elliott Hughes4eeb1f12013-10-25 17:38:02 -07001584#if defined(__arm__)
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001585/* Return the address and size of the .ARM.exidx section in memory,
1586 * if present.
1587 *
1588 * Input:
1589 * phdr_table -> program header table
Elliott Hughes105bc262012-08-15 16:56:00 -07001590 * phdr_count -> number of entries in tables
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001591 * load_bias -> load bias
1592 * Output:
Dmitriy Ivanov851135b2014-08-29 12:02:36 -07001593 * arm_exidx -> address of table in memory (null on failure).
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001594 * arm_exidx_count -> number of items in table (0 on failure).
1595 * Return:
Mitch Phillips117e45e2023-10-20 13:32:33 +00001596 * 0 on success, -1 on failure (_no_ error code in errno)
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001597 */
Elliott Hughes0266ae52014-02-10 17:46:57 -08001598int phdr_table_get_arm_exidx(const ElfW(Phdr)* phdr_table, size_t phdr_count,
1599 ElfW(Addr) load_bias,
Dmitriy Ivanov1649e7e2015-01-22 16:04:25 -08001600 ElfW(Addr)** arm_exidx, size_t* arm_exidx_count) {
Elliott Hughes0266ae52014-02-10 17:46:57 -08001601 const ElfW(Phdr)* phdr = phdr_table;
1602 const ElfW(Phdr)* phdr_limit = phdr + phdr_count;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001603
Elliott Hughes0266ae52014-02-10 17:46:57 -08001604 for (phdr = phdr_table; phdr < phdr_limit; phdr++) {
1605 if (phdr->p_type != PT_ARM_EXIDX) {
1606 continue;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001607 }
Elliott Hughes0266ae52014-02-10 17:46:57 -08001608
1609 *arm_exidx = reinterpret_cast<ElfW(Addr)*>(load_bias + phdr->p_vaddr);
Dmitriy Ivanov1649e7e2015-01-22 16:04:25 -08001610 *arm_exidx_count = phdr->p_memsz / 8;
Elliott Hughes0266ae52014-02-10 17:46:57 -08001611 return 0;
1612 }
Dmitriy Ivanov851135b2014-08-29 12:02:36 -07001613 *arm_exidx = nullptr;
Elliott Hughes0266ae52014-02-10 17:46:57 -08001614 *arm_exidx_count = 0;
1615 return -1;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001616}
Elliott Hughes4eeb1f12013-10-25 17:38:02 -07001617#endif
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001618
Ard Biesheuvel12c78bb2012-08-14 12:30:09 +02001619/* Return the address and size of the ELF file's .dynamic section in memory,
Dmitriy Ivanov851135b2014-08-29 12:02:36 -07001620 * or null if missing.
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001621 *
1622 * Input:
1623 * phdr_table -> program header table
Elliott Hughes105bc262012-08-15 16:56:00 -07001624 * phdr_count -> number of entries in tables
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001625 * load_bias -> load bias
Ard Biesheuvel12c78bb2012-08-14 12:30:09 +02001626 * Output:
Dmitriy Ivanov851135b2014-08-29 12:02:36 -07001627 * dynamic -> address of table in memory (null on failure).
Ningsheng Jiane93be992014-09-16 15:22:10 +08001628 * dynamic_flags -> protection flags for section (unset on failure)
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001629 * Return:
Ard Biesheuvel12c78bb2012-08-14 12:30:09 +02001630 * void
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001631 */
Elliott Hughes0266ae52014-02-10 17:46:57 -08001632void phdr_table_get_dynamic_section(const ElfW(Phdr)* phdr_table, size_t phdr_count,
Ningsheng Jiane93be992014-09-16 15:22:10 +08001633 ElfW(Addr) load_bias, ElfW(Dyn)** dynamic,
1634 ElfW(Word)* dynamic_flags) {
Dmitriy Ivanov498eb182014-09-05 14:57:59 -07001635 *dynamic = nullptr;
Dmitriy Ivanov20d89cb2015-03-30 18:43:38 -07001636 for (size_t i = 0; i<phdr_count; ++i) {
1637 const ElfW(Phdr)& phdr = phdr_table[i];
1638 if (phdr.p_type == PT_DYNAMIC) {
1639 *dynamic = reinterpret_cast<ElfW(Dyn)*>(load_bias + phdr.p_vaddr);
Ningsheng Jiane93be992014-09-16 15:22:10 +08001640 if (dynamic_flags) {
Dmitriy Ivanov20d89cb2015-03-30 18:43:38 -07001641 *dynamic_flags = phdr.p_flags;
Ningsheng Jiane93be992014-09-16 15:22:10 +08001642 }
Dmitriy Ivanov14669a92014-09-05 16:42:53 -07001643 return;
1644 }
Elliott Hughes0266ae52014-02-10 17:46:57 -08001645 }
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001646}
1647
Evgenii Stepanovd640b222015-07-10 17:54:01 -07001648/* Return the program interpreter string, or nullptr if missing.
1649 *
1650 * Input:
1651 * phdr_table -> program header table
1652 * phdr_count -> number of entries in tables
1653 * load_bias -> load bias
1654 * Return:
1655 * pointer to the program interpreter string.
1656 */
Tamas Petz8d55d182020-02-24 14:15:25 +01001657const char* phdr_table_get_interpreter_name(const ElfW(Phdr)* phdr_table, size_t phdr_count,
Evgenii Stepanovd640b222015-07-10 17:54:01 -07001658 ElfW(Addr) load_bias) {
1659 for (size_t i = 0; i<phdr_count; ++i) {
1660 const ElfW(Phdr)& phdr = phdr_table[i];
1661 if (phdr.p_type == PT_INTERP) {
1662 return reinterpret_cast<const char*>(load_bias + phdr.p_vaddr);
1663 }
1664 }
1665 return nullptr;
1666}
1667
Robert Grosse4544d9f2014-10-15 14:32:19 -07001668// Sets loaded_phdr_ to the address of the program header table as it appears
1669// in the loaded segments in memory. This is in contrast with phdr_table_,
1670// which is temporary and will be released before the library is relocated.
Elliott Hughes650be4e2013-03-05 18:47:58 -08001671bool ElfReader::FindPhdr() {
Elliott Hughes0266ae52014-02-10 17:46:57 -08001672 const ElfW(Phdr)* phdr_limit = phdr_table_ + phdr_num_;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001673
Elliott Hughes650be4e2013-03-05 18:47:58 -08001674 // If there is a PT_PHDR, use it directly.
Elliott Hughes0266ae52014-02-10 17:46:57 -08001675 for (const ElfW(Phdr)* phdr = phdr_table_; phdr < phdr_limit; ++phdr) {
Elliott Hughes650be4e2013-03-05 18:47:58 -08001676 if (phdr->p_type == PT_PHDR) {
1677 return CheckPhdr(load_bias_ + phdr->p_vaddr);
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001678 }
Elliott Hughes650be4e2013-03-05 18:47:58 -08001679 }
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001680
Elliott Hughes650be4e2013-03-05 18:47:58 -08001681 // Otherwise, check the first loadable segment. If its file offset
1682 // is 0, it starts with the ELF header, and we can trivially find the
1683 // loaded program header from it.
Elliott Hughes0266ae52014-02-10 17:46:57 -08001684 for (const ElfW(Phdr)* phdr = phdr_table_; phdr < phdr_limit; ++phdr) {
Elliott Hughes650be4e2013-03-05 18:47:58 -08001685 if (phdr->p_type == PT_LOAD) {
1686 if (phdr->p_offset == 0) {
Elliott Hughes0266ae52014-02-10 17:46:57 -08001687 ElfW(Addr) elf_addr = load_bias_ + phdr->p_vaddr;
Elliott Hughesfaf05ba2014-02-11 16:59:37 -08001688 const ElfW(Ehdr)* ehdr = reinterpret_cast<const ElfW(Ehdr)*>(elf_addr);
Elliott Hughes0266ae52014-02-10 17:46:57 -08001689 ElfW(Addr) offset = ehdr->e_phoff;
Dmitriy Ivanov1649e7e2015-01-22 16:04:25 -08001690 return CheckPhdr(reinterpret_cast<ElfW(Addr)>(ehdr) + offset);
Elliott Hughes650be4e2013-03-05 18:47:58 -08001691 }
1692 break;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001693 }
Elliott Hughes650be4e2013-03-05 18:47:58 -08001694 }
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001695
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -07001696 DL_ERR("can't find loaded phdr for \"%s\"", name_.c_str());
Elliott Hughes650be4e2013-03-05 18:47:58 -08001697 return false;
1698}
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001699
Tamas Petz8d55d182020-02-24 14:15:25 +01001700// Tries to find .note.gnu.property section.
1701// It is not considered an error if such section is missing.
1702bool ElfReader::FindGnuPropertySection() {
1703#if defined(__aarch64__)
1704 note_gnu_property_ = GnuPropertySection(phdr_table_, phdr_num_, load_start(), name_.c_str());
1705#endif
1706 return true;
1707}
1708
Elliott Hughes650be4e2013-03-05 18:47:58 -08001709// Ensures that our program header is actually within a loadable
1710// segment. This should help catch badly-formed ELF files that
1711// would cause the linker to crash later when trying to access it.
Elliott Hughes0266ae52014-02-10 17:46:57 -08001712bool ElfReader::CheckPhdr(ElfW(Addr) loaded) {
1713 const ElfW(Phdr)* phdr_limit = phdr_table_ + phdr_num_;
1714 ElfW(Addr) loaded_end = loaded + (phdr_num_ * sizeof(ElfW(Phdr)));
Dmitriy Ivanovcf1cbbe2015-10-19 16:57:46 -07001715 for (const ElfW(Phdr)* phdr = phdr_table_; phdr < phdr_limit; ++phdr) {
Elliott Hughes650be4e2013-03-05 18:47:58 -08001716 if (phdr->p_type != PT_LOAD) {
1717 continue;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001718 }
Elliott Hughes0266ae52014-02-10 17:46:57 -08001719 ElfW(Addr) seg_start = phdr->p_vaddr + load_bias_;
1720 ElfW(Addr) seg_end = phdr->p_filesz + seg_start;
Elliott Hughes650be4e2013-03-05 18:47:58 -08001721 if (seg_start <= loaded && loaded_end <= seg_end) {
Elliott Hughes0266ae52014-02-10 17:46:57 -08001722 loaded_phdr_ = reinterpret_cast<const ElfW(Phdr)*>(loaded);
Elliott Hughes650be4e2013-03-05 18:47:58 -08001723 return true;
1724 }
1725 }
Dmitriy Ivanov4f7a7ad2015-10-15 12:07:25 -07001726 DL_ERR("\"%s\" loaded phdr %p not in loadable segment",
1727 name_.c_str(), reinterpret_cast<void*>(loaded));
Elliott Hughes650be4e2013-03-05 18:47:58 -08001728 return false;
David 'Digit' Turnerc1bd5592012-06-19 11:21:29 +02001729}