linker: map large portion of ELF file to read its fragments
ElfReader::Read() maps ELF file fragments separately to read their
data, which leads to at least 4 separate mmap() syscalls and additional
mmap() call for every PT_NOTE section. This can be replaced with one
mmap() syscall which covers large enough address space to include all
required areas in majority of cases and use the old approach for a few
outliers. Local testing on a Pixel 7 device reveals that 1MB is enough
to cover 97.5% of the cases, therefore we map the first 1MB of the Elf
file (which starts at file_offset_). pread64() syscall in ReadElfHeader()
can also be eliminated since it's always at the beginning of the Elf
file and will be included in this large mapping.
This reduction in syscalls should benefit app launch time since
ElfReader::Read() is used by load_library(), which is one of the heavily
used functions during application launch.
Bug: 322132947
Test: atest -c linker-unit-tests
Change-Id: I7b25151aba52da4116c2b6afd8a254646d0b622c
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
diff --git a/linker/linker_phdr.cpp b/linker/linker_phdr.cpp
index b9229ca..cf1cfb3 100644
--- a/linker/linker_phdr.cpp
+++ b/linker/linker_phdr.cpp
@@ -221,17 +221,22 @@
}
bool ElfReader::ReadElfHeader() {
- ssize_t rc = TEMP_FAILURE_RETRY(pread64(fd_, &header_, sizeof(header_), file_offset_));
- if (rc < 0) {
- DL_ERR("can't read file \"%s\": %s", name_.c_str(), strerror(errno));
+ size_t map_size = file_size_ - file_offset_;
+ if (map_size < sizeof(header_)) {
+ DL_ERR("\"%s\" is too small to be an ELF executable: only found %zd bytes", name_.c_str(),
+ map_size);
return false;
}
- if (rc != sizeof(header_)) {
- DL_ERR("\"%s\" is too small to be an ELF executable: only found %zd bytes", name_.c_str(),
- static_cast<size_t>(rc));
+ // Map at most 1MiB which should cover most cases
+ map_size = std::min(map_size, static_cast<size_t>(1 * 1024 * 1024));
+
+ if (!file_fragment_.Map(fd_, file_offset_, 0, map_size)) {
+ DL_ERR("\"%s\" header mmap failed: %s", name_.c_str(), strerror(errno));
return false;
}
+
+ header_ = *static_cast<ElfW(Ehdr)*>(file_fragment_.data());
return true;
}
@@ -340,6 +345,24 @@
((offset % alignment) == 0);
}
+void* ElfReader::MapData(MappedFileFragment* fragment, off64_t offs, off64_t size) {
+ off64_t end;
+ CHECK(safe_add(&end, offs, size));
+
+ // If the data is already mapped just return it
+ if (static_cast<off64_t>(file_fragment_.size()) >= end) {
+ return static_cast<char*>(file_fragment_.data()) + offs;
+ }
+ // Use the passed-in fragment if area is not mapped. We can't remap the original fragment
+ // because that invalidates all previous pointers if the file is remapped to a different
+ // virtual address. A local variable can't be used in place of the passed-in fragment because
+ // the area would be unmapped as soon as the local object goes out of scope.
+ if (fragment->Map(fd_, file_offset_, offs, size)) {
+ return fragment->data();
+ }
+ return nullptr;
+}
+
// Loads the program header table from an ELF file into a read-only private
// anonymous mmap-ed block.
bool ElfReader::ReadProgramHeaders() {
@@ -362,12 +385,13 @@
return false;
}
- if (!phdr_fragment_.Map(fd_, file_offset_, header_.e_phoff, size)) {
+ void* phdr_data = MapData(&phdr_fragment_, header_.e_phoff, size);
+ if (phdr_data == nullptr) {
DL_ERR("\"%s\" phdr mmap failed: %s", name_.c_str(), strerror(errno));
return false;
}
- phdr_table_ = static_cast<ElfW(Phdr)*>(phdr_fragment_.data());
+ phdr_table_ = static_cast<ElfW(Phdr)*>(phdr_data);
return true;
}
@@ -388,12 +412,13 @@
return false;
}
- if (!shdr_fragment_.Map(fd_, file_offset_, header_.e_shoff, size)) {
+ void* shdr_data = MapData(&shdr_fragment_, header_.e_shoff, size);
+ if (shdr_data == nullptr) {
DL_ERR("\"%s\" shdr mmap failed: %s", name_.c_str(), strerror(errno));
return false;
}
- shdr_table_ = static_cast<const ElfW(Shdr)*>(shdr_fragment_.data());
+ shdr_table_ = static_cast<const ElfW(Shdr)*>(shdr_data);
return true;
}
@@ -481,12 +506,13 @@
return false;
}
- if (!dynamic_fragment_.Map(fd_, file_offset_, dynamic_shdr->sh_offset, dynamic_shdr->sh_size)) {
+ void* dynamic_data = MapData(&dynamic_fragment_, dynamic_shdr->sh_offset, dynamic_shdr->sh_size);
+ if (dynamic_data == nullptr) {
DL_ERR("\"%s\" dynamic section mmap failed: %s", name_.c_str(), strerror(errno));
return false;
}
- dynamic_ = static_cast<const ElfW(Dyn)*>(dynamic_fragment_.data());
+ dynamic_ = static_cast<const ElfW(Dyn)*>(dynamic_data);
if (!CheckFileRange(strtab_shdr->sh_offset, strtab_shdr->sh_size, alignof(const char))) {
DL_ERR_AND_LOG("\"%s\" has invalid offset/size of the .strtab section linked from .dynamic section",
@@ -494,13 +520,14 @@
return false;
}
- if (!strtab_fragment_.Map(fd_, file_offset_, strtab_shdr->sh_offset, strtab_shdr->sh_size)) {
+ void* strtab_data = MapData(&strtab_fragment_, strtab_shdr->sh_offset, strtab_shdr->sh_size);
+ if (strtab_data == nullptr) {
DL_ERR("\"%s\" strtab section mmap failed: %s", name_.c_str(), strerror(errno));
return false;
}
- strtab_ = static_cast<const char*>(strtab_fragment_.data());
- strtab_size_ = strtab_fragment_.size();
+ strtab_ = static_cast<const char*>(strtab_data);
+ strtab_size_ = strtab_shdr->sh_size;
return true;
}
@@ -756,7 +783,8 @@
// note_fragment is scoped to within the loop so that there is
// at most 1 PT_NOTE mapped at anytime during this search.
MappedFileFragment note_fragment;
- if (!note_fragment.Map(fd_, file_offset_, phdr->p_offset, phdr->p_memsz)) {
+ void* note_data = MapData(¬e_fragment, phdr->p_offset, phdr->p_memsz);
+ if (note_data == nullptr) {
DL_ERR("\"%s\": PT_NOTE mmap(nullptr, %p, PROT_READ, MAP_PRIVATE, %d, %p) failed: %m",
name_.c_str(), reinterpret_cast<void*>(phdr->p_memsz), fd_,
reinterpret_cast<void*>(page_start(file_offset_ + phdr->p_offset)));
@@ -766,7 +794,7 @@
const ElfW(Nhdr)* note_hdr = nullptr;
const char* note_desc = nullptr;
if (!__get_elf_note(NT_ANDROID_TYPE_PAD_SEGMENT, "Android",
- reinterpret_cast<ElfW(Addr)>(note_fragment.data()),
+ reinterpret_cast<ElfW(Addr)>(note_data),
phdr, ¬e_hdr, ¬e_desc)) {
continue;
}
diff --git a/linker/linker_phdr.h b/linker/linker_phdr.h
index aab9018..dc1ca99 100644
--- a/linker/linker_phdr.h
+++ b/linker/linker_phdr.h
@@ -73,6 +73,7 @@
[[nodiscard]] bool FindGnuPropertySection();
[[nodiscard]] bool CheckPhdr(ElfW(Addr));
[[nodiscard]] bool CheckFileRange(ElfW(Addr) offset, size_t size, size_t alignment);
+ [[nodiscard]] void* MapData(MappedFileFragment* fragment, off64_t offs, off64_t size);
bool did_read_;
bool did_load_;
@@ -81,6 +82,8 @@
off64_t file_offset_;
off64_t file_size_;
+ MappedFileFragment file_fragment_;
+
ElfW(Ehdr) header_;
size_t phdr_num_;