blob: 4dd64b7012112bb3d3076db4f2de8ba828377f83 [file] [log] [blame]
Joel Fernandesd76a2002018-10-16 13:19:58 -07001/*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Maciej Żenczykowskie2f2b822024-05-01 05:16:51 -070017#define LOG_TAG "BpfLoader"
Joel Fernandesd76a2002018-10-16 13:19:58 -070018
19#include <errno.h>
Maciej Żenczykowskie15229f2023-04-17 07:16:33 +000020#include <fcntl.h>
Joel Fernandesd76a2002018-10-16 13:19:58 -070021#include <linux/bpf.h>
22#include <linux/elf.h>
23#include <log/log.h>
24#include <stdint.h>
25#include <stdio.h>
26#include <stdlib.h>
27#include <string.h>
Connor O'Brien35425e52022-01-18 21:41:16 -080028#include <sysexits.h>
Maciej Żenczykowski83f29772020-01-27 03:11:51 -080029#include <sys/stat.h>
Joel Fernandesd76a2002018-10-16 13:19:58 -070030#include <sys/utsname.h>
Connor O'Brien35425e52022-01-18 21:41:16 -080031#include <sys/wait.h>
Joel Fernandesd76a2002018-10-16 13:19:58 -070032#include <unistd.h>
33
Maciej Żenczykowski300c51f2022-12-14 04:18:02 -080034#include "BpfSyscallWrappers.h"
Maciej Żenczykowski07375e22020-02-19 14:23:59 -080035#include "bpf/BpfUtils.h"
Ken Chend5689472021-12-20 18:07:21 +080036#include "bpf/bpf_map_def.h"
Joel Fernandesd76a2002018-10-16 13:19:58 -070037#include "include/libbpf_android.h"
38
39#include <cstdlib>
40#include <fstream>
41#include <iostream>
Connor O'Brien3278a162020-02-13 21:45:22 -080042#include <optional>
Joel Fernandesd76a2002018-10-16 13:19:58 -070043#include <string>
Connor O'Brien35425e52022-01-18 21:41:16 -080044#include <unordered_map>
Christopher Ferrisc151c672019-02-01 15:31:26 -080045#include <vector>
Joel Fernandesd76a2002018-10-16 13:19:58 -070046
Connor O'Brien35425e52022-01-18 21:41:16 -080047#include <android-base/cmsg.h>
48#include <android-base/file.h>
Maciej Żenczykowski6d4d9d82024-08-06 15:32:00 -070049#include <android-base/logging.h>
Joel Fernandesd76a2002018-10-16 13:19:58 -070050#include <android-base/strings.h>
Connor O'Brien8d49fc72019-10-24 18:23:49 -070051#include <android-base/unique_fd.h>
Joel Fernandesd76a2002018-10-16 13:19:58 -070052
53#define BPF_FS_PATH "/sys/fs/bpf/"
54
55// Size of the BPF log buffer for verifier logging
Stephane Leeeb61b732021-10-21 17:03:11 -070056#define BPF_LOAD_LOG_SZ 0xfffff
Joel Fernandesd76a2002018-10-16 13:19:58 -070057
Tyler Wear4e2f4602022-02-03 09:46:01 -080058// Unspecified attach type is 0 which is BPF_CGROUP_INET_INGRESS.
59#define BPF_ATTACH_TYPE_UNSPEC BPF_CGROUP_INET_INGRESS
60
Maciej Żenczykowski6d4d9d82024-08-06 15:32:00 -070061using android::base::EndsWith;
Joel Fernandesd76a2002018-10-16 13:19:58 -070062using android::base::StartsWith;
Connor O'Brien8d49fc72019-10-24 18:23:49 -070063using android::base::unique_fd;
Joel Fernandesd76a2002018-10-16 13:19:58 -070064using std::ifstream;
65using std::ios;
Connor O'Brien3278a162020-02-13 21:45:22 -080066using std::optional;
Maciej Żenczykowski6d4d9d82024-08-06 15:32:00 -070067using std::strerror;
Christopher Ferrisc151c672019-02-01 15:31:26 -080068using std::string;
Joel Fernandesd76a2002018-10-16 13:19:58 -070069using std::vector;
70
71namespace android {
72namespace bpf {
73
Maciej Żenczykowski8a117a32023-06-16 08:18:49 +000074static unsigned int page_size = static_cast<unsigned int>(getpagesize());
75
Maciej Żenczykowski21869ef2022-07-07 06:01:57 -070076static string pathToObjName(const string& path) {
77 // extract everything after the final slash, ie. this is the filename 'foo@1.o' or 'bar.o'
78 string filename = android::base::Split(path, "/").back();
79 // strip off everything from the final period onwards (strip '.o' suffix), ie. 'foo@1' or 'bar'
80 string name = filename.substr(0, filename.find_last_of('.'));
81 // strip any potential @1 suffix, this will leave us with just 'foo' or 'bar'
82 // this can be used to provide duplicate programs (mux based on the bpfloader version)
83 return name.substr(0, name.find_last_of('@'));
Maciej Żenczykowskif7c0d992021-01-21 16:01:04 -080084}
85
Joel Fernandesd76a2002018-10-16 13:19:58 -070086typedef struct {
87 const char* name;
88 enum bpf_prog_type type;
Tyler Wear4e2f4602022-02-03 09:46:01 -080089 enum bpf_attach_type expected_attach_type;
Joel Fernandesd76a2002018-10-16 13:19:58 -070090} sectionType;
91
92/*
93 * Map section name prefixes to program types, the section name will be:
Maciej Żenczykowski3adb1d52021-10-22 19:27:10 -070094 * SECTION(<prefix>/<name-of-program>)
Joel Fernandesd76a2002018-10-16 13:19:58 -070095 * For example:
Maciej Żenczykowski3adb1d52021-10-22 19:27:10 -070096 * SECTION("tracepoint/sched_switch_func") where sched_switch_funcs
Joel Fernandesd76a2002018-10-16 13:19:58 -070097 * is the name of the program, and tracepoint is the type.
Maciej Żenczykowski3adb1d52021-10-22 19:27:10 -070098 *
99 * However, be aware that you should not be directly using the SECTION() macro.
100 * Instead use the DEFINE_(BPF|XDP)_(PROG|MAP)... & LICENSE/CRITICAL macros.
Joel Fernandesd76a2002018-10-16 13:19:58 -0700101 */
102sectionType sectionNameTypes[] = {
Maciej Żenczykowskie245fa92023-04-05 01:16:05 +0000103 {"kprobe/", BPF_PROG_TYPE_KPROBE, BPF_ATTACH_TYPE_UNSPEC},
104 {"kretprobe/", BPF_PROG_TYPE_KPROBE, BPF_ATTACH_TYPE_UNSPEC},
Maciej Żenczykowskie245fa92023-04-05 01:16:05 +0000105 {"perf_event/", BPF_PROG_TYPE_PERF_EVENT, BPF_ATTACH_TYPE_UNSPEC},
Maciej Żenczykowskie245fa92023-04-05 01:16:05 +0000106 {"skfilter/", BPF_PROG_TYPE_SOCKET_FILTER, BPF_ATTACH_TYPE_UNSPEC},
Maciej Żenczykowskie245fa92023-04-05 01:16:05 +0000107 {"tracepoint/", BPF_PROG_TYPE_TRACEPOINT, BPF_ATTACH_TYPE_UNSPEC},
108 {"uprobe/", BPF_PROG_TYPE_KPROBE, BPF_ATTACH_TYPE_UNSPEC},
109 {"uretprobe/", BPF_PROG_TYPE_KPROBE, BPF_ATTACH_TYPE_UNSPEC},
Joel Fernandesd76a2002018-10-16 13:19:58 -0700110};
111
112typedef struct {
113 enum bpf_prog_type type;
Tyler Wear4e2f4602022-02-03 09:46:01 -0800114 enum bpf_attach_type expected_attach_type;
Joel Fernandesd76a2002018-10-16 13:19:58 -0700115 string name;
116 vector<char> data;
117 vector<char> rel_data;
Connor O'Brien3278a162020-02-13 21:45:22 -0800118 optional<struct bpf_prog_def> prog_def;
Joel Fernandesd76a2002018-10-16 13:19:58 -0700119
Connor O'Brien8d49fc72019-10-24 18:23:49 -0700120 unique_fd prog_fd; /* fd after loading */
Joel Fernandesd76a2002018-10-16 13:19:58 -0700121} codeSection;
122
Joel Fernandesd76a2002018-10-16 13:19:58 -0700123static int readElfHeader(ifstream& elfFile, Elf64_Ehdr* eh) {
124 elfFile.seekg(0);
125 if (elfFile.fail()) return -1;
126
127 if (!elfFile.read((char*)eh, sizeof(*eh))) return -1;
128
129 return 0;
130}
131
132/* Reads all section header tables into an Shdr array */
133static int readSectionHeadersAll(ifstream& elfFile, vector<Elf64_Shdr>& shTable) {
134 Elf64_Ehdr eh;
135 int ret = 0;
136
137 ret = readElfHeader(elfFile, &eh);
138 if (ret) return ret;
139
140 elfFile.seekg(eh.e_shoff);
141 if (elfFile.fail()) return -1;
142
143 /* Read shdr table entries */
144 shTable.resize(eh.e_shnum);
145
146 if (!elfFile.read((char*)shTable.data(), (eh.e_shnum * eh.e_shentsize))) return -ENOMEM;
147
148 return 0;
149}
150
151/* Read a section by its index - for ex to get sec hdr strtab blob */
152static int readSectionByIdx(ifstream& elfFile, int id, vector<char>& sec) {
153 vector<Elf64_Shdr> shTable;
Maciej Żenczykowskid56ec052021-01-15 00:27:04 -0800154 int ret = readSectionHeadersAll(elfFile, shTable);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700155 if (ret) return ret;
Joel Fernandesd76a2002018-10-16 13:19:58 -0700156
157 elfFile.seekg(shTable[id].sh_offset);
158 if (elfFile.fail()) return -1;
159
160 sec.resize(shTable[id].sh_size);
161 if (!elfFile.read(sec.data(), shTable[id].sh_size)) return -1;
162
163 return 0;
164}
165
166/* Read whole section header string table */
167static int readSectionHeaderStrtab(ifstream& elfFile, vector<char>& strtab) {
168 Elf64_Ehdr eh;
Maciej Żenczykowskid56ec052021-01-15 00:27:04 -0800169 int ret = readElfHeader(elfFile, &eh);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700170 if (ret) return ret;
171
172 ret = readSectionByIdx(elfFile, eh.e_shstrndx, strtab);
173 if (ret) return ret;
174
175 return 0;
176}
177
178/* Get name from offset in strtab */
179static int getSymName(ifstream& elfFile, int nameOff, string& name) {
180 int ret;
181 vector<char> secStrTab;
182
183 ret = readSectionHeaderStrtab(elfFile, secStrTab);
184 if (ret) return ret;
185
186 if (nameOff >= (int)secStrTab.size()) return -1;
187
188 name = string((char*)secStrTab.data() + nameOff);
189 return 0;
190}
191
192/* Reads a full section by name - example to get the GPL license */
193static int readSectionByName(const char* name, ifstream& elfFile, vector<char>& data) {
194 vector<char> secStrTab;
195 vector<Elf64_Shdr> shTable;
196 int ret;
197
198 ret = readSectionHeadersAll(elfFile, shTable);
199 if (ret) return ret;
200
201 ret = readSectionHeaderStrtab(elfFile, secStrTab);
202 if (ret) return ret;
203
204 for (int i = 0; i < (int)shTable.size(); i++) {
205 char* secname = secStrTab.data() + shTable[i].sh_name;
206 if (!secname) continue;
207
208 if (!strcmp(secname, name)) {
209 vector<char> dataTmp;
210 dataTmp.resize(shTable[i].sh_size);
211
212 elfFile.seekg(shTable[i].sh_offset);
213 if (elfFile.fail()) return -1;
214
215 if (!elfFile.read((char*)dataTmp.data(), shTable[i].sh_size)) return -1;
216
217 data = dataTmp;
218 return 0;
219 }
220 }
221 return -2;
222}
223
Maciej Żenczykowski7ed94ef2021-07-06 01:47:15 -0700224unsigned int readSectionUint(const char* name, ifstream& elfFile, unsigned int defVal) {
Maciej Żenczykowski9217eee2021-03-03 05:28:52 -0800225 vector<char> theBytes;
226 int ret = readSectionByName(name, elfFile, theBytes);
227 if (ret) {
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700228 ALOGV("Couldn't find section %s (defaulting to %u [0x%x]).", name, defVal, defVal);
Maciej Żenczykowski9217eee2021-03-03 05:28:52 -0800229 return defVal;
230 } else if (theBytes.size() < sizeof(unsigned int)) {
Maciej Żenczykowskie626a952022-06-17 02:35:36 -0700231 ALOGE("Section %s too short (defaulting to %u [0x%x]).", name, defVal, defVal);
Maciej Żenczykowski9217eee2021-03-03 05:28:52 -0800232 return defVal;
233 } else {
234 // decode first 4 bytes as LE32 uint, there will likely be more bytes due to alignment.
235 unsigned int value = static_cast<unsigned char>(theBytes[3]);
236 value <<= 8;
237 value += static_cast<unsigned char>(theBytes[2]);
238 value <<= 8;
239 value += static_cast<unsigned char>(theBytes[1]);
240 value <<= 8;
241 value += static_cast<unsigned char>(theBytes[0]);
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700242 ALOGV("Section %s value is %u [0x%x]", name, value, value);
Maciej Żenczykowski9217eee2021-03-03 05:28:52 -0800243 return value;
244 }
245}
246
Joel Fernandesd76a2002018-10-16 13:19:58 -0700247static int readSectionByType(ifstream& elfFile, int type, vector<char>& data) {
248 int ret;
249 vector<Elf64_Shdr> shTable;
250
251 ret = readSectionHeadersAll(elfFile, shTable);
252 if (ret) return ret;
253
254 for (int i = 0; i < (int)shTable.size(); i++) {
255 if ((int)shTable[i].sh_type != type) continue;
256
257 vector<char> dataTmp;
258 dataTmp.resize(shTable[i].sh_size);
259
260 elfFile.seekg(shTable[i].sh_offset);
261 if (elfFile.fail()) return -1;
262
263 if (!elfFile.read((char*)dataTmp.data(), shTable[i].sh_size)) return -1;
264
265 data = dataTmp;
266 return 0;
267 }
268 return -2;
269}
270
271static bool symCompare(Elf64_Sym a, Elf64_Sym b) {
272 return (a.st_value < b.st_value);
273}
274
275static int readSymTab(ifstream& elfFile, int sort, vector<Elf64_Sym>& data) {
276 int ret, numElems;
277 Elf64_Sym* buf;
278 vector<char> secData;
279
280 ret = readSectionByType(elfFile, SHT_SYMTAB, secData);
281 if (ret) return ret;
282
283 buf = (Elf64_Sym*)secData.data();
284 numElems = (secData.size() / sizeof(Elf64_Sym));
285 data.assign(buf, buf + numElems);
286
287 if (sort) std::sort(data.begin(), data.end(), symCompare);
288 return 0;
289}
290
Paul Lawrence7fb8b542022-07-19 16:13:49 -0700291static enum bpf_prog_type getFuseProgType() {
292 int result = BPF_PROG_TYPE_UNSPEC;
293 ifstream("/sys/fs/fuse/bpf_prog_type_fuse") >> result;
294 return static_cast<bpf_prog_type>(result);
295}
296
Joel Fernandesd76a2002018-10-16 13:19:58 -0700297static enum bpf_prog_type getSectionType(string& name) {
Maciej Żenczykowski2b203132021-11-18 15:13:36 -0800298 for (auto& snt : sectionNameTypes)
299 if (StartsWith(name, snt.name)) return snt.type;
Joel Fernandesd76a2002018-10-16 13:19:58 -0700300
Paul Lawrence9548f9f2021-11-09 16:32:43 +0000301 // TODO Remove this code when fuse-bpf is upstream and this BPF_PROG_TYPE_FUSE is fixed
Paul Lawrence7fb8b542022-07-19 16:13:49 -0700302 if (StartsWith(name, "fuse/")) return getFuseProgType();
Paul Lawrence9548f9f2021-11-09 16:32:43 +0000303
Joel Fernandesd76a2002018-10-16 13:19:58 -0700304 return BPF_PROG_TYPE_UNSPEC;
305}
306
Tyler Wear4e2f4602022-02-03 09:46:01 -0800307static enum bpf_attach_type getExpectedAttachType(string& name) {
308 for (auto& snt : sectionNameTypes)
309 if (StartsWith(name, snt.name)) return snt.expected_attach_type;
310 return BPF_ATTACH_TYPE_UNSPEC;
311}
312
Joel Fernandesd76a2002018-10-16 13:19:58 -0700313static string getSectionName(enum bpf_prog_type type)
314{
Maciej Żenczykowski2b203132021-11-18 15:13:36 -0800315 for (auto& snt : sectionNameTypes)
316 if (snt.type == type)
317 return string(snt.name);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700318
Steven Moreland0f10f3f2019-12-12 14:22:34 -0800319 return "UNKNOWN SECTION NAME " + std::to_string(type);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700320}
Joel Fernandesd76a2002018-10-16 13:19:58 -0700321
Maciej Żenczykowskic82f5662024-03-10 14:37:24 -0700322static int readProgDefs(ifstream& elfFile, vector<struct bpf_prog_def>& pd) {
Connor O'Brien3278a162020-02-13 21:45:22 -0800323 vector<char> pdData;
324 int ret = readSectionByName("progs", elfFile, pdData);
Connor O'Brien3278a162020-02-13 21:45:22 -0800325 if (ret) return ret;
326
Maciej Żenczykowskic82f5662024-03-10 14:37:24 -0700327 if (pdData.size() % sizeof(struct bpf_prog_def)) {
Maciej Żenczykowskie626a952022-06-17 02:35:36 -0700328 ALOGE("readProgDefs failed due to improper sized progs section, %zu %% %zu != 0",
Maciej Żenczykowskic82f5662024-03-10 14:37:24 -0700329 pdData.size(), sizeof(struct bpf_prog_def));
Maciej Żenczykowski9217eee2021-03-03 05:28:52 -0800330 return -1;
331 };
332
Maciej Żenczykowskic82f5662024-03-10 14:37:24 -0700333 pd.resize(pdData.size() / sizeof(struct bpf_prog_def));
334 memcpy(pd.data(), pdData.data(), pdData.size());
Connor O'Brien3278a162020-02-13 21:45:22 -0800335 return 0;
336}
337
Connor O'Brien0ea4c6b2022-01-14 00:18:11 -0800338static int getSectionSymNames(ifstream& elfFile, const string& sectionName, vector<string>& names,
339 optional<unsigned> symbolType = std::nullopt) {
Connor O'Brien3278a162020-02-13 21:45:22 -0800340 int ret;
341 string name;
342 vector<Elf64_Sym> symtab;
343 vector<Elf64_Shdr> shTable;
344
345 ret = readSymTab(elfFile, 1 /* sort */, symtab);
346 if (ret) return ret;
347
348 /* Get index of section */
349 ret = readSectionHeadersAll(elfFile, shTable);
350 if (ret) return ret;
351
352 int sec_idx = -1;
353 for (int i = 0; i < (int)shTable.size(); i++) {
354 ret = getSymName(elfFile, shTable[i].sh_name, name);
355 if (ret) return ret;
356
357 if (!name.compare(sectionName)) {
358 sec_idx = i;
359 break;
360 }
361 }
362
363 /* No section found with matching name*/
364 if (sec_idx == -1) {
Maciej Żenczykowskie626a952022-06-17 02:35:36 -0700365 ALOGW("No %s section could be found in elf object", sectionName.c_str());
Connor O'Brien3278a162020-02-13 21:45:22 -0800366 return -1;
367 }
368
369 for (int i = 0; i < (int)symtab.size(); i++) {
Connor O'Brien0ea4c6b2022-01-14 00:18:11 -0800370 if (symbolType.has_value() && ELF_ST_TYPE(symtab[i].st_info) != symbolType) continue;
371
Connor O'Brien3278a162020-02-13 21:45:22 -0800372 if (symtab[i].st_shndx == sec_idx) {
373 string s;
374 ret = getSymName(elfFile, symtab[i].st_name, s);
375 if (ret) return ret;
376 names.push_back(s);
377 }
378 }
379
380 return 0;
381}
382
Steven Moreland0f10f3f2019-12-12 14:22:34 -0800383static bool IsAllowed(bpf_prog_type type, const bpf_prog_type* allowed, size_t numAllowed) {
384 if (allowed == nullptr) return true;
385
386 for (size_t i = 0; i < numAllowed; i++) {
Paul Lawrence7fb8b542022-07-19 16:13:49 -0700387 if (allowed[i] == BPF_PROG_TYPE_UNSPEC) {
388 if (type == getFuseProgType()) return true;
389 } else if (type == allowed[i])
390 return true;
Steven Moreland0f10f3f2019-12-12 14:22:34 -0800391 }
392
393 return false;
394}
395
Joel Fernandesd76a2002018-10-16 13:19:58 -0700396/* Read a section by its index - for ex to get sec hdr strtab blob */
Maciej Żenczykowskic82f5662024-03-10 14:37:24 -0700397static int readCodeSections(ifstream& elfFile, vector<codeSection>& cs,
Steven Moreland0f10f3f2019-12-12 14:22:34 -0800398 const bpf_prog_type* allowed, size_t numAllowed) {
Joel Fernandesd76a2002018-10-16 13:19:58 -0700399 vector<Elf64_Shdr> shTable;
400 int entries, ret = 0;
401
402 ret = readSectionHeadersAll(elfFile, shTable);
403 if (ret) return ret;
404 entries = shTable.size();
405
Connor O'Brien3278a162020-02-13 21:45:22 -0800406 vector<struct bpf_prog_def> pd;
Maciej Żenczykowskic82f5662024-03-10 14:37:24 -0700407 ret = readProgDefs(elfFile, pd);
Connor O'Brien3278a162020-02-13 21:45:22 -0800408 if (ret) return ret;
409 vector<string> progDefNames;
410 ret = getSectionSymNames(elfFile, "progs", progDefNames);
411 if (!pd.empty() && ret) return ret;
412
Joel Fernandesd76a2002018-10-16 13:19:58 -0700413 for (int i = 0; i < entries; i++) {
414 string name;
415 codeSection cs_temp;
416 cs_temp.type = BPF_PROG_TYPE_UNSPEC;
417
418 ret = getSymName(elfFile, shTable[i].sh_name, name);
419 if (ret) return ret;
420
421 enum bpf_prog_type ptype = getSectionType(name);
Steven Moreland0f10f3f2019-12-12 14:22:34 -0800422
Tyler Wear4e2f4602022-02-03 09:46:01 -0800423 if (ptype == BPF_PROG_TYPE_UNSPEC) continue;
Maciej Żenczykowskif7c0d992021-01-21 16:01:04 -0800424
Steven Moreland0f10f3f2019-12-12 14:22:34 -0800425 if (!IsAllowed(ptype, allowed, numAllowed)) {
426 ALOGE("Program type %s not permitted here", getSectionName(ptype).c_str());
427 return -1;
428 }
429
Tyler Wear4e2f4602022-02-03 09:46:01 -0800430 // This must be done before '/' is replaced with '_'.
431 cs_temp.expected_attach_type = getExpectedAttachType(name);
Maciej Żenczykowskif7c0d992021-01-21 16:01:04 -0800432
Tyler Wear4e2f4602022-02-03 09:46:01 -0800433 string oldName = name;
Joel Fernandesd76a2002018-10-16 13:19:58 -0700434
Tyler Wear4e2f4602022-02-03 09:46:01 -0800435 // convert all slashes to underscores
436 std::replace(name.begin(), name.end(), '/', '_');
Connor O'Brien3278a162020-02-13 21:45:22 -0800437
Tyler Wear4e2f4602022-02-03 09:46:01 -0800438 cs_temp.type = ptype;
439 cs_temp.name = name;
440
441 ret = readSectionByIdx(elfFile, i, cs_temp.data);
442 if (ret) return ret;
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700443 ALOGV("Loaded code section %d (%s)", i, name.c_str());
Tyler Wear4e2f4602022-02-03 09:46:01 -0800444
445 vector<string> csSymNames;
446 ret = getSectionSymNames(elfFile, oldName, csSymNames, STT_FUNC);
447 if (ret || !csSymNames.size()) return ret;
448 for (size_t i = 0; i < progDefNames.size(); ++i) {
449 if (!progDefNames[i].compare(csSymNames[0] + "_def")) {
450 cs_temp.prog_def = pd[i];
451 break;
Connor O'Brien3278a162020-02-13 21:45:22 -0800452 }
Joel Fernandesd76a2002018-10-16 13:19:58 -0700453 }
454
455 /* Check for rel section */
456 if (cs_temp.data.size() > 0 && i < entries) {
457 ret = getSymName(elfFile, shTable[i + 1].sh_name, name);
458 if (ret) return ret;
459
Tyler Wear4e2f4602022-02-03 09:46:01 -0800460 if (name == (".rel" + oldName)) {
Joel Fernandesd76a2002018-10-16 13:19:58 -0700461 ret = readSectionByIdx(elfFile, i + 1, cs_temp.rel_data);
462 if (ret) return ret;
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700463 ALOGV("Loaded relo section %d (%s)", i, name.c_str());
Joel Fernandesd76a2002018-10-16 13:19:58 -0700464 }
465 }
466
467 if (cs_temp.data.size() > 0) {
Connor O'Brien8d49fc72019-10-24 18:23:49 -0700468 cs.push_back(std::move(cs_temp));
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700469 ALOGV("Adding section %d to cs list", i);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700470 }
471 }
472 return 0;
473}
474
475static int getSymNameByIdx(ifstream& elfFile, int index, string& name) {
476 vector<Elf64_Sym> symtab;
477 int ret = 0;
478
479 ret = readSymTab(elfFile, 0 /* !sort */, symtab);
480 if (ret) return ret;
481
482 if (index >= (int)symtab.size()) return -1;
483
484 return getSymName(elfFile, symtab[index].st_name, name);
485}
486
Maciej Żenczykowski1a7fff32022-06-20 18:16:24 -0700487static bool mapMatchesExpectations(const unique_fd& fd, const string& mapName,
488 const struct bpf_map_def& mapDef, const enum bpf_map_type type) {
Maciej Żenczykowski1a7fff32022-06-20 18:16:24 -0700489 // Assuming fd is a valid Bpf Map file descriptor then
490 // all the following should always succeed on a 4.14+ kernel.
491 // If they somehow do fail, they'll return -1 (and set errno),
492 // which should then cause (among others) a key_size mismatch.
Maciej Żenczykowski12bb5202022-06-16 15:50:58 -0700493 int fd_type = bpfGetFdMapType(fd);
494 int fd_key_size = bpfGetFdKeySize(fd);
495 int fd_value_size = bpfGetFdValueSize(fd);
496 int fd_max_entries = bpfGetFdMaxEntries(fd);
497 int fd_map_flags = bpfGetFdMapFlags(fd);
498
499 // DEVMAPs are readonly from the bpf program side's point of view, as such
500 // the kernel in kernel/bpf/devmap.c dev_map_init_map() will set the flag
501 int desired_map_flags = (int)mapDef.map_flags;
502 if (type == BPF_MAP_TYPE_DEVMAP || type == BPF_MAP_TYPE_DEVMAP_HASH)
503 desired_map_flags |= BPF_F_RDONLY_PROG;
504
Maciej Żenczykowski28f01bb2023-06-20 19:40:22 +0000505 // The .h file enforces that this is a power of two, and page size will
506 // also always be a power of two, so this logic is actually enough to
507 // force it to be a multiple of the page size, as required by the kernel.
Maciej Żenczykowski8a117a32023-06-16 08:18:49 +0000508 unsigned int desired_max_entries = mapDef.max_entries;
509 if (type == BPF_MAP_TYPE_RINGBUF) {
510 if (desired_max_entries < page_size) desired_max_entries = page_size;
511 }
512
Maciej Żenczykowski1a7fff32022-06-20 18:16:24 -0700513 // The following checks should *never* trigger, if one of them somehow does,
514 // it probably means a bpf .o file has been changed/replaced at runtime
515 // and bpfloader was manually rerun (normally it should only run *once*
516 // early during the boot process).
517 // Another possibility is that something is misconfigured in the code:
518 // most likely a shared map is declared twice differently.
519 // But such a change should never be checked into the source tree...
520 if ((fd_type == type) &&
521 (fd_key_size == (int)mapDef.key_size) &&
522 (fd_value_size == (int)mapDef.value_size) &&
Maciej Żenczykowski8a117a32023-06-16 08:18:49 +0000523 (fd_max_entries == (int)desired_max_entries) &&
Maciej Żenczykowski1a7fff32022-06-20 18:16:24 -0700524 (fd_map_flags == desired_map_flags)) {
525 return true;
526 }
Maciej Żenczykowski12bb5202022-06-16 15:50:58 -0700527
528 ALOGE("bpf map name %s mismatch: desired/found: "
529 "type:%d/%d key:%u/%d value:%u/%d entries:%u/%d flags:%u/%d",
530 mapName.c_str(), type, fd_type, mapDef.key_size, fd_key_size, mapDef.value_size,
531 fd_value_size, mapDef.max_entries, fd_max_entries, desired_map_flags, fd_map_flags);
532 return false;
533}
534
Maciej Żenczykowskid8a45782021-01-14 23:36:32 -0800535static int createMaps(const char* elfPath, ifstream& elfFile, vector<unique_fd>& mapFds,
Maciej Żenczykowskic1647d72024-03-10 19:11:58 -0700536 const char* prefix) {
Connor O'Brien8d49fc72019-10-24 18:23:49 -0700537 int ret;
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700538 vector<char> mdData;
Joel Fernandesd76a2002018-10-16 13:19:58 -0700539 vector<struct bpf_map_def> md;
540 vector<string> mapNames;
Maciej Żenczykowski21869ef2022-07-07 06:01:57 -0700541 string objName = pathToObjName(string(elfPath));
Joel Fernandesd76a2002018-10-16 13:19:58 -0700542
543 ret = readSectionByName("maps", elfFile, mdData);
Steven Morelandc0905b42019-12-12 14:21:20 -0800544 if (ret == -2) return 0; // no maps to read
Joel Fernandesd76a2002018-10-16 13:19:58 -0700545 if (ret) return ret;
Maciej Żenczykowski9217eee2021-03-03 05:28:52 -0800546
Maciej Żenczykowskic82f5662024-03-10 14:37:24 -0700547 if (mdData.size() % sizeof(struct bpf_map_def)) {
Maciej Żenczykowskie626a952022-06-17 02:35:36 -0700548 ALOGE("createMaps failed due to improper sized maps section, %zu %% %zu != 0",
Maciej Żenczykowskic82f5662024-03-10 14:37:24 -0700549 mdData.size(), sizeof(struct bpf_map_def));
Maciej Żenczykowski9217eee2021-03-03 05:28:52 -0800550 return -1;
Maciej Żenczykowski9217eee2021-03-03 05:28:52 -0800551 }
Maciej Żenczykowskic82f5662024-03-10 14:37:24 -0700552 md.resize(mdData.size() / sizeof(struct bpf_map_def));
553 memcpy(md.data(), mdData.data(), mdData.size());
Joel Fernandesd76a2002018-10-16 13:19:58 -0700554
Connor O'Brien3278a162020-02-13 21:45:22 -0800555 ret = getSectionSymNames(elfFile, "maps", mapNames);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700556 if (ret) return ret;
557
Maciej Żenczykowskibbab8182022-06-22 22:51:07 -0700558 unsigned kvers = kernelVersion();
559
Maciej Żenczykowski36c53ba2021-07-05 15:20:34 -0700560 for (int i = 0; i < (int)mapNames.size(); i++) {
Maciej Żenczykowski2a5d0162022-07-21 13:27:24 +0000561 if (md[i].zero != 0) abort();
562
Maciej Żenczykowski36c53ba2021-07-05 15:20:34 -0700563 if (kvers < md[i].min_kver) {
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700564 ALOGD("skipping map %s which requires kernel version 0x%x >= 0x%x",
Maciej Żenczykowski36c53ba2021-07-05 15:20:34 -0700565 mapNames[i].c_str(), kvers, md[i].min_kver);
566 mapFds.push_back(unique_fd());
567 continue;
568 }
569
570 if (kvers >= md[i].max_kver) {
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700571 ALOGD("skipping map %s which requires kernel version 0x%x < 0x%x",
Maciej Żenczykowski36c53ba2021-07-05 15:20:34 -0700572 mapNames[i].c_str(), kvers, md[i].max_kver);
573 mapFds.push_back(unique_fd());
574 continue;
575 }
576
Maciej Żenczykowski12bb5202022-06-16 15:50:58 -0700577 enum bpf_map_type type = md[i].type;
Maciej Żenczykowski12bb5202022-06-16 15:50:58 -0700578 if (type == BPF_MAP_TYPE_DEVMAP_HASH && !isAtLeastKernelVersion(5, 4, 0)) {
579 // On Linux Kernels older than 5.4 this map type doesn't exist, but it can kind
580 // of be approximated: HASH has the same userspace visible api.
581 // However it cannot be used by ebpf programs in the same way.
582 // Since bpf_redirect_map() only requires 4.14, a program using a DEVMAP_HASH map
583 // would fail to load (due to trying to redirect to a HASH instead of DEVMAP_HASH).
584 // One must thus tag any BPF_MAP_TYPE_DEVMAP_HASH + bpf_redirect_map() using
585 // programs as being 5.4+...
586 type = BPF_MAP_TYPE_HASH;
587 }
588
Maciej Żenczykowski28f01bb2023-06-20 19:40:22 +0000589 // The .h file enforces that this is a power of two, and page size will
590 // also always be a power of two, so this logic is actually enough to
591 // force it to be a multiple of the page size, as required by the kernel.
Maciej Żenczykowski8a117a32023-06-16 08:18:49 +0000592 unsigned int max_entries = md[i].max_entries;
593 if (type == BPF_MAP_TYPE_RINGBUF) {
594 if (max_entries < page_size) max_entries = page_size;
595 }
596
Maciej Żenczykowskic1647d72024-03-10 19:11:58 -0700597 // Format of pin location is /sys/fs/bpf/<prefix>map_<objName>_<mapName>
Maciej Żenczykowski21869ef2022-07-07 06:01:57 -0700598 // except that maps shared across .o's have empty <objName>
599 // Note: <objName> refers to the extension-less basename of the .o file (without @ suffix).
Maciej Żenczykowskic1647d72024-03-10 19:11:58 -0700600 string mapPinLoc = string(BPF_FS_PATH) + prefix + "map_" +
Maciej Żenczykowski21869ef2022-07-07 06:01:57 -0700601 (md[i].shared ? "" : objName) + "_" + mapNames[i];
Maciej Żenczykowski36c53ba2021-07-05 15:20:34 -0700602 bool reuse = false;
603 unique_fd fd;
604 int saved_errno;
605
Joel Fernandesd76a2002018-10-16 13:19:58 -0700606 if (access(mapPinLoc.c_str(), F_OK) == 0) {
Maciej Żenczykowskieb199dd2022-07-01 03:21:40 -0700607 fd.reset(mapRetrieveRO(mapPinLoc.c_str()));
Maciej Żenczykowski2c372132021-03-01 23:09:54 -0800608 saved_errno = errno;
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700609 ALOGV("bpf_create_map reusing map %s, ret: %d", mapNames[i].c_str(), fd.get());
Joel Fernandesd76a2002018-10-16 13:19:58 -0700610 reuse = true;
611 } else {
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700612 union bpf_attr req = {
613 .map_type = type,
614 .key_size = md[i].key_size,
615 .value_size = md[i].value_size,
616 .max_entries = max_entries,
617 .map_flags = md[i].map_flags,
Connor O'Brien35425e52022-01-18 21:41:16 -0800618 };
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700619 strlcpy(req.map_name, mapNames[i].c_str(), sizeof(req.map_name));
620 fd.reset(bpf(BPF_MAP_CREATE, req));
Maciej Żenczykowski2c372132021-03-01 23:09:54 -0800621 saved_errno = errno;
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700622 ALOGV("bpf_create_map name %s, ret: %d", mapNames[i].c_str(), fd.get());
Joel Fernandesd76a2002018-10-16 13:19:58 -0700623 }
624
Maciej Żenczykowski12bb5202022-06-16 15:50:58 -0700625 if (!fd.ok()) return -saved_errno;
626
627 // When reusing a pinned map, we need to check the map type/sizes/etc match, but for
628 // safety (since reuse code path is rare) run these checks even if we just created it.
629 // We assume failure is due to pinned map mismatch, hence the 'NOT UNIQUE' return code.
630 if (!mapMatchesExpectations(fd, mapNames[i], md[i], type)) return -ENOTUNIQ;
Joel Fernandesd76a2002018-10-16 13:19:58 -0700631
632 if (!reuse) {
Maciej Żenczykowskic1647d72024-03-10 19:11:58 -0700633 ret = bpfFdPin(fd, mapPinLoc.c_str());
634 if (ret) {
635 int err = errno;
636 ALOGE("pin %s -> %d [%d:%s]", mapPinLoc.c_str(), ret, err, strerror(err));
637 return -err;
Maciej Żenczykowski41817132022-06-03 08:41:04 -0700638 }
Maciej Żenczykowski83f29772020-01-27 03:11:51 -0800639 ret = chmod(mapPinLoc.c_str(), md[i].mode);
Maciej Żenczykowski41817132022-06-03 08:41:04 -0700640 if (ret) {
641 int err = errno;
642 ALOGE("chmod(%s, 0%o) = %d [%d:%s]", mapPinLoc.c_str(), md[i].mode, ret, err,
643 strerror(err));
644 return -err;
645 }
Maciej Żenczykowski5e4aabf2022-06-27 01:15:53 -0700646 ret = chown(mapPinLoc.c_str(), (uid_t)md[i].uid, (gid_t)md[i].gid);
647 if (ret) {
648 int err = errno;
649 ALOGE("chown(%s, %u, %u) = %d [%d:%s]", mapPinLoc.c_str(), md[i].uid, md[i].gid,
650 ret, err, strerror(err));
651 return -err;
652 }
Joel Fernandesd76a2002018-10-16 13:19:58 -0700653 }
654
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700655 int mapId = bpfGetFdMapId(fd);
656 if (mapId == -1) {
657 ALOGE("bpfGetFdMapId failed, ret: %d [%d]", mapId, errno);
Maciej Żenczykowski57412c22022-05-20 16:44:06 -0700658 } else {
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700659 ALOGD("map %s id %d", mapPinLoc.c_str(), mapId);
Maciej Żenczykowski57412c22022-05-20 16:44:06 -0700660 }
661
Connor O'Brien8d49fc72019-10-24 18:23:49 -0700662 mapFds.push_back(std::move(fd));
Joel Fernandesd76a2002018-10-16 13:19:58 -0700663 }
664
665 return ret;
666}
667
Joel Fernandesd76a2002018-10-16 13:19:58 -0700668static void applyRelo(void* insnsPtr, Elf64_Addr offset, int fd) {
669 int insnIndex;
670 struct bpf_insn *insn, *insns;
671
672 insns = (struct bpf_insn*)(insnsPtr);
673
674 insnIndex = offset / sizeof(struct bpf_insn);
675 insn = &insns[insnIndex];
676
Maciej Żenczykowski509b1b92023-03-07 01:11:20 +0000677 // Occasionally might be useful for relocation debugging, but pretty spammy
678 if (0) {
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700679 ALOGV("applying relo to instruction at byte offset: %llu, "
Maciej Żenczykowski509b1b92023-03-07 01:11:20 +0000680 "insn offset %d, insn %llx",
681 (unsigned long long)offset, insnIndex, *(unsigned long long*)insn);
682 }
Joel Fernandesd76a2002018-10-16 13:19:58 -0700683
684 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) {
Maciej Żenczykowskie626a952022-06-17 02:35:36 -0700685 ALOGE("invalid relo for insn %d: code 0x%x", insnIndex, insn->code);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700686 return;
687 }
688
689 insn->imm = fd;
690 insn->src_reg = BPF_PSEUDO_MAP_FD;
691}
692
Connor O'Brien8d49fc72019-10-24 18:23:49 -0700693static void applyMapRelo(ifstream& elfFile, vector<unique_fd> &mapFds, vector<codeSection>& cs) {
Joel Fernandesd76a2002018-10-16 13:19:58 -0700694 vector<string> mapNames;
695
Connor O'Brien3278a162020-02-13 21:45:22 -0800696 int ret = getSectionSymNames(elfFile, "maps", mapNames);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700697 if (ret) return;
698
699 for (int k = 0; k != (int)cs.size(); k++) {
700 Elf64_Rel* rel = (Elf64_Rel*)(cs[k].rel_data.data());
701 int n_rel = cs[k].rel_data.size() / sizeof(*rel);
702
703 for (int i = 0; i < n_rel; i++) {
704 int symIndex = ELF64_R_SYM(rel[i].r_info);
705 string symName;
706
707 ret = getSymNameByIdx(elfFile, symIndex, symName);
708 if (ret) return;
709
710 /* Find the map fd and apply relo */
711 for (int j = 0; j < (int)mapNames.size(); j++) {
712 if (!mapNames[j].compare(symName)) {
713 applyRelo(cs[k].data.data(), rel[i].r_offset, mapFds[j]);
714 break;
715 }
716 }
717 }
718 }
719}
720
Maciej Żenczykowskid8a45782021-01-14 23:36:32 -0800721static int loadCodeSections(const char* elfPath, vector<codeSection>& cs, const string& license,
Maciej Żenczykowskic1647d72024-03-10 19:11:58 -0700722 const char* prefix) {
Maciej Żenczykowski07375e22020-02-19 14:23:59 -0800723 unsigned kvers = kernelVersion();
Joel Fernandesd76a2002018-10-16 13:19:58 -0700724
Maciej Żenczykowski5c791652022-08-03 23:49:08 +0000725 if (!kvers) {
726 ALOGE("unable to get kernel version");
727 return -EINVAL;
728 }
Joel Fernandesd76a2002018-10-16 13:19:58 -0700729
Maciej Żenczykowski21869ef2022-07-07 06:01:57 -0700730 string objName = pathToObjName(string(elfPath));
Joel Fernandesd76a2002018-10-16 13:19:58 -0700731
732 for (int i = 0; i < (int)cs.size(); i++) {
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700733 unique_fd& fd = cs[i].prog_fd;
734 int ret;
Maciej Żenczykowski681f6042020-04-21 15:34:18 -0700735 string name = cs[i].name;
Joel Fernandesd76a2002018-10-16 13:19:58 -0700736
Maciej Żenczykowski5c791652022-08-03 23:49:08 +0000737 if (!cs[i].prog_def.has_value()) {
738 ALOGE("[%d] '%s' missing program definition! bad bpf.o build?", i, name.c_str());
739 return -EINVAL;
Maciej Żenczykowski07375e22020-02-19 14:23:59 -0800740 }
741
Maciej Żenczykowski5c791652022-08-03 23:49:08 +0000742 unsigned min_kver = cs[i].prog_def->min_kver;
743 unsigned max_kver = cs[i].prog_def->max_kver;
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700744 if (kvers < min_kver || kvers >= max_kver) {
745 ALOGD("skipping program cs[%d].name:%s min_kver:%x max_kver:%x (kvers:%x)",
746 i, name.c_str(), min_kver, max_kver, kvers);
747 continue;
748 }
Maciej Żenczykowski5c791652022-08-03 23:49:08 +0000749
Maciej Żenczykowski681f6042020-04-21 15:34:18 -0700750 // strip any potential $foo suffix
751 // this can be used to provide duplicate programs
752 // conditionally loaded based on running kernel version
Maciej Żenczykowski428843d2020-04-23 12:43:44 -0700753 name = name.substr(0, name.find_last_of('$'));
Maciej Żenczykowski681f6042020-04-21 15:34:18 -0700754
755 bool reuse = false;
Joel Fernandesd76a2002018-10-16 13:19:58 -0700756 // Format of pin location is
Maciej Żenczykowski21869ef2022-07-07 06:01:57 -0700757 // /sys/fs/bpf/<prefix>prog_<objName>_<progName>
Maciej Żenczykowskic1647d72024-03-10 19:11:58 -0700758 string progPinLoc = string(BPF_FS_PATH) + prefix + "prog_" +
Maciej Żenczykowski21869ef2022-07-07 06:01:57 -0700759 objName + '_' + string(name);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700760 if (access(progPinLoc.c_str(), F_OK) == 0) {
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700761 fd.reset(retrieveProgram(progPinLoc.c_str()));
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700762 ALOGV("New bpf prog load reusing prog %s, ret: %d (%s)", progPinLoc.c_str(), fd.get(),
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700763 (!fd.ok() ? std::strerror(errno) : "no error"));
Joel Fernandesd76a2002018-10-16 13:19:58 -0700764 reuse = true;
765 } else {
766 vector<char> log_buf(BPF_LOAD_LOG_SZ, 0);
767
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700768 union bpf_attr req = {
769 .prog_type = cs[i].type,
770 .kern_version = kvers,
771 .license = ptr_to_u64(license.c_str()),
772 .insns = ptr_to_u64(cs[i].data.data()),
773 .insn_cnt = static_cast<__u32>(cs[i].data.size() / sizeof(struct bpf_insn)),
774 .log_level = 1,
775 .log_buf = ptr_to_u64(log_buf.data()),
776 .log_size = static_cast<__u32>(log_buf.size()),
777 .expected_attach_type = cs[i].expected_attach_type,
Tyler Wear4e2f4602022-02-03 09:46:01 -0800778 };
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700779 strlcpy(req.prog_name, cs[i].name.c_str(), sizeof(req.prog_name));
780 fd.reset(bpf(BPF_PROG_LOAD, req));
Tyler Wear4e2f4602022-02-03 09:46:01 -0800781
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700782 if (!fd.ok()) {
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700783 ALOGW("BPF_PROG_LOAD call for %s (%s) returned fd: %d (%s)", elfPath,
784 cs[i].name.c_str(), fd.get(), std::strerror(errno));
785
Maciej Żenczykowskif7c0d992021-01-21 16:01:04 -0800786 vector<string> lines = android::base::Split(log_buf.data(), "\n");
Maciej Żenczykowski524deef2020-02-11 11:12:37 -0800787
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700788 ALOGW("BPF_PROG_LOAD - BEGIN log_buf contents:");
Maciej Żenczykowskiaa295c82020-06-16 17:02:48 -0700789 for (const auto& line : lines) ALOGW("%s", line.c_str());
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700790 ALOGW("BPF_PROG_LOAD - END log_buf contents.");
Maciej Żenczykowskiaa295c82020-06-16 17:02:48 -0700791
Maciej Żenczykowski5c791652022-08-03 23:49:08 +0000792 if (cs[i].prog_def->optional) {
Maciej Żenczykowskiaa295c82020-06-16 17:02:48 -0700793 ALOGW("failed program is marked optional - continuing...");
794 continue;
795 }
796 ALOGE("non-optional program failed to load.");
Maciej Żenczykowski524deef2020-02-11 11:12:37 -0800797 }
Joel Fernandesd76a2002018-10-16 13:19:58 -0700798 }
799
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700800 if (!fd.ok()) return fd.get();
Joel Fernandesd76a2002018-10-16 13:19:58 -0700801
802 if (!reuse) {
Maciej Żenczykowskic1647d72024-03-10 19:11:58 -0700803 ret = bpfFdPin(fd, progPinLoc.c_str());
804 if (ret) {
805 int err = errno;
806 ALOGE("create %s -> %d [%d:%s]", progPinLoc.c_str(), ret, err, strerror(err));
807 return -err;
Maciej Żenczykowski41817132022-06-03 08:41:04 -0700808 }
809 if (chmod(progPinLoc.c_str(), 0440)) {
810 int err = errno;
811 ALOGE("chmod %s 0440 -> [%d:%s]", progPinLoc.c_str(), err, strerror(err));
812 return -err;
813 }
Maciej Żenczykowski5c791652022-08-03 23:49:08 +0000814 if (chown(progPinLoc.c_str(), (uid_t)cs[i].prog_def->uid,
815 (gid_t)cs[i].prog_def->gid)) {
816 int err = errno;
817 ALOGE("chown %s %d %d -> [%d:%s]", progPinLoc.c_str(), cs[i].prog_def->uid,
818 cs[i].prog_def->gid, err, strerror(err));
819 return -err;
Connor O'Brien3278a162020-02-13 21:45:22 -0800820 }
Joel Fernandesd76a2002018-10-16 13:19:58 -0700821 }
822
Maciej Żenczykowskib44e2872023-06-12 23:10:52 -0700823 int progId = bpfGetFdProgId(fd);
824 if (progId == -1) {
825 ALOGE("bpfGetFdProgId failed, ret: %d [%d]", progId, errno);
Maciej Żenczykowski57412c22022-05-20 16:44:06 -0700826 } else {
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700827 ALOGD("prog %s id %d", progPinLoc.c_str(), progId);
Maciej Żenczykowski57412c22022-05-20 16:44:06 -0700828 }
Joel Fernandesd76a2002018-10-16 13:19:58 -0700829 }
830
831 return 0;
832}
833
Connor O'Brien6c0ce9f2022-12-01 20:08:17 -0800834int loadProg(const char* elfPath, bool* isCritical, const Location& location) {
Joel Fernandesd76a2002018-10-16 13:19:58 -0700835 vector<char> license;
Maciej Żenczykowski4ba8c1c2020-06-10 15:49:31 -0700836 vector<char> critical;
Joel Fernandesd76a2002018-10-16 13:19:58 -0700837 vector<codeSection> cs;
Connor O'Brien8d49fc72019-10-24 18:23:49 -0700838 vector<unique_fd> mapFds;
Joel Fernandesd76a2002018-10-16 13:19:58 -0700839 int ret;
840
Maciej Żenczykowski89515d92020-06-14 19:27:33 -0700841 if (!isCritical) return -1;
842 *isCritical = false;
843
Joel Fernandesd76a2002018-10-16 13:19:58 -0700844 ifstream elfFile(elfPath, ios::in | ios::binary);
845 if (!elfFile.is_open()) return -1;
846
Maciej Żenczykowski4ba8c1c2020-06-10 15:49:31 -0700847 ret = readSectionByName("critical", elfFile, critical);
Maciej Żenczykowski89515d92020-06-14 19:27:33 -0700848 *isCritical = !ret;
Maciej Żenczykowski4ba8c1c2020-06-10 15:49:31 -0700849
Joel Fernandesd76a2002018-10-16 13:19:58 -0700850 ret = readSectionByName("license", elfFile, license);
851 if (ret) {
Maciej Żenczykowskie626a952022-06-17 02:35:36 -0700852 ALOGE("Couldn't find license in %s", elfPath);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700853 return ret;
Joel Fernandesd76a2002018-10-16 13:19:58 -0700854 }
855
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700856 ALOGI("Platform BpfLoader loading %s%s ELF object %s with license %s",
857 *isCritical ? "critical for " : "optional", *isCritical ? (char*)critical.data() : "",
858 elfPath, (char*)license.data());
Maciej Żenczykowski9217eee2021-03-03 05:28:52 -0800859
Maciej Żenczykowskic82f5662024-03-10 14:37:24 -0700860 ret = readCodeSections(elfFile, cs, location.allowedProgTypes, location.allowedProgTypesLength);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700861 if (ret) {
Maciej Żenczykowskie626a952022-06-17 02:35:36 -0700862 ALOGE("Couldn't read all code sections in %s", elfPath);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700863 return ret;
864 }
865
Maciej Żenczykowskic1647d72024-03-10 19:11:58 -0700866 ret = createMaps(elfPath, elfFile, mapFds, location.prefix);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700867 if (ret) {
Maciej Żenczykowskie626a952022-06-17 02:35:36 -0700868 ALOGE("Failed to create maps: (ret=%d) in %s", ret, elfPath);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700869 return ret;
870 }
871
872 for (int i = 0; i < (int)mapFds.size(); i++)
Maciej Żenczykowski0eb98a22024-04-30 22:35:08 -0700873 ALOGV("map_fd found at %d is %d in %s", i, mapFds[i].get(), elfPath);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700874
875 applyMapRelo(elfFile, mapFds, cs);
876
Maciej Żenczykowskic1647d72024-03-10 19:11:58 -0700877 ret = loadCodeSections(elfPath, cs, string(license.data()), location.prefix);
Maciej Żenczykowskie626a952022-06-17 02:35:36 -0700878 if (ret) ALOGE("Failed to load programs, loadCodeSections ret=%d", ret);
Joel Fernandesd76a2002018-10-16 13:19:58 -0700879
880 return ret;
881}
882
Maciej Żenczykowskie0f11522024-08-06 15:25:55 -0700883// Networking-related program types are limited to the Tethering Apex
884// to prevent things from breaking due to conflicts on mainline updates
885// (exception made for socket filters, ie. xt_bpf for potential use in iptables,
886// or for attaching to sockets directly)
887constexpr bpf_prog_type kPlatformAllowedProgTypes[] = {
888 BPF_PROG_TYPE_KPROBE,
889 BPF_PROG_TYPE_PERF_EVENT,
890 BPF_PROG_TYPE_SOCKET_FILTER,
891 BPF_PROG_TYPE_TRACEPOINT,
892 BPF_PROG_TYPE_UNSPEC, // Will be replaced with fuse bpf program type
893};
894
895constexpr bpf_prog_type kUprobestatsAllowedProgTypes[] = {
896 BPF_PROG_TYPE_KPROBE,
897};
898
899// see b/162057235. For arbitrary program types, the concern is that due to the lack of
900// SELinux access controls over BPF program attachpoints, we have no way to control the
901// attachment of programs to shared resources (or to detect when a shared resource
902// has one BPF program replace another that is attached there)
903constexpr bpf_prog_type kVendorAllowedProgTypes[] = {
904 BPF_PROG_TYPE_SOCKET_FILTER,
905};
906
Maciej Żenczykowski6d4d9d82024-08-06 15:32:00 -0700907const Location locations[] = {
Maciej Żenczykowskie0f11522024-08-06 15:25:55 -0700908 // Core operating system
909 {
910 .dir = "/system/etc/bpf/",
911 .prefix = "",
912 .allowedProgTypes = kPlatformAllowedProgTypes,
913 .allowedProgTypesLength = arraysize(kPlatformAllowedProgTypes),
914 },
915 // uprobestats
916 {
917 .dir = "/system/etc/bpf/uprobestats/",
918 .prefix = "uprobestats/",
919 .allowedProgTypes = kUprobestatsAllowedProgTypes,
920 .allowedProgTypesLength = arraysize(kUprobestatsAllowedProgTypes),
921 },
922 // Vendor operating system
923 {
924 .dir = "/vendor/etc/bpf/",
925 .prefix = "vendor/",
926 .allowedProgTypes = kVendorAllowedProgTypes,
927 .allowedProgTypesLength = arraysize(kVendorAllowedProgTypes),
928 },
929};
930
Maciej Żenczykowski6d4d9d82024-08-06 15:32:00 -0700931int loadAllElfObjects(const Location& location) {
Maciej Żenczykowskie0f11522024-08-06 15:25:55 -0700932 int retVal = 0;
933 DIR* dir;
934 struct dirent* ent;
935
936 if ((dir = opendir(location.dir)) != NULL) {
937 while ((ent = readdir(dir)) != NULL) {
938 string s = ent->d_name;
939 if (!EndsWith(s, ".o")) continue;
940
941 string progPath(location.dir);
942 progPath += s;
943
944 bool critical;
Maciej Żenczykowski6d4d9d82024-08-06 15:32:00 -0700945 int ret = loadProg(progPath.c_str(), &critical, location);
Maciej Żenczykowskie0f11522024-08-06 15:25:55 -0700946 if (ret) {
947 if (critical) retVal = ret;
Maciej Żenczykowski6d4d9d82024-08-06 15:32:00 -0700948 ALOGE("Failed to load object: %s, ret: %s", progPath.c_str(), strerror(-ret));
Maciej Żenczykowskie0f11522024-08-06 15:25:55 -0700949 } else {
950 ALOGV("Loaded object: %s", progPath.c_str());
951 }
952 }
953 closedir(dir);
954 }
955 return retVal;
956}
957
958int createSysFsBpfSubDir(const char* const prefix) {
959 if (*prefix) {
960 mode_t prevUmask = umask(0);
961
962 string s = "/sys/fs/bpf/";
963 s += prefix;
964
965 errno = 0;
966 int ret = mkdir(s.c_str(), S_ISVTX | S_IRWXU | S_IRWXG | S_IRWXO);
967 if (ret && errno != EEXIST) {
968 const int err = errno;
Maciej Żenczykowski6d4d9d82024-08-06 15:32:00 -0700969 ALOGE("Failed to create directory: %s, ret: %s", s.c_str(), strerror(err));
Maciej Żenczykowskie0f11522024-08-06 15:25:55 -0700970 return -err;
971 }
972
973 umask(prevUmask);
974 }
975 return 0;
976}
977
Maciej Żenczykowski6d4d9d82024-08-06 15:32:00 -0700978int legacyBpfLoader(int __unused argc, char** argv, char * const envp[]) {
979 base::InitLogging(argv, &base::KernelLogger);
Maciej Żenczykowskie0f11522024-08-06 15:25:55 -0700980
981 // Load all ELF objects, create programs and maps, and pin them
982 for (const auto& location : locations) {
983 if (createSysFsBpfSubDir(location.prefix) || loadAllElfObjects(location)) {
984 ALOGE("=== CRITICAL FAILURE LOADING BPF PROGRAMS FROM %s ===", location.dir);
985 ALOGE("If this triggers reliably, you're probably missing kernel options or patches.");
986 ALOGE("If this triggers randomly, you might be hitting some memory allocation "
987 "problems or startup script race.");
988 ALOGE("--- DO NOT EXPECT SYSTEM TO BOOT SUCCESSFULLY ---");
989 sleep(20);
990 return 120;
991 }
992 }
993
994 const char * args[] = { "/apex/com.android.tethering/bin/netbpfload", "done", NULL, };
995 execve(args[0], (char**)args, envp);
996 ALOGE("FATAL: execve(): %d[%s]", errno, strerror(errno));
997 return 121;
998}
Maciej Żenczykowski6d4d9d82024-08-06 15:32:00 -0700999
1000} // namespace bpf
1001} // namespace android