bpf: switch to mainline netbpfload on Android S/SV2
Test: TreeHugger, manually installing tethering apex
(with and without dnsresolver apex) on pixel 6 running S
incl. 'atest bpf_existence_test'
Signed-off-by: Maciej Żenczykowski <maze@google.com>
Change-Id: I63d4b84287e9c98dc0d2a517ac58d3f3ce9d7760
diff --git a/Tethering/apex/Android.bp b/Tethering/apex/Android.bp
index 19dd492..2878f79 100644
--- a/Tethering/apex/Android.bp
+++ b/Tethering/apex/Android.bp
@@ -102,15 +102,14 @@
"dscpPolicy.o",
"netd.o",
"offload.o",
- "offload@mainline.o",
"test.o",
- "test@mainline.o",
],
apps: [
"ServiceConnectivityResources",
],
prebuilts: [
"current_sdkinfo",
+ "netbpfload.31rc",
"netbpfload.33rc",
"netbpfload.35rc",
"ot-daemon.34rc",
diff --git a/bpf/dns_helper/DnsBpfHelper.cpp b/bpf/dns_helper/DnsBpfHelper.cpp
index 0719ade..cf2fa2b 100644
--- a/bpf/dns_helper/DnsBpfHelper.cpp
+++ b/bpf/dns_helper/DnsBpfHelper.cpp
@@ -32,12 +32,44 @@
} \
} while (0)
+// copied from BpfHandler.cpp
+static bool mainlineNetBpfLoadDone() {
+ return !access("/sys/fs/bpf/netd_shared/mainline_done", F_OK);
+}
+
+// copied from BpfHandler.cpp
+static inline void waitForNetProgsLoaded() {
+ // infinite loop until success with 5/10/20/40/60/60/60... delay
+ for (int delay = 5;; delay *= 2) {
+ if (delay > 60) delay = 60;
+ if (base::WaitForProperty("init.svc.mdnsd_netbpfload", "stopped", std::chrono::seconds(delay))
+ && mainlineNetBpfLoadDone()) return;
+ LOG(WARNING) << "Waited " << delay << "s for init.svc.mdnsd_netbpfload=stopped, still waiting.";
+ }
+}
+
base::Result<void> DnsBpfHelper::init() {
- if (!android::modules::sdklevel::IsAtLeastT()) {
- LOG(ERROR) << __func__ << ": Unsupported before Android T.";
+ if (!android::modules::sdklevel::IsAtLeastS()) {
+ LOG(ERROR) << __func__ << ": Unsupported before Android S.";
return base::Error(EOPNOTSUPP);
}
+ if (!android::modules::sdklevel::IsAtLeastT()) {
+ LOG(INFO) << "performing Android S mainline NetBpfload magic!";
+ if (!mainlineNetBpfLoadDone()) {
+ // We're on S/Sv2 & it's the first time netd is starting up (unless crashlooping)
+ if (!base::SetProperty("ctl.start", "mdnsd_netbpfload")) {
+ LOG(ERROR) << "Failed to set property ctl.start=mdnsd_netbpfload, see dmesg for reason.";
+ return base::Error(ENOEXEC);
+ }
+
+ LOG(INFO) << "Waiting for Networking BPF programs";
+ waitForNetProgsLoaded();
+ LOG(INFO) << "Networking BPF programs are loaded";
+ }
+ return {};
+ }
+
RETURN_IF_RESULT_NOT_OK(mConfigurationMap.init(CONFIGURATION_MAP_PATH));
RETURN_IF_RESULT_NOT_OK(mUidOwnerMap.init(UID_OWNER_MAP_PATH));
RETURN_IF_RESULT_NOT_OK(mDataSaverEnabledMap.init(DATA_SAVER_ENABLED_MAP_PATH));
diff --git a/bpf/loader/Android.bp b/bpf/loader/Android.bp
index b08913a..780fe20 100644
--- a/bpf/loader/Android.bp
+++ b/bpf/loader/Android.bp
@@ -56,11 +56,22 @@
installable: false,
}
-// Versioned netbpfload init rc: init system will process it only on api T/33+ devices
+// Versioned netbpfload init rc: init system will process it only on api R/30 S/31 Sv2/32 devices
// Note: R[30] S[31] Sv2[32] T[33] U[34] V[35])
//
// For details of versioned rc files see:
// https://android.googlesource.com/platform/system/core/+/HEAD/init/README.md#versioned-rc-files-within-apexs
+//
+// However, .Xrc versioning doesn't work on S, so we use unversioned, and thus *do* trigger on R,
+// luckily nothing ever uses the new service on R, so you can think of it as being S/Sv2 only
+prebuilt_etc {
+ name: "netbpfload.31rc",
+ src: "netbpfload.31rc",
+ filename: "netbpfload.rc", // intentional: .31rc wouldn't take effect on S
+ installable: false,
+}
+
+// Versioned netbpfload init rc: init system will process it only on api T/33+ devices
prebuilt_etc {
name: "netbpfload.33rc",
src: "netbpfload.33rc",
diff --git a/bpf/loader/NetBpfLoad.cpp b/bpf/loader/NetBpfLoad.cpp
index 9c62e74..bad506f 100644
--- a/bpf/loader/NetBpfLoad.cpp
+++ b/bpf/loader/NetBpfLoad.cpp
@@ -122,6 +122,7 @@
struct Location {
const char* const dir = "";
const char* const prefix = "";
+ const bool t_plus = true;
};
// Returns the build type string (from ro.build.type).
@@ -1216,8 +1217,9 @@
const Location locations[] = {
// S+ Tethering mainline module (network_stack): tether offload
{
- .dir = BPFROOT "/",
+ .dir = BPFROOT "/tethering/",
.prefix = "tethering/",
+ .t_plus = false,
},
// T+ Tethering mainline module (shared with netd & system server)
// netutils_wrapper (for iptables xt_bpf) has access to programs
@@ -1412,6 +1414,13 @@
}
static int doLoad(char** argv, char * const envp[]) {
+ if (!isAtLeastS) {
+ ALOGE("Impossible - not reachable on Android <S.");
+ // for safety, we don't fail, this is a just-in-case workaround
+ // for any possible busted 'optimized' start everything vendor init hacks on R
+ return 0;
+ }
+
const bool runningAsRoot = !getuid(); // true iff U QPR3 or V+
const int first_api_level = GetIntProperty("ro.board.first_api_level", api_level);
@@ -1446,14 +1455,9 @@
logTetheringApexVersion();
- if (!isAtLeastT) {
- ALOGE("Impossible - not reachable on Android <T.");
- return 1;
- }
-
// both S and T require kernel 4.9 (and eBpf support)
- if (isAtLeastT && !isAtLeastKernelVersion(4, 9, 0)) {
- ALOGE("Android T requires kernel 4.9.");
+ if (!isAtLeastKernelVersion(4, 9, 0)) {
+ ALOGE("Android S & T require kernel 4.9.");
return 1;
}
@@ -1622,18 +1626,22 @@
// which could otherwise fail with ENOENT during object pinning or renaming,
// due to ordering issues)
for (const auto& location : locations) {
+ if (location.t_plus && !isAtLeastT) continue;
if (createSysFsBpfSubDir(location.prefix)) return 1;
}
- // Note: there's no actual src dir for fs_bpf_loader .o's,
- // so it is not listed in 'locations[].prefix'.
- // This is because this is primarily meant for triggering genfscon rules,
- // and as such this will likely always be the case.
- // Thus we need to manually create the /sys/fs/bpf/loader subdirectory.
- if (createSysFsBpfSubDir("loader")) return 1;
+ if (isAtLeastT) {
+ // Note: there's no actual src dir for fs_bpf_loader .o's,
+ // so it is not listed in 'locations[].prefix'.
+ // This is because this is primarily meant for triggering genfscon rules,
+ // and as such this will likely always be the case.
+ // Thus we need to manually create the /sys/fs/bpf/loader subdirectory.
+ if (createSysFsBpfSubDir("loader")) return 1;
+ }
// Load all ELF objects, create programs and maps, and pin them
for (const auto& location : locations) {
+ if (location.t_plus && !isAtLeastT) continue;
if (loadAllElfObjects(bpfloader_ver, location) != 0) {
ALOGE("=== CRITICAL FAILURE LOADING BPF PROGRAMS FROM %s ===", location.dir);
ALOGE("If this triggers reliably, you're probably missing kernel options or patches.");
@@ -1654,6 +1662,9 @@
return 1;
}
+ // on S we haven't created this subdir yet, but we need it for 'mainline_done' flag below
+ if (!isAtLeastT && createSysFsBpfSubDir("netd_shared")) return 1;
+
// leave a flag that we're done
if (createSysFsBpfSubDir("netd_shared/mainline_done")) return 1;
@@ -1688,7 +1699,12 @@
} // namespace android
int main(int argc, char** argv, char * const envp[]) {
- InitLogging(argv, &KernelLogger);
+ if (android::bpf::isAtLeastT) {
+ InitLogging(argv, &KernelLogger);
+ } else {
+ // S lacks the sepolicy to make non-root uid KernelLogger viable
+ InitLogging(argv);
+ }
if (argc == 2 && !strcmp(argv[1], "done")) {
// we're being re-exec'ed from platform bpfloader to 'finalize' things
diff --git a/bpf/loader/netbpfload.31rc b/bpf/loader/netbpfload.31rc
new file mode 100644
index 0000000..bca7dc8
--- /dev/null
+++ b/bpf/loader/netbpfload.31rc
@@ -0,0 +1,13 @@
+# This file takes effect only on S and Sv2
+# (Note: it does take effect on R as well, but isn't actually used)
+#
+# The service is started from netd's dnsresolver call into ADnsHelper_init()
+# on initial (boot time) startup of netd.
+
+service mdnsd_netbpfload /apex/com.android.tethering/bin/netbpfload
+ capabilities CHOWN SYS_ADMIN NET_ADMIN
+ group system root graphics network_stack net_admin net_bw_acct net_bw_stats net_raw
+ user system
+ rlimit memlock 1073741824 1073741824
+ oneshot
+ reboot_on_failure reboot,netbpfload-failed
diff --git a/bpf/progs/Android.bp b/bpf/progs/Android.bp
index 20d194c..2bfe613 100644
--- a/bpf/progs/Android.bp
+++ b/bpf/progs/Android.bp
@@ -69,32 +69,16 @@
sub_dir: "net_shared",
}
-// Ships to Android S, the bpfloader of which fails to parse BTF enabled .o's.
bpf {
name: "offload.o",
srcs: ["offload.c"],
- btf: false,
+ sub_dir: "tethering",
}
-// This version ships to Android T+ which uses mainline netbpfload.
-bpf {
- name: "offload@mainline.o",
- srcs: ["offload@mainline.c"],
- cflags: ["-DMAINLINE"],
-}
-
-// Ships to Android S, the bpfloader of which fails to parse BTF enabled .o's.
bpf {
name: "test.o",
srcs: ["test.c"],
- btf: false,
-}
-
-// This version ships to Android T+ which uses mainline netbpfload.
-bpf {
- name: "test@mainline.o",
- srcs: ["test@mainline.c"],
- cflags: ["-DMAINLINE"],
+ sub_dir: "tethering",
}
bpf {
diff --git a/bpf/progs/offload.c b/bpf/progs/offload.c
index 0f23844..b34fe6f 100644
--- a/bpf/progs/offload.c
+++ b/bpf/progs/offload.c
@@ -14,16 +14,8 @@
* limitations under the License.
*/
-#ifdef MAINLINE
-// BTF is incompatible with bpfloaders < v0.10, hence for S (v0.2) we must
-// ship a different file than for later versions, but we need bpfloader v0.25+
-// for obj@ver.o support
-#define BPFLOADER_MIN_VER BPFLOADER_MAINLINE_T_VERSION
-#else /* MAINLINE */
-// The resulting .o needs to load on the Android S bpfloader
-#define BPFLOADER_MIN_VER BPFLOADER_S_VERSION
-#define BPFLOADER_MAX_VER BPFLOADER_T_VERSION
-#endif /* MAINLINE */
+// The resulting .o needs to load on Android S+
+#define BPFLOADER_MIN_VER BPFLOADER_MAINLINE_S_VERSION
#include "bpf_net_helpers.h"
#include "offload.h"
diff --git a/bpf/progs/test.c b/bpf/progs/test.c
index 8585118..4dba6b9 100644
--- a/bpf/progs/test.c
+++ b/bpf/progs/test.c
@@ -14,16 +14,8 @@
* limitations under the License.
*/
-#ifdef MAINLINE
-// BTF is incompatible with bpfloaders < v0.10, hence for S (v0.2) we must
-// ship a different file than for later versions, but we need bpfloader v0.25+
-// for obj@ver.o support
-#define BPFLOADER_MIN_VER BPFLOADER_MAINLINE_T_VERSION
-#else /* MAINLINE */
-// The resulting .o needs to load on the Android S bpfloader
-#define BPFLOADER_MIN_VER BPFLOADER_S_VERSION
-#define BPFLOADER_MAX_VER BPFLOADER_T_VERSION
-#endif /* MAINLINE */
+// The resulting .o needs to load on Android S+
+#define BPFLOADER_MIN_VER BPFLOADER_MAINLINE_S_VERSION
// This is non production code, only used for testing
// Needed because the bitmap array definition is non-kosher for pre-T OS devices.
diff --git a/bpf/tests/mts/bpf_existence_test.cpp b/bpf/tests/mts/bpf_existence_test.cpp
index 75fb8e9..4d5f9b5 100644
--- a/bpf/tests/mts/bpf_existence_test.cpp
+++ b/bpf/tests/mts/bpf_existence_test.cpp
@@ -196,7 +196,12 @@
// S requires Linux Kernel 4.9+ and thus requires eBPF support.
if (isAtLeastS) ASSERT_TRUE(isAtLeastKernelVersion(4, 9, 0));
- DO_EXPECT(isAtLeastS, MAINLINE_FOR_S_PLUS);
+
+ // on S without a new enough DnsResolver apex, NetBpfLoad doesn't get triggered,
+ // and thus no mainline programs get loaded.
+ bool mainlineBpfCapableResolve = !access("/apex/com.android.resolv/NetBpfLoad-S.flag", F_OK);
+ bool mainlineNetBpfLoad = isAtLeastT || mainlineBpfCapableResolve;
+ DO_EXPECT(isAtLeastS && mainlineNetBpfLoad, MAINLINE_FOR_S_PLUS);
// Nothing added or removed in SCv2.