Bind devices to VFIO with VirtualizationService
vfio_handler service is added, which is a minimal service for tasks
which should be done as root. It will interact to sysfs to bind
VFIO devices.
Bug: 287379025
Bug: 278008182
Test: adb shell /apex/com.android.virt/bin/vm run-microdroid \
--devices /sys/bus/platform/devices/16d00000.eh --protected
Change-Id: Ia99f2be86c33b171297f76f7e30eacfc083aeaa0
diff --git a/apex/Android.bp b/apex/Android.bp
index 7ef2c79..765372a 100644
--- a/apex/Android.bp
+++ b/apex/Android.bp
@@ -76,6 +76,7 @@
arm64: {
binaries: [
"crosvm",
+ "vfio_handler",
"virtmgr",
"virtualizationservice",
],
@@ -84,6 +85,7 @@
x86_64: {
binaries: [
"crosvm",
+ "vfio_handler",
"virtmgr",
"virtualizationservice",
],
diff --git a/apex/virtualizationservice.rc b/apex/virtualizationservice.rc
index 02b2081..be90904 100644
--- a/apex/virtualizationservice.rc
+++ b/apex/virtualizationservice.rc
@@ -19,3 +19,10 @@
interface aidl android.system.virtualizationservice
disabled
oneshot
+
+service vfio_handler /apex/com.android.virt/bin/vfio_handler
+ user root
+ group root
+ interface aidl android.system.virtualizationservice_internal.IVfioHandler
+ disabled
+ oneshot
diff --git a/virtualizationservice/Android.bp b/virtualizationservice/Android.bp
index 0732c04..6b39ff9 100644
--- a/virtualizationservice/Android.bp
+++ b/virtualizationservice/Android.bp
@@ -36,7 +36,6 @@
"libstatslog_virtualization_rust",
"libtombstoned_client_rust",
"libvsock",
- "liblazy_static",
],
apex_available: ["com.android.virt"],
}
diff --git a/virtualizationservice/aidl/android/system/virtualizationservice_internal/IVfioHandler.aidl b/virtualizationservice/aidl/android/system/virtualizationservice_internal/IVfioHandler.aidl
new file mode 100644
index 0000000..516b7a1
--- /dev/null
+++ b/virtualizationservice/aidl/android/system/virtualizationservice_internal/IVfioHandler.aidl
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.system.virtualizationservice_internal;
+
+import android.system.virtualizationservice.AssignableDevice;
+import android.system.virtualizationservice.VirtualMachineDebugInfo;
+import android.system.virtualizationservice_internal.AtomVmBooted;
+import android.system.virtualizationservice_internal.AtomVmCreationRequested;
+import android.system.virtualizationservice_internal.AtomVmExited;
+import android.system.virtualizationservice_internal.IGlobalVmContext;
+
+/** VFIO related methods which should be done as root. */
+interface IVfioHandler {
+ /**
+ * Bind given devices to vfio driver.
+ *
+ * @param devices paths of sysfs nodes of devices to assign.
+ * @return a file descriptor containing DTBO for VM.
+ */
+ ParcelFileDescriptor bindDevicesToVfioDriver(in String[] devices);
+}
diff --git a/virtualizationservice/src/aidl.rs b/virtualizationservice/src/aidl.rs
index 8aea556..384915c 100644
--- a/virtualizationservice/src/aidl.rs
+++ b/virtualizationservice/src/aidl.rs
@@ -29,26 +29,24 @@
AtomVmExited::AtomVmExited,
IGlobalVmContext::{BnGlobalVmContext, IGlobalVmContext},
IVirtualizationServiceInternal::IVirtualizationServiceInternal,
+ IVfioHandler::{BpVfioHandler, IVfioHandler},
};
use android_system_virtualmachineservice::aidl::android::system::virtualmachineservice::IVirtualMachineService::VM_TOMBSTONES_SERVICE_PORT;
use anyhow::{anyhow, ensure, Context, Result};
-use binder::{self, BinderFeatures, ExceptionCode, Interface, LazyServiceGuard, Status, Strong};
-use lazy_static::lazy_static;
+use binder::{self, wait_for_interface, BinderFeatures, ExceptionCode, Interface, LazyServiceGuard, Status, Strong};
use libc::VMADDR_CID_HOST;
use log::{error, info, warn};
use rustutils::system_properties;
use std::collections::HashMap;
-use std::fs::{canonicalize, create_dir, remove_dir_all, set_permissions, File, Permissions};
+use std::fs::{create_dir, remove_dir_all, set_permissions, Permissions};
use std::io::{Read, Write};
-use std::os::fd::FromRawFd;
use std::os::unix::fs::PermissionsExt;
use std::os::unix::raw::{pid_t, uid_t};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
use std::sync::{Arc, Mutex, Weak};
use tombstoned_client::{DebuggerdDumpType, TombstonedConnection};
use vsock::{VsockListener, VsockStream};
-use nix::fcntl::OFlag;
-use nix::unistd::{chown, pipe2, Uid};
+use nix::unistd::{chown, Uid};
/// The unique ID of a VM used (together with a port number) for vsock communication.
pub type Cid = u32;
@@ -188,64 +186,12 @@
fn bindDevicesToVfioDriver(&self, devices: &[String]) -> binder::Result<ParcelFileDescriptor> {
check_use_custom_virtual_machine()?;
- devices.iter().try_for_each(|x| bind_device(x))?;
-
- // TODO(b/278008182): create a file descriptor containing DTBO for devices.
- let (raw_read, raw_write) = pipe2(OFlag::O_CLOEXEC).map_err(|e| {
- Status::new_exception_str(
- ExceptionCode::SERVICE_SPECIFIC,
- Some(format!("can't create fd for DTBO: {e:?}")),
- )
- })?;
- // SAFETY: We are the sole owner of this FD as we just created it, and it is valid and open.
- let read_fd = unsafe { File::from_raw_fd(raw_read) };
- // SAFETY: We are the sole owner of this FD as we just created it, and it is valid and open.
- let _write_fd = unsafe { File::from_raw_fd(raw_write) };
-
- Ok(ParcelFileDescriptor::new(read_fd))
+ let vfio_service: Strong<dyn IVfioHandler> =
+ wait_for_interface(<BpVfioHandler as IVfioHandler>::get_descriptor())?;
+ vfio_service.bindDevicesToVfioDriver(devices)
}
}
-lazy_static! {
- static ref SYSFS_PLATFORM_DEVICES: &'static Path = Path::new("/sys/devices/platform/");
- static ref VFIO_PLATFORM_DRIVER: &'static Path =
- Path::new("/sys/bus/platform/drivers/vfio-platform");
-}
-
-fn bind_device(device: &str) -> binder::Result<()> {
- // Check platform device exists
- let dev_sysfs_path = canonicalize(device).map_err(|e| {
- Status::new_exception_str(
- ExceptionCode::SERVICE_SPECIFIC,
- Some(format!("can't canonicalize: {e:?}")),
- )
- })?;
- if !dev_sysfs_path.starts_with(*SYSFS_PLATFORM_DEVICES) {
- return Err(Status::new_exception_str(
- ExceptionCode::ILLEGAL_ARGUMENT,
- Some(format!("{device} is not a platform device")),
- ));
- }
-
- // Check platform device is bound to VFIO driver
- let dev_driver_path = canonicalize(dev_sysfs_path.join("driver")).map_err(|e| {
- Status::new_exception_str(
- ExceptionCode::SERVICE_SPECIFIC,
- Some(format!("can't canonicalize: {e:?}")),
- )
- })?;
- if dev_driver_path != *VFIO_PLATFORM_DRIVER {
- // TODO(b/278008182): unbind driver and bind to VFIO
- return Err(Status::new_exception_str(
- ExceptionCode::UNSUPPORTED_OPERATION,
- Some("not implemented".to_owned()),
- ));
- }
-
- // already bound to VFIO driver
- Ok(())
-}
-
#[derive(Debug, Default)]
struct GlobalVmInstance {
/// The unique CID assigned to the VM for vsock communication.
diff --git a/virtualizationservice/vfio_handler/Android.bp b/virtualizationservice/vfio_handler/Android.bp
new file mode 100644
index 0000000..efbb7b5
--- /dev/null
+++ b/virtualizationservice/vfio_handler/Android.bp
@@ -0,0 +1,31 @@
+package {
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+rust_binary {
+ name: "vfio_handler",
+ crate_name: "vfio_handler",
+ edition: "2021",
+ srcs: ["src/main.rs"],
+ // Only build on targets which crosvm builds on.
+ enabled: false,
+ target: {
+ android64: {
+ compile_multilib: "64",
+ enabled: true,
+ },
+ linux_bionic_arm64: {
+ enabled: true,
+ },
+ },
+ prefer_rlib: true,
+ rustlibs: [
+ "android.system.virtualizationservice_internal-rust",
+ "libandroid_logger",
+ "libbinder_rs",
+ "liblog_rust",
+ "libnix",
+ "liblazy_static",
+ ],
+ apex_available: ["com.android.virt"],
+}
diff --git a/virtualizationservice/vfio_handler/src/aidl.rs b/virtualizationservice/vfio_handler/src/aidl.rs
new file mode 100644
index 0000000..9a50fd3
--- /dev/null
+++ b/virtualizationservice/vfio_handler/src/aidl.rs
@@ -0,0 +1,182 @@
+// Copyright 2023, The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Implementation of the AIDL interface of the VirtualizationService.
+
+use android_system_virtualizationservice_internal::aidl::android::system::virtualizationservice_internal::IVfioHandler::IVfioHandler;
+use android_system_virtualizationservice_internal::binder::ParcelFileDescriptor;
+use binder::{self, ExceptionCode, Interface, Status};
+use lazy_static::lazy_static;
+use std::fs::{read_link, write, File};
+use std::os::fd::FromRawFd;
+use std::path::Path;
+use nix::fcntl::OFlag;
+use nix::unistd::pipe2;
+
+#[derive(Debug, Default)]
+pub struct VfioHandler {}
+
+impl VfioHandler {
+ pub fn init() -> VfioHandler {
+ VfioHandler::default()
+ }
+}
+
+impl Interface for VfioHandler {}
+
+impl IVfioHandler for VfioHandler {
+ fn bindDevicesToVfioDriver(&self, devices: &[String]) -> binder::Result<ParcelFileDescriptor> {
+ // permission check is already done by IVirtualizationServiceInternal.
+ if !*IS_VFIO_SUPPORTED {
+ return Err(Status::new_exception_str(
+ ExceptionCode::UNSUPPORTED_OPERATION,
+ Some("VFIO-platform not supported"),
+ ));
+ }
+
+ devices.iter().try_for_each(|x| bind_device(Path::new(x)))?;
+
+ // TODO(b/278008182): create a file descriptor containing DTBO for devices.
+ let (raw_read, raw_write) = pipe2(OFlag::O_CLOEXEC).map_err(|e| {
+ Status::new_exception_str(
+ ExceptionCode::SERVICE_SPECIFIC,
+ Some(format!("can't create fd for DTBO: {e:?}")),
+ )
+ })?;
+ // SAFETY: We are the sole owner of this FD as we just created it, and it is valid and open.
+ let read_fd = unsafe { File::from_raw_fd(raw_read) };
+ // SAFETY: We are the sole owner of this FD as we just created it, and it is valid and open.
+ let _write_fd = unsafe { File::from_raw_fd(raw_write) };
+
+ Ok(ParcelFileDescriptor::new(read_fd))
+ }
+}
+
+const DEV_VFIO_PATH: &str = "/dev/vfio/vfio";
+const SYSFS_PLATFORM_DEVICES_PATH: &str = "/sys/devices/platform/";
+const VFIO_PLATFORM_DRIVER_PATH: &str = "/sys/bus/platform/drivers/vfio-platform";
+const SYSFS_PLATFORM_DRIVERS_PROBE_PATH: &str = "/sys/bus/platform/drivers_probe";
+
+lazy_static! {
+ static ref IS_VFIO_SUPPORTED: bool = is_vfio_supported();
+}
+
+fn is_vfio_supported() -> bool {
+ Path::new(DEV_VFIO_PATH).exists() && Path::new(VFIO_PLATFORM_DRIVER_PATH).exists()
+}
+
+fn check_platform_device(path: &Path) -> binder::Result<()> {
+ if !path.exists() {
+ return Err(Status::new_exception_str(
+ ExceptionCode::ILLEGAL_ARGUMENT,
+ Some(format!("no such device {path:?}")),
+ ));
+ }
+
+ if !path.starts_with(SYSFS_PLATFORM_DEVICES_PATH) {
+ return Err(Status::new_exception_str(
+ ExceptionCode::ILLEGAL_ARGUMENT,
+ Some(format!("{path:?} is not a platform device")),
+ ));
+ }
+
+ Ok(())
+}
+
+fn get_device_iommu_group(path: &Path) -> Option<u64> {
+ let group_path = read_link(path.join("iommu_group")).ok()?;
+ let group = group_path.file_name()?;
+ group.to_str()?.parse().ok()
+}
+
+fn is_bound_to_vfio_driver(path: &Path) -> bool {
+ let Ok(driver_path) = read_link(path.join("driver")) else {
+ return false;
+ };
+ let Some(driver) = driver_path.file_name() else {
+ return false;
+ };
+ driver.to_str().unwrap_or("") == "vfio-platform"
+}
+
+fn bind_vfio_driver(path: &Path) -> binder::Result<()> {
+ if is_bound_to_vfio_driver(path) {
+ // already bound
+ return Ok(());
+ }
+
+ // unbind
+ let Some(device) = path.file_name() else {
+ return Err(Status::new_exception_str(
+ ExceptionCode::ILLEGAL_ARGUMENT,
+ Some(format!("can't get device name from {path:?}"))
+ ));
+ };
+ let Some(device_str) = device.to_str() else {
+ return Err(Status::new_exception_str(
+ ExceptionCode::ILLEGAL_ARGUMENT,
+ Some(format!("invalid filename {device:?}"))
+ ));
+ };
+ write(path.join("driver/unbind"), device_str.as_bytes()).map_err(|e| {
+ Status::new_exception_str(
+ ExceptionCode::SERVICE_SPECIFIC,
+ Some(format!("could not unbind {device_str}: {e:?}")),
+ )
+ })?;
+
+ // bind to VFIO
+ write(path.join("driver_override"), b"vfio-platform").map_err(|e| {
+ Status::new_exception_str(
+ ExceptionCode::SERVICE_SPECIFIC,
+ Some(format!("could not bind {device_str} to vfio-platform: {e:?}")),
+ )
+ })?;
+
+ write(SYSFS_PLATFORM_DRIVERS_PROBE_PATH, device_str.as_bytes()).map_err(|e| {
+ Status::new_exception_str(
+ ExceptionCode::SERVICE_SPECIFIC,
+ Some(format!("could not write {device_str} to drivers-probe: {e:?}")),
+ )
+ })?;
+
+ // final check
+ if !is_bound_to_vfio_driver(path) {
+ return Err(Status::new_exception_str(
+ ExceptionCode::SERVICE_SPECIFIC,
+ Some(format!("{path:?} still not bound to vfio driver")),
+ ));
+ }
+
+ if get_device_iommu_group(path).is_none() {
+ return Err(Status::new_exception_str(
+ ExceptionCode::SERVICE_SPECIFIC,
+ Some(format!("can't get iommu group for {path:?}")),
+ ));
+ }
+
+ Ok(())
+}
+
+fn bind_device(path: &Path) -> binder::Result<()> {
+ let path = path.canonicalize().map_err(|e| {
+ Status::new_exception_str(
+ ExceptionCode::ILLEGAL_ARGUMENT,
+ Some(format!("can't canonicalize {path:?}: {e:?}")),
+ )
+ })?;
+
+ check_platform_device(&path)?;
+ bind_vfio_driver(&path)
+}
diff --git a/virtualizationservice/vfio_handler/src/main.rs b/virtualizationservice/vfio_handler/src/main.rs
new file mode 100644
index 0000000..1a1cce8
--- /dev/null
+++ b/virtualizationservice/vfio_handler/src/main.rs
@@ -0,0 +1,45 @@
+// Copyright 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Android VfioHandler
+
+mod aidl;
+
+use crate::aidl::VfioHandler;
+use android_logger::Config;
+use android_system_virtualizationservice_internal::aidl::android::system::virtualizationservice_internal::IVfioHandler::{
+ BnVfioHandler,
+ BpVfioHandler,
+ IVfioHandler,
+};
+use binder::{register_lazy_service, BinderFeatures, ProcessState};
+use log::{info, Level};
+
+const LOG_TAG: &str = "VfioHandler";
+
+fn main() {
+ android_logger::init_once(
+ Config::default()
+ .with_tag(LOG_TAG)
+ .with_min_level(Level::Info)
+ .with_log_id(android_logger::LogId::System),
+ );
+
+ let service = VfioHandler::init();
+ let service = BnVfioHandler::new_binder(service, BinderFeatures::default());
+ register_lazy_service(<BpVfioHandler as IVfioHandler>::get_descriptor(), service.as_binder())
+ .unwrap();
+ info!("Registered Binder service, joining threadpool.");
+ ProcessState::join_thread_pool();
+}