blob: dfb1cbb3952dbe40cb14516a2f5818b9019a0185 [file] [log] [blame]
Andrew Walbrand6dce6f2021-03-05 16:39:08 +00001// Copyright 2021, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Functions for running instances of `crosvm`.
16
Andrew Walbrandae07162021-03-12 17:05:20 +000017use crate::aidl::VirtualMachineCallbacks;
Andrew Walbrand6dce6f2021-03-05 16:39:08 +000018use crate::Cid;
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000019use anyhow::{bail, Error};
Andrew Walbran02b8ec02021-06-22 13:07:02 +000020use command_fds::CommandFdExt;
Andrew Walbran3a5a9212021-05-04 17:09:08 +000021use log::{debug, error, info};
Andrew Walbrandae07162021-03-12 17:05:20 +000022use shared_child::SharedChild;
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000023use std::fs::{remove_dir_all, File};
Andrew Walbranf8d94112021-09-07 11:45:36 +000024use std::mem;
Andrew Walbranb15cd6e2021-07-05 16:38:07 +000025use std::num::NonZeroU32;
Andrew Walbran02b8ec02021-06-22 13:07:02 +000026use std::os::unix::io::{AsRawFd, RawFd};
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000027use std::path::PathBuf;
Andrew Walbrandae07162021-03-12 17:05:20 +000028use std::process::Command;
Inseob Kim7f61fe72021-08-20 20:50:47 +090029use std::sync::{Arc, Mutex};
Andrew Walbrandae07162021-03-12 17:05:20 +000030use std::thread;
Inseob Kim7f61fe72021-08-20 20:50:47 +090031use vsock::VsockStream;
Inseob Kimc7d28c72021-10-25 14:28:10 +000032use android_system_virtualmachineservice::binder::Strong;
33use android_system_virtualmachineservice::aidl::android::system::virtualmachineservice::IVirtualMachineService::IVirtualMachineService;
Andrew Walbrand6dce6f2021-03-05 16:39:08 +000034
35const CROSVM_PATH: &str = "/apex/com.android.virt/bin/crosvm";
36
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000037/// Configuration for a VM to run with crosvm.
38#[derive(Debug)]
Andrew Walbrand3a84182021-09-07 14:48:52 +000039pub struct CrosvmConfig {
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000040 pub cid: Cid,
Andrew Walbrand3a84182021-09-07 14:48:52 +000041 pub bootloader: Option<File>,
42 pub kernel: Option<File>,
43 pub initrd: Option<File>,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000044 pub disks: Vec<DiskFile>,
45 pub params: Option<String>,
Andrew Walbranf8650422021-06-09 15:54:09 +000046 pub protected: bool,
Andrew Walbranb15cd6e2021-07-05 16:38:07 +000047 pub memory_mib: Option<NonZeroU32>,
Andrew Walbrand3a84182021-09-07 14:48:52 +000048 pub log_fd: Option<File>,
49 pub indirect_files: Vec<File>,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000050}
51
52/// A disk image to pass to crosvm for a VM.
53#[derive(Debug)]
54pub struct DiskFile {
55 pub image: File,
56 pub writable: bool,
57}
58
Andrew Walbran6b650662021-09-07 13:13:23 +000059/// The lifecycle state which the payload in the VM has reported itself to be in.
60///
61/// Note that the order of enum variants is significant; only forward transitions are allowed by
62/// [`VmInstance::update_payload_state`].
63#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
64pub enum PayloadState {
65 Starting,
66 Started,
67 Ready,
68 Finished,
69}
70
Andrew Walbranf8d94112021-09-07 11:45:36 +000071/// The current state of the VM itself.
72#[derive(Debug)]
73pub enum VmState {
74 /// The VM has not yet tried to start.
75 NotStarted {
76 ///The configuration needed to start the VM, if it has not yet been started.
77 config: CrosvmConfig,
78 },
79 /// The VM has been started.
80 Running {
81 /// The crosvm child process.
82 child: Arc<SharedChild>,
83 },
84 /// The VM died or was killed.
85 Dead,
86 /// The VM failed to start.
87 Failed,
88}
89
90impl VmState {
91 /// Tries to start the VM, if it is in the `NotStarted` state.
92 ///
93 /// Returns an error if the VM is in the wrong state, or fails to start.
94 fn start(&mut self, instance: Arc<VmInstance>) -> Result<(), Error> {
95 let state = mem::replace(self, VmState::Failed);
96 if let VmState::NotStarted { config } = state {
97 // If this fails and returns an error, `self` will be left in the `Failed` state.
98 let child = Arc::new(run_vm(config)?);
99
100 let child_clone = child.clone();
101 thread::spawn(move || {
102 instance.monitor(child_clone);
103 });
104
105 // If it started correctly, update the state.
106 *self = VmState::Running { child };
107 Ok(())
108 } else {
109 *self = state;
110 bail!("VM already started or failed")
111 }
112 }
113}
114
115/// Information about a particular instance of a VM which may be running.
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000116#[derive(Debug)]
117pub struct VmInstance {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000118 /// The current state of the VM.
119 pub vm_state: Mutex<VmState>,
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000120 /// The CID assigned to the VM for vsock communication.
121 pub cid: Cid,
Andrew Walbranf8650422021-06-09 15:54:09 +0000122 /// Whether the VM is a protected VM.
123 pub protected: bool,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000124 /// Directory of temporary files used by the VM while it is running.
125 pub temporary_directory: PathBuf,
Andrew Walbranf6a1eb92021-04-01 11:16:02 +0000126 /// The UID of the process which requested the VM.
127 pub requester_uid: u32,
128 /// The SID of the process which requested the VM.
Andrew Walbran02034492021-04-13 15:05:07 +0000129 pub requester_sid: String,
Andrew Walbranf6a1eb92021-04-01 11:16:02 +0000130 /// The PID of the process which requested the VM. Note that this process may no longer exist
131 /// and the PID may have been reused for a different process, so this should not be trusted.
Andrew Walbran02034492021-04-13 15:05:07 +0000132 pub requester_debug_pid: i32,
Andrew Walbrandae07162021-03-12 17:05:20 +0000133 /// Callbacks to clients of the VM.
134 pub callbacks: VirtualMachineCallbacks,
Inseob Kim7f61fe72021-08-20 20:50:47 +0900135 /// Input/output stream of the payload run in the VM.
136 pub stream: Mutex<Option<VsockStream>>,
Inseob Kimc7d28c72021-10-25 14:28:10 +0000137 /// VirtualMachineService binder object for the VM.
138 pub vm_service: Mutex<Option<Strong<dyn IVirtualMachineService>>>,
Andrew Walbran6b650662021-09-07 13:13:23 +0000139 /// The latest lifecycle state which the payload reported itself to be in.
140 payload_state: Mutex<PayloadState>,
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000141}
142
143impl VmInstance {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000144 /// Validates the given config and creates a new `VmInstance` but doesn't start running it.
145 pub fn new(
Andrew Walbrand3a84182021-09-07 14:48:52 +0000146 config: CrosvmConfig,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000147 temporary_directory: PathBuf,
Andrew Walbranf6a1eb92021-04-01 11:16:02 +0000148 requester_uid: u32,
Andrew Walbran02034492021-04-13 15:05:07 +0000149 requester_sid: String,
150 requester_debug_pid: i32,
Andrew Walbranf8d94112021-09-07 11:45:36 +0000151 ) -> Result<VmInstance, Error> {
152 validate_config(&config)?;
Andrew Walbrand3a84182021-09-07 14:48:52 +0000153 let cid = config.cid;
154 let protected = config.protected;
Andrew Walbranf8d94112021-09-07 11:45:36 +0000155 Ok(VmInstance {
156 vm_state: Mutex::new(VmState::NotStarted { config }),
Andrew Walbrand3a84182021-09-07 14:48:52 +0000157 cid,
158 protected,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000159 temporary_directory,
Andrew Walbran02034492021-04-13 15:05:07 +0000160 requester_uid,
161 requester_sid,
162 requester_debug_pid,
Andrew Walbranf8d94112021-09-07 11:45:36 +0000163 callbacks: Default::default(),
164 stream: Mutex::new(None),
Inseob Kimc7d28c72021-10-25 14:28:10 +0000165 vm_service: Mutex::new(None),
Andrew Walbranf8d94112021-09-07 11:45:36 +0000166 payload_state: Mutex::new(PayloadState::Starting),
167 })
Andrew Walbrandae07162021-03-12 17:05:20 +0000168 }
169
Andrew Walbranf8d94112021-09-07 11:45:36 +0000170 /// Starts an instance of `crosvm` to manage the VM. The `crosvm` instance will be killed when
171 /// the `VmInstance` is dropped.
172 pub fn start(self: &Arc<Self>) -> Result<(), Error> {
173 self.vm_state.lock().unwrap().start(self.clone())
174 }
175
176 /// Waits for the crosvm child process to finish, then marks the VM as no longer running and
177 /// calls any callbacks.
178 ///
179 /// This takes a separate reference to the `SharedChild` rather than using the one in
180 /// `self.vm_state` to avoid holding the lock on `vm_state` while it is running.
181 fn monitor(&self, child: Arc<SharedChild>) {
182 match child.wait() {
Andrew Walbrandae07162021-03-12 17:05:20 +0000183 Err(e) => error!("Error waiting for crosvm instance to die: {}", e),
184 Ok(status) => info!("crosvm exited with status {}", status),
185 }
Andrew Walbranf8d94112021-09-07 11:45:36 +0000186
187 let mut vm_state = self.vm_state.lock().unwrap();
188 *vm_state = VmState::Dead;
189 // Ensure that the mutex is released before calling the callbacks.
190 drop(vm_state);
191
Andrew Walbrandae07162021-03-12 17:05:20 +0000192 self.callbacks.callback_on_died(self.cid);
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000193
194 // Delete temporary files.
195 if let Err(e) = remove_dir_all(&self.temporary_directory) {
Andrew Walbran806f1542021-06-10 14:07:12 +0000196 error!("Error removing temporary directory {:?}: {}", self.temporary_directory, e);
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000197 }
Andrew Walbrandae07162021-03-12 17:05:20 +0000198 }
199
Andrew Walbran6b650662021-09-07 13:13:23 +0000200 /// Returns the last reported state of the VM payload.
201 pub fn payload_state(&self) -> PayloadState {
202 *self.payload_state.lock().unwrap()
203 }
204
205 /// Updates the payload state to the given value, if it is a valid state transition.
206 pub fn update_payload_state(&self, new_state: PayloadState) -> Result<(), Error> {
207 let mut state_locked = self.payload_state.lock().unwrap();
208 // Only allow forward transitions, e.g. from starting to started or finished, not back in
209 // the other direction.
210 if new_state > *state_locked {
211 *state_locked = new_state;
212 Ok(())
213 } else {
214 bail!("Invalid payload state transition from {:?} to {:?}", *state_locked, new_state)
215 }
216 }
217
Andrew Walbranf8d94112021-09-07 11:45:36 +0000218 /// Kills the crosvm instance, if it is running.
Andrew Walbrandae07162021-03-12 17:05:20 +0000219 pub fn kill(&self) {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000220 let vm_state = &*self.vm_state.lock().unwrap();
221 if let VmState::Running { child } = vm_state {
222 // TODO: Talk to crosvm to shutdown cleanly.
223 if let Err(e) = child.kill() {
224 error!("Error killing crosvm instance: {}", e);
225 }
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000226 }
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000227 }
228}
229
Andrew Walbrand3a84182021-09-07 14:48:52 +0000230/// Starts an instance of `crosvm` to manage a new VM.
231fn run_vm(config: CrosvmConfig) -> Result<SharedChild, Error> {
232 validate_config(&config)?;
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000233
234 let mut command = Command::new(CROSVM_PATH);
235 // TODO(qwandor): Remove --disable-sandbox.
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000236 command.arg("run").arg("--disable-sandbox").arg("--cid").arg(config.cid.to_string());
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000237
Andrew Walbranf8650422021-06-09 15:54:09 +0000238 if config.protected {
239 command.arg("--protected-vm");
240 }
241
Andrew Walbranb15cd6e2021-07-05 16:38:07 +0000242 if let Some(memory_mib) = config.memory_mib {
243 command.arg("--mem").arg(memory_mib.to_string());
244 }
245
Jiyong Parkfa91d702021-10-18 23:51:39 +0900246 // Keep track of what file descriptors should be mapped to the crosvm process.
247 let mut preserved_fds = config.indirect_files.iter().map(|file| file.as_raw_fd()).collect();
248
Jiyong Park747d6362021-10-19 17:12:52 +0900249 // Setup the serial devices.
250 // 1. uart device: used as the output device by bootloaders and as early console by linux
251 // 2. virtio-console device: used as the console device
Jiyong Parkfa91d702021-10-18 23:51:39 +0900252 // 3. virtio-console device: used as the logcat output
Jiyong Park747d6362021-10-19 17:12:52 +0900253 //
254 // When log_fd is not specified, the devices are attached to sink, which means what's written
255 // there is discarded.
Jiyong Parkfa91d702021-10-18 23:51:39 +0900256 let path = config.log_fd.as_ref().map(|fd| add_preserved_fd(&mut preserved_fds, fd));
257 let backend = path.as_ref().map_or("sink", |_| "file");
258 let path_arg = path.as_ref().map_or(String::new(), |path| format!(",path={}", path));
259
Jiyong Park747d6362021-10-19 17:12:52 +0900260 // Warning: Adding more serial devices requires you to shift the PCI device ID of the boot
261 // disks in bootconfig.x86_64. This is because x86 crosvm puts serial devices and the block
262 // devices in the same PCI bus and serial devices comes before the block devices. Arm crosvm
263 // doesn't have the issue.
Jiyong Parkfa91d702021-10-18 23:51:39 +0900264 // /dev/ttyS0
265 command.arg(format!("--serial=type={}{},hardware=serial", backend, &path_arg));
266 // /dev/hvc0
267 command.arg(format!("--serial=type={}{},hardware=virtio-console,num=1", backend, &path_arg));
268 // /dev/hvc1
269 // TODO(b/200914564) use a different fd for logcat log
270 command.arg(format!("--serial=type={}{},hardware=virtio-console,num=2", backend, &path_arg));
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000271
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000272 if let Some(bootloader) = &config.bootloader {
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000273 command.arg("--bios").arg(add_preserved_fd(&mut preserved_fds, bootloader));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000274 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000275
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000276 if let Some(initrd) = &config.initrd {
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000277 command.arg("--initrd").arg(add_preserved_fd(&mut preserved_fds, initrd));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000278 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000279
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000280 if let Some(params) = &config.params {
281 command.arg("--params").arg(params);
282 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000283
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000284 for disk in &config.disks {
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000285 command
286 .arg(if disk.writable { "--rwdisk" } else { "--disk" })
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000287 .arg(add_preserved_fd(&mut preserved_fds, &disk.image));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000288 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000289
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000290 if let Some(kernel) = &config.kernel {
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000291 command.arg(add_preserved_fd(&mut preserved_fds, kernel));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000292 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000293
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000294 debug!("Preserving FDs {:?}", preserved_fds);
295 command.preserved_fds(preserved_fds);
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000296
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000297 info!("Running {:?}", command);
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000298 let result = SharedChild::spawn(&mut command)?;
299 Ok(result)
300}
301
302/// Ensure that the configuration has a valid combination of fields set, or return an error if not.
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000303fn validate_config(config: &CrosvmConfig) -> Result<(), Error> {
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000304 if config.bootloader.is_none() && config.kernel.is_none() {
305 bail!("VM must have either a bootloader or a kernel image.");
306 }
307 if config.bootloader.is_some() && (config.kernel.is_some() || config.initrd.is_some()) {
308 bail!("Can't have both bootloader and kernel/initrd image.");
309 }
310 Ok(())
311}
312
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000313/// Adds the file descriptor for `file` to `preserved_fds`, and returns a string of the form
314/// "/proc/self/fd/N" where N is the file descriptor.
315fn add_preserved_fd(preserved_fds: &mut Vec<RawFd>, file: &File) -> String {
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000316 let fd = file.as_raw_fd();
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000317 preserved_fds.push(fd);
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000318 format!("/proc/self/fd/{}", fd)
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000319}