blob: 29040b76707c53744ba2acecf64a396532683b67 [file] [log] [blame]
Andrew Walbrand6dce6f2021-03-05 16:39:08 +00001// Copyright 2021, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Functions for running instances of `crosvm`.
16
David Brazdil41d1a872022-10-05 14:44:19 +010017use crate::aidl::{Cid, VirtualMachineCallbacks};
Seungjae Yoob4c07ba2022-08-12 04:44:52 +000018use crate::atom::write_vm_exited_stats;
Jiyong Park1612b902022-08-22 14:47:39 +090019use anyhow::{anyhow, bail, Context, Error};
Andrew Walbran02b8ec02021-06-22 13:07:02 +000020use command_fds::CommandFdExt;
Jiyong Parke6ed0f92022-06-22 00:13:00 +090021use lazy_static::lazy_static;
Andrew Walbran3a5a9212021-05-04 17:09:08 +000022use log::{debug, error, info};
Jiyong Parkdcf17412022-02-08 15:07:23 +090023use semver::{Version, VersionReq};
Andrew Walbranb27681f2022-02-23 15:11:52 +000024use nix::{fcntl::OFlag, unistd::pipe2};
Jiyong Park2d736562022-10-24 22:40:12 +090025use regex::{Captures, Regex};
Andrew Walbrandae07162021-03-12 17:05:20 +000026use shared_child::SharedChild;
Jiyong Parke6ed0f92022-06-22 00:13:00 +090027use std::borrow::Cow;
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000028use std::fs::{remove_dir_all, File};
Andrew Walbranb27681f2022-02-23 15:11:52 +000029use std::io::{self, Read};
Andrew Walbranf8d94112021-09-07 11:45:36 +000030use std::mem;
Andrew Walbranb15cd6e2021-07-05 16:38:07 +000031use std::num::NonZeroU32;
Andrew Walbranb27681f2022-02-23 15:11:52 +000032use std::os::unix::io::{AsRawFd, RawFd, FromRawFd};
Jiyong Park1612b902022-08-22 14:47:39 +090033use std::path::{Path, PathBuf};
Andrew Walbranc92d35f2022-01-12 12:45:19 +000034use std::process::{Command, ExitStatus};
Jiyong Parke6ed0f92022-06-22 00:13:00 +090035use std::sync::{Arc, Condvar, Mutex};
Seungjae Yoo2e7beea2022-08-24 16:09:12 +090036use std::time::{Duration, SystemTime};
Andrew Walbrandae07162021-03-12 17:05:20 +000037use std::thread;
Andrew Walbranc92d35f2022-01-12 12:45:19 +000038use android_system_virtualizationservice::aidl::android::system::virtualizationservice::DeathReason::DeathReason;
Alan Stokes0e82b502022-08-08 14:44:48 +010039use binder::Strong;
Inseob Kimc7d28c72021-10-25 14:28:10 +000040use android_system_virtualmachineservice::aidl::android::system::virtualmachineservice::IVirtualMachineService::IVirtualMachineService;
Jiyong Park1612b902022-08-22 14:47:39 +090041use tombstoned_client::{TombstonedConnection, DebuggerdDumpType};
Andrew Walbrand6dce6f2021-03-05 16:39:08 +000042
Keir Fraser13a956a2022-07-14 14:20:46 +000043/// external/crosvm
44use base::UnixSeqpacketListener;
45
Andrew Walbrand6dce6f2021-03-05 16:39:08 +000046const CROSVM_PATH: &str = "/apex/com.android.virt/bin/crosvm";
47
Jiyong Parkdcf17412022-02-08 15:07:23 +090048/// Version of the platform that crosvm currently implements. The format follows SemVer. This
49/// should be updated when there is a platform change in the crosvm side. Having this value here is
50/// fine because virtualizationservice and crosvm are supposed to be updated together in the virt
51/// APEX.
52const CROSVM_PLATFORM_VERSION: &str = "1.0.0";
53
Andrew Walbrand15c5632022-02-03 13:38:31 +000054/// The exit status which crosvm returns when it has an error starting a VM.
55const CROSVM_ERROR_STATUS: i32 = 1;
Andrew Walbranc92d35f2022-01-12 12:45:19 +000056/// The exit status which crosvm returns when a VM requests a reboot.
57const CROSVM_REBOOT_STATUS: i32 = 32;
Andrew Walbrand15c5632022-02-03 13:38:31 +000058/// The exit status which crosvm returns when it crashes due to an error.
59const CROSVM_CRASH_STATUS: i32 = 33;
Sebastian Ene23167d82022-10-07 14:09:53 +000060/// The exit status which crosvm returns when vcpu is stalled.
61const CROSVM_WATCHDOG_REBOOT_STATUS: i32 = 36;
Andrew Walbranc92d35f2022-01-12 12:45:19 +000062
Jiyong Parke6ed0f92022-06-22 00:13:00 +090063lazy_static! {
64 /// If the VM doesn't move to the Started state within this amount time, a hang-up error is
65 /// triggered.
Alan Stokesc3f2ac22022-06-23 12:19:46 +010066 static ref BOOT_HANGUP_TIMEOUT: Duration = if nested_virt::is_nested_virtualization().unwrap() {
Jiyong Parke6ed0f92022-06-22 00:13:00 +090067 // Nested virtualization is slow, so we need a longer timeout.
68 Duration::from_secs(100)
69 } else {
70 Duration::from_secs(10)
71 };
72}
73
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000074/// Configuration for a VM to run with crosvm.
75#[derive(Debug)]
Andrew Walbrand3a84182021-09-07 14:48:52 +000076pub struct CrosvmConfig {
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000077 pub cid: Cid,
Seungjae Yoo62085c02022-08-12 04:44:52 +000078 pub name: String,
Andrew Walbrand3a84182021-09-07 14:48:52 +000079 pub bootloader: Option<File>,
80 pub kernel: Option<File>,
81 pub initrd: Option<File>,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000082 pub disks: Vec<DiskFile>,
83 pub params: Option<String>,
Andrew Walbranf8650422021-06-09 15:54:09 +000084 pub protected: bool,
Andrew Walbranb15cd6e2021-07-05 16:38:07 +000085 pub memory_mib: Option<NonZeroU32>,
Jiyong Park032615f2022-01-10 13:55:34 +090086 pub cpus: Option<NonZeroU32>,
Jiyong Parkdfe16d62022-04-20 17:32:12 +090087 pub task_profiles: Vec<String>,
Jiyong Parkb8182bb2021-10-26 22:53:08 +090088 pub console_fd: Option<File>,
Andrew Walbrand3a84182021-09-07 14:48:52 +000089 pub log_fd: Option<File>,
Jiyong Parke558ab12022-07-07 20:18:55 +090090 pub ramdump: Option<File>,
Andrew Walbrand3a84182021-09-07 14:48:52 +000091 pub indirect_files: Vec<File>,
Jiyong Parkdcf17412022-02-08 15:07:23 +090092 pub platform_version: VersionReq,
Jiyong Parke6ed0f92022-06-22 00:13:00 +090093 pub detect_hangup: bool,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000094}
95
96/// A disk image to pass to crosvm for a VM.
97#[derive(Debug)]
98pub struct DiskFile {
99 pub image: File,
100 pub writable: bool,
101}
102
Andrew Walbran6b650662021-09-07 13:13:23 +0000103/// The lifecycle state which the payload in the VM has reported itself to be in.
104///
105/// Note that the order of enum variants is significant; only forward transitions are allowed by
106/// [`VmInstance::update_payload_state`].
107#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
108pub enum PayloadState {
109 Starting,
110 Started,
111 Ready,
112 Finished,
Jiyong Parka4eebde2022-07-12 18:01:12 +0900113 Hangup, // Hasn't reached to Ready before timeout expires
Andrew Walbran6b650662021-09-07 13:13:23 +0000114}
115
Andrew Walbranf8d94112021-09-07 11:45:36 +0000116/// The current state of the VM itself.
117#[derive(Debug)]
118pub enum VmState {
119 /// The VM has not yet tried to start.
120 NotStarted {
121 ///The configuration needed to start the VM, if it has not yet been started.
122 config: CrosvmConfig,
123 },
124 /// The VM has been started.
125 Running {
126 /// The crosvm child process.
127 child: Arc<SharedChild>,
128 },
129 /// The VM died or was killed.
130 Dead,
131 /// The VM failed to start.
132 Failed,
133}
134
135impl VmState {
136 /// Tries to start the VM, if it is in the `NotStarted` state.
137 ///
138 /// Returns an error if the VM is in the wrong state, or fails to start.
139 fn start(&mut self, instance: Arc<VmInstance>) -> Result<(), Error> {
140 let state = mem::replace(self, VmState::Failed);
141 if let VmState::NotStarted { config } = state {
Jiyong Parke6ed0f92022-06-22 00:13:00 +0900142 let detect_hangup = config.detect_hangup;
Andrew Walbranb27681f2022-02-23 15:11:52 +0000143 let (failure_pipe_read, failure_pipe_write) = create_pipe()?;
144
Andrew Walbranf8d94112021-09-07 11:45:36 +0000145 // If this fails and returns an error, `self` will be left in the `Failed` state.
Keir Fraser13a956a2022-07-14 14:20:46 +0000146 let child =
147 Arc::new(run_vm(config, &instance.temporary_directory, failure_pipe_write)?);
Andrew Walbranf8d94112021-09-07 11:45:36 +0000148
149 let child_clone = child.clone();
Jiyong Parka4eebde2022-07-12 18:01:12 +0900150 let instance_clone = instance.clone();
Andrew Walbranf8d94112021-09-07 11:45:36 +0000151 thread::spawn(move || {
Jiyong Parka4eebde2022-07-12 18:01:12 +0900152 instance_clone.monitor_vm_exit(child_clone, failure_pipe_read);
Andrew Walbranf8d94112021-09-07 11:45:36 +0000153 });
154
Jiyong Parka4eebde2022-07-12 18:01:12 +0900155 if detect_hangup {
156 let child_clone = child.clone();
157 thread::spawn(move || {
158 instance.monitor_payload_hangup(child_clone);
159 });
160 }
161
Andrew Walbranf8d94112021-09-07 11:45:36 +0000162 // If it started correctly, update the state.
163 *self = VmState::Running { child };
164 Ok(())
165 } else {
166 *self = state;
167 bail!("VM already started or failed")
168 }
169 }
170}
171
172/// Information about a particular instance of a VM which may be running.
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000173#[derive(Debug)]
174pub struct VmInstance {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000175 /// The current state of the VM.
176 pub vm_state: Mutex<VmState>,
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000177 /// The CID assigned to the VM for vsock communication.
178 pub cid: Cid,
Seungjae Yoo62085c02022-08-12 04:44:52 +0000179 /// The name of the VM.
180 pub name: String,
Andrew Walbranf8650422021-06-09 15:54:09 +0000181 /// Whether the VM is a protected VM.
182 pub protected: bool,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000183 /// Directory of temporary files used by the VM while it is running.
184 pub temporary_directory: PathBuf,
Andrew Walbranf6a1eb92021-04-01 11:16:02 +0000185 /// The UID of the process which requested the VM.
186 pub requester_uid: u32,
Andrew Walbranf6a1eb92021-04-01 11:16:02 +0000187 /// The PID of the process which requested the VM. Note that this process may no longer exist
188 /// and the PID may have been reused for a different process, so this should not be trusted.
Andrew Walbran02034492021-04-13 15:05:07 +0000189 pub requester_debug_pid: i32,
Andrew Walbrandae07162021-03-12 17:05:20 +0000190 /// Callbacks to clients of the VM.
191 pub callbacks: VirtualMachineCallbacks,
Inseob Kimc7d28c72021-10-25 14:28:10 +0000192 /// VirtualMachineService binder object for the VM.
193 pub vm_service: Mutex<Option<Strong<dyn IVirtualMachineService>>>,
Seungjae Yoo2e7beea2022-08-24 16:09:12 +0900194 /// Recorded timestamp when the VM is started.
195 pub vm_start_timestamp: Mutex<Option<SystemTime>>,
Andrew Walbran6b650662021-09-07 13:13:23 +0000196 /// The latest lifecycle state which the payload reported itself to be in.
197 payload_state: Mutex<PayloadState>,
Jiyong Parka4eebde2022-07-12 18:01:12 +0900198 /// Represents the condition that payload_state was updated
199 payload_state_updated: Condvar,
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000200}
201
202impl VmInstance {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000203 /// Validates the given config and creates a new `VmInstance` but doesn't start running it.
204 pub fn new(
Andrew Walbrand3a84182021-09-07 14:48:52 +0000205 config: CrosvmConfig,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000206 temporary_directory: PathBuf,
Andrew Walbranf6a1eb92021-04-01 11:16:02 +0000207 requester_uid: u32,
Andrew Walbran02034492021-04-13 15:05:07 +0000208 requester_debug_pid: i32,
Andrew Walbranf8d94112021-09-07 11:45:36 +0000209 ) -> Result<VmInstance, Error> {
210 validate_config(&config)?;
Andrew Walbrand3a84182021-09-07 14:48:52 +0000211 let cid = config.cid;
Seungjae Yoo62085c02022-08-12 04:44:52 +0000212 let name = config.name.clone();
Andrew Walbrand3a84182021-09-07 14:48:52 +0000213 let protected = config.protected;
Andrew Walbranf8d94112021-09-07 11:45:36 +0000214 Ok(VmInstance {
215 vm_state: Mutex::new(VmState::NotStarted { config }),
Andrew Walbrand3a84182021-09-07 14:48:52 +0000216 cid,
Seungjae Yoo62085c02022-08-12 04:44:52 +0000217 name,
Andrew Walbrand3a84182021-09-07 14:48:52 +0000218 protected,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000219 temporary_directory,
Andrew Walbran02034492021-04-13 15:05:07 +0000220 requester_uid,
Andrew Walbran02034492021-04-13 15:05:07 +0000221 requester_debug_pid,
Andrew Walbranf8d94112021-09-07 11:45:36 +0000222 callbacks: Default::default(),
Inseob Kimc7d28c72021-10-25 14:28:10 +0000223 vm_service: Mutex::new(None),
Seungjae Yoo2e7beea2022-08-24 16:09:12 +0900224 vm_start_timestamp: Mutex::new(None),
Andrew Walbranf8d94112021-09-07 11:45:36 +0000225 payload_state: Mutex::new(PayloadState::Starting),
Jiyong Parka4eebde2022-07-12 18:01:12 +0900226 payload_state_updated: Condvar::new(),
Andrew Walbranf8d94112021-09-07 11:45:36 +0000227 })
Andrew Walbrandae07162021-03-12 17:05:20 +0000228 }
229
Andrew Walbranf8d94112021-09-07 11:45:36 +0000230 /// Starts an instance of `crosvm` to manage the VM. The `crosvm` instance will be killed when
231 /// the `VmInstance` is dropped.
232 pub fn start(self: &Arc<Self>) -> Result<(), Error> {
Seungjae Yoo2e7beea2022-08-24 16:09:12 +0900233 *self.vm_start_timestamp.lock().unwrap() = Some(SystemTime::now());
Andrew Walbranf8d94112021-09-07 11:45:36 +0000234 self.vm_state.lock().unwrap().start(self.clone())
235 }
236
Jiyong Parka4eebde2022-07-12 18:01:12 +0900237 /// Monitors the exit of the VM (i.e. termination of the `child` process). When that happens,
238 /// handles the event by updating the state, noityfing the event to clients by calling
239 /// callbacks, and removing temporary files for the VM.
240 fn monitor_vm_exit(&self, child: Arc<SharedChild>, mut failure_pipe_read: File) {
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000241 let result = child.wait();
242 match &result {
Jooyung Hanbfe086f2021-10-28 10:15:45 +0900243 Err(e) => error!("Error waiting for crosvm({}) instance to die: {}", child.id(), e),
Sebastian Ene23167d82022-10-07 14:09:53 +0000244 Ok(status) => {
245 info!("crosvm({}) exited with status {}", child.id(), status);
246 if let Some(exit_status_code) = status.code() {
247 if exit_status_code == CROSVM_WATCHDOG_REBOOT_STATUS {
248 info!("detected vcpu stall on crosvm");
249 }
250 }
251 }
Andrew Walbrandae07162021-03-12 17:05:20 +0000252 }
Andrew Walbranf8d94112021-09-07 11:45:36 +0000253
254 let mut vm_state = self.vm_state.lock().unwrap();
255 *vm_state = VmState::Dead;
256 // Ensure that the mutex is released before calling the callbacks.
257 drop(vm_state);
258
Jiyong Parka4eebde2022-07-12 18:01:12 +0900259 // Read the pipe to see if any failure reason is written
260 let mut failure_reason = String::new();
261 match failure_pipe_read.read_to_string(&mut failure_reason) {
262 Err(e) => error!("Error reading VM failure reason from pipe: {}", e),
263 Ok(len) if len > 0 => info!("VM returned failure reason '{}'", &failure_reason),
264 _ => (),
Jiyong Parke6ed0f92022-06-22 00:13:00 +0900265 };
Andrew Walbranb27681f2022-02-23 15:11:52 +0000266
Jiyong Parka4eebde2022-07-12 18:01:12 +0900267 // In case of hangup, the pipe doesn't give us any information because the hangup can't be
268 // detected on the VM side (otherwise, it isn't a hangup), but in the
269 // monitor_payload_hangup function below which updates the payload state to Hangup.
270 let failure_reason =
271 if failure_reason.is_empty() && self.payload_state() == PayloadState::Hangup {
272 Cow::from("HANGUP")
273 } else {
274 Cow::from(failure_reason)
275 };
276
Jiyong Parke558ab12022-07-07 20:18:55 +0900277 self.handle_ramdump().unwrap_or_else(|e| error!("Error handling ramdump: {}", e));
Seungjae Yoob4c07ba2022-08-12 04:44:52 +0000278
279 let death_reason = death_reason(&result, &failure_reason);
280 self.callbacks.callback_on_died(self.cid, death_reason);
Seungjae Yoo2e7beea2022-08-24 16:09:12 +0900281
282 let vm_start_timestamp = self.vm_start_timestamp.lock().unwrap();
283 write_vm_exited_stats(
284 self.requester_uid as i32,
285 &self.name,
286 death_reason,
287 *vm_start_timestamp,
288 );
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000289
290 // Delete temporary files.
291 if let Err(e) = remove_dir_all(&self.temporary_directory) {
Andrew Walbran806f1542021-06-10 14:07:12 +0000292 error!("Error removing temporary directory {:?}: {}", self.temporary_directory, e);
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000293 }
Andrew Walbrandae07162021-03-12 17:05:20 +0000294 }
295
Jiyong Parka4eebde2022-07-12 18:01:12 +0900296 /// Waits until payload is started, or timeout expires. When timeout occurs, kill
297 /// the VM to prevent indefinite hangup and update the payload_state accordingly.
298 fn monitor_payload_hangup(&self, child: Arc<SharedChild>) {
299 debug!("Starting to monitor hangup for Microdroid({})", child.id());
300 let (_, result) = self
301 .payload_state_updated
302 .wait_timeout_while(self.payload_state.lock().unwrap(), *BOOT_HANGUP_TIMEOUT, |s| {
303 *s < PayloadState::Started
304 })
305 .unwrap();
306 let child_still_running = child.try_wait().ok() == Some(None);
307 if result.timed_out() && child_still_running {
308 error!(
309 "Microdroid({}) failed to start payload within {} secs timeout. Shutting down.",
310 child.id(),
311 BOOT_HANGUP_TIMEOUT.as_secs()
312 );
313 self.update_payload_state(PayloadState::Hangup).unwrap();
314 if let Err(e) = self.kill() {
315 error!("Error stopping timed-out VM with CID {}: {:?}", child.id(), e);
316 }
317 }
318 }
319
Andrew Walbran6b650662021-09-07 13:13:23 +0000320 /// Returns the last reported state of the VM payload.
321 pub fn payload_state(&self) -> PayloadState {
322 *self.payload_state.lock().unwrap()
323 }
324
325 /// Updates the payload state to the given value, if it is a valid state transition.
326 pub fn update_payload_state(&self, new_state: PayloadState) -> Result<(), Error> {
327 let mut state_locked = self.payload_state.lock().unwrap();
328 // Only allow forward transitions, e.g. from starting to started or finished, not back in
329 // the other direction.
330 if new_state > *state_locked {
331 *state_locked = new_state;
Jiyong Parka4eebde2022-07-12 18:01:12 +0900332 self.payload_state_updated.notify_all();
Andrew Walbran6b650662021-09-07 13:13:23 +0000333 Ok(())
334 } else {
335 bail!("Invalid payload state transition from {:?} to {:?}", *state_locked, new_state)
336 }
337 }
338
Andrew Walbranf8d94112021-09-07 11:45:36 +0000339 /// Kills the crosvm instance, if it is running.
Inseob Kima446f802022-07-11 19:46:37 +0900340 pub fn kill(&self) -> Result<(), Error> {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000341 let vm_state = &*self.vm_state.lock().unwrap();
342 if let VmState::Running { child } = vm_state {
Jooyung Hanbfe086f2021-10-28 10:15:45 +0900343 let id = child.id();
344 debug!("Killing crosvm({})", id);
Andrew Walbranf8d94112021-09-07 11:45:36 +0000345 // TODO: Talk to crosvm to shutdown cleanly.
346 if let Err(e) = child.kill() {
Inseob Kima446f802022-07-11 19:46:37 +0900347 bail!("Error killing crosvm({}) instance: {}", id, e);
348 } else {
349 Ok(())
Andrew Walbranf8d94112021-09-07 11:45:36 +0000350 }
Inseob Kima446f802022-07-11 19:46:37 +0900351 } else {
352 bail!("VM is not running")
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000353 }
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000354 }
Jiyong Parke558ab12022-07-07 20:18:55 +0900355
356 /// Checks if ramdump has been created. If so, send a notification to the user with the handle
357 /// to read the ramdump.
358 fn handle_ramdump(&self) -> Result<(), Error> {
359 let ramdump_path = self.temporary_directory.join("ramdump");
360 if std::fs::metadata(&ramdump_path)?.len() > 0 {
361 let ramdump = File::open(&ramdump_path)
362 .context(format!("Failed to open ramdump {:?} for reading", &ramdump_path))?;
363 self.callbacks.callback_on_ramdump(self.cid, ramdump);
Jiyong Park1612b902022-08-22 14:47:39 +0900364
365 Self::send_ramdump_to_tombstoned(&ramdump_path)?;
Jiyong Parke558ab12022-07-07 20:18:55 +0900366 }
367 Ok(())
368 }
Jiyong Park1612b902022-08-22 14:47:39 +0900369
370 fn send_ramdump_to_tombstoned(ramdump_path: &Path) -> Result<(), Error> {
371 let mut input = File::open(ramdump_path)
372 .context(format!("Failed to open raudmp {:?} for reading", ramdump_path))?;
373
374 let pid = std::process::id() as i32;
375 let conn = TombstonedConnection::connect(pid, DebuggerdDumpType::Tombstone)
376 .context("Failed to connect to tombstoned")?;
377 let mut output = conn
378 .text_output
379 .as_ref()
380 .ok_or_else(|| anyhow!("Could not get file to write the tombstones on"))?;
381
382 std::io::copy(&mut input, &mut output).context("Failed to send ramdump to tombstoned")?;
383 info!("Ramdump {:?} sent to tombstoned", ramdump_path);
384
385 conn.notify_completion()?;
386 Ok(())
387 }
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000388}
389
Alan Stokes3ba10fd2022-10-06 15:46:51 +0100390fn death_reason(result: &Result<ExitStatus, io::Error>, mut failure_reason: &str) -> DeathReason {
391 if let Some(position) = failure_reason.find('|') {
392 // Separator indicates extra context information is present after the failure name.
393 error!("Failure info: {}", &failure_reason[(position + 1)..]);
394 failure_reason = &failure_reason[..position];
395 }
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000396 if let Ok(status) = result {
Andrew Walbranb27681f2022-02-23 15:11:52 +0000397 match failure_reason {
398 "PVM_FIRMWARE_PUBLIC_KEY_MISMATCH" => {
399 return DeathReason::PVM_FIRMWARE_PUBLIC_KEY_MISMATCH
400 }
401 "PVM_FIRMWARE_INSTANCE_IMAGE_CHANGED" => {
402 return DeathReason::PVM_FIRMWARE_INSTANCE_IMAGE_CHANGED
403 }
404 "BOOTLOADER_PUBLIC_KEY_MISMATCH" => return DeathReason::BOOTLOADER_PUBLIC_KEY_MISMATCH,
405 "BOOTLOADER_INSTANCE_IMAGE_CHANGED" => {
406 return DeathReason::BOOTLOADER_INSTANCE_IMAGE_CHANGED
407 }
Inseob Kim272f5722022-06-13 17:14:51 +0900408 "MICRODROID_FAILED_TO_CONNECT_TO_VIRTUALIZATION_SERVICE" => {
409 return DeathReason::MICRODROID_FAILED_TO_CONNECT_TO_VIRTUALIZATION_SERVICE
410 }
411 "MICRODROID_PAYLOAD_HAS_CHANGED" => return DeathReason::MICRODROID_PAYLOAD_HAS_CHANGED,
412 "MICRODROID_PAYLOAD_VERIFICATION_FAILED" => {
413 return DeathReason::MICRODROID_PAYLOAD_VERIFICATION_FAILED
414 }
415 "MICRODROID_INVALID_PAYLOAD_CONFIG" => {
416 return DeathReason::MICRODROID_INVALID_PAYLOAD_CONFIG
417 }
418 "MICRODROID_UNKNOWN_RUNTIME_ERROR" => {
419 return DeathReason::MICRODROID_UNKNOWN_RUNTIME_ERROR
420 }
Jiyong Parke6ed0f92022-06-22 00:13:00 +0900421 "HANGUP" => return DeathReason::HANGUP,
Andrew Walbranb27681f2022-02-23 15:11:52 +0000422 _ => {}
423 }
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000424 match status.code() {
425 None => DeathReason::KILLED,
426 Some(0) => DeathReason::SHUTDOWN,
Andrew Walbrand15c5632022-02-03 13:38:31 +0000427 Some(CROSVM_ERROR_STATUS) => DeathReason::ERROR,
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000428 Some(CROSVM_REBOOT_STATUS) => DeathReason::REBOOT,
Andrew Walbrand15c5632022-02-03 13:38:31 +0000429 Some(CROSVM_CRASH_STATUS) => DeathReason::CRASH,
Sebastian Ene23167d82022-10-07 14:09:53 +0000430 Some(CROSVM_WATCHDOG_REBOOT_STATUS) => DeathReason::WATCHDOG_REBOOT,
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000431 Some(_) => DeathReason::UNKNOWN,
432 }
433 } else {
434 DeathReason::INFRASTRUCTURE_ERROR
435 }
436}
437
Andrew Walbrand3a84182021-09-07 14:48:52 +0000438/// Starts an instance of `crosvm` to manage a new VM.
Keir Fraser13a956a2022-07-14 14:20:46 +0000439fn run_vm(
440 config: CrosvmConfig,
441 temporary_directory: &Path,
442 failure_pipe_write: File,
443) -> Result<SharedChild, Error> {
Andrew Walbrand3a84182021-09-07 14:48:52 +0000444 validate_config(&config)?;
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000445
446 let mut command = Command::new(CROSVM_PATH);
447 // TODO(qwandor): Remove --disable-sandbox.
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000448 command
449 .arg("--extended-status")
Jiyong Park6c60fea2022-10-24 16:10:01 +0900450 // Configure the logger for the crosvm process to silence logs from the disk crate which
451 // don't provide much information to us (but do spamming us).
452 .arg("--log-level")
453 .arg("info,disk=off")
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000454 .arg("run")
455 .arg("--disable-sandbox")
Keir Fraser72762722022-09-30 16:12:06 +0000456 .arg("--no-balloon")
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000457 .arg("--cid")
458 .arg(config.cid.to_string());
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000459
Andrew Walbranf8650422021-06-09 15:54:09 +0000460 if config.protected {
David Brazdil86c76fa2022-02-04 15:50:57 +0000461 command.arg("--protected-vm");
Andrew Walbran0b5789f2022-02-04 13:57:57 +0000462
463 // 3 virtio-console devices + vsock = 4.
464 let virtio_pci_device_count = 4 + config.disks.len();
465 // crosvm virtio queue has 256 entries, so 2 MiB per device (2 pages per entry) should be
466 // enough.
467 let swiotlb_size_mib = 2 * virtio_pci_device_count;
468 command.arg("--swiotlb").arg(swiotlb_size_mib.to_string());
Andrew Walbranf8650422021-06-09 15:54:09 +0000469 }
470
Andrew Walbranb15cd6e2021-07-05 16:38:07 +0000471 if let Some(memory_mib) = config.memory_mib {
472 command.arg("--mem").arg(memory_mib.to_string());
473 }
474
Jiyong Park032615f2022-01-10 13:55:34 +0900475 if let Some(cpus) = config.cpus {
476 command.arg("--cpus").arg(cpus.to_string());
477 }
478
Jiyong Parkdfe16d62022-04-20 17:32:12 +0900479 if !config.task_profiles.is_empty() {
480 command.arg("--task-profiles").arg(config.task_profiles.join(","));
481 }
482
Jiyong Parkfa91d702021-10-18 23:51:39 +0900483 // Keep track of what file descriptors should be mapped to the crosvm process.
484 let mut preserved_fds = config.indirect_files.iter().map(|file| file.as_raw_fd()).collect();
485
Jiyong Park747d6362021-10-19 17:12:52 +0900486 // Setup the serial devices.
487 // 1. uart device: used as the output device by bootloaders and as early console by linux
Andrew Walbranb27681f2022-02-23 15:11:52 +0000488 // 2. uart device: used to report the reason for the VM failing.
489 // 3. virtio-console device: used as the console device where kmsg is redirected to
Jiyong Park4afe2012022-07-08 05:38:49 +0900490 // 4. virtio-console device: used as the ramdump output
Andrew Walbranb27681f2022-02-23 15:11:52 +0000491 // 5. virtio-console device: used as the logcat output
Jiyong Park747d6362021-10-19 17:12:52 +0900492 //
Jiyong Parkb8182bb2021-10-26 22:53:08 +0900493 // When [console|log]_fd is not specified, the devices are attached to sink, which means what's
494 // written there is discarded.
Andrew Walbranb27681f2022-02-23 15:11:52 +0000495 let console_arg = format_serial_arg(&mut preserved_fds, &config.console_fd);
496 let log_arg = format_serial_arg(&mut preserved_fds, &config.log_fd);
497 let failure_serial_path = add_preserved_fd(&mut preserved_fds, &failure_pipe_write);
Jiyong Parke558ab12022-07-07 20:18:55 +0900498 let ramdump_arg = format_serial_arg(&mut preserved_fds, &config.ramdump);
Jiyong Parkfa91d702021-10-18 23:51:39 +0900499
Jiyong Park747d6362021-10-19 17:12:52 +0900500 // Warning: Adding more serial devices requires you to shift the PCI device ID of the boot
501 // disks in bootconfig.x86_64. This is because x86 crosvm puts serial devices and the block
502 // devices in the same PCI bus and serial devices comes before the block devices. Arm crosvm
503 // doesn't have the issue.
Jiyong Parkfa91d702021-10-18 23:51:39 +0900504 // /dev/ttyS0
Andrew Walbranb27681f2022-02-23 15:11:52 +0000505 command.arg(format!("--serial={},hardware=serial,num=1", &console_arg));
506 // /dev/ttyS1
507 command.arg(format!("--serial=type=file,path={},hardware=serial,num=2", &failure_serial_path));
Jiyong Parkfa91d702021-10-18 23:51:39 +0900508 // /dev/hvc0
Jiyong Parkb8182bb2021-10-26 22:53:08 +0900509 command.arg(format!("--serial={},hardware=virtio-console,num=1", &console_arg));
Jiyong Park4afe2012022-07-08 05:38:49 +0900510 // /dev/hvc1
511 command.arg(format!("--serial={},hardware=virtio-console,num=2", &ramdump_arg));
Jiyong Parkae5a4ed2021-11-01 18:36:28 +0900512 // /dev/hvc2
513 command.arg(format!("--serial={},hardware=virtio-console,num=3", &log_arg));
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000514
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000515 if let Some(bootloader) = &config.bootloader {
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000516 command.arg("--bios").arg(add_preserved_fd(&mut preserved_fds, bootloader));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000517 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000518
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000519 if let Some(initrd) = &config.initrd {
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000520 command.arg("--initrd").arg(add_preserved_fd(&mut preserved_fds, initrd));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000521 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000522
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000523 if let Some(params) = &config.params {
524 command.arg("--params").arg(params);
525 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000526
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000527 for disk in &config.disks {
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000528 command
529 .arg(if disk.writable { "--rwdisk" } else { "--disk" })
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000530 .arg(add_preserved_fd(&mut preserved_fds, &disk.image));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000531 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000532
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000533 if let Some(kernel) = &config.kernel {
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000534 command.arg(add_preserved_fd(&mut preserved_fds, kernel));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000535 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000536
Keir Fraser13a956a2022-07-14 14:20:46 +0000537 let control_server_socket =
538 UnixSeqpacketListener::bind(temporary_directory.join("crosvm.sock"))
539 .context("failed to create control server")?;
540 command.arg("--socket").arg(add_preserved_fd(&mut preserved_fds, &control_server_socket));
541
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000542 debug!("Preserving FDs {:?}", preserved_fds);
543 command.preserved_fds(preserved_fds);
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000544
Jaewan Kimb2814062022-11-14 13:21:40 +0900545 command.arg("--params").arg("crashkernel=17M");
Jiyong Park2d736562022-10-24 22:40:12 +0900546 print_crosvm_args(&command);
547
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000548 let result = SharedChild::spawn(&mut command)?;
Jooyung Hanbfe086f2021-10-28 10:15:45 +0900549 debug!("Spawned crosvm({}).", result.id());
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000550 Ok(result)
551}
552
553/// Ensure that the configuration has a valid combination of fields set, or return an error if not.
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000554fn validate_config(config: &CrosvmConfig) -> Result<(), Error> {
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000555 if config.bootloader.is_none() && config.kernel.is_none() {
556 bail!("VM must have either a bootloader or a kernel image.");
557 }
558 if config.bootloader.is_some() && (config.kernel.is_some() || config.initrd.is_some()) {
559 bail!("Can't have both bootloader and kernel/initrd image.");
560 }
Jiyong Parkdcf17412022-02-08 15:07:23 +0900561 let version = Version::parse(CROSVM_PLATFORM_VERSION).unwrap();
562 if !config.platform_version.matches(&version) {
563 bail!(
564 "Incompatible platform version. The config is compatible with platform version(s) \
565 {}, but the actual platform version is {}",
566 config.platform_version,
567 version
568 );
569 }
570
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000571 Ok(())
572}
573
Jiyong Park2d736562022-10-24 22:40:12 +0900574/// Print arguments of the crosvm command. In doing so, /proc/self/fd/XX is annotated with the
575/// actual file path if the FD is backed by a regular file. If not, the /proc path is printed
576/// unmodified.
577fn print_crosvm_args(command: &Command) {
578 let re = Regex::new(r"/proc/self/fd/[\d]+").unwrap();
579 info!(
580 "Running crosvm with args: {:?}",
581 command
582 .get_args()
583 .map(|s| s.to_string_lossy())
584 .map(|s| {
585 re.replace_all(&s, |caps: &Captures| {
586 let path = &caps[0];
587 if let Ok(realpath) = std::fs::canonicalize(path) {
588 format!("{} ({})", path, realpath.to_string_lossy())
589 } else {
590 path.to_owned()
591 }
592 })
593 .into_owned()
594 })
595 .collect::<Vec<_>>()
596 );
597}
598
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000599/// Adds the file descriptor for `file` to `preserved_fds`, and returns a string of the form
600/// "/proc/self/fd/N" where N is the file descriptor.
Keir Fraser13a956a2022-07-14 14:20:46 +0000601fn add_preserved_fd(preserved_fds: &mut Vec<RawFd>, file: &dyn AsRawFd) -> String {
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000602 let fd = file.as_raw_fd();
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000603 preserved_fds.push(fd);
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000604 format!("/proc/self/fd/{}", fd)
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000605}
Andrew Walbranb27681f2022-02-23 15:11:52 +0000606
607/// Adds the file descriptor for `file` (if any) to `preserved_fds`, and returns the appropriate
608/// string for a crosvm `--serial` flag. If `file` is none, creates a dummy sink device.
609fn format_serial_arg(preserved_fds: &mut Vec<RawFd>, file: &Option<File>) -> String {
610 if let Some(file) = file {
611 format!("type=file,path={}", add_preserved_fd(preserved_fds, file))
612 } else {
613 "type=sink".to_string()
614 }
615}
616
617/// Creates a new pipe with the `O_CLOEXEC` flag set, and returns the read side and write side.
618fn create_pipe() -> Result<(File, File), Error> {
619 let (raw_read, raw_write) = pipe2(OFlag::O_CLOEXEC)?;
620 // SAFETY: We are the sole owners of these fds as they were just created.
621 let read_fd = unsafe { File::from_raw_fd(raw_read) };
622 let write_fd = unsafe { File::from_raw_fd(raw_write) };
623 Ok((read_fd, write_fd))
624}