blob: 299411dfc0b4a15d73aea9723df9b3fd6deecd23 [file] [log] [blame]
Andrew Walbrand6dce6f2021-03-05 16:39:08 +00001// Copyright 2021, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Functions for running instances of `crosvm`.
16
David Brazdil41d1a872022-10-05 14:44:19 +010017use crate::aidl::{Cid, VirtualMachineCallbacks};
Seungjae Yoob4c07ba2022-08-12 04:44:52 +000018use crate::atom::write_vm_exited_stats;
Seungjae Yoo6d265d92022-11-15 10:51:33 +090019use anyhow::{anyhow, bail, Context, Error, Result};
Andrew Walbran02b8ec02021-06-22 13:07:02 +000020use command_fds::CommandFdExt;
Jiyong Parke6ed0f92022-06-22 00:13:00 +090021use lazy_static::lazy_static;
Seungjae Yoo6d265d92022-11-15 10:51:33 +090022use libc::{sysconf, _SC_CLK_TCK};
Andrew Walbran3a5a9212021-05-04 17:09:08 +000023use log::{debug, error, info};
Jiyong Parkdcf17412022-02-08 15:07:23 +090024use semver::{Version, VersionReq};
Jiyong Park08eee7b2022-11-11 15:01:26 +090025use nix::{fcntl::OFlag, unistd::pipe2, unistd::Uid, unistd::User};
Jiyong Park2d736562022-10-24 22:40:12 +090026use regex::{Captures, Regex};
Andrew Walbrandae07162021-03-12 17:05:20 +000027use shared_child::SharedChild;
Jiyong Parke6ed0f92022-06-22 00:13:00 +090028use std::borrow::Cow;
Seungjae Yoo6d265d92022-11-15 10:51:33 +090029use std::cmp::max;
Jiyong Park08eee7b2022-11-11 15:01:26 +090030use std::fmt;
Seungjae Yoo6d265d92022-11-15 10:51:33 +090031use std::fs::{read_to_string, remove_dir_all, File};
Andrew Walbranb27681f2022-02-23 15:11:52 +000032use std::io::{self, Read};
Andrew Walbranf8d94112021-09-07 11:45:36 +000033use std::mem;
Andrew Walbranb15cd6e2021-07-05 16:38:07 +000034use std::num::NonZeroU32;
Andrew Walbranb27681f2022-02-23 15:11:52 +000035use std::os::unix::io::{AsRawFd, RawFd, FromRawFd};
Jiyong Park1612b902022-08-22 14:47:39 +090036use std::path::{Path, PathBuf};
Andrew Walbranc92d35f2022-01-12 12:45:19 +000037use std::process::{Command, ExitStatus};
Jiyong Parke6ed0f92022-06-22 00:13:00 +090038use std::sync::{Arc, Condvar, Mutex};
Seungjae Yoo2e7beea2022-08-24 16:09:12 +090039use std::time::{Duration, SystemTime};
Andrew Walbrandae07162021-03-12 17:05:20 +000040use std::thread;
Andrew Walbranc92d35f2022-01-12 12:45:19 +000041use android_system_virtualizationservice::aidl::android::system::virtualizationservice::DeathReason::DeathReason;
Alan Stokes0e82b502022-08-08 14:44:48 +010042use binder::Strong;
Inseob Kimc7d28c72021-10-25 14:28:10 +000043use android_system_virtualmachineservice::aidl::android::system::virtualmachineservice::IVirtualMachineService::IVirtualMachineService;
Jiyong Park1612b902022-08-22 14:47:39 +090044use tombstoned_client::{TombstonedConnection, DebuggerdDumpType};
Andrew Walbrand6dce6f2021-03-05 16:39:08 +000045
Keir Fraser13a956a2022-07-14 14:20:46 +000046/// external/crosvm
47use base::UnixSeqpacketListener;
48
Andrew Walbrand6dce6f2021-03-05 16:39:08 +000049const CROSVM_PATH: &str = "/apex/com.android.virt/bin/crosvm";
50
Jiyong Parkdcf17412022-02-08 15:07:23 +090051/// Version of the platform that crosvm currently implements. The format follows SemVer. This
52/// should be updated when there is a platform change in the crosvm side. Having this value here is
53/// fine because virtualizationservice and crosvm are supposed to be updated together in the virt
54/// APEX.
55const CROSVM_PLATFORM_VERSION: &str = "1.0.0";
56
Andrew Walbrand15c5632022-02-03 13:38:31 +000057/// The exit status which crosvm returns when it has an error starting a VM.
58const CROSVM_ERROR_STATUS: i32 = 1;
Andrew Walbranc92d35f2022-01-12 12:45:19 +000059/// The exit status which crosvm returns when a VM requests a reboot.
60const CROSVM_REBOOT_STATUS: i32 = 32;
Andrew Walbrand15c5632022-02-03 13:38:31 +000061/// The exit status which crosvm returns when it crashes due to an error.
62const CROSVM_CRASH_STATUS: i32 = 33;
Sebastian Ene23167d82022-10-07 14:09:53 +000063/// The exit status which crosvm returns when vcpu is stalled.
64const CROSVM_WATCHDOG_REBOOT_STATUS: i32 = 36;
Andrew Walbranc92d35f2022-01-12 12:45:19 +000065
Seungjae Yoo6d265d92022-11-15 10:51:33 +090066const MILLIS_PER_SEC: i64 = 1000;
67
Jiyong Parke6ed0f92022-06-22 00:13:00 +090068lazy_static! {
69 /// If the VM doesn't move to the Started state within this amount time, a hang-up error is
70 /// triggered.
Alan Stokesc3f2ac22022-06-23 12:19:46 +010071 static ref BOOT_HANGUP_TIMEOUT: Duration = if nested_virt::is_nested_virtualization().unwrap() {
Jiyong Parke6ed0f92022-06-22 00:13:00 +090072 // Nested virtualization is slow, so we need a longer timeout.
73 Duration::from_secs(100)
74 } else {
75 Duration::from_secs(10)
76 };
77}
78
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000079/// Configuration for a VM to run with crosvm.
80#[derive(Debug)]
Andrew Walbrand3a84182021-09-07 14:48:52 +000081pub struct CrosvmConfig {
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000082 pub cid: Cid,
Seungjae Yoo62085c02022-08-12 04:44:52 +000083 pub name: String,
Andrew Walbrand3a84182021-09-07 14:48:52 +000084 pub bootloader: Option<File>,
85 pub kernel: Option<File>,
86 pub initrd: Option<File>,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000087 pub disks: Vec<DiskFile>,
88 pub params: Option<String>,
Andrew Walbranf8650422021-06-09 15:54:09 +000089 pub protected: bool,
Andrew Walbranb15cd6e2021-07-05 16:38:07 +000090 pub memory_mib: Option<NonZeroU32>,
Jiyong Park032615f2022-01-10 13:55:34 +090091 pub cpus: Option<NonZeroU32>,
Jiyong Parkdfe16d62022-04-20 17:32:12 +090092 pub task_profiles: Vec<String>,
Jiyong Parkb8182bb2021-10-26 22:53:08 +090093 pub console_fd: Option<File>,
Andrew Walbrand3a84182021-09-07 14:48:52 +000094 pub log_fd: Option<File>,
Jiyong Parke558ab12022-07-07 20:18:55 +090095 pub ramdump: Option<File>,
Andrew Walbrand3a84182021-09-07 14:48:52 +000096 pub indirect_files: Vec<File>,
Jiyong Parkdcf17412022-02-08 15:07:23 +090097 pub platform_version: VersionReq,
Jiyong Parke6ed0f92022-06-22 00:13:00 +090098 pub detect_hangup: bool,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000099}
100
101/// A disk image to pass to crosvm for a VM.
102#[derive(Debug)]
103pub struct DiskFile {
104 pub image: File,
105 pub writable: bool,
106}
107
Andrew Walbran6b650662021-09-07 13:13:23 +0000108/// The lifecycle state which the payload in the VM has reported itself to be in.
109///
110/// Note that the order of enum variants is significant; only forward transitions are allowed by
111/// [`VmInstance::update_payload_state`].
112#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
113pub enum PayloadState {
114 Starting,
115 Started,
116 Ready,
117 Finished,
Jiyong Parka4eebde2022-07-12 18:01:12 +0900118 Hangup, // Hasn't reached to Ready before timeout expires
Andrew Walbran6b650662021-09-07 13:13:23 +0000119}
120
Andrew Walbranf8d94112021-09-07 11:45:36 +0000121/// The current state of the VM itself.
122#[derive(Debug)]
123pub enum VmState {
124 /// The VM has not yet tried to start.
125 NotStarted {
126 ///The configuration needed to start the VM, if it has not yet been started.
127 config: CrosvmConfig,
128 },
129 /// The VM has been started.
130 Running {
131 /// The crosvm child process.
132 child: Arc<SharedChild>,
133 },
134 /// The VM died or was killed.
135 Dead,
136 /// The VM failed to start.
137 Failed,
138}
139
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900140/// RSS values of VM and CrosVM process itself.
141#[derive(Copy, Clone, Debug, Default)]
142pub struct Rss {
143 pub vm: i64,
144 pub crosvm: i64,
145}
146
147/// Metrics regarding the VM.
148#[derive(Debug, Default)]
149pub struct VmMetric {
150 /// Recorded timestamp when the VM is started.
151 pub start_timestamp: Option<SystemTime>,
152 /// Update most recent guest_time periodically from /proc/[crosvm pid]/stat while VM is running.
153 pub cpu_guest_time: Option<i64>,
154 /// Update maximum RSS values periodically from /proc/[crosvm pid]/smaps while VM is running.
155 pub rss: Option<Rss>,
156}
157
Andrew Walbranf8d94112021-09-07 11:45:36 +0000158impl VmState {
159 /// Tries to start the VM, if it is in the `NotStarted` state.
160 ///
161 /// Returns an error if the VM is in the wrong state, or fails to start.
162 fn start(&mut self, instance: Arc<VmInstance>) -> Result<(), Error> {
163 let state = mem::replace(self, VmState::Failed);
164 if let VmState::NotStarted { config } = state {
Jiyong Parke6ed0f92022-06-22 00:13:00 +0900165 let detect_hangup = config.detect_hangup;
Andrew Walbranb27681f2022-02-23 15:11:52 +0000166 let (failure_pipe_read, failure_pipe_write) = create_pipe()?;
167
Andrew Walbranf8d94112021-09-07 11:45:36 +0000168 // If this fails and returns an error, `self` will be left in the `Failed` state.
Keir Fraser13a956a2022-07-14 14:20:46 +0000169 let child =
170 Arc::new(run_vm(config, &instance.temporary_directory, failure_pipe_write)?);
Andrew Walbranf8d94112021-09-07 11:45:36 +0000171
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900172 let instance_monitor_status = instance.clone();
173 let child_monitor_status = child.clone();
174 thread::spawn(move || {
175 instance_monitor_status.clone().monitor_vm_status(child_monitor_status);
176 });
177
Andrew Walbranf8d94112021-09-07 11:45:36 +0000178 let child_clone = child.clone();
Jiyong Parka4eebde2022-07-12 18:01:12 +0900179 let instance_clone = instance.clone();
Andrew Walbranf8d94112021-09-07 11:45:36 +0000180 thread::spawn(move || {
Jiyong Parka4eebde2022-07-12 18:01:12 +0900181 instance_clone.monitor_vm_exit(child_clone, failure_pipe_read);
Andrew Walbranf8d94112021-09-07 11:45:36 +0000182 });
183
Jiyong Parka4eebde2022-07-12 18:01:12 +0900184 if detect_hangup {
185 let child_clone = child.clone();
186 thread::spawn(move || {
187 instance.monitor_payload_hangup(child_clone);
188 });
189 }
190
Andrew Walbranf8d94112021-09-07 11:45:36 +0000191 // If it started correctly, update the state.
192 *self = VmState::Running { child };
193 Ok(())
194 } else {
195 *self = state;
196 bail!("VM already started or failed")
197 }
198 }
199}
200
201/// Information about a particular instance of a VM which may be running.
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000202#[derive(Debug)]
203pub struct VmInstance {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000204 /// The current state of the VM.
205 pub vm_state: Mutex<VmState>,
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000206 /// The CID assigned to the VM for vsock communication.
207 pub cid: Cid,
Seungjae Yoo62085c02022-08-12 04:44:52 +0000208 /// The name of the VM.
209 pub name: String,
Andrew Walbranf8650422021-06-09 15:54:09 +0000210 /// Whether the VM is a protected VM.
211 pub protected: bool,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000212 /// Directory of temporary files used by the VM while it is running.
213 pub temporary_directory: PathBuf,
Andrew Walbranf6a1eb92021-04-01 11:16:02 +0000214 /// The UID of the process which requested the VM.
215 pub requester_uid: u32,
Andrew Walbranf6a1eb92021-04-01 11:16:02 +0000216 /// The PID of the process which requested the VM. Note that this process may no longer exist
217 /// and the PID may have been reused for a different process, so this should not be trusted.
Andrew Walbran02034492021-04-13 15:05:07 +0000218 pub requester_debug_pid: i32,
Andrew Walbrandae07162021-03-12 17:05:20 +0000219 /// Callbacks to clients of the VM.
220 pub callbacks: VirtualMachineCallbacks,
Inseob Kimc7d28c72021-10-25 14:28:10 +0000221 /// VirtualMachineService binder object for the VM.
222 pub vm_service: Mutex<Option<Strong<dyn IVirtualMachineService>>>,
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900223 /// Recorded metrics of VM such as timestamp or cpu / memory usage.
224 pub vm_metric: Mutex<VmMetric>,
Andrew Walbran6b650662021-09-07 13:13:23 +0000225 /// The latest lifecycle state which the payload reported itself to be in.
226 payload_state: Mutex<PayloadState>,
Jiyong Parka4eebde2022-07-12 18:01:12 +0900227 /// Represents the condition that payload_state was updated
228 payload_state_updated: Condvar,
Jiyong Park08eee7b2022-11-11 15:01:26 +0900229 /// The human readable name of requester_uid
230 requester_uid_name: String,
231}
232
233impl fmt::Display for VmInstance {
234 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
235 let adj = if self.protected { "Protected" } else { "Non-protected" };
236 write!(
237 f,
238 "{} virtual machine \"{}\" (owner: {}, cid: {})",
239 adj, self.name, self.requester_uid_name, self.cid
240 )
241 }
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000242}
243
244impl VmInstance {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000245 /// Validates the given config and creates a new `VmInstance` but doesn't start running it.
246 pub fn new(
Andrew Walbrand3a84182021-09-07 14:48:52 +0000247 config: CrosvmConfig,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000248 temporary_directory: PathBuf,
Andrew Walbranf6a1eb92021-04-01 11:16:02 +0000249 requester_uid: u32,
Andrew Walbran02034492021-04-13 15:05:07 +0000250 requester_debug_pid: i32,
Andrew Walbranf8d94112021-09-07 11:45:36 +0000251 ) -> Result<VmInstance, Error> {
252 validate_config(&config)?;
Andrew Walbrand3a84182021-09-07 14:48:52 +0000253 let cid = config.cid;
Seungjae Yoo62085c02022-08-12 04:44:52 +0000254 let name = config.name.clone();
Andrew Walbrand3a84182021-09-07 14:48:52 +0000255 let protected = config.protected;
Jiyong Park08eee7b2022-11-11 15:01:26 +0900256 let requester_uid_name = User::from_uid(Uid::from_raw(requester_uid))
257 .ok()
258 .flatten()
259 .map_or_else(|| format!("{}", requester_uid), |u| u.name);
260 let instance = VmInstance {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000261 vm_state: Mutex::new(VmState::NotStarted { config }),
Andrew Walbrand3a84182021-09-07 14:48:52 +0000262 cid,
Seungjae Yoo62085c02022-08-12 04:44:52 +0000263 name,
Andrew Walbrand3a84182021-09-07 14:48:52 +0000264 protected,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000265 temporary_directory,
Andrew Walbran02034492021-04-13 15:05:07 +0000266 requester_uid,
Andrew Walbran02034492021-04-13 15:05:07 +0000267 requester_debug_pid,
Andrew Walbranf8d94112021-09-07 11:45:36 +0000268 callbacks: Default::default(),
Inseob Kimc7d28c72021-10-25 14:28:10 +0000269 vm_service: Mutex::new(None),
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900270 vm_metric: Mutex::new(Default::default()),
Andrew Walbranf8d94112021-09-07 11:45:36 +0000271 payload_state: Mutex::new(PayloadState::Starting),
Jiyong Parka4eebde2022-07-12 18:01:12 +0900272 payload_state_updated: Condvar::new(),
Jiyong Park08eee7b2022-11-11 15:01:26 +0900273 requester_uid_name,
274 };
275 info!("{} created", &instance);
276 Ok(instance)
Andrew Walbrandae07162021-03-12 17:05:20 +0000277 }
278
Andrew Walbranf8d94112021-09-07 11:45:36 +0000279 /// Starts an instance of `crosvm` to manage the VM. The `crosvm` instance will be killed when
280 /// the `VmInstance` is dropped.
281 pub fn start(self: &Arc<Self>) -> Result<(), Error> {
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900282 let mut vm_metric = self.vm_metric.lock().unwrap();
283 vm_metric.start_timestamp = Some(SystemTime::now());
Jiyong Park08eee7b2022-11-11 15:01:26 +0900284 let ret = self.vm_state.lock().unwrap().start(self.clone());
285 if ret.is_ok() {
286 info!("{} started", &self);
287 }
288 ret.with_context(|| format!("{} failed to start", &self))
Andrew Walbranf8d94112021-09-07 11:45:36 +0000289 }
290
Jiyong Parka4eebde2022-07-12 18:01:12 +0900291 /// Monitors the exit of the VM (i.e. termination of the `child` process). When that happens,
292 /// handles the event by updating the state, noityfing the event to clients by calling
293 /// callbacks, and removing temporary files for the VM.
294 fn monitor_vm_exit(&self, child: Arc<SharedChild>, mut failure_pipe_read: File) {
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000295 let result = child.wait();
296 match &result {
Jooyung Hanbfe086f2021-10-28 10:15:45 +0900297 Err(e) => error!("Error waiting for crosvm({}) instance to die: {}", child.id(), e),
Sebastian Ene23167d82022-10-07 14:09:53 +0000298 Ok(status) => {
299 info!("crosvm({}) exited with status {}", child.id(), status);
300 if let Some(exit_status_code) = status.code() {
301 if exit_status_code == CROSVM_WATCHDOG_REBOOT_STATUS {
302 info!("detected vcpu stall on crosvm");
303 }
304 }
305 }
Andrew Walbrandae07162021-03-12 17:05:20 +0000306 }
Andrew Walbranf8d94112021-09-07 11:45:36 +0000307
308 let mut vm_state = self.vm_state.lock().unwrap();
309 *vm_state = VmState::Dead;
310 // Ensure that the mutex is released before calling the callbacks.
311 drop(vm_state);
Jiyong Park08eee7b2022-11-11 15:01:26 +0900312 info!("{} exited", &self);
Andrew Walbranf8d94112021-09-07 11:45:36 +0000313
Jiyong Parka4eebde2022-07-12 18:01:12 +0900314 // Read the pipe to see if any failure reason is written
315 let mut failure_reason = String::new();
316 match failure_pipe_read.read_to_string(&mut failure_reason) {
317 Err(e) => error!("Error reading VM failure reason from pipe: {}", e),
318 Ok(len) if len > 0 => info!("VM returned failure reason '{}'", &failure_reason),
319 _ => (),
Jiyong Parke6ed0f92022-06-22 00:13:00 +0900320 };
Andrew Walbranb27681f2022-02-23 15:11:52 +0000321
Jiyong Parka4eebde2022-07-12 18:01:12 +0900322 // In case of hangup, the pipe doesn't give us any information because the hangup can't be
323 // detected on the VM side (otherwise, it isn't a hangup), but in the
324 // monitor_payload_hangup function below which updates the payload state to Hangup.
325 let failure_reason =
326 if failure_reason.is_empty() && self.payload_state() == PayloadState::Hangup {
327 Cow::from("HANGUP")
328 } else {
329 Cow::from(failure_reason)
330 };
331
Jiyong Parke558ab12022-07-07 20:18:55 +0900332 self.handle_ramdump().unwrap_or_else(|e| error!("Error handling ramdump: {}", e));
Seungjae Yoob4c07ba2022-08-12 04:44:52 +0000333
334 let death_reason = death_reason(&result, &failure_reason);
335 self.callbacks.callback_on_died(self.cid, death_reason);
Seungjae Yoo2e7beea2022-08-24 16:09:12 +0900336
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900337 let vm_metric = self.vm_metric.lock().unwrap();
338 write_vm_exited_stats(self.requester_uid as i32, &self.name, death_reason, &*vm_metric);
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000339
340 // Delete temporary files.
341 if let Err(e) = remove_dir_all(&self.temporary_directory) {
Andrew Walbran806f1542021-06-10 14:07:12 +0000342 error!("Error removing temporary directory {:?}: {}", self.temporary_directory, e);
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000343 }
Andrew Walbrandae07162021-03-12 17:05:20 +0000344 }
345
Jiyong Parka4eebde2022-07-12 18:01:12 +0900346 /// Waits until payload is started, or timeout expires. When timeout occurs, kill
347 /// the VM to prevent indefinite hangup and update the payload_state accordingly.
348 fn monitor_payload_hangup(&self, child: Arc<SharedChild>) {
349 debug!("Starting to monitor hangup for Microdroid({})", child.id());
350 let (_, result) = self
351 .payload_state_updated
352 .wait_timeout_while(self.payload_state.lock().unwrap(), *BOOT_HANGUP_TIMEOUT, |s| {
353 *s < PayloadState::Started
354 })
355 .unwrap();
356 let child_still_running = child.try_wait().ok() == Some(None);
357 if result.timed_out() && child_still_running {
358 error!(
359 "Microdroid({}) failed to start payload within {} secs timeout. Shutting down.",
360 child.id(),
361 BOOT_HANGUP_TIMEOUT.as_secs()
362 );
363 self.update_payload_state(PayloadState::Hangup).unwrap();
364 if let Err(e) = self.kill() {
365 error!("Error stopping timed-out VM with CID {}: {:?}", child.id(), e);
366 }
367 }
368 }
369
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900370 fn monitor_vm_status(&self, child: Arc<SharedChild>) {
371 let pid = child.id();
372
373 loop {
374 {
375 // Check VM state
376 let vm_state = &*self.vm_state.lock().unwrap();
377 if let VmState::Dead = vm_state {
378 break;
379 }
380
381 let mut vm_metric = self.vm_metric.lock().unwrap();
382
383 // Get CPU Information
384 // TODO: Collect it once right before VM dies using SIGCHLD
385 if let Ok(guest_time) = get_guest_time(pid) {
386 vm_metric.cpu_guest_time = Some(guest_time);
387 } else {
388 error!("Failed to parse /proc/[pid]/stat");
389 }
390
391 // Get Memory Information
392 if let Ok(rss) = get_rss(pid) {
393 vm_metric.rss = match &vm_metric.rss {
394 Some(x) => Some(Rss::extract_max(x, &rss)),
395 None => Some(rss),
396 }
397 } else {
398 error!("Failed to parse /proc/[pid]/smaps");
399 }
400 }
401
402 thread::sleep(Duration::from_secs(1));
403 }
404 }
405
Andrew Walbran6b650662021-09-07 13:13:23 +0000406 /// Returns the last reported state of the VM payload.
407 pub fn payload_state(&self) -> PayloadState {
408 *self.payload_state.lock().unwrap()
409 }
410
411 /// Updates the payload state to the given value, if it is a valid state transition.
412 pub fn update_payload_state(&self, new_state: PayloadState) -> Result<(), Error> {
413 let mut state_locked = self.payload_state.lock().unwrap();
414 // Only allow forward transitions, e.g. from starting to started or finished, not back in
415 // the other direction.
416 if new_state > *state_locked {
417 *state_locked = new_state;
Jiyong Parka4eebde2022-07-12 18:01:12 +0900418 self.payload_state_updated.notify_all();
Andrew Walbran6b650662021-09-07 13:13:23 +0000419 Ok(())
420 } else {
421 bail!("Invalid payload state transition from {:?} to {:?}", *state_locked, new_state)
422 }
423 }
424
Andrew Walbranf8d94112021-09-07 11:45:36 +0000425 /// Kills the crosvm instance, if it is running.
Inseob Kima446f802022-07-11 19:46:37 +0900426 pub fn kill(&self) -> Result<(), Error> {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000427 let vm_state = &*self.vm_state.lock().unwrap();
428 if let VmState::Running { child } = vm_state {
Jooyung Hanbfe086f2021-10-28 10:15:45 +0900429 let id = child.id();
430 debug!("Killing crosvm({})", id);
Andrew Walbranf8d94112021-09-07 11:45:36 +0000431 // TODO: Talk to crosvm to shutdown cleanly.
432 if let Err(e) = child.kill() {
Inseob Kima446f802022-07-11 19:46:37 +0900433 bail!("Error killing crosvm({}) instance: {}", id, e);
434 } else {
435 Ok(())
Andrew Walbranf8d94112021-09-07 11:45:36 +0000436 }
Inseob Kima446f802022-07-11 19:46:37 +0900437 } else {
438 bail!("VM is not running")
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000439 }
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000440 }
Jiyong Parke558ab12022-07-07 20:18:55 +0900441
442 /// Checks if ramdump has been created. If so, send a notification to the user with the handle
443 /// to read the ramdump.
444 fn handle_ramdump(&self) -> Result<(), Error> {
445 let ramdump_path = self.temporary_directory.join("ramdump");
446 if std::fs::metadata(&ramdump_path)?.len() > 0 {
447 let ramdump = File::open(&ramdump_path)
448 .context(format!("Failed to open ramdump {:?} for reading", &ramdump_path))?;
449 self.callbacks.callback_on_ramdump(self.cid, ramdump);
Jiyong Park1612b902022-08-22 14:47:39 +0900450
451 Self::send_ramdump_to_tombstoned(&ramdump_path)?;
Jiyong Parke558ab12022-07-07 20:18:55 +0900452 }
453 Ok(())
454 }
Jiyong Park1612b902022-08-22 14:47:39 +0900455
456 fn send_ramdump_to_tombstoned(ramdump_path: &Path) -> Result<(), Error> {
457 let mut input = File::open(ramdump_path)
458 .context(format!("Failed to open raudmp {:?} for reading", ramdump_path))?;
459
460 let pid = std::process::id() as i32;
461 let conn = TombstonedConnection::connect(pid, DebuggerdDumpType::Tombstone)
462 .context("Failed to connect to tombstoned")?;
463 let mut output = conn
464 .text_output
465 .as_ref()
466 .ok_or_else(|| anyhow!("Could not get file to write the tombstones on"))?;
467
468 std::io::copy(&mut input, &mut output).context("Failed to send ramdump to tombstoned")?;
469 info!("Ramdump {:?} sent to tombstoned", ramdump_path);
470
471 conn.notify_completion()?;
472 Ok(())
473 }
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000474}
475
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900476impl Rss {
477 fn extract_max(x: &Rss, y: &Rss) -> Rss {
478 Rss { vm: max(x.vm, y.vm), crosvm: max(x.crosvm, y.crosvm) }
479 }
480}
481
482// Get guest time from /proc/[crosvm pid]/stat
483fn get_guest_time(pid: u32) -> Result<i64> {
484 let file = read_to_string(format!("/proc/{}/stat", pid))?;
485 let data_list: Vec<_> = file.split_whitespace().collect();
486
487 // Information about guest_time is at 43th place of the file split with the whitespace.
488 // Example of /proc/[pid]/stat :
489 // 6603 (kworker/104:1H-kblockd) I 2 0 0 0 -1 69238880 0 0 0 0 0 88 0 0 0 -20 1 0 1845 0 0
490 // 18446744073709551615 0 0 0 0 0 0 0 2147483647 0 0 0 0 17 104 0 0 0 0 0 0 0 0 0 0 0 0 0
491 if data_list.len() < 43 {
492 bail!("Failed to parse command result for getting guest time : {}", file);
493 }
494
495 let guest_time_ticks = data_list[42].parse::<i64>()?;
496 // SAFETY : It just returns an integer about CPU tick information.
497 let ticks_per_sec = unsafe { sysconf(_SC_CLK_TCK) } as i64;
498 Ok(guest_time_ticks * MILLIS_PER_SEC / ticks_per_sec)
499}
500
501// Get rss from /proc/[crosvm pid]/smaps
502fn get_rss(pid: u32) -> Result<Rss> {
503 let file = read_to_string(format!("/proc/{}/smaps", pid))?;
504 let lines: Vec<_> = file.split('\n').collect();
505
506 let mut rss_vm_total = 0i64;
507 let mut rss_crosvm_total = 0i64;
508 let mut is_vm = false;
509 for line in lines {
510 if line.contains("crosvm_guest") {
511 is_vm = true;
512 } else if line.contains("Rss:") {
513 let data_list: Vec<_> = line.split_whitespace().collect();
514 if data_list.len() < 2 {
515 bail!("Failed to parse command result for getting rss :\n{}", line);
516 }
517 let rss = data_list[1].parse::<i64>()?;
518
519 if is_vm {
520 rss_vm_total += rss;
521 is_vm = false;
522 }
523 rss_crosvm_total += rss;
524 }
525 }
526
527 Ok(Rss { vm: rss_vm_total, crosvm: rss_crosvm_total })
528}
529
Alan Stokes3ba10fd2022-10-06 15:46:51 +0100530fn death_reason(result: &Result<ExitStatus, io::Error>, mut failure_reason: &str) -> DeathReason {
531 if let Some(position) = failure_reason.find('|') {
532 // Separator indicates extra context information is present after the failure name.
533 error!("Failure info: {}", &failure_reason[(position + 1)..]);
534 failure_reason = &failure_reason[..position];
535 }
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000536 if let Ok(status) = result {
Andrew Walbranb27681f2022-02-23 15:11:52 +0000537 match failure_reason {
538 "PVM_FIRMWARE_PUBLIC_KEY_MISMATCH" => {
539 return DeathReason::PVM_FIRMWARE_PUBLIC_KEY_MISMATCH
540 }
541 "PVM_FIRMWARE_INSTANCE_IMAGE_CHANGED" => {
542 return DeathReason::PVM_FIRMWARE_INSTANCE_IMAGE_CHANGED
543 }
544 "BOOTLOADER_PUBLIC_KEY_MISMATCH" => return DeathReason::BOOTLOADER_PUBLIC_KEY_MISMATCH,
545 "BOOTLOADER_INSTANCE_IMAGE_CHANGED" => {
546 return DeathReason::BOOTLOADER_INSTANCE_IMAGE_CHANGED
547 }
Inseob Kim272f5722022-06-13 17:14:51 +0900548 "MICRODROID_FAILED_TO_CONNECT_TO_VIRTUALIZATION_SERVICE" => {
549 return DeathReason::MICRODROID_FAILED_TO_CONNECT_TO_VIRTUALIZATION_SERVICE
550 }
551 "MICRODROID_PAYLOAD_HAS_CHANGED" => return DeathReason::MICRODROID_PAYLOAD_HAS_CHANGED,
552 "MICRODROID_PAYLOAD_VERIFICATION_FAILED" => {
553 return DeathReason::MICRODROID_PAYLOAD_VERIFICATION_FAILED
554 }
555 "MICRODROID_INVALID_PAYLOAD_CONFIG" => {
556 return DeathReason::MICRODROID_INVALID_PAYLOAD_CONFIG
557 }
558 "MICRODROID_UNKNOWN_RUNTIME_ERROR" => {
559 return DeathReason::MICRODROID_UNKNOWN_RUNTIME_ERROR
560 }
Jiyong Parke6ed0f92022-06-22 00:13:00 +0900561 "HANGUP" => return DeathReason::HANGUP,
Andrew Walbranb27681f2022-02-23 15:11:52 +0000562 _ => {}
563 }
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000564 match status.code() {
565 None => DeathReason::KILLED,
566 Some(0) => DeathReason::SHUTDOWN,
Andrew Walbrand15c5632022-02-03 13:38:31 +0000567 Some(CROSVM_ERROR_STATUS) => DeathReason::ERROR,
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000568 Some(CROSVM_REBOOT_STATUS) => DeathReason::REBOOT,
Andrew Walbrand15c5632022-02-03 13:38:31 +0000569 Some(CROSVM_CRASH_STATUS) => DeathReason::CRASH,
Sebastian Ene23167d82022-10-07 14:09:53 +0000570 Some(CROSVM_WATCHDOG_REBOOT_STATUS) => DeathReason::WATCHDOG_REBOOT,
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000571 Some(_) => DeathReason::UNKNOWN,
572 }
573 } else {
574 DeathReason::INFRASTRUCTURE_ERROR
575 }
576}
577
Andrew Walbrand3a84182021-09-07 14:48:52 +0000578/// Starts an instance of `crosvm` to manage a new VM.
Keir Fraser13a956a2022-07-14 14:20:46 +0000579fn run_vm(
580 config: CrosvmConfig,
581 temporary_directory: &Path,
582 failure_pipe_write: File,
583) -> Result<SharedChild, Error> {
Andrew Walbrand3a84182021-09-07 14:48:52 +0000584 validate_config(&config)?;
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000585
586 let mut command = Command::new(CROSVM_PATH);
587 // TODO(qwandor): Remove --disable-sandbox.
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000588 command
589 .arg("--extended-status")
Jiyong Park6c60fea2022-10-24 16:10:01 +0900590 // Configure the logger for the crosvm process to silence logs from the disk crate which
591 // don't provide much information to us (but do spamming us).
592 .arg("--log-level")
593 .arg("info,disk=off")
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000594 .arg("run")
595 .arg("--disable-sandbox")
Keir Fraser72762722022-09-30 16:12:06 +0000596 .arg("--no-balloon")
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000597 .arg("--cid")
598 .arg(config.cid.to_string());
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000599
Andrew Walbranf8650422021-06-09 15:54:09 +0000600 if config.protected {
David Brazdil86c76fa2022-02-04 15:50:57 +0000601 command.arg("--protected-vm");
Andrew Walbran0b5789f2022-02-04 13:57:57 +0000602
603 // 3 virtio-console devices + vsock = 4.
604 let virtio_pci_device_count = 4 + config.disks.len();
605 // crosvm virtio queue has 256 entries, so 2 MiB per device (2 pages per entry) should be
606 // enough.
607 let swiotlb_size_mib = 2 * virtio_pci_device_count;
608 command.arg("--swiotlb").arg(swiotlb_size_mib.to_string());
Andrew Walbranf8650422021-06-09 15:54:09 +0000609 }
610
Andrew Walbranb15cd6e2021-07-05 16:38:07 +0000611 if let Some(memory_mib) = config.memory_mib {
612 command.arg("--mem").arg(memory_mib.to_string());
613 }
614
Jiyong Park032615f2022-01-10 13:55:34 +0900615 if let Some(cpus) = config.cpus {
616 command.arg("--cpus").arg(cpus.to_string());
617 }
618
Jiyong Parkdfe16d62022-04-20 17:32:12 +0900619 if !config.task_profiles.is_empty() {
620 command.arg("--task-profiles").arg(config.task_profiles.join(","));
621 }
622
Jiyong Parkfa91d702021-10-18 23:51:39 +0900623 // Keep track of what file descriptors should be mapped to the crosvm process.
624 let mut preserved_fds = config.indirect_files.iter().map(|file| file.as_raw_fd()).collect();
625
Jiyong Park747d6362021-10-19 17:12:52 +0900626 // Setup the serial devices.
627 // 1. uart device: used as the output device by bootloaders and as early console by linux
Andrew Walbranb27681f2022-02-23 15:11:52 +0000628 // 2. uart device: used to report the reason for the VM failing.
629 // 3. virtio-console device: used as the console device where kmsg is redirected to
Jiyong Park4afe2012022-07-08 05:38:49 +0900630 // 4. virtio-console device: used as the ramdump output
Andrew Walbranb27681f2022-02-23 15:11:52 +0000631 // 5. virtio-console device: used as the logcat output
Jiyong Park747d6362021-10-19 17:12:52 +0900632 //
Jiyong Parkb8182bb2021-10-26 22:53:08 +0900633 // When [console|log]_fd is not specified, the devices are attached to sink, which means what's
634 // written there is discarded.
Andrew Walbranb27681f2022-02-23 15:11:52 +0000635 let console_arg = format_serial_arg(&mut preserved_fds, &config.console_fd);
636 let log_arg = format_serial_arg(&mut preserved_fds, &config.log_fd);
637 let failure_serial_path = add_preserved_fd(&mut preserved_fds, &failure_pipe_write);
Jiyong Parke558ab12022-07-07 20:18:55 +0900638 let ramdump_arg = format_serial_arg(&mut preserved_fds, &config.ramdump);
Jiyong Parkfa91d702021-10-18 23:51:39 +0900639
Jiyong Park747d6362021-10-19 17:12:52 +0900640 // Warning: Adding more serial devices requires you to shift the PCI device ID of the boot
641 // disks in bootconfig.x86_64. This is because x86 crosvm puts serial devices and the block
642 // devices in the same PCI bus and serial devices comes before the block devices. Arm crosvm
643 // doesn't have the issue.
Jiyong Parkfa91d702021-10-18 23:51:39 +0900644 // /dev/ttyS0
Andrew Walbranb27681f2022-02-23 15:11:52 +0000645 command.arg(format!("--serial={},hardware=serial,num=1", &console_arg));
646 // /dev/ttyS1
647 command.arg(format!("--serial=type=file,path={},hardware=serial,num=2", &failure_serial_path));
Jiyong Parkfa91d702021-10-18 23:51:39 +0900648 // /dev/hvc0
Jiyong Parkb8182bb2021-10-26 22:53:08 +0900649 command.arg(format!("--serial={},hardware=virtio-console,num=1", &console_arg));
Jiyong Park4afe2012022-07-08 05:38:49 +0900650 // /dev/hvc1
651 command.arg(format!("--serial={},hardware=virtio-console,num=2", &ramdump_arg));
Jiyong Parkae5a4ed2021-11-01 18:36:28 +0900652 // /dev/hvc2
653 command.arg(format!("--serial={},hardware=virtio-console,num=3", &log_arg));
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000654
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000655 if let Some(bootloader) = &config.bootloader {
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000656 command.arg("--bios").arg(add_preserved_fd(&mut preserved_fds, bootloader));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000657 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000658
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000659 if let Some(initrd) = &config.initrd {
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000660 command.arg("--initrd").arg(add_preserved_fd(&mut preserved_fds, initrd));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000661 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000662
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000663 if let Some(params) = &config.params {
664 command.arg("--params").arg(params);
665 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000666
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000667 for disk in &config.disks {
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000668 command
669 .arg(if disk.writable { "--rwdisk" } else { "--disk" })
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000670 .arg(add_preserved_fd(&mut preserved_fds, &disk.image));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000671 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000672
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000673 if let Some(kernel) = &config.kernel {
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000674 command.arg(add_preserved_fd(&mut preserved_fds, kernel));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000675 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000676
Keir Fraser13a956a2022-07-14 14:20:46 +0000677 let control_server_socket =
678 UnixSeqpacketListener::bind(temporary_directory.join("crosvm.sock"))
679 .context("failed to create control server")?;
680 command.arg("--socket").arg(add_preserved_fd(&mut preserved_fds, &control_server_socket));
681
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000682 debug!("Preserving FDs {:?}", preserved_fds);
683 command.preserved_fds(preserved_fds);
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000684
Jaewan Kimb2814062022-11-14 13:21:40 +0900685 command.arg("--params").arg("crashkernel=17M");
Jiyong Park2d736562022-10-24 22:40:12 +0900686 print_crosvm_args(&command);
687
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000688 let result = SharedChild::spawn(&mut command)?;
Jooyung Hanbfe086f2021-10-28 10:15:45 +0900689 debug!("Spawned crosvm({}).", result.id());
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000690 Ok(result)
691}
692
693/// Ensure that the configuration has a valid combination of fields set, or return an error if not.
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000694fn validate_config(config: &CrosvmConfig) -> Result<(), Error> {
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000695 if config.bootloader.is_none() && config.kernel.is_none() {
696 bail!("VM must have either a bootloader or a kernel image.");
697 }
698 if config.bootloader.is_some() && (config.kernel.is_some() || config.initrd.is_some()) {
699 bail!("Can't have both bootloader and kernel/initrd image.");
700 }
Jiyong Parkdcf17412022-02-08 15:07:23 +0900701 let version = Version::parse(CROSVM_PLATFORM_VERSION).unwrap();
702 if !config.platform_version.matches(&version) {
703 bail!(
704 "Incompatible platform version. The config is compatible with platform version(s) \
705 {}, but the actual platform version is {}",
706 config.platform_version,
707 version
708 );
709 }
710
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000711 Ok(())
712}
713
Jiyong Park2d736562022-10-24 22:40:12 +0900714/// Print arguments of the crosvm command. In doing so, /proc/self/fd/XX is annotated with the
715/// actual file path if the FD is backed by a regular file. If not, the /proc path is printed
716/// unmodified.
717fn print_crosvm_args(command: &Command) {
718 let re = Regex::new(r"/proc/self/fd/[\d]+").unwrap();
719 info!(
720 "Running crosvm with args: {:?}",
721 command
722 .get_args()
723 .map(|s| s.to_string_lossy())
724 .map(|s| {
725 re.replace_all(&s, |caps: &Captures| {
726 let path = &caps[0];
727 if let Ok(realpath) = std::fs::canonicalize(path) {
728 format!("{} ({})", path, realpath.to_string_lossy())
729 } else {
730 path.to_owned()
731 }
732 })
733 .into_owned()
734 })
735 .collect::<Vec<_>>()
736 );
737}
738
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000739/// Adds the file descriptor for `file` to `preserved_fds`, and returns a string of the form
740/// "/proc/self/fd/N" where N is the file descriptor.
Keir Fraser13a956a2022-07-14 14:20:46 +0000741fn add_preserved_fd(preserved_fds: &mut Vec<RawFd>, file: &dyn AsRawFd) -> String {
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000742 let fd = file.as_raw_fd();
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000743 preserved_fds.push(fd);
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000744 format!("/proc/self/fd/{}", fd)
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000745}
Andrew Walbranb27681f2022-02-23 15:11:52 +0000746
747/// Adds the file descriptor for `file` (if any) to `preserved_fds`, and returns the appropriate
748/// string for a crosvm `--serial` flag. If `file` is none, creates a dummy sink device.
749fn format_serial_arg(preserved_fds: &mut Vec<RawFd>, file: &Option<File>) -> String {
750 if let Some(file) = file {
751 format!("type=file,path={}", add_preserved_fd(preserved_fds, file))
752 } else {
753 "type=sink".to_string()
754 }
755}
756
757/// Creates a new pipe with the `O_CLOEXEC` flag set, and returns the read side and write side.
758fn create_pipe() -> Result<(File, File), Error> {
759 let (raw_read, raw_write) = pipe2(OFlag::O_CLOEXEC)?;
760 // SAFETY: We are the sole owners of these fds as they were just created.
761 let read_fd = unsafe { File::from_raw_fd(raw_read) };
762 let write_fd = unsafe { File::from_raw_fd(raw_write) };
763 Ok((read_fd, write_fd))
764}