blob: 745d4f66eaa9137c45f646b261b800afdc4512ff [file] [log] [blame]
Andrew Walbrand6dce6f2021-03-05 16:39:08 +00001// Copyright 2021, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Functions for running instances of `crosvm`.
16
David Brazdil4b4c5102022-12-19 22:56:20 +000017use crate::aidl::{remove_temporary_files, Cid, VirtualMachineCallbacks};
David Brazdil25e8c052023-02-17 12:53:01 +000018use crate::atom::{get_num_cpus, write_vm_exited_stats};
Jaewan Kim84b91212023-02-28 00:11:57 +090019use crate::debug_config::should_prepare_console_output;
Seungjae Yoo6d265d92022-11-15 10:51:33 +090020use anyhow::{anyhow, bail, Context, Error, Result};
Andrew Walbran02b8ec02021-06-22 13:07:02 +000021use command_fds::CommandFdExt;
Jiyong Parke6ed0f92022-06-22 00:13:00 +090022use lazy_static::lazy_static;
Seungjae Yoo6d265d92022-11-15 10:51:33 +090023use libc::{sysconf, _SC_CLK_TCK};
Andrew Walbran3a5a9212021-05-04 17:09:08 +000024use log::{debug, error, info};
Jiyong Parkdcf17412022-02-08 15:07:23 +090025use semver::{Version, VersionReq};
Jiyong Park08eee7b2022-11-11 15:01:26 +090026use nix::{fcntl::OFlag, unistd::pipe2, unistd::Uid, unistd::User};
Jiyong Park2d736562022-10-24 22:40:12 +090027use regex::{Captures, Regex};
Keir Fraserf25cb922022-11-23 14:26:00 +000028use rustutils::system_properties;
Andrew Walbrandae07162021-03-12 17:05:20 +000029use shared_child::SharedChild;
Jiyong Parke6ed0f92022-06-22 00:13:00 +090030use std::borrow::Cow;
Seungjae Yoo6d265d92022-11-15 10:51:33 +090031use std::cmp::max;
Jiyong Park08eee7b2022-11-11 15:01:26 +090032use std::fmt;
David Brazdil4b4c5102022-12-19 22:56:20 +000033use std::fs::{read_to_string, File};
Andrew Walbranb27681f2022-02-23 15:11:52 +000034use std::io::{self, Read};
Andrew Walbranf8d94112021-09-07 11:45:36 +000035use std::mem;
Nikita Ioffe5776f082023-02-10 21:38:26 +000036use std::num::{NonZeroU16, NonZeroU32};
Andrew Walbranb27681f2022-02-23 15:11:52 +000037use std::os::unix::io::{AsRawFd, RawFd, FromRawFd};
Seungjae Yoo93430e82022-12-05 16:37:42 +090038use std::os::unix::process::ExitStatusExt;
Jiyong Park1612b902022-08-22 14:47:39 +090039use std::path::{Path, PathBuf};
Andrew Walbranc92d35f2022-01-12 12:45:19 +000040use std::process::{Command, ExitStatus};
Jiyong Parke6ed0f92022-06-22 00:13:00 +090041use std::sync::{Arc, Condvar, Mutex};
Seungjae Yoo2e7beea2022-08-24 16:09:12 +090042use std::time::{Duration, SystemTime};
David Brazdilf18e1162022-12-18 18:27:23 +000043use std::thread::{self, JoinHandle};
David Brazdil49f96f52022-12-16 21:29:13 +000044use android_system_virtualizationcommon::aidl::android::system::virtualizationcommon::DeathReason::DeathReason;
Jaewan Kim84b91212023-02-28 00:11:57 +090045use android_system_virtualizationservice::aidl::android::system::virtualizationservice::{
46 MemoryTrimLevel::MemoryTrimLevel,
47 VirtualMachineAppConfig::DebugLevel::DebugLevel
48};
David Brazdil528e0472022-10-10 15:06:02 +010049use android_system_virtualizationservice_internal::aidl::android::system::virtualizationservice_internal::IGlobalVmContext::IGlobalVmContext;
Alan Stokes0e82b502022-08-08 14:44:48 +010050use binder::Strong;
Inseob Kimc7d28c72021-10-25 14:28:10 +000051use android_system_virtualmachineservice::aidl::android::system::virtualmachineservice::IVirtualMachineService::IVirtualMachineService;
Jiyong Park1612b902022-08-22 14:47:39 +090052use tombstoned_client::{TombstonedConnection, DebuggerdDumpType};
David Brazdil73988ea2022-11-11 15:10:32 +000053use rpcbinder::RpcServer;
Andrew Walbrand6dce6f2021-03-05 16:39:08 +000054
Keir Fraser13a956a2022-07-14 14:20:46 +000055/// external/crosvm
Elie Kheirallah2d12ff32023-03-20 18:12:07 +000056use base::AsRawDescriptor;
Keir Fraser13a956a2022-07-14 14:20:46 +000057use base::UnixSeqpacketListener;
Keir Frasercdd4b112022-11-24 14:02:25 +000058use vm_control::{BalloonControlCommand, VmRequest, VmResponse};
Keir Fraser13a956a2022-07-14 14:20:46 +000059
Andrew Walbrand6dce6f2021-03-05 16:39:08 +000060const CROSVM_PATH: &str = "/apex/com.android.virt/bin/crosvm";
61
Jiyong Parkdcf17412022-02-08 15:07:23 +090062/// Version of the platform that crosvm currently implements. The format follows SemVer. This
63/// should be updated when there is a platform change in the crosvm side. Having this value here is
64/// fine because virtualizationservice and crosvm are supposed to be updated together in the virt
65/// APEX.
66const CROSVM_PLATFORM_VERSION: &str = "1.0.0";
67
Andrew Walbrand15c5632022-02-03 13:38:31 +000068/// The exit status which crosvm returns when it has an error starting a VM.
Alan Stokes7c459e82022-12-06 12:21:49 +000069const CROSVM_START_ERROR_STATUS: i32 = 1;
Andrew Walbranc92d35f2022-01-12 12:45:19 +000070/// The exit status which crosvm returns when a VM requests a reboot.
71const CROSVM_REBOOT_STATUS: i32 = 32;
Andrew Walbrand15c5632022-02-03 13:38:31 +000072/// The exit status which crosvm returns when it crashes due to an error.
73const CROSVM_CRASH_STATUS: i32 = 33;
Sebastian Ene23167d82022-10-07 14:09:53 +000074/// The exit status which crosvm returns when vcpu is stalled.
75const CROSVM_WATCHDOG_REBOOT_STATUS: i32 = 36;
Jiyong Park3115cdb2023-02-24 18:04:44 +090076/// The size of memory (in MiB) reserved for ramdump
77const RAMDUMP_RESERVED_MIB: u32 = 17;
Andrew Walbranc92d35f2022-01-12 12:45:19 +000078
Seungjae Yoo6d265d92022-11-15 10:51:33 +090079const MILLIS_PER_SEC: i64 = 1000;
80
Jaewan Kim46422812022-11-25 10:59:39 +090081const SYSPROP_CUSTOM_PVMFW_PATH: &str = "hypervisor.pvmfw.path";
82
Jiyong Parke6ed0f92022-06-22 00:13:00 +090083lazy_static! {
84 /// If the VM doesn't move to the Started state within this amount time, a hang-up error is
85 /// triggered.
Alan Stokesc3f2ac22022-06-23 12:19:46 +010086 static ref BOOT_HANGUP_TIMEOUT: Duration = if nested_virt::is_nested_virtualization().unwrap() {
Jiyong Parke6ed0f92022-06-22 00:13:00 +090087 // Nested virtualization is slow, so we need a longer timeout.
Inseob Kime557cec2023-01-26 22:49:58 +090088 Duration::from_secs(300)
Jiyong Parke6ed0f92022-06-22 00:13:00 +090089 } else {
Inseob Kime557cec2023-01-26 22:49:58 +090090 Duration::from_secs(30)
Jiyong Parke6ed0f92022-06-22 00:13:00 +090091 };
92}
93
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000094/// Configuration for a VM to run with crosvm.
95#[derive(Debug)]
Andrew Walbrand3a84182021-09-07 14:48:52 +000096pub struct CrosvmConfig {
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000097 pub cid: Cid,
Seungjae Yoo62085c02022-08-12 04:44:52 +000098 pub name: String,
Andrew Walbrand3a84182021-09-07 14:48:52 +000099 pub bootloader: Option<File>,
100 pub kernel: Option<File>,
101 pub initrd: Option<File>,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000102 pub disks: Vec<DiskFile>,
103 pub params: Option<String>,
Andrew Walbranf8650422021-06-09 15:54:09 +0000104 pub protected: bool,
Jaewan Kim84b91212023-02-28 00:11:57 +0900105 pub debug_level: DebugLevel,
Andrew Walbranb15cd6e2021-07-05 16:38:07 +0000106 pub memory_mib: Option<NonZeroU32>,
Jiyong Park032615f2022-01-10 13:55:34 +0900107 pub cpus: Option<NonZeroU32>,
David Brazdil7d1e5ec2023-02-06 17:56:29 +0000108 pub host_cpu_topology: bool,
Jiyong Parkdfe16d62022-04-20 17:32:12 +0900109 pub task_profiles: Vec<String>,
Jiyong Parkb8182bb2021-10-26 22:53:08 +0900110 pub console_fd: Option<File>,
Andrew Walbrand3a84182021-09-07 14:48:52 +0000111 pub log_fd: Option<File>,
Jiyong Parke558ab12022-07-07 20:18:55 +0900112 pub ramdump: Option<File>,
Andrew Walbrand3a84182021-09-07 14:48:52 +0000113 pub indirect_files: Vec<File>,
Jiyong Parkdcf17412022-02-08 15:07:23 +0900114 pub platform_version: VersionReq,
Jiyong Parke6ed0f92022-06-22 00:13:00 +0900115 pub detect_hangup: bool,
Nikita Ioffe5776f082023-02-10 21:38:26 +0000116 pub gdb_port: Option<NonZeroU16>,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000117}
118
119/// A disk image to pass to crosvm for a VM.
120#[derive(Debug)]
121pub struct DiskFile {
122 pub image: File,
123 pub writable: bool,
124}
125
Andrew Walbran6b650662021-09-07 13:13:23 +0000126/// The lifecycle state which the payload in the VM has reported itself to be in.
127///
128/// Note that the order of enum variants is significant; only forward transitions are allowed by
129/// [`VmInstance::update_payload_state`].
130#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
131pub enum PayloadState {
132 Starting,
133 Started,
134 Ready,
135 Finished,
Jiyong Parka4eebde2022-07-12 18:01:12 +0900136 Hangup, // Hasn't reached to Ready before timeout expires
Andrew Walbran6b650662021-09-07 13:13:23 +0000137}
138
Andrew Walbranf8d94112021-09-07 11:45:36 +0000139/// The current state of the VM itself.
140#[derive(Debug)]
141pub enum VmState {
142 /// The VM has not yet tried to start.
143 NotStarted {
144 ///The configuration needed to start the VM, if it has not yet been started.
145 config: CrosvmConfig,
146 },
147 /// The VM has been started.
148 Running {
149 /// The crosvm child process.
150 child: Arc<SharedChild>,
David Brazdilf18e1162022-12-18 18:27:23 +0000151 /// The thread waiting for crosvm to finish.
152 monitor_vm_exit_thread: Option<JoinHandle<()>>,
Andrew Walbranf8d94112021-09-07 11:45:36 +0000153 },
154 /// The VM died or was killed.
155 Dead,
156 /// The VM failed to start.
157 Failed,
158}
159
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900160/// RSS values of VM and CrosVM process itself.
161#[derive(Copy, Clone, Debug, Default)]
162pub struct Rss {
163 pub vm: i64,
164 pub crosvm: i64,
165}
166
167/// Metrics regarding the VM.
168#[derive(Debug, Default)]
169pub struct VmMetric {
170 /// Recorded timestamp when the VM is started.
171 pub start_timestamp: Option<SystemTime>,
172 /// Update most recent guest_time periodically from /proc/[crosvm pid]/stat while VM is running.
173 pub cpu_guest_time: Option<i64>,
174 /// Update maximum RSS values periodically from /proc/[crosvm pid]/smaps while VM is running.
175 pub rss: Option<Rss>,
176}
177
Andrew Walbranf8d94112021-09-07 11:45:36 +0000178impl VmState {
179 /// Tries to start the VM, if it is in the `NotStarted` state.
180 ///
181 /// Returns an error if the VM is in the wrong state, or fails to start.
182 fn start(&mut self, instance: Arc<VmInstance>) -> Result<(), Error> {
183 let state = mem::replace(self, VmState::Failed);
184 if let VmState::NotStarted { config } = state {
Jiyong Parke6ed0f92022-06-22 00:13:00 +0900185 let detect_hangup = config.detect_hangup;
Andrew Walbranb27681f2022-02-23 15:11:52 +0000186 let (failure_pipe_read, failure_pipe_write) = create_pipe()?;
187
Andrew Walbranf8d94112021-09-07 11:45:36 +0000188 // If this fails and returns an error, `self` will be left in the `Failed` state.
Keir Fraser13a956a2022-07-14 14:20:46 +0000189 let child =
Keir Frasercdd4b112022-11-24 14:02:25 +0000190 Arc::new(run_vm(config, &instance.crosvm_control_socket_path, failure_pipe_write)?);
Andrew Walbranf8d94112021-09-07 11:45:36 +0000191
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900192 let instance_monitor_status = instance.clone();
193 let child_monitor_status = child.clone();
194 thread::spawn(move || {
195 instance_monitor_status.clone().monitor_vm_status(child_monitor_status);
196 });
197
Andrew Walbranf8d94112021-09-07 11:45:36 +0000198 let child_clone = child.clone();
Jiyong Parka4eebde2022-07-12 18:01:12 +0900199 let instance_clone = instance.clone();
David Brazdilf18e1162022-12-18 18:27:23 +0000200 let monitor_vm_exit_thread = Some(thread::spawn(move || {
Jiyong Parka4eebde2022-07-12 18:01:12 +0900201 instance_clone.monitor_vm_exit(child_clone, failure_pipe_read);
David Brazdilf18e1162022-12-18 18:27:23 +0000202 }));
Andrew Walbranf8d94112021-09-07 11:45:36 +0000203
Jiyong Parka4eebde2022-07-12 18:01:12 +0900204 if detect_hangup {
205 let child_clone = child.clone();
206 thread::spawn(move || {
207 instance.monitor_payload_hangup(child_clone);
208 });
209 }
210
Andrew Walbranf8d94112021-09-07 11:45:36 +0000211 // If it started correctly, update the state.
David Brazdilf18e1162022-12-18 18:27:23 +0000212 *self = VmState::Running { child, monitor_vm_exit_thread };
Andrew Walbranf8d94112021-09-07 11:45:36 +0000213 Ok(())
214 } else {
215 *self = state;
216 bail!("VM already started or failed")
217 }
218 }
219}
220
David Brazdil8cf8f482022-11-23 14:21:26 +0000221/// Internal struct that holds the handles to globally unique resources of a VM.
222#[derive(Debug)]
223pub struct VmContext {
224 #[allow(dead_code)] // Keeps the global context alive
225 global_context: Strong<dyn IGlobalVmContext>,
226 #[allow(dead_code)] // Keeps the server alive
227 vm_server: RpcServer,
228}
229
230impl VmContext {
231 /// Construct new VmContext.
232 pub fn new(global_context: Strong<dyn IGlobalVmContext>, vm_server: RpcServer) -> VmContext {
233 VmContext { global_context, vm_server }
234 }
235}
236
Andrew Walbranf8d94112021-09-07 11:45:36 +0000237/// Information about a particular instance of a VM which may be running.
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000238#[derive(Debug)]
239pub struct VmInstance {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000240 /// The current state of the VM.
241 pub vm_state: Mutex<VmState>,
David Brazdil8cf8f482022-11-23 14:21:26 +0000242 /// Global resources allocated for this VM.
243 #[allow(dead_code)] // Keeps the context alive
244 vm_context: VmContext,
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000245 /// The CID assigned to the VM for vsock communication.
246 pub cid: Cid,
Keir Frasercdd4b112022-11-24 14:02:25 +0000247 /// Path to crosvm control socket
248 crosvm_control_socket_path: PathBuf,
Seungjae Yoo62085c02022-08-12 04:44:52 +0000249 /// The name of the VM.
250 pub name: String,
Andrew Walbranf8650422021-06-09 15:54:09 +0000251 /// Whether the VM is a protected VM.
252 pub protected: bool,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000253 /// Directory of temporary files used by the VM while it is running.
254 pub temporary_directory: PathBuf,
Andrew Walbranf6a1eb92021-04-01 11:16:02 +0000255 /// The UID of the process which requested the VM.
256 pub requester_uid: u32,
Andrew Walbranf6a1eb92021-04-01 11:16:02 +0000257 /// The PID of the process which requested the VM. Note that this process may no longer exist
258 /// and the PID may have been reused for a different process, so this should not be trusted.
Andrew Walbran02034492021-04-13 15:05:07 +0000259 pub requester_debug_pid: i32,
Andrew Walbrandae07162021-03-12 17:05:20 +0000260 /// Callbacks to clients of the VM.
261 pub callbacks: VirtualMachineCallbacks,
Inseob Kimc7d28c72021-10-25 14:28:10 +0000262 /// VirtualMachineService binder object for the VM.
263 pub vm_service: Mutex<Option<Strong<dyn IVirtualMachineService>>>,
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900264 /// Recorded metrics of VM such as timestamp or cpu / memory usage.
265 pub vm_metric: Mutex<VmMetric>,
Andrew Walbran6b650662021-09-07 13:13:23 +0000266 /// The latest lifecycle state which the payload reported itself to be in.
267 payload_state: Mutex<PayloadState>,
Jiyong Parka4eebde2022-07-12 18:01:12 +0900268 /// Represents the condition that payload_state was updated
269 payload_state_updated: Condvar,
Jiyong Park08eee7b2022-11-11 15:01:26 +0900270 /// The human readable name of requester_uid
271 requester_uid_name: String,
272}
273
274impl fmt::Display for VmInstance {
275 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
276 let adj = if self.protected { "Protected" } else { "Non-protected" };
277 write!(
278 f,
279 "{} virtual machine \"{}\" (owner: {}, cid: {})",
280 adj, self.name, self.requester_uid_name, self.cid
281 )
282 }
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000283}
284
285impl VmInstance {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000286 /// Validates the given config and creates a new `VmInstance` but doesn't start running it.
287 pub fn new(
Andrew Walbrand3a84182021-09-07 14:48:52 +0000288 config: CrosvmConfig,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000289 temporary_directory: PathBuf,
Andrew Walbranf6a1eb92021-04-01 11:16:02 +0000290 requester_uid: u32,
Andrew Walbran02034492021-04-13 15:05:07 +0000291 requester_debug_pid: i32,
David Brazdil8cf8f482022-11-23 14:21:26 +0000292 vm_context: VmContext,
Andrew Walbranf8d94112021-09-07 11:45:36 +0000293 ) -> Result<VmInstance, Error> {
294 validate_config(&config)?;
Andrew Walbrand3a84182021-09-07 14:48:52 +0000295 let cid = config.cid;
Seungjae Yoo62085c02022-08-12 04:44:52 +0000296 let name = config.name.clone();
Andrew Walbrand3a84182021-09-07 14:48:52 +0000297 let protected = config.protected;
Jiyong Park08eee7b2022-11-11 15:01:26 +0900298 let requester_uid_name = User::from_uid(Uid::from_raw(requester_uid))
299 .ok()
300 .flatten()
301 .map_or_else(|| format!("{}", requester_uid), |u| u.name);
302 let instance = VmInstance {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000303 vm_state: Mutex::new(VmState::NotStarted { config }),
David Brazdil528e0472022-10-10 15:06:02 +0100304 vm_context,
Andrew Walbrand3a84182021-09-07 14:48:52 +0000305 cid,
Keir Frasercdd4b112022-11-24 14:02:25 +0000306 crosvm_control_socket_path: temporary_directory.join("crosvm.sock"),
Seungjae Yoo62085c02022-08-12 04:44:52 +0000307 name,
Andrew Walbrand3a84182021-09-07 14:48:52 +0000308 protected,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000309 temporary_directory,
Andrew Walbran02034492021-04-13 15:05:07 +0000310 requester_uid,
Andrew Walbran02034492021-04-13 15:05:07 +0000311 requester_debug_pid,
Andrew Walbranf8d94112021-09-07 11:45:36 +0000312 callbacks: Default::default(),
Inseob Kimc7d28c72021-10-25 14:28:10 +0000313 vm_service: Mutex::new(None),
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900314 vm_metric: Mutex::new(Default::default()),
Andrew Walbranf8d94112021-09-07 11:45:36 +0000315 payload_state: Mutex::new(PayloadState::Starting),
Jiyong Parka4eebde2022-07-12 18:01:12 +0900316 payload_state_updated: Condvar::new(),
Jiyong Park08eee7b2022-11-11 15:01:26 +0900317 requester_uid_name,
318 };
319 info!("{} created", &instance);
320 Ok(instance)
Andrew Walbrandae07162021-03-12 17:05:20 +0000321 }
322
Andrew Walbranf8d94112021-09-07 11:45:36 +0000323 /// Starts an instance of `crosvm` to manage the VM. The `crosvm` instance will be killed when
324 /// the `VmInstance` is dropped.
325 pub fn start(self: &Arc<Self>) -> Result<(), Error> {
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900326 let mut vm_metric = self.vm_metric.lock().unwrap();
327 vm_metric.start_timestamp = Some(SystemTime::now());
Jiyong Park08eee7b2022-11-11 15:01:26 +0900328 let ret = self.vm_state.lock().unwrap().start(self.clone());
329 if ret.is_ok() {
330 info!("{} started", &self);
331 }
332 ret.with_context(|| format!("{} failed to start", &self))
Andrew Walbranf8d94112021-09-07 11:45:36 +0000333 }
334
Jiyong Parka4eebde2022-07-12 18:01:12 +0900335 /// Monitors the exit of the VM (i.e. termination of the `child` process). When that happens,
336 /// handles the event by updating the state, noityfing the event to clients by calling
337 /// callbacks, and removing temporary files for the VM.
338 fn monitor_vm_exit(&self, child: Arc<SharedChild>, mut failure_pipe_read: File) {
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000339 let result = child.wait();
340 match &result {
Jooyung Hanbfe086f2021-10-28 10:15:45 +0900341 Err(e) => error!("Error waiting for crosvm({}) instance to die: {}", child.id(), e),
Sebastian Ene23167d82022-10-07 14:09:53 +0000342 Ok(status) => {
343 info!("crosvm({}) exited with status {}", child.id(), status);
344 if let Some(exit_status_code) = status.code() {
345 if exit_status_code == CROSVM_WATCHDOG_REBOOT_STATUS {
346 info!("detected vcpu stall on crosvm");
347 }
348 }
349 }
Andrew Walbrandae07162021-03-12 17:05:20 +0000350 }
Andrew Walbranf8d94112021-09-07 11:45:36 +0000351
352 let mut vm_state = self.vm_state.lock().unwrap();
353 *vm_state = VmState::Dead;
354 // Ensure that the mutex is released before calling the callbacks.
355 drop(vm_state);
Jiyong Park08eee7b2022-11-11 15:01:26 +0900356 info!("{} exited", &self);
Andrew Walbranf8d94112021-09-07 11:45:36 +0000357
Jiyong Parka4eebde2022-07-12 18:01:12 +0900358 // Read the pipe to see if any failure reason is written
359 let mut failure_reason = String::new();
360 match failure_pipe_read.read_to_string(&mut failure_reason) {
361 Err(e) => error!("Error reading VM failure reason from pipe: {}", e),
362 Ok(len) if len > 0 => info!("VM returned failure reason '{}'", &failure_reason),
363 _ => (),
Jiyong Parke6ed0f92022-06-22 00:13:00 +0900364 };
Andrew Walbranb27681f2022-02-23 15:11:52 +0000365
Jiyong Parka4eebde2022-07-12 18:01:12 +0900366 // In case of hangup, the pipe doesn't give us any information because the hangup can't be
367 // detected on the VM side (otherwise, it isn't a hangup), but in the
368 // monitor_payload_hangup function below which updates the payload state to Hangup.
369 let failure_reason =
370 if failure_reason.is_empty() && self.payload_state() == PayloadState::Hangup {
371 Cow::from("HANGUP")
372 } else {
373 Cow::from(failure_reason)
374 };
375
Jiyong Parke558ab12022-07-07 20:18:55 +0900376 self.handle_ramdump().unwrap_or_else(|e| error!("Error handling ramdump: {}", e));
Seungjae Yoob4c07ba2022-08-12 04:44:52 +0000377
378 let death_reason = death_reason(&result, &failure_reason);
Seungjae Yoo93430e82022-12-05 16:37:42 +0900379 let exit_signal = exit_signal(&result);
380
Seungjae Yoob4c07ba2022-08-12 04:44:52 +0000381 self.callbacks.callback_on_died(self.cid, death_reason);
Seungjae Yoo2e7beea2022-08-24 16:09:12 +0900382
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900383 let vm_metric = self.vm_metric.lock().unwrap();
Seungjae Yoo93430e82022-12-05 16:37:42 +0900384 write_vm_exited_stats(
385 self.requester_uid as i32,
386 &self.name,
387 death_reason,
388 exit_signal,
Chris Wailes75269622022-12-05 23:01:44 -0800389 &vm_metric,
Seungjae Yoo93430e82022-12-05 16:37:42 +0900390 );
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000391
David Brazdil4b4c5102022-12-19 22:56:20 +0000392 // Delete temporary files. The folder itself is removed by VirtualizationServiceInternal.
393 remove_temporary_files(&self.temporary_directory).unwrap_or_else(|e| {
394 error!("Error removing temporary files from {:?}: {}", self.temporary_directory, e);
395 });
Andrew Walbrandae07162021-03-12 17:05:20 +0000396 }
397
Jiyong Parka4eebde2022-07-12 18:01:12 +0900398 /// Waits until payload is started, or timeout expires. When timeout occurs, kill
399 /// the VM to prevent indefinite hangup and update the payload_state accordingly.
400 fn monitor_payload_hangup(&self, child: Arc<SharedChild>) {
401 debug!("Starting to monitor hangup for Microdroid({})", child.id());
402 let (_, result) = self
403 .payload_state_updated
404 .wait_timeout_while(self.payload_state.lock().unwrap(), *BOOT_HANGUP_TIMEOUT, |s| {
405 *s < PayloadState::Started
406 })
407 .unwrap();
408 let child_still_running = child.try_wait().ok() == Some(None);
409 if result.timed_out() && child_still_running {
410 error!(
411 "Microdroid({}) failed to start payload within {} secs timeout. Shutting down.",
412 child.id(),
413 BOOT_HANGUP_TIMEOUT.as_secs()
414 );
415 self.update_payload_state(PayloadState::Hangup).unwrap();
416 if let Err(e) = self.kill() {
417 error!("Error stopping timed-out VM with CID {}: {:?}", child.id(), e);
418 }
419 }
420 }
421
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900422 fn monitor_vm_status(&self, child: Arc<SharedChild>) {
423 let pid = child.id();
424
425 loop {
426 {
427 // Check VM state
428 let vm_state = &*self.vm_state.lock().unwrap();
429 if let VmState::Dead = vm_state {
430 break;
431 }
432
433 let mut vm_metric = self.vm_metric.lock().unwrap();
434
435 // Get CPU Information
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900436 if let Ok(guest_time) = get_guest_time(pid) {
437 vm_metric.cpu_guest_time = Some(guest_time);
438 } else {
439 error!("Failed to parse /proc/[pid]/stat");
440 }
441
442 // Get Memory Information
443 if let Ok(rss) = get_rss(pid) {
444 vm_metric.rss = match &vm_metric.rss {
445 Some(x) => Some(Rss::extract_max(x, &rss)),
446 None => Some(rss),
447 }
448 } else {
449 error!("Failed to parse /proc/[pid]/smaps");
450 }
451 }
452
453 thread::sleep(Duration::from_secs(1));
454 }
455 }
456
Andrew Walbran6b650662021-09-07 13:13:23 +0000457 /// Returns the last reported state of the VM payload.
458 pub fn payload_state(&self) -> PayloadState {
459 *self.payload_state.lock().unwrap()
460 }
461
462 /// Updates the payload state to the given value, if it is a valid state transition.
463 pub fn update_payload_state(&self, new_state: PayloadState) -> Result<(), Error> {
464 let mut state_locked = self.payload_state.lock().unwrap();
465 // Only allow forward transitions, e.g. from starting to started or finished, not back in
466 // the other direction.
467 if new_state > *state_locked {
468 *state_locked = new_state;
Jiyong Parka4eebde2022-07-12 18:01:12 +0900469 self.payload_state_updated.notify_all();
Andrew Walbran6b650662021-09-07 13:13:23 +0000470 Ok(())
471 } else {
472 bail!("Invalid payload state transition from {:?} to {:?}", *state_locked, new_state)
473 }
474 }
475
Andrew Walbranf8d94112021-09-07 11:45:36 +0000476 /// Kills the crosvm instance, if it is running.
Inseob Kima446f802022-07-11 19:46:37 +0900477 pub fn kill(&self) -> Result<(), Error> {
David Brazdilf18e1162022-12-18 18:27:23 +0000478 let monitor_vm_exit_thread = {
479 let vm_state = &mut *self.vm_state.lock().unwrap();
480 if let VmState::Running { child, monitor_vm_exit_thread } = vm_state {
481 let id = child.id();
482 debug!("Killing crosvm({})", id);
483 // TODO: Talk to crosvm to shutdown cleanly.
484 child.kill().with_context(|| format!("Error killing crosvm({id}) instance"))?;
485 monitor_vm_exit_thread.take()
Inseob Kima446f802022-07-11 19:46:37 +0900486 } else {
David Brazdilf18e1162022-12-18 18:27:23 +0000487 bail!("VM is not running")
Andrew Walbranf8d94112021-09-07 11:45:36 +0000488 }
David Brazdilf18e1162022-12-18 18:27:23 +0000489 };
490
491 // Wait for monitor_vm_exit() to finish. Must release vm_state lock
492 // first, as monitor_vm_exit() takes it as well.
493 monitor_vm_exit_thread.map(JoinHandle::join);
494
495 Ok(())
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000496 }
Jiyong Parke558ab12022-07-07 20:18:55 +0900497
Keir Frasercdd4b112022-11-24 14:02:25 +0000498 /// Responds to memory-trimming notifications by inflating the virtio
499 /// balloon to reclaim guest memory.
500 pub fn trim_memory(&self, level: MemoryTrimLevel) -> Result<(), Error> {
501 let request = VmRequest::BalloonCommand(BalloonControlCommand::Stats {});
502 match vm_control::client::handle_request(&request, &self.crosvm_control_socket_path) {
503 Ok(VmResponse::BalloonStats { stats, balloon_actual: _ }) => {
504 if let Some(total_memory) = stats.total_memory {
505 // Reclaim up to 50% of total memory assuming worst case
506 // most memory is anonymous and must be swapped to zram
507 // with an approximate 2:1 compression ratio.
508 let pct = match level {
509 MemoryTrimLevel::TRIM_MEMORY_RUNNING_CRITICAL => 50,
510 MemoryTrimLevel::TRIM_MEMORY_RUNNING_LOW => 30,
511 MemoryTrimLevel::TRIM_MEMORY_RUNNING_MODERATE => 10,
512 _ => bail!("Invalid memory trim level {:?}", level),
513 };
514 let command =
515 BalloonControlCommand::Adjust { num_bytes: total_memory * pct / 100 };
516 if let Err(e) = vm_control::client::handle_request(
517 &VmRequest::BalloonCommand(command),
518 &self.crosvm_control_socket_path,
519 ) {
520 bail!("Error sending balloon adjustment: {:?}", e);
521 }
522 }
523 }
524 Ok(VmResponse::Err(e)) => {
525 // ENOTSUP is returned when the balloon protocol is not initialised. This
526 // can occur for numerous reasons: Guest is still booting, guest doesn't
527 // support ballooning, host doesn't support ballooning. We don't log or
528 // raise an error in this case: trim is just a hint and we can ignore it.
529 if e.errno() != libc::ENOTSUP {
530 bail!("Errno return when requesting balloon stats: {}", e.errno())
531 }
532 }
533 e => bail!("Error requesting balloon stats: {:?}", e),
534 }
535 Ok(())
536 }
537
Alan Stokes3e98d292022-12-14 15:18:22 +0000538 /// Checks if ramdump has been created. If so, send it to tombstoned.
Jiyong Parke558ab12022-07-07 20:18:55 +0900539 fn handle_ramdump(&self) -> Result<(), Error> {
540 let ramdump_path = self.temporary_directory.join("ramdump");
Jiyong Park933d2b22023-02-28 11:41:19 +0900541 if !ramdump_path.as_path().try_exists()? {
542 return Ok(());
543 }
Jiyong Parke558ab12022-07-07 20:18:55 +0900544 if std::fs::metadata(&ramdump_path)?.len() > 0 {
Jiyong Park1612b902022-08-22 14:47:39 +0900545 Self::send_ramdump_to_tombstoned(&ramdump_path)?;
Jiyong Parke558ab12022-07-07 20:18:55 +0900546 }
547 Ok(())
548 }
Jiyong Park1612b902022-08-22 14:47:39 +0900549
550 fn send_ramdump_to_tombstoned(ramdump_path: &Path) -> Result<(), Error> {
551 let mut input = File::open(ramdump_path)
Alan Stokes3e98d292022-12-14 15:18:22 +0000552 .context(format!("Failed to open ramdump {:?} for reading", ramdump_path))?;
Jiyong Park1612b902022-08-22 14:47:39 +0900553
554 let pid = std::process::id() as i32;
555 let conn = TombstonedConnection::connect(pid, DebuggerdDumpType::Tombstone)
556 .context("Failed to connect to tombstoned")?;
557 let mut output = conn
558 .text_output
559 .as_ref()
560 .ok_or_else(|| anyhow!("Could not get file to write the tombstones on"))?;
561
562 std::io::copy(&mut input, &mut output).context("Failed to send ramdump to tombstoned")?;
563 info!("Ramdump {:?} sent to tombstoned", ramdump_path);
564
565 conn.notify_completion()?;
566 Ok(())
567 }
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000568}
569
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900570impl Rss {
571 fn extract_max(x: &Rss, y: &Rss) -> Rss {
572 Rss { vm: max(x.vm, y.vm), crosvm: max(x.crosvm, y.crosvm) }
573 }
574}
575
576// Get guest time from /proc/[crosvm pid]/stat
577fn get_guest_time(pid: u32) -> Result<i64> {
578 let file = read_to_string(format!("/proc/{}/stat", pid))?;
579 let data_list: Vec<_> = file.split_whitespace().collect();
580
581 // Information about guest_time is at 43th place of the file split with the whitespace.
582 // Example of /proc/[pid]/stat :
583 // 6603 (kworker/104:1H-kblockd) I 2 0 0 0 -1 69238880 0 0 0 0 0 88 0 0 0 -20 1 0 1845 0 0
584 // 18446744073709551615 0 0 0 0 0 0 0 2147483647 0 0 0 0 17 104 0 0 0 0 0 0 0 0 0 0 0 0 0
585 if data_list.len() < 43 {
586 bail!("Failed to parse command result for getting guest time : {}", file);
587 }
588
589 let guest_time_ticks = data_list[42].parse::<i64>()?;
590 // SAFETY : It just returns an integer about CPU tick information.
Charisee96113f32023-01-26 09:00:42 +0000591 let ticks_per_sec = unsafe { sysconf(_SC_CLK_TCK) };
Seungjae Yoo6d265d92022-11-15 10:51:33 +0900592 Ok(guest_time_ticks * MILLIS_PER_SEC / ticks_per_sec)
593}
594
595// Get rss from /proc/[crosvm pid]/smaps
596fn get_rss(pid: u32) -> Result<Rss> {
597 let file = read_to_string(format!("/proc/{}/smaps", pid))?;
598 let lines: Vec<_> = file.split('\n').collect();
599
600 let mut rss_vm_total = 0i64;
601 let mut rss_crosvm_total = 0i64;
602 let mut is_vm = false;
603 for line in lines {
604 if line.contains("crosvm_guest") {
605 is_vm = true;
606 } else if line.contains("Rss:") {
607 let data_list: Vec<_> = line.split_whitespace().collect();
608 if data_list.len() < 2 {
609 bail!("Failed to parse command result for getting rss :\n{}", line);
610 }
611 let rss = data_list[1].parse::<i64>()?;
612
613 if is_vm {
614 rss_vm_total += rss;
615 is_vm = false;
616 }
617 rss_crosvm_total += rss;
618 }
619 }
620
621 Ok(Rss { vm: rss_vm_total, crosvm: rss_crosvm_total })
622}
623
Alan Stokes3ba10fd2022-10-06 15:46:51 +0100624fn death_reason(result: &Result<ExitStatus, io::Error>, mut failure_reason: &str) -> DeathReason {
625 if let Some(position) = failure_reason.find('|') {
626 // Separator indicates extra context information is present after the failure name.
627 error!("Failure info: {}", &failure_reason[(position + 1)..]);
628 failure_reason = &failure_reason[..position];
629 }
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000630 if let Ok(status) = result {
Andrew Walbranb27681f2022-02-23 15:11:52 +0000631 match failure_reason {
632 "PVM_FIRMWARE_PUBLIC_KEY_MISMATCH" => {
633 return DeathReason::PVM_FIRMWARE_PUBLIC_KEY_MISMATCH
634 }
635 "PVM_FIRMWARE_INSTANCE_IMAGE_CHANGED" => {
636 return DeathReason::PVM_FIRMWARE_INSTANCE_IMAGE_CHANGED
637 }
638 "BOOTLOADER_PUBLIC_KEY_MISMATCH" => return DeathReason::BOOTLOADER_PUBLIC_KEY_MISMATCH,
639 "BOOTLOADER_INSTANCE_IMAGE_CHANGED" => {
640 return DeathReason::BOOTLOADER_INSTANCE_IMAGE_CHANGED
641 }
Inseob Kim272f5722022-06-13 17:14:51 +0900642 "MICRODROID_FAILED_TO_CONNECT_TO_VIRTUALIZATION_SERVICE" => {
643 return DeathReason::MICRODROID_FAILED_TO_CONNECT_TO_VIRTUALIZATION_SERVICE
644 }
645 "MICRODROID_PAYLOAD_HAS_CHANGED" => return DeathReason::MICRODROID_PAYLOAD_HAS_CHANGED,
646 "MICRODROID_PAYLOAD_VERIFICATION_FAILED" => {
647 return DeathReason::MICRODROID_PAYLOAD_VERIFICATION_FAILED
648 }
649 "MICRODROID_INVALID_PAYLOAD_CONFIG" => {
650 return DeathReason::MICRODROID_INVALID_PAYLOAD_CONFIG
651 }
652 "MICRODROID_UNKNOWN_RUNTIME_ERROR" => {
653 return DeathReason::MICRODROID_UNKNOWN_RUNTIME_ERROR
654 }
Jiyong Parke6ed0f92022-06-22 00:13:00 +0900655 "HANGUP" => return DeathReason::HANGUP,
Andrew Walbranb27681f2022-02-23 15:11:52 +0000656 _ => {}
657 }
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000658 match status.code() {
659 None => DeathReason::KILLED,
660 Some(0) => DeathReason::SHUTDOWN,
Alan Stokes7c459e82022-12-06 12:21:49 +0000661 Some(CROSVM_START_ERROR_STATUS) => DeathReason::START_FAILED,
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000662 Some(CROSVM_REBOOT_STATUS) => DeathReason::REBOOT,
Andrew Walbrand15c5632022-02-03 13:38:31 +0000663 Some(CROSVM_CRASH_STATUS) => DeathReason::CRASH,
Sebastian Ene23167d82022-10-07 14:09:53 +0000664 Some(CROSVM_WATCHDOG_REBOOT_STATUS) => DeathReason::WATCHDOG_REBOOT,
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000665 Some(_) => DeathReason::UNKNOWN,
666 }
667 } else {
668 DeathReason::INFRASTRUCTURE_ERROR
669 }
670}
671
Seungjae Yoo93430e82022-12-05 16:37:42 +0900672fn exit_signal(result: &Result<ExitStatus, io::Error>) -> Option<i32> {
673 match result {
674 Ok(status) => status.signal(),
675 Err(_) => None,
676 }
677}
678
Andrew Walbrand3a84182021-09-07 14:48:52 +0000679/// Starts an instance of `crosvm` to manage a new VM.
Keir Fraser13a956a2022-07-14 14:20:46 +0000680fn run_vm(
681 config: CrosvmConfig,
Keir Frasercdd4b112022-11-24 14:02:25 +0000682 crosvm_control_socket_path: &Path,
Keir Fraser13a956a2022-07-14 14:20:46 +0000683 failure_pipe_write: File,
684) -> Result<SharedChild, Error> {
Andrew Walbrand3a84182021-09-07 14:48:52 +0000685 validate_config(&config)?;
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000686
687 let mut command = Command::new(CROSVM_PATH);
688 // TODO(qwandor): Remove --disable-sandbox.
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000689 command
690 .arg("--extended-status")
Jiyong Park6c60fea2022-10-24 16:10:01 +0900691 // Configure the logger for the crosvm process to silence logs from the disk crate which
692 // don't provide much information to us (but do spamming us).
693 .arg("--log-level")
Frederick Mayle138f53c2023-01-24 18:55:26 -0800694 .arg("info,disk=warn")
Andrew Walbranc92d35f2022-01-12 12:45:19 +0000695 .arg("run")
696 .arg("--disable-sandbox")
697 .arg("--cid")
698 .arg(config.cid.to_string());
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000699
Keir Fraserf25cb922022-11-23 14:26:00 +0000700 if system_properties::read_bool("hypervisor.memory_reclaim.supported", false)? {
701 command.arg("--balloon-page-reporting");
702 } else {
703 command.arg("--no-balloon");
704 }
705
Andrew Walbranf8650422021-06-09 15:54:09 +0000706 if config.protected {
Jaewan Kim46422812022-11-25 10:59:39 +0900707 match system_properties::read(SYSPROP_CUSTOM_PVMFW_PATH)? {
708 Some(pvmfw_path) if !pvmfw_path.is_empty() => {
709 command.arg("--protected-vm-with-firmware").arg(pvmfw_path)
710 }
711 _ => command.arg("--protected-vm"),
712 };
Andrew Walbran0b5789f2022-02-04 13:57:57 +0000713
714 // 3 virtio-console devices + vsock = 4.
715 let virtio_pci_device_count = 4 + config.disks.len();
716 // crosvm virtio queue has 256 entries, so 2 MiB per device (2 pages per entry) should be
717 // enough.
Jiyong Park3115cdb2023-02-24 18:04:44 +0900718 let swiotlb_size_mib = 2 * virtio_pci_device_count as u32;
Andrew Walbran0b5789f2022-02-04 13:57:57 +0000719 command.arg("--swiotlb").arg(swiotlb_size_mib.to_string());
Frederick Mayle58baa302023-02-14 19:06:37 -0800720
721 // Workaround to keep crash_dump from trying to read protected guest memory.
722 // Context in b/238324526.
723 command.arg("--unmap-guest-memory-on-fork");
Jiyong Park3115cdb2023-02-24 18:04:44 +0900724
Jiyong Parked180932023-02-24 19:55:41 +0900725 if config.ramdump.is_some() {
Jaewan Kim3124ef02023-03-23 19:25:20 +0900726 // Protected VM needs to reserve memory for ramdump here. Note that we reserve more
Jiyong Parked180932023-02-24 19:55:41 +0900727 // memory for the restricted dma pool.
728 let ramdump_reserve = RAMDUMP_RESERVED_MIB + swiotlb_size_mib;
729 command.arg("--params").arg(format!("crashkernel={ramdump_reserve}M"));
730 }
Jaewan Kim84b91212023-02-28 00:11:57 +0900731 } else {
732 if config.ramdump.is_some() {
733 command.arg("--params").arg(format!("crashkernel={RAMDUMP_RESERVED_MIB}M"));
734 }
735 if config.debug_level == DebugLevel::NONE
736 && should_prepare_console_output(config.debug_level)
737 {
738 // bootconfig.normal will be used, but we need log.
739 // pvmfw will add following commands by itself, but non-protected VM should do so here.
740 command.arg("--params").arg("printk.devkmsg=on");
741 command.arg("--params").arg("console=hvc0");
742 }
Andrew Walbranf8650422021-06-09 15:54:09 +0000743 }
744
Andrew Walbranb15cd6e2021-07-05 16:38:07 +0000745 if let Some(memory_mib) = config.memory_mib {
746 command.arg("--mem").arg(memory_mib.to_string());
747 }
748
Jiyong Park032615f2022-01-10 13:55:34 +0900749 if let Some(cpus) = config.cpus {
750 command.arg("--cpus").arg(cpus.to_string());
751 }
752
David Brazdil7d1e5ec2023-02-06 17:56:29 +0000753 if config.host_cpu_topology {
David Brazdil25e8c052023-02-17 12:53:01 +0000754 // TODO(b/266664564): replace with --host-cpu-topology once available
755 if let Some(cpus) = get_num_cpus() {
756 command.arg("--cpus").arg(cpus.to_string());
757 } else {
758 bail!("Could not determine the number of CPUs in the system");
759 }
David Brazdil7d1e5ec2023-02-06 17:56:29 +0000760 }
761
Jiyong Parkdfe16d62022-04-20 17:32:12 +0900762 if !config.task_profiles.is_empty() {
763 command.arg("--task-profiles").arg(config.task_profiles.join(","));
764 }
765
Nikita Ioffe5776f082023-02-10 21:38:26 +0000766 if let Some(gdb_port) = config.gdb_port {
767 command.arg("--gdb").arg(gdb_port.to_string());
768 }
769
Jiyong Parkfa91d702021-10-18 23:51:39 +0900770 // Keep track of what file descriptors should be mapped to the crosvm process.
771 let mut preserved_fds = config.indirect_files.iter().map(|file| file.as_raw_fd()).collect();
772
Jiyong Park747d6362021-10-19 17:12:52 +0900773 // Setup the serial devices.
774 // 1. uart device: used as the output device by bootloaders and as early console by linux
Andrew Walbranb27681f2022-02-23 15:11:52 +0000775 // 2. uart device: used to report the reason for the VM failing.
776 // 3. virtio-console device: used as the console device where kmsg is redirected to
Jiyong Park4afe2012022-07-08 05:38:49 +0900777 // 4. virtio-console device: used as the ramdump output
Andrew Walbranb27681f2022-02-23 15:11:52 +0000778 // 5. virtio-console device: used as the logcat output
Jiyong Park747d6362021-10-19 17:12:52 +0900779 //
Jiyong Parkb8182bb2021-10-26 22:53:08 +0900780 // When [console|log]_fd is not specified, the devices are attached to sink, which means what's
781 // written there is discarded.
Andrew Walbranb27681f2022-02-23 15:11:52 +0000782 let console_arg = format_serial_arg(&mut preserved_fds, &config.console_fd);
783 let log_arg = format_serial_arg(&mut preserved_fds, &config.log_fd);
784 let failure_serial_path = add_preserved_fd(&mut preserved_fds, &failure_pipe_write);
Jiyong Parke558ab12022-07-07 20:18:55 +0900785 let ramdump_arg = format_serial_arg(&mut preserved_fds, &config.ramdump);
Jiyong Parkfa91d702021-10-18 23:51:39 +0900786
Jiyong Park747d6362021-10-19 17:12:52 +0900787 // Warning: Adding more serial devices requires you to shift the PCI device ID of the boot
788 // disks in bootconfig.x86_64. This is because x86 crosvm puts serial devices and the block
789 // devices in the same PCI bus and serial devices comes before the block devices. Arm crosvm
790 // doesn't have the issue.
Jiyong Parkfa91d702021-10-18 23:51:39 +0900791 // /dev/ttyS0
Andrew Walbranb27681f2022-02-23 15:11:52 +0000792 command.arg(format!("--serial={},hardware=serial,num=1", &console_arg));
793 // /dev/ttyS1
794 command.arg(format!("--serial=type=file,path={},hardware=serial,num=2", &failure_serial_path));
Jiyong Parkfa91d702021-10-18 23:51:39 +0900795 // /dev/hvc0
Jiyong Parkb8182bb2021-10-26 22:53:08 +0900796 command.arg(format!("--serial={},hardware=virtio-console,num=1", &console_arg));
Jiyong Park4afe2012022-07-08 05:38:49 +0900797 // /dev/hvc1
798 command.arg(format!("--serial={},hardware=virtio-console,num=2", &ramdump_arg));
Jiyong Parkae5a4ed2021-11-01 18:36:28 +0900799 // /dev/hvc2
800 command.arg(format!("--serial={},hardware=virtio-console,num=3", &log_arg));
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000801
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000802 if let Some(bootloader) = &config.bootloader {
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000803 command.arg("--bios").arg(add_preserved_fd(&mut preserved_fds, bootloader));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000804 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000805
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000806 if let Some(initrd) = &config.initrd {
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000807 command.arg("--initrd").arg(add_preserved_fd(&mut preserved_fds, initrd));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000808 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000809
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000810 if let Some(params) = &config.params {
811 command.arg("--params").arg(params);
812 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000813
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000814 for disk in &config.disks {
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000815 command
816 .arg(if disk.writable { "--rwdisk" } else { "--disk" })
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000817 .arg(add_preserved_fd(&mut preserved_fds, &disk.image));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000818 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000819
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000820 if let Some(kernel) = &config.kernel {
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000821 command.arg(add_preserved_fd(&mut preserved_fds, kernel));
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000822 }
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000823
Keir Frasercdd4b112022-11-24 14:02:25 +0000824 let control_server_socket = UnixSeqpacketListener::bind(crosvm_control_socket_path)
825 .context("failed to create control server")?;
Elie Kheirallah2d12ff32023-03-20 18:12:07 +0000826 command
827 .arg("--socket")
828 .arg(add_preserved_fd(&mut preserved_fds, &control_server_socket.as_raw_descriptor()));
Keir Fraser13a956a2022-07-14 14:20:46 +0000829
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000830 debug!("Preserving FDs {:?}", preserved_fds);
831 command.preserved_fds(preserved_fds);
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000832
Jiyong Park2d736562022-10-24 22:40:12 +0900833 print_crosvm_args(&command);
834
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000835 let result = SharedChild::spawn(&mut command)?;
Jooyung Hanbfe086f2021-10-28 10:15:45 +0900836 debug!("Spawned crosvm({}).", result.id());
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000837 Ok(result)
838}
839
840/// Ensure that the configuration has a valid combination of fields set, or return an error if not.
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000841fn validate_config(config: &CrosvmConfig) -> Result<(), Error> {
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000842 if config.bootloader.is_none() && config.kernel.is_none() {
843 bail!("VM must have either a bootloader or a kernel image.");
844 }
845 if config.bootloader.is_some() && (config.kernel.is_some() || config.initrd.is_some()) {
846 bail!("Can't have both bootloader and kernel/initrd image.");
847 }
Jiyong Parkdcf17412022-02-08 15:07:23 +0900848 let version = Version::parse(CROSVM_PLATFORM_VERSION).unwrap();
849 if !config.platform_version.matches(&version) {
850 bail!(
851 "Incompatible platform version. The config is compatible with platform version(s) \
852 {}, but the actual platform version is {}",
853 config.platform_version,
854 version
855 );
856 }
857
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000858 Ok(())
859}
860
Jiyong Park2d736562022-10-24 22:40:12 +0900861/// Print arguments of the crosvm command. In doing so, /proc/self/fd/XX is annotated with the
862/// actual file path if the FD is backed by a regular file. If not, the /proc path is printed
863/// unmodified.
864fn print_crosvm_args(command: &Command) {
865 let re = Regex::new(r"/proc/self/fd/[\d]+").unwrap();
866 info!(
867 "Running crosvm with args: {:?}",
868 command
869 .get_args()
870 .map(|s| s.to_string_lossy())
871 .map(|s| {
872 re.replace_all(&s, |caps: &Captures| {
873 let path = &caps[0];
874 if let Ok(realpath) = std::fs::canonicalize(path) {
875 format!("{} ({})", path, realpath.to_string_lossy())
876 } else {
877 path.to_owned()
878 }
879 })
880 .into_owned()
881 })
882 .collect::<Vec<_>>()
883 );
884}
885
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000886/// Adds the file descriptor for `file` to `preserved_fds`, and returns a string of the form
887/// "/proc/self/fd/N" where N is the file descriptor.
Keir Fraser13a956a2022-07-14 14:20:46 +0000888fn add_preserved_fd(preserved_fds: &mut Vec<RawFd>, file: &dyn AsRawFd) -> String {
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000889 let fd = file.as_raw_fd();
Andrew Walbran02b8ec02021-06-22 13:07:02 +0000890 preserved_fds.push(fd);
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000891 format!("/proc/self/fd/{}", fd)
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000892}
Andrew Walbranb27681f2022-02-23 15:11:52 +0000893
894/// Adds the file descriptor for `file` (if any) to `preserved_fds`, and returns the appropriate
895/// string for a crosvm `--serial` flag. If `file` is none, creates a dummy sink device.
896fn format_serial_arg(preserved_fds: &mut Vec<RawFd>, file: &Option<File>) -> String {
897 if let Some(file) = file {
898 format!("type=file,path={}", add_preserved_fd(preserved_fds, file))
899 } else {
900 "type=sink".to_string()
901 }
902}
903
904/// Creates a new pipe with the `O_CLOEXEC` flag set, and returns the read side and write side.
905fn create_pipe() -> Result<(File, File), Error> {
906 let (raw_read, raw_write) = pipe2(OFlag::O_CLOEXEC)?;
907 // SAFETY: We are the sole owners of these fds as they were just created.
908 let read_fd = unsafe { File::from_raw_fd(raw_read) };
909 let write_fd = unsafe { File::from_raw_fd(raw_write) };
910 Ok((read_fd, write_fd))
911}