blob: 2d83963cc74cbfce75d66dbc68164d0da163b4dd [file] [log] [blame]
Andrew Walbrand6dce6f2021-03-05 16:39:08 +00001// Copyright 2021, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Andrew Walbranf6bf6862021-05-21 12:41:13 +000015//! Implementation of the AIDL interface of the VirtualizationService.
Andrew Walbrand6dce6f2021-03-05 16:39:08 +000016
Seungjae Yoo13af0b62024-05-20 14:15:13 +090017use crate::{get_calling_pid, get_calling_uid, get_this_pid};
Alan Stokesfda70842023-12-20 17:50:14 +000018use crate::atom::{write_vm_booted_stats, write_vm_creation_stats};
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000019use crate::composite::make_composite_image;
Mu-Le Leeb2d5a312024-06-05 10:52:47 +080020use crate::crosvm::{AudioConfig, CrosvmConfig, DiskFile, DisplayConfig, GpuConfig, InputDeviceOption, PayloadState, VmContext, VmInstance, VmState};
Jaewan Kim61f86142023-03-28 15:12:52 +090021use crate::debug_config::DebugConfig;
Shikha Panwar55e10ec2024-02-13 12:53:49 +000022use crate::dt_overlay::{create_device_tree_overlay, VM_DT_OVERLAY_MAX_SIZE, VM_DT_OVERLAY_PATH};
Nikita Ioffe5dfddf22023-06-29 16:11:26 +010023use crate::payload::{add_microdroid_payload_images, add_microdroid_system_images, add_microdroid_vendor_image};
Seungjae Yoofd9a0622022-10-14 10:01:29 +090024use crate::selinux::{getfilecon, SeContext};
Jiyong Park753553b2021-07-12 21:21:09 +090025use android_os_permissions_aidl::aidl::android::os::IPermissionController;
David Brazdil49f96f52022-12-16 21:29:13 +000026use android_system_virtualizationcommon::aidl::android::system::virtualizationcommon::{
Alice Wang4e3015d2023-10-10 09:35:37 +000027 Certificate::Certificate,
Andrew Walbranc92d35f2022-01-12 12:45:19 +000028 DeathReason::DeathReason,
David Brazdil49f96f52022-12-16 21:29:13 +000029 ErrorCode::ErrorCode,
30};
31use android_system_virtualizationservice::aidl::android::system::virtualizationservice::{
Inseob Kim53d0b212023-07-20 16:58:37 +090032 AssignableDevice::AssignableDevice,
David Brazdil7d1e5ec2023-02-06 17:56:29 +000033 CpuTopology::CpuTopology,
Andrew Walbran6b650662021-09-07 13:13:23 +000034 DiskImage::DiskImage,
Jeongik Chac181be72024-03-27 00:18:30 +090035 InputDevice::InputDevice,
Alan Stokes0cc59ee2021-09-24 11:20:34 +010036 IVirtualMachine::{BnVirtualMachine, IVirtualMachine},
Andrew Walbran6b650662021-09-07 13:13:23 +000037 IVirtualMachineCallback::IVirtualMachineCallback,
38 IVirtualizationService::IVirtualizationService,
Jiyong Park029977d2021-11-24 21:56:49 +090039 Partition::Partition,
Andrew Walbran6b650662021-09-07 13:13:23 +000040 PartitionType::PartitionType,
Inseob Kim0168b462022-12-27 14:54:35 +090041 VirtualMachineAppConfig::{DebugLevel::DebugLevel, Payload::Payload, VirtualMachineAppConfig},
Jooyung Han21e9b922021-06-26 04:14:16 +090042 VirtualMachineConfig::VirtualMachineConfig,
Andrew Walbran6b650662021-09-07 13:13:23 +000043 VirtualMachineDebugInfo::VirtualMachineDebugInfo,
Alan Stokes0d1ef782022-09-27 13:46:35 +010044 VirtualMachinePayloadConfig::VirtualMachinePayloadConfig,
Jooyung Han21e9b922021-06-26 04:14:16 +090045 VirtualMachineRawConfig::VirtualMachineRawConfig,
Andrew Walbran6b650662021-09-07 13:13:23 +000046 VirtualMachineState::VirtualMachineState,
Jooyung Han21e9b922021-06-26 04:14:16 +090047};
David Brazdilafc9a9e2023-01-12 16:08:10 +000048use android_system_virtualizationservice_internal::aidl::android::system::virtualizationservice_internal::IVirtualizationServiceInternal::IVirtualizationServiceInternal;
Seungjae Yoodd91f0f2022-11-09 15:25:21 +090049use android_system_virtualmachineservice::aidl::android::system::virtualmachineservice::IVirtualMachineService::{
David Brazdilafc9a9e2023-01-12 16:08:10 +000050 BnVirtualMachineService, IVirtualMachineService,
Seungjae Yoofd9a0622022-10-14 10:01:29 +090051};
Shikha Panware45e9422024-02-28 21:18:10 +000052use android_hardware_security_secretkeeper::aidl::android::hardware::security::secretkeeper::ISecretkeeper::{BnSecretkeeper, ISecretkeeper};
Shikha Panwar5d6a6752023-12-14 22:08:26 +000053use android_hardware_security_secretkeeper::aidl::android::hardware::security::secretkeeper::SecretId::SecretId;
54use android_hardware_security_authgraph::aidl::android::hardware::security::authgraph::{
55 Arc::Arc as AuthgraphArc, IAuthGraphKeyExchange::IAuthGraphKeyExchange,
56 IAuthGraphKeyExchange::BnAuthGraphKeyExchange, Identity::Identity, KeInitResult::KeInitResult,
57 Key::Key, PubKey::PubKey, SessionIdSignature::SessionIdSignature, SessionInfo::SessionInfo,
58 SessionInitiationInfo::SessionInitiationInfo,
59};
Alan Stokes25f69362023-03-06 16:51:54 +000060use anyhow::{anyhow, bail, Context, Result};
Seungjae Yoofd9a0622022-10-14 10:01:29 +090061use apkverify::{HashAlgorithm, V4Signature};
Jiyong Parkd7bd2f22023-08-10 20:41:19 +090062use avflog::LogResult;
Alan Stokes0e82b502022-08-08 14:44:48 +010063use binder::{
David Brazdilafc9a9e2023-01-12 16:08:10 +000064 self, wait_for_interface, BinderFeatures, ExceptionCode, Interface, ParcelFileDescriptor,
65 Status, StatusCode, Strong,
Jiyong Park2227eaa2023-08-04 11:59:18 +090066 IntoBinderResult,
Andrew Walbrana89fc132021-03-17 17:08:36 +000067};
Shikha Panwar55e10ec2024-02-13 12:53:49 +000068use cstr::cstr;
Inseob Kim46257382024-01-03 15:41:22 +090069use glob::glob;
David Brazdil49f96f52022-12-16 21:29:13 +000070use lazy_static::lazy_static;
Seungjae Yoo0a8c84c2022-07-11 08:19:15 +000071use log::{debug, error, info, warn};
Inseob Kim89b24592024-02-23 18:59:43 +090072use microdroid_payload_config::{ApkConfig, Task, TaskType, VmPayloadConfig};
Inseob Kim0168b462022-12-27 14:54:35 +090073use nix::unistd::pipe;
David Brazdil73988ea2022-11-11 15:10:32 +000074use rpcbinder::RpcServer;
Alan Stokes25f69362023-03-06 16:51:54 +000075use rustutils::system_properties;
Jiyong Parkdcf17412022-02-08 15:07:23 +090076use semver::VersionReq;
Inseob Kim6ef80972023-07-20 17:23:36 +090077use std::collections::HashSet;
Andrew Walbrandff3b942021-06-09 15:20:36 +000078use std::convert::TryInto;
Shikha Panwar55e10ec2024-02-13 12:53:49 +000079use std::fs;
Jaewan Kim39952072024-01-19 17:04:53 +090080use std::ffi::CStr;
Inseob Kim6ef80972023-07-20 17:23:36 +090081use std::fs::{canonicalize, read_dir, remove_file, File, OpenOptions};
Shikha Panwar9ae2e622024-01-30 12:22:30 +000082use std::io::{BufRead, BufReader, Error, ErrorKind, Seek, SeekFrom, Write};
Inseob Kim46257382024-01-03 15:41:22 +090083use std::iter;
Nikita Ioffe5776f082023-02-10 21:38:26 +000084use std::num::{NonZeroU16, NonZeroU32};
Inseob Kimff48a7c2024-02-26 22:07:21 +090085use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd};
David Brazdilafc9a9e2023-01-12 16:08:10 +000086use std::os::unix::raw::pid_t;
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +000087use std::path::{Path, PathBuf};
Andrew Walbran320b5602021-03-04 16:11:12 +000088use std::sync::{Arc, Mutex, Weak};
Seungjae Yoo14e60182024-02-21 13:28:31 +090089use vbmeta::VbMetaImage;
Pierre-Clément Tosid3bbe1d2024-04-15 18:03:51 +010090use vmconfig::{VmConfig, get_debug_level};
David Brazdilafc9a9e2023-01-12 16:08:10 +000091use vsock::VsockStream;
Jooyung Han35edb8f2021-07-01 16:17:16 +090092use zip::ZipArchive;
Andrew Walbrand6dce6f2021-03-05 16:39:08 +000093
David Brazdil41d1a872022-10-05 14:44:19 +010094/// The unique ID of a VM used (together with a port number) for vsock communication.
95pub type Cid = u32;
96
David Brazdil4b4c5102022-12-19 22:56:20 +000097pub const BINDER_SERVICE_IDENTIFIER: &str = "android.system.virtualizationservice";
98
Jooyung Han95884632021-07-06 22:27:54 +090099/// The size of zero.img.
100/// Gaps in composite disk images are filled with a shared zero.img.
101const ZERO_FILLER_SIZE: u64 = 4096;
102
David Brazdilf50c7a62023-04-19 14:22:42 +0000103/// Magic string for the instance image
104const ANDROID_VM_INSTANCE_MAGIC: &str = "Android-VM-instance";
105
106/// Version of the instance image format
107const ANDROID_VM_INSTANCE_VERSION: u16 = 1;
108
Alan Stokes0d1ef782022-09-27 13:46:35 +0100109const MICRODROID_OS_NAME: &str = "microdroid";
110
Shikha Panwar5d6a6752023-12-14 22:08:26 +0000111const SECRETKEEPER_IDENTIFIER: &str =
Shikha Panwarbe5dee72024-02-21 19:06:20 +0000112 "android.hardware.security.secretkeeper.ISecretkeeper/default";
Shikha Panwar5d6a6752023-12-14 22:08:26 +0000113
David Brazdilf50c7a62023-04-19 14:22:42 +0000114const UNFORMATTED_STORAGE_MAGIC: &str = "UNFORMATTED-STORAGE";
115
116/// crosvm requires all partitions to be a multiple of 4KiB.
117const PARTITION_GRANULARITY_BYTES: u64 = 4096;
118
Shikha Panwar55e10ec2024-02-13 12:53:49 +0000119const VM_REFERENCE_DT_ON_HOST_PATH: &str = "/proc/device-tree/avf/reference";
120
David Brazdil49f96f52022-12-16 21:29:13 +0000121lazy_static! {
David Brazdil4b4c5102022-12-19 22:56:20 +0000122 pub static ref GLOBAL_SERVICE: Strong<dyn IVirtualizationServiceInternal> =
123 wait_for_interface(BINDER_SERVICE_IDENTIFIER)
124 .expect("Could not connect to VirtualizationServiceInternal");
Inseob Kim46257382024-01-03 15:41:22 +0900125 static ref SUPPORTED_OS_NAMES: HashSet<String> =
126 get_supported_os_names().expect("Failed to get list of supported os names");
David Brazdil49f96f52022-12-16 21:29:13 +0000127}
128
Nikita Ioffef1ce9872022-12-09 13:31:59 +0000129fn create_or_update_idsig_file(
130 input_fd: &ParcelFileDescriptor,
131 idsig_fd: &ParcelFileDescriptor,
132) -> Result<()> {
133 let mut input = clone_file(input_fd)?;
134 let metadata = input.metadata().context("failed to get input metadata")?;
135 if !metadata.is_file() {
136 bail!("input is not a regular file");
137 }
Alan Stokes25f69362023-03-06 16:51:54 +0000138 let mut sig =
139 V4Signature::create(&mut input, get_current_sdk()?, 4096, &[], HashAlgorithm::SHA256)
140 .context("failed to create idsig")?;
Nikita Ioffef1ce9872022-12-09 13:31:59 +0000141
142 let mut output = clone_file(idsig_fd)?;
Jiyong Park8d192952023-06-26 14:29:51 +0900143
144 // Optimization. We don't have to update idsig file whenever a VM is started. Don't update it,
145 // if the idsig file already has the same APK digest.
146 if output.metadata()?.len() > 0 {
147 if let Ok(out_sig) = V4Signature::from_idsig(&mut output) {
148 if out_sig.signing_info.apk_digest == sig.signing_info.apk_digest {
149 debug!("idsig {:?} is up-to-date with apk {:?}.", output, input);
150 return Ok(());
151 }
152 }
153 // if we fail to read v4signature from output, that's fine. User can pass a random file.
154 // We will anyway overwrite the file to the v4signature generated from input_fd.
155 }
156
Shikha Panwar9ae2e622024-01-30 12:22:30 +0000157 output
158 .seek(SeekFrom::Start(0))
159 .context("failed to move cursor to start on the idsig output")?;
Nikita Ioffec09b0492022-12-14 20:18:33 +0000160 output.set_len(0).context("failed to set_len on the idsig output")?;
161 sig.write_into(&mut output).context("failed to write idsig")?;
Nikita Ioffef1ce9872022-12-09 13:31:59 +0000162 Ok(())
163}
164
Alan Stokes25f69362023-03-06 16:51:54 +0000165fn get_current_sdk() -> Result<u32> {
166 let current_sdk = system_properties::read("ro.build.version.sdk")?;
167 let current_sdk = current_sdk.ok_or_else(|| anyhow!("SDK version missing"))?;
168 current_sdk.parse().context("Malformed SDK version")
169}
170
David Brazdil4b4c5102022-12-19 22:56:20 +0000171pub fn remove_temporary_files(path: &PathBuf) -> Result<()> {
172 for dir_entry in read_dir(path)? {
173 remove_file(dir_entry?.path())?;
174 }
175 Ok(())
David Brazdil528e0472022-10-10 15:06:02 +0100176}
177
David Brazdil528e0472022-10-10 15:06:02 +0100178/// Implementation of `IVirtualizationService`, the entry point of the AIDL service.
David Brazdil49f96f52022-12-16 21:29:13 +0000179#[derive(Debug, Default)]
Andrew Walbranf6bf6862021-05-21 12:41:13 +0000180pub struct VirtualizationService {
Jiyong Park8611a6c2021-07-09 18:17:44 +0900181 state: Arc<Mutex<State>>,
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000182}
183
Shikha Panward8e35422021-10-11 13:51:27 +0000184impl Interface for VirtualizationService {
Andrei Homescu0cf8e222023-11-09 04:27:55 +0000185 fn dump(&self, writer: &mut dyn Write, _args: &[&CStr]) -> Result<(), StatusCode> {
Shikha Panward8e35422021-10-11 13:51:27 +0000186 check_permission("android.permission.DUMP").or(Err(StatusCode::PERMISSION_DENIED))?;
187 let state = &mut *self.state.lock().unwrap();
188 let vms = state.vms();
Andrei Homescu0cf8e222023-11-09 04:27:55 +0000189 writeln!(writer, "Running {0} VMs:", vms.len()).or(Err(StatusCode::UNKNOWN_ERROR))?;
Shikha Panward8e35422021-10-11 13:51:27 +0000190 for vm in vms {
Andrei Homescu0cf8e222023-11-09 04:27:55 +0000191 writeln!(writer, "VM CID: {}", vm.cid).or(Err(StatusCode::UNKNOWN_ERROR))?;
192 writeln!(writer, "\tState: {:?}", vm.vm_state.lock().unwrap())
Shikha Panward8e35422021-10-11 13:51:27 +0000193 .or(Err(StatusCode::UNKNOWN_ERROR))?;
Andrei Homescu0cf8e222023-11-09 04:27:55 +0000194 writeln!(writer, "\tPayload state {:?}", vm.payload_state())
Shikha Panward8e35422021-10-11 13:51:27 +0000195 .or(Err(StatusCode::UNKNOWN_ERROR))?;
Andrei Homescu0cf8e222023-11-09 04:27:55 +0000196 writeln!(writer, "\tProtected: {}", vm.protected).or(Err(StatusCode::UNKNOWN_ERROR))?;
197 writeln!(writer, "\ttemporary_directory: {}", vm.temporary_directory.to_string_lossy())
Shikha Panward8e35422021-10-11 13:51:27 +0000198 .or(Err(StatusCode::UNKNOWN_ERROR))?;
Andrei Homescu0cf8e222023-11-09 04:27:55 +0000199 writeln!(writer, "\trequester_uid: {}", vm.requester_uid)
Shikha Panward8e35422021-10-11 13:51:27 +0000200 .or(Err(StatusCode::UNKNOWN_ERROR))?;
Andrei Homescu0cf8e222023-11-09 04:27:55 +0000201 writeln!(writer, "\trequester_debug_pid: {}", vm.requester_debug_pid)
Shikha Panward8e35422021-10-11 13:51:27 +0000202 .or(Err(StatusCode::UNKNOWN_ERROR))?;
203 }
204 Ok(())
205 }
206}
Andrew Walbranf6bf6862021-05-21 12:41:13 +0000207impl IVirtualizationService for VirtualizationService {
Andrew Walbranf8d94112021-09-07 11:45:36 +0000208 /// Creates (but does not start) a new VM with the given configuration, assigning it the next
209 /// available CID.
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000210 ///
211 /// Returns a binder `IVirtualMachine` object referring to it, as a handle for the client.
Andrew Walbranf8d94112021-09-07 11:45:36 +0000212 fn createVm(
Andrew Walbrana89fc132021-03-17 17:08:36 +0000213 &self,
Andrew Walbran3a5a9212021-05-04 17:09:08 +0000214 config: &VirtualMachineConfig,
Jiyong Parke6fb1672023-06-26 16:45:55 +0900215 console_out_fd: Option<&ParcelFileDescriptor>,
216 console_in_fd: Option<&ParcelFileDescriptor>,
Andrew Walbrana89fc132021-03-17 17:08:36 +0000217 log_fd: Option<&ParcelFileDescriptor>,
218 ) -> binder::Result<Strong<dyn IVirtualMachine>> {
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000219 let mut is_protected = false;
Jiyong Parke6fb1672023-06-26 16:45:55 +0900220 let ret = self.create_vm_internal(
221 config,
222 console_out_fd,
223 console_in_fd,
224 log_fd,
225 &mut is_protected,
226 );
Seungjae Yoo0a8c84c2022-07-11 08:19:15 +0000227 write_vm_creation_stats(config, is_protected, &ret);
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000228 ret
Andrew Walbrand6dce6f2021-03-05 16:39:08 +0000229 }
Andrew Walbran320b5602021-03-04 16:11:12 +0000230
Shikha Panwar61a74b52024-02-16 13:17:01 +0000231 /// Allocate a new instance_id to the VM
232 fn allocateInstanceId(&self) -> binder::Result<[u8; 64]> {
Shikha Panwarbd5eba92024-03-27 20:23:31 +0000233 check_manage_access()?;
Shikha Panwar61a74b52024-02-16 13:17:01 +0000234 GLOBAL_SERVICE.allocateInstanceId()
235 }
236
Andrew Walbrandff3b942021-06-09 15:20:36 +0000237 /// Initialise an empty partition image of the given size to be used as a writable partition.
238 fn initializeWritablePartition(
239 &self,
240 image_fd: &ParcelFileDescriptor,
Alan Stokesff0005f2023-01-30 09:53:00 +0000241 size_bytes: i64,
Jiyong Park9dd389e2021-08-23 20:42:59 +0900242 partition_type: PartitionType,
Andrew Walbrandff3b942021-06-09 15:20:36 +0000243 ) -> binder::Result<()> {
Jiyong Park753553b2021-07-12 21:21:09 +0900244 check_manage_access()?;
Jiyong Park2227eaa2023-08-04 11:59:18 +0900245 let size_bytes = size_bytes
246 .try_into()
247 .with_context(|| format!("Invalid size: {}", size_bytes))
248 .or_binder_exception(ExceptionCode::ILLEGAL_ARGUMENT)?;
David Brazdilf50c7a62023-04-19 14:22:42 +0000249 let size_bytes = round_up(size_bytes, PARTITION_GRANULARITY_BYTES);
Shikha Panwar9ae2e622024-01-30 12:22:30 +0000250 let mut image = clone_file(image_fd)?;
David Brazdilf50c7a62023-04-19 14:22:42 +0000251 // initialize the file. Any data in the file will be erased.
Shikha Panwar9ae2e622024-01-30 12:22:30 +0000252 image
253 .seek(SeekFrom::Start(0))
254 .context("failed to move cursor to start")
255 .or_service_specific_exception(-1)?;
Jiyong Park2227eaa2023-08-04 11:59:18 +0900256 image.set_len(0).context("Failed to reset a file").or_service_specific_exception(-1)?;
Frederick Mayle0d310572024-05-02 12:34:58 -0700257 // Set the file length. In most filesystems, this will not allocate any physical disk
258 // space, it will only change the logical size.
259 image
260 .set_len(size_bytes)
261 .context("Failed to extend file")
Jiyong Park2227eaa2023-08-04 11:59:18 +0900262 .or_service_specific_exception(-1)?;
David Brazdilf50c7a62023-04-19 14:22:42 +0000263
264 match partition_type {
265 PartitionType::RAW => Ok(()),
Frederick Mayle0d310572024-05-02 12:34:58 -0700266 PartitionType::ANDROID_VM_INSTANCE => format_as_android_vm_instance(&mut image),
267 PartitionType::ENCRYPTEDSTORE => format_as_encryptedstore(&mut image),
David Brazdilf50c7a62023-04-19 14:22:42 +0000268 _ => Err(Error::new(
269 ErrorKind::Unsupported,
270 format!("Unsupported partition type {:?}", partition_type),
271 )),
272 }
Jiyong Park2227eaa2023-08-04 11:59:18 +0900273 .with_context(|| format!("Failed to initialize partition as {:?}", partition_type))
274 .or_service_specific_exception(-1)?;
David Brazdilf50c7a62023-04-19 14:22:42 +0000275
276 Ok(())
Andrew Walbrandff3b942021-06-09 15:20:36 +0000277 }
278
Jiyong Park0a248432021-08-20 23:32:39 +0900279 /// Creates or update the idsig file by digesting the input APK file.
280 fn createOrUpdateIdsigFile(
281 &self,
282 input_fd: &ParcelFileDescriptor,
283 idsig_fd: &ParcelFileDescriptor,
284 ) -> binder::Result<()> {
Jiyong Parkc3ca24f2022-06-28 10:45:15 +0900285 check_manage_access()?;
286
Jiyong Park2227eaa2023-08-04 11:59:18 +0900287 create_or_update_idsig_file(input_fd, idsig_fd).or_service_specific_exception(-1)?;
Jiyong Park0a248432021-08-20 23:32:39 +0900288 Ok(())
289 }
290
Andrew Walbran320b5602021-03-04 16:11:12 +0000291 /// Get a list of all currently running VMs. This method is only intended for debug purposes,
292 /// and as such is only permitted from the shell user.
293 fn debugListVms(&self) -> binder::Result<Vec<VirtualMachineDebugInfo>> {
David Brazdil209074a2023-01-12 16:44:51 +0000294 // Delegate to the global service, including checking the debug permission.
David Brazdild4f51a52023-01-11 14:09:27 +0000295 GLOBAL_SERVICE.debugListVms()
Andrew Walbran320b5602021-03-04 16:11:12 +0000296 }
Inseob Kim53d0b212023-07-20 16:58:37 +0900297
298 /// Get a list of assignable device types.
299 fn getAssignableDevices(&self) -> binder::Result<Vec<AssignableDevice>> {
300 // Delegate to the global service, including checking the permission.
301 GLOBAL_SERVICE.getAssignableDevices()
302 }
Nikita Ioffef7c742a2023-09-04 16:57:59 +0100303
Inseob Kim46257382024-01-03 15:41:22 +0900304 /// Get a list of supported OSes.
305 fn getSupportedOSList(&self) -> binder::Result<Vec<String>> {
306 Ok(Vec::from_iter(SUPPORTED_OS_NAMES.iter().cloned()))
307 }
308
Nikita Ioffef7c742a2023-09-04 16:57:59 +0100309 /// Returns whether given feature is enabled
310 fn isFeatureEnabled(&self, feature: &str) -> binder::Result<bool> {
311 check_manage_access()?;
Shikha Panwar6d306412024-02-17 21:37:49 +0000312 Ok(avf_features::is_feature_enabled(feature))
Nikita Ioffef7c742a2023-09-04 16:57:59 +0100313 }
Alice Wange64dd182024-01-17 15:57:55 +0000314
315 fn enableTestAttestation(&self) -> binder::Result<()> {
316 GLOBAL_SERVICE.enableTestAttestation()
317 }
Alice Wang0362f7f2024-03-21 08:16:26 +0000318
319 fn isRemoteAttestationSupported(&self) -> binder::Result<bool> {
320 check_manage_access()?;
321 GLOBAL_SERVICE.isRemoteAttestationSupported()
322 }
Shikha Panwar7aef66d2024-03-13 00:28:41 +0000323
324 fn isUpdatableVmSupported(&self) -> binder::Result<bool> {
325 // The response is specific to Microdroid. Updatable VMs are only possible if device
326 // supports Secretkeeper. Guest OS needs to use Secretkeeper based secrets. Microdroid does
327 // this, however other guest OSes may do things differently.
328 check_manage_access()?;
329 Ok(is_secretkeeper_supported())
330 }
Shikha Panwarbd5eba92024-03-27 20:23:31 +0000331
Shikha Panwar07d701c2024-03-28 14:43:52 +0000332 fn removeVmInstance(&self, instance_id: &[u8; 64]) -> binder::Result<()> {
333 check_manage_access()?;
334 GLOBAL_SERVICE.removeVmInstance(instance_id)
335 }
336
Shikha Panwarbd5eba92024-03-27 20:23:31 +0000337 fn claimVmInstance(&self, instance_id: &[u8; 64]) -> binder::Result<()> {
338 check_manage_access()?;
339 GLOBAL_SERVICE.claimVmInstance(instance_id)
340 }
Andrew Walbran320b5602021-03-04 16:11:12 +0000341}
342
Jiyong Park8611a6c2021-07-09 18:17:44 +0900343impl VirtualizationService {
344 pub fn init() -> VirtualizationService {
David Brazdil49f96f52022-12-16 21:29:13 +0000345 VirtualizationService::default()
Jiyong Park8611a6c2021-07-09 18:17:44 +0900346 }
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000347
David Brazdil209074a2023-01-12 16:44:51 +0000348 fn create_vm_context(
349 &self,
350 requester_debug_pid: pid_t,
351 ) -> binder::Result<(VmContext, Cid, PathBuf)> {
David Brazdil8cf8f482022-11-23 14:21:26 +0000352 const NUM_ATTEMPTS: usize = 5;
353
354 for _ in 0..NUM_ATTEMPTS {
Charisee96113f32023-01-26 09:00:42 +0000355 let vm_context = GLOBAL_SERVICE.allocateGlobalVmContext(requester_debug_pid)?;
David Brazdild4f51a52023-01-11 14:09:27 +0000356 let cid = vm_context.getCid()? as Cid;
357 let temp_dir: PathBuf = vm_context.getTemporaryDirectory()?.into();
David Brazdil8cf8f482022-11-23 14:21:26 +0000358 let service = VirtualMachineService::new_binder(self.state.clone(), cid).as_binder();
359
360 // Start VM service listening for connections from the new CID on port=CID.
David Brazdil8cf8f482022-11-23 14:21:26 +0000361 let port = cid;
David Brazdil3238da42022-11-18 10:04:51 +0000362 match RpcServer::new_vsock(service, cid, port) {
David Brazdil8cf8f482022-11-23 14:21:26 +0000363 Ok(vm_server) => {
364 vm_server.start();
David Brazdild4f51a52023-01-11 14:09:27 +0000365 return Ok((VmContext::new(vm_context, vm_server), cid, temp_dir));
David Brazdil8cf8f482022-11-23 14:21:26 +0000366 }
367 Err(err) => {
368 warn!("Could not start RpcServer on port {}: {}", port, err);
369 }
370 }
371 }
Jiyong Park2227eaa2023-08-04 11:59:18 +0900372 Err(anyhow!("Too many attempts to create VM context failed"))
373 .or_service_specific_exception(-1)
David Brazdil8cf8f482022-11-23 14:21:26 +0000374 }
375
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000376 fn create_vm_internal(
377 &self,
378 config: &VirtualMachineConfig,
Jiyong Parke6fb1672023-06-26 16:45:55 +0900379 console_out_fd: Option<&ParcelFileDescriptor>,
380 console_in_fd: Option<&ParcelFileDescriptor>,
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000381 log_fd: Option<&ParcelFileDescriptor>,
382 is_protected: &mut bool,
383 ) -> binder::Result<Strong<dyn IVirtualMachine>> {
David Brazdil209074a2023-01-12 16:44:51 +0000384 let requester_uid = get_calling_uid();
385 let requester_debug_pid = get_calling_pid();
386
Nikita Ioffe631717e2023-09-05 13:38:07 +0100387 check_config_features(config)?;
388
David Brazdil209074a2023-01-12 16:44:51 +0000389 // Allocating VM context checks the MANAGE_VIRTUAL_MACHINE permission.
390 let (vm_context, cid, temporary_directory) = self.create_vm_context(requester_debug_pid)?;
Inseob Kim1119d702022-05-02 18:01:58 +0900391
Alan Stokesfda70842023-12-20 17:50:14 +0000392 if is_custom_config(config) {
Alan Stokes7bc146c2022-10-20 17:10:32 +0100393 check_use_custom_virtual_machine()?;
Inseob Kim1119d702022-05-02 18:01:58 +0900394 }
395
Nikita Ioffe5776f082023-02-10 21:38:26 +0000396 let gdb_port = extract_gdb_port(config);
397
398 // Additional permission checks if caller request gdb.
399 if gdb_port.is_some() {
400 check_gdb_allowed(config)?;
401 }
402
Nikita Ioffeb2c6f3a2024-06-06 12:51:53 +0000403 let device_tree_overlay = maybe_create_device_tree_overlay(config, &temporary_directory)?;
Seungjae Yooec3bc522023-11-09 10:14:30 +0900404
Jaewan Kimf3143242024-03-15 06:56:31 +0000405 let debug_config = DebugConfig::new(config);
Nikita Ioffe2cb75cf2024-06-04 16:43:56 +0000406 let ramdump = if !uses_gki_kernel(config) && debug_config.is_ramdump_needed() {
Jiyong Parked180932023-02-24 19:55:41 +0900407 Some(prepare_ramdump_file(&temporary_directory)?)
408 } else {
409 None
410 };
411
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000412 let state = &mut *self.state.lock().unwrap();
Jiyong Parke6fb1672023-06-26 16:45:55 +0900413 let console_out_fd =
414 clone_or_prepare_logger_fd(&debug_config, console_out_fd, format!("Console({})", cid))?;
415 let console_in_fd = console_in_fd.map(clone_file).transpose()?;
Jaewan Kim61f86142023-03-28 15:12:52 +0900416 let log_fd = clone_or_prepare_logger_fd(&debug_config, log_fd, format!("Log({})", cid))?;
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000417
418 // Counter to generate unique IDs for temporary image files.
419 let mut next_temporary_image_id = 0;
420 // Files which are referred to from composite images. These must be mapped to the crosvm
421 // child process, and not closed before it is started.
422 let mut indirect_files = vec![];
423
Alan Stokes7bc146c2022-10-20 17:10:32 +0100424 let (is_app_config, config) = match config {
425 VirtualMachineConfig::RawConfig(config) => (false, BorrowedOrOwned::Borrowed(config)),
426 VirtualMachineConfig::AppConfig(config) => {
Jiyong Park2227eaa2023-08-04 11:59:18 +0900427 let config = load_app_config(config, &debug_config, &temporary_directory)
428 .or_service_specific_exception_with(-1, |e| {
Jaewan Kim61f86142023-03-28 15:12:52 +0900429 *is_protected = config.protectedVm;
430 let message = format!("Failed to load app config: {:?}", e);
431 error!("{}", message);
Jiyong Park2227eaa2023-08-04 11:59:18 +0900432 message
Jaewan Kim61f86142023-03-28 15:12:52 +0900433 })?;
Alan Stokes7bc146c2022-10-20 17:10:32 +0100434 (true, BorrowedOrOwned::Owned(config))
435 }
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000436 };
437 let config = config.as_ref();
438 *is_protected = config.protectedVm;
439
440 // Check if partition images are labeled incorrectly. This is to prevent random images
441 // which are not protected by the Android Verified Boot (e.g. bits downloaded by apps) from
Alan Stokes3e5eec12023-09-07 12:10:00 +0100442 // being loaded in a pVM. This applies to everything but the instance image in the raw
443 // config, and everything but the non-executable, generated partitions in the app
444 // config.
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000445 config
446 .disks
447 .iter()
448 .flat_map(|disk| disk.partitions.iter())
449 .filter(|partition| {
450 if is_app_config {
Alan Stokes53cc5ca2022-08-30 14:28:19 +0100451 !is_safe_app_partition(&partition.label)
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000452 } else {
Alice Wangc206b9b2023-08-28 14:13:51 +0000453 !is_safe_raw_partition(&partition.label)
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000454 }
455 })
456 .try_for_each(check_label_for_partition)
Jiyong Park2227eaa2023-08-04 11:59:18 +0900457 .or_service_specific_exception(-1)?;
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000458
Inseob Kimff48a7c2024-02-26 22:07:21 +0900459 // Check if files for payloads and bases are NOT coming from /vendor and /odm, as they may
460 // have unstable interfaces.
461 // TODO(b/316431494): remove once Treble interfaces are stabilized.
462 check_partitions_for_files(config).or_service_specific_exception(-1)?;
463
Alan Stokes185fe112023-01-10 16:20:55 +0000464 let kernel = maybe_clone_file(&config.kernel)?;
465 let initrd = maybe_clone_file(&config.initrd)?;
466
Alan Stokes185fe112023-01-10 16:20:55 +0000467 if config.protectedVm {
Nikita Ioffef934e812024-07-05 15:44:41 +0000468 // In a protected VM, we require custom kernels to come from a trusted source
469 // (b/237054515).
Jiyong Park2227eaa2023-08-04 11:59:18 +0900470 check_label_for_kernel_files(&kernel, &initrd).or_service_specific_exception(-1)?;
Nikita Ioffef934e812024-07-05 15:44:41 +0000471 // Fail fast with a meaningful error message in case device doesn't support pVMs.
472 check_protected_vm_is_supported()?;
Alan Stokes185fe112023-01-10 16:20:55 +0000473 }
474
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000475 let zero_filler_path = temporary_directory.join("zero.img");
Jiyong Park2227eaa2023-08-04 11:59:18 +0900476 write_zero_filler(&zero_filler_path)
477 .context("Failed to make composite image")
478 .with_log()
479 .or_service_specific_exception(-1)?;
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000480
481 // Assemble disk images if needed.
482 let disks = config
483 .disks
484 .iter()
485 .map(|disk| {
486 assemble_disk_image(
487 disk,
488 &zero_filler_path,
489 &temporary_directory,
490 &mut next_temporary_image_id,
491 &mut indirect_files,
492 )
493 })
494 .collect::<Result<Vec<DiskFile>, _>>()?;
495
David Brazdil7d1e5ec2023-02-06 17:56:29 +0000496 let (cpus, host_cpu_topology) = match config.cpuTopology {
497 CpuTopology::MATCH_HOST => (None, true),
498 CpuTopology::ONE_CPU => (NonZeroU32::new(1), false),
499 val => {
Jiyong Park2227eaa2023-08-04 11:59:18 +0900500 return Err(anyhow!("Failed to parse CPU topology value {:?}", val))
501 .with_log()
502 .or_service_specific_exception(-1);
David Brazdil7d1e5ec2023-02-06 17:56:29 +0000503 }
504 };
505
David Brazdil2dfefd12023-11-17 14:07:36 +0000506 let (vfio_devices, dtbo) = if !config.devices.is_empty() {
Inseob Kim6ef80972023-07-20 17:23:36 +0900507 let mut set = HashSet::new();
508 for device in config.devices.iter() {
Jiyong Park2227eaa2023-08-04 11:59:18 +0900509 let path = canonicalize(device)
510 .with_context(|| format!("can't canonicalize {device}"))
511 .or_service_specific_exception(-1)?;
Inseob Kim6ef80972023-07-20 17:23:36 +0900512 if !set.insert(path) {
Jiyong Park2227eaa2023-08-04 11:59:18 +0900513 return Err(anyhow!("duplicated device {device}"))
514 .or_binder_exception(ExceptionCode::ILLEGAL_ARGUMENT);
Inseob Kim6ef80972023-07-20 17:23:36 +0900515 }
516 }
Jakob Vukalovicd42aa2c2023-11-09 16:04:00 +0000517 let devices = GLOBAL_SERVICE.bindDevicesToVfioDriver(&config.devices)?;
David Brazdil2dfefd12023-11-17 14:07:36 +0000518 let dtbo_file = File::from(
519 GLOBAL_SERVICE
520 .getDtboFile()?
521 .as_ref()
522 .try_clone()
Jaewan Kim39952072024-01-19 17:04:53 +0900523 .context("Failed to create VM DTBO from ParcelFileDescriptor")
David Brazdil2dfefd12023-11-17 14:07:36 +0000524 .or_binder_exception(ExceptionCode::BAD_PARCELABLE)?,
525 );
526 (devices, Some(dtbo_file))
Inseob Kim7307a892023-09-14 13:37:58 +0900527 } else {
David Brazdil2dfefd12023-11-17 14:07:36 +0000528 (vec![], None)
Inseob Kim7307a892023-09-14 13:37:58 +0900529 };
Jeongik Cha798401c2024-04-11 21:54:49 +0900530 let display_config = if cfg!(paravirtualized_devices) {
531 config
532 .displayConfig
533 .as_ref()
534 .map(DisplayConfig::new)
535 .transpose()
536 .or_binder_exception(ExceptionCode::ILLEGAL_ARGUMENT)?
537 } else {
538 None
539 };
Jason Macnakea7b0fa2024-05-08 17:02:44 -0700540 let gpu_config = if cfg!(paravirtualized_devices) {
541 config
542 .gpuConfig
543 .as_ref()
544 .map(GpuConfig::new)
545 .transpose()
546 .or_binder_exception(ExceptionCode::ILLEGAL_ARGUMENT)?
547 } else {
548 None
549 };
Jeongik Cha1df1c032024-04-03 16:03:12 +0900550
Jeongik Chac181be72024-03-27 00:18:30 +0900551 let input_device_options = if cfg!(paravirtualized_devices) {
552 config
553 .inputDevices
554 .iter()
555 .map(to_input_device_option_from)
556 .collect::<Result<Vec<InputDeviceOption>, _>>()
557 .or_binder_exception(ExceptionCode::ILLEGAL_ARGUMENT)?
558 } else {
559 vec![]
560 };
561
Seungjae Yoo13af0b62024-05-20 14:15:13 +0900562 // Create TAP network interface if the VM supports network.
Seungjae Yoo986d7a42024-05-28 14:06:58 +0900563 let tap = if cfg!(network) && config.networkSupported {
Seungjae Yoo13af0b62024-05-20 14:15:13 +0900564 if *is_protected {
565 return Err(anyhow!("Network feature is not supported for pVM yet"))
566 .with_log()
567 .or_binder_exception(ExceptionCode::UNSUPPORTED_OPERATION)?;
568 }
Seungjae Yoo986d7a42024-05-28 14:06:58 +0900569 Some(File::from(
570 GLOBAL_SERVICE
571 .createTapInterface(&get_this_pid().to_string())?
572 .as_ref()
573 .try_clone()
574 .context("Failed to get TAP interface from ParcelFileDescriptor")
575 .or_binder_exception(ExceptionCode::BAD_PARCELABLE)?,
576 ))
Seungjae Yoo13af0b62024-05-20 14:15:13 +0900577 } else {
578 None
579 };
Mu-Le Leeb2d5a312024-06-05 10:52:47 +0800580
581 let audio_config = if cfg!(paravirtualized_devices) {
582 config.audioConfig.as_ref().map(AudioConfig::new)
583 } else {
584 None
585 };
Seungjae Yoo13af0b62024-05-20 14:15:13 +0900586
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000587 // Actually start the VM.
588 let crosvm_config = CrosvmConfig {
589 cid,
Seungjae Yoo62085c02022-08-12 04:44:52 +0000590 name: config.name.clone(),
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000591 bootloader: maybe_clone_file(&config.bootloader)?,
Alan Stokes185fe112023-01-10 16:20:55 +0000592 kernel,
593 initrd,
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000594 disks,
595 params: config.params.to_owned(),
596 protected: *is_protected,
Jaewan Kim61f86142023-03-28 15:12:52 +0900597 debug_config,
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000598 memory_mib: config.memoryMib.try_into().ok().and_then(NonZeroU32::new),
David Brazdil7d1e5ec2023-02-06 17:56:29 +0000599 cpus,
600 host_cpu_topology,
Jiyong Parke6fb1672023-06-26 16:45:55 +0900601 console_out_fd,
602 console_in_fd,
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000603 log_fd,
Jiyong Parked180932023-02-24 19:55:41 +0900604 ramdump,
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000605 indirect_files,
606 platform_version: parse_platform_version_req(&config.platformVersion)?,
Jiyong Parke6ed0f92022-06-22 00:13:00 +0900607 detect_hangup: is_app_config,
Nikita Ioffe5776f082023-02-10 21:38:26 +0000608 gdb_port,
Inseob Kim7307a892023-09-14 13:37:58 +0900609 vfio_devices,
David Brazdil2dfefd12023-11-17 14:07:36 +0000610 dtbo,
Shikha Panwar55e10ec2024-02-13 12:53:49 +0000611 device_tree_overlay,
Jeongik Cha1df1c032024-04-03 16:03:12 +0900612 display_config,
Jeongik Chac181be72024-03-27 00:18:30 +0900613 input_device_options,
Vincent Donnefort538a2c62024-03-20 16:01:10 +0000614 hugepages: config.hugePages,
Seungjae Yoo986d7a42024-05-28 14:06:58 +0900615 tap,
Yi-Yo Chiang8dd32552024-05-22 19:38:16 +0800616 console_input_device: config.consoleInputDevice.clone(),
David Dai23cff712024-06-13 19:23:45 +0000617 boost_uclamp: config.boostUclamp,
Jason Macnakea7b0fa2024-05-08 17:02:44 -0700618 gpu_config,
Mu-Le Leeb2d5a312024-06-05 10:52:47 +0800619 audio_config,
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000620 };
621 let instance = Arc::new(
David Brazdil528e0472022-10-10 15:06:02 +0100622 VmInstance::new(
623 crosvm_config,
624 temporary_directory,
625 requester_uid,
626 requester_debug_pid,
627 vm_context,
628 )
Jiyong Park2227eaa2023-08-04 11:59:18 +0900629 .with_context(|| format!("Failed to create VM with config {:?}", config))
630 .with_log()
631 .or_service_specific_exception(-1)?,
Shikha Panwar061aa2c2022-04-05 12:52:56 +0000632 );
633 state.add_vm(Arc::downgrade(&instance));
634 Ok(VirtualMachine::create(instance))
635 }
Jiyong Park8611a6c2021-07-09 18:17:44 +0900636}
637
Alan Stokesfda70842023-12-20 17:50:14 +0000638/// Returns whether a VM config represents a "custom" virtual machine, which requires the
639/// USE_CUSTOM_VIRTUAL_MACHINE.
640fn is_custom_config(config: &VirtualMachineConfig) -> bool {
641 match config {
642 // Any raw (non-Microdroid) VM is considered custom.
643 VirtualMachineConfig::RawConfig(_) => true,
644 VirtualMachineConfig::AppConfig(config) => {
645 // Some features are reserved for platform apps only, even when using
646 // VirtualMachineAppConfig. Almost all of these features are grouped in the
647 // CustomConfig struct:
648 // - controlling CPUs;
649 // - gdbPort is set, meaning that crosvm will start a gdb server;
650 // - using anything other than the default kernel;
651 // - specifying devices to be assigned.
652 if config.customConfig.is_some() {
653 true
654 } else {
655 // Additional custom features not included in CustomConfig:
656 // - specifying a config file;
Alan Stokes736f6822024-02-16 16:08:00 +0000657 // - specifying extra APKs;
658 // - specifying an OS other than Microdroid.
Inseob Kim89b24592024-02-23 18:59:43 +0900659 (match &config.payload {
Alan Stokesfda70842023-12-20 17:50:14 +0000660 Payload::ConfigPath(_) => true,
Inseob Kim89b24592024-02-23 18:59:43 +0900661 Payload::PayloadConfig(payload_config) => !payload_config.extraApks.is_empty(),
662 }) || config.osName != MICRODROID_OS_NAME
Alan Stokesfda70842023-12-20 17:50:14 +0000663 }
664 }
665 }
666}
667
Seungjae Yoo14e60182024-02-21 13:28:31 +0900668fn extract_vendor_hashtree_digest(config: &VirtualMachineConfig) -> Result<Option<Vec<u8>>> {
669 let VirtualMachineConfig::AppConfig(config) = config else {
670 return Ok(None);
671 };
672 let Some(custom_config) = &config.customConfig else {
673 return Ok(None);
674 };
675 let Some(file) = custom_config.vendorImage.as_ref() else {
676 return Ok(None);
677 };
678
679 let file = clone_file(file)?;
680 let size =
681 file.metadata().context("Failed to get metadata from microdroid vendor image")?.len();
682 let vbmeta = VbMetaImage::verify_reader_region(&file, 0, size)
683 .context("Failed to get vbmeta from microdroid-vendor.img")?;
684
685 for descriptor in vbmeta.descriptors()?.iter() {
686 if let vbmeta::Descriptor::Hashtree(_) = descriptor {
687 let root_digest = hex::encode(descriptor.to_hashtree()?.root_digest());
688 return Ok(Some(root_digest.as_bytes().to_vec()));
689 }
690 }
691 Err(anyhow!("No hashtree digest is extracted from microdroid vendor image"))
692}
693
Nikita Ioffeb2c6f3a2024-06-06 12:51:53 +0000694fn maybe_create_device_tree_overlay(
695 config: &VirtualMachineConfig,
696 temporary_directory: &Path,
697) -> binder::Result<Option<File>> {
698 // Currently, VirtMgr adds the host copy of reference DT & untrusted properties
699 // (e.g. instance-id)
700 let host_ref_dt = Path::new(VM_REFERENCE_DT_ON_HOST_PATH);
701 let host_ref_dt = if host_ref_dt.exists()
702 && read_dir(host_ref_dt).or_service_specific_exception(-1)?.next().is_some()
703 {
704 Some(host_ref_dt)
705 } else {
706 warn!("VM reference DT doesn't exist in host DT");
707 None
708 };
709
710 let vendor_hashtree_digest = extract_vendor_hashtree_digest(config)
711 .context("Failed to extract vendor hashtree digest")
712 .or_service_specific_exception(-1)?;
713
714 let trusted_props = if let Some(ref vendor_hashtree_digest) = vendor_hashtree_digest {
715 info!(
716 "Passing vendor hashtree digest to pvmfw. This will be rejected if it doesn't \
717 match the trusted digest in the pvmfw config, causing the VM to fail to start."
718 );
719 vec![(cstr!("vendor_hashtree_descriptor_root_digest"), vendor_hashtree_digest.as_slice())]
720 } else {
721 vec![]
722 };
723
724 let instance_id;
725 let mut untrusted_props = Vec::with_capacity(2);
726 if cfg!(llpvm_changes) {
727 instance_id = extract_instance_id(config);
728 untrusted_props.push((cstr!("instance-id"), &instance_id[..]));
729 let want_updatable = extract_want_updatable(config);
730 if want_updatable && is_secretkeeper_supported() {
731 // Let guest know that it can defer rollback protection to Secretkeeper by setting
732 // an empty property in untrusted node in DT. This enables Updatable VMs.
733 untrusted_props.push((cstr!("defer-rollback-protection"), &[]))
734 }
735 }
736
737 let device_tree_overlay = if host_ref_dt.is_some()
738 || !untrusted_props.is_empty()
739 || !trusted_props.is_empty()
740 {
741 let dt_output = temporary_directory.join(VM_DT_OVERLAY_PATH);
742 let mut data = [0_u8; VM_DT_OVERLAY_MAX_SIZE];
743 let fdt =
744 create_device_tree_overlay(&mut data, host_ref_dt, &untrusted_props, &trusted_props)
745 .map_err(|e| anyhow!("Failed to create DT overlay, {e:?}"))
746 .or_service_specific_exception(-1)?;
747 fs::write(&dt_output, fdt.as_slice()).or_service_specific_exception(-1)?;
748 Some(File::open(dt_output).or_service_specific_exception(-1)?)
749 } else {
750 None
751 };
752 Ok(device_tree_overlay)
753}
754
Andrew Walbranfbb39d22021-07-28 17:01:25 +0000755fn write_zero_filler(zero_filler_path: &Path) -> Result<()> {
Jooyung Han95884632021-07-06 22:27:54 +0900756 let file = OpenOptions::new()
757 .create_new(true)
758 .read(true)
759 .write(true)
760 .open(zero_filler_path)
761 .with_context(|| "Failed to create zero.img")?;
762 file.set_len(ZERO_FILLER_SIZE)?;
Andrew Walbranfbb39d22021-07-28 17:01:25 +0000763 Ok(())
Jooyung Han95884632021-07-06 22:27:54 +0900764}
765
David Brazdilf50c7a62023-04-19 14:22:42 +0000766fn format_as_android_vm_instance(part: &mut dyn Write) -> std::io::Result<()> {
767 part.write_all(ANDROID_VM_INSTANCE_MAGIC.as_bytes())?;
768 part.write_all(&ANDROID_VM_INSTANCE_VERSION.to_le_bytes())?;
769 part.flush()
770}
771
772fn format_as_encryptedstore(part: &mut dyn Write) -> std::io::Result<()> {
773 part.write_all(UNFORMATTED_STORAGE_MAGIC.as_bytes())?;
774 part.flush()
775}
776
777fn round_up(input: u64, granularity: u64) -> u64 {
778 if granularity == 0 {
779 return input;
780 }
781 // If the input is absurdly large we round down instead of up; it's going to fail anyway.
782 let result = input.checked_add(granularity - 1).unwrap_or(input);
783 (result / granularity) * granularity
784}
785
Jeongik Chac181be72024-03-27 00:18:30 +0900786fn to_input_device_option_from(input_device: &InputDevice) -> Result<InputDeviceOption> {
787 Ok(match input_device {
788 InputDevice::SingleTouch(single_touch) => InputDeviceOption::SingleTouch {
789 file: clone_file(single_touch.pfd.as_ref().ok_or(anyhow!("pfd should have value"))?)?,
790 height: u32::try_from(single_touch.height)?,
791 width: u32::try_from(single_touch.width)?,
792 name: if !single_touch.name.is_empty() {
793 Some(single_touch.name.clone())
794 } else {
795 None
796 },
797 },
798 InputDevice::EvDev(evdev) => InputDeviceOption::EvDev(clone_file(
799 evdev.pfd.as_ref().ok_or(anyhow!("pfd should have value"))?,
800 )?),
Jeongik Cha985b6402024-04-15 18:13:00 +0900801 InputDevice::Keyboard(keyboard) => InputDeviceOption::Keyboard(clone_file(
802 keyboard.pfd.as_ref().ok_or(anyhow!("pfd should have value"))?,
803 )?),
Jeongik Cha343894e2024-05-17 20:39:31 +0900804 InputDevice::Mouse(mouse) => InputDeviceOption::Mouse(clone_file(
805 mouse.pfd.as_ref().ok_or(anyhow!("pfd should have value"))?,
806 )?),
Jiyong Park6c1b9f02024-07-05 16:21:10 +0900807 InputDevice::Switches(switches) => InputDeviceOption::Switches(clone_file(
808 switches.pfd.as_ref().ok_or(anyhow!("pfd should have value"))?,
809 )?),
Jeongik Cha112a2682024-07-09 12:28:45 +0900810 InputDevice::Trackpad(trackpad) => InputDeviceOption::MultiTouchTrackpad {
811 file: clone_file(trackpad.pfd.as_ref().ok_or(anyhow!("pfd should have value"))?)?,
812 height: u32::try_from(trackpad.height)?,
813 width: u32::try_from(trackpad.width)?,
814 name: if !trackpad.name.is_empty() { Some(trackpad.name.clone()) } else { None },
815 },
Jeongik Chac181be72024-03-27 00:18:30 +0900816 })
817}
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000818/// Given the configuration for a disk image, assembles the `DiskFile` to pass to crosvm.
819///
820/// This may involve assembling a composite disk from a set of partition images.
821fn assemble_disk_image(
822 disk: &DiskImage,
Jooyung Han95884632021-07-06 22:27:54 +0900823 zero_filler_path: &Path,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000824 temporary_directory: &Path,
825 next_temporary_image_id: &mut u64,
826 indirect_files: &mut Vec<File>,
Andrew Walbran806f1542021-06-10 14:07:12 +0000827) -> Result<DiskFile, Status> {
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000828 let image = if !disk.partitions.is_empty() {
829 if disk.image.is_some() {
830 warn!("DiskImage {:?} contains both image and partitions.", disk);
Jiyong Park2227eaa2023-08-04 11:59:18 +0900831 return Err(anyhow!("DiskImage contains both image and partitions"))
832 .or_binder_exception(ExceptionCode::ILLEGAL_ARGUMENT);
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000833 }
834
Andrew Walbran3eca16c2021-06-14 11:15:14 +0000835 let composite_image_filenames =
836 make_composite_image_filenames(temporary_directory, next_temporary_image_id);
837 let (image, partition_files) = make_composite_image(
838 &disk.partitions,
Jooyung Han95884632021-07-06 22:27:54 +0900839 zero_filler_path,
Andrew Walbran3eca16c2021-06-14 11:15:14 +0000840 &composite_image_filenames.composite,
841 &composite_image_filenames.header,
842 &composite_image_filenames.footer,
843 )
Jiyong Park2227eaa2023-08-04 11:59:18 +0900844 .with_context(|| format!("Failed to make composite disk image with config {:?}", disk))
845 .with_log()
846 .or_service_specific_exception(-1)?;
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000847
848 // Pass the file descriptors for the various partition files to crosvm when it
849 // is run.
850 indirect_files.extend(partition_files);
851
852 image
853 } else if let Some(image) = &disk.image {
854 clone_file(image)?
855 } else {
856 warn!("DiskImage {:?} didn't contain image or partitions.", disk);
Jiyong Park2227eaa2023-08-04 11:59:18 +0900857 return Err(anyhow!("DiskImage didn't contain image or partitions."))
858 .or_binder_exception(ExceptionCode::ILLEGAL_ARGUMENT);
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +0000859 };
860
861 Ok(DiskFile { image, writable: disk.writable })
862}
863
Nikita Ioffeaa6858c2023-07-04 01:37:41 +0100864fn append_kernel_param(param: &str, vm_config: &mut VirtualMachineRawConfig) {
865 if let Some(ref mut params) = vm_config.params {
866 params.push(' ');
867 params.push_str(param)
868 } else {
869 vm_config.params = Some(param.to_owned())
870 }
871}
872
Inseob Kim46257382024-01-03 15:41:22 +0900873fn extract_os_name_from_config_path(config: &Path) -> Option<String> {
874 if config.extension()?.to_str()? != "json" {
875 return None;
Inseob Kim172f9eb2023-11-06 17:02:08 +0900876 }
Inseob Kim46257382024-01-03 15:41:22 +0900877
878 Some(config.with_extension("").file_name()?.to_str()?.to_owned())
879}
880
881fn extract_os_names_from_configs(config_glob_pattern: &str) -> Result<HashSet<String>> {
882 let configs = glob(config_glob_pattern)?.collect::<Result<Vec<_>, _>>()?;
883 let os_names =
884 configs.iter().filter_map(|x| extract_os_name_from_config_path(x)).collect::<HashSet<_>>();
885
886 Ok(os_names)
887}
888
889fn get_supported_os_names() -> Result<HashSet<String>> {
890 if !cfg!(vendor_modules) {
891 return Ok(iter::once(MICRODROID_OS_NAME.to_owned()).collect());
892 }
893
894 extract_os_names_from_configs("/apex/com.android.virt/etc/microdroid*.json")
895}
896
897fn is_valid_os(os_name: &str) -> bool {
898 SUPPORTED_OS_NAMES.contains(os_name)
Inseob Kim172f9eb2023-11-06 17:02:08 +0900899}
900
Nikita Ioffe2cb75cf2024-06-04 16:43:56 +0000901fn uses_gki_kernel(config: &VirtualMachineConfig) -> bool {
902 if !cfg!(vendor_modules) {
903 return false;
904 }
905 match config {
906 VirtualMachineConfig::RawConfig(_) => false,
907 VirtualMachineConfig::AppConfig(config) => config.osName.starts_with("microdroid_gki-"),
908 }
909}
910
Jooyung Han21e9b922021-06-26 04:14:16 +0900911fn load_app_config(
912 config: &VirtualMachineAppConfig,
Jaewan Kim61f86142023-03-28 15:12:52 +0900913 debug_config: &DebugConfig,
Jooyung Han21e9b922021-06-26 04:14:16 +0900914 temporary_directory: &Path,
Jooyung Hanadfb76c2021-06-28 17:29:30 +0900915) -> Result<VirtualMachineRawConfig> {
Andrew Walbrancc0db522021-07-12 17:03:42 +0000916 let apk_file = clone_file(config.apk.as_ref().unwrap())?;
917 let idsig_file = clone_file(config.idsig.as_ref().unwrap())?;
Jiyong Park8d081812021-07-23 17:45:04 +0900918 let instance_file = clone_file(config.instanceImage.as_ref().unwrap())?;
Jooyung Han21e9b922021-06-26 04:14:16 +0900919
Shikha Panwar22e70452022-10-10 18:32:55 +0000920 let storage_image = if let Some(file) = config.encryptedStorageImage.as_ref() {
921 Some(clone_file(file)?)
922 } else {
923 None
924 };
925
Alan Stokesfda70842023-12-20 17:50:14 +0000926 let vm_payload_config;
927 let extra_apk_files: Vec<_>;
928 match &config.payload {
Alan Stokes0d1ef782022-09-27 13:46:35 +0100929 Payload::ConfigPath(config_path) => {
Alan Stokesfda70842023-12-20 17:50:14 +0000930 vm_payload_config =
931 load_vm_payload_config_from_file(&apk_file, config_path.as_str())
932 .with_context(|| format!("Couldn't read config from {}", config_path))?;
933 extra_apk_files = vm_payload_config
934 .extra_apks
935 .iter()
936 .enumerate()
937 .map(|(i, apk)| {
938 File::open(PathBuf::from(&apk.path))
939 .with_context(|| format!("Failed to open extra apk #{i} {}", apk.path))
940 })
941 .collect::<Result<_>>()?;
Alan Stokes0d1ef782022-09-27 13:46:35 +0100942 }
Alan Stokesfda70842023-12-20 17:50:14 +0000943 Payload::PayloadConfig(payload_config) => {
944 vm_payload_config = create_vm_payload_config(payload_config)?;
945 extra_apk_files =
946 payload_config.extraApks.iter().map(clone_file).collect::<binder::Result<_>>()?;
947 }
Alan Stokes0d1ef782022-09-27 13:46:35 +0100948 };
Jooyung Han21e9b922021-06-26 04:14:16 +0900949
Inseob Kim89b24592024-02-23 18:59:43 +0900950 let payload_config_os = vm_payload_config.os.name.as_str();
951 if !payload_config_os.is_empty() && payload_config_os != "microdroid" {
952 bail!("'os' in payload config is deprecated");
953 }
954
Inseob Kim172f9eb2023-11-06 17:02:08 +0900955 // For now, the only supported OS is Microdroid and Microdroid GKI
Inseob Kim89b24592024-02-23 18:59:43 +0900956 let os_name = config.osName.as_str();
Inseob Kim172f9eb2023-11-06 17:02:08 +0900957 if !is_valid_os(os_name) {
Andrew Walbrancc0db522021-07-12 17:03:42 +0000958 bail!("Unknown OS \"{}\"", os_name);
Jooyung Han35edb8f2021-07-01 16:17:16 +0900959 }
Andrew Walbrancc0db522021-07-12 17:03:42 +0000960
961 // It is safe to construct a filename based on the os_name because we've already checked that it
962 // is one of the allowed values.
Jooyung Han21e9b922021-06-26 04:14:16 +0900963 let vm_config_path = PathBuf::from(format!("/apex/com.android.virt/etc/{}.json", os_name));
964 let vm_config_file = File::open(vm_config_path)?;
Andrew Walbrancc0db522021-07-12 17:03:42 +0000965 let mut vm_config = VmConfig::load(&vm_config_file)?.to_parcelable()?;
Jooyung Han21e9b922021-06-26 04:14:16 +0900966
Nikita Ioffea0eb5ee2023-06-26 18:18:21 +0100967 if let Some(custom_config) = &config.customConfig {
968 if let Some(file) = custom_config.customKernelImage.as_ref() {
969 vm_config.kernel = Some(ParcelFileDescriptor::new(clone_file(file)?))
970 }
Nikita Ioffea0eb5ee2023-06-26 18:18:21 +0100971 vm_config.gdbPort = custom_config.gdbPort;
Nikita Ioffe5dfddf22023-06-29 16:11:26 +0100972
973 if let Some(file) = custom_config.vendorImage.as_ref() {
Nikita Ioffeaa6858c2023-07-04 01:37:41 +0100974 add_microdroid_vendor_image(clone_file(file)?, &mut vm_config);
975 append_kernel_param("androidboot.microdroid.mount_vendor=1", &mut vm_config)
Nikita Ioffe5dfddf22023-06-29 16:11:26 +0100976 }
Inseob Kim6ef80972023-07-20 17:23:36 +0900977
Chris Wailes6177c662024-05-09 14:37:44 -0700978 vm_config.devices.clone_from(&custom_config.devices);
Seungjae Yoo13af0b62024-05-20 14:15:13 +0900979 vm_config.networkSupported = custom_config.networkSupported;
Nikita Ioffe26c35ed2023-06-05 17:49:08 +0100980 }
981
Andrew Walbrancc045902021-07-27 16:06:17 +0000982 if config.memoryMib > 0 {
983 vm_config.memoryMib = config.memoryMib;
Andrew Walbran45bcb0c2021-07-14 15:02:06 +0000984 }
985
Chris Wailes6177c662024-05-09 14:37:44 -0700986 vm_config.name.clone_from(&config.name);
Andrew Walbran3994f002022-01-27 17:33:45 +0000987 vm_config.protectedVm = config.protectedVm;
David Brazdil7d1e5ec2023-02-06 17:56:29 +0000988 vm_config.cpuTopology = config.cpuTopology;
Vincent Donnefort538a2c62024-03-20 16:01:10 +0000989 vm_config.hugePages = config.hugePages || vm_payload_config.hugepages;
David Dai23cff712024-06-13 19:23:45 +0000990 vm_config.boostUclamp = config.boostUclamp;
Jiyong Park032615f2022-01-10 13:55:34 +0900991
Shikha Panwar22e70452022-10-10 18:32:55 +0000992 // Microdroid takes additional init ramdisk & (optionally) storage image
Inseob Kim172f9eb2023-11-06 17:02:08 +0900993 add_microdroid_system_images(config, instance_file, storage_image, os_name, &mut vm_config)?;
Shikha Panwar22e70452022-10-10 18:32:55 +0000994
995 // Include Microdroid payload disk (contains apks, idsigs) in vm config
996 add_microdroid_payload_images(
Alan Stokes0d1ef782022-09-27 13:46:35 +0100997 config,
Jaewan Kim61f86142023-03-28 15:12:52 +0900998 debug_config,
Alan Stokes0d1ef782022-09-27 13:46:35 +0100999 temporary_directory,
1000 apk_file,
1001 idsig_file,
Alan Stokesfda70842023-12-20 17:50:14 +00001002 extra_apk_files,
Alan Stokes0d1ef782022-09-27 13:46:35 +01001003 &vm_payload_config,
1004 &mut vm_config,
1005 )?;
Jooyung Han21e9b922021-06-26 04:14:16 +09001006
Andrew Walbrancc0db522021-07-12 17:03:42 +00001007 Ok(vm_config)
Jooyung Han21e9b922021-06-26 04:14:16 +09001008}
1009
Inseob Kimff48a7c2024-02-26 22:07:21 +09001010fn check_partition_for_file(fd: &ParcelFileDescriptor) -> Result<()> {
1011 let path = format!("/proc/self/fd/{}", fd.as_raw_fd());
1012 let link = fs::read_link(&path).context(format!("can't read_link {path}"))?;
1013
1014 // microdroid vendor image is OK
1015 if cfg!(vendor_modules) && link == Path::new("/vendor/etc/avf/microdroid/microdroid_vendor.img")
1016 {
1017 return Ok(());
1018 }
1019
1020 if link.starts_with("/vendor") || link.starts_with("/odm") {
1021 bail!("vendor or odm file {} can't be used for VM", link.display());
1022 }
1023
1024 Ok(())
1025}
1026
1027fn check_partitions_for_files(config: &VirtualMachineRawConfig) -> Result<()> {
1028 config
1029 .disks
1030 .iter()
1031 .flat_map(|disk| disk.partitions.iter())
1032 .filter_map(|partition| partition.image.as_ref())
1033 .try_for_each(check_partition_for_file)?;
1034
1035 config.kernel.as_ref().map_or(Ok(()), check_partition_for_file)?;
1036 config.initrd.as_ref().map_or(Ok(()), check_partition_for_file)?;
1037 config.bootloader.as_ref().map_or(Ok(()), check_partition_for_file)?;
1038
1039 Ok(())
1040}
1041
Alan Stokes0d1ef782022-09-27 13:46:35 +01001042fn load_vm_payload_config_from_file(apk_file: &File, config_path: &str) -> Result<VmPayloadConfig> {
1043 let mut apk_zip = ZipArchive::new(apk_file)?;
1044 let config_file = apk_zip.by_name(config_path)?;
1045 Ok(serde_json::from_reader(config_file)?)
1046}
1047
Alan Stokes8f12f2b2023-01-09 09:19:20 +00001048fn create_vm_payload_config(
1049 payload_config: &VirtualMachinePayloadConfig,
1050) -> Result<VmPayloadConfig> {
Alan Stokes0d1ef782022-09-27 13:46:35 +01001051 // There isn't an actual config file. Construct a synthetic VmPayloadConfig from the explicit
1052 // parameters we've been given. Microdroid will do something equivalent inside the VM using the
1053 // payload config that we send it via the metadata file.
Alan Stokes8f12f2b2023-01-09 09:19:20 +00001054
1055 let payload_binary_name = &payload_config.payloadBinaryName;
1056 if payload_binary_name.contains('/') {
1057 bail!("Payload binary name must not specify a path: {payload_binary_name}");
1058 }
1059
1060 let task = Task { type_: TaskType::MicrodroidLauncher, command: payload_binary_name.clone() };
Alan Stokesfda70842023-12-20 17:50:14 +00001061
1062 // The VM only cares about how many there are, these names are actually ignored.
1063 let extra_apk_count = payload_config.extraApks.len();
1064 let extra_apks =
1065 (0..extra_apk_count).map(|i| ApkConfig { path: format!("extra-apk-{i}") }).collect();
1066
Inseob Kim89b24592024-02-23 18:59:43 +09001067 Ok(VmPayloadConfig { task: Some(task), extra_apks, ..Default::default() })
Alan Stokes0d1ef782022-09-27 13:46:35 +01001068}
1069
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +00001070/// Generates a unique filename to use for a composite disk image.
Andrew Walbran3eca16c2021-06-14 11:15:14 +00001071fn make_composite_image_filenames(
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +00001072 temporary_directory: &Path,
1073 next_temporary_image_id: &mut u64,
Andrew Walbran3eca16c2021-06-14 11:15:14 +00001074) -> CompositeImageFilenames {
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +00001075 let id = *next_temporary_image_id;
1076 *next_temporary_image_id += 1;
Andrew Walbran3eca16c2021-06-14 11:15:14 +00001077 CompositeImageFilenames {
1078 composite: temporary_directory.join(format!("composite-{}.img", id)),
1079 header: temporary_directory.join(format!("composite-{}-header.img", id)),
1080 footer: temporary_directory.join(format!("composite-{}-footer.img", id)),
1081 }
1082}
1083
1084/// Filenames for a composite disk image, including header and footer partitions.
1085#[derive(Clone, Debug, Eq, PartialEq)]
1086struct CompositeImageFilenames {
1087 /// The composite disk image itself.
1088 composite: PathBuf,
1089 /// The header partition image.
1090 header: PathBuf,
1091 /// The footer partition image.
1092 footer: PathBuf,
Andrew Walbranf5fbb7d2021-05-12 17:15:48 +00001093}
1094
Jiyong Park753553b2021-07-12 21:21:09 +09001095/// Checks whether the caller has a specific permission
1096fn check_permission(perm: &str) -> binder::Result<()> {
David Brazdil1f530702022-10-03 12:18:10 +01001097 let calling_pid = get_calling_pid();
1098 let calling_uid = get_calling_uid();
Jiyong Park753553b2021-07-12 21:21:09 +09001099 // Root can do anything
1100 if calling_uid == 0 {
1101 return Ok(());
1102 }
1103 let perm_svc: Strong<dyn IPermissionController::IPermissionController> =
Frederick Mayleb2a02872024-05-08 13:35:12 -07001104 binder::wait_for_interface("permission")?;
Jiyong Park753553b2021-07-12 21:21:09 +09001105 if perm_svc.checkPermission(perm, calling_pid, calling_uid as i32)? {
Andrew Walbran806f1542021-06-10 14:07:12 +00001106 Ok(())
1107 } else {
Jiyong Park2227eaa2023-08-04 11:59:18 +09001108 Err(anyhow!("does not have the {} permission", perm))
1109 .or_binder_exception(ExceptionCode::SECURITY)
Andrew Walbran806f1542021-06-10 14:07:12 +00001110 }
Andrew Walbrand6dce6f2021-03-05 16:39:08 +00001111}
1112
Jiyong Park753553b2021-07-12 21:21:09 +09001113/// Check whether the caller of the current Binder method is allowed to manage VMs
1114fn check_manage_access() -> binder::Result<()> {
1115 check_permission("android.permission.MANAGE_VIRTUAL_MACHINE")
1116}
1117
Inseob Kim1119d702022-05-02 18:01:58 +09001118/// Check whether the caller of the current Binder method is allowed to create custom VMs
1119fn check_use_custom_virtual_machine() -> binder::Result<()> {
1120 check_permission("android.permission.USE_CUSTOM_VIRTUAL_MACHINE")
1121}
1122
Alan Stokes185fe112023-01-10 16:20:55 +00001123/// Return whether a partition is exempt from selinux label checks, because we know that it does
1124/// not contain code and is likely to be generated in an app-writable directory.
Alan Stokes53cc5ca2022-08-30 14:28:19 +01001125fn is_safe_app_partition(label: &str) -> bool {
Shikha Panwara2ff8c52022-11-30 19:25:46 +00001126 // See add_microdroid_system_images & add_microdroid_payload_images in payload.rs.
Alan Stokes53cc5ca2022-08-30 14:28:19 +01001127 label == "vm-instance"
Shikha Panwara2ff8c52022-11-30 19:25:46 +00001128 || label == "encryptedstore"
Alan Stokes53cc5ca2022-08-30 14:28:19 +01001129 || label == "microdroid-apk-idsig"
1130 || label == "payload-metadata"
1131 || label.starts_with("extra-idsig-")
1132}
1133
Alice Wangc206b9b2023-08-28 14:13:51 +00001134/// Returns whether a partition with the given label is safe for a raw config VM.
1135fn is_safe_raw_partition(label: &str) -> bool {
1136 label == "vm-instance"
1137}
1138
Alan Stokes185fe112023-01-10 16:20:55 +00001139/// Check that a file SELinux label is acceptable.
1140///
1141/// We only want to allow code in a VM to be sourced from places that apps, and the
Seungjae Yoo2b74c442023-11-15 18:05:07 +09001142/// system or vendor, do not have write access to.
Alan Stokes185fe112023-01-10 16:20:55 +00001143///
1144/// Note that sepolicy must also grant read access for these types to both virtualization
1145/// service and crosvm.
1146///
1147/// App private data files are deliberately excluded, to avoid arbitrary payloads being run on
1148/// user devices (W^X).
1149fn check_label_is_allowed(context: &SeContext) -> Result<()> {
1150 match context.selinux_type()? {
Alan Stokes53cc5ca2022-08-30 14:28:19 +01001151 | "apk_data_file" // APKs of an installed app
Alan Stokes53cc5ca2022-08-30 14:28:19 +01001152 | "shell_data_file" // test files created via adb shell
Alan Stokesfe4bb0c2023-03-20 14:15:36 +00001153 | "staging_data_file" // updated/staged APEX images
1154 | "system_file" // immutable dm-verity protected partition
1155 | "virtualizationservice_data_file" // files created by VS / VirtMgr
Seungjae Yoo2b74c442023-11-15 18:05:07 +09001156 | "vendor_microdroid_file" // immutable dm-verity protected partition (/vendor/etc/avf/microdroid/.*)
Alan Stokes53cc5ca2022-08-30 14:28:19 +01001157 => Ok(()),
Alan Stokes185fe112023-01-10 16:20:55 +00001158 _ => bail!("Label {} is not allowed", context),
Jiyong Park029977d2021-11-24 21:56:49 +09001159 }
1160}
1161
Alan Stokes185fe112023-01-10 16:20:55 +00001162fn check_label_for_partition(partition: &Partition) -> Result<()> {
1163 let file = partition.image.as_ref().unwrap().as_ref();
1164 check_label_is_allowed(&getfilecon(file)?)
1165 .with_context(|| format!("Partition {} invalid", &partition.label))
1166}
1167
1168fn check_label_for_kernel_files(kernel: &Option<File>, initrd: &Option<File>) -> Result<()> {
1169 if let Some(f) = kernel {
1170 check_label_for_file(f, "kernel")?;
1171 }
1172 if let Some(f) = initrd {
1173 check_label_for_file(f, "initrd")?;
1174 }
1175 Ok(())
1176}
1177fn check_label_for_file(file: &File, name: &str) -> Result<()> {
1178 check_label_is_allowed(&getfilecon(file)?).with_context(|| format!("{} file invalid", name))
1179}
1180
Andrew Walbrand6dce6f2021-03-05 16:39:08 +00001181/// Implementation of the AIDL `IVirtualMachine` interface. Used as a handle to a VM.
1182#[derive(Debug)]
1183struct VirtualMachine {
1184 instance: Arc<VmInstance>,
1185}
1186
1187impl VirtualMachine {
1188 fn create(instance: Arc<VmInstance>) -> Strong<dyn IVirtualMachine> {
David Brazdil4b4c5102022-12-19 22:56:20 +00001189 BnVirtualMachine::new_binder(VirtualMachine { instance }, BinderFeatures::default())
Andrew Walbrand6dce6f2021-03-05 16:39:08 +00001190 }
1191}
1192
1193impl Interface for VirtualMachine {}
1194
1195impl IVirtualMachine for VirtualMachine {
1196 fn getCid(&self) -> binder::Result<i32> {
Jiyong Park753553b2021-07-12 21:21:09 +09001197 // Don't check permission. The owner of the VM might have passed this binder object to
1198 // others.
Andrew Walbrand6dce6f2021-03-05 16:39:08 +00001199 Ok(self.instance.cid as i32)
1200 }
Andrew Walbrandae07162021-03-12 17:05:20 +00001201
Andrew Walbran6b650662021-09-07 13:13:23 +00001202 fn getState(&self) -> binder::Result<VirtualMachineState> {
Jiyong Park753553b2021-07-12 21:21:09 +09001203 // Don't check permission. The owner of the VM might have passed this binder object to
1204 // others.
Andrew Walbran6b650662021-09-07 13:13:23 +00001205 Ok(get_state(&self.instance))
Andrew Walbrandae07162021-03-12 17:05:20 +00001206 }
1207
1208 fn registerCallback(
1209 &self,
1210 callback: &Strong<dyn IVirtualMachineCallback>,
1211 ) -> binder::Result<()> {
Jiyong Park753553b2021-07-12 21:21:09 +09001212 // Don't check permission. The owner of the VM might have passed this binder object to
1213 // others.
1214 //
Andrew Walbrandae07162021-03-12 17:05:20 +00001215 // TODO: Should this give an error if the VM is already dead?
1216 self.instance.callbacks.add(callback.clone());
1217 Ok(())
1218 }
Andrew Walbrancbe8b082021-08-06 15:42:11 +00001219
Andrew Walbranf8d94112021-09-07 11:45:36 +00001220 fn start(&self) -> binder::Result<()> {
Jiyong Park2227eaa2023-08-04 11:59:18 +09001221 self.instance
1222 .start()
1223 .with_context(|| format!("Error starting VM with CID {}", self.instance.cid))
1224 .with_log()
1225 .or_service_specific_exception(-1)
Andrew Walbranf8d94112021-09-07 11:45:36 +00001226 }
1227
Inseob Kima446f802022-07-11 19:46:37 +09001228 fn stop(&self) -> binder::Result<()> {
Jiyong Park2227eaa2023-08-04 11:59:18 +09001229 self.instance
1230 .kill()
1231 .with_context(|| format!("Error stopping VM with CID {}", self.instance.cid))
1232 .with_log()
1233 .or_service_specific_exception(-1)
Inseob Kima446f802022-07-11 19:46:37 +09001234 }
1235
Keir Fraser48221ba2024-07-12 14:05:02 +00001236 fn getMemoryBalloon(&self) -> binder::Result<i64> {
1237 let balloon = self
1238 .instance
1239 .get_memory_balloon()
1240 .with_context(|| format!("Error getting balloon for VM with CID {}", self.instance.cid))
1241 .with_log()
1242 .or_service_specific_exception(-1)?;
1243 Ok(balloon.try_into().unwrap())
1244 }
1245
1246 fn setMemoryBalloon(&self, num_bytes: i64) -> binder::Result<()> {
Jiyong Park2227eaa2023-08-04 11:59:18 +09001247 self.instance
Keir Fraser48221ba2024-07-12 14:05:02 +00001248 .set_memory_balloon(num_bytes.try_into().unwrap())
1249 .with_context(|| format!("Error setting balloon for VM with CID {}", self.instance.cid))
Jiyong Park2227eaa2023-08-04 11:59:18 +09001250 .with_log()
1251 .or_service_specific_exception(-1)
Keir Frasercdd4b112022-11-24 14:02:25 +00001252 }
1253
Andrew Walbrancbe8b082021-08-06 15:42:11 +00001254 fn connectVsock(&self, port: i32) -> binder::Result<ParcelFileDescriptor> {
Andrew Walbranf8d94112021-09-07 11:45:36 +00001255 if !matches!(&*self.instance.vm_state.lock().unwrap(), VmState::Running { .. }) {
Jiyong Park2227eaa2023-08-04 11:59:18 +09001256 return Err(anyhow!("VM is not running")).or_service_specific_exception(-1);
Andrew Walbrancbe8b082021-08-06 15:42:11 +00001257 }
Alan Stokes10c47672022-12-13 17:17:08 +00001258 let port = port as u32;
1259 if port < 1024 {
Jiyong Park2227eaa2023-08-04 11:59:18 +09001260 return Err(anyhow!("Can't connect to privileged port {port}"))
1261 .or_service_specific_exception(-1);
Alan Stokes10c47672022-12-13 17:17:08 +00001262 }
Jiyong Park2227eaa2023-08-04 11:59:18 +09001263 let stream = VsockStream::connect_with_cid_port(self.instance.cid, port)
1264 .context("Failed to connect")
1265 .or_service_specific_exception(-1)?;
Andrew Walbrancbe8b082021-08-06 15:42:11 +00001266 Ok(vsock_stream_to_pfd(stream))
1267 }
Yi-Yo Chiang2fbf0da2024-06-14 22:56:56 +08001268
1269 fn setHostConsoleName(&self, ptsname: &str) -> binder::Result<()> {
1270 self.instance.vm_context.global_context.setHostConsoleName(ptsname)
1271 }
Jiyong Park23b67392024-07-04 13:51:42 +09001272
1273 fn suspend(&self) -> binder::Result<()> {
1274 self.instance
1275 .suspend()
1276 .with_context(|| format!("Error suspending VM with CID {}", self.instance.cid))
1277 .with_log()
1278 .or_service_specific_exception(-1)
1279 }
1280
1281 fn resume(&self) -> binder::Result<()> {
1282 self.instance
1283 .resume()
1284 .with_context(|| format!("Error resuming VM with CID {}", self.instance.cid))
1285 .with_log()
1286 .or_service_specific_exception(-1)
1287 }
Andrew Walbrandae07162021-03-12 17:05:20 +00001288}
1289
1290impl Drop for VirtualMachine {
1291 fn drop(&mut self) {
1292 debug!("Dropping {:?}", self);
Inseob Kima446f802022-07-11 19:46:37 +09001293 if let Err(e) = self.instance.kill() {
1294 debug!("Error stopping dropped VM with CID {}: {:?}", self.instance.cid, e);
1295 }
Andrew Walbrandae07162021-03-12 17:05:20 +00001296 }
1297}
1298
1299/// A set of Binders to be called back in response to various events on the VM, such as when it
1300/// dies.
1301#[derive(Debug, Default)]
1302pub struct VirtualMachineCallbacks(Mutex<Vec<Strong<dyn IVirtualMachineCallback>>>);
1303
1304impl VirtualMachineCallbacks {
Jiyong Park8611a6c2021-07-09 18:17:44 +09001305 /// Call all registered callbacks to notify that the payload has started.
David Brazdil451cc962022-10-14 14:08:12 +01001306 pub fn notify_payload_started(&self, cid: Cid) {
Jiyong Park8611a6c2021-07-09 18:17:44 +09001307 let callbacks = &*self.0.lock().unwrap();
Jiyong Park8611a6c2021-07-09 18:17:44 +09001308 for callback in callbacks {
David Brazdil451cc962022-10-14 14:08:12 +01001309 if let Err(e) = callback.onPayloadStarted(cid as i32) {
Alan Stokes70ccf162022-07-08 11:05:03 +01001310 error!("Error notifying payload start event from VM CID {}: {:?}", cid, e);
Jiyong Park8611a6c2021-07-09 18:17:44 +09001311 }
1312 }
1313 }
1314
Inseob Kim14cb8692021-08-31 21:50:39 +09001315 /// Call all registered callbacks to notify that the payload is ready to serve.
1316 pub fn notify_payload_ready(&self, cid: Cid) {
1317 let callbacks = &*self.0.lock().unwrap();
1318 for callback in callbacks {
1319 if let Err(e) = callback.onPayloadReady(cid as i32) {
Alan Stokes70ccf162022-07-08 11:05:03 +01001320 error!("Error notifying payload ready event from VM CID {}: {:?}", cid, e);
Inseob Kim14cb8692021-08-31 21:50:39 +09001321 }
1322 }
1323 }
1324
Inseob Kim2444af92021-08-31 01:22:50 +09001325 /// Call all registered callbacks to notify that the payload has finished.
1326 pub fn notify_payload_finished(&self, cid: Cid, exit_code: i32) {
1327 let callbacks = &*self.0.lock().unwrap();
1328 for callback in callbacks {
1329 if let Err(e) = callback.onPayloadFinished(cid as i32, exit_code) {
Alan Stokes70ccf162022-07-08 11:05:03 +01001330 error!("Error notifying payload finish event from VM CID {}: {:?}", cid, e);
Inseob Kim2444af92021-08-31 01:22:50 +09001331 }
1332 }
1333 }
1334
Jooyung Handd0a1732021-11-23 15:26:20 +09001335 /// Call all registered callbacks to say that the VM encountered an error.
Alan Stokes2bead0d2022-09-05 16:58:34 +01001336 pub fn notify_error(&self, cid: Cid, error_code: ErrorCode, message: &str) {
Jooyung Handd0a1732021-11-23 15:26:20 +09001337 let callbacks = &*self.0.lock().unwrap();
1338 for callback in callbacks {
1339 if let Err(e) = callback.onError(cid as i32, error_code, message) {
Alan Stokes70ccf162022-07-08 11:05:03 +01001340 error!("Error notifying error event from VM CID {}: {:?}", cid, e);
Jooyung Handd0a1732021-11-23 15:26:20 +09001341 }
1342 }
1343 }
1344
Andrew Walbrandae07162021-03-12 17:05:20 +00001345 /// Call all registered callbacks to say that the VM has died.
Andrew Walbranc92d35f2022-01-12 12:45:19 +00001346 pub fn callback_on_died(&self, cid: Cid, reason: DeathReason) {
Andrew Walbrandae07162021-03-12 17:05:20 +00001347 let callbacks = &*self.0.lock().unwrap();
1348 for callback in callbacks {
Andrew Walbranc92d35f2022-01-12 12:45:19 +00001349 if let Err(e) = callback.onDied(cid as i32, reason) {
Alan Stokes70ccf162022-07-08 11:05:03 +01001350 error!("Error notifying exit of VM CID {}: {:?}", cid, e);
Andrew Walbrandae07162021-03-12 17:05:20 +00001351 }
1352 }
1353 }
1354
1355 /// Add a new callback to the set.
1356 fn add(&self, callback: Strong<dyn IVirtualMachineCallback>) {
1357 self.0.lock().unwrap().push(callback);
1358 }
Andrew Walbrand6dce6f2021-03-05 16:39:08 +00001359}
1360
Andrew Walbranf6bf6862021-05-21 12:41:13 +00001361/// The mutable state of the VirtualizationService. There should only be one instance of this
1362/// struct.
Chris Wailes641fc4a2021-12-01 15:03:21 -08001363#[derive(Debug, Default)]
Andrew Walbrand6dce6f2021-03-05 16:39:08 +00001364struct State {
Alan Stokes3e5eec12023-09-07 12:10:00 +01001365 /// The VMs which have been started. When VMs are started a weak reference is added to this
1366 /// list while a strong reference is returned to the caller over Binder. Once all copies of
1367 /// the Binder client are dropped the weak reference here will become invalid, and will be
1368 /// removed from the list opportunistically the next time `add_vm` is called.
Andrew Walbran320b5602021-03-04 16:11:12 +00001369 vms: Vec<Weak<VmInstance>>,
1370}
1371
1372impl State {
Andrew Walbrandae07162021-03-12 17:05:20 +00001373 /// Get a list of VMs which still have Binder references to them.
Andrew Walbran320b5602021-03-04 16:11:12 +00001374 fn vms(&self) -> Vec<Arc<VmInstance>> {
1375 // Attempt to upgrade the weak pointers to strong pointers.
1376 self.vms.iter().filter_map(Weak::upgrade).collect()
1377 }
1378
1379 /// Add a new VM to the list.
1380 fn add_vm(&mut self, vm: Weak<VmInstance>) {
1381 // Garbage collect any entries from the stored list which no longer exist.
1382 self.vms.retain(|vm| vm.strong_count() > 0);
1383
1384 // Actually add the new VM.
1385 self.vms.push(vm);
1386 }
David Brazdil3c2ddef2021-03-18 13:09:57 +00001387
Jiyong Park8611a6c2021-07-09 18:17:44 +09001388 /// Get a VM that corresponds to the given cid
1389 fn get_vm(&self, cid: Cid) -> Option<Arc<VmInstance>> {
1390 self.vms().into_iter().find(|vm| vm.cid == cid)
1391 }
Jiyong Parkd50a0242021-09-16 21:00:14 +09001392}
1393
Andrew Walbran6b650662021-09-07 13:13:23 +00001394/// Gets the `VirtualMachineState` of the given `VmInstance`.
1395fn get_state(instance: &VmInstance) -> VirtualMachineState {
Andrew Walbranf8d94112021-09-07 11:45:36 +00001396 match &*instance.vm_state.lock().unwrap() {
1397 VmState::NotStarted { .. } => VirtualMachineState::NOT_STARTED,
1398 VmState::Running { .. } => match instance.payload_state() {
Andrew Walbran6b650662021-09-07 13:13:23 +00001399 PayloadState::Starting => VirtualMachineState::STARTING,
1400 PayloadState::Started => VirtualMachineState::STARTED,
1401 PayloadState::Ready => VirtualMachineState::READY,
1402 PayloadState::Finished => VirtualMachineState::FINISHED,
Jiyong Parka4eebde2022-07-12 18:01:12 +09001403 PayloadState::Hangup => VirtualMachineState::DEAD,
Andrew Walbranf8d94112021-09-07 11:45:36 +00001404 },
1405 VmState::Dead => VirtualMachineState::DEAD,
1406 VmState::Failed => VirtualMachineState::DEAD,
Andrew Walbran6b650662021-09-07 13:13:23 +00001407 }
1408}
1409
David Brazdilf50c7a62023-04-19 14:22:42 +00001410/// Converts a `&ParcelFileDescriptor` to a `File` by cloning the file.
Jiyong Park2227eaa2023-08-04 11:59:18 +09001411pub fn clone_file(file: &ParcelFileDescriptor) -> binder::Result<File> {
1412 file.as_ref()
1413 .try_clone()
1414 .context("Failed to clone File from ParcelFileDescriptor")
1415 .or_binder_exception(ExceptionCode::BAD_PARCELABLE)
Andrei Homescu11333c62023-11-09 04:26:39 +00001416 .map(File::from)
David Brazdilf50c7a62023-04-19 14:22:42 +00001417}
1418
Andrew Walbrand3a84182021-09-07 14:48:52 +00001419/// Converts an `&Option<ParcelFileDescriptor>` to an `Option<File>` by cloning the file.
Jiyong Park2227eaa2023-08-04 11:59:18 +09001420fn maybe_clone_file(file: &Option<ParcelFileDescriptor>) -> binder::Result<Option<File>> {
Andrew Walbrand3a84182021-09-07 14:48:52 +00001421 file.as_ref().map(clone_file).transpose()
1422}
1423
Andrew Walbrancbe8b082021-08-06 15:42:11 +00001424/// Converts a `VsockStream` to a `ParcelFileDescriptor`.
1425fn vsock_stream_to_pfd(stream: VsockStream) -> ParcelFileDescriptor {
1426 // SAFETY: ownership is transferred from stream to f
1427 let f = unsafe { File::from_raw_fd(stream.into_raw_fd()) };
1428 ParcelFileDescriptor::new(f)
1429}
1430
Jiyong Parkdcf17412022-02-08 15:07:23 +09001431/// Parses the platform version requirement string.
Jiyong Park2227eaa2023-08-04 11:59:18 +09001432fn parse_platform_version_req(s: &str) -> binder::Result<VersionReq> {
1433 VersionReq::parse(s)
1434 .with_context(|| format!("Invalid platform version requirement {}", s))
1435 .or_binder_exception(ExceptionCode::BAD_PARCELABLE)
Jiyong Parkdcf17412022-02-08 15:07:23 +09001436}
1437
Jiyong Parked180932023-02-24 19:55:41 +09001438/// Create the empty ramdump file
1439fn prepare_ramdump_file(temporary_directory: &Path) -> binder::Result<File> {
1440 // `ramdump_write` is sent to crosvm and will be the backing store for the /dev/hvc1 where
1441 // VM will emit ramdump to. `ramdump_read` will be sent back to the client (i.e. the VM
1442 // owner) for readout.
1443 let ramdump_path = temporary_directory.join("ramdump");
Jiyong Park2227eaa2023-08-04 11:59:18 +09001444 let ramdump = File::create(ramdump_path)
1445 .context("Failed to prepare ramdump file")
1446 .with_log()
1447 .or_service_specific_exception(-1)?;
Jiyong Parked180932023-02-24 19:55:41 +09001448 Ok(ramdump)
1449}
1450
Nikita Ioffe5776f082023-02-10 21:38:26 +00001451fn is_protected(config: &VirtualMachineConfig) -> bool {
1452 match config {
1453 VirtualMachineConfig::RawConfig(config) => config.protectedVm,
1454 VirtualMachineConfig::AppConfig(config) => config.protectedVm,
1455 }
1456}
1457
1458fn check_gdb_allowed(config: &VirtualMachineConfig) -> binder::Result<()> {
1459 if is_protected(config) {
Jiyong Park2227eaa2023-08-04 11:59:18 +09001460 return Err(anyhow!("Can't use gdb with protected VMs"))
1461 .or_binder_exception(ExceptionCode::SECURITY);
Nikita Ioffe5776f082023-02-10 21:38:26 +00001462 }
1463
Pierre-Clément Tosid3bbe1d2024-04-15 18:03:51 +01001464 if get_debug_level(config) == Some(DebugLevel::NONE) {
1465 return Err(anyhow!("Can't use gdb with non-debuggable VMs"))
1466 .or_binder_exception(ExceptionCode::SECURITY);
Nikita Ioffe5776f082023-02-10 21:38:26 +00001467 }
Pierre-Clément Tosid3bbe1d2024-04-15 18:03:51 +01001468
1469 Ok(())
Nikita Ioffe5776f082023-02-10 21:38:26 +00001470}
1471
Shikha Panwar61a74b52024-02-16 13:17:01 +00001472fn extract_instance_id(config: &VirtualMachineConfig) -> [u8; 64] {
1473 match config {
1474 VirtualMachineConfig::RawConfig(config) => config.instanceId,
1475 VirtualMachineConfig::AppConfig(config) => config.instanceId,
1476 }
1477}
1478
Alan Stokesc96b35e2024-05-03 13:21:20 +01001479fn extract_want_updatable(config: &VirtualMachineConfig) -> bool {
1480 match config {
1481 VirtualMachineConfig::RawConfig(_) => true,
1482 VirtualMachineConfig::AppConfig(config) => {
1483 let Some(custom) = &config.customConfig else { return true };
1484 custom.wantUpdatable
1485 }
1486 }
1487}
1488
Nikita Ioffe5776f082023-02-10 21:38:26 +00001489fn extract_gdb_port(config: &VirtualMachineConfig) -> Option<NonZeroU16> {
1490 match config {
1491 VirtualMachineConfig::RawConfig(config) => NonZeroU16::new(config.gdbPort as u16),
Nikita Ioffea0eb5ee2023-06-26 18:18:21 +01001492 VirtualMachineConfig::AppConfig(config) => {
1493 NonZeroU16::new(config.customConfig.as_ref().map(|c| c.gdbPort).unwrap_or(0) as u16)
1494 }
Nikita Ioffe5776f082023-02-10 21:38:26 +00001495 }
1496}
1497
Nikita Ioffe631717e2023-09-05 13:38:07 +01001498fn check_no_vendor_modules(config: &VirtualMachineConfig) -> binder::Result<()> {
1499 let VirtualMachineConfig::AppConfig(config) = config else { return Ok(()) };
1500 if let Some(custom_config) = &config.customConfig {
1501 if custom_config.vendorImage.is_some() || custom_config.customKernelImage.is_some() {
1502 return Err(anyhow!("vendor modules feature is disabled"))
1503 .or_binder_exception(ExceptionCode::UNSUPPORTED_OPERATION);
1504 }
1505 }
1506 Ok(())
1507}
1508
Nikita Ioffe94a8a182023-11-16 16:37:48 +00001509fn check_no_devices(config: &VirtualMachineConfig) -> binder::Result<()> {
1510 let VirtualMachineConfig::AppConfig(config) = config else { return Ok(()) };
1511 if let Some(custom_config) = &config.customConfig {
1512 if !custom_config.devices.is_empty() {
1513 return Err(anyhow!("device assignment feature is disabled"))
1514 .or_binder_exception(ExceptionCode::UNSUPPORTED_OPERATION);
1515 }
1516 }
1517 Ok(())
1518}
1519
Alan Stokesfda70842023-12-20 17:50:14 +00001520fn check_no_extra_apks(config: &VirtualMachineConfig) -> binder::Result<()> {
1521 let VirtualMachineConfig::AppConfig(config) = config else { return Ok(()) };
1522 let Payload::PayloadConfig(payload_config) = &config.payload else { return Ok(()) };
1523 if !payload_config.extraApks.is_empty() {
1524 return Err(anyhow!("multi-tenant feature is disabled"))
1525 .or_binder_exception(ExceptionCode::UNSUPPORTED_OPERATION);
1526 }
1527 Ok(())
1528}
1529
Nikita Ioffef934e812024-07-05 15:44:41 +00001530fn check_protected_vm_is_supported() -> binder::Result<()> {
1531 let is_pvm_supported =
1532 hypervisor_props::is_protected_vm_supported().or_service_specific_exception(-1)?;
1533 if is_pvm_supported {
1534 Ok(())
1535 } else {
1536 Err(anyhow!("pVM is not supported"))
1537 .or_binder_exception(ExceptionCode::UNSUPPORTED_OPERATION)
1538 }
1539}
1540
Nikita Ioffe631717e2023-09-05 13:38:07 +01001541fn check_config_features(config: &VirtualMachineConfig) -> binder::Result<()> {
1542 if !cfg!(vendor_modules) {
1543 check_no_vendor_modules(config)?;
1544 }
Nikita Ioffe94a8a182023-11-16 16:37:48 +00001545 if !cfg!(device_assignment) {
1546 check_no_devices(config)?;
1547 }
Alan Stokesfda70842023-12-20 17:50:14 +00001548 if !cfg!(multi_tenant) {
1549 check_no_extra_apks(config)?;
1550 }
Nikita Ioffe631717e2023-09-05 13:38:07 +01001551 Ok(())
1552}
1553
Inseob Kim0168b462022-12-27 14:54:35 +09001554fn clone_or_prepare_logger_fd(
Jaewan Kim61f86142023-03-28 15:12:52 +09001555 debug_config: &DebugConfig,
Inseob Kim0168b462022-12-27 14:54:35 +09001556 fd: Option<&ParcelFileDescriptor>,
1557 tag: String,
1558) -> Result<Option<File>, Status> {
1559 if let Some(fd) = fd {
1560 return Ok(Some(clone_file(fd)?));
1561 }
1562
Jaewan Kim61f86142023-03-28 15:12:52 +09001563 if !debug_config.should_prepare_console_output() {
Jaewan Kim66f062e2023-02-25 01:07:43 +09001564 return Ok(None);
1565 };
Inseob Kim0168b462022-12-27 14:54:35 +09001566
Frederick Maylefbbcfcd2024-04-08 16:31:54 -07001567 let (read_fd, write_fd) =
Jiyong Park2227eaa2023-08-04 11:59:18 +09001568 pipe().context("Failed to create pipe").or_service_specific_exception(-1)?;
Inseob Kim0168b462022-12-27 14:54:35 +09001569
Frederick Maylefbbcfcd2024-04-08 16:31:54 -07001570 let mut reader = BufReader::new(File::from(read_fd));
1571 let write_fd = File::from(write_fd);
Inseob Kim0168b462022-12-27 14:54:35 +09001572
1573 std::thread::spawn(move || loop {
1574 let mut buf = vec![];
1575 match reader.read_until(b'\n', &mut buf) {
1576 Ok(0) => {
1577 // EOF
1578 return;
1579 }
1580 Ok(size) => {
1581 if buf[size - 1] == b'\n' {
1582 buf.pop();
1583 }
1584 info!("{}: {}", &tag, &String::from_utf8_lossy(&buf));
1585 }
1586 Err(e) => {
1587 error!("Could not read console pipe: {:?}", e);
1588 return;
1589 }
1590 };
1591 });
1592
1593 Ok(Some(write_fd))
1594}
1595
Jooyung Han35edb8f2021-07-01 16:17:16 +09001596/// Simple utility for referencing Borrowed or Owned. Similar to std::borrow::Cow, but
1597/// it doesn't require that T implements Clone.
1598enum BorrowedOrOwned<'a, T> {
1599 Borrowed(&'a T),
1600 Owned(T),
1601}
1602
1603impl<'a, T> AsRef<T> for BorrowedOrOwned<'a, T> {
1604 fn as_ref(&self) -> &T {
1605 match self {
1606 Self::Borrowed(b) => b,
Chris Wailes68c39f82021-07-27 16:03:44 -07001607 Self::Owned(o) => o,
Jooyung Han35edb8f2021-07-01 16:17:16 +09001608 }
1609 }
1610}
Inseob Kim1b95f2e2021-08-19 13:17:40 +09001611
1612/// Implementation of `IVirtualMachineService`, the entry point of the AIDL service.
1613#[derive(Debug, Default)]
1614struct VirtualMachineService {
1615 state: Arc<Mutex<State>>,
Inseob Kimc7d28c72021-10-25 14:28:10 +00001616 cid: Cid,
Inseob Kim1b95f2e2021-08-19 13:17:40 +09001617}
1618
1619impl Interface for VirtualMachineService {}
1620
1621impl IVirtualMachineService for VirtualMachineService {
Inseob Kimc7d28c72021-10-25 14:28:10 +00001622 fn notifyPayloadStarted(&self) -> binder::Result<()> {
1623 let cid = self.cid;
Inseob Kim7f61fe72021-08-20 20:50:47 +09001624 if let Some(vm) = self.state.lock().unwrap().get_vm(cid) {
David Brazdil415097c2022-10-21 14:17:05 +01001625 info!("VM with CID {} started payload", cid);
Jiyong Park2227eaa2023-08-04 11:59:18 +09001626 vm.update_payload_state(PayloadState::Started)
1627 .or_binder_exception(ExceptionCode::ILLEGAL_STATE)?;
David Brazdil451cc962022-10-14 14:08:12 +01001628 vm.callbacks.notify_payload_started(cid);
Seungjae Yoo62085c02022-08-12 04:44:52 +00001629
Seungjae Yoo6d265d92022-11-15 10:51:33 +09001630 let vm_start_timestamp = vm.vm_metric.lock().unwrap().start_timestamp;
1631 write_vm_booted_stats(vm.requester_uid as i32, &vm.name, vm_start_timestamp);
Inseob Kim7f61fe72021-08-20 20:50:47 +09001632 Ok(())
1633 } else {
Jooyung Handd0a1732021-11-23 15:26:20 +09001634 error!("notifyPayloadStarted is called from an unknown CID {}", cid);
Jiyong Park2227eaa2023-08-04 11:59:18 +09001635 Err(anyhow!("cannot find a VM with CID {}", cid)).or_service_specific_exception(-1)
Inseob Kim1b95f2e2021-08-19 13:17:40 +09001636 }
Inseob Kim1b95f2e2021-08-19 13:17:40 +09001637 }
Inseob Kim2444af92021-08-31 01:22:50 +09001638
Inseob Kimc7d28c72021-10-25 14:28:10 +00001639 fn notifyPayloadReady(&self) -> binder::Result<()> {
1640 let cid = self.cid;
Inseob Kim14cb8692021-08-31 21:50:39 +09001641 if let Some(vm) = self.state.lock().unwrap().get_vm(cid) {
David Brazdil415097c2022-10-21 14:17:05 +01001642 info!("VM with CID {} reported payload is ready", cid);
Jiyong Park2227eaa2023-08-04 11:59:18 +09001643 vm.update_payload_state(PayloadState::Ready)
1644 .or_binder_exception(ExceptionCode::ILLEGAL_STATE)?;
Inseob Kim14cb8692021-08-31 21:50:39 +09001645 vm.callbacks.notify_payload_ready(cid);
1646 Ok(())
1647 } else {
Jooyung Handd0a1732021-11-23 15:26:20 +09001648 error!("notifyPayloadReady is called from an unknown CID {}", cid);
Jiyong Park2227eaa2023-08-04 11:59:18 +09001649 Err(anyhow!("cannot find a VM with CID {}", cid)).or_service_specific_exception(-1)
Inseob Kim14cb8692021-08-31 21:50:39 +09001650 }
1651 }
1652
Inseob Kimc7d28c72021-10-25 14:28:10 +00001653 fn notifyPayloadFinished(&self, exit_code: i32) -> binder::Result<()> {
1654 let cid = self.cid;
Inseob Kim2444af92021-08-31 01:22:50 +09001655 if let Some(vm) = self.state.lock().unwrap().get_vm(cid) {
David Brazdil415097c2022-10-21 14:17:05 +01001656 info!("VM with CID {} finished payload", cid);
Jiyong Park2227eaa2023-08-04 11:59:18 +09001657 vm.update_payload_state(PayloadState::Finished)
1658 .or_binder_exception(ExceptionCode::ILLEGAL_STATE)?;
Inseob Kim2444af92021-08-31 01:22:50 +09001659 vm.callbacks.notify_payload_finished(cid, exit_code);
1660 Ok(())
1661 } else {
Jooyung Handd0a1732021-11-23 15:26:20 +09001662 error!("notifyPayloadFinished is called from an unknown CID {}", cid);
Jiyong Park2227eaa2023-08-04 11:59:18 +09001663 Err(anyhow!("cannot find a VM with CID {}", cid)).or_service_specific_exception(-1)
Jooyung Handd0a1732021-11-23 15:26:20 +09001664 }
1665 }
1666
Alan Stokes2bead0d2022-09-05 16:58:34 +01001667 fn notifyError(&self, error_code: ErrorCode, message: &str) -> binder::Result<()> {
Jooyung Handd0a1732021-11-23 15:26:20 +09001668 let cid = self.cid;
1669 if let Some(vm) = self.state.lock().unwrap().get_vm(cid) {
David Brazdil415097c2022-10-21 14:17:05 +01001670 info!("VM with CID {} encountered an error", cid);
Jiyong Park2227eaa2023-08-04 11:59:18 +09001671 vm.update_payload_state(PayloadState::Finished)
1672 .or_binder_exception(ExceptionCode::ILLEGAL_STATE)?;
Jooyung Handd0a1732021-11-23 15:26:20 +09001673 vm.callbacks.notify_error(cid, error_code, message);
1674 Ok(())
1675 } else {
Seungjae Yooec8c1602022-06-20 05:28:00 +00001676 error!("notifyError is called from an unknown CID {}", cid);
Jiyong Park2227eaa2023-08-04 11:59:18 +09001677 Err(anyhow!("cannot find a VM with CID {}", cid)).or_service_specific_exception(-1)
Inseob Kim2444af92021-08-31 01:22:50 +09001678 }
1679 }
Alice Wangc2fec932023-02-23 16:24:02 +00001680
Shikha Panware45e9422024-02-28 21:18:10 +00001681 fn getSecretkeeper(&self) -> binder::Result<Strong<dyn ISecretkeeper>> {
1682 if !is_secretkeeper_supported() {
1683 return Err(StatusCode::NAME_NOT_FOUND)?;
1684 }
1685 let sk = binder::wait_for_interface(SECRETKEEPER_IDENTIFIER)?;
1686 Ok(BnSecretkeeper::new_binder(SecretkeeperProxy(sk), BinderFeatures::default()))
Shikha Panwar5d6a6752023-12-14 22:08:26 +00001687 }
1688
Alice Wange64dd182024-01-17 15:57:55 +00001689 fn requestAttestation(&self, csr: &[u8], test_mode: bool) -> binder::Result<Vec<Certificate>> {
1690 GLOBAL_SERVICE.requestAttestation(csr, get_calling_uid() as i32, test_mode)
Alice Wangc2fec932023-02-23 16:24:02 +00001691 }
Inseob Kim1b95f2e2021-08-19 13:17:40 +09001692}
1693
Shikha Panwar8707f0f2024-02-28 20:40:42 +00001694fn is_secretkeeper_supported() -> bool {
Shikha Panwar81d13d32024-03-10 19:39:23 +00001695 binder::is_declared(SECRETKEEPER_IDENTIFIER)
1696 .expect("Could not check for declared Secretkeeper interface")
Shikha Panwar8187ca22024-01-04 08:33:08 +00001697}
1698
Inseob Kim1b95f2e2021-08-19 13:17:40 +09001699impl VirtualMachineService {
Inseob Kimc7d28c72021-10-25 14:28:10 +00001700 fn new_binder(state: Arc<Mutex<State>>, cid: Cid) -> Strong<dyn IVirtualMachineService> {
Inseob Kim1b95f2e2021-08-19 13:17:40 +09001701 BnVirtualMachineService::new_binder(
Inseob Kimc7d28c72021-10-25 14:28:10 +00001702 VirtualMachineService { state, cid },
Inseob Kim1b95f2e2021-08-19 13:17:40 +09001703 BinderFeatures::default(),
1704 )
1705 }
1706}
Alan Stokes53cc5ca2022-08-30 14:28:19 +01001707
Stephen Hines4c1e2fc2024-02-09 16:34:47 -08001708struct SecretkeeperProxy(Strong<dyn ISecretkeeper>);
1709
1710impl Interface for SecretkeeperProxy {}
1711
1712impl ISecretkeeper for SecretkeeperProxy {
1713 fn processSecretManagementRequest(&self, req: &[u8]) -> binder::Result<Vec<u8>> {
1714 // Pass the request to the channel, and read the response.
1715 self.0.processSecretManagementRequest(req)
1716 }
1717
1718 fn getAuthGraphKe(&self) -> binder::Result<Strong<dyn IAuthGraphKeyExchange>> {
1719 let ag = AuthGraphKeyExchangeProxy(self.0.getAuthGraphKe()?);
1720 Ok(BnAuthGraphKeyExchange::new_binder(ag, BinderFeatures::default()))
1721 }
1722
1723 fn deleteIds(&self, ids: &[SecretId]) -> binder::Result<()> {
1724 self.0.deleteIds(ids)
1725 }
1726
1727 fn deleteAll(&self) -> binder::Result<()> {
1728 self.0.deleteAll()
1729 }
1730}
1731
1732struct AuthGraphKeyExchangeProxy(Strong<dyn IAuthGraphKeyExchange>);
1733
1734impl Interface for AuthGraphKeyExchangeProxy {}
1735
1736impl IAuthGraphKeyExchange for AuthGraphKeyExchangeProxy {
1737 fn create(&self) -> binder::Result<SessionInitiationInfo> {
1738 self.0.create()
1739 }
1740
1741 fn init(
1742 &self,
1743 peer_pub_key: &PubKey,
1744 peer_id: &Identity,
1745 peer_nonce: &[u8],
1746 peer_version: i32,
1747 ) -> binder::Result<KeInitResult> {
1748 self.0.init(peer_pub_key, peer_id, peer_nonce, peer_version)
1749 }
1750
1751 fn finish(
1752 &self,
1753 peer_pub_key: &PubKey,
1754 peer_id: &Identity,
1755 peer_signature: &SessionIdSignature,
1756 peer_nonce: &[u8],
1757 peer_version: i32,
1758 own_key: &Key,
1759 ) -> binder::Result<SessionInfo> {
1760 self.0.finish(peer_pub_key, peer_id, peer_signature, peer_nonce, peer_version, own_key)
1761 }
1762
1763 fn authenticationComplete(
1764 &self,
1765 peer_signature: &SessionIdSignature,
1766 shared_keys: &[AuthgraphArc; 2],
1767 ) -> binder::Result<[AuthgraphArc; 2]> {
1768 self.0.authenticationComplete(peer_signature, shared_keys)
1769 }
1770}
1771
Alan Stokes53cc5ca2022-08-30 14:28:19 +01001772#[cfg(test)]
1773mod tests {
1774 use super::*;
1775
1776 #[test]
1777 fn test_is_allowed_label_for_partition() -> Result<()> {
1778 let expected_results = vec![
1779 ("u:object_r:system_file:s0", true),
1780 ("u:object_r:apk_data_file:s0", true),
1781 ("u:object_r:app_data_file:s0", false),
1782 ("u:object_r:app_data_file:s0:c512,c768", false),
1783 ("u:object_r:privapp_data_file:s0:c512,c768", false),
1784 ("invalid", false),
1785 ("user:role:apk_data_file:severity:categories", true),
1786 ("user:role:apk_data_file:severity:categories:extraneous", false),
1787 ];
1788
1789 for (label, expected_valid) in expected_results {
1790 let context = SeContext::new(label)?;
1791 let result = check_label_is_allowed(&context);
1792 if expected_valid {
1793 assert!(result.is_ok(), "Expected label {} to be allowed, got {:?}", label, result);
1794 } else if result.is_ok() {
1795 bail!("Expected label {} to be disallowed", label);
1796 }
1797 }
1798 Ok(())
1799 }
Nikita Ioffef1ce9872022-12-09 13:31:59 +00001800
1801 #[test]
1802 fn test_create_or_update_idsig_file_empty_apk() -> Result<()> {
1803 let apk = tempfile::tempfile().unwrap();
1804 let idsig = tempfile::tempfile().unwrap();
1805
1806 let ret = create_or_update_idsig_file(
1807 &ParcelFileDescriptor::new(apk),
1808 &ParcelFileDescriptor::new(idsig),
1809 );
1810 assert!(ret.is_err(), "should fail");
1811 Ok(())
1812 }
1813
1814 #[test]
1815 fn test_create_or_update_idsig_dir_instead_of_file_for_apk() -> Result<()> {
1816 let tmp_dir = tempfile::TempDir::new().unwrap();
1817 let apk = File::open(tmp_dir.path()).unwrap();
1818 let idsig = tempfile::tempfile().unwrap();
1819
1820 let ret = create_or_update_idsig_file(
1821 &ParcelFileDescriptor::new(apk),
1822 &ParcelFileDescriptor::new(idsig),
1823 );
1824 assert!(ret.is_err(), "should fail");
1825 Ok(())
1826 }
1827
1828 /// Verifies that create_or_update_idsig_file won't oom if a fd that corresponds to a directory
1829 /// on ext4 filesystem is passed.
1830 /// On ext4 lseek on a directory fd will return (off_t)-1 (see:
1831 /// https://bugzilla.kernel.org/show_bug.cgi?id=200043), which will result in
1832 /// create_or_update_idsig_file ooming while attempting to allocate petabytes of memory.
1833 #[test]
1834 fn test_create_or_update_idsig_does_not_crash_dir_on_ext4() -> Result<()> {
1835 // APEXes are backed by the ext4.
1836 let apk = File::open("/apex/com.android.virt/").unwrap();
1837 let idsig = tempfile::tempfile().unwrap();
1838
1839 let ret = create_or_update_idsig_file(
1840 &ParcelFileDescriptor::new(apk),
1841 &ParcelFileDescriptor::new(idsig),
1842 );
1843 assert!(ret.is_err(), "should fail");
1844 Ok(())
1845 }
Jiyong Park8d192952023-06-26 14:29:51 +09001846
1847 #[test]
1848 fn test_create_or_update_idsig_does_not_update_if_already_valid() -> Result<()> {
1849 use std::io::Seek;
1850
1851 // Pick any APK
1852 let mut apk = File::open("/system/priv-app/Shell/Shell.apk").unwrap();
1853 let mut idsig = tempfile::tempfile().unwrap();
1854
1855 create_or_update_idsig_file(
1856 &ParcelFileDescriptor::new(apk.try_clone()?),
1857 &ParcelFileDescriptor::new(idsig.try_clone()?),
1858 )?;
1859 let modified_orig = idsig.metadata()?.modified()?;
1860 apk.rewind()?;
1861 idsig.rewind()?;
1862
1863 // Call the function again
1864 create_or_update_idsig_file(
1865 &ParcelFileDescriptor::new(apk.try_clone()?),
1866 &ParcelFileDescriptor::new(idsig.try_clone()?),
1867 )?;
1868 let modified_new = idsig.metadata()?.modified()?;
1869 assert!(modified_orig == modified_new, "idsig file was updated unnecessarily");
1870 Ok(())
1871 }
Nikita Ioffeaa6858c2023-07-04 01:37:41 +01001872
1873 #[test]
Shikha Panwar9ae2e622024-01-30 12:22:30 +00001874 fn test_create_or_update_idsig_on_non_empty_file() -> Result<()> {
1875 use std::io::Read;
1876
1877 // Pick any APK
1878 let mut apk = File::open("/system/priv-app/Shell/Shell.apk").unwrap();
1879 let idsig_empty = tempfile::tempfile().unwrap();
1880 let mut idsig_invalid = tempfile::tempfile().unwrap();
1881 idsig_invalid.write_all(b"Oops")?;
1882
1883 // Create new idsig
1884 create_or_update_idsig_file(
1885 &ParcelFileDescriptor::new(apk.try_clone()?),
1886 &ParcelFileDescriptor::new(idsig_empty.try_clone()?),
1887 )?;
1888 apk.rewind()?;
1889
1890 // Update idsig_invalid
1891 create_or_update_idsig_file(
1892 &ParcelFileDescriptor::new(apk.try_clone()?),
1893 &ParcelFileDescriptor::new(idsig_invalid.try_clone()?),
1894 )?;
1895
1896 // Ensure the 2 idsig files have same size!
1897 assert!(
1898 idsig_empty.metadata()?.len() == idsig_invalid.metadata()?.len(),
1899 "idsig files differ in size"
1900 );
1901 // Ensure the 2 idsig files have same content!
1902 for (b1, b2) in idsig_empty.bytes().zip(idsig_invalid.bytes()) {
1903 assert!(b1.unwrap() == b2.unwrap(), "idsig files differ")
1904 }
1905 Ok(())
1906 }
1907 #[test]
Nikita Ioffeaa6858c2023-07-04 01:37:41 +01001908 fn test_append_kernel_param_first_param() {
1909 let mut vm_config = VirtualMachineRawConfig { ..Default::default() };
1910 append_kernel_param("foo=1", &mut vm_config);
1911 assert_eq!(vm_config.params, Some("foo=1".to_owned()))
1912 }
1913
1914 #[test]
1915 fn test_append_kernel_param() {
1916 let mut vm_config =
1917 VirtualMachineRawConfig { params: Some("foo=5".to_owned()), ..Default::default() };
1918 append_kernel_param("bar=42", &mut vm_config);
1919 assert_eq!(vm_config.params, Some("foo=5 bar=42".to_owned()))
1920 }
Seungjae Yooec3bc522023-11-09 10:14:30 +09001921
Inseob Kim46257382024-01-03 15:41:22 +09001922 fn test_extract_os_name_from_config_path(
1923 path: &Path,
1924 expected_result: Option<&str>,
1925 ) -> Result<()> {
1926 let result = extract_os_name_from_config_path(path);
1927 if result.as_deref() != expected_result {
1928 bail!("Expected {:?} but was {:?}", expected_result, &result)
1929 }
1930 Ok(())
1931 }
1932
1933 #[test]
1934 fn test_extract_os_name_from_microdroid_config() -> Result<()> {
1935 test_extract_os_name_from_config_path(
1936 Path::new("/apex/com.android.virt/etc/microdroid.json"),
1937 Some("microdroid"),
1938 )
1939 }
1940
1941 #[test]
1942 fn test_extract_os_name_from_microdroid_gki_config() -> Result<()> {
1943 test_extract_os_name_from_config_path(
1944 Path::new("/apex/com.android.virt/etc/microdroid_gki-android14-6.1.json"),
1945 Some("microdroid_gki-android14-6.1"),
1946 )
1947 }
1948
1949 #[test]
1950 fn test_extract_os_name_from_invalid_path() -> Result<()> {
1951 test_extract_os_name_from_config_path(
1952 Path::new("/apex/com.android.virt/etc/microdroid.img"),
1953 None,
1954 )
1955 }
1956
1957 #[test]
1958 fn test_extract_os_name_from_configs() -> Result<()> {
1959 let tmp_dir = tempfile::TempDir::new()?;
1960 let tmp_dir_path = tmp_dir.path().to_owned();
1961
1962 let mut os_names: HashSet<String> = HashSet::new();
1963 os_names.insert("microdroid".to_owned());
1964 os_names.insert("microdroid_gki-android14-6.1".to_owned());
1965 os_names.insert("microdroid_gki-android15-6.1".to_owned());
1966
1967 // config files
1968 for os_name in &os_names {
1969 std::fs::write(tmp_dir_path.join(os_name.to_owned() + ".json"), b"")?;
1970 }
1971
1972 // fake files not related to configs
1973 std::fs::write(tmp_dir_path.join("microdroid_super.img"), b"")?;
1974 std::fs::write(tmp_dir_path.join("microdroid_foobar.apk"), b"")?;
1975
1976 let glob_pattern = match tmp_dir_path.join("microdroid*.json").to_str() {
1977 Some(s) => s.to_owned(),
1978 None => bail!("tmp_dir_path {:?} is not UTF-8", tmp_dir_path),
1979 };
1980
1981 let result = extract_os_names_from_configs(&glob_pattern)?;
1982 if result != os_names {
1983 bail!("Expected {:?} but was {:?}", os_names, result);
1984 }
1985 Ok(())
1986 }
Alan Stokes53cc5ca2022-08-30 14:28:19 +01001987}