blob: 604aa80247b257e831883e321461b5114ed48f59 [file] [log] [blame]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +00001// Copyright 2022, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Low-level allocation and tracking of main memory.
16
Andrew Walbran848decf2022-12-15 14:39:38 +000017#![deny(unsafe_op_in_unsafe_fn)]
18
19use crate::helpers::{self, align_down, align_up, page_4kb_of, SIZE_4KB};
Andrew Walbran41ebe932022-12-14 15:22:30 +000020use crate::hvc::{hyp_meminfo, mem_share, mem_unshare};
Andrew Walbran19690632022-12-07 16:41:30 +000021use crate::mmio_guard;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000022use crate::mmu;
Andrew Walbran41ebe932022-12-14 15:22:30 +000023use crate::smccc;
Andrew Walbran848decf2022-12-15 14:39:38 +000024use alloc::alloc::alloc_zeroed;
25use alloc::alloc::dealloc;
26use alloc::alloc::handle_alloc_error;
27use core::alloc::Layout;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000028use core::cmp::max;
29use core::cmp::min;
30use core::fmt;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000031use core::num::NonZeroUsize;
32use core::ops::Range;
Andrew Walbran848decf2022-12-15 14:39:38 +000033use core::ptr::NonNull;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000034use core::result;
35use log::error;
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000036use tinyvec::ArrayVec;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000037
Andrew Walbran0d8b54d2022-12-08 16:32:33 +000038pub type MemoryRange = Range<usize>;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000039
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000040#[derive(Clone, Copy, Debug, Default)]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000041enum MemoryType {
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000042 #[default]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000043 ReadOnly,
44 ReadWrite,
45}
46
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000047#[derive(Clone, Debug, Default)]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000048struct MemoryRegion {
49 range: MemoryRange,
50 mem_type: MemoryType,
51}
52
53impl MemoryRegion {
54 /// True if the instance overlaps with the passed range.
55 pub fn overlaps(&self, range: &MemoryRange) -> bool {
Andrew Walbran19690632022-12-07 16:41:30 +000056 overlaps(&self.range, range)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000057 }
58
59 /// True if the instance is fully contained within the passed range.
60 pub fn is_within(&self, range: &MemoryRange) -> bool {
61 let our: &MemoryRange = self.as_ref();
62 self.as_ref() == &(max(our.start, range.start)..min(our.end, range.end))
63 }
64}
65
66impl AsRef<MemoryRange> for MemoryRegion {
67 fn as_ref(&self) -> &MemoryRange {
68 &self.range
69 }
70}
71
Andrew Walbran19690632022-12-07 16:41:30 +000072/// Returns true if one range overlaps with the other at all.
73fn overlaps<T: Copy + Ord>(a: &Range<T>, b: &Range<T>) -> bool {
74 max(a.start, b.start) < min(a.end, b.end)
75}
76
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000077/// Tracks non-overlapping slices of main memory.
78pub struct MemoryTracker {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000079 total: MemoryRange,
80 page_table: mmu::PageTable,
Andrew Walbran19690632022-12-07 16:41:30 +000081 regions: ArrayVec<[MemoryRegion; MemoryTracker::CAPACITY]>,
82 mmio_regions: ArrayVec<[MemoryRange; MemoryTracker::MMIO_CAPACITY]>,
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000083}
84
85/// Errors for MemoryTracker operations.
86#[derive(Debug, Clone)]
87pub enum MemoryTrackerError {
88 /// Tried to modify the memory base address.
89 DifferentBaseAddress,
90 /// Tried to shrink to a larger memory size.
91 SizeTooLarge,
92 /// Tracked regions would not fit in memory size.
93 SizeTooSmall,
94 /// Reached limit number of tracked regions.
95 Full,
96 /// Region is out of the tracked memory address space.
97 OutOfRange,
98 /// New region overlaps with tracked regions.
99 Overlaps,
100 /// Region couldn't be mapped.
101 FailedToMap,
Andrew Walbran19690632022-12-07 16:41:30 +0000102 /// Error from an MMIO guard call.
103 MmioGuard(mmio_guard::Error),
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000104}
105
106impl fmt::Display for MemoryTrackerError {
107 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
108 match self {
109 Self::DifferentBaseAddress => write!(f, "Received different base address"),
110 Self::SizeTooLarge => write!(f, "Tried to shrink to a larger memory size"),
111 Self::SizeTooSmall => write!(f, "Tracked regions would not fit in memory size"),
112 Self::Full => write!(f, "Reached limit number of tracked regions"),
113 Self::OutOfRange => write!(f, "Region is out of the tracked memory address space"),
114 Self::Overlaps => write!(f, "New region overlaps with tracked regions"),
115 Self::FailedToMap => write!(f, "Failed to map the new region"),
Andrew Walbran19690632022-12-07 16:41:30 +0000116 Self::MmioGuard(e) => e.fmt(f),
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000117 }
118 }
119}
120
Andrew Walbran19690632022-12-07 16:41:30 +0000121impl From<mmio_guard::Error> for MemoryTrackerError {
122 fn from(e: mmio_guard::Error) -> Self {
123 Self::MmioGuard(e)
124 }
125}
126
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000127type Result<T> = result::Result<T, MemoryTrackerError>;
128
129impl MemoryTracker {
130 const CAPACITY: usize = 5;
Andrew Walbran19690632022-12-07 16:41:30 +0000131 const MMIO_CAPACITY: usize = 5;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000132 /// Base of the system's contiguous "main" memory.
133 const BASE: usize = 0x8000_0000;
134 /// First address that can't be translated by a level 1 TTBR0_EL1.
135 const MAX_ADDR: usize = 1 << 39;
136
137 /// Create a new instance from an active page table, covering the maximum RAM size.
138 pub fn new(page_table: mmu::PageTable) -> Self {
Andrew Walbran19690632022-12-07 16:41:30 +0000139 Self {
140 total: Self::BASE..Self::MAX_ADDR,
141 page_table,
142 regions: ArrayVec::new(),
143 mmio_regions: ArrayVec::new(),
144 }
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000145 }
146
147 /// Resize the total RAM size.
148 ///
149 /// This function fails if it contains regions that are not included within the new size.
150 pub fn shrink(&mut self, range: &MemoryRange) -> Result<()> {
151 if range.start != self.total.start {
152 return Err(MemoryTrackerError::DifferentBaseAddress);
153 }
154 if self.total.end < range.end {
155 return Err(MemoryTrackerError::SizeTooLarge);
156 }
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +0000157 if !self.regions.iter().all(|r| r.is_within(range)) {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000158 return Err(MemoryTrackerError::SizeTooSmall);
159 }
160
161 self.total = range.clone();
162 Ok(())
163 }
164
165 /// Allocate the address range for a const slice; returns None if failed.
166 pub fn alloc_range(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
Andrew Walbranda65ab12022-12-07 15:10:13 +0000167 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadOnly };
168 self.check(&region)?;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000169 self.page_table.map_rodata(range).map_err(|e| {
170 error!("Error during range allocation: {e}");
171 MemoryTrackerError::FailedToMap
172 })?;
Andrew Walbranda65ab12022-12-07 15:10:13 +0000173 self.add(region)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000174 }
175
176 /// Allocate the address range for a mutable slice; returns None if failed.
177 pub fn alloc_range_mut(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
Andrew Walbranda65ab12022-12-07 15:10:13 +0000178 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadWrite };
179 self.check(&region)?;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000180 self.page_table.map_data(range).map_err(|e| {
181 error!("Error during mutable range allocation: {e}");
182 MemoryTrackerError::FailedToMap
183 })?;
Andrew Walbranda65ab12022-12-07 15:10:13 +0000184 self.add(region)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000185 }
186
187 /// Allocate the address range for a const slice; returns None if failed.
188 pub fn alloc(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
189 self.alloc_range(&(base..(base + size.get())))
190 }
191
192 /// Allocate the address range for a mutable slice; returns None if failed.
193 pub fn alloc_mut(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
194 self.alloc_range_mut(&(base..(base + size.get())))
195 }
196
Andrew Walbran19690632022-12-07 16:41:30 +0000197 /// Checks that the given range of addresses is within the MMIO region, and then maps it
198 /// appropriately.
199 pub fn map_mmio_range(&mut self, range: MemoryRange) -> Result<()> {
200 // MMIO space is below the main memory region.
201 if range.end > self.total.start {
202 return Err(MemoryTrackerError::OutOfRange);
203 }
204 if self.mmio_regions.iter().any(|r| overlaps(r, &range)) {
205 return Err(MemoryTrackerError::Overlaps);
206 }
207 if self.mmio_regions.len() == self.mmio_regions.capacity() {
208 return Err(MemoryTrackerError::Full);
209 }
210
211 self.page_table.map_device(&range).map_err(|e| {
212 error!("Error during MMIO device mapping: {e}");
213 MemoryTrackerError::FailedToMap
214 })?;
215
216 for page_base in page_iterator(&range) {
217 mmio_guard::map(page_base)?;
218 }
219
220 if self.mmio_regions.try_push(range).is_some() {
221 return Err(MemoryTrackerError::Full);
222 }
223
224 Ok(())
225 }
226
Andrew Walbranda65ab12022-12-07 15:10:13 +0000227 /// Checks that the given region is within the range of the `MemoryTracker` and doesn't overlap
228 /// with any other previously allocated regions, and that the regions ArrayVec has capacity to
229 /// add it.
230 fn check(&self, region: &MemoryRegion) -> Result<()> {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000231 if !region.is_within(&self.total) {
232 return Err(MemoryTrackerError::OutOfRange);
233 }
Andrew Walbranda65ab12022-12-07 15:10:13 +0000234 if self.regions.iter().any(|r| r.overlaps(&region.range)) {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000235 return Err(MemoryTrackerError::Overlaps);
236 }
Andrew Walbranda65ab12022-12-07 15:10:13 +0000237 if self.regions.len() == self.regions.capacity() {
238 return Err(MemoryTrackerError::Full);
239 }
240 Ok(())
241 }
242
243 fn add(&mut self, region: MemoryRegion) -> Result<MemoryRange> {
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +0000244 if self.regions.try_push(region).is_some() {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000245 return Err(MemoryTrackerError::Full);
246 }
247
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +0000248 Ok(self.regions.last().unwrap().as_ref().clone())
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000249 }
Andrew Walbran19690632022-12-07 16:41:30 +0000250
251 /// Unmaps all tracked MMIO regions from the MMIO guard.
252 ///
253 /// Note that they are not unmapped from the page table.
254 pub fn mmio_unmap_all(&self) -> Result<()> {
255 for region in &self.mmio_regions {
256 for page_base in page_iterator(region) {
257 mmio_guard::unmap(page_base)?;
258 }
259 }
260
261 Ok(())
262 }
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000263}
264
265impl Drop for MemoryTracker {
266 fn drop(&mut self) {
Andrew Walbran19690632022-12-07 16:41:30 +0000267 for region in &self.regions {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000268 match region.mem_type {
269 MemoryType::ReadWrite => {
270 // TODO: Use page table's dirty bit to only flush pages that were touched.
271 helpers::flush_region(region.range.start, region.range.len())
272 }
273 MemoryType::ReadOnly => {}
274 }
275 }
276 }
277}
Andrew Walbran19690632022-12-07 16:41:30 +0000278
Andrew Walbran41ebe932022-12-14 15:22:30 +0000279/// Gives the KVM host read, write and execute permissions on the given memory range. If the range
280/// is not aligned with the memory protection granule then it will be extended on either end to
281/// align.
Andrew Walbran848decf2022-12-15 14:39:38 +0000282fn share_range(range: &MemoryRange, granule: usize) -> smccc::Result<()> {
Andrew Walbran41ebe932022-12-14 15:22:30 +0000283 for base in (align_down(range.start, granule)
284 .expect("Memory protection granule was not a power of two")..range.end)
285 .step_by(granule)
286 {
287 mem_share(base as u64)?;
288 }
289 Ok(())
290}
291
292/// Removes permission from the KVM host to access the given memory range which was previously
293/// shared. If the range is not aligned with the memory protection granule then it will be extended
294/// on either end to align.
Andrew Walbran848decf2022-12-15 14:39:38 +0000295fn unshare_range(range: &MemoryRange, granule: usize) -> smccc::Result<()> {
Andrew Walbran41ebe932022-12-14 15:22:30 +0000296 for base in (align_down(range.start, granule)
297 .expect("Memory protection granule was not a power of two")..range.end)
298 .step_by(granule)
299 {
300 mem_unshare(base as u64)?;
301 }
302 Ok(())
303}
304
Andrew Walbran848decf2022-12-15 14:39:38 +0000305/// Allocates a memory range of at least the given size from the global allocator, and shares it
306/// with the host. Returns a pointer to the buffer.
307///
308/// It will be aligned to the memory sharing granule size supported by the hypervisor.
309pub fn alloc_shared(size: usize) -> smccc::Result<NonNull<u8>> {
310 let layout = shared_buffer_layout(size)?;
311 let granule = layout.align();
312
313 // Safe because `shared_buffer_layout` panics if the size is 0, so the layout must have a
314 // non-zero size.
315 let buffer = unsafe { alloc_zeroed(layout) };
316
317 // TODO: Use let-else once we have Rust 1.65 in AOSP.
318 let buffer = if let Some(buffer) = NonNull::new(buffer) {
319 buffer
320 } else {
321 handle_alloc_error(layout);
322 };
323
324 let vaddr = buffer.as_ptr() as usize;
325 let paddr = virt_to_phys(vaddr);
326 // If share_range fails then we will leak the allocation, but that seems better than having it
327 // be reused while maybe still partially shared with the host.
328 share_range(&(paddr..paddr + layout.size()), granule)?;
329
330 Ok(buffer)
331}
332
333/// Unshares and deallocates a memory range which was previously allocated by `alloc_shared`.
334///
335/// The size passed in must be the size passed to the original `alloc_shared` call.
336///
337/// # Safety
338///
339/// The memory must have been allocated by `alloc_shared` with the same size, and not yet
340/// deallocated.
341pub unsafe fn dealloc_shared(vaddr: usize, size: usize) -> smccc::Result<()> {
342 let layout = shared_buffer_layout(size)?;
343 let granule = layout.align();
344
345 let paddr = virt_to_phys(vaddr);
346 unshare_range(&(paddr..paddr + layout.size()), granule)?;
347 // Safe because the memory was allocated by `alloc_shared` above using the same allocator, and
348 // the layout is the same as was used then.
349 unsafe { dealloc(vaddr as *mut u8, layout) };
350
351 Ok(())
352}
353
354/// Returns the layout to use for allocating a buffer of at least the given size shared with the
355/// host.
356///
357/// It will be aligned to the memory sharing granule size supported by the hypervisor.
358///
359/// Panics if `size` is 0.
360fn shared_buffer_layout(size: usize) -> smccc::Result<Layout> {
361 assert_ne!(size, 0);
362 let granule = hyp_meminfo()? as usize;
363 let allocated_size =
364 align_up(size, granule).expect("Memory protection granule was not a power of two");
365 Ok(Layout::from_size_align(allocated_size, granule).unwrap())
366}
367
Andrew Walbran19690632022-12-07 16:41:30 +0000368/// Returns an iterator which yields the base address of each 4 KiB page within the given range.
369fn page_iterator(range: &MemoryRange) -> impl Iterator<Item = usize> {
370 (page_4kb_of(range.start)..range.end).step_by(SIZE_4KB)
371}
Andrew Walbran848decf2022-12-15 14:39:38 +0000372
373/// Returns the intermediate physical address corresponding to the given virtual address.
374///
375/// As we use identity mapping for everything, this is just the identity function, but it's useful
376/// to use it to be explicit about where we are converting from virtual to physical address.
377pub fn virt_to_phys(vaddr: usize) -> usize {
378 vaddr
379}