blob: 06563214fcc03e9f9307951cec2c30aebf3c9304 [file] [log] [blame]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +00001// Copyright 2022, The Android Open Source Project
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Low-level allocation and tracking of main memory.
16
Andrew Walbran848decf2022-12-15 14:39:38 +000017#![deny(unsafe_op_in_unsafe_fn)]
18
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +000019use crate::helpers::{self, page_4kb_of, RangeExt, SIZE_4KB, SIZE_4MB};
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000020use crate::mmu;
Andrew Walbran848decf2022-12-15 14:39:38 +000021use alloc::alloc::alloc_zeroed;
22use alloc::alloc::dealloc;
23use alloc::alloc::handle_alloc_error;
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -070024use alloc::boxed::Box;
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +000025use alloc::vec::Vec;
Andrew Walbran87933f32023-05-09 15:29:06 +000026use buddy_system_allocator::{FrameAllocator, LockedFrameAllocator};
Andrew Walbran848decf2022-12-15 14:39:38 +000027use core::alloc::Layout;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000028use core::cmp::max;
29use core::cmp::min;
30use core::fmt;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000031use core::num::NonZeroUsize;
32use core::ops::Range;
Andrew Walbran848decf2022-12-15 14:39:38 +000033use core::ptr::NonNull;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000034use core::result;
Alice Wang90e6f162023-04-17 13:49:45 +000035use hyp::get_hypervisor;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000036use log::error;
Pierre-Clément Tosi90238c52023-04-27 17:59:10 +000037use log::trace;
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -070038use once_cell::race::OnceBox;
Jakob Vukalovic85a00d72023-04-20 09:51:10 +010039use spin::mutex::SpinMutex;
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000040use tinyvec::ArrayVec;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000041
Jiyong Park0ee65392023-03-27 20:52:45 +090042/// Base of the system's contiguous "main" memory.
43pub const BASE_ADDR: usize = 0x8000_0000;
44/// First address that can't be translated by a level 1 TTBR0_EL1.
45pub const MAX_ADDR: usize = 1 << 40;
46
Andrew Walbran0d8b54d2022-12-08 16:32:33 +000047pub type MemoryRange = Range<usize>;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000048
Jakob Vukalovic85a00d72023-04-20 09:51:10 +010049pub static MEMORY: SpinMutex<Option<MemoryTracker>> = SpinMutex::new(None);
50unsafe impl Send for MemoryTracker {}
51
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000052#[derive(Clone, Copy, Debug, Default)]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000053enum MemoryType {
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000054 #[default]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000055 ReadOnly,
56 ReadWrite,
57}
58
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +000059#[derive(Clone, Debug, Default)]
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000060struct MemoryRegion {
61 range: MemoryRange,
62 mem_type: MemoryType,
63}
64
65impl MemoryRegion {
66 /// True if the instance overlaps with the passed range.
67 pub fn overlaps(&self, range: &MemoryRange) -> bool {
Andrew Walbran19690632022-12-07 16:41:30 +000068 overlaps(&self.range, range)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000069 }
70
71 /// True if the instance is fully contained within the passed range.
72 pub fn is_within(&self, range: &MemoryRange) -> bool {
Srivatsa Vaddagiric25d68e2023-04-19 22:56:33 -070073 self.as_ref().is_within(range)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000074 }
75}
76
77impl AsRef<MemoryRange> for MemoryRegion {
78 fn as_ref(&self) -> &MemoryRange {
79 &self.range
80 }
81}
82
Andrew Walbran19690632022-12-07 16:41:30 +000083/// Returns true if one range overlaps with the other at all.
84fn overlaps<T: Copy + Ord>(a: &Range<T>, b: &Range<T>) -> bool {
85 max(a.start, b.start) < min(a.end, b.end)
86}
87
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000088/// Tracks non-overlapping slices of main memory.
89pub struct MemoryTracker {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000090 total: MemoryRange,
91 page_table: mmu::PageTable,
Andrew Walbran19690632022-12-07 16:41:30 +000092 regions: ArrayVec<[MemoryRegion; MemoryTracker::CAPACITY]>,
93 mmio_regions: ArrayVec<[MemoryRange; MemoryTracker::MMIO_CAPACITY]>,
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +000094}
95
96/// Errors for MemoryTracker operations.
97#[derive(Debug, Clone)]
98pub enum MemoryTrackerError {
99 /// Tried to modify the memory base address.
100 DifferentBaseAddress,
101 /// Tried to shrink to a larger memory size.
102 SizeTooLarge,
103 /// Tracked regions would not fit in memory size.
104 SizeTooSmall,
105 /// Reached limit number of tracked regions.
106 Full,
107 /// Region is out of the tracked memory address space.
108 OutOfRange,
109 /// New region overlaps with tracked regions.
110 Overlaps,
111 /// Region couldn't be mapped.
112 FailedToMap,
Alice Wang90e6f162023-04-17 13:49:45 +0000113 /// Error from the interaction with the hypervisor.
114 Hypervisor(hyp::Error),
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000115 /// Failure to set `SHARED_MEMORY`.
116 SharedMemorySetFailure,
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700117 /// Failure to set `SHARED_POOL`.
118 SharedPoolSetFailure,
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000119}
120
121impl fmt::Display for MemoryTrackerError {
122 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
123 match self {
124 Self::DifferentBaseAddress => write!(f, "Received different base address"),
125 Self::SizeTooLarge => write!(f, "Tried to shrink to a larger memory size"),
126 Self::SizeTooSmall => write!(f, "Tracked regions would not fit in memory size"),
127 Self::Full => write!(f, "Reached limit number of tracked regions"),
128 Self::OutOfRange => write!(f, "Region is out of the tracked memory address space"),
129 Self::Overlaps => write!(f, "New region overlaps with tracked regions"),
130 Self::FailedToMap => write!(f, "Failed to map the new region"),
Alice Wang90e6f162023-04-17 13:49:45 +0000131 Self::Hypervisor(e) => e.fmt(f),
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000132 Self::SharedMemorySetFailure => write!(f, "Failed to set SHARED_MEMORY"),
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700133 Self::SharedPoolSetFailure => write!(f, "Failed to set SHARED_POOL"),
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000134 }
135 }
136}
137
Alice Wang90e6f162023-04-17 13:49:45 +0000138impl From<hyp::Error> for MemoryTrackerError {
139 fn from(e: hyp::Error) -> Self {
140 Self::Hypervisor(e)
Andrew Walbran19690632022-12-07 16:41:30 +0000141 }
142}
143
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000144type Result<T> = result::Result<T, MemoryTrackerError>;
145
Andrew Walbran87933f32023-05-09 15:29:06 +0000146static SHARED_POOL: OnceBox<LockedFrameAllocator<32>> = OnceBox::new();
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000147static SHARED_MEMORY: SpinMutex<Option<MemorySharer>> = SpinMutex::new(None);
148
149/// Allocates memory on the heap and shares it with the host.
150///
151/// Unshares all pages when dropped.
152pub struct MemorySharer {
153 granule: usize,
154 shared_regions: Vec<(usize, Layout)>,
155}
156
157impl MemorySharer {
158 const INIT_CAP: usize = 10;
159
160 pub fn new(granule: usize) -> Self {
161 assert!(granule.is_power_of_two());
162 Self { granule, shared_regions: Vec::with_capacity(Self::INIT_CAP) }
163 }
164
165 /// Get from the global allocator a granule-aligned region that suits `hint` and share it.
Andrew Walbran87933f32023-05-09 15:29:06 +0000166 pub fn refill(&mut self, pool: &mut FrameAllocator<32>, hint: Layout) {
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000167 let layout = hint.align_to(self.granule).unwrap().pad_to_align();
168 assert_ne!(layout.size(), 0);
169 // SAFETY - layout has non-zero size.
170 let Some(shared) = NonNull::new(unsafe { alloc_zeroed(layout) }) else {
171 handle_alloc_error(layout);
172 };
173
174 let base = shared.as_ptr() as usize;
175 let end = base.checked_add(layout.size()).unwrap();
176 trace!("Sharing memory region {:#x?}", base..end);
177 for vaddr in (base..end).step_by(self.granule) {
178 let vaddr = NonNull::new(vaddr as *mut _).unwrap();
179 get_hypervisor().mem_share(virt_to_phys(vaddr).try_into().unwrap()).unwrap();
180 }
181 self.shared_regions.push((base, layout));
182
Andrew Walbran87933f32023-05-09 15:29:06 +0000183 pool.add_frame(base, end);
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000184 }
185}
186
187impl Drop for MemorySharer {
188 fn drop(&mut self) {
189 while let Some((base, layout)) = self.shared_regions.pop() {
190 let end = base.checked_add(layout.size()).unwrap();
191 trace!("Unsharing memory region {:#x?}", base..end);
192 for vaddr in (base..end).step_by(self.granule) {
193 let vaddr = NonNull::new(vaddr as *mut _).unwrap();
194 get_hypervisor().mem_unshare(virt_to_phys(vaddr).try_into().unwrap()).unwrap();
195 }
196
197 // SAFETY - The region was obtained from alloc_zeroed() with the recorded layout.
198 unsafe { dealloc(base as *mut _, layout) };
199 }
200 }
201}
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700202
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000203impl MemoryTracker {
204 const CAPACITY: usize = 5;
Andrew Walbran19690632022-12-07 16:41:30 +0000205 const MMIO_CAPACITY: usize = 5;
Pierre-Clément Tosi164a6f52023-04-18 19:29:11 +0100206 const PVMFW_RANGE: MemoryRange = (BASE_ADDR - SIZE_4MB)..BASE_ADDR;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000207
208 /// Create a new instance from an active page table, covering the maximum RAM size.
209 pub fn new(page_table: mmu::PageTable) -> Self {
Andrew Walbran19690632022-12-07 16:41:30 +0000210 Self {
Jiyong Park0ee65392023-03-27 20:52:45 +0900211 total: BASE_ADDR..MAX_ADDR,
Andrew Walbran19690632022-12-07 16:41:30 +0000212 page_table,
213 regions: ArrayVec::new(),
214 mmio_regions: ArrayVec::new(),
215 }
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000216 }
217
218 /// Resize the total RAM size.
219 ///
220 /// This function fails if it contains regions that are not included within the new size.
221 pub fn shrink(&mut self, range: &MemoryRange) -> Result<()> {
222 if range.start != self.total.start {
223 return Err(MemoryTrackerError::DifferentBaseAddress);
224 }
225 if self.total.end < range.end {
226 return Err(MemoryTrackerError::SizeTooLarge);
227 }
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +0000228 if !self.regions.iter().all(|r| r.is_within(range)) {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000229 return Err(MemoryTrackerError::SizeTooSmall);
230 }
231
232 self.total = range.clone();
233 Ok(())
234 }
235
236 /// Allocate the address range for a const slice; returns None if failed.
237 pub fn alloc_range(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
Andrew Walbranda65ab12022-12-07 15:10:13 +0000238 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadOnly };
239 self.check(&region)?;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000240 self.page_table.map_rodata(range).map_err(|e| {
241 error!("Error during range allocation: {e}");
242 MemoryTrackerError::FailedToMap
243 })?;
Andrew Walbranda65ab12022-12-07 15:10:13 +0000244 self.add(region)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000245 }
246
247 /// Allocate the address range for a mutable slice; returns None if failed.
248 pub fn alloc_range_mut(&mut self, range: &MemoryRange) -> Result<MemoryRange> {
Andrew Walbranda65ab12022-12-07 15:10:13 +0000249 let region = MemoryRegion { range: range.clone(), mem_type: MemoryType::ReadWrite };
250 self.check(&region)?;
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000251 self.page_table.map_data(range).map_err(|e| {
252 error!("Error during mutable range allocation: {e}");
253 MemoryTrackerError::FailedToMap
254 })?;
Andrew Walbranda65ab12022-12-07 15:10:13 +0000255 self.add(region)
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000256 }
257
258 /// Allocate the address range for a const slice; returns None if failed.
259 pub fn alloc(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
260 self.alloc_range(&(base..(base + size.get())))
261 }
262
263 /// Allocate the address range for a mutable slice; returns None if failed.
264 pub fn alloc_mut(&mut self, base: usize, size: NonZeroUsize) -> Result<MemoryRange> {
265 self.alloc_range_mut(&(base..(base + size.get())))
266 }
267
Andrew Walbran19690632022-12-07 16:41:30 +0000268 /// Checks that the given range of addresses is within the MMIO region, and then maps it
269 /// appropriately.
270 pub fn map_mmio_range(&mut self, range: MemoryRange) -> Result<()> {
271 // MMIO space is below the main memory region.
Pierre-Clément Tosi164a6f52023-04-18 19:29:11 +0100272 if range.end > self.total.start || overlaps(&Self::PVMFW_RANGE, &range) {
Andrew Walbran19690632022-12-07 16:41:30 +0000273 return Err(MemoryTrackerError::OutOfRange);
274 }
275 if self.mmio_regions.iter().any(|r| overlaps(r, &range)) {
276 return Err(MemoryTrackerError::Overlaps);
277 }
278 if self.mmio_regions.len() == self.mmio_regions.capacity() {
279 return Err(MemoryTrackerError::Full);
280 }
281
282 self.page_table.map_device(&range).map_err(|e| {
283 error!("Error during MMIO device mapping: {e}");
284 MemoryTrackerError::FailedToMap
285 })?;
286
287 for page_base in page_iterator(&range) {
Alice Wang90e6f162023-04-17 13:49:45 +0000288 get_hypervisor().mmio_guard_map(page_base)?;
Andrew Walbran19690632022-12-07 16:41:30 +0000289 }
290
291 if self.mmio_regions.try_push(range).is_some() {
292 return Err(MemoryTrackerError::Full);
293 }
294
295 Ok(())
296 }
297
Andrew Walbranda65ab12022-12-07 15:10:13 +0000298 /// Checks that the given region is within the range of the `MemoryTracker` and doesn't overlap
299 /// with any other previously allocated regions, and that the regions ArrayVec has capacity to
300 /// add it.
301 fn check(&self, region: &MemoryRegion) -> Result<()> {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000302 if !region.is_within(&self.total) {
303 return Err(MemoryTrackerError::OutOfRange);
304 }
Andrew Walbranda65ab12022-12-07 15:10:13 +0000305 if self.regions.iter().any(|r| r.overlaps(&region.range)) {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000306 return Err(MemoryTrackerError::Overlaps);
307 }
Andrew Walbranda65ab12022-12-07 15:10:13 +0000308 if self.regions.len() == self.regions.capacity() {
309 return Err(MemoryTrackerError::Full);
310 }
311 Ok(())
312 }
313
314 fn add(&mut self, region: MemoryRegion) -> Result<MemoryRange> {
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +0000315 if self.regions.try_push(region).is_some() {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000316 return Err(MemoryTrackerError::Full);
317 }
318
Pierre-Clément Tosi328dfb62022-11-25 18:20:42 +0000319 Ok(self.regions.last().unwrap().as_ref().clone())
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000320 }
Andrew Walbran19690632022-12-07 16:41:30 +0000321
322 /// Unmaps all tracked MMIO regions from the MMIO guard.
323 ///
324 /// Note that they are not unmapped from the page table.
325 pub fn mmio_unmap_all(&self) -> Result<()> {
326 for region in &self.mmio_regions {
327 for page_base in page_iterator(region) {
Alice Wang90e6f162023-04-17 13:49:45 +0000328 get_hypervisor().mmio_guard_unmap(page_base)?;
Andrew Walbran19690632022-12-07 16:41:30 +0000329 }
330 }
331
332 Ok(())
333 }
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700334
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000335 /// Initialize the shared heap to dynamically share memory from the global allocator.
336 pub fn init_dynamic_shared_pool(&mut self) -> Result<()> {
337 let granule = get_hypervisor().memory_protection_granule()?;
338 let previous = SHARED_MEMORY.lock().replace(MemorySharer::new(granule));
339 if previous.is_some() {
340 return Err(MemoryTrackerError::SharedMemorySetFailure);
341 }
342
343 SHARED_POOL
Andrew Walbran87933f32023-05-09 15:29:06 +0000344 .set(Box::new(LockedFrameAllocator::new()))
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000345 .map_err(|_| MemoryTrackerError::SharedPoolSetFailure)?;
346
347 Ok(())
348 }
349
350 /// Initialize the shared heap from a static region of memory.
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700351 ///
352 /// Some hypervisors such as Gunyah do not support a MemShare API for guest
353 /// to share its memory with host. Instead they allow host to designate part
354 /// of guest memory as "shared" ahead of guest starting its execution. The
355 /// shared memory region is indicated in swiotlb node. On such platforms use
356 /// a separate heap to allocate buffers that can be shared with host.
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000357 pub fn init_static_shared_pool(&mut self, range: Range<usize>) -> Result<()> {
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700358 let size = NonZeroUsize::new(range.len()).unwrap();
359 let range = self.alloc_mut(range.start, size)?;
Andrew Walbran87933f32023-05-09 15:29:06 +0000360 let shared_pool = LockedFrameAllocator::<32>::new();
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700361
Andrew Walbran87933f32023-05-09 15:29:06 +0000362 shared_pool.lock().insert(range);
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700363
364 SHARED_POOL
365 .set(Box::new(shared_pool))
366 .map_err(|_| MemoryTrackerError::SharedPoolSetFailure)?;
367
368 Ok(())
369 }
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000370
371 /// Unshares any memory that may have been shared.
372 pub fn unshare_all_memory(&mut self) {
373 drop(SHARED_MEMORY.lock().take());
374 }
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000375}
376
377impl Drop for MemoryTracker {
378 fn drop(&mut self) {
Andrew Walbran19690632022-12-07 16:41:30 +0000379 for region in &self.regions {
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000380 match region.mem_type {
381 MemoryType::ReadWrite => {
Pierre-Clément Tosi73c2d642023-02-17 14:56:48 +0000382 // TODO(b/269738062): Use PT's dirty bit to only flush pages that were touched.
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000383 helpers::flush_region(region.range.start, region.range.len())
384 }
385 MemoryType::ReadOnly => {}
386 }
387 }
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000388 self.unshare_all_memory()
Pierre-Clément Tosia0934c12022-11-25 20:54:11 +0000389 }
390}
Andrew Walbran19690632022-12-07 16:41:30 +0000391
Andrew Walbran2b0c7fb2023-05-09 12:16:20 +0000392/// Allocates a memory range of at least the given size and alignment that is shared with the host.
393/// Returns a pointer to the buffer.
Pierre-Clément Tosi2d5bc582023-05-03 11:23:11 +0000394pub fn alloc_shared(layout: Layout) -> hyp::Result<NonNull<u8>> {
395 assert_ne!(layout.size(), 0);
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000396 let Some(buffer) = try_shared_alloc(layout) else {
Andrew Walbran848decf2022-12-15 14:39:38 +0000397 handle_alloc_error(layout);
398 };
399
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000400 trace!("Allocated shared buffer at {buffer:?} with {layout:?}");
Andrew Walbran848decf2022-12-15 14:39:38 +0000401 Ok(buffer)
402}
403
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000404fn try_shared_alloc(layout: Layout) -> Option<NonNull<u8>> {
405 let mut shared_pool = SHARED_POOL.get().unwrap().lock();
406
Andrew Walbran87933f32023-05-09 15:29:06 +0000407 if let Some(buffer) = shared_pool.alloc_aligned(layout) {
408 Some(NonNull::new(buffer as _).unwrap())
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000409 } else if let Some(shared_memory) = SHARED_MEMORY.lock().as_mut() {
410 shared_memory.refill(&mut shared_pool, layout);
Andrew Walbran87933f32023-05-09 15:29:06 +0000411 shared_pool.alloc_aligned(layout).map(|buffer| NonNull::new(buffer as _).unwrap())
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000412 } else {
413 None
414 }
415}
416
Andrew Walbran848decf2022-12-15 14:39:38 +0000417/// Unshares and deallocates a memory range which was previously allocated by `alloc_shared`.
418///
Andrew Walbran2b0c7fb2023-05-09 12:16:20 +0000419/// The layout passed in must be the same layout passed to the original `alloc_shared` call.
Andrew Walbran848decf2022-12-15 14:39:38 +0000420///
421/// # Safety
422///
Andrew Walbran2b0c7fb2023-05-09 12:16:20 +0000423/// The memory must have been allocated by `alloc_shared` with the same layout, and not yet
Andrew Walbran848decf2022-12-15 14:39:38 +0000424/// deallocated.
Pierre-Clément Tosi2d5bc582023-05-03 11:23:11 +0000425pub unsafe fn dealloc_shared(vaddr: NonNull<u8>, layout: Layout) -> hyp::Result<()> {
Andrew Walbran87933f32023-05-09 15:29:06 +0000426 SHARED_POOL.get().unwrap().lock().dealloc_aligned(vaddr.as_ptr() as usize, layout);
Srivatsa Vaddagiri37713ec2023-04-20 04:04:08 -0700427
Pierre-Clément Tosif19c0e62023-05-02 13:56:58 +0000428 trace!("Deallocated shared buffer at {vaddr:?} with {layout:?}");
Andrew Walbran848decf2022-12-15 14:39:38 +0000429 Ok(())
430}
431
Andrew Walbran19690632022-12-07 16:41:30 +0000432/// Returns an iterator which yields the base address of each 4 KiB page within the given range.
433fn page_iterator(range: &MemoryRange) -> impl Iterator<Item = usize> {
434 (page_4kb_of(range.start)..range.end).step_by(SIZE_4KB)
435}
Andrew Walbran848decf2022-12-15 14:39:38 +0000436
437/// Returns the intermediate physical address corresponding to the given virtual address.
438///
Andrew Walbran272bd7a2023-01-24 14:02:36 +0000439/// As we use identity mapping for everything, this is just a cast, but it's useful to use it to be
440/// explicit about where we are converting from virtual to physical address.
441pub fn virt_to_phys(vaddr: NonNull<u8>) -> usize {
442 vaddr.as_ptr() as _
443}
444
445/// Returns a pointer for the virtual address corresponding to the given non-zero intermediate
446/// physical address.
447///
448/// Panics if `paddr` is 0.
449pub fn phys_to_virt(paddr: usize) -> NonNull<u8> {
450 NonNull::new(paddr as _).unwrap()
Andrew Walbran848decf2022-12-15 14:39:38 +0000451}