From 2f8dbfa1e5d4130b70ded771f7935005b193fc04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=87a=C4=9Fatay=20Yi=C4=9Fit=20=C5=9Eahin?= Date: Tue, 25 Jun 2024 18:08:52 +0200 Subject: [PATCH] virtq: don't expose TransferTokens to the drivers TransferTokens carry the control descriptor that represents the indirect descriptor table in them, which is not of the same type for split queues and packed queues. This type difference is currently hidden behind the type erasing MemDescr type, but when we change the structure to use the actual type for it, we will need to make TransferToken generic. An actually typed TransferToken would not work for driver code that decides the type of queue to use at runtime, as we cannot make a boxed slice (which control descriptors represent) work with dynamically-sized elements. Furthermore, TransferTokens are tied to a particular type of queue, as it would be an error to dispatch a TransferToken that has a control descriptor that is suitable for a PackedVq through a SplitVq. To ensure dispatch through the correct type of queue, BufferTokens are created through queues and have a reference to the queue that created them, which forces the use of Rc, when a regular Box would likely suffice. --- src/drivers/fs/virtio_fs.rs | 11 +- src/drivers/net/virtio/mod.rs | 159 +-- src/drivers/virtio/virtqueue/mod.rs | 1304 ++++++++++-------------- src/drivers/virtio/virtqueue/packed.rs | 74 +- src/drivers/virtio/virtqueue/split.rs | 42 +- 5 files changed, 687 insertions(+), 903 deletions(-) diff --git a/src/drivers/fs/virtio_fs.rs b/src/drivers/fs/virtio_fs.rs index f9b118ecc4..99e267d4cf 100644 --- a/src/drivers/fs/virtio_fs.rs +++ b/src/drivers/fs/virtio_fs.rs @@ -19,7 +19,9 @@ use crate::drivers::virtio::transport::mmio::{ComCfg, IsrStatus, NotifCfg}; use crate::drivers::virtio::transport::pci::{ComCfg, IsrStatus, NotifCfg}; use crate::drivers::virtio::virtqueue::error::VirtqError; use crate::drivers::virtio::virtqueue::split::SplitVq; -use crate::drivers::virtio::virtqueue::{AsSliceU8, BufferType, Virtq, VqIndex, VqSize}; +use crate::drivers::virtio::virtqueue::{ + AsSliceU8, BufferToken, BufferType, Virtq, VqIndex, VqSize, +}; use crate::fs::fuse::{self, FuseInterface, Rsp, RspHeader}; /// A wrapper struct for the raw configuration structure. @@ -176,11 +178,8 @@ impl FuseInterface for VirtioFsDriver { ] }; - let transfer_tkn = self.vqueues[1] - .clone() - .prep_transfer_from_raw(send, recv, BufferType::Direct) - .unwrap(); - transfer_tkn.dispatch_blocking()?; + let buffer_tkn = BufferToken::from_existing(send, recv).unwrap(); + self.vqueues[1].dispatch_blocking(buffer_tkn, BufferType::Direct)?; Ok(unsafe { Rsp { headers: rsp_headers.assume_init(), diff --git a/src/drivers/net/virtio/mod.rs b/src/drivers/net/virtio/mod.rs index 4ed8a54f65..a6514e429e 100644 --- a/src/drivers/net/virtio/mod.rs +++ b/src/drivers/net/virtio/mod.rs @@ -10,7 +10,6 @@ cfg_if::cfg_if! { } } -use alloc::boxed::Box; use alloc::rc::Rc; use alloc::vec::Vec; use core::cmp::Ordering; @@ -61,8 +60,8 @@ impl CtrlQueue { pub struct RxQueues { vqs: Vec>, - poll_sender: async_channel::Sender>, - poll_receiver: async_channel::Receiver>, + poll_sender: async_channel::Sender, + poll_receiver: async_channel::Receiver, is_multi: bool, } @@ -77,12 +76,12 @@ impl RxQueues { } } - /// Takes care if handling packets correctly which need some processing after being received. - /// This currently include nothing. But in the future it might include among others:: + /// Takes care of handling packets correctly which need some processing after being received. + /// This currently include nothing. But in the future it might include among others: /// * Calculating missing checksums /// * Merging receive buffers, by simply checking the poll_queue (if VIRTIO_NET_F_MRG_BUF) - fn post_processing(buffer_tkn: Box) -> Result, VirtioNetError> { - Ok(buffer_tkn) + fn post_processing(_buffer_tkn: &mut BufferToken) -> Result<(), VirtioNetError> { + Ok(()) } /// Adds a given queue to the underlying vector and populates the queue with RecvBuffers. @@ -102,7 +101,7 @@ impl RxQueues { // let spec = BuffSpec::Single(Bytes::new(rx_size).unwrap()); for _ in 0..num_buff { - let buff_tkn = match vq.clone().prep_buffer(None, Some(spec.clone())) { + let buff_tkn = match BufferToken::new(None, Some(spec.clone())) { Ok(tkn) => tkn, Err(_vq_err) => { error!("Setup of network queue failed, which should not happen!"); @@ -113,10 +112,12 @@ impl RxQueues { // BufferTokens are directly provided to the queue // TransferTokens are directly dispatched // Transfers will be awaited at the queue - match buff_tkn - .provide(BufferType::Direct) - .dispatch_await(self.poll_sender.clone(), false) - { + match vq.dispatch_await( + buff_tkn, + self.poll_sender.clone(), + false, + BufferType::Direct, + ) { Ok(_) => (), Err(_) => { error!("Descriptor IDs were exhausted earlier than expected."); @@ -133,7 +134,7 @@ impl RxQueues { } } - fn get_next(&mut self) -> Option> { + fn get_next(&mut self) -> Option { let transfer = self.poll_receiver.try_recv(); transfer @@ -181,8 +182,8 @@ impl RxQueues { /// to the respective queue structures. pub struct TxQueues { vqs: Vec>, - poll_sender: async_channel::Sender>, - poll_receiver: async_channel::Receiver>, + poll_sender: async_channel::Sender, + poll_receiver: async_channel::Receiver, ready_queue: Vec, /// Indicates, whether the Driver/Device are using multiple /// queues for communication. @@ -253,13 +254,11 @@ impl TxQueues { let num_buff: u16 = vq.size().into(); for _ in 0..num_buff { - self.ready_queue.push( - vq.clone() - .prep_buffer(Some(spec.clone()), None) - .unwrap() - .write_seq(Some(&Hdr::default()), None::<&Hdr>) - .unwrap(), - ) + let mut buffer_tkn = BufferToken::new(Some(spec.clone()), None).unwrap(); + buffer_tkn + .write_seq(Some(&Hdr::default()), None::<&Hdr>) + .unwrap(); + self.ready_queue.push(buffer_tkn) } } else { // Virtio specification v1.1. - 5.1.6.2 point 5. @@ -275,13 +274,11 @@ impl TxQueues { let num_buff: u16 = vq.size().into(); for _ in 0..num_buff { - self.ready_queue.push( - vq.clone() - .prep_buffer(Some(spec.clone()), None) - .unwrap() - .write_seq(Some(&Hdr::default()), None::<&Hdr>) - .unwrap(), - ) + let mut buffer_tkn = BufferToken::new(Some(spec.clone()), None).unwrap(); + buffer_tkn + .write_seq(Some(&Hdr::default()), None::<&Hdr>) + .unwrap(); + self.ready_queue.push(buffer_tkn) } } } else { @@ -317,16 +314,16 @@ impl TxQueues { self.poll(); } - while let Ok(buffer_token) = self.poll_receiver.try_recv() { - let mut tkn = buffer_token.reset(); - let (send_len, _) = tkn.len(); + while let Ok(mut buffer_token) = self.poll_receiver.try_recv() { + buffer_token.reset(); + let (send_len, _) = buffer_token.len(); match send_len.cmp(&len) { Ordering::Less => {} - Ordering::Equal => return Some((tkn, 0)), + Ordering::Equal => return Some((buffer_token, 0)), Ordering::Greater => { - tkn.restr_size(Some(len), None).unwrap(); - return Some((tkn, 0)); + buffer_token.restr_size(Some(len), None).unwrap(); + return Some((buffer_token, 0)); } } } @@ -334,7 +331,7 @@ impl TxQueues { // As usize is currently safe as the minimal usize is defined as 16bit in rust. let spec = BuffSpec::Single(Bytes::new(len).unwrap()); - match self.vqs[0].clone().prep_buffer(Some(spec), None) { + match BufferToken::new(Some(spec), None) { Ok(tkn) => Some((tkn, 0)), Err(_) => { // Here it is possible if multiple queues are enabled to get another buffertoken from them! @@ -457,9 +454,13 @@ impl NetworkDriver for VirtioNetDriver { .into(); } - buff_tkn - .provide(BufferType::Direct) - .dispatch_await(self.send_vqs.poll_sender.clone(), false) + self.send_vqs.vqs[0] + .dispatch_await( + buff_tkn, + self.send_vqs.poll_sender.clone(), + false, + BufferType::Direct, + ) .unwrap(); result @@ -470,8 +471,8 @@ impl NetworkDriver for VirtioNetDriver { fn receive_packet(&mut self) -> Option<(RxToken, TxToken)> { match self.recv_vqs.get_next() { - Some(transfer) => { - let transfer = match RxQueues::post_processing(transfer) { + Some(mut buffer_tkn) => { + match RxQueues::post_processing(&mut buffer_tkn) { Ok(trf) => trf, Err(vnet_err) => { warn!("Post processing failed. Err: {:?}", vnet_err); @@ -479,7 +480,7 @@ impl NetworkDriver for VirtioNetDriver { } }; - let (_, recv_data_opt) = transfer.as_slices().unwrap(); + let (_, recv_data_opt) = buffer_tkn.as_slices().unwrap(); let mut recv_data = recv_data_opt.unwrap(); // If the given length isn't 1, we currently fail. @@ -491,10 +492,15 @@ impl NetworkDriver for VirtioNetDriver { // drop packets with invalid packet size if packet.len() < HEADER_SIZE { - transfer - .reset() - .provide(BufferType::Direct) - .dispatch_await(self.recv_vqs.poll_sender.clone(), false) + buffer_tkn.reset(); + + self.recv_vqs.vqs[0] + .dispatch_await( + buffer_tkn, + self.recv_vqs.poll_sender.clone(), + false, + BufferType::Direct, + ) .unwrap(); return None; @@ -509,45 +515,60 @@ impl NetworkDriver for VirtioNetDriver { let num_buffers = header.num_buffers; vec_data.extend_from_slice(&packet[mem::size_of::()..]); - transfer - .reset() - .provide(BufferType::Direct) - .dispatch_await(self.recv_vqs.poll_sender.clone(), false) + buffer_tkn.reset(); + self.recv_vqs.vqs[0] + .dispatch_await( + buffer_tkn, + self.recv_vqs.poll_sender.clone(), + false, + BufferType::Direct, + ) .unwrap(); num_buffers }; for _ in 1..num_buffers.to_ne() { - let transfer = - match RxQueues::post_processing(self.recv_vqs.get_next().unwrap()) { - Ok(trf) => trf, - Err(vnet_err) => { - warn!("Post processing failed. Err: {:?}", vnet_err); - return None; - } - }; - - let (_, recv_data_opt) = transfer.as_slices().unwrap(); + let mut buffer_tkn = self.recv_vqs.get_next().unwrap(); + match RxQueues::post_processing(&mut buffer_tkn) { + Ok(trf) => trf, + Err(vnet_err) => { + warn!("Post processing failed. Err: {:?}", vnet_err); + return None; + } + }; + + let (_, recv_data_opt) = buffer_tkn.as_slices().unwrap(); let mut recv_data = recv_data_opt.unwrap(); let packet = recv_data.pop().unwrap(); vec_data.extend_from_slice(packet); - transfer - .reset() - .provide(BufferType::Direct) - .dispatch_await(self.recv_vqs.poll_sender.clone(), false) + buffer_tkn.reset(); + + self.recv_vqs.vqs[0] + .dispatch_await( + buffer_tkn, + self.recv_vqs.poll_sender.clone(), + false, + BufferType::Direct, + ) .unwrap(); } Some((RxToken::new(vec_data), TxToken::new())) } else { error!("Empty transfer, or with wrong buffer layout. Reusing and returning error to user-space network driver..."); - transfer - .reset() + buffer_tkn.reset(); + buffer_tkn .write_seq(None::<&Hdr>, Some(&Hdr::default())) - .unwrap() - .provide(BufferType::Direct) - .dispatch_await(self.recv_vqs.poll_sender.clone(), false) + .unwrap(); + + self.recv_vqs.vqs[0] + .dispatch_await( + buffer_tkn, + self.recv_vqs.poll_sender.clone(), + false, + BufferType::Direct, + ) .unwrap(); None diff --git a/src/drivers/virtio/virtqueue/mod.rs b/src/drivers/virtio/virtqueue/mod.rs index f307361a41..d6233637ca 100644 --- a/src/drivers/virtio/virtqueue/mod.rs +++ b/src/drivers/virtio/virtqueue/mod.rs @@ -14,7 +14,6 @@ pub mod packed; pub mod split; use alloc::boxed::Box; -use alloc::rc::Rc; use alloc::vec::Vec; use core::alloc::{Allocator, Layout}; use core::cell::RefCell; @@ -91,7 +90,7 @@ impl From for u16 { } } -type BufferTokenSender = async_channel::Sender>; +type BufferTokenSender = async_channel::Sender; // Public interface of Virtq @@ -103,14 +102,54 @@ type BufferTokenSender = async_channel::Sender>; /// do need these features should refrain from providing support for both /// Virtqueue types and use the structs directly instead. #[allow(private_bounds)] -pub trait Virtq: VirtqPrivate { - /// Entry function which the TransferTokens can use, when they are dispatching - /// themselves via their `Rc` reference - /// +pub trait Virtq { /// The `notif` parameter indicates if the driver wants to have a notification for this specific /// transfer. This is only for performance optimization. As it is NOT ensured, that the device sees the /// updated notification flags before finishing transfers! - fn dispatch(&self, tkn: TransferToken, notif: bool) -> Result<(), VirtqError>; + fn dispatch_await( + &self, + tkn: BufferToken, + sender: BufferTokenSender, + notif: bool, + buffer_type: BufferType, + ) -> Result<(), VirtqError>; + + /// Dispatches the provided TransferToken to the respective queue and does + /// return when, the queue finished the transfer. + /// + /// The returned [BufferToken] can be reused, copied from + /// or return the underlying buffers. + /// + /// **INFO:** + /// Currently this function is constantly polling the queue while keeping the notifications disabled. + /// Upon finish notifications are enabled again. + fn dispatch_blocking( + &self, + tkn: BufferToken, + buffer_type: BufferType, + ) -> Result { + let (sender, receiver) = async_channel::bounded(1); + self.dispatch_await(tkn, sender, false, buffer_type)?; + + self.disable_notifs(); + + let result: BufferToken; + // Keep Spinning until the receive queue is filled + loop { + match receiver.try_recv() { + Ok(buffer_tkn) => { + result = buffer_tkn; + break; + } + Err(TryRecvError::Closed) => return Err(VirtqError::General), + Err(TryRecvError::Empty) => self.poll(), + } + } + + self.enable_notifs(); + + Ok(result) + } /// Enables interrupts for this virtqueue upon receiving a transfer fn enable_notifs(&self); @@ -125,30 +164,34 @@ pub trait Virtq: VirtqPrivate { /// these queues. fn poll(&self); - /// Dispatches a batch of transfer token. The buffers of the respective transfers are provided to the queue in + /// Dispatches a batch of [BufferToken]s. The buffers are provided to the queue in /// sequence. After the last buffer has been written, the queue marks the first buffer as available and triggers /// a device notification if wanted by the device. /// /// The `notif` parameter indicates if the driver wants to have a notification for this specific /// transfer. This is only for performance optimization. As it is NOT ensured, that the device sees the /// updated notification flags before finishing transfers! - fn dispatch_batch(&self, tkns: Vec, notif: bool) -> Result<(), VirtqError>; + fn dispatch_batch( + &self, + tkns: Vec<(BufferToken, BufferType)>, + notif: bool, + ) -> Result<(), VirtqError>; - /// Dispatches a batch of TransferTokens. The Transfers will be placed in to the `await_queue` + /// Dispatches a batch of [BufferToken]s. The tokens will be placed in to the `await_queue` /// upon finish. /// /// The `notif` parameter indicates if the driver wants to have a notification for this specific /// transfer. This is only for performance optimization. As it is NOT ensured, that the device sees the /// updated notification flags before finishing transfers! /// - /// Dispatches a batch of transfer token. The buffers of the respective transfers are provided to the queue in + /// The buffers are provided to the queue in /// sequence. After the last buffer has been written, the queue marks the first buffer as available and triggers /// a device notification if wanted by the device. /// /// Tokens to get a reference to the provided await_queue, where they will be placed upon finish. fn dispatch_batch_await( &self, - tkns: Vec, + tkns: Vec<(BufferToken, BufferType)>, await_queue: BufferTokenSender, notif: bool, ) -> Result<(), VirtqError>; @@ -174,245 +217,335 @@ pub trait Virtq: VirtqPrivate { // Returns the index (ID) of a Virtqueue. fn index(&self) -> VqIndex; +} - /// Provides the calley with a TransferToken. Fails upon multiple circumstances. - /// - /// **INFO:** - /// * Data behind the respective raw pointers will NOT be deallocated. Under no circumstances. - /// * Calley is responsible for ensuring the raw pointers will remain valid from start till end of transfer. - /// * start: call of `fn prep_transfer_from_raw()` - /// * end: return of the [BufferToken] via [TransferToken::dispatch_blocking] or its push to the [TransferToken::await_queue]. - /// * In case the underlying BufferToken is reused, the raw pointers MUST still be valid all the time - /// [BufferToken] exists. - /// * [BufferToken] created from this TransferTokens will ONLY allow to return a copy of the data. - /// * This is due to the fact, that the `Transfer.ret()` returns a `Box[u8]`, which must own - /// the array. This would lead to unwanted frees, if not handled carefully - /// * Drivers must take care of keeping a copy of the respective `*mut T` and `*mut K` for themselves - /// - /// **Parameters** - /// * send: `Option<(*mut T, BuffSpec)>` - /// * None: No send buffers are provided to the device - /// * Some: - /// * `T` defines the structure which will be provided to the device - /// * [BuffSpec] defines how this struct will be presented to the device. - /// See documentation on `BuffSpec` for details. - /// * recv: `Option<(*mut K, BuffSpec)>` - /// * None: No buffers, which are writable for the device are provided to the device. - /// * Some: - /// * `K` defines the structure which will be provided to the device - /// * [BuffSpec] defines how this struct will be presented to the device. - /// See documentation on `BuffSpec` for details. - /// - /// **Reasons for Failure:** - /// * Queue does not have enough descriptors left, to split `T` or `K` into the desired amount of memory chunks. - /// * Calley mixed `Indirect (Direct::Indirect())` with `Direct(BuffSpec::Single() or BuffSpec::Multiple())` descriptors. - /// - /// **Details on Usage:** - /// * `(Single, _ )` or `(_ , Single)` -> Results in one descriptor in the queue, hence Consumes one element. - /// * `(Multiple, _ )` or `(_ , Multiple)` -> Results in a list of descriptors in the queue. Consumes `Multiple.len()` elements. - /// * `(Single, Single)` -> Results in a descriptor list of two chained descriptors, hence Consumes two elements in the queue - /// * `(Single, Multiple)` or `(Multiple, Single)` -> Results in a descripotr list of `1 + Multiple.len(). Consumes equally - /// many elements in the queue. - /// * `(Indirect, _ )` or `(_, Indirect)` -> Resulsts in one descriptor in the queue, hence Consumes one element. - /// * `(Indirect, Indirect)` -> Resulsts in one descriptor in the queue, hence Consumes one element. - /// * Calley is not allowed to mix `Indirect` and `Direct` descriptors. Furthermore if the calley decides to use `Indirect` - /// descriptors, the queue will merge the send and recv structure as follows: - /// ```text - /// //+++++++++++++++++++++++ - /// //+ Queue + - /// //+++++++++++++++++++++++ - /// //+ Indirect descriptor + -> refers to a descriptor list in the form of -> ++++++++++++++++++++++++++ - /// //+ ... + + Descriptors for T + - /// //+++++++++++++++++++++++ + Descriptors for K + - /// // ++++++++++++++++++++++++++ - /// ``` - /// As a result indirect descriptors result in a single descriptor consumption in the actual queue. +/// These methods are an implementation detail and are meant only for consumption by the default method +/// implementations in [Virtq]. +trait VirtqPrivate { + fn create_indirect_ctrl( + &self, + send: Option<&[MemDescr]>, + recv: Option<&[MemDescr]>, + ) -> Result; + + /// Consumes the [BufferToken] and returns a [TransferToken], that can be used to actually start the transfer. /// - /// * If one wants to have a structure in the style of: - /// ``` - /// struct send_recv_struct { - /// // send_part: ... - /// // recv_part: ... - /// } - /// ``` - /// Then he must split the structure after the send part and provide the respective part via the send argument and the respective other - /// part via the recv argument. - fn prep_transfer_from_raw( - self: Rc, - send: &[&[u8]], - recv: &[&mut [MaybeUninit]], + /// After this call, the buffers are no longer writable. + fn transfer_token_from_buffer_token( + &self, + buff_tkn: BufferToken, + await_queue: Option, buffer_type: BufferType, - ) -> Result; + ) -> TransferToken { + let ctrl_desc = match buffer_type { + BufferType::Direct => None, + BufferType::Indirect => Some( + self.create_indirect_ctrl( + buff_tkn.send_buff.as_ref().map(Buffer::as_slice), + buff_tkn.recv_buff.as_ref().map(Buffer::as_slice), + ) + .unwrap(), + ), + }; - /// The implementation of the method requires constraints that are incompatible with a trait object. - /// Because of this, we constrain it to static objects (via Sized) and call it from the implementation - /// of [Self::prep_buffer] inside the implementor. - fn prep_transfer_from_raw_static( - self: Rc, - send: &[&[u8]], - recv: &[&mut [MaybeUninit]], - buffer_type: BufferType, - ) -> Result - where - Self: Sized + 'static, - { - if send.is_empty() && recv.is_empty() { - return Err(VirtqError::BufferNotSpecified); + TransferToken { + buff_tkn, + await_queue, + ctrl_desc, } + } +} - let total_send_len = send.iter().map(|slice| slice.len()).sum(); - let total_recv_len = recv.iter().map(|slice| slice.len()).sum(); +/// Allows to check, if a given structure crosses a physical page boundary. +/// Returns true, if the structure does NOT cross a boundary or crosses only +/// contiguous physical page boundaries. +/// +/// Structures provided to the Queue must pass this test, otherwise the queue +/// currently panics. +pub fn check_bounds(data: &T) -> bool { + let slice = data.as_slice_u8(); - let send_desc_lst: Vec<_> = send - .iter() - .map(|slice| MemDescr::pull_from_raw(slice)) - .collect(); + let start_virt = ptr::from_ref(slice.first().unwrap()).addr(); + let end_virt = ptr::from_ref(slice.last().unwrap()).addr(); + let end_phy_calc = paging::virt_to_phys(VirtAddr::from(start_virt)) + (slice.len() - 1); + let end_phy = paging::virt_to_phys(VirtAddr::from(end_virt)); - let recv_desc_lst: Vec<_> = recv - .iter() - .map(|slice| MemDescr::pull_from_raw(slice)) - .collect(); + end_phy == end_phy_calc +} - let ctrl_desc = match buffer_type { - BufferType::Direct => None, - BufferType::Indirect => Some(self.create_indirect_ctrl( - if !send.is_empty() { - Some(&send_desc_lst) - } else { - None - }, - if !recv.is_empty() { - Some(&recv_desc_lst) - } else { - None - }, - )?), - }; +/// Allows to check, if a given slice crosses a physical page boundary. +/// Returns true, if the slice does NOT cross a boundary or crosses only +/// contiguous physical page boundaries. +/// Slice MUST come from a boxed value. Otherwise the slice might be moved and +/// the test of this function is not longer valid. +/// +/// This check is especially useful if one wants to check if slices +/// into which the queue will destructure a structure are valid for the queue. +/// +/// Slices provided to the Queue must pass this test, otherwise the queue +/// currently panics. +pub fn check_bounds_slice(slice: &[u8]) -> bool { + let start_virt = ptr::from_ref(slice.first().unwrap()).addr(); + let end_virt = ptr::from_ref(slice.last().unwrap()).addr(); + let end_phy_calc = paging::virt_to_phys(VirtAddr::from(start_virt)) + (slice.len() - 1); + let end_phy = paging::virt_to_phys(VirtAddr::from(end_virt)); - let send_buff = if !send.is_empty() { - Some(Buffer { - desc_lst: send_desc_lst.into_boxed_slice(), - len: total_send_len, - next_write: 0, - }) - } else { - None - }; + end_phy == end_phy_calc +} - let recv_buff = if !recv.is_empty() { - Some(Buffer { - desc_lst: recv_desc_lst.into_boxed_slice(), - len: total_recv_len, - next_write: 0, - }) - } else { - None - }; +/// Frees memory regions gained access to via `Transfer.ret_raw()`. +pub fn free_raw(ptr: *mut u8, len: usize) { + crate::mm::deallocate(VirtAddr::from(ptr as usize), len); +} - Ok(TransferToken { - buff_tkn: BufferToken { - recv_buff, - send_buff, - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }, - await_queue: None, - ctrl_desc, - }) +/// The trait needs to be implemented for +/// structures which are to be used to write data into buffers of a [BufferToken] via [BufferToken::write] or +/// `BufferToken.write_seq()`. +/// +/// **INFO:* +/// The trait provides a decent default implementation. Please look at the code for details. +/// The provided default implementation computes the size of the given structure via `core::mem::size_of_val(&self)` +/// and then casts the given `*const Self` pointer of the structure into an `*const u8`. +/// +/// Users must be really careful, and check, whether the memory representation of the given structure equals +/// the representation the device expects. It is advised to only use `#[repr(C)]` and to check the output +/// of `as_slice_u8` and `as_slice_u8_mut`. +pub trait AsSliceU8 { + /// Retruns the size of the structure + /// + /// In case of an unsized structure, the function should returns + /// the exact value of the structure. + fn len(&self) -> usize { + core::mem::size_of_val(self) } - /// Provides the calley with empty buffers as specified via the `send` and `recv` function parameters, (see [BuffSpec]), in form of - /// a [BufferToken]. - /// Fails upon multiple circumstances. - /// - /// **Parameters** - /// * send: `Option` - /// * None: No send buffers are provided to the device - /// * Some: - /// * [BuffSpec] defines the size of the buffer and how the buffer is - /// Buffer will be structured. See documentation on `BuffSpec` for details. - /// * recv: `Option` - /// * None: No buffers, which are writable for the device are provided to the device. - /// * Some: - /// * [BuffSpec] defines the size of the buffer and how the buffer is - /// Buffer will be structured. See documentation on `BuffSpec` for details. - /// - /// **Reasons for Failure:** - /// * Queue does not have enough descriptors left to create the desired amount of descriptors as indicated by the `BuffSpec`. - /// * Calley mixed `Indirect (Direct::Indirect())` with `Direct(BuffSpec::Single() or BuffSpec::Multiple())` descriptors. - /// * Systerm does not have enough memory resources left. + /// Returns a slice of the given structure. /// - /// **Details on Usage:** - /// * `(Single, _ )` or `(_ , Single)` -> Results in one descriptor in the queue, hence Consumes one element. - /// * `(Multiple, _ )` or `(_ , Multiple)` -> Results in a list of descriptors in the queue. Consumes `Multiple.len()` elements. - /// * `(Single, Single)` -> Results in a descriptor list of two chained descriptors, hence Consumes two elements in the queue - /// * `(Single, Multiple)` or `(Multiple, Single)` -> Results in a descripotr list of `1 + Multiple.len(). Consumes equally - /// many elements in the queue. - /// * `(Indirect, _ )` or `(_, Indirect)` -> Resulsts in one descriptor in the queue, hence Consumes one element. - /// * `(Indirect, Indirect)` -> Resulsts in one descriptor in the queue, hence Consumes one element. - /// * Calley is not allowed to mix `Indirect` and `Direct` descriptors. Furthermore if the calley decides to use `Indirect` - /// descriptors, the queue will merge the send and recv structure as follows: - /// ```text - /// //+++++++++++++++++++++++ - /// //+ Queue + - /// //+++++++++++++++++++++++ - /// //+ Indirect descriptor + -> refers to a descriptor list in the form of -> ++++++++++++++++++++++++++ - /// //+ ... + + Descriptors for T + - /// //+++++++++++++++++++++++ + Descriptors for K + - /// // ++++++++++++++++++++++++++ - /// ``` - /// As a result indirect descriptors result in a single descriptor consumption in the actual queue. - fn prep_buffer( - self: Rc, - send: Option>, - recv: Option>, - ) -> Result; - - /// The implementation of the method requires constraints that are incompatible with a trait object. - /// Because of this, we constrain it to static objects (via Sized) and call it from the implementation - /// of [Self::prep_buffer] inside the implementor. - fn prep_buffer_static( - self: Rc, - send: Option>, - recv: Option>, - ) -> Result - where - Self: Sized + 'static, - { - match (send, recv) { - // No buffers specified - (None, None) => Err(VirtqError::BufferNotSpecified), - // Send buffer specified, No recv buffer - (Some(spec), None) => { - match spec { - BuffSpec::Single(size) => match MemDescr::pull(size) { - Ok(desc) => { - let buffer = Buffer { - desc_lst: vec![desc].into_boxed_slice(), - len: size.into(), - next_write: 0, - }; + /// ** WARN:** + /// * The slice must be little endian coded in order to be understood by the device + /// * The slice must serialize the actual structure the device expects, as the queue will use + /// the addresses of the slice in order to refer to the structure. + fn as_slice_u8(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(ptr::from_ref(self) as *const u8, self.len()) } + } - Ok(BufferToken { - send_buff: Some(buffer), - recv_buff: None, - vq: self.clone(), - ret_send: true, - ret_recv: false, - reusable: true, - }) - } - Err(vq_err) => Err(vq_err), - }, - BuffSpec::Multiple(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut len = 0usize; + /// Returns a mutable slice of the given structure. + /// + /// ** WARN:** + /// * The slice must be little endian coded in order to be understood by the device + /// * The slice must serialize the actual structure the device expects, as the queue will use + /// the addresses of the slice in order to refer to the structure. + fn as_slice_u8_mut(&mut self) -> &mut [u8] { + unsafe { core::slice::from_raw_parts_mut(ptr::from_mut(self) as *mut u8, self.len()) } + } +} - for size in size_lst { - match MemDescr::pull(*size) { - Ok(desc) => desc_lst.push(desc), +/// The struct represents buffers which are ready to be send via the +/// virtqueue. Buffers can no longer be written or retrieved. +pub struct TransferToken { + /// Must be some in order to prevent drop + /// upon reuse. + buff_tkn: BufferToken, + /// Structure which allows to await Transfers + /// If Some, finished TransferTokens will be placed here + /// as finished `Transfers`. If None, only the state + /// of the Token will be changed. + await_queue: Option, + // Contains the [MemDescr] for the indirect table if the transfer is indirect. + ctrl_desc: Option, +} + +/// Public Interface for TransferToken +impl TransferToken { + /// Returns the number of descritprors that will be placed in the queue. + /// This number can differ from the `BufferToken.num_descr()` function value + /// as indirect buffers only consume one descriptor in the queue, but can have + /// more descriptors that are accessible via the descriptor in the queue. + fn num_consuming_descr(&self) -> u16 { + if self.ctrl_desc.is_some() { + 1 + } else { + self.buff_tkn.num_descr() + } + } +} + +/// The struct represents buffers which are ready to be written or to be send. +/// +/// BufferTokens can be written in two ways: +/// * in one step via `BufferToken.write() +/// * consumes BufferToken and returns a TransferToken +/// * sequentially via `BufferToken.write_seq() +/// +/// # Structure of the Token +/// The token can potentially hold both a *send* and a *recv* buffer, but MUST hold +/// one. +/// The *send* buffer is the data the device will read during a transfer, the *recv* buffer +/// is the data the device will write to during a transfer. +/// +/// # What are Buffers +/// A buffer represents multiple chunks of memory. Where each chunk can be of different size. +/// The chunks are named descriptors in the following. +/// +/// **For Example:** +/// A buffer could consist of 3 descriptors: +/// 1. First descriptor of 30 bytes +/// 2. Second descriptor of 10 bytes +/// 3. Third descriptor of 100 bytes +/// +/// Each of these descriptors consumes one "element" of the +/// respective virtqueue. +/// The maximum number of descriptors per buffer is bounded by the size of the virtqueue. +pub struct BufferToken { + send_buff: Option, + //send_desc_lst: Option>, + recv_buff: Option, + //recv_desc_lst: Option>, + /// Indicates whether the buff is returnable + ret_send: bool, + ret_recv: bool, + /// Indicates if the token is allowed + /// to be reused. + reusable: bool, +} + +// Private interface of BufferToken +impl BufferToken { + /// Returns the overall number of descriptors. + fn num_descr(&self) -> u16 { + let mut len = 0; + + if let Some(buffer) = &self.recv_buff { + len += buffer.num_descr(); + } + + if let Some(buffer) = &self.send_buff { + len += buffer.num_descr(); + } + len + } + + /// Resets all properties from the previous transfer. + /// + /// Includes: + /// * Resetting the write status inside the MemDescr. -> Allowing to rewrite the buffers + /// * Resetting the MemDescr length at initialization. This length might be reduced upon writes + /// of the driver or the device. + /// * Erazing all memory areas with zeros + fn reset_purge(&mut self) { + if let Some(buff) = self.send_buff.as_mut() { + buff.reset_write(); + let mut init_buff_len = 0usize; + for desc in buff.as_mut_slice() { + desc.len = desc._init_len; + init_buff_len += desc._init_len; + + // Resetting written memory + for byte in desc.deref_mut() { + *byte = 0; + } + } + buff.reset_len(init_buff_len); + } + + if let Some(buff) = self.recv_buff.as_mut() { + buff.reset_write(); + let mut init_buff_len = 0usize; + for desc in buff.as_mut_slice() { + desc.len = desc._init_len; + init_buff_len += desc._init_len; + + // Resetting written memory + for byte in desc.deref_mut() { + *byte = 0; + } + } + buff.reset_len(init_buff_len); + } + } + + /// Resets all properties from the previous transfer. + /// + /// Includes: + /// * Resetting the write status inside the MemDescr. -> Allowing to rewrite the buffers + /// * Resetting the MemDescr length at initialization. This length might be reduced upon writes + /// of the driver or the device. + pub fn reset(&mut self) { + if let Some(buff) = self.send_buff.as_mut() { + buff.reset_write(); + let mut init_buff_len = 0usize; + for desc in buff.as_mut_slice() { + desc.len = desc._init_len; + init_buff_len += desc._init_len; + } + buff.reset_len(init_buff_len); + } + + if let Some(buff) = self.recv_buff.as_mut() { + buff.reset_write(); + let mut init_buff_len = 0usize; + for desc in buff.as_mut_slice() { + desc.len = desc._init_len; + init_buff_len += desc._init_len; + } + buff.reset_len(init_buff_len); + } + } +} + +// Public interface of BufferToken +impl BufferToken { + /// Provides the caller with empty buffers as specified via the `send` and `recv` function parameters, (see [BuffSpec]), in form of + /// a [BufferToken]. + /// Fails upon multiple circumstances. + /// + /// **Parameters** + /// * send: `Option` + /// * None: No send buffers are provided to the device + /// * Some: + /// * [BuffSpec] defines the size of the buffer and how the buffer is + /// Buffer will be structured. See documentation on `BuffSpec` for details. + /// * recv: `Option` + /// * None: No buffers, which are writable for the device are provided to the device. + /// * Some: + /// * [BuffSpec] defines the size of the buffer and how the buffer is + /// Buffer will be structured. See documentation on `BuffSpec` for details. + /// + /// **Reasons for Failure:** + /// * Both `send` and `recv` are empty, which is not allowed by Virtio. + /// * System does not have enough heap memory left. + pub fn new(send: Option>, recv: Option>) -> Result { + match (send, recv) { + // No buffers specified + (None, None) => Err(VirtqError::BufferNotSpecified), + // Send buffer specified, No recv buffer + (Some(spec), None) => { + match spec { + BuffSpec::Single(size) => match MemDescr::pull(size) { + Ok(desc) => { + let buffer = Buffer { + desc_lst: vec![desc].into_boxed_slice(), + len: size.into(), + next_write: 0, + }; + + Ok(Self { + send_buff: Some(buffer), + recv_buff: None, + ret_send: true, + ret_recv: false, + reusable: true, + }) + } + Err(vq_err) => Err(vq_err), + }, + BuffSpec::Multiple(size_lst) => { + let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); + let mut len = 0usize; + + for size in size_lst { + match MemDescr::pull(*size) { + Ok(desc) => desc_lst.push(desc), Err(vq_err) => return Err(vq_err), } len += usize::from(*size); @@ -424,10 +557,9 @@ pub trait Virtq: VirtqPrivate { next_write: 0, }; - Ok(BufferToken { + Ok(Self { send_buff: Some(buffer), recv_buff: None, - vq: self.clone(), ret_send: true, ret_recv: false, reusable: true, @@ -450,10 +582,9 @@ pub trait Virtq: VirtqPrivate { next_write: 0, }; - Ok(BufferToken { + Ok(Self { send_buff: Some(buffer), recv_buff: None, - vq: self.clone(), ret_send: true, ret_recv: false, reusable: true, @@ -472,10 +603,9 @@ pub trait Virtq: VirtqPrivate { next_write: 0, }; - Ok(BufferToken { + Ok(Self { send_buff: None, recv_buff: Some(buffer), - vq: self.clone(), ret_send: false, ret_recv: true, reusable: true, @@ -501,10 +631,9 @@ pub trait Virtq: VirtqPrivate { next_write: 0, }; - Ok(BufferToken { + Ok(Self { send_buff: None, recv_buff: Some(buffer), - vq: self.clone(), ret_send: false, ret_recv: true, reusable: true, @@ -527,10 +656,9 @@ pub trait Virtq: VirtqPrivate { next_write: 0, }; - Ok(BufferToken { + Ok(Self { send_buff: None, recv_buff: Some(buffer), - vq: self.clone(), ret_send: false, ret_recv: true, reusable: true, @@ -560,10 +688,9 @@ pub trait Virtq: VirtqPrivate { Err(vq_err) => return Err(vq_err), }; - Ok(BufferToken { + Ok(Self { send_buff, recv_buff, - vq: self.clone(), ret_send: true, ret_recv: true, reusable: true, @@ -597,10 +724,9 @@ pub trait Virtq: VirtqPrivate { next_write: 0, }); - Ok(BufferToken { + Ok(Self { send_buff, recv_buff, - vq: self.clone(), ret_send: true, ret_recv: true, reusable: true, @@ -642,10 +768,9 @@ pub trait Virtq: VirtqPrivate { next_write: 0, }); - Ok(BufferToken { + Ok(Self { send_buff, recv_buff, - vq: self.clone(), ret_send: true, ret_recv: true, reusable: true, @@ -656,511 +781,164 @@ pub trait Virtq: VirtqPrivate { Vec::with_capacity(send_size_lst.len()); let mut send_len = 0usize; - for size in send_size_lst { - match MemDescr::pull(*size) { - Ok(desc) => send_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - send_len += usize::from(*size); - } - - let send_buff = Some(Buffer { - desc_lst: send_desc_lst.into_boxed_slice(), - len: send_len, - next_write: 0, - }); - - let recv_buff = match MemDescr::pull(recv_size) { - Ok(recv_desc) => Some(Buffer { - desc_lst: vec![recv_desc].into_boxed_slice(), - len: recv_size.into(), - next_write: 0, - }), - Err(vq_err) => return Err(vq_err), - }; - - Ok(BufferToken { - send_buff, - recv_buff, - vq: self.clone(), - ret_send: true, - ret_recv: true, - reusable: true, - }) - } - (BuffSpec::Indirect(send_size_lst), BuffSpec::Indirect(recv_size_lst)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut send_len = 0usize; - - for size in send_size_lst { - // As the indirect list does only consume one descriptor for the - // control descriptor, the actual list is untracked - send_desc_lst.push(MemDescr::pull(*size)?); - send_len += usize::from(*size); - } - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut recv_len = 0usize; - - for size in recv_size_lst { - // As the indirect list does only consume one descriptor for the - // control descriptor, the actual list is untracked - recv_desc_lst.push(MemDescr::pull(*size)?); - recv_len += usize::from(*size); - } - - let recv_buff = Some(Buffer { - desc_lst: recv_desc_lst.into_boxed_slice(), - len: recv_len, - next_write: 0, - }); - let send_buff = Some(Buffer { - desc_lst: send_desc_lst.into_boxed_slice(), - len: send_len, - next_write: 0, - }); - - Ok(BufferToken { - send_buff, - recv_buff, - vq: self.clone(), - ret_send: true, - ret_recv: true, - reusable: true, - }) - } - (BuffSpec::Indirect(_), BuffSpec::Single(_)) - | (BuffSpec::Indirect(_), BuffSpec::Multiple(_)) => Err(VirtqError::BufferInWithDirect), - (BuffSpec::Single(_), BuffSpec::Indirect(_)) - | (BuffSpec::Multiple(_), BuffSpec::Indirect(_)) => Err(VirtqError::BufferInWithDirect), - } - } - } - } -} - -/// These methods are an implementation detail and are meant only for consumption by the default method -/// implementations in [Virtq]. -trait VirtqPrivate { - fn create_indirect_ctrl( - &self, - send: Option<&[MemDescr]>, - recv: Option<&[MemDescr]>, - ) -> Result; -} - -/// Allows to check, if a given structure crosses a physical page boundary. -/// Returns true, if the structure does NOT cross a boundary or crosses only -/// contiguous physical page boundaries. -/// -/// Structures provided to the Queue must pass this test, otherwise the queue -/// currently panics. -pub fn check_bounds(data: &T) -> bool { - let slice = data.as_slice_u8(); - - let start_virt = ptr::from_ref(slice.first().unwrap()).addr(); - let end_virt = ptr::from_ref(slice.last().unwrap()).addr(); - let end_phy_calc = paging::virt_to_phys(VirtAddr::from(start_virt)) + (slice.len() - 1); - let end_phy = paging::virt_to_phys(VirtAddr::from(end_virt)); - - end_phy == end_phy_calc -} - -/// Allows to check, if a given slice crosses a physical page boundary. -/// Returns true, if the slice does NOT cross a boundary or crosses only -/// contiguous physical page boundaries. -/// Slice MUST come from a boxed value. Otherwise the slice might be moved and -/// the test of this function is not longer valid. -/// -/// This check is especially useful if one wants to check if slices -/// into which the queue will destructure a structure are valid for the queue. -/// -/// Slices provided to the Queue must pass this test, otherwise the queue -/// currently panics. -pub fn check_bounds_slice(slice: &[u8]) -> bool { - let start_virt = ptr::from_ref(slice.first().unwrap()).addr(); - let end_virt = ptr::from_ref(slice.last().unwrap()).addr(); - let end_phy_calc = paging::virt_to_phys(VirtAddr::from(start_virt)) + (slice.len() - 1); - let end_phy = paging::virt_to_phys(VirtAddr::from(end_virt)); - - end_phy == end_phy_calc -} - -/// Frees memory regions gained access to via `Transfer.ret_raw()`. -pub fn free_raw(ptr: *mut u8, len: usize) { - crate::mm::deallocate(VirtAddr::from(ptr as usize), len); -} - -/// Dispatches a batch of TransferTokens. The actual behaviour depends on the respective -/// virtqueue implementation. Please see the respective docs for details. -/// -/// **INFO:** -/// Due to the missing HashMap implementation in the kernel, this function currently uses a nested -/// for-loop. The first iteration is over the number if dispatched tokens. Inside this loop, the -/// function iterates over a list of all already "used" virtqueues. If the given token belongs to an -/// existing queue it is inserted into the corresponding list of tokens, if it belongs to no queue, -/// a new entry in the "used" virtqueues list is made. -/// This procedure can possibly be very slow. -/// -/// The `notif` parameter indicates if the driver wants to have a notification for this specific -/// transfer. This is only for performance optimization. As it is NOT ensured, that the device sees the -/// updated notification flags before finishing transfers! -pub fn dispatch_batch(tkns: Vec, notif: bool) -> Result<(), VirtqError> { - let mut used_vqs: Vec<(Rc, Vec)> = Vec::new(); - - // Sort the TransferTokens depending in the queue their coming from. - // then call dispatch_batch of that queue - for tkn in tkns { - let index = tkn.get_vq().index(); - let mut used = false; - let mut index_used = 0usize; - - for (pos, (vq, _)) in used_vqs.iter_mut().enumerate() { - if index == vq.index() { - index_used = pos; - used = true; - break; - } - } - - if used { - let (_, tkn_lst) = &mut used_vqs[index_used]; - tkn_lst.push(tkn); - } else { - let mut new_tkn_lst = Vec::new(); - let vq = tkn.get_vq(); - new_tkn_lst.push(tkn); - - used_vqs.push((vq, new_tkn_lst)) - } - } - - for (vq_ref, tkn_lst) in used_vqs { - vq_ref.dispatch_batch(tkn_lst, notif)?; - } - Ok(()) -} - -/// Dispatches a batch of TransferTokens. The Transfers will be placed in to the `await_queue` -/// upon finish. -/// -/// **INFO:** -/// Due to the missing HashMap implementation in the kernel, this function currently uses a nested -/// for-loop. The first iteration is over the number if dispatched tokens. Inside this loop, the -/// function iterates over a list of all already "used" virtqueues. If the given token belongs to an -/// existing queue it is inserted into the corresponding list of tokens, if it belongs to no queue, -/// a new entry in the "used" virtqueues list is made. -/// This procedure can possibly be very slow. -/// -/// The `notif` parameter indicates if the driver wants to have a notification for this specific -/// transfer. This is only for performance optimization. As it is NOT ensured, that the device sees the -/// updated notification flags before finishing transfers! -pub fn dispatch_batch_await( - tkns: Vec, - await_queue: BufferTokenSender, - notif: bool, -) -> Result<(), VirtqError> { - let mut used_vqs: Vec<(Rc, Vec)> = Vec::new(); - - // Sort the TransferTokens depending in the queue their coming from. - // then call dispatch_batch of that queue - for tkn in tkns { - let index = tkn.get_vq().index(); - let mut used = false; - let mut index_used = 0usize; - - for (pos, (vq, _)) in used_vqs.iter_mut().enumerate() { - if index == vq.index() { - index_used = pos; - used = true; - break; - } - } - - if used { - let (_, tkn_lst) = &mut used_vqs[index_used]; - tkn_lst.push(tkn); - } else { - let mut new_tkn_lst = Vec::new(); - let vq = tkn.get_vq(); - new_tkn_lst.push(tkn); - - used_vqs.push((vq, new_tkn_lst)) - } - } - - for (vq, tkn_lst) in used_vqs { - vq.dispatch_batch_await(tkn_lst, await_queue.clone(), notif)?; - } - Ok(()) -} - -/// The trait needs to be implemented for -/// structures which are to be used to write data into buffers of a [BufferToken] via [BufferToken::write] or -/// `BufferToken.write_seq()`. -/// -/// **INFO:* -/// The trait provides a decent default implementation. Please look at the code for details. -/// The provided default implementation computes the size of the given structure via `core::mem::size_of_val(&self)` -/// and then casts the given `*const Self` pointer of the structure into an `*const u8`. -/// -/// Users must be really careful, and check, whether the memory representation of the given structure equals -/// the representation the device expects. It is advised to only use `#[repr(C)]` and to check the output -/// of `as_slice_u8` and `as_slice_u8_mut`. -pub trait AsSliceU8 { - /// Retruns the size of the structure - /// - /// In case of an unsized structure, the function should returns - /// the exact value of the structure. - fn len(&self) -> usize { - core::mem::size_of_val(self) - } - - /// Returns a slice of the given structure. - /// - /// ** WARN:** - /// * The slice must be little endian coded in order to be understood by the device - /// * The slice must serialize the actual structure the device expects, as the queue will use - /// the addresses of the slice in order to refer to the structure. - fn as_slice_u8(&self) -> &[u8] { - unsafe { core::slice::from_raw_parts(ptr::from_ref(self) as *const u8, self.len()) } - } - - /// Returns a mutable slice of the given structure. - /// - /// ** WARN:** - /// * The slice must be little endian coded in order to be understood by the device - /// * The slice must serialize the actual structure the device expects, as the queue will use - /// the addresses of the slice in order to refer to the structure. - fn as_slice_u8_mut(&mut self) -> &mut [u8] { - unsafe { core::slice::from_raw_parts_mut(ptr::from_mut(self) as *mut u8, self.len()) } - } -} - -/// The struct represents buffers which are ready to be send via the -/// virtqueue. Buffers can no longer be written or retrieved. -pub struct TransferToken { - /// Must be some in order to prevent drop - /// upon reuse. - buff_tkn: BufferToken, - /// Structure which allows to await Transfers - /// If Some, finished TransferTokens will be placed here - /// as finished `Transfers`. If None, only the state - /// of the Token will be changed. - await_queue: Option, - // Contains the [MemDescr] for the indirect table if the transfer is indirect. - ctrl_desc: Option, -} - -/// Public Interface for TransferToken -impl TransferToken { - /// Returns a reference to the holding virtqueue - pub fn get_vq(&self) -> Rc { - // Unwrapping is okay here, as TransferToken must hold a BufferToken - Rc::clone(&self.buff_tkn.vq) - } - - /// Dispatches a TransferToken and awaits it at the specified queue. - /// - /// The `notif` parameter indicates if the driver wants to have a notification for this specific - /// transfer. This is only for performance optimization. As it is NOT ensured, that the device sees the - /// updated notification flags before finishing transfers! - pub fn dispatch_await( - mut self, - await_queue: BufferTokenSender, - notif: bool, - ) -> Result<(), VirtqError> { - self.await_queue = Some(await_queue.clone()); - - self.get_vq().dispatch(self, notif) - } - - /// Dispatches the provided TransferToken to the respective queue. - /// - /// The `notif` parameter indicates if the driver wants to have a notification for this specific - /// transfer. This is only for performance optimization. As it is NOT ensured, that the device sees the - /// updated notification flags before finishing transfers! - pub fn dispatch(self, notif: bool) -> Result<(), VirtqError> { - self.get_vq().dispatch(self, notif) - } - - /// Dispatches the provided TransferToken to the respectuve queue and does - /// return when, the queue finished the transfer. - /// - /// The returned [BufferToken] can be reused, copied from - /// or return the underlying buffers. - /// - /// **INFO:** - /// Currently this function is constantly polling the queue while keeping the notifications disabled. - /// Upon finish notifications are enabled again. - pub fn dispatch_blocking(self) -> Result, VirtqError> { - let vq = self.get_vq(); - let (sender, receiver) = async_channel::bounded(1); - self.dispatch_await(sender, false)?; - - vq.disable_notifs(); - - let result: Box; - // Keep Spinning until the receive queue is filled - loop { - match receiver.try_recv() { - Ok(buffer_tkn) => { - result = buffer_tkn; - break; - } - Err(TryRecvError::Closed) => return Err(VirtqError::General), - Err(TryRecvError::Empty) => vq.poll(), - } - } - - vq.enable_notifs(); - - Ok(result) - } - - /// Returns the number of descritprors that will be placed in the queue. - /// This number can differ from the `BufferToken.num_descr()` function value - /// as indirect buffers only consume one descriptor in the queue, but can have - /// more descriptors that are accessible via the descriptor in the queue. - fn num_consuming_descr(&self) -> u16 { - if self.ctrl_desc.is_some() { - 1 - } else { - self.buff_tkn.num_descr() - } - } -} - -/// The struct represents buffers which are ready to be written or to be send. -/// -/// BufferTokens can be written in two ways: -/// * in one step via `BufferToken.write() -/// * consumes BufferToken and returns a TransferToken -/// * sequentially via `BufferToken.write_seq() -/// -/// # Structure of the Token -/// The token can potentially hold both a *send* and a *recv* buffer, but MUST hold -/// one. -/// The *send* buffer is the data the device will read during a transfer, the *recv* buffer -/// is the data the device will write to during a transfer. -/// -/// # What are Buffers -/// A buffer represents multiple chunks of memory. Where each chunk can be of different size. -/// The chunks are named descriptors in the following. -/// -/// **For Example:** -/// A buffer could consist of 3 descriptors: -/// 1. First descriptor of 30 bytes -/// 2. Second descriptor of 10 bytes -/// 3. Third descriptor of 100 bytes -/// -/// Each of these descriptors consumes one "element" of the -/// respective virtqueue. -/// The maximum number of descriptors per buffer is bounded by the size of the virtqueue. -pub struct BufferToken { - send_buff: Option, - //send_desc_lst: Option>, - recv_buff: Option, - //recv_desc_lst: Option>, - vq: Rc, - /// Indicates whether the buff is returnable - ret_send: bool, - ret_recv: bool, - /// Indicates if the token is allowed - /// to be reused. - reusable: bool, -} + for size in send_size_lst { + match MemDescr::pull(*size) { + Ok(desc) => send_desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + } + send_len += usize::from(*size); + } -// Private interface of BufferToken -impl BufferToken { - /// Returns the overall number of descriptors. - fn num_descr(&self) -> u16 { - let mut len = 0; + let send_buff = Some(Buffer { + desc_lst: send_desc_lst.into_boxed_slice(), + len: send_len, + next_write: 0, + }); - if let Some(buffer) = &self.recv_buff { - len += buffer.num_descr(); - } + let recv_buff = match MemDescr::pull(recv_size) { + Ok(recv_desc) => Some(Buffer { + desc_lst: vec![recv_desc].into_boxed_slice(), + len: recv_size.into(), + next_write: 0, + }), + Err(vq_err) => return Err(vq_err), + }; - if let Some(buffer) = &self.send_buff { - len += buffer.num_descr(); - } - len - } + Ok(Self { + send_buff, + recv_buff, + ret_send: true, + ret_recv: true, + reusable: true, + }) + } + (BuffSpec::Indirect(send_size_lst), BuffSpec::Indirect(recv_size_lst)) => { + let mut send_desc_lst: Vec = + Vec::with_capacity(send_size_lst.len()); + let mut send_len = 0usize; - /// Resets all properties from the previous transfer. - /// - /// Includes: - /// * Resetting the write status inside the MemDescr. -> Allowing to rewrite the buffers - /// * Resetting the MemDescr length at initialization. This length might be reduced upon writes - /// of the driver or the device. - /// * Erazing all memory areas with zeros - fn reset_purge(mut self) -> Self { - if let Some(buff) = self.send_buff.as_mut() { - buff.reset_write(); - let mut init_buff_len = 0usize; - for desc in buff.as_mut_slice() { - desc.len = desc._init_len; - init_buff_len += desc._init_len; + for size in send_size_lst { + // As the indirect list does only consume one descriptor for the + // control descriptor, the actual list is untracked + send_desc_lst.push(MemDescr::pull(*size)?); + send_len += usize::from(*size); + } - // Resetting written memory - for byte in desc.deref_mut() { - *byte = 0; - } - } - buff.reset_len(init_buff_len); - } + let mut recv_desc_lst: Vec = + Vec::with_capacity(recv_size_lst.len()); + let mut recv_len = 0usize; - if let Some(buff) = self.recv_buff.as_mut() { - buff.reset_write(); - let mut init_buff_len = 0usize; - for desc in buff.as_mut_slice() { - desc.len = desc._init_len; - init_buff_len += desc._init_len; + for size in recv_size_lst { + // As the indirect list does only consume one descriptor for the + // control descriptor, the actual list is untracked + recv_desc_lst.push(MemDescr::pull(*size)?); + recv_len += usize::from(*size); + } - // Resetting written memory - for byte in desc.deref_mut() { - *byte = 0; + let recv_buff = Some(Buffer { + desc_lst: recv_desc_lst.into_boxed_slice(), + len: recv_len, + next_write: 0, + }); + let send_buff = Some(Buffer { + desc_lst: send_desc_lst.into_boxed_slice(), + len: send_len, + next_write: 0, + }); + + Ok(Self { + send_buff, + recv_buff, + ret_send: true, + ret_recv: true, + reusable: true, + }) + } + (BuffSpec::Indirect(_), BuffSpec::Single(_)) + | (BuffSpec::Indirect(_), BuffSpec::Multiple(_)) => Err(VirtqError::BufferInWithDirect), + (BuffSpec::Single(_), BuffSpec::Indirect(_)) + | (BuffSpec::Multiple(_), BuffSpec::Indirect(_)) => Err(VirtqError::BufferInWithDirect), } } - buff.reset_len(init_buff_len); } - self } - /// Resets all properties from the previous transfer. + /// * Data behind the respective slices will NOT be deallocated under any circumstances. + /// * Caller is responsible for ensuring that the slices remain valid from the start till the end of the transfer. + /// * start: call of [Self::from_existing] + /// * end: return of the [BufferToken] via [Virtq::dispatch_blocking] or its push to the [BufferTokenSender] provided to the dispatch function. + /// * In case the underlying [BufferToken] is reused, the slices MUST still be valid the whole time [BufferToken] exists. + /// * [BufferToken] created by this function will ONLY allow to return a copy of the data. + /// * This is due to the fact, that the [Buffer::cpy] returns a [Box<[u8]>], which must own + /// the slice. This would lead to unwanted frees, if not handled carefully + /// * Drivers must take care of keeping a copy of the respective [&[u8]] and [&mut [u8]] for themselves. /// - /// Includes: - /// * Resetting the write status inside the MemDescr. -> Allowing to rewrite the buffers - /// * Resetting the MemDescr length at initialization. This length might be reduced upon writes - /// of the driver or the device. - pub fn reset(mut self) -> Self { - if let Some(buff) = self.send_buff.as_mut() { - buff.reset_write(); - let mut init_buff_len = 0usize; - for desc in buff.as_mut_slice() { - desc.len = desc._init_len; - init_buff_len += desc._init_len; - } - buff.reset_len(init_buff_len); + /// **Parameters** + /// * send: The slices that will make up the elements of the driver-writable buffer. + /// * recv: The slices that will make up the elements of the device-writable buffer. + /// + /// **Reasons for Failure:** + /// * Both `send` and `recv` are empty, which is not allowed by Virtio. + /// + /// * If one wants to have a structure in the style of: + /// ``` + /// struct send_recv_struct { + /// // send_part: ... + /// // recv_part: ... + /// } + /// ``` + /// they must split the structure after the send part and provide the respective part via the send argument and the respective other + /// part via the recv argument. + pub fn from_existing( + send: &[&[u8]], + recv: &[&mut [MaybeUninit]], + ) -> Result { + if send.is_empty() && recv.is_empty() { + return Err(VirtqError::BufferNotSpecified); } - if let Some(buff) = self.recv_buff.as_mut() { - buff.reset_write(); - let mut init_buff_len = 0usize; - for desc in buff.as_mut_slice() { - desc.len = desc._init_len; - init_buff_len += desc._init_len; - } - buff.reset_len(init_buff_len); - } - self + let total_send_len = send.iter().map(|slice| slice.len()).sum(); + let total_recv_len = recv.iter().map(|slice| slice.len()).sum(); + + let send_desc_lst: Vec<_> = send + .iter() + .map(|slice| MemDescr::pull_from_raw(slice)) + .collect(); + + let recv_desc_lst: Vec<_> = recv + .iter() + .map(|slice| MemDescr::pull_from_raw(slice)) + .collect(); + + let send_buff = if !send.is_empty() { + Some(Buffer { + desc_lst: send_desc_lst.into_boxed_slice(), + len: total_send_len, + next_write: 0, + }) + } else { + None + }; + + let recv_buff = if !recv.is_empty() { + Some(Buffer { + desc_lst: recv_desc_lst.into_boxed_slice(), + len: total_recv_len, + next_write: 0, + }) + } else { + None + }; + + Ok(Self { + recv_buff, + send_buff, + ret_send: false, + ret_recv: false, + reusable: false, + }) } -} -// Public interface of BufferToken -impl BufferToken { /// Restricts the size of a given BufferToken. One must specify either a `new_send_len` or/and `new_recv_len`. If possible /// the function will restrict the respective buffers size to this value. This is especially useful if one has to provide the /// user-space or the device with a buffer and has already a free buffer at hand, which is to large. With this method the user @@ -1411,11 +1189,10 @@ impl BufferToken { /// The `write()` function does NOT take into account the distinct descriptors of a buffer but treats the buffer as a single continuous /// memory element and as a result writes `T` or `H` as a slice of bytes into this memory. pub fn write( - mut self, + &mut self, send: Option<&K>, recv: Option<&H>, - buffer_type: BufferType, - ) -> Result { + ) -> Result<(), VirtqError> { if let Some(data) = send { match self.send_buff.as_mut() { Some(buff) => { @@ -1470,7 +1247,7 @@ impl BufferToken { None => return Err(VirtqError::NoBufferAvail), } } - Ok(self.provide(buffer_type)) + Ok(()) } /// Writes `K` or `H` respectively into the next buffer descriptor. @@ -1486,10 +1263,10 @@ impl BufferToken { /// * Third Write: `write_seq(Some(10 bytes, Some(4 bytes))`: /// * Will result in 10 bytes written to the second buffer descriptor of the send buffer and 4 bytes written to the third buffer descriptor of the recv buffer. pub fn write_seq( - mut self, + &mut self, send_seq: Option<&K>, recv_seq: Option<&H>, - ) -> Result { + ) -> Result<(), VirtqError> { if let Some(data) = send_seq { match self.send_buff.as_mut() { Some(buff) => { @@ -1521,36 +1298,23 @@ impl BufferToken { None => return Err(VirtqError::NoBufferAvail), } } - - Ok(self) - } - - /// Consumes the [BufferToken] and returns a [TransferToken], that can be used to actually start the transfer. - /// - /// After this call, the buffers are no longer writable. - pub fn provide(self, buffer_type: BufferType) -> TransferToken { - let ctrl_desc = match buffer_type { - BufferType::Direct => None, - BufferType::Indirect => Some( - self.vq - .create_indirect_ctrl( - self.send_buff.as_ref().map(Buffer::as_slice), - self.recv_buff.as_ref().map(Buffer::as_slice), - ) - .unwrap(), - ), - }; - - TransferToken { - buff_tkn: self, - await_queue: None, - ctrl_desc, - } + Ok(()) } } - pub enum BufferType { + /// As many descriptors get consumed in the descriptor table as the sum of the numbers of slices in [BufferToken::send_buff] and [BufferToken::recv_buff]. Direct, + /// Results in one descriptor in the queue, hence consumes one element in the main descriptor table. The queue will merge the send and recv buffers as follows: + /// ```text + /// //+++++++++++++++++++++++ + /// //+ Queue + + /// //+++++++++++++++++++++++ + /// //+ Indirect descriptor + -> refers to a descriptor list in the form of -> ++++++++++++++++++++++++++ + /// //+ ... + + Descriptors for send + + /// //+++++++++++++++++++++++ + Descriptors for recv + + /// // ++++++++++++++++++++++++++ + /// ``` + /// As a result indirect descriptors result in a single descriptor consumption in the actual queue. Indirect, } diff --git a/src/drivers/virtio/virtqueue/packed.rs b/src/drivers/virtio/virtqueue/packed.rs index 7e15892479..56c8f7a3ed 100644 --- a/src/drivers/virtio/virtqueue/packed.rs +++ b/src/drivers/virtio/virtqueue/packed.rs @@ -3,10 +3,8 @@ #![allow(dead_code)] use alloc::boxed::Box; -use alloc::rc::Rc; use alloc::vec::Vec; use core::cell::{Cell, RefCell}; -use core::mem::MaybeUninit; use core::sync::atomic::{fence, Ordering}; use core::{iter, ops, ptr}; @@ -21,8 +19,8 @@ use super::super::transport::mmio::{ComCfg, NotifCfg, NotifCtrl}; use super::super::transport::pci::{ComCfg, NotifCfg, NotifCtrl}; use super::error::VirtqError; use super::{ - BuffSpec, Buffer, BufferToken, BufferType, Bytes, MemDescr, MemDescrId, MemPool, TransferToken, - Virtq, VirtqPrivate, VqIndex, VqSize, + Buffer, BufferToken, BufferTokenSender, BufferType, Bytes, MemDescr, MemDescrId, MemPool, + TransferToken, Virtq, VirtqPrivate, VqIndex, VqSize, }; use crate::arch::mm::paging::{BasePageSize, PageSize}; use crate::arch::mm::{paging, VirtAddr}; @@ -164,7 +162,7 @@ impl DescriptorRing { if let Some(mut tkn) = ctrl.poll_next() { if let Some(queue) = tkn.await_queue.take() { // Place the TransferToken in a Transfer, which will hold ownership of the token - queue.try_send(Box::new(tkn.buff_tkn)).unwrap(); + queue.try_send(tkn.buff_tkn).unwrap(); } } } @@ -671,11 +669,22 @@ impl Virtq for PackedVq { self.descr_ring.borrow_mut().poll(); } - fn dispatch_batch(&self, tkns: Vec, notif: bool) -> Result<(), VirtqError> { + fn dispatch_batch( + &self, + buffer_tkns: Vec<(BufferToken, BufferType)>, + notif: bool, + ) -> Result<(), VirtqError> { // Zero transfers are not allowed - assert!(!tkns.is_empty()); + assert!(!buffer_tkns.is_empty()); + + let transfer_tkns = buffer_tkns + .into_iter() + .map(|(buffer_tkn, buffer_type)| { + self.transfer_token_from_buffer_token(buffer_tkn, None, buffer_type) + }) + .collect(); - let next_idx = self.descr_ring.borrow_mut().push_batch(tkns)?; + let next_idx = self.descr_ring.borrow_mut().push_batch(transfer_tkns)?; if notif { self.drv_event.borrow_mut().enable_specific(next_idx); @@ -700,19 +709,25 @@ impl Virtq for PackedVq { fn dispatch_batch_await( &self, - mut tkns: Vec, + buffer_tkns: Vec<(BufferToken, BufferType)>, await_queue: super::BufferTokenSender, notif: bool, ) -> Result<(), VirtqError> { // Zero transfers are not allowed - assert!(!tkns.is_empty()); + assert!(!buffer_tkns.is_empty()); - // We have to iterate here too, in order to ensure, tokens are placed into the await_queue - for tkn in tkns.iter_mut() { - tkn.await_queue = Some(await_queue.clone()); - } + let transfer_tkns = buffer_tkns + .into_iter() + .map(|(buffer_tkn, buffer_type)| { + self.transfer_token_from_buffer_token( + buffer_tkn, + Some(await_queue.clone()), + buffer_type, + ) + }) + .collect(); - let next_idx = self.descr_ring.borrow_mut().push_batch(tkns)?; + let next_idx = self.descr_ring.borrow_mut().push_batch(transfer_tkns)?; if notif { self.drv_event.borrow_mut().enable_specific(next_idx); @@ -735,8 +750,16 @@ impl Virtq for PackedVq { Ok(()) } - fn dispatch(&self, tkn: TransferToken, notif: bool) -> Result<(), VirtqError> { - let next_idx = self.descr_ring.borrow_mut().push(tkn)?; + fn dispatch_await( + &self, + buffer_tkn: BufferToken, + sender: BufferTokenSender, + notif: bool, + buffer_type: BufferType, + ) -> Result<(), VirtqError> { + let transfer_tkn = + self.transfer_token_from_buffer_token(buffer_tkn, Some(sender), buffer_type); + let next_idx = self.descr_ring.borrow_mut().push(transfer_tkn)?; if notif { self.drv_event.borrow_mut().enable_specific(next_idx); @@ -851,23 +874,6 @@ impl Virtq for PackedVq { }) } - fn prep_transfer_from_raw( - self: Rc, - send: &[&[u8]], - recv: &[&mut [MaybeUninit]], - buffer_type: BufferType, - ) -> Result { - self.prep_transfer_from_raw_static(send, recv, buffer_type) - } - - fn prep_buffer( - self: Rc, - send: Option>, - recv: Option>, - ) -> Result { - self.prep_buffer_static(send, recv) - } - fn size(&self) -> VqSize { self.size } diff --git a/src/drivers/virtio/virtqueue/split.rs b/src/drivers/virtio/virtqueue/split.rs index 89b060c589..6bd8bdaf4a 100644 --- a/src/drivers/virtio/virtqueue/split.rs +++ b/src/drivers/virtio/virtqueue/split.rs @@ -3,7 +3,6 @@ #![allow(dead_code)] use alloc::boxed::Box; -use alloc::rc::Rc; use alloc::vec::Vec; use core::alloc::{Allocator, Layout}; use core::cell::{RefCell, UnsafeCell}; @@ -22,7 +21,7 @@ use super::super::transport::mmio::{ComCfg, NotifCfg, NotifCtrl}; use super::super::transport::pci::{ComCfg, NotifCfg, NotifCtrl}; use super::error::VirtqError; use super::{ - BuffSpec, BufferToken, BufferType, Bytes, MemDescr, MemPool, TransferToken, Virtq, + BufferToken, BufferTokenSender, BufferType, Bytes, MemDescr, MemPool, TransferToken, Virtq, VirtqPrivate, VqIndex, VqSize, }; use crate::arch::memory_barrier; @@ -266,7 +265,7 @@ impl DescrRing { .unwrap(); } if let Some(queue) = tkn.await_queue.take() { - queue.try_send(Box::new(tkn.buff_tkn)).unwrap() + queue.try_send(tkn.buff_tkn).unwrap() } let mut id_ret_idx = u16::try_from(cur_ring_index).unwrap(); @@ -332,21 +331,33 @@ impl Virtq for SplitVq { self.ring.borrow_mut().poll() } - fn dispatch_batch(&self, _tkns: Vec, _notif: bool) -> Result<(), VirtqError> { + fn dispatch_batch( + &self, + _tkns: Vec<(BufferToken, BufferType)>, + _notif: bool, + ) -> Result<(), VirtqError> { unimplemented!(); } fn dispatch_batch_await( &self, - _tkns: Vec, + _tkns: Vec<(BufferToken, BufferType)>, _await_queue: super::BufferTokenSender, _notif: bool, ) -> Result<(), VirtqError> { unimplemented!() } - fn dispatch(&self, tkn: TransferToken, notif: bool) -> Result<(), VirtqError> { - let next_idx = self.ring.borrow_mut().push(tkn)?; + fn dispatch_await( + &self, + buffer_tkn: BufferToken, + sender: BufferTokenSender, + notif: bool, + buffer_type: BufferType, + ) -> Result<(), VirtqError> { + let transfer_tkn = + self.transfer_token_from_buffer_token(buffer_tkn, Some(sender), buffer_type); + let next_idx = self.ring.borrow_mut().push(transfer_tkn)?; if notif { // TODO: Check whether the splitvirtquue has notifications for specific descriptors @@ -479,23 +490,6 @@ impl Virtq for SplitVq { }) } - fn prep_transfer_from_raw( - self: Rc, - send: &[&[u8]], - recv: &[&mut [MaybeUninit]], - buffer_type: BufferType, - ) -> Result { - self.prep_transfer_from_raw_static(send, recv, buffer_type) - } - - fn prep_buffer( - self: Rc, - send: Option>, - recv: Option>, - ) -> Result { - self.prep_buffer_static(send, recv) - } - fn size(&self) -> VqSize { self.size }