diff --git a/src/drivers/virtio/virtqueue/mod.rs b/src/drivers/virtio/virtqueue/mod.rs index f2e07b738c..7059ea3cb1 100644 --- a/src/drivers/virtio/virtqueue/mod.rs +++ b/src/drivers/virtio/virtqueue/mod.rs @@ -108,7 +108,8 @@ struct Descriptor { /// might not provide the complete feature set of each queue. Drivers who /// do need these features should refrain from providing support for both /// Virtqueue types and use the structs directly instead. -pub trait Virtq { +#[allow(private_bounds)] +pub trait Virtq: VirtqPrivate { /// Entry function which the TransferTokens can use, when they are dispatching /// themselves via their `Rc` reference /// @@ -249,6 +250,577 @@ pub trait Virtq { recv: Option<(&mut [u8], BuffSpec<'_>)>, ) -> Result; + /// The implementation of the method requires constraints that are incompatible with a trait object. + /// Because of this, we constrain it to static objects (via Sized) and call it from the implementation + /// of [Self::prep_buffer] inside the implementor. + fn prep_transfer_from_raw_static( + self: Rc, + send: Option<(&[u8], BuffSpec<'_>)>, + recv: Option<(&mut [u8], BuffSpec<'_>)>, + ) -> Result + where + Self: Sized + 'static, + { + match (send, recv) { + (None, None) => Err(VirtqError::BufferNotSpecified), + (Some((send_data, send_spec)), None) => { + match send_spec { + BuffSpec::Single(size) => { + // Buffer must have the right size + if send_data.len() != size.into() { + return Err(VirtqError::BufferSizeWrong(send_data.len())); + } + + let desc = match self + .mem_pool() + .pull_from_raw(Rc::clone(&self.mem_pool()), send_data) + { + Ok(desc) => desc, + Err(vq_err) => return Err(vq_err), + }; + + Ok(TransferToken { + state: TransferState::Ready, + buff_tkn: Some(BufferToken { + send_buff: Some(Buffer::Single { + desc_lst: vec![desc].into_boxed_slice(), + len: send_data.len(), + next_write: 0, + }), + recv_buff: None, + vq: self, + ret_send: false, + ret_recv: false, + reusable: false, + }), + await_queue: None, + }) + } + BuffSpec::Multiple(size_lst) => { + let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); + let mut index = 0usize; + + for byte in size_lst { + let end_index = index + usize::from(*byte); + let next_slice = match send_data.get(index..end_index) { + Some(slice) => slice, + None => return Err(VirtqError::BufferSizeWrong(send_data.len())), + }; + + match self + .mem_pool() + .pull_from_raw(Rc::clone(&self.mem_pool()), next_slice) + { + Ok(desc) => desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + }; + + // update the starting index for the next iteration + index += usize::from(*byte); + } + + Ok(TransferToken { + state: TransferState::Ready, + buff_tkn: Some(BufferToken { + send_buff: Some(Buffer::Multiple { + desc_lst: desc_lst.into_boxed_slice(), + len: send_data.len(), + next_write: 0, + }), + recv_buff: None, + vq: self, + ret_send: false, + ret_recv: false, + reusable: false, + }), + await_queue: None, + }) + } + BuffSpec::Indirect(size_lst) => { + let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); + let mut index = 0usize; + + for byte in size_lst { + let end_index = index + usize::from(*byte); + let next_slice = match send_data.get(index..end_index) { + Some(slice) => slice, + None => return Err(VirtqError::BufferSizeWrong(send_data.len())), + }; + + desc_lst.push( + self.mem_pool().pull_from_raw_untracked( + Rc::clone(&self.mem_pool()), + next_slice, + ), + ); + + // update the starting index for the next iteration + index += usize::from(*byte); + } + + let ctrl_desc = match self.create_indirect_ctrl(Some(&desc_lst), None) { + Ok(desc) => desc, + Err(vq_err) => return Err(vq_err), + }; + + Ok(TransferToken { + state: TransferState::Ready, + buff_tkn: Some(BufferToken { + send_buff: Some(Buffer::Indirect { + desc_lst: desc_lst.into_boxed_slice(), + ctrl_desc, + len: send_data.len(), + next_write: 0, + }), + recv_buff: None, + vq: self, + ret_send: false, + ret_recv: false, + reusable: false, + }), + await_queue: None, + }) + } + } + } + (None, Some((recv_data, recv_spec))) => { + match recv_spec { + BuffSpec::Single(size) => { + // Buffer must have the right size + if recv_data.len() != size.into() { + return Err(VirtqError::BufferSizeWrong(recv_data.len())); + } + + let desc = match self + .mem_pool() + .pull_from_raw(Rc::clone(&self.mem_pool()), recv_data) + { + Ok(desc) => desc, + Err(vq_err) => return Err(vq_err), + }; + + Ok(TransferToken { + state: TransferState::Ready, + buff_tkn: Some(BufferToken { + send_buff: None, + recv_buff: Some(Buffer::Single { + desc_lst: vec![desc].into_boxed_slice(), + len: recv_data.len(), + next_write: 0, + }), + vq: self, + ret_send: false, + ret_recv: false, + reusable: false, + }), + await_queue: None, + }) + } + BuffSpec::Multiple(size_lst) => { + let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); + let mut index = 0usize; + + for byte in size_lst { + let end_index = index + usize::from(*byte); + let next_slice = match recv_data.get(index..end_index) { + Some(slice) => slice, + None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), + }; + + match self + .mem_pool() + .pull_from_raw(Rc::clone(&self.mem_pool()), next_slice) + { + Ok(desc) => desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + }; + + // update the starting index for the next iteration + index += usize::from(*byte); + } + + Ok(TransferToken { + state: TransferState::Ready, + buff_tkn: Some(BufferToken { + send_buff: None, + recv_buff: Some(Buffer::Multiple { + desc_lst: desc_lst.into_boxed_slice(), + len: recv_data.len(), + next_write: 0, + }), + vq: self, + ret_send: false, + ret_recv: false, + reusable: false, + }), + await_queue: None, + }) + } + BuffSpec::Indirect(size_lst) => { + let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); + let mut index = 0usize; + + for byte in size_lst { + let end_index = index + usize::from(*byte); + let next_slice = match recv_data.get(index..end_index) { + Some(slice) => slice, + None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), + }; + + desc_lst.push( + self.mem_pool().pull_from_raw_untracked( + Rc::clone(&self.mem_pool()), + next_slice, + ), + ); + + // update the starting index for the next iteration + index += usize::from(*byte); + } + + let ctrl_desc = match self.create_indirect_ctrl(None, Some(&desc_lst)) { + Ok(desc) => desc, + Err(vq_err) => return Err(vq_err), + }; + + Ok(TransferToken { + state: TransferState::Ready, + buff_tkn: Some(BufferToken { + send_buff: None, + recv_buff: Some(Buffer::Indirect { + desc_lst: desc_lst.into_boxed_slice(), + ctrl_desc, + len: recv_data.len(), + next_write: 0, + }), + vq: self, + ret_send: false, + ret_recv: false, + reusable: false, + }), + await_queue: None, + }) + } + } + } + (Some((send_data, send_spec)), Some((recv_data, recv_spec))) => { + match (send_spec, recv_spec) { + (BuffSpec::Single(send_size), BuffSpec::Single(recv_size)) => { + // Buffer must have the right size + if send_data.len() != send_size.into() { + return Err(VirtqError::BufferSizeWrong(send_data.len())); + } + + let send_desc = match self + .mem_pool() + .pull_from_raw(Rc::clone(&self.mem_pool()), send_data) + { + Ok(desc) => desc, + Err(vq_err) => return Err(vq_err), + }; + + // Buffer must have the right size + if recv_data.len() != recv_size.into() { + return Err(VirtqError::BufferSizeWrong(recv_data.len())); + } + + let recv_desc = match self + .mem_pool() + .pull_from_raw(Rc::clone(&self.mem_pool()), recv_data) + { + Ok(desc) => desc, + Err(vq_err) => return Err(vq_err), + }; + + Ok(TransferToken { + state: TransferState::Ready, + buff_tkn: Some(BufferToken { + send_buff: Some(Buffer::Single { + desc_lst: vec![send_desc].into_boxed_slice(), + len: send_data.len(), + next_write: 0, + }), + recv_buff: Some(Buffer::Single { + desc_lst: vec![recv_desc].into_boxed_slice(), + len: recv_data.len(), + next_write: 0, + }), + vq: self, + ret_send: false, + ret_recv: false, + reusable: false, + }), + await_queue: None, + }) + } + (BuffSpec::Single(send_size), BuffSpec::Multiple(recv_size_lst)) => { + // Buffer must have the right size + if send_data.len() != send_size.into() { + return Err(VirtqError::BufferSizeWrong(send_data.len())); + } + + let send_desc = match self + .mem_pool() + .pull_from_raw(Rc::clone(&self.mem_pool()), send_data) + { + Ok(desc) => desc, + Err(vq_err) => return Err(vq_err), + }; + + let mut recv_desc_lst: Vec = + Vec::with_capacity(recv_size_lst.len()); + let mut index = 0usize; + + for byte in recv_size_lst { + let end_index = index + usize::from(*byte); + let next_slice = match recv_data.get(index..end_index) { + Some(slice) => slice, + None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), + }; + + match self + .mem_pool() + .pull_from_raw(Rc::clone(&self.mem_pool()), next_slice) + { + Ok(desc) => recv_desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + }; + + // update the starting index for the next iteration + index += usize::from(*byte); + } + + Ok(TransferToken { + state: TransferState::Ready, + buff_tkn: Some(BufferToken { + send_buff: Some(Buffer::Single { + desc_lst: vec![send_desc].into_boxed_slice(), + len: send_data.len(), + next_write: 0, + }), + recv_buff: Some(Buffer::Multiple { + desc_lst: recv_desc_lst.into_boxed_slice(), + len: recv_data.len(), + next_write: 0, + }), + vq: self, + ret_send: false, + ret_recv: false, + reusable: false, + }), + await_queue: None, + }) + } + (BuffSpec::Multiple(send_size_lst), BuffSpec::Multiple(recv_size_lst)) => { + let mut send_desc_lst: Vec = + Vec::with_capacity(send_size_lst.len()); + let mut index = 0usize; + + for byte in send_size_lst { + let end_index = index + usize::from(*byte); + let next_slice = match send_data.get(index..end_index) { + Some(slice) => slice, + None => return Err(VirtqError::BufferSizeWrong(send_data.len())), + }; + + match self + .mem_pool() + .pull_from_raw(Rc::clone(&self.mem_pool()), next_slice) + { + Ok(desc) => send_desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + }; + + // update the starting index for the next iteration + index += usize::from(*byte); + } + + let mut recv_desc_lst: Vec = + Vec::with_capacity(recv_size_lst.len()); + let mut index = 0usize; + + for byte in recv_size_lst { + let end_index = index + usize::from(*byte); + let next_slice = match recv_data.get(index..end_index) { + Some(slice) => slice, + None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), + }; + + match self + .mem_pool() + .pull_from_raw(Rc::clone(&self.mem_pool()), next_slice) + { + Ok(desc) => recv_desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + }; + + // update the starting index for the next iteration + index += usize::from(*byte); + } + + Ok(TransferToken { + state: TransferState::Ready, + buff_tkn: Some(BufferToken { + send_buff: Some(Buffer::Multiple { + desc_lst: send_desc_lst.into_boxed_slice(), + len: send_data.len(), + next_write: 0, + }), + recv_buff: Some(Buffer::Multiple { + desc_lst: recv_desc_lst.into_boxed_slice(), + len: recv_data.len(), + next_write: 0, + }), + vq: self, + ret_send: false, + ret_recv: false, + reusable: false, + }), + await_queue: None, + }) + } + (BuffSpec::Multiple(send_size_lst), BuffSpec::Single(recv_size)) => { + let mut send_desc_lst: Vec = + Vec::with_capacity(send_size_lst.len()); + let mut index = 0usize; + + for byte in send_size_lst { + let end_index = index + usize::from(*byte); + let next_slice = match send_data.get(index..end_index) { + Some(slice) => slice, + None => return Err(VirtqError::BufferSizeWrong(send_data.len())), + }; + + match self + .mem_pool() + .pull_from_raw(Rc::clone(&self.mem_pool()), next_slice) + { + Ok(desc) => send_desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + }; + + // update the starting index for the next iteration + index += usize::from(*byte); + } + + // Buffer must have the right size + if recv_data.len() != recv_size.into() { + return Err(VirtqError::BufferSizeWrong(recv_data.len())); + } + + let recv_desc = match self + .mem_pool() + .pull_from_raw(Rc::clone(&self.mem_pool()), recv_data) + { + Ok(desc) => desc, + Err(vq_err) => return Err(vq_err), + }; + + Ok(TransferToken { + state: TransferState::Ready, + buff_tkn: Some(BufferToken { + send_buff: Some(Buffer::Multiple { + desc_lst: send_desc_lst.into_boxed_slice(), + len: send_data.len(), + next_write: 0, + }), + recv_buff: Some(Buffer::Single { + desc_lst: vec![recv_desc].into_boxed_slice(), + len: recv_data.len(), + next_write: 0, + }), + vq: self, + ret_send: false, + ret_recv: false, + reusable: false, + }), + await_queue: None, + }) + } + (BuffSpec::Indirect(send_size_lst), BuffSpec::Indirect(recv_size_lst)) => { + let mut send_desc_lst: Vec = + Vec::with_capacity(send_size_lst.len()); + let mut index = 0usize; + + for byte in send_size_lst { + let end_index = index + usize::from(*byte); + let next_slice = match send_data.get(index..end_index) { + Some(slice) => slice, + None => return Err(VirtqError::BufferSizeWrong(send_data.len())), + }; + + send_desc_lst.push( + self.mem_pool().pull_from_raw_untracked( + Rc::clone(&self.mem_pool()), + next_slice, + ), + ); + + // update the starting index for the next iteration + index += usize::from(*byte); + } + + let mut recv_desc_lst: Vec = + Vec::with_capacity(recv_size_lst.len()); + let mut index = 0usize; + + for byte in recv_size_lst { + let end_index = index + usize::from(*byte); + let next_slice = match recv_data.get(index..end_index) { + Some(slice) => slice, + None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), + }; + + recv_desc_lst.push( + self.mem_pool().pull_from_raw_untracked( + Rc::clone(&self.mem_pool()), + next_slice, + ), + ); + + // update the starting index for the next iteration + index += usize::from(*byte); + } + + let ctrl_desc = match self + .create_indirect_ctrl(Some(&send_desc_lst), Some(&recv_desc_lst)) + { + Ok(desc) => desc, + Err(vq_err) => return Err(vq_err), + }; + + Ok(TransferToken { + state: TransferState::Ready, + buff_tkn: Some(BufferToken { + recv_buff: Some(Buffer::Indirect { + desc_lst: recv_desc_lst.into_boxed_slice(), + ctrl_desc: ctrl_desc.no_dealloc_clone(), + len: recv_data.len(), + next_write: 0, + }), + send_buff: Some(Buffer::Indirect { + desc_lst: send_desc_lst.into_boxed_slice(), + ctrl_desc, + len: send_data.len(), + next_write: 0, + }), + vq: self, + ret_send: false, + ret_recv: false, + reusable: false, + }), + await_queue: None, + }) + } + (BuffSpec::Indirect(_), BuffSpec::Single(_)) + | (BuffSpec::Indirect(_), BuffSpec::Multiple(_)) => Err(VirtqError::BufferInWithDirect), + (BuffSpec::Single(_), BuffSpec::Indirect(_)) + | (BuffSpec::Multiple(_), BuffSpec::Indirect(_)) => Err(VirtqError::BufferInWithDirect), + } + } + } + } + /// Provides the calley with empty buffers as specified via the `send` and `recv` function parameters, (see [BuffSpec]), in form of /// a [BufferToken]. /// Fails upon multiple circumstances. @@ -295,6 +867,429 @@ pub trait Virtq { send: Option>, recv: Option>, ) -> Result; + + /// The implementation of the method requires constraints that are incompatible with a trait object. + /// Because of this, we constrain it to static objects (via Sized) and call it from the implementation + /// of [Self::prep_buffer] inside the implementor. + fn prep_buffer_static( + self: Rc, + send: Option>, + recv: Option>, + ) -> Result + where + Self: Sized + 'static, + { + match (send, recv) { + // No buffers specified + (None, None) => Err(VirtqError::BufferNotSpecified), + // Send buffer specified, No recv buffer + (Some(spec), None) => { + match spec { + BuffSpec::Single(size) => { + match self.mem_pool().pull(Rc::clone(&self.mem_pool()), size) { + Ok(desc) => { + let buffer = Buffer::Single { + desc_lst: vec![desc].into_boxed_slice(), + len: size.into(), + next_write: 0, + }; + + Ok(BufferToken { + send_buff: Some(buffer), + recv_buff: None, + vq: self.clone(), + ret_send: true, + ret_recv: false, + reusable: true, + }) + } + Err(vq_err) => Err(vq_err), + } + } + BuffSpec::Multiple(size_lst) => { + let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); + let mut len = 0usize; + + for size in size_lst { + match self.mem_pool().pull(Rc::clone(&self.mem_pool()), *size) { + Ok(desc) => desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + } + len += usize::from(*size); + } + + let buffer = Buffer::Multiple { + desc_lst: desc_lst.into_boxed_slice(), + len, + next_write: 0, + }; + + Ok(BufferToken { + send_buff: Some(buffer), + recv_buff: None, + vq: self.clone(), + ret_send: true, + ret_recv: false, + reusable: true, + }) + } + BuffSpec::Indirect(size_lst) => { + let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); + let mut len = 0usize; + + for size in size_lst { + // As the indirect list does only consume one descriptor for the + // control descriptor, the actual list is untracked + desc_lst.push( + self.mem_pool() + .pull_untracked(Rc::clone(&self.mem_pool()), *size), + ); + len += usize::from(*size); + } + + let ctrl_desc = match self.create_indirect_ctrl(Some(&desc_lst), None) { + Ok(desc) => desc, + Err(vq_err) => return Err(vq_err), + }; + + let buffer = Buffer::Indirect { + desc_lst: desc_lst.into_boxed_slice(), + ctrl_desc, + len, + next_write: 0, + }; + + Ok(BufferToken { + send_buff: Some(buffer), + recv_buff: None, + vq: self.clone(), + ret_send: true, + ret_recv: false, + reusable: true, + }) + } + } + } + // No send buffer, recv buffer is specified + (None, Some(spec)) => { + match spec { + BuffSpec::Single(size) => { + match self.mem_pool().pull(Rc::clone(&self.mem_pool()), size) { + Ok(desc) => { + let buffer = Buffer::Single { + desc_lst: vec![desc].into_boxed_slice(), + len: size.into(), + next_write: 0, + }; + + Ok(BufferToken { + send_buff: None, + recv_buff: Some(buffer), + vq: self.clone(), + ret_send: false, + ret_recv: true, + reusable: true, + }) + } + Err(vq_err) => Err(vq_err), + } + } + BuffSpec::Multiple(size_lst) => { + let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); + let mut len = 0usize; + + for size in size_lst { + match self.mem_pool().pull(Rc::clone(&self.mem_pool()), *size) { + Ok(desc) => desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + } + len += usize::from(*size); + } + + let buffer = Buffer::Multiple { + desc_lst: desc_lst.into_boxed_slice(), + len, + next_write: 0, + }; + + Ok(BufferToken { + send_buff: None, + recv_buff: Some(buffer), + vq: self.clone(), + ret_send: false, + ret_recv: true, + reusable: true, + }) + } + BuffSpec::Indirect(size_lst) => { + let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); + let mut len = 0usize; + + for size in size_lst { + // As the indirect list does only consume one descriptor for the + // control descriptor, the actual list is untracked + desc_lst.push( + self.mem_pool() + .pull_untracked(Rc::clone(&self.mem_pool()), *size), + ); + len += usize::from(*size); + } + + let ctrl_desc = match self.create_indirect_ctrl(None, Some(&desc_lst)) { + Ok(desc) => desc, + Err(vq_err) => return Err(vq_err), + }; + + let buffer = Buffer::Indirect { + desc_lst: desc_lst.into_boxed_slice(), + ctrl_desc, + len, + next_write: 0, + }; + + Ok(BufferToken { + send_buff: None, + recv_buff: Some(buffer), + vq: self.clone(), + ret_send: false, + ret_recv: true, + reusable: true, + }) + } + } + } + // Send buffer specified, recv buffer specified + (Some(send_spec), Some(recv_spec)) => { + match (send_spec, recv_spec) { + (BuffSpec::Single(send_size), BuffSpec::Single(recv_size)) => { + let send_buff = + match self.mem_pool().pull(Rc::clone(&self.mem_pool()), send_size) { + Ok(send_desc) => Some(Buffer::Single { + desc_lst: vec![send_desc].into_boxed_slice(), + len: send_size.into(), + next_write: 0, + }), + Err(vq_err) => return Err(vq_err), + }; + + let recv_buff = + match self.mem_pool().pull(Rc::clone(&self.mem_pool()), recv_size) { + Ok(recv_desc) => Some(Buffer::Single { + desc_lst: vec![recv_desc].into_boxed_slice(), + len: recv_size.into(), + next_write: 0, + }), + Err(vq_err) => return Err(vq_err), + }; + + Ok(BufferToken { + send_buff, + recv_buff, + vq: self.clone(), + ret_send: true, + ret_recv: true, + reusable: true, + }) + } + (BuffSpec::Single(send_size), BuffSpec::Multiple(recv_size_lst)) => { + let send_buff = + match self.mem_pool().pull(Rc::clone(&self.mem_pool()), send_size) { + Ok(send_desc) => Some(Buffer::Single { + desc_lst: vec![send_desc].into_boxed_slice(), + len: send_size.into(), + next_write: 0, + }), + Err(vq_err) => return Err(vq_err), + }; + + let mut recv_desc_lst: Vec = + Vec::with_capacity(recv_size_lst.len()); + let mut recv_len = 0usize; + + for size in recv_size_lst { + match self.mem_pool().pull(Rc::clone(&self.mem_pool()), *size) { + Ok(desc) => recv_desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + } + recv_len += usize::from(*size); + } + + let recv_buff = Some(Buffer::Multiple { + desc_lst: recv_desc_lst.into_boxed_slice(), + len: recv_len, + next_write: 0, + }); + + Ok(BufferToken { + send_buff, + recv_buff, + vq: self.clone(), + ret_send: true, + ret_recv: true, + reusable: true, + }) + } + (BuffSpec::Multiple(send_size_lst), BuffSpec::Multiple(recv_size_lst)) => { + let mut send_desc_lst: Vec = + Vec::with_capacity(send_size_lst.len()); + let mut send_len = 0usize; + for size in send_size_lst { + match self.mem_pool().pull(Rc::clone(&self.mem_pool()), *size) { + Ok(desc) => send_desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + } + send_len += usize::from(*size); + } + + let send_buff = Some(Buffer::Multiple { + desc_lst: send_desc_lst.into_boxed_slice(), + len: send_len, + next_write: 0, + }); + + let mut recv_desc_lst: Vec = + Vec::with_capacity(recv_size_lst.len()); + let mut recv_len = 0usize; + + for size in recv_size_lst { + match self.mem_pool().pull(Rc::clone(&self.mem_pool()), *size) { + Ok(desc) => recv_desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + } + recv_len += usize::from(*size); + } + + let recv_buff = Some(Buffer::Multiple { + desc_lst: recv_desc_lst.into_boxed_slice(), + len: recv_len, + next_write: 0, + }); + + Ok(BufferToken { + send_buff, + recv_buff, + vq: self.clone(), + ret_send: true, + ret_recv: true, + reusable: true, + }) + } + (BuffSpec::Multiple(send_size_lst), BuffSpec::Single(recv_size)) => { + let mut send_desc_lst: Vec = + Vec::with_capacity(send_size_lst.len()); + let mut send_len = 0usize; + + for size in send_size_lst { + match self.mem_pool().pull(Rc::clone(&self.mem_pool()), *size) { + Ok(desc) => send_desc_lst.push(desc), + Err(vq_err) => return Err(vq_err), + } + send_len += usize::from(*size); + } + + let send_buff = Some(Buffer::Multiple { + desc_lst: send_desc_lst.into_boxed_slice(), + len: send_len, + next_write: 0, + }); + + let recv_buff = + match self.mem_pool().pull(Rc::clone(&self.mem_pool()), recv_size) { + Ok(recv_desc) => Some(Buffer::Single { + desc_lst: vec![recv_desc].into_boxed_slice(), + len: recv_size.into(), + next_write: 0, + }), + Err(vq_err) => return Err(vq_err), + }; + + Ok(BufferToken { + send_buff, + recv_buff, + vq: self.clone(), + ret_send: true, + ret_recv: true, + reusable: true, + }) + } + (BuffSpec::Indirect(send_size_lst), BuffSpec::Indirect(recv_size_lst)) => { + let mut send_desc_lst: Vec = + Vec::with_capacity(send_size_lst.len()); + let mut send_len = 0usize; + + for size in send_size_lst { + // As the indirect list does only consume one descriptor for the + // control descriptor, the actual list is untracked + send_desc_lst.push( + self.mem_pool() + .pull_untracked(Rc::clone(&self.mem_pool()), *size), + ); + send_len += usize::from(*size); + } + + let mut recv_desc_lst: Vec = + Vec::with_capacity(recv_size_lst.len()); + let mut recv_len = 0usize; + + for size in recv_size_lst { + // As the indirect list does only consume one descriptor for the + // control descriptor, the actual list is untracked + recv_desc_lst.push( + self.mem_pool() + .pull_untracked(Rc::clone(&self.mem_pool()), *size), + ); + recv_len += usize::from(*size); + } + + let ctrl_desc = match self + .create_indirect_ctrl(Some(&send_desc_lst), Some(&recv_desc_lst)) + { + Ok(desc) => desc, + Err(vq_err) => return Err(vq_err), + }; + + let recv_buff = Some(Buffer::Indirect { + desc_lst: recv_desc_lst.into_boxed_slice(), + ctrl_desc: ctrl_desc.no_dealloc_clone(), + len: recv_len, + next_write: 0, + }); + let send_buff = Some(Buffer::Indirect { + desc_lst: send_desc_lst.into_boxed_slice(), + ctrl_desc, + len: send_len, + next_write: 0, + }); + + Ok(BufferToken { + send_buff, + recv_buff, + vq: self.clone(), + ret_send: true, + ret_recv: true, + reusable: true, + }) + } + (BuffSpec::Indirect(_), BuffSpec::Single(_)) + | (BuffSpec::Indirect(_), BuffSpec::Multiple(_)) => Err(VirtqError::BufferInWithDirect), + (BuffSpec::Single(_), BuffSpec::Indirect(_)) + | (BuffSpec::Multiple(_), BuffSpec::Indirect(_)) => Err(VirtqError::BufferInWithDirect), + } + } + } + } +} + +/// These methods are an implementation detail and are meant only for consumption by the default method +/// implementations in [Virtq]. +trait VirtqPrivate { + fn create_indirect_ctrl( + &self, + send: Option<&Vec>, + recv: Option<&Vec>, + ) -> Result; + + fn mem_pool(&self) -> Rc; } /// Allows to check, if a given structure crosses a physical page boundary. diff --git a/src/drivers/virtio/virtqueue/packed.rs b/src/drivers/virtio/virtqueue/packed.rs index 8a0d1d9191..e93944a7b1 100644 --- a/src/drivers/virtio/virtqueue/packed.rs +++ b/src/drivers/virtio/virtqueue/packed.rs @@ -20,7 +20,7 @@ use super::super::transport::pci::{ComCfg, NotifCfg, NotifCtrl}; use super::error::VirtqError; use super::{ BuffSpec, Buffer, BufferToken, Bytes, DescrFlags, MemDescr, MemPool, Transfer, TransferState, - TransferToken, Virtq, VqIndex, VqSize, + TransferToken, Virtq, VirtqPrivate, VqIndex, VqSize, }; use crate::arch::mm::paging::{BasePageSize, PageSize}; use crate::arch::mm::{paging, VirtAddr}; @@ -1201,556 +1201,7 @@ impl Virtq for PackedVq { send: Option<(&[u8], BuffSpec<'_>)>, recv: Option<(&mut [u8], BuffSpec<'_>)>, ) -> Result { - match (send, recv) { - (None, None) => Err(VirtqError::BufferNotSpecified), - (Some((send_data, send_spec)), None) => { - match send_spec { - BuffSpec::Single(size) => { - // Buffer must have the right size - if send_data.len() != size.into() { - return Err(VirtqError::BufferSizeWrong(send_data.len())); - } - - let desc = match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), send_data) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Single { - desc_lst: vec![desc].into_boxed_slice(), - len: send_data.len(), - next_write: 0, - }), - recv_buff: None, - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - BuffSpec::Multiple(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut index = 0usize; - - for byte in size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match send_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(send_data.len())), - }; - - match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), next_slice) - { - Ok(desc) => desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - }; - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Multiple { - desc_lst: desc_lst.into_boxed_slice(), - len: send_data.len(), - next_write: 0, - }), - recv_buff: None, - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - BuffSpec::Indirect(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut index = 0usize; - - for byte in size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match send_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(send_data.len())), - }; - - desc_lst.push( - self.mem_pool - .pull_from_raw_untracked(Rc::clone(&self.mem_pool), next_slice), - ); - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - let ctrl_desc = match self.create_indirect_ctrl(Some(&desc_lst), None) { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Indirect { - desc_lst: desc_lst.into_boxed_slice(), - ctrl_desc, - len: send_data.len(), - next_write: 0, - }), - recv_buff: None, - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - } - } - (None, Some((recv_data, recv_spec))) => { - match recv_spec { - BuffSpec::Single(size) => { - // Buffer must have the right size - if recv_data.len() != size.into() { - return Err(VirtqError::BufferSizeWrong(recv_data.len())); - } - - let desc = match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), recv_data) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: None, - recv_buff: Some(Buffer::Single { - desc_lst: vec![desc].into_boxed_slice(), - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - BuffSpec::Multiple(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut index = 0usize; - - for byte in size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match recv_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), - }; - - match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), next_slice) - { - Ok(desc) => desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - }; - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: None, - recv_buff: Some(Buffer::Multiple { - desc_lst: desc_lst.into_boxed_slice(), - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - BuffSpec::Indirect(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut index = 0usize; - - for byte in size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match recv_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), - }; - - desc_lst.push( - self.mem_pool - .pull_from_raw_untracked(Rc::clone(&self.mem_pool), next_slice), - ); - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - let ctrl_desc = match self.create_indirect_ctrl(None, Some(&desc_lst)) { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: None, - recv_buff: Some(Buffer::Indirect { - desc_lst: desc_lst.into_boxed_slice(), - ctrl_desc, - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - } - } - (Some((send_data, send_spec)), Some((recv_data, recv_spec))) => { - match (send_spec, recv_spec) { - (BuffSpec::Single(send_size), BuffSpec::Single(recv_size)) => { - // Buffer must have the right size - if send_data.len() != send_size.into() { - return Err(VirtqError::BufferSizeWrong(send_data.len())); - } - - let send_desc = match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), send_data) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - // Buffer must have the right size - if recv_data.len() != recv_size.into() { - return Err(VirtqError::BufferSizeWrong(recv_data.len())); - } - - let recv_desc = match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), recv_data) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Single { - desc_lst: vec![send_desc].into_boxed_slice(), - len: send_data.len(), - next_write: 0, - }), - recv_buff: Some(Buffer::Single { - desc_lst: vec![recv_desc].into_boxed_slice(), - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - (BuffSpec::Single(send_size), BuffSpec::Multiple(recv_size_lst)) => { - // Buffer must have the right size - if send_data.len() != send_size.into() { - return Err(VirtqError::BufferSizeWrong(send_data.len())); - } - - let send_desc = match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), send_data) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut index = 0usize; - - for byte in recv_size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match recv_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), - }; - - match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), next_slice) - { - Ok(desc) => recv_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - }; - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Single { - desc_lst: vec![send_desc].into_boxed_slice(), - len: send_data.len(), - next_write: 0, - }), - recv_buff: Some(Buffer::Multiple { - desc_lst: recv_desc_lst.into_boxed_slice(), - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - (BuffSpec::Multiple(send_size_lst), BuffSpec::Multiple(recv_size_lst)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut index = 0usize; - - for byte in send_size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match send_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(send_data.len())), - }; - - match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), next_slice) - { - Ok(desc) => send_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - }; - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut index = 0usize; - - for byte in recv_size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match recv_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), - }; - - match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), next_slice) - { - Ok(desc) => recv_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - }; - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Multiple { - desc_lst: send_desc_lst.into_boxed_slice(), - len: send_data.len(), - next_write: 0, - }), - recv_buff: Some(Buffer::Multiple { - desc_lst: recv_desc_lst.into_boxed_slice(), - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - (BuffSpec::Multiple(send_size_lst), BuffSpec::Single(recv_size)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut index = 0usize; - - for byte in send_size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match send_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(send_data.len())), - }; - - match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), next_slice) - { - Ok(desc) => send_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - }; - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - // Buffer must have the right size - if recv_data.len() != recv_size.into() { - return Err(VirtqError::BufferSizeWrong(recv_data.len())); - } - - let recv_desc = match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), recv_data) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Multiple { - desc_lst: send_desc_lst.into_boxed_slice(), - len: send_data.len(), - next_write: 0, - }), - recv_buff: Some(Buffer::Single { - desc_lst: vec![recv_desc].into_boxed_slice(), - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - (BuffSpec::Indirect(send_size_lst), BuffSpec::Indirect(recv_size_lst)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut index = 0usize; - - for byte in send_size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match send_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(send_data.len())), - }; - - send_desc_lst.push( - self.mem_pool - .pull_from_raw_untracked(Rc::clone(&self.mem_pool), next_slice), - ); - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut index = 0usize; - - for byte in recv_size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match recv_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), - }; - - recv_desc_lst.push( - self.mem_pool - .pull_from_raw_untracked(Rc::clone(&self.mem_pool), next_slice), - ); - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - let ctrl_desc = match self - .create_indirect_ctrl(Some(&send_desc_lst), Some(&recv_desc_lst)) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - recv_buff: Some(Buffer::Indirect { - desc_lst: recv_desc_lst.into_boxed_slice(), - ctrl_desc: ctrl_desc.no_dealloc_clone(), - len: recv_data.len(), - next_write: 0, - }), - send_buff: Some(Buffer::Indirect { - desc_lst: send_desc_lst.into_boxed_slice(), - ctrl_desc, - len: send_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - (BuffSpec::Indirect(_), BuffSpec::Single(_)) - | (BuffSpec::Indirect(_), BuffSpec::Multiple(_)) => Err(VirtqError::BufferInWithDirect), - (BuffSpec::Single(_), BuffSpec::Indirect(_)) - | (BuffSpec::Multiple(_), BuffSpec::Indirect(_)) => Err(VirtqError::BufferInWithDirect), - } - } - } + self.prep_transfer_from_raw_static(send, recv) } fn prep_buffer( @@ -1758,404 +1209,7 @@ impl Virtq for PackedVq { send: Option>, recv: Option>, ) -> Result { - match (send, recv) { - // No buffers specified - (None, None) => Err(VirtqError::BufferNotSpecified), - // Send buffer specified, No recv buffer - (Some(spec), None) => { - match spec { - BuffSpec::Single(size) => { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), size) { - Ok(desc) => { - let buffer = Buffer::Single { - desc_lst: vec![desc].into_boxed_slice(), - len: size.into(), - next_write: 0, - }; - - Ok(BufferToken { - send_buff: Some(buffer), - recv_buff: None, - vq: self.clone(), - ret_send: true, - ret_recv: false, - reusable: true, - }) - } - Err(vq_err) => Err(vq_err), - } - } - BuffSpec::Multiple(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut len = 0usize; - - for size in size_lst { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) { - Ok(desc) => desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - len += usize::from(*size); - } - - let buffer = Buffer::Multiple { - desc_lst: desc_lst.into_boxed_slice(), - len, - next_write: 0, - }; - - Ok(BufferToken { - send_buff: Some(buffer), - recv_buff: None, - vq: self.clone(), - ret_send: true, - ret_recv: false, - reusable: true, - }) - } - BuffSpec::Indirect(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut len = 0usize; - - for size in size_lst { - // As the indirect list does only consume one descriptor for the - // control descriptor, the actual list is untracked - desc_lst.push( - self.mem_pool - .pull_untracked(Rc::clone(&self.mem_pool), *size), - ); - len += usize::from(*size); - } - - let ctrl_desc = match self.create_indirect_ctrl(Some(&desc_lst), None) { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - let buffer = Buffer::Indirect { - desc_lst: desc_lst.into_boxed_slice(), - ctrl_desc, - len, - next_write: 0, - }; - - Ok(BufferToken { - send_buff: Some(buffer), - recv_buff: None, - vq: self.clone(), - ret_send: true, - ret_recv: false, - reusable: true, - }) - } - } - } - // No send buffer, recv buffer is specified - (None, Some(spec)) => { - match spec { - BuffSpec::Single(size) => { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), size) { - Ok(desc) => { - let buffer = Buffer::Single { - desc_lst: vec![desc].into_boxed_slice(), - len: size.into(), - next_write: 0, - }; - - Ok(BufferToken { - send_buff: None, - recv_buff: Some(buffer), - vq: self.clone(), - ret_send: false, - ret_recv: true, - reusable: true, - }) - } - Err(vq_err) => Err(vq_err), - } - } - BuffSpec::Multiple(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut len = 0usize; - - for size in size_lst { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) { - Ok(desc) => desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - len += usize::from(*size); - } - - let buffer = Buffer::Multiple { - desc_lst: desc_lst.into_boxed_slice(), - len, - next_write: 0, - }; - - Ok(BufferToken { - send_buff: None, - recv_buff: Some(buffer), - vq: self.clone(), - ret_send: false, - ret_recv: true, - reusable: true, - }) - } - BuffSpec::Indirect(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut len = 0usize; - - for size in size_lst { - // As the indirect list does only consume one descriptor for the - // control descriptor, the actual list is untracked - desc_lst.push( - self.mem_pool - .pull_untracked(Rc::clone(&self.mem_pool), *size), - ); - len += usize::from(*size); - } - - let ctrl_desc = match self.create_indirect_ctrl(None, Some(&desc_lst)) { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - let buffer = Buffer::Indirect { - desc_lst: desc_lst.into_boxed_slice(), - ctrl_desc, - len, - next_write: 0, - }; - - Ok(BufferToken { - send_buff: None, - recv_buff: Some(buffer), - vq: self.clone(), - ret_send: false, - ret_recv: true, - reusable: true, - }) - } - } - } - // Send buffer specified, recv buffer specified - (Some(send_spec), Some(recv_spec)) => { - match (send_spec, recv_spec) { - (BuffSpec::Single(send_size), BuffSpec::Single(recv_size)) => { - let send_buff = - match self.mem_pool.pull(Rc::clone(&self.mem_pool), send_size) { - Ok(send_desc) => Some(Buffer::Single { - desc_lst: vec![send_desc].into_boxed_slice(), - len: send_size.into(), - next_write: 0, - }), - Err(vq_err) => return Err(vq_err), - }; - - let recv_buff = - match self.mem_pool.pull(Rc::clone(&self.mem_pool), recv_size) { - Ok(recv_desc) => Some(Buffer::Single { - desc_lst: vec![recv_desc].into_boxed_slice(), - len: recv_size.into(), - next_write: 0, - }), - Err(vq_err) => return Err(vq_err), - }; - - Ok(BufferToken { - send_buff, - recv_buff, - vq: self.clone(), - ret_send: true, - ret_recv: true, - reusable: true, - }) - } - (BuffSpec::Single(send_size), BuffSpec::Multiple(recv_size_lst)) => { - let send_buff = - match self.mem_pool.pull(Rc::clone(&self.mem_pool), send_size) { - Ok(send_desc) => Some(Buffer::Single { - desc_lst: vec![send_desc].into_boxed_slice(), - len: send_size.into(), - next_write: 0, - }), - Err(vq_err) => return Err(vq_err), - }; - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut recv_len = 0usize; - - for size in recv_size_lst { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) { - Ok(desc) => recv_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - recv_len += usize::from(*size); - } - - let recv_buff = Some(Buffer::Multiple { - desc_lst: recv_desc_lst.into_boxed_slice(), - len: recv_len, - next_write: 0, - }); - - Ok(BufferToken { - send_buff, - recv_buff, - vq: self.clone(), - ret_send: true, - ret_recv: true, - reusable: true, - }) - } - (BuffSpec::Multiple(send_size_lst), BuffSpec::Multiple(recv_size_lst)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut send_len = 0usize; - for size in send_size_lst { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) { - Ok(desc) => send_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - send_len += usize::from(*size); - } - - let send_buff = Some(Buffer::Multiple { - desc_lst: send_desc_lst.into_boxed_slice(), - len: send_len, - next_write: 0, - }); - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut recv_len = 0usize; - - for size in recv_size_lst { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) { - Ok(desc) => recv_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - recv_len += usize::from(*size); - } - - let recv_buff = Some(Buffer::Multiple { - desc_lst: recv_desc_lst.into_boxed_slice(), - len: recv_len, - next_write: 0, - }); - - Ok(BufferToken { - send_buff, - recv_buff, - vq: self.clone(), - ret_send: true, - ret_recv: true, - reusable: true, - }) - } - (BuffSpec::Multiple(send_size_lst), BuffSpec::Single(recv_size)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut send_len = 0usize; - - for size in send_size_lst { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) { - Ok(desc) => send_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - send_len += usize::from(*size); - } - - let send_buff = Some(Buffer::Multiple { - desc_lst: send_desc_lst.into_boxed_slice(), - len: send_len, - next_write: 0, - }); - - let recv_buff = - match self.mem_pool.pull(Rc::clone(&self.mem_pool), recv_size) { - Ok(recv_desc) => Some(Buffer::Single { - desc_lst: vec![recv_desc].into_boxed_slice(), - len: recv_size.into(), - next_write: 0, - }), - Err(vq_err) => return Err(vq_err), - }; - - Ok(BufferToken { - send_buff, - recv_buff, - vq: self.clone(), - ret_send: true, - ret_recv: true, - reusable: true, - }) - } - (BuffSpec::Indirect(send_size_lst), BuffSpec::Indirect(recv_size_lst)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut send_len = 0usize; - - for size in send_size_lst { - // As the indirect list does only consume one descriptor for the - // control descriptor, the actual list is untracked - send_desc_lst.push( - self.mem_pool - .pull_untracked(Rc::clone(&self.mem_pool), *size), - ); - send_len += usize::from(*size); - } - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut recv_len = 0usize; - - for size in recv_size_lst { - // As the indirect list does only consume one descriptor for the - // control descriptor, the actual list is untracked - recv_desc_lst.push( - self.mem_pool - .pull_untracked(Rc::clone(&self.mem_pool), *size), - ); - recv_len += usize::from(*size); - } - - let ctrl_desc = match self - .create_indirect_ctrl(Some(&send_desc_lst), Some(&recv_desc_lst)) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - let recv_buff = Some(Buffer::Indirect { - desc_lst: recv_desc_lst.into_boxed_slice(), - ctrl_desc: ctrl_desc.no_dealloc_clone(), - len: recv_len, - next_write: 0, - }); - let send_buff = Some(Buffer::Indirect { - desc_lst: send_desc_lst.into_boxed_slice(), - ctrl_desc, - len: send_len, - next_write: 0, - }); - - Ok(BufferToken { - send_buff, - recv_buff, - vq: self.clone(), - ret_send: true, - ret_recv: true, - reusable: true, - }) - } - (BuffSpec::Indirect(_), BuffSpec::Single(_)) - | (BuffSpec::Indirect(_), BuffSpec::Multiple(_)) => Err(VirtqError::BufferInWithDirect), - (BuffSpec::Single(_), BuffSpec::Indirect(_)) - | (BuffSpec::Multiple(_), BuffSpec::Indirect(_)) => Err(VirtqError::BufferInWithDirect), - } - } - } + self.prep_buffer_static(send, recv) } fn size(&self) -> VqSize { @@ -2163,8 +1217,11 @@ impl Virtq for PackedVq { } } -// Private Interface for PackedVq -impl PackedVq { +impl VirtqPrivate for PackedVq { + fn mem_pool(&self) -> Rc { + self.mem_pool.clone() + } + fn create_indirect_ctrl( &self, send: Option<&Vec>, diff --git a/src/drivers/virtio/virtqueue/split.rs b/src/drivers/virtio/virtqueue/split.rs index 17c23fd32a..3906f3d83b 100644 --- a/src/drivers/virtio/virtqueue/split.rs +++ b/src/drivers/virtio/virtqueue/split.rs @@ -17,8 +17,8 @@ use super::super::transport::mmio::{ComCfg, NotifCfg, NotifCtrl}; use super::super::transport::pci::{ComCfg, NotifCfg, NotifCtrl}; use super::error::VirtqError; use super::{ - BuffSpec, Buffer, BufferToken, Bytes, DescrFlags, MemDescr, MemPool, Transfer, TransferState, - TransferToken, Virtq, VqIndex, VqSize, + BuffSpec, BufferToken, Bytes, DescrFlags, MemDescr, MemPool, Transfer, TransferState, + TransferToken, Virtq, VirtqPrivate, VqIndex, VqSize, }; use crate::arch::memory_barrier; use crate::arch::mm::paging::{BasePageSize, PageSize}; @@ -418,556 +418,7 @@ impl Virtq for SplitVq { send: Option<(&[u8], BuffSpec<'_>)>, recv: Option<(&mut [u8], BuffSpec<'_>)>, ) -> Result { - match (send, recv) { - (None, None) => Err(VirtqError::BufferNotSpecified), - (Some((send_data, send_spec)), None) => { - match send_spec { - BuffSpec::Single(size) => { - // Buffer must have the right size - if send_data.len() != size.into() { - return Err(VirtqError::BufferSizeWrong(send_data.len())); - } - - let desc = match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), send_data) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Single { - desc_lst: vec![desc].into_boxed_slice(), - len: send_data.len(), - next_write: 0, - }), - recv_buff: None, - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - BuffSpec::Multiple(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut index = 0usize; - - for byte in size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match send_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(send_data.len())), - }; - - match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), next_slice) - { - Ok(desc) => desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - }; - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Multiple { - desc_lst: desc_lst.into_boxed_slice(), - len: send_data.len(), - next_write: 0, - }), - recv_buff: None, - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - BuffSpec::Indirect(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut index = 0usize; - - for byte in size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match send_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(send_data.len())), - }; - - desc_lst.push( - self.mem_pool - .pull_from_raw_untracked(Rc::clone(&self.mem_pool), next_slice), - ); - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - let ctrl_desc = match self.create_indirect_ctrl(Some(&desc_lst), None) { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Indirect { - desc_lst: desc_lst.into_boxed_slice(), - ctrl_desc, - len: send_data.len(), - next_write: 0, - }), - recv_buff: None, - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - } - } - (None, Some((recv_data, recv_spec))) => { - match recv_spec { - BuffSpec::Single(size) => { - // Buffer must have the right size - if recv_data.len() != size.into() { - return Err(VirtqError::BufferSizeWrong(recv_data.len())); - } - - let desc = match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), recv_data) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: None, - recv_buff: Some(Buffer::Single { - desc_lst: vec![desc].into_boxed_slice(), - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - BuffSpec::Multiple(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut index = 0usize; - - for byte in size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match recv_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), - }; - - match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), next_slice) - { - Ok(desc) => desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - }; - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: None, - recv_buff: Some(Buffer::Multiple { - desc_lst: desc_lst.into_boxed_slice(), - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - BuffSpec::Indirect(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut index = 0usize; - - for byte in size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match recv_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), - }; - - desc_lst.push( - self.mem_pool - .pull_from_raw_untracked(Rc::clone(&self.mem_pool), next_slice), - ); - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - let ctrl_desc = match self.create_indirect_ctrl(None, Some(&desc_lst)) { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: None, - recv_buff: Some(Buffer::Indirect { - desc_lst: desc_lst.into_boxed_slice(), - ctrl_desc, - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - } - } - (Some((send_data, send_spec)), Some((recv_data, recv_spec))) => { - match (send_spec, recv_spec) { - (BuffSpec::Single(send_size), BuffSpec::Single(recv_size)) => { - // Buffer must have the right size - if send_data.len() != send_size.into() { - return Err(VirtqError::BufferSizeWrong(send_data.len())); - } - - let send_desc = match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), send_data) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - // Buffer must have the right size - if recv_data.len() != recv_size.into() { - return Err(VirtqError::BufferSizeWrong(recv_data.len())); - } - - let recv_desc = match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), recv_data) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Single { - desc_lst: vec![send_desc].into_boxed_slice(), - len: send_data.len(), - next_write: 0, - }), - recv_buff: Some(Buffer::Single { - desc_lst: vec![recv_desc].into_boxed_slice(), - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - (BuffSpec::Single(send_size), BuffSpec::Multiple(recv_size_lst)) => { - // Buffer must have the right size - if send_data.len() != send_size.into() { - return Err(VirtqError::BufferSizeWrong(send_data.len())); - } - - let send_desc = match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), send_data) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut index = 0usize; - - for byte in recv_size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match recv_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), - }; - - match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), next_slice) - { - Ok(desc) => recv_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - }; - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Single { - desc_lst: vec![send_desc].into_boxed_slice(), - len: send_data.len(), - next_write: 0, - }), - recv_buff: Some(Buffer::Multiple { - desc_lst: recv_desc_lst.into_boxed_slice(), - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - (BuffSpec::Multiple(send_size_lst), BuffSpec::Multiple(recv_size_lst)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut index = 0usize; - - for byte in send_size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match send_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(send_data.len())), - }; - - match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), next_slice) - { - Ok(desc) => send_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - }; - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut index = 0usize; - - for byte in recv_size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match recv_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), - }; - - match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), next_slice) - { - Ok(desc) => recv_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - }; - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Multiple { - desc_lst: send_desc_lst.into_boxed_slice(), - len: send_data.len(), - next_write: 0, - }), - recv_buff: Some(Buffer::Multiple { - desc_lst: recv_desc_lst.into_boxed_slice(), - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - (BuffSpec::Multiple(send_size_lst), BuffSpec::Single(recv_size)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut index = 0usize; - - for byte in send_size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match send_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(send_data.len())), - }; - - match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), next_slice) - { - Ok(desc) => send_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - }; - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - // Buffer must have the right size - if recv_data.len() != recv_size.into() { - return Err(VirtqError::BufferSizeWrong(recv_data.len())); - } - - let recv_desc = match self - .mem_pool - .pull_from_raw(Rc::clone(&self.mem_pool), recv_data) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - send_buff: Some(Buffer::Multiple { - desc_lst: send_desc_lst.into_boxed_slice(), - len: send_data.len(), - next_write: 0, - }), - recv_buff: Some(Buffer::Single { - desc_lst: vec![recv_desc].into_boxed_slice(), - len: recv_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - (BuffSpec::Indirect(send_size_lst), BuffSpec::Indirect(recv_size_lst)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut index = 0usize; - - for byte in send_size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match send_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(send_data.len())), - }; - - send_desc_lst.push( - self.mem_pool - .pull_from_raw_untracked(Rc::clone(&self.mem_pool), next_slice), - ); - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut index = 0usize; - - for byte in recv_size_lst { - let end_index = index + usize::from(*byte); - let next_slice = match recv_data.get(index..end_index) { - Some(slice) => slice, - None => return Err(VirtqError::BufferSizeWrong(recv_data.len())), - }; - - recv_desc_lst.push( - self.mem_pool - .pull_from_raw_untracked(Rc::clone(&self.mem_pool), next_slice), - ); - - // update the starting index for the next iteration - index += usize::from(*byte); - } - - let ctrl_desc = match self - .create_indirect_ctrl(Some(&send_desc_lst), Some(&recv_desc_lst)) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - Ok(TransferToken { - state: TransferState::Ready, - buff_tkn: Some(BufferToken { - recv_buff: Some(Buffer::Indirect { - desc_lst: recv_desc_lst.into_boxed_slice(), - ctrl_desc: ctrl_desc.no_dealloc_clone(), - len: recv_data.len(), - next_write: 0, - }), - send_buff: Some(Buffer::Indirect { - desc_lst: send_desc_lst.into_boxed_slice(), - ctrl_desc, - len: send_data.len(), - next_write: 0, - }), - vq: self, - ret_send: false, - ret_recv: false, - reusable: false, - }), - await_queue: None, - }) - } - (BuffSpec::Indirect(_), BuffSpec::Single(_)) - | (BuffSpec::Indirect(_), BuffSpec::Multiple(_)) => Err(VirtqError::BufferInWithDirect), - (BuffSpec::Single(_), BuffSpec::Indirect(_)) - | (BuffSpec::Multiple(_), BuffSpec::Indirect(_)) => Err(VirtqError::BufferInWithDirect), - } - } - } + self.prep_transfer_from_raw_static(send, recv) } fn prep_buffer( @@ -975,404 +426,7 @@ impl Virtq for SplitVq { send: Option>, recv: Option>, ) -> Result { - match (send, recv) { - // No buffers specified - (None, None) => Err(VirtqError::BufferNotSpecified), - // Send buffer specified, No recv buffer - (Some(spec), None) => { - match spec { - BuffSpec::Single(size) => { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), size) { - Ok(desc) => { - let buffer = Buffer::Single { - desc_lst: vec![desc].into_boxed_slice(), - len: size.into(), - next_write: 0, - }; - - Ok(BufferToken { - send_buff: Some(buffer), - recv_buff: None, - vq: self.clone(), - ret_send: true, - ret_recv: false, - reusable: true, - }) - } - Err(vq_err) => Err(vq_err), - } - } - BuffSpec::Multiple(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut len = 0usize; - - for size in size_lst { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) { - Ok(desc) => desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - len += usize::from(*size); - } - - let buffer = Buffer::Multiple { - desc_lst: desc_lst.into_boxed_slice(), - len, - next_write: 0, - }; - - Ok(BufferToken { - send_buff: Some(buffer), - recv_buff: None, - vq: self.clone(), - ret_send: true, - ret_recv: false, - reusable: true, - }) - } - BuffSpec::Indirect(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut len = 0usize; - - for size in size_lst { - // As the indirect list does only consume one descriptor for the - // control descriptor, the actual list is untracked - desc_lst.push( - self.mem_pool - .pull_untracked(Rc::clone(&self.mem_pool), *size), - ); - len += usize::from(*size); - } - - let ctrl_desc = match self.create_indirect_ctrl(Some(&desc_lst), None) { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - let buffer = Buffer::Indirect { - desc_lst: desc_lst.into_boxed_slice(), - ctrl_desc, - len, - next_write: 0, - }; - - Ok(BufferToken { - send_buff: Some(buffer), - recv_buff: None, - vq: self.clone(), - ret_send: true, - ret_recv: false, - reusable: true, - }) - } - } - } - // No send buffer, recv buffer is specified - (None, Some(spec)) => { - match spec { - BuffSpec::Single(size) => { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), size) { - Ok(desc) => { - let buffer = Buffer::Single { - desc_lst: vec![desc].into_boxed_slice(), - len: size.into(), - next_write: 0, - }; - - Ok(BufferToken { - send_buff: None, - recv_buff: Some(buffer), - vq: self.clone(), - ret_send: false, - ret_recv: true, - reusable: true, - }) - } - Err(vq_err) => Err(vq_err), - } - } - BuffSpec::Multiple(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut len = 0usize; - - for size in size_lst { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) { - Ok(desc) => desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - len += usize::from(*size); - } - - let buffer = Buffer::Multiple { - desc_lst: desc_lst.into_boxed_slice(), - len, - next_write: 0, - }; - - Ok(BufferToken { - send_buff: None, - recv_buff: Some(buffer), - vq: self.clone(), - ret_send: false, - ret_recv: true, - reusable: true, - }) - } - BuffSpec::Indirect(size_lst) => { - let mut desc_lst: Vec = Vec::with_capacity(size_lst.len()); - let mut len = 0usize; - - for size in size_lst { - // As the indirect list does only consume one descriptor for the - // control descriptor, the actual list is untracked - desc_lst.push( - self.mem_pool - .pull_untracked(Rc::clone(&self.mem_pool), *size), - ); - len += usize::from(*size); - } - - let ctrl_desc = match self.create_indirect_ctrl(None, Some(&desc_lst)) { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - let buffer = Buffer::Indirect { - desc_lst: desc_lst.into_boxed_slice(), - ctrl_desc, - len, - next_write: 0, - }; - - Ok(BufferToken { - send_buff: None, - recv_buff: Some(buffer), - vq: self.clone(), - ret_send: false, - ret_recv: true, - reusable: true, - }) - } - } - } - // Send buffer specified, recv buffer specified - (Some(send_spec), Some(recv_spec)) => { - match (send_spec, recv_spec) { - (BuffSpec::Single(send_size), BuffSpec::Single(recv_size)) => { - let send_buff = - match self.mem_pool.pull(Rc::clone(&self.mem_pool), send_size) { - Ok(send_desc) => Some(Buffer::Single { - desc_lst: vec![send_desc].into_boxed_slice(), - len: send_size.into(), - next_write: 0, - }), - Err(vq_err) => return Err(vq_err), - }; - - let recv_buff = - match self.mem_pool.pull(Rc::clone(&self.mem_pool), recv_size) { - Ok(recv_desc) => Some(Buffer::Single { - desc_lst: vec![recv_desc].into_boxed_slice(), - len: recv_size.into(), - next_write: 0, - }), - Err(vq_err) => return Err(vq_err), - }; - - Ok(BufferToken { - send_buff, - recv_buff, - vq: self.clone(), - ret_send: true, - ret_recv: true, - reusable: true, - }) - } - (BuffSpec::Single(send_size), BuffSpec::Multiple(recv_size_lst)) => { - let send_buff = - match self.mem_pool.pull(Rc::clone(&self.mem_pool), send_size) { - Ok(send_desc) => Some(Buffer::Single { - desc_lst: vec![send_desc].into_boxed_slice(), - len: send_size.into(), - next_write: 0, - }), - Err(vq_err) => return Err(vq_err), - }; - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut recv_len = 0usize; - - for size in recv_size_lst { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) { - Ok(desc) => recv_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - recv_len += usize::from(*size); - } - - let recv_buff = Some(Buffer::Multiple { - desc_lst: recv_desc_lst.into_boxed_slice(), - len: recv_len, - next_write: 0, - }); - - Ok(BufferToken { - send_buff, - recv_buff, - vq: self.clone(), - ret_send: true, - ret_recv: true, - reusable: true, - }) - } - (BuffSpec::Multiple(send_size_lst), BuffSpec::Multiple(recv_size_lst)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut send_len = 0usize; - for size in send_size_lst { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) { - Ok(desc) => send_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - send_len += usize::from(*size); - } - - let send_buff = Some(Buffer::Multiple { - desc_lst: send_desc_lst.into_boxed_slice(), - len: send_len, - next_write: 0, - }); - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut recv_len = 0usize; - - for size in recv_size_lst { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) { - Ok(desc) => recv_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - recv_len += usize::from(*size); - } - - let recv_buff = Some(Buffer::Multiple { - desc_lst: recv_desc_lst.into_boxed_slice(), - len: recv_len, - next_write: 0, - }); - - Ok(BufferToken { - send_buff, - recv_buff, - vq: self.clone(), - ret_send: true, - ret_recv: true, - reusable: true, - }) - } - (BuffSpec::Multiple(send_size_lst), BuffSpec::Single(recv_size)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut send_len = 0usize; - - for size in send_size_lst { - match self.mem_pool.pull(Rc::clone(&self.mem_pool), *size) { - Ok(desc) => send_desc_lst.push(desc), - Err(vq_err) => return Err(vq_err), - } - send_len += usize::from(*size); - } - - let send_buff = Some(Buffer::Multiple { - desc_lst: send_desc_lst.into_boxed_slice(), - len: send_len, - next_write: 0, - }); - - let recv_buff = - match self.mem_pool.pull(Rc::clone(&self.mem_pool), recv_size) { - Ok(recv_desc) => Some(Buffer::Single { - desc_lst: vec![recv_desc].into_boxed_slice(), - len: recv_size.into(), - next_write: 0, - }), - Err(vq_err) => return Err(vq_err), - }; - - Ok(BufferToken { - send_buff, - recv_buff, - vq: self.clone(), - ret_send: true, - ret_recv: true, - reusable: true, - }) - } - (BuffSpec::Indirect(send_size_lst), BuffSpec::Indirect(recv_size_lst)) => { - let mut send_desc_lst: Vec = - Vec::with_capacity(send_size_lst.len()); - let mut send_len = 0usize; - - for size in send_size_lst { - // As the indirect list does only consume one descriptor for the - // control descriptor, the actual list is untracked - send_desc_lst.push( - self.mem_pool - .pull_untracked(Rc::clone(&self.mem_pool), *size), - ); - send_len += usize::from(*size); - } - - let mut recv_desc_lst: Vec = - Vec::with_capacity(recv_size_lst.len()); - let mut recv_len = 0usize; - - for size in recv_size_lst { - // As the indirect list does only consume one descriptor for the - // control descriptor, the actual list is untracked - recv_desc_lst.push( - self.mem_pool - .pull_untracked(Rc::clone(&self.mem_pool), *size), - ); - recv_len += usize::from(*size); - } - - let ctrl_desc = match self - .create_indirect_ctrl(Some(&send_desc_lst), Some(&recv_desc_lst)) - { - Ok(desc) => desc, - Err(vq_err) => return Err(vq_err), - }; - - let recv_buff = Some(Buffer::Indirect { - desc_lst: recv_desc_lst.into_boxed_slice(), - ctrl_desc: ctrl_desc.no_dealloc_clone(), - len: recv_len, - next_write: 0, - }); - let send_buff = Some(Buffer::Indirect { - desc_lst: send_desc_lst.into_boxed_slice(), - ctrl_desc, - len: send_len, - next_write: 0, - }); - - Ok(BufferToken { - send_buff, - recv_buff, - vq: self.clone(), - ret_send: true, - ret_recv: true, - reusable: true, - }) - } - (BuffSpec::Indirect(_), BuffSpec::Single(_)) - | (BuffSpec::Indirect(_), BuffSpec::Multiple(_)) => Err(VirtqError::BufferInWithDirect), - (BuffSpec::Single(_), BuffSpec::Indirect(_)) - | (BuffSpec::Multiple(_), BuffSpec::Indirect(_)) => Err(VirtqError::BufferInWithDirect), - } - } - } + self.prep_buffer_static(send, recv) } fn size(&self) -> VqSize { @@ -1380,8 +434,7 @@ impl Virtq for SplitVq { } } -// Private Interface for PackedVq -impl SplitVq { +impl VirtqPrivate for SplitVq { fn create_indirect_ctrl( &self, send: Option<&Vec>, @@ -1515,4 +568,7 @@ impl SplitVq { } } } + fn mem_pool(&self) -> Rc { + self.mem_pool.clone() + } }