Implement device-mapper and bio

dm-sworndisk/src/bio.rs

@@ -0,0 +1,343 @@
+use core::{
+    ops::{Deref, DerefMut, Range},
+    ptr::NonNull,
+};
+use kernel::{
+    bindings::{self, GFP_KERNEL},
+    error::to_result,
+    prelude::*,
+    types::Opaque,
+};
+
+use crate::block_device::{BlockDevice, BLOCK_SIZE};
+
+const GFP_NOIO: bindings::gfp_t = bindings::___GFP_DIRECT_RECLAIM | bindings::___GFP_KSWAPD_RECLAIM;
+
+const BIO_INLINE_VECS: core::ffi::c_ushort = 4;
+
+/// A wrapper for kernel's struct `bio`
+pub struct Bio {
+    inner: NonNull<bindings::bio>,
+    with_owned_pages: bool,
+    nr_pages: usize,
+}
+
+impl Bio {
+    /// Constructs a `Bio` from raw pointer.
+    ///
+    /// # Safety
+    ///
+    /// User must provide a valid pointer to the kernel's `bio`.
+    pub unsafe fn from_raw(ptr: *mut bindings::bio) -> Self {
+        // Get a reference to a `bio`, so it won't disappear. And `bio_put` is
+        // called when it's been dropped.
+        bindings::bio_get(ptr);
+        Self {
+            inner: NonNull::new_unchecked(ptr),
+            with_owned_pages: false,
+            nr_pages: 0,
+        }
+    }
+
+    /// Allocates a bio with `nr_blocks` as its io buffer.
+    pub fn alloc(bdev: &BlockDevice, nr_blocks: usize, op: BioOp) -> Result<Self> {
+        // SAFETY: no safety requirements on this FFI call.
+        let bio = unsafe {
+            bindings::bio_alloc_bioset(
+                bdev.as_ptr(),
+                BIO_INLINE_VECS,
+                op as _,
+                GFP_NOIO,
+                &mut bindings::fs_bio_set,
+            )
+        };
+        if bio.is_null() {
+            return Err(ENOMEM);
+        }
+
+        // SAFETY: no safety requirements on this FFI call.
+        let virt = unsafe { bindings::alloc_pages_exact(nr_blocks * BLOCK_SIZE, GFP_KERNEL) };
+        if virt.is_null() {
+            // SAFETY: the bio is allocated by `bio_alloc_bioset`, and it is
+            // non-null (checked above) and valid.
+            unsafe { bindings::bio_put(bio) };
+            return Err(ENOMEM);
+        }
+
+        // SAFETY: the `bio` and `virt` address is valid (checked above).
+        unsafe {
+            let page = bindings::virt_to_page(virt);
+            bindings::__bio_add_page(bio, page, (nr_blocks * BLOCK_SIZE) as _, 0);
+            Ok(Self {
+                inner: NonNull::new_unchecked(bio),
+                with_owned_pages: true,
+                nr_pages: nr_blocks,
+            })
+        }
+    }
+
+    // Clone a bio that shares the original bio's biovec.
+    pub fn alloc_clone(&self) -> Result<Self> {
+        let src = self.inner.as_ptr();
+        // SAFETY: the `src.bi_bdev` is a valid `BlockDevice`.
+        let cloned = unsafe {
+            bindings::bio_alloc_clone((*src).bi_bdev, src, GFP_NOIO, &mut bindings::fs_bio_set)
+        };
+        if cloned.is_null() {
+            return Err(ENOMEM);
+        }
+
+        // SAFETY: the `cloned` is non-null and valid.
+        unsafe {
+            Ok(Self {
+                inner: NonNull::new_unchecked(cloned),
+                with_owned_pages: false,
+                nr_pages: 0,
+            })
+        }
+    }
+
+    /// Split a smaller bio from current bio.
+    pub fn split(&self, sectors: usize) -> Result<Self> {
+        let bio = self.inner.as_ptr();
+        // SAFETY: `self.inner` is a valid bio.
+        let split = unsafe {
+            bindings::bio_split(
+                bio,
+                sectors as core::ffi::c_int,
+                GFP_NOIO,
+                &mut bindings::fs_bio_set,
+            )
+        };
+        if split.is_null() {
+            return Err(EINVAL);
+        }
+
+        // SAFETY: the `split` is non-null and valid.
+        unsafe {
+            Ok(Self {
+                inner: NonNull::new_unchecked(split),
+                with_owned_pages: false,
+                nr_pages: 0,
+            })
+        }
+    }
+
+    /// Get the operation and flags. The bottom 8 bits are encoding the operation,
+    /// and the remaining 24 for flags.
+    pub fn opf(&self) -> u32 {
+        // SAFETY: `self.inner` is a valid bio.
+        unsafe { (*self.inner.as_ptr()).bi_opf }
+    }
+
+    /// Set the operation and flags.
+    pub fn set_opf(&mut self, opf: u32) {
+        // SAFETY: `self.inner` is a valid bio.
+        unsafe { (*self.inner.as_ptr()).bi_opf = opf }
+    }
+
+    /// Return `true` if the bio request is write.
+    pub fn is_write(&self) -> bool {
+        self.opf() & bindings::req_op_REQ_OP_WRITE != 0
+    }
+
+    /// Return the block range of the bio.
+    pub fn region(&self) -> Range<usize> {
+        // SAFETY: `self.inner` is a valid bio.
+        let start_sector = unsafe { (*self.inner.as_ptr()).bi_iter.bi_sector as usize };
+        let start_block = start_sector / (bindings::PAGE_SECTORS as usize);
+        let nr_bytes = unsafe { (*self.inner.as_ptr()).bi_iter.bi_size as usize };
+        let nr_blocks = (nr_bytes + BLOCK_SIZE - 1) / BLOCK_SIZE;
+        Range {
+            start: start_block,
+            end: start_block + nr_blocks,
+        }
+    }
+
+    /// Set the block range of the bio.
+    pub fn set_region(&mut self, region: Range<usize>) {
+        let start_sector = region.start * (bindings::PAGE_SECTORS as usize);
+        let nr_bytes = region.len() * BLOCK_SIZE;
+        // SAFETY: `self.inner` is a valid bio.
+        unsafe {
+            (*self.inner.as_ptr()).bi_iter.bi_sector = start_sector as _;
+            (*self.inner.as_ptr()).bi_iter.bi_size = nr_bytes as _;
+        }
+    }
+
+    /// Set the block device of bio request.
+    pub fn set_dev(&mut self, bdev: &BlockDevice) {
+        // SAFETY: `self.inner` is a valid bio.
+        unsafe { bindings::bio_set_dev(self.inner.as_ptr(), bdev.as_ptr()) }
+    }
+
+    /// Submit the bio.
+    ///
+    /// The success/failure status of the request, along with notification of
+    /// completion, is delivered asynchronously through the `bi_end_io` callback
+    /// in bio.
+    pub fn submit(&self) {
+        // SAFETY: `self.inner` is a valid bio.
+        unsafe { bindings::submit_bio(self.inner.as_ptr()) }
+    }
+
+    /// Submit a bio (synchronously).
+    pub fn submit_sync(&self) -> Result<()> {
+        // SAFETY: `self.inner` is a valid bio.
+        let err = unsafe { bindings::submit_bio_wait(self.inner.as_ptr()) };
+        to_result(err)
+    }
+
+    /// End the bio.
+    ///
+    /// This will end I/O on the whole bio. No one should call bi_end_io()
+    /// directly on a bio unless they own it and thus know that it has an
+    /// end_io function.
+    pub fn end(&self) {
+        // SAFETY: `self.inner` is a valid bio.
+        unsafe { bindings::bio_endio(self.inner.as_ptr()) }
+    }
+
+    /// Return a iterator on the bio_vec.
+    pub fn iter(&self) -> BioVecIter {
+        let bio = self.inner.as_ptr();
+        // SAFETY: `self.inner` is a valid bio.
+        unsafe { BioVecIter::from_raw((*bio).bi_io_vec, (*bio).bi_vcnt as _) }
+    }
+}
+
+impl Drop for Bio {
+    fn drop(&mut self) {
+        let bio = self.inner.as_ptr();
+        if self.with_owned_pages {
+            // SAFETY: the bio is constructed by `alloc` with owned pages.
+            // We should free those pages here.
+            unsafe {
+                let page = (*(*bio).bi_io_vec).bv_page;
+                let virt = bindings::page_to_virt(page);
+                bindings::free_pages_exact(virt, self.nr_pages * BLOCK_SIZE);
+            }
+        }
+        // SAFETY: `self.inner` is a valid bio.
+        unsafe {
+            // Put a reference to a &struct bio, either one you have gotten
+            // with `bio_alloc`, `bio_get` or `bio_clone_*`. The last put of
+            // a bio will free it.
+            bindings::bio_put(bio);
+        }
+    }
+}
+
+/// A contiguous range of physical memory addresses.
+///
+/// Used to iterate the io buffer in `Bio`.
+pub struct BioVec {
+    ptr: NonNull<bindings::bio_vec>,
+    virt_addr: usize,
+    len: usize,
+}
+
+impl BioVec {
+    /// Constructs a `BioVec` from raw pointer.
+    ///
+    /// # Safety
+    ///
+    /// User must provide a valid pointer to the kernel's `bio_vec`.
+    pub unsafe fn from_raw(bio_vec: *mut bindings::bio_vec) -> Self {
+        let mut ptr = bindings::kmap_local_page((*bio_vec).bv_page);
+        let virt_addr = ptr.add((*bio_vec).bv_offset as _) as usize;
+        Self {
+            ptr: NonNull::new_unchecked(bio_vec),
+            virt_addr,
+            len: (*bio_vec).bv_len as usize,
+        }
+    }
+}
+
+impl Drop for BioVec {
+    fn drop(&mut self) {
+        // SAFETY: the `virt_addr` is mapped by `kmap_local_page`, it's valid.
+        unsafe { bindings::kunmap_local(self.virt_addr as _) }
+    }
+}
+
+impl Deref for BioVec {
+    type Target = [u8];
+    fn deref(&self) -> &Self::Target {
+        // SAFETY: the `virt_addr` is mapped by `kmap_local_page`, it's valid.
+        unsafe { core::slice::from_raw_parts(self.virt_addr as _, self.len) }
+    }
+}
+
+impl DerefMut for BioVec {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        // SAFETY: the `virt_addr` is mapped by `kmap_local_page`, it's valid.
+        unsafe { core::slice::from_raw_parts_mut(self.virt_addr as _, self.len) }
+    }
+}
+
+/// An iterator on `BioVec`.
+pub struct BioVecIter {
+    bio_vec: NonNull<bindings::bio_vec>,
+    bi_vcnt: usize,
+}
+
+impl BioVecIter {
+    /// Constructs a `BioVecIter` from raw pointer.
+    ///
+    /// # Safety
+    ///
+    /// User must provide a valid array to the kernel's `bio_vec`, whose length
+    /// is exactly `bi_vcnt`.
+    pub unsafe fn from_raw(bio_vec: *mut bindings::bio_vec, bi_vcnt: usize) -> Self {
+        Self {
+            bio_vec: NonNull::new_unchecked(bio_vec),
+            bi_vcnt,
+        }
+    }
+}
+
+impl Iterator for BioVecIter {
+    type Item = BioVec;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.bi_vcnt == 0 {
+            return None;
+        }
+        let next = unsafe { self.bio_vec.as_ptr().add(1) };
+        let vcnt = self.bi_vcnt - 1;
+        // SAFETY: the `next` bio_vec is existed, so it's valid.
+        unsafe {
+            let bio_vec = BioVec::from_raw(next);
+            self.bio_vec = NonNull::new_unchecked(next);
+            self.bi_vcnt = vcnt;
+            Some(bio_vec)
+        }
+    }
+}
+
+/// Wrap the bio operations (see [req_op]).
+///
+/// [`req_op`]: include/linux/blk_types.h
+#[allow(missing_docs)]
+#[repr(u8)]
+#[derive(Clone, Copy, Debug)]
+pub enum BioOp {
+    Read,
+    Write,
+    Flush,
+    Discard,
+    Undefined,
+}
+
+impl From<u8> for BioOp {
+    fn from(value: u8) -> Self {
+        match value {
+            0 => Self::Read,
+            1 => Self::Write,
+            2 => Self::Flush,
+            3 => Self::Discard,
+            _ => Self::Undefined,
+        }
+    }
+}