feat(fs-gen): implement initramfs building and archiving

Cargo.toml

@@ -1,3 +1,3 @@
 [workspace]
-members = ["src/api", "src/vmm", "src/cli"]
+members = ["src/api", "src/vmm", "src/cli", "src/fs-gen"]
 resolver = "2"

src/fs-gen/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "fs-gen"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+clap = { version = "4.5.3", features = ["derive", "wrap_help", "string"] }
+dircpy = "0.3.16"
+fuse-backend-rs = "0.12.0"
+flate2 = "1.0.28"
+once_cell = "1.19.0"
+regex = "1.10.4"
+reqwest = { version = "0.12.3", features = ["blocking", "json"] }
+serde = { version = "1.0.197", features = ["derive"] }
+serde_json = "1.0.115"
+signal-hook = "0.3.17"
+tar = "0.4.40"
+validator = { version = "0.17.0", features = ["derive"] }
+anyhow = "1.0.82"

src/fs-gen/src/cli_args.rs

@@ -0,0 +1,80 @@
+use std::{env, path::PathBuf};
+
+use clap::{command, error::ErrorKind, CommandFactory, Parser};
+use regex::Regex;
+
+use once_cell::sync::Lazy;
+
+// So, for any of you who may be scared, this is the regex from the OCI Distribution Sepcification for the image name + the tag
+static RE_IMAGE_NAME: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(\/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*:[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}").unwrap()
+});
+
+/// Convert an OCI image into a CPIO file
+#[derive(Parser, Debug)]
+#[command(version, about, long_about = None)]
+pub struct CliArgs {
+    /// The name of the image to download
+    pub image_name: String,
+
+    /// The path to the output file
+    #[arg(short='o', long="output", default_value=get_default_output_file().into_os_string())]
+    pub output_file: PathBuf,
+
+    /// The path to the temporary folder
+    #[arg(short='t', long="tempdir", default_value=get_default_temp_directory().into_os_string())]
+    pub temp_directory: PathBuf,
+
+    /// The host path to the guest agent binary
+    pub agent_host_path: PathBuf,
+}
+
+impl CliArgs {
+    /// Get the cli arguments with additional validation
+    pub fn get_args() -> Self {
+        let args = CliArgs::parse();
+
+        args.validate_image();
+        args.validate_host_path();
+
+        args
+    }
+
+    fn validate_image(&self) {
+        if !RE_IMAGE_NAME.is_match(&self.image_name) {
+            let mut cmd = CliArgs::command();
+            cmd.error(
+                ErrorKind::InvalidValue,
+                format!("Invalid image name: \"{}\"", self.image_name),
+            )
+            .exit();
+        }
+    }
+
+    fn validate_host_path(&self) {
+        if !self.agent_host_path.exists() {
+            let mut cmd = CliArgs::command();
+            cmd.error(
+                ErrorKind::InvalidValue,
+                format!(
+                    "File not found for agent binary: \"{}\"",
+                    self.agent_host_path.to_string_lossy()
+                ),
+            )
+            .exit();
+        }
+    }
+}
+
+/// Get the default output path for the cpio file.
+fn get_default_temp_directory() -> PathBuf {
+    let mut path = env::current_dir().unwrap();
+    path.push(".cloudlet_temp/");
+    path
+}
+
+fn get_default_output_file() -> PathBuf {
+    let mut path = env::current_dir().unwrap();
+    path.push("initramfs.img");
+    path
+}

src/fs-gen/src/image_builder.rs

@@ -0,0 +1,159 @@
+use std::{
+    fs::{self},
+    path::{Path, PathBuf},
+    sync::Arc,
+    thread,
+};
+
+use anyhow::{Context, Result};
+use fuse_backend_rs::{
+    api::{filesystem::Layer, server::Server},
+    overlayfs::{config::Config, OverlayFs},
+    passthrough::{self, PassthroughFs},
+    transport::{FuseChannel, FuseSession},
+};
+
+pub struct FuseServer {
+    server: Arc<Server<Arc<OverlayFs>>>,
+    ch: FuseChannel,
+}
+
+type BoxedLayer = Box<dyn Layer<Inode = u64, Handle = u64> + Send + Sync>;
+
+/// Initialiazes a passthrough fs for a given layer
+/// a passthrough fs is just a dummy implementation to map to the physical disk
+/// # Usage
+/// ```
+/// let passthrough_layer = new_passthroughfs_layer("/path/to/layer")
+/// ```
+fn new_passthroughfs_layer(rootdir: &str) -> Result<BoxedLayer> {
+    let config = passthrough::Config {
+        root_dir: String::from(rootdir),
+        xattr: true,
+        do_import: true,
+        ..Default::default()
+    };
+    let fs = Box::new(PassthroughFs::<()>::new(config)?);
+    fs.import()
+        .with_context(|| format!("Failed to create the passthrough layer: {}", rootdir))?;
+    Ok(fs as BoxedLayer)
+}
+
+/// Ensure a destination folder is created
+fn ensure_folder_created(output_folder: &Path) -> Result<()> {
+    fs::create_dir(output_folder).with_context(|| {
+        format!(
+            "Failed to ensure folder creation: {}",
+            output_folder.to_string_lossy()
+        )
+    })
+}
+
+/// Merges all the layers into a single folder for further manipulation
+/// It works by instantiating an overlay fs via FUSE then copying the files to the desired target
+/// # Usage
+/// ```
+/// merge_layer(vec!["source/layer_1", "source/layer_2"], "/tmp/fused_layers")
+/// ```
+pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path) -> Result<()> {
+    // Stack all lower layers
+    let mut lower_layers = Vec::new();
+    for lower in blob_paths {
+        lower_layers.push(Arc::new(new_passthroughfs_layer(&lower.to_string_lossy())?));
+    }
+
+    let mountpoint = Path::new("/tmp/cloudlet_internal");
+    let fs_name = "cloudlet_overlay";
+
+    ensure_folder_created(mountpoint)?;
+    ensure_folder_created(output_folder)?;
+
+    // Setup the overlay fs config
+    let config = Config {
+        work: "/work".into(),
+        mountpoint: output_folder.to_string_lossy().into(),
+        do_import: true,
+        ..Default::default()
+    };
+
+    let fs = OverlayFs::new(None, lower_layers, config)
+        .with_context(|| "Failed to construct the Overlay fs struct !".to_string())?;
+    fs.import()
+        .with_context(|| "Failed to initialize the overlay fs".to_string())?;
+
+    // Enable a fuse session to make the fs available
+    let mut se = FuseSession::new(mountpoint, fs_name, "", true)
+        .with_context(|| "Failed to construct the Fuse session")?;
+    se.set_allow_other(false);
+    se.mount()
+        .with_context(|| "Failed to mount the overlay fs".to_string())?;
+
+    // Fuse session
+    let mut server = FuseServer {
+        server: Arc::new(Server::new(Arc::new(fs))),
+        ch: se
+            .new_channel()
+            .with_context(|| "Failed to create a new channel".to_string())?,
+    };
+
+    let handle = thread::spawn(move || {
+        let _ = server.svc_loop();
+    });
+
+    println!("copy starting !");
+    //So now we need to copy the files
+    dircpy::copy_dir(mountpoint, output_folder).with_context(|| {
+        format!(
+            "Failed to copy directories into the output folder: {}",
+            output_folder.to_string_lossy()
+        )
+    })?;
+    println!("copy finished");
+
+    // Unmount sessions so it can be re-used in later executions of the program
+    se.wake()
+        .with_context(|| "Failed to exit the fuse session".to_string())?;
+    se.umount()
+        .with_context(|| "Failed to unmount the fuse session".to_string())?;
+
+    let _ = handle.join();
+    Ok(())
+}
+
+impl FuseServer {
+    /// Run a loop to execute requests from the FUSE session
+    ///
+    pub fn svc_loop(&mut self) -> Result<()> {
+        println!("entering server loop");
+        loop {
+            let value = self
+                .ch
+                .get_request()
+                .with_context(|| "Failed to get message from fuse session".to_string())?;
+
+            if value.is_none() {
+                println!("fuse server exits");
+                break;
+            }
+
+            // Technically the unwrap is safe
+            let (reader, writer) = value.unwrap();
+
+            if let Err(e) = self
+                .server
+                .handle_message(reader, writer.into(), None, None)
+            {
+                match e {
+                    fuse_backend_rs::Error::EncodeMessage(_ebadf) => {
+                        break;
+                    }
+                    _ => {
+                        print!("Handling fuse message failed");
+                        continue;
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+}