Feat: add initramfs implementation for vmm (#34)

* feat(vmm): implemented automatic generation of rootfs with initramfs

Signed-off-by: Muriel Paraire <[email protected]>

* feat: image generation based off language

Signed-off-by: Muriel Paraire <[email protected]>

* feat(vmm): implemented automatic generation of rootfs with initramfs

Signed-off-by: Muriel Paraire <[email protected]>

* fix(vmm): fix logging & language order

Signed-off-by: Muriel Paraire <[email protected]>

* feat(vmm): one image per language

Signed-off-by: Muriel Paraire <[email protected]>

* feat(vmm): implemented initramfs

Signed-off-by: Muriel Paraire <[email protected]>

* fix(vmm): code cleanup

Signed-off-by: Muriel Paraire <[email protected]>

* fix(vmm): code cleanup

Signed-off-by: Muriel Paraire <[email protected]>

* fix(vmm): code cleanup

Signed-off-by: Muriel Paraire <[email protected]>

* fix: rust export for cargo agent and increase MMIO_GAP_END

Signed-off-by: Mauran <[email protected]>

* chore: lint

Signed-off-by: Mauran <[email protected]>

* fix: add back tracing

Signed-off-by: Mauran <[email protected]>

---------

Signed-off-by: Muriel Paraire <[email protected]>
Signed-off-by: Mauran <[email protected]>
Co-authored-by: Mauran <[email protected]>

Justfile

@@ -29,6 +29,11 @@ build-agent args = "":
     -w /volume \
     -t clux/muslrust \
     cargo build --release --bin agent {{args}}
+
+build-musl-agent args = "":
+  #!/bin/bash
+  rustup target add x86_64-unknown-linux-musl
+  cargo build --release --bin agent --target=x86_64-unknown-linux-musl
 
 build-rootfs mode = "dev":
   #!/bin/bash

proto/vmm.proto

@@ -3,9 +3,9 @@ syntax = "proto3";
 package vmmorchestrator;
 
 enum Language {
-  PYTHON = 0;
-  NODE = 1;
-  RUST = 2;
+  RUST = 0;
+  PYTHON = 1;
+  NODE = 2;
 }
 
 enum LogLevel {

src/api/src/service.rs

@@ -22,9 +22,9 @@ pub async fn run(req_body: web::Json<CloudletDtoRequest>) -> impl Responder {
         workload_name: req.workload_name,
         code: req.code,
         language: match req.language {
-            Language::PYTHON => 0,
-            Language::NODE => 1,
-            Language::RUST => 2,
+            Language::RUST => 0,
+            Language::PYTHON => 1,
+            Language::NODE => 2,
         },
         log_level: req.log_level as i32,
     };

src/fs-gen/resources/initfile

@@ -1,21 +1,19 @@
 #! /bin/sh
 #
-# Cloudlet initramfs generation
+# /init executable file in the initramfs
 #
 mount -t devtmpfs dev /dev
 mount -t proc proc /proc
 mount -t sysfs sysfs /sys
 
 ip link set up dev lo
 
-slattach -L /dev/ttyS1&
+export CARGO_HOME='/usr/local/cargo'
+export RUSTUP_HOME='/usr/local/rustup'
+export RUST_VERSION='1.77.2'
 
-while ! ifconfig sl0 &> /dev/null; do
-    sleep 1
-done
-
-ifconfig sl0 172.30.0.11 netmask 255.255.0.0 up
+export PATH=$CARGO_HOME/bin:$PATH
 
 /agent
 
-reboot
+reboot

src/vmm/src/core/vmm.rs

@@ -29,7 +29,7 @@ use super::irq_allocator::IrqAllocator;
 use super::slip_pty::SlipPty;
 
 #[cfg(target_arch = "x86_64")]
-pub(crate) const MMIO_GAP_END: u64 = 1 << 32;
+pub(crate) const MMIO_GAP_END: u64 = 1 << 34;
 /// Size of the MMIO gap.
 #[cfg(target_arch = "x86_64")]
 pub(crate) const MMIO_GAP_SIZE: u64 = 768 << 20;

src/vmm/src/grpc/server.rs

@@ -1,10 +1,13 @@
-use self::vmmorchestrator::{vmm_service_server::VmmService as VmmServiceTrait, RunVmmRequest};
+use self::vmmorchestrator::{
+    vmm_service_server::VmmService as VmmServiceTrait, Language, RunVmmRequest,
+};
 use crate::grpc::client::agent::ExecuteRequest;
 use crate::VmmErrors;
 use crate::{core::vmm::VMM, grpc::client::WorkloadClient};
 use std::time::Duration;
 use std::{
     convert::From,
+    env::current_dir,
     net::Ipv4Addr,
     path::{Path, PathBuf},
     process::{Command, Stdio},
@@ -50,10 +53,21 @@ impl VmmServiceTrait for VmmService {
         const HOST_NETMASK: Ipv4Addr = Ipv4Addr::new(255, 255, 0, 0);
         const GUEST_IP: Ipv4Addr = Ipv4Addr::new(172, 29, 0, 2);
 
+        // get current directory
+        let mut curr_dir =
+            current_dir().expect("Need to be able to access current directory path.");
+
+        // define kernel path
+        let mut kernel_entire_path = curr_dir.as_os_str().to_owned();
+        kernel_entire_path
+            .push("/tools/kernel/linux-cloud-hypervisor/arch/x86/boot/compressed/vmlinux.bin");
+
         // Check if the kernel is on the system, else build it
-        if !Path::new("./tools/kernel/linux-cloud-hypervisor/arch/x86/boot/compressed/vmlinux.bin")
-            .exists()
-        {
+        let kernel_exists = Path::new(&kernel_entire_path)
+            .try_exists()
+            .unwrap_or_else(|_| panic!("Could not access folder {:?}", &kernel_entire_path));
+
+        if !kernel_exists {
             info!("Kernel not found, building kernel");
             // Execute the script using sh and capture output and error streams
             let output = Command::new("sh")
@@ -64,23 +78,89 @@ impl VmmServiceTrait for VmmService {
                 .expect("Failed to execute the kernel build script");
 
             // Print output and error streams
-            error!("Script output: {}", String::from_utf8_lossy(&output.stdout));
+            info!("Script output: {}", String::from_utf8_lossy(&output.stdout));
             error!("Script errors: {}", String::from_utf8_lossy(&output.stderr));
         };
+        let kernel_path = Path::new(&kernel_entire_path);
+
+        // define initramfs file placement
+        let mut initramfs_entire_file_path = curr_dir.as_os_str().to_owned();
+        initramfs_entire_file_path.push("/tools/rootfs/");
+
+        // get request with the language
+        let vmm_request = request.into_inner();
+        let language: Language =
+            Language::from_i32(vmm_request.language).expect("Unknown language");
 
-        let kernel_path = &Path::new(
-            "./tools/kernel/linux-cloud-hypervisor/arch/x86/boot/compressed/vmlinux.bin",
-        );
-        let mut initramfs_path: PathBuf = PathBuf::new();
+        let image = match language {
+            Language::Rust => {
+                initramfs_entire_file_path.push("rust.img");
+                "rust:alpine"
+            }
+            Language::Python => {
+                initramfs_entire_file_path.push("python.img");
+                "python:alpine"
+            }
+            Language::Node => {
+                initramfs_entire_file_path.push("node.img");
+                "node:alpine"
+            }
+        };
 
-        // Todo - Check if the initramfs for the specified language is on the system, else build it
-        initramfs_path.push("./tools/rootfs/initramfs.img");
+        let rootfs_exists = Path::new(&initramfs_entire_file_path)
+            .try_exists()
+            .unwrap_or_else(|_| {
+                panic!("Could not access folder {:?}", &initramfs_entire_file_path)
+            });
+        if !rootfs_exists {
+            // check if agent binary exists
+            let agent_file_name = curr_dir.as_mut_os_string();
+            agent_file_name.push("/target/x86_64-unknown-linux-musl/release/agent");
+
+            // if agent hasn't been build, build it
+            let agent_exists = Path::new(&agent_file_name)
+                .try_exists()
+                .unwrap_or_else(|_| panic!("Could not access folder {:?}", &agent_file_name));
+            if !agent_exists {
+                //build agent
+                info!("Building agent binary");
+                // Execute the script using sh and capture output and error streams
+                let output = Command::new("just")
+                    .arg("build-musl-agent")
+                    .stdout(Stdio::piped())
+                    .stderr(Stdio::piped())
+                    .output()
+                    .expect("Failed to execute the just build script for the agent");
+
+                // Print output and error streams
+                info!("Script output: {}", String::from_utf8_lossy(&output.stdout));
+                error!("Script errors: {}", String::from_utf8_lossy(&output.stderr));
+                info!("Agent binary successfully built.")
+            }
+
+            info!("Building initramfs");
+            // Execute the script using sh and capture output and error streams
+            let output = Command::new("sh")
+                .arg("./tools/rootfs/mkrootfs.sh")
+                .arg(image)
+                .arg(&agent_file_name)
+                .arg(&initramfs_entire_file_path)
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped())
+                .output()
+                .expect("Failed to execute the initramfs build script");
+
+            // Print output and error streams
+            info!("Script output: {}", String::from_utf8_lossy(&output.stdout));
+            error!("Script errors: {}", String::from_utf8_lossy(&output.stderr));
+            info!("Initramfs successfully built.")
+        }
+        let initramfs_path = PathBuf::from(&initramfs_entire_file_path);
 
-        // // Create a new VMM
         let mut vmm = VMM::new(HOST_IP, HOST_NETMASK, GUEST_IP).map_err(VmmErrors::VmmNew)?;
 
         // Configure the VMM parameters might need to be calculated rather than hardcoded
-        vmm.configure(1, 512, kernel_path, &Some(initramfs_path))
+        vmm.configure(1, 4000, kernel_path, &Some(initramfs_path))
             .map_err(VmmErrors::VmmConfigure)?;
 
         // Run the VMM in a separate task
@@ -102,13 +182,12 @@ impl VmmServiceTrait for VmmService {
         .unwrap();
 
         // Send the grpc request to start the agent
-        let vmm_request = request.into_inner();
         let agent_request = ExecuteRequest {
             workload_name: vmm_request.workload_name,
             language: match vmm_request.language {
-                0 => "python".to_string(),
-                1 => "node".to_string(),
-                2 => "rust".to_string(),
+                0 => "rust".to_string(),
+                1 => "python".to_string(),
+                2 => "node".to_string(),
                 _ => unreachable!("Invalid language"),
             },
             action: 2, // Prepare and run

src/vmm/src/main.rs

@@ -26,6 +26,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // check if the args is grpc or command
     match args.command {
         Commands::Grpc => {
+            tracing_subscriber::fmt().init();
             Server::builder()
                 .add_service(vmmorchestrator::vmm_service_server::VmmServiceServer::new(
                     vmm_service,
@@ -37,6 +38,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             tracing_subscriber::fmt()
                 .with_max_level(cli_args.convert_log_to_tracing())
                 .init();
+
             // Create a new VMM
             let mut vmm = VMM::new(
                 cli_args.iface_host_addr,

tools/kernel/linux-config-x86_64

@@ -1475,6 +1475,8 @@ CONFIG_VIRTIO_NET=y
 # CONFIG_PPP is not set
 CONFIG_SLIP=y
 CONFIG_SLIP_COMPRESSED=y
+CONFIG_SLIP_SMART=n
+CONFIG_SLIP_MODE_SLIP6=n
 
 #
 # Host-side USB support is needed for USB Network Adapter support

tools/kernel/mkkernel.sh

@@ -1,13 +1,13 @@
 #!/usr/bin/bash
 
 LINUX_REPO=linux-cloud-hypervisor
+cd ./tools/kernel/
 
 if [ ! -d $LINUX_REPO ]
 then
     git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "ch-6.2" $LINUX_REPO
 fi
 
-pushd $LINUX_REPO
+cd $LINUX_REPO
 cp ../linux-config-x86_64 .config
 KCFLAGS="-Wa,-mx86-used-note=no" make bzImage -j `nproc`
-popd

tools/rootfs/mkrootfs.sh

@@ -1,43 +1,17 @@
 #!/usr/bin/bash
-
-set -e
-
-if [ ! -d alpine-minirootfs ]
+# test if its already in src folder => fs-gen available
+# launch from src folder at the same level as tools
+if [ -d src ]
 then
-    curl -O https://dl-cdn.alpinelinux.org/alpine/v3.14/releases/x86_64/alpine-minirootfs-3.14.2-x86_64.tar.gz
-
-    mkdir alpine-minirootfs
-    tar xf alpine-minirootfs-3.14.2-x86_64.tar.gz -C alpine-minirootfs
+    cd src
 fi
 
+# augment the open file limit
+ulimit -Sn 8192
 
-pushd alpine-minirootfs
-mkdir -p etc/cloudlet/agent
-cp ../../../target/x86_64-unknown-linux-musl/release/agent agent
-cp ../config.toml etc/cloudlet/agent/config.toml
-
-cat > init <<EOF
-#! /bin/sh
-#
-# /init executable file in the initramfs
-#
-mount -t devtmpfs dev /dev
-mount -t proc proc /proc
-mount -t sysfs sysfs /sys
-
-ip link set up dev lo
-
-ifconfig eth0 172.29.0.2 netmask 255.255.0.0 up
-
-/agent
-
-reboot
-EOF
-
-chmod +x init
-
-find . -print0 |
-    cpio --null --create --verbose --owner root:root --format=newc |
-    xz -9 --format=lzma  > ../initramfs.img
-
-popd
+if [ -d fs-gen ]
+then
+    cargo run --bin fs-gen -- $1 $2 -o $3
+else
+    echo "Module fs-gen not found"
+fi