Add NixOS image for bare-metal Kata

docs/docs/features-limitations.md

@@ -34,3 +34,10 @@ Currently, this requires inspecting the iptables rules on startup or terminating
 The Contrast Coordinator is a singleton and can't be scaled to more than one instance.
 When this instance's pod is restarted, for example for node maintenance, it needs to be recovered manually.
 In a future release, we plan to support distributed Coordinator instances that can recover automatically.
+
+## Overriding Kata configuration
+
+Kata Containers supports [overriding certain configuration values via Kubernetes annotations](https://github.com/kata-containers/kata-containers/blob/b4da4b5e3b9b21048af9333b071235a57a3e9493/docs/how-to/how-to-set-sandbox-config-kata.md).
+
+It needs to be noted that setting these values is unsupported, and doing so may lead to unexpected
+behaviour, as Contrast isn't tested against all possible configuration combinations.

nodeinstaller/internal/constants/configuration-qemu-tdx.toml

@@ -18,6 +18,9 @@ cpu_features="-vmx-rdseed-exit,pmu=off"
 default_vcpus = 1
 default_maxvcpus = 0
 default_bridges = 1
+# On TDX, when lowering this, the patch:
+# packages/by-name/qemu-tdx-static/0004-hw-x86-load-initrd-to-static-address.patch
+# needs to be updated accordingly.
 default_memory = 2048
 default_maxmemory = 0
 disable_block_device_use = false

nodeinstaller/internal/constants/constants.go

@@ -64,37 +64,36 @@ func KataRuntimeConfig(baseDir string, platform platforms.Platform, qemuExtraKer
 		config.Hypervisor["qemu"]["path"] = filepath.Join(baseDir, "tdx", "bin", "qemu-system-x86_64")
 		config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "tdx", "share", "OVMF.fd")
 		config.Hypervisor["qemu"]["image"] = filepath.Join(baseDir, "share", "kata-containers.img")
-		config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel")
 		config.Hypervisor["qemu"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "tdx", "bin", "qemu-system-x86_64")}
 		config.Hypervisor["qemu"]["block_device_aio"] = "threads"
 		config.Hypervisor["qemu"]["shared_fs"] = "none"
-		kernelParams := qemuExtraKernelParams
+		config.Hypervisor["qemu"]["initrd"] = filepath.Join(baseDir, "share", "kata-initrd.zst")
+		config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel")
+		// Replace the kernel params entirely (and don't append) since that's
+		// also what we do when calculating the launch measurement.
+		config.Hypervisor["qemu"]["kernel_params"] = qemuExtraKernelParams
 		if debug {
 			config.Hypervisor["qemu"]["enable_debug"] = true
 		}
-		// Replace the kernel params entirely (and don't append) since that's
-		// also what we do when calculating the launch measurement.
-		config.Hypervisor["qemu"]["kernel_params"] = kernelParams
 	case platforms.K3sQEMUSNP:
 		if err := toml.Unmarshal([]byte(kataBareMetalQEMUSNPBaseConfig), &config); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal kata runtime configuration: %w", err)
 		}
 		config.Hypervisor["qemu"]["path"] = filepath.Join(baseDir, "snp", "bin", "qemu-system-x86_64")
 		config.Hypervisor["qemu"]["firmware"] = filepath.Join(baseDir, "snp", "share", "OVMF.fd")
 		config.Hypervisor["qemu"]["image"] = filepath.Join(baseDir, "share", "kata-containers.img")
-		config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel")
-		delete(config.Hypervisor["qemu"], "initrd")
 		config.Hypervisor["qemu"]["block_device_aio"] = "threads"
 		config.Hypervisor["qemu"]["shared_fs"] = "none"
 		config.Hypervisor["qemu"]["valid_hypervisor_paths"] = []string{filepath.Join(baseDir, "snp", "bin", "qemu-system-x86_64")}
 		config.Hypervisor["qemu"]["rootfs_type"] = "erofs"
-		kernelParams := qemuExtraKernelParams
+		config.Hypervisor["qemu"]["initrd"] = filepath.Join(baseDir, "share", "kata-initrd.zst")
+		config.Hypervisor["qemu"]["kernel"] = filepath.Join(baseDir, "share", "kata-kernel")
+		// Replace the kernel params entirely (and don't append) since that's
+		// also what we do when calculating the launch measurement.
+		config.Hypervisor["qemu"]["kernel_params"] = qemuExtraKernelParams
 		if debug {
 			config.Hypervisor["qemu"]["enable_debug"] = true
 		}
-		// Replace the kernel params entirely (and don't append) since that's
-		// also what we do when calculating the launch measurement.
-		config.Hypervisor["qemu"]["kernel_params"] = kernelParams
 	default:
 		return nil, fmt.Errorf("unsupported platform: %s", platform)
 	}

packages/by-name/OVMF-TDX/package.nix

@@ -9,7 +9,7 @@
   debug ? false,
 }:
 
-edk2.mkDerivation "OvmfPkg/IntelTdx/IntelTdxX64.dsc" rec {
+edk2.mkDerivation "OvmfPkg/IntelTdx/IntelTdxX64.dsc" {
   name = "OVMF-TDX";
 
   buildFlags = lib.optionals debug [ "-D DEBUG_ON_SERIAL_PORT=TRUE" ];

packages/by-name/boot-microvm/package.nix

@@ -0,0 +1,36 @@
+# Copyright 2024 Edgeless Systems GmbH
+# SPDX-License-Identifier: AGPL-3.0-only
+
+{
+  writeShellApplication,
+  qemu,
+  OVMF,
+}:
+
+# Usage example:
+# outPath=$(nix build .#kata.kata-image --print-out-paths); nix run .#boot-microvm -- "${outPath}/bzImage" "${outPath}/initrd" "${outPath}/image-podvm-gpu_1-rc1.raw" "$(nix eval --raw .#kata.kata-image.cmdline)"
+
+writeShellApplication {
+  name = "boot-microvm";
+  runtimeInputs = [ qemu ];
+  text = ''
+    if [ $# -ne 4 ]; then
+      echo "Usage: $0 <kernel> <initrd> <rootfs> <cmdline>";
+      exit 1;
+    fi
+
+    tmpFile=$(mktemp)
+    cp "$3" "$tmpFile"
+
+    qemu-system-x86_64 \
+      -enable-kvm \
+      -m 3G \
+      -nographic \
+      -drive if=pflash,format=raw,readonly=on,file=${OVMF.firmware} \
+      -drive if=pflash,format=raw,readonly=on,file=${OVMF.variables} \
+      -kernel "$1" \
+      -initrd "$2" \
+      -append "$4" \
+      -drive "if=virtio,format=raw,file=$tmpFile" 
+  '';
+}

packages/by-name/buildVerityMicroVM/package.nix

@@ -0,0 +1,58 @@
+# Copyright 2024 Edgeless Systems GmbH
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# Builds a micro VM image (i.e. rootfs, kernel and kernel cmdline) from a NixOS
+# configuration. These components can then be booted in a microVM-fashion
+# with QEMU's direct Linux boot feature.
+# See: https://qemu-project.gitlab.io/qemu/system/linuxboot.html
+
+{
+  symlinkJoin,
+  lib,
+}:
+
+nixos-config:
+
+let
+  image = nixos-config.image.overrideAttrs (oldAttrs: {
+    passthru = oldAttrs.passthru // {
+      imageFileName = "${oldAttrs.pname}_${oldAttrs.version}.raw";
+    };
+  });
+in
+
+lib.throwIf
+  (lib.foldlAttrs (
+    acc: _: partConfig:
+    acc || (partConfig.repartConfig.Type == "esp")
+  ) false nixos-config.config.image.repart.partitions)
+  "MicroVM images should not contain an ESP."
+
+  symlinkJoin
+  {
+    pname = "microvm-image";
+    inherit (nixos-config.config.system.image) version;
+
+    paths = [
+      nixos-config.config.system.build.kernel
+      nixos-config.config.system.build.initialRamdisk
+      image
+    ];
+
+    passthru =
+      let
+        roothash = builtins.head (
+          lib.map (e: e.roothash) (builtins.fromJSON (builtins.readFile "${image}/repart-output.json"))
+        );
+      in
+      {
+        cmdline = lib.concatStringsSep " " (
+          nixos-config.config.boot.kernelParams
+          ++ [
+            "init=${nixos-config.config.system.build.toplevel}/init"
+            "roothash=${roothash}"
+          ]
+        );
+        inherit (image) imageFileName;
+      };
+  }

packages/by-name/image-podvm/package.nix

@@ -15,5 +15,6 @@ buildVerityUKI (mkNixosConfig {
     debug.enable = withDebug;
     gpu.enable = withGPU;
     azure.enable = withCSP == "azure";
+    peerpods.enable = true;
   };
 })

packages/by-name/kata/contrast-node-installer-image/package.nix

@@ -47,6 +47,10 @@ let
               url = "file:///opt/edgeless/share/kata-kernel";
               path = "/opt/edgeless/@@runtimeName@@/share/kata-kernel";
             }
+            {
+              url = "file:///opt/edgeless/share/kata-initrd.zst";
+              path = "/opt/edgeless/@@runtimeName@@/share/kata-initrd.zst";
+            }
             {
               url = "file:///opt/edgeless/snp/bin/qemu-system-x86_64";
               path = "/opt/edgeless/@@runtimeName@@/snp/bin/qemu-system-x86_64";
@@ -106,7 +110,7 @@ let
             }
           ];
           inherit debugRuntime;
-          qemuExtraKernelParams = kata.snp-launch-digest.dmVerityArgs;
+          qemuExtraKernelParams = kata.kata-image.cmdline;
         };
         destination = "/config/contrast-node-install.json";
       }
@@ -116,13 +120,17 @@ let
   kata-container-img = ociLayerTar {
     files = [
       {
-        source = kata.kata-image;
+        source = "${kata.kata-image}/${kata.kata-image.imageFileName}";
         destination = "/opt/edgeless/share/kata-containers.img";
       }
       {
-        source = "${kata.kata-kernel-uvm}/bzImage";
+        source = "${kata.kata-image}/bzImage";
         destination = "/opt/edgeless/share/kata-kernel";
       }
+      {
+        source = "${kata.kata-image}/initrd.zst";
+        destination = "/opt/edgeless/share/kata-initrd.zst";
+      }
     ];
   };