attempt to make ajinomoto happen

machines/ajinomoto/boot.nix

@@ -0,0 +1,56 @@
+inputs:
+{ config, lib, pkgs, ... }:
+let constants = import ./constants.nix;
+in {
+  boot.initrd.availableKernelModules = [
+    "ehci_pci"
+    "ata_piix"
+    "mpt3sas"
+    "usbhid"
+    "usb_storage"
+    "sd_mod"
+    "e1000e" # initrd networking
+  ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  # legacy boot moment
+  boot.loader.grub.devices = [
+    "/dev/disk/by-id/usb-Generic_Flash_Disk_5AF232B0-0:0"
+  ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.intel.updateMicrocode =
+    lib.mkDefault config.hardware.enableRedistributableFirmware;
+
+  # because we want to be able to decrypt host keys over SSH
+  boot.initrd.network = {
+    enable = true;
+    udhcpc = {
+      enable = true;
+      extraArgs = [
+        "-i"
+        constants.mgmt_if
+        "-x"
+        "hostname:boop"
+        "-b" # background if lease not obtained
+      ];
+    };
+    postCommands = ''
+      ip addr
+    '';
+    ssh = {
+      enable = true;
+      port = 2222; # because we are using a different host key
+      hostKeys = [
+        (pkgs.writeText "ssh_host_rsa_key"
+          (builtins.readFile ./initrd/ssh_host_rsa_key))
+        (pkgs.writeText "ssh_host_ed25519_key"
+          (builtins.readFile ./initrd/ssh_host_ed25519_key))
+      ];
+      authorizedKeys = inputs.self.lib.sshKeyDatabase.users.astrid;
+    };
+  };
+}
+

machines/ajinomoto/bootstrap.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+make_zpool() {
+  zpool create ajinomoto raidz3 \
+    /dev/disk/by-id/wwn-0x5000cca03b9b5f00 \
+    /dev/disk/by-id/wwn-0x5000cca05c5b9eec \
+    /dev/disk/by-id/wwn-0x5000cca05c9fd908 \
+    /dev/disk/by-id/wwn-0x5000cca05ca387b0 \
+    /dev/disk/by-id/wwn-0x5000cca05ca82394 \
+    /dev/disk/by-id/wwn-0x5000cca05caa4814 \
+    /dev/disk/by-id/wwn-0x5000cca05caa6590 \
+    /dev/disk/by-id/wwn-0x5000cca05cab52e0 \
+    /dev/disk/by-id/wwn-0x5000cca05cbc8f9c \
+    /dev/disk/by-id/wwn-0x5000cca05cc7c110 \
+    /dev/disk/by-id/wwn-0x5000cca05ccc91e0 \
+    /dev/disk/by-id/wwn-0x5000cca05cd50144
+}
+
+setup_zpool() {
+  zfs set mountpoint=none compression=lz4 ajinomoto
+  zfs create -o mountpoint=legacy ajinomoto/boot
+  zfs create -o mountpoint=legacy ajinomoto/nix
+  zfs create -o encryption=on -o keyformat=passphrase ajinomoto/enc
+  zfs create -o mountpoint=legacy ajinomoto/enc/var
+  zfs create -o mountpoint=legacy ajinomoto/enc/etc
+  zfs create -o mountpoint=legacy ajinomoto/enc/tmp
+  zfs create -o mountpoint=legacy ajinomoto/enc/home
+}
+
+mount_disks() {
+  mount -t tmpfs -o size=256M,mode=755 rootfs /mnt
+  mount -o x-mount.mkdir /dev/disk/by-uuid/1f75c5dd-1fc2-4300-8946-cb8c327231af /mnt/boot
+  mount -t zfs -o x-mount.mkdir ajinomoto/nix /mnt/nix
+  mount -t zfs -o x-mount.mkdir ajinomoto/enc/etc /mnt/etc
+  mount -t zfs -o x-mount.mkdir ajinomoto/enc/tmp /mnt/tmp
+  mount -t zfs -o x-mount.mkdir ajinomoto/enc/var /mnt/var
+  mount -t zfs -o x-mount.mkdir ajinomoto/enc/home /mnt/home
+}

machines/ajinomoto/configuration.nix

@@ -0,0 +1,35 @@
+inputs:
+{ config, lib, pkgs, modulesPath, ... }:
+with lib; {
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+
+    inputs.self.nixosModules.server
+
+    (import ./boot.nix inputs)
+    ./fs.nix
+  ];
+
+  astral = {
+    users.alia.enable = true;
+    users.astrid.enable = true;
+    virt = {
+      docker.enable = true;
+      libvirt.enable = true;
+    };
+    tailscale.enable = mkForce false;
+    monitoring-node.enable = mkForce false;
+
+    backup.db.enable = false;
+  };
+
+  time.timeZone = "US/Pacific";
+
+  networking = {
+    hostName = "boop";
+    domain = "h.astrid.tech";
+
+    hostId = "681441e3"; # Required for ZFS
+    useDHCP = true;
+  };
+}

machines/ajinomoto/constants.nix

@@ -0,0 +1,6 @@
+{
+  /**
+    Interface on the management port
+   */
+  mgmt_if = "enp3s0";
+}

machines/ajinomoto/fio-file.conf

@@ -0,0 +1,30 @@
+[global]
+bs=256K
+iodepth=64
+direct=1
+ioengine=libaio
+stonewall
+group_reporting
+time_based
+runtime=120
+numjobs=4
+filename=/tmp/fio
+size=1G
+
+[seqread-file]
+rw=read
+
+[seqwrite-file]
+rw=write
+
+[seqrw-file]
+rw=rw
+
+[randread-file]
+rw=randread
+
+[randwrite-file]
+rw=randwrite
+
+[randrw-file]
+rw=randrw

machines/ajinomoto/fio-results.txt

@@ -0,0 +1,193 @@
+seqread-file: (g=0): rw=read, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64
+...
+seqwrite-file: (g=1): rw=write, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64
+...
+seqrw-file: (g=2): rw=rw, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64
+...
+randread-file: (g=3): rw=randread, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64
+...
+randwrite-file: (g=4): rw=randwrite, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64
+...
+randrw-file: (g=5): rw=randrw, bs=(R) 256KiB-256KiB, (W) 256KiB-256KiB, (T) 256KiB-256KiB, ioengine=libaio, iodepth=64
+...
+fio-3.36
+Starting 24 processes
+
+seqread-file: (groupid=0, jobs=4): err= 0: pid=13161: Thu Apr 11 17:34:44 2024
+  read: IOPS=19.3k, BW=4836MiB/s (5071MB/s)(567GiB/120001msec)
+    slat (usec): min=74, max=1250, avg=201.71, stdev=15.64
+    clat (usec): min=5, max=26880, avg=13027.62, stdev=557.58
+     lat (usec): min=198, max=27274, avg=13229.33, stdev=566.13
+    clat percentiles (usec):
+     |  1.00th=[12518],  5.00th=[12649], 10.00th=[12911], 20.00th=[12911],
+     | 30.00th=[12911], 40.00th=[12911], 50.00th=[12911], 60.00th=[12911],
+     | 70.00th=[13042], 80.00th=[13042], 90.00th=[13042], 95.00th=[13173],
+     | 99.00th=[15926], 99.50th=[17957], 99.90th=[18482], 99.95th=[18482],
+     | 99.99th=[18744]
+   bw (  MiB/s): min= 3984, max= 4949, per=100.00%, avg=4838.63, stdev=42.96, samples=956
+   iops        : min=15936, max=19798, avg=19353.74, stdev=171.84, samples=956
+  lat (usec)   : 10=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%
+  lat (msec)   : 2=0.01%, 4=0.01%, 10=0.01%, 20=99.98%, 50=0.01%
+  cpu          : usr=2.51%, sys=83.72%, ctx=1911136, majf=0, minf=16439
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=2321528,0,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+seqwrite-file: (groupid=1, jobs=4): err= 0: pid=13188: Thu Apr 11 17:34:44 2024
+  write: IOPS=2756, BW=689MiB/s (723MB/s)(80.7GiB/120003msec); 0 zone resets
+    slat (usec): min=74, max=79714, avg=1445.12, stdev=1311.83
+    clat (usec): min=6, max=246057, avg=91425.25, stdev=79459.94
+     lat (usec): min=1698, max=249617, avg=92870.37, stdev=80719.29
+    clat percentiles (msec):
+     |  1.00th=[   12],  5.00th=[   22], 10.00th=[   23], 20.00th=[   24],
+     | 30.00th=[   28], 40.00th=[   37], 50.00th=[   51], 60.00th=[   74],
+     | 70.00th=[  128], 80.00th=[  201], 90.00th=[  228], 95.00th=[  232],
+     | 99.00th=[  234], 99.50th=[  234], 99.90th=[  243], 99.95th=[  245],
+     | 99.99th=[  247]
+   bw (  KiB/s): min=273031, max=3852127, per=99.67%, avg=703275.33, stdev=181621.72, samples=956
+   iops        : min= 1066, max=15044, avg=2746.87, stdev=709.44, samples=956
+  lat (usec)   : 10=0.01%
+  lat (msec)   : 2=0.01%, 4=0.01%, 10=0.28%, 20=2.92%, 50=46.81%
+  lat (msec)   : 100=16.13%, 250=33.85%
+  cpu          : usr=1.73%, sys=7.35%, ctx=460131, majf=0, minf=57
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=0,330751,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+seqrw-file: (groupid=2, jobs=4): err= 0: pid=13202: Thu Apr 11 17:34:44 2024
+  read: IOPS=1949, BW=487MiB/s (511MB/s)(57.1GiB/120001msec)
+    slat (usec): min=35, max=44930, avg=758.35, stdev=1391.88
+    clat (usec): min=5, max=521104, avg=64455.12, stdev=45775.54
+     lat (usec): min=94, max=522066, avg=65213.46, stdev=46457.39
+    clat percentiles (msec):
+     |  1.00th=[   17],  5.00th=[   19], 10.00th=[   21], 20.00th=[   23],
+     | 30.00th=[   26], 40.00th=[   34], 50.00th=[   50], 60.00th=[   75],
+     | 70.00th=[  100], 80.00th=[  111], 90.00th=[  121], 95.00th=[  126],
+     | 99.00th=[  169], 99.50th=[  266], 99.90th=[  384], 99.95th=[  409],
+     | 99.99th=[  468]
+   bw (  KiB/s): min=88253, max=1697597, per=99.84%, avg=498379.43, stdev=97942.96, samples=956
+   iops        : min=  344, max= 6631, avg=1946.39, stdev=382.58, samples=956
+  write: IOPS=1957, BW=489MiB/s (513MB/s)(57.3GiB/120001msec); 0 zone resets
+    slat (usec): min=180, max=34997, avg=1276.27, stdev=1033.40
+    clat (usec): min=6, max=509705, avg=64470.53, stdev=45351.02
+     lat (usec): min=450, max=510228, avg=65746.80, stdev=46031.96
+    clat percentiles (msec):
+     |  1.00th=[   17],  5.00th=[   19], 10.00th=[   21], 20.00th=[   23],
+     | 30.00th=[   26], 40.00th=[   34], 50.00th=[   50], 60.00th=[   75],
+     | 70.00th=[  100], 80.00th=[  112], 90.00th=[  121], 95.00th=[  126],
+     | 99.00th=[  159], 99.50th=[  253], 99.90th=[  372], 99.95th=[  405],
+     | 99.99th=[  447]
+   bw (  KiB/s): min=84623, max=1697792, per=99.83%, avg=500153.26, stdev=97891.30, samples=956
+   iops        : min=  329, max= 6632, avg=1953.30, stdev=382.38, samples=956
+  lat (usec)   : 10=0.01%, 20=0.01%, 100=0.01%, 250=0.01%, 500=0.01%
+  lat (usec)   : 750=0.01%, 1000=0.01%
+  lat (msec)   : 2=0.01%, 4=0.01%, 10=0.01%, 20=9.57%, 50=40.62%
+  lat (msec)   : 100=20.22%, 250=29.02%, 500=0.54%, 750=0.01%
+  cpu          : usr=1.51%, sys=12.15%, ctx=460029, majf=0, minf=60
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=233998,234857,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+randread-file: (groupid=3, jobs=4): err= 0: pid=13214: Thu Apr 11 17:34:44 2024
+  read: IOPS=9149, BW=2287MiB/s (2398MB/s)(268GiB/120001msec)
+    slat (usec): min=39, max=1595.1k, avg=430.26, stdev=3632.47
+    clat (usec): min=6, max=5791.4k, avg=27543.95, stdev=79393.88
+     lat (usec): min=152, max=5815.3k, avg=27974.22, stdev=80514.22
+    clat percentiles (msec):
+     |  1.00th=[   16],  5.00th=[   21], 10.00th=[   22], 20.00th=[   22],
+     | 30.00th=[   22], 40.00th=[   22], 50.00th=[   22], 60.00th=[   22],
+     | 70.00th=[   22], 80.00th=[   23], 90.00th=[   23], 95.00th=[   23],
+     | 99.00th=[  155], 99.50th=[  393], 99.90th=[ 1150], 99.95th=[ 1519],
+     | 99.99th=[ 3339]
+   bw (  MiB/s): min=    5, max= 4065, per=100.00%, avg=2322.27, stdev=285.55, samples=941
+   iops        : min=   18, max=16259, avg=9287.83, stdev=1142.22, samples=941
+  lat (usec)   : 10=0.01%, 20=0.01%, 250=0.01%, 500=0.01%, 750=0.01%
+  lat (usec)   : 1000=0.01%
+  lat (msec)   : 2=0.01%, 4=0.01%, 10=0.09%, 20=3.09%, 50=95.11%
+  lat (msec)   : 100=0.45%, 250=0.54%, 500=0.31%, 750=0.22%, 1000=0.07%
+  lat (msec)   : 2000=0.09%, >=2000=0.03%
+  cpu          : usr=1.59%, sys=89.96%, ctx=15896, majf=0, minf=16433
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=1097892,0,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+randwrite-file: (groupid=4, jobs=4): err= 0: pid=13229: Thu Apr 11 17:34:44 2024
+  write: IOPS=2480, BW=620MiB/s (650MB/s)(72.7GiB/120003msec); 0 zone resets
+    slat (usec): min=89, max=25481, avg=1605.87, stdev=1833.22
+    clat (usec): min=6, max=453700, avg=101568.87, stdev=113825.57
+     lat (msec): min=2, max=460, avg=103.17, stdev=115.63
+    clat percentiles (msec):
+     |  1.00th=[   21],  5.00th=[   22], 10.00th=[   23], 20.00th=[   25],
+     | 30.00th=[   28], 40.00th=[   36], 50.00th=[   47], 60.00th=[   66],
+     | 70.00th=[   96], 80.00th=[  169], 90.00th=[  305], 95.00th=[  388],
+     | 99.00th=[  435], 99.50th=[  439], 99.90th=[  447], 99.95th=[  447],
+     | 99.99th=[  447]
+   bw (  KiB/s): min=143872, max=2924272, per=99.93%, avg=634679.49, stdev=180457.86, samples=956
+   iops        : min=  562, max=11422, avg=2478.98, stdev=704.89, samples=956
+  lat (usec)   : 10=0.01%
+  lat (msec)   : 4=0.01%, 10=0.01%, 20=0.93%, 50=51.30%, 100=18.54%
+  lat (msec)   : 250=15.65%, 500=13.58%
+  cpu          : usr=1.61%, sys=6.89%, ctx=387992, majf=0, minf=49
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=0,297721,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+randrw-file: (groupid=5, jobs=4): err= 0: pid=13262: Thu Apr 11 17:34:44 2024
+  read: IOPS=1655, BW=414MiB/s (434MB/s)(48.5GiB/120005msec)
+    slat (usec): min=38, max=51516, avg=1060.30, stdev=2907.67
+    clat (usec): min=6, max=375406, avg=75953.69, stdev=73535.15
+     lat (usec): min=1061, max=375499, avg=77013.99, stdev=74754.14
+    clat percentiles (msec):
+     |  1.00th=[   17],  5.00th=[   19], 10.00th=[   21], 20.00th=[   22],
+     | 30.00th=[   24], 40.00th=[   29], 50.00th=[   37], 60.00th=[   55],
+     | 70.00th=[   88], 80.00th=[  138], 90.00th=[  203], 95.00th=[  236],
+     | 99.00th=[  284], 99.50th=[  300], 99.90th=[  330], 99.95th=[  338],
+     | 99.99th=[  359]
+   bw (  KiB/s): min=54328, max=1715400, per=100.00%, avg=424333.50, stdev=107301.22, samples=956
+   iops        : min=  212, max= 6700, avg=1657.07, stdev=419.19, samples=956
+  write: IOPS=1654, BW=414MiB/s (434MB/s)(48.5GiB/120005msec); 0 zone resets
+    slat (usec): min=184, max=41765, avg=1343.90, stdev=1508.67
+    clat (usec): min=6, max=372928, avg=76181.12, stdev=73944.72
+     lat (usec): min=1669, max=376598, avg=77525.01, stdev=75003.38
+    clat percentiles (msec):
+     |  1.00th=[   17],  5.00th=[   19], 10.00th=[   20], 20.00th=[   22],
+     | 30.00th=[   24], 40.00th=[   29], 50.00th=[   37], 60.00th=[   55],
+     | 70.00th=[   88], 80.00th=[  138], 90.00th=[  205], 95.00th=[  239],
+     | 99.00th=[  284], 99.50th=[  300], 99.90th=[  330], 99.95th=[  338],
+     | 99.99th=[  355]
+   bw (  KiB/s): min=56879, max=1689379, per=100.00%, avg=423993.52, stdev=107211.87, samples=956
+   iops        : min=  222, max= 6599, avg=1655.72, stdev=418.85, samples=956
+  lat (usec)   : 10=0.01%
+  lat (msec)   : 2=0.01%, 4=0.01%, 10=0.01%, 20=10.35%, 50=47.47%
+  lat (msec)   : 100=15.01%, 250=23.64%, 500=3.52%
+  cpu          : usr=1.32%, sys=19.51%, ctx=404351, majf=0, minf=53
+  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
+     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
+     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
+     issued rwts: total=198652,198490,0,0 short=0,0,0,0 dropped=0,0,0,0
+     latency   : target=0, window=0, percentile=100.00%, depth=64
+
+Run status group 0 (all jobs):
+   READ: bw=4836MiB/s (5071MB/s), 4836MiB/s-4836MiB/s (5071MB/s-5071MB/s), io=567GiB (609GB), run=120001-120001msec
+
+Run status group 1 (all jobs):
+  WRITE: bw=689MiB/s (723MB/s), 689MiB/s-689MiB/s (723MB/s-723MB/s), io=80.7GiB (86.7GB), run=120003-120003msec
+
+Run status group 2 (all jobs):
+   READ: bw=487MiB/s (511MB/s), 487MiB/s-487MiB/s (511MB/s-511MB/s), io=57.1GiB (61.3GB), run=120001-120001msec
+  WRITE: bw=489MiB/s (513MB/s), 489MiB/s-489MiB/s (513MB/s-513MB/s), io=57.3GiB (61.6GB), run=120001-120001msec
+
+Run status group 3 (all jobs):
+   READ: bw=2287MiB/s (2398MB/s), 2287MiB/s-2287MiB/s (2398MB/s-2398MB/s), io=268GiB (288GB), run=120001-120001msec
+
+Run status group 4 (all jobs):
+  WRITE: bw=620MiB/s (650MB/s), 620MiB/s-620MiB/s (650MB/s-650MB/s), io=72.7GiB (78.0GB), run=120003-120003msec
+
+Run status group 5 (all jobs):
+   READ: bw=414MiB/s (434MB/s), 414MiB/s-414MiB/s (434MB/s-434MB/s), io=48.5GiB (52.1GB), run=120005-120005msec
+  WRITE: bw=414MiB/s (434MB/s), 414MiB/s-414MiB/s (434MB/s-434MB/s), io=48.5GiB (52.0GB), run=120005-120005msec