feat: add custom k3s image (#55)

* wip: add custom k3s image

* wip: be more sane

* note

* not default, publish on main

* workflow diff

* multi-version publish

* matrix time

* fix matrix

* test bettererer

* vvvvvvvvvvvvvvvvvvvvvvvvvv

* fixy wixy

* buildx, naming vars

.github/workflows/build-test.yaml

@@ -0,0 +1,44 @@
+name: Test UDS Capability
+
+on:
+  pull_request:
+    paths-ignore:
+      - "**.md"
+      - "docs/**"
+      - "CODEOWNERS"
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  test-clean-install:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        image: ["ghcr.io/defenseunicorns/oss/uds-k3d-k3s"]
+        version: ["v1.27.11-k3s1", "v1.28.7-k3s1", "v1.29.2-k3s1"]
+        # Test the default image as well
+        include:
+          - image: "rancher/k3s"
+            version: "v1.27.4-k3s1"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup UDS
+        if: always()
+        uses: defenseunicorns/uds-common/.github/actions/[email protected]
+        with:
+          username: ${{secrets.IRON_BANK_ROBOT_USERNAME}}
+          password: ${{secrets.IRON_BANK_ROBOT_PASSWORD}}
+
+      - name: Build the custom k3s image
+        if: ${{matrix.image}} != "rancher/k3s"
+        run: uds run build-image --set VERSION=${{matrix.version}}
+
+      - name: Create and deploy the uds-k3d package
+        run: uds run --set IMAGE_NAME=${{matrix.image}} --set VERSION=${{matrix.version}}
+
+      - name: Validate uds-k3d package
+        run: uds run validate

.github/workflows/publish-image.yaml

@@ -0,0 +1,40 @@
+name: Publish k3s image
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - docker/**
+      - .github/workflows/publish-image.yaml
+
+jobs:
+  publish-k3s-image:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: ["v1.27.11-k3s1", "v1.28.7-k3s1", "v1.29.2-k3s1"]
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup UDS
+        if: always()
+        uses: defenseunicorns/uds-common/.github/actions/[email protected]
+        with:
+          username: ${{secrets.IRON_BANK_ROBOT_USERNAME}}
+          password: ${{secrets.IRON_BANK_ROBOT_PASSWORD}}
+
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: dummy
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish the custom k3s image
+        run: uds run publish-image --set VERSION=${{matrix.version}}

.github/workflows/tag-and-release.yml → .github/workflows/tag-and-release.yaml

@@ -34,7 +34,10 @@ jobs:
 
       - name: Setup UDS
         if: always()
-        uses: defenseunicorns/uds-common/.github/actions/[email protected]
+        uses: defenseunicorns/uds-common/.github/actions/[email protected]
+        with:
+          username: ${{secrets.IRON_BANK_ROBOT_USERNAME}}
+          password: ${{secrets.IRON_BANK_ROBOT_PASSWORD}}
 
       - name: Login to GHCR
         uses: docker/login-action@v3

docker/Dockerfile

@@ -0,0 +1,5 @@
+ARG K3S_TAG="v1.28.7-k3s1"
+
+FROM rancher/k3s:$K3S_TAG as k3s
+
+COPY config.toml.tmpl /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl

docker/config.toml.tmpl

@@ -0,0 +1,27 @@
+# This is a custom configuration that has a specific registry cert config removed to prevent zarf init issues.
+version = 2
+
+[plugins."io.containerd.internal.v1.opt"]
+  path = "/var/lib/rancher/k3s/agent/containerd"
+
+[plugins."io.containerd.grpc.v1.cri"]
+  stream_server_address = "127.0.0.1"
+  stream_server_port = "10010"
+  enable_selinux = false
+  enable_unprivileged_ports = true
+  enable_unprivileged_icmp = true
+  sandbox_image = "rancher/mirrored-pause:3.6"
+
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  snapshotter = "overlayfs"
+  disable_snapshot_annotations = true
+
+[plugins."io.containerd.grpc.v1.cri".cni]
+  bin_dir = "/bin"
+  conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d"
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+  SystemdCgroup = false

tasks.yaml

@@ -1,3 +1,9 @@
+variables:
+  - name: VERSION
+    default: "v1.28.7-k3s1"
+  - name: IMAGE_NAME
+    default: "ghcr.io/defenseunicorns/oss/uds-k3d-k3s"
+
 tasks:
   - name: default
     description: "Build and deploy uds-k3d"
@@ -6,7 +12,7 @@ tasks:
         cmd: "uds zarf package create --confirm"
 
       - description: "Deploy UDS K3d package"
-        cmd: "uds zarf package deploy zarf-package-uds-k3d-*.tar.zst --confirm"
+        cmd: "uds zarf package deploy zarf-package-uds-k3d-*.tar.zst --confirm --set K3D_IMAGE=${IMAGE_NAME}:${VERSION}"
 
   - name: validate
     actions:
@@ -27,3 +33,21 @@ tasks:
             echo "CoreDNS patch failed, foo.uds.dev is resolving to 127.0.0.1"
             exit 1
           fi
+      - description: Validate zarf init
+        cmd: |
+          set -e
+          # uds zarf tools download-init does not work in 0.9.4 - https://github.com/defenseunicorns/uds-cli/issues/517
+          uds zarf package pull oci://ghcr.io/defenseunicorns/packages/init:v$(uds zarf version)
+          mv zarf-init-amd64-v$(uds zarf version).tar.zst zarf-init-amd64-$(uds zarf version).tar.zst
+          # Test zarf init due to containerd issue - https://github.com/defenseunicorns/zarf/issues/592
+          uds zarf init --confirm
+
+  - name: build-image
+    actions:
+      - description: Build the custom k3s image
+        cmd: docker build -t ${IMAGE_NAME}:${VERSION} --build-arg K3S_TAG=${VERSION} docker/
+
+  - name: publish-image 
+    actions:
+      - description: Publish the custom k3s image
+        cmd: docker buildx build --push --platform linux/arm64/v8,linux/amd64 --tag ${IMAGE_NAME}:${VERSION} docker