node: introduce layered OCI image

https://github.com/coreos/fedora-coreos-docs/blob/2a55dc37c49c8bb298aea53b5642bd42d69d3fe7/modules/ROOT/pages/deriving-container.adoc the node/ Dockerfile packages up all the non-sensitive stuff previously in the Ignition config. the idea is to slim control-plane Ignition further, by pushing more of that stuff into cluster workloads. related to #464
samcday · Jun 20, 2024 · 250572c · 250572c
1 parent 182612c
commit 250572c
Show file tree

Hide file tree

Showing 19 changed files with 248 additions and 165 deletions.
diff --git a/.github/workflows/node-image.yml b/.github/workflows/node-image.yml
@@ -0,0 +1,94 @@
+name: node/ Image
+
+on:
+  schedule:
+    - cron: '22 9 * * *'
+  push:
+    branches: [ main ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+  pull_request:
+    branches: [ main ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}-node
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
+        with:
+          cosign-release: 'v2.2.4'
+
+      # Set up BuildKit Docker container builder to be able to build
+      # multi-platform images and export cache
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+        with:
+          context: node/
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Sign the resulting Docker image digest except on PRs.
+      # This will only write to the public Rekor transparency log when the Docker
+      # repository is public to avoid leaking data.  If you would like to publish
+      # transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      - name: Sign the published Docker image
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
+          TAGS: ${{ steps.meta.outputs.tags }}
+          DIGEST: ${{ steps.build-and-push.outputs.digest }}
+        # This step uses the identity token to provision an ephemeral certificate
+        # against the sigstore community Fulcio instance.
+        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
diff --git a/control-plane/config.bu b/control-plane/config.bu
@@ -24,9 +24,6 @@ passwd:
         - systemd-journal
       ssh_authorized_keys: [ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFwawprQXEkGl38Q7T0PNseL0vpoyr4TbATMkEaZJTWQ]
 storage:
-  directories:
-    - path: /etc/haproxy
-    - path: /etc/haproxy/conf.d
   disks:
     - device: /dev/disk/by-id/coreos-boot-disk
       wipe_table: false
@@ -38,10 +35,6 @@ storage:
         - size_mib: 0
           label: var
   files:
-    - path: /etc/containerd/config.toml
-      overwrite: true
-    - path: /etc/kubeadm.yaml
-      mode: 0600
     - path: /etc/ssh/ssh_host_ecdsa_key
       mode: 0600
     - path: /etc/ssh/ssh_host_ed25519_key
@@ -68,43 +61,16 @@ storage:
     - local: tree
 systemd:
   units:
-    - name: containerd.service
-      enabled: true
-    - name: etc-kubernetes.mount
-      enabled: true
-      contents: |
-        [Mount]
-        What=/var/lib/kubernetes
-        Where=/etc/kubernetes
-        Type=none
-        Options=bind
-
-        [Install]
-        WantedBy=multi-user.target
-    - name: init.service
+    - name: rebase-custom.service
       enabled: true
       contents: |
         [Unit]
-        Wants=network-online.target
+        ConditionFirstBoot=true
         After=network-online.target
-        Before=zincati.service
-        ConditionPathExists=!/etc/%N.stamp
-
         [Service]
-        LoadCredential=authkey:/etc/tailscale-auth.key
-        RemainAfterExit=yes
+        After=coreos-ignition-firstboot-complete.service
         Type=oneshot
-        # Install stuff.
-        ExecStart=/usr/bin/rpm-ostree install --apply-live --allow-inactive dmidecode haproxy kubeadm kubectl kubelet man-db man-pages tailscale tcpdump usbutils vim
-
-        # Start stuff just installed.
-        ExecStart=systemctl enable --now haproxy.service kubelet.service tailscaled.service
-
-        # Wait until tailscale is up.
-        ExecStart=bash -c 'until tailscale status --json; do sleep 0.5; done'
-        ExecStart=bash -c 'while [[ "$(tailscale status --json | jq -r .BackendState)" == "NoState" ]]; do sleep 0.5; done'
-        ExecStart=bash -c '[[ "$(tailscale status --json | jq -r .BackendState)" == "NeedsLogin" ]] && tailscale up --auth-key=file:${CREDENTIALS_DIRECTORY}/authkey --accept-dns=false --advertise-routes=172.29.0.1/32 || exit 0'
-        ExecStart=/bin/touch /etc/%N.stamp
-
+        RemainAfterExit=yes
+        ExecStart=rpm-ostree rebase --reboot ostree-unverified-registry:ghcr.io/samcday/home-cluster-node:latest
         [Install]
         WantedBy=multi-user.target
diff --git a/control-plane/ignition-files.enc/tree/etc/kubeadm.conf.d/90-control-plane.yaml b/control-plane/ignition-files.enc/tree/etc/kubeadm.conf.d/90-control-plane.yaml
@@ -0,0 +1,63 @@
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: JoinConfiguration
+controlPlane:
+    certificateKey: ENC[AES256_GCM,data:mGVlFWlaxfBhgTzQF3UKP9QExlvcC0cOHweXPAfYc6v3a+iUiYrQ9ZsoL5Eeb5h4Mac5XrrrQwvffb7mkIIwKg==,iv:QGmR9P0fWfFtYKVgzzYD+rigg9gmr82+WGvPT7qLun8=,tag:oaMcsc6xvNxPPnW3qM3CTw==,type:str]
+nodeRegistration:
+    taints: []
+discovery:
+    bootstrapToken:
+        apiServerEndpoint: 172.29.0.1:6444
+        token: ENC[AES256_GCM,data:iQ1Yszkh5MoWb0CcLhe46XqA0BBSwI0=,iv:HOmkqDz+aDRJ3s3WTi+NzuuojGzm5BpFt5+HZWBGiwQ=,tag:OTas+BeLeOgwIlBPzw1pZg==,type:str]
+        caCertHashes:
+            - sha256:115e5a84bdb260f15defdbd836acf108769cff1e18bc9e2c473eb6f208051c58
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age10c9vvuvfkflc7zypu6zm8dtw0gdn028nlr3gslt35df8vdqrap5q36xav4
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIV05yUmI2WDA3dkd6Nmt5
+            amdwUFU0NHl0MElNOG8zQnMwYzFFekNoZUVVCktpaXUyMGw4Ung0SjZYRDlpQi9F
+            amY1WngrMnZIWXdHanREbWpKbGN0d2sKLS0tIEwzbjM1emE0ZFJKRloxTDZkeFVy
+            Uk9rRndTVmNKS0h5N3E3QkFNbVVrdlEK+SIz9paPmHB7Cfyv080/H4I9fWKEbhyk
+            wyd01z7xNBq29HI3UoOnuGmVXHP8ZCz3bfRKpdv96Th6BZn9sL5ZqQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-06-20T13:48:40Z"
+    mac: ENC[AES256_GCM,data:g+BrOR2vUK+NMIzwLKlfJQZj737aOpd5fDZ330kzLm3BhTumQL+qLC8D2AqYKv30NvLR3qnFEQJ54FNBzeCauXiIwN7IuwAGTLP60lKSxl/Up7zRHoumfuCDw6TwVjOIS2NHqZ13htuSWC9JRKr4ds1iCzcBLkYeUU2yzjIoyiQ=,iv:TSjiSx8nLhQk45ZdmeQO26P00yZIazOKbl6DBJd0KvI=,tag:Ll/3GmactsVwW+3nZV0etg==,type:str]
+    pgp: []
+    encrypted_regex: token|certificateKey
+    version: 3.8.1
+---
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: InitConfiguration
+bootstrapTokens:
+    - token: ENC[AES256_GCM,data:nUO2q0UL0BhNI4wKwKbIO4buRGEnjwo=,iv:QfYQuQEDATIagygqINw32GN2C145SCZiAH5u8104rK8=,tag:VaGflIjLkc2GygGEFDYejA==,type:str]
+      ttl: "0"
+certificateKey: ENC[AES256_GCM,data:LhTFvyPlA/haZ/G24Op1Uy/IL1os8aeuAjqeqxRZv6wUJILm3KUKTAmNBdKzWn6Eo7fVKSZFwkUtju/dWoeh7Q==,iv:vkEwjBh1z+De4aLCWwjlSFBFN39uifXZ804ZfLje5Ro=,tag:JF1D9w/pUtACHCmDQbde2g==,type:str]
+nodeRegistration:
+    kubeletExtraArgs:
+        volume-plugin-dir: /var/lib/kubelet/volume-plugins
+    taints: []
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age10c9vvuvfkflc7zypu6zm8dtw0gdn028nlr3gslt35df8vdqrap5q36xav4
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIV05yUmI2WDA3dkd6Nmt5
+            amdwUFU0NHl0MElNOG8zQnMwYzFFekNoZUVVCktpaXUyMGw4Ung0SjZYRDlpQi9F
+            amY1WngrMnZIWXdHanREbWpKbGN0d2sKLS0tIEwzbjM1emE0ZFJKRloxTDZkeFVy
+            Uk9rRndTVmNKS0h5N3E3QkFNbVVrdlEK+SIz9paPmHB7Cfyv080/H4I9fWKEbhyk
+            wyd01z7xNBq29HI3UoOnuGmVXHP8ZCz3bfRKpdv96Th6BZn9sL5ZqQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-06-20T13:48:40Z"
+    mac: ENC[AES256_GCM,data:g+BrOR2vUK+NMIzwLKlfJQZj737aOpd5fDZ330kzLm3BhTumQL+qLC8D2AqYKv30NvLR3qnFEQJ54FNBzeCauXiIwN7IuwAGTLP60lKSxl/Up7zRHoumfuCDw6TwVjOIS2NHqZ13htuSWC9JRKr4ds1iCzcBLkYeUU2yzjIoyiQ=,iv:TSjiSx8nLhQk45ZdmeQO26P00yZIazOKbl6DBJd0KvI=,tag:Ll/3GmactsVwW+3nZV0etg==,type:str]
+    pgp: []
+    encrypted_regex: token|certificateKey
+    version: 3.8.1
diff --git a/control-plane/ignition-files.enc/tree/etc/kubeadm.yaml b/control-plane/ignition-files.enc/tree/etc/kubeadm.yaml
diff --git a/node/Dockerfile b/node/Dockerfile
@@ -0,0 +1,36 @@
+FROM quay.io/fedora/fedora-coreos:stable
+
+# extra packages
+COPY etc/yum.repos.d/ /etc/yum.repos.d
+RUN rpm-ostree install \
+        dmidecode \
+        etcd \
+        haproxy \
+        kubeadm \
+        kubectl \
+        kubelet \
+        man-db \
+        man-pages \
+        tailscale \
+        tcpdump \
+        usbutils \
+        vim \
+    && \
+    ostree container commit
+
+COPY etc/ /etc
+COPY usr/ /usr
+RUN mkdir -p /etc/haproxy/conf.d && \
+    chmod 700 /etc/kubeadm.conf.d && \
+    ostree container commit
+
+# enable services
+RUN systemctl enable \
+        containerd.service \
+        etc-kubernetes.mount \
+        haproxy.service \
+        kubelet.service \
+        tailscale-init.service \
+        tailscaled.service \
+    && \
+    ostree container commit
diff --git a/.../etc/NetworkManager/dispatcher.d/kube-vip → .../etc/NetworkManager/dispatcher.d/kube-vip b/.../etc/NetworkManager/dispatcher.d/kube-vip → .../etc/NetworkManager/dispatcher.d/kube-vip
diff --git a/...ion-files/tree/etc/containerd/config.toml → node/etc/containerd/config.toml b/...ion-files/tree/etc/containerd/config.toml → node/etc/containerd/config.toml
diff --git a/...nition-files/tree/etc/haproxy/haproxy.cfg → node/etc/haproxy/haproxy.cfg b/...nition-files/tree/etc/haproxy/haproxy.cfg → node/etc/haproxy/haproxy.cfg