From 250572c74f42ad11cddbeb8ec32e56b424d9b16c Mon Sep 17 00:00:00 2001 From: Sam Day Date: Thu, 20 Jun 2024 16:56:12 +0200 Subject: [PATCH] node: introduce layered OCI image https://github.com/coreos/fedora-coreos-docs/blob/2a55dc37c49c8bb298aea53b5642bd42d69d3fe7/modules/ROOT/pages/deriving-container.adoc the node/ Dockerfile packages up all the non-sensitive stuff previously in the Ignition config. the idea is to slim control-plane Ignition further, by pushing more of that stuff into cluster workloads. related to #464 --- .github/workflows/node-image.yml | 94 +++++++++++++ control-plane/config.bu | 44 +----- .../etc/kubeadm.conf.d/90-control-plane.yaml | 63 +++++++++ .../ignition-files.enc/tree/etc/kubeadm.yaml | 126 ------------------ node/Dockerfile | 36 +++++ .../etc/NetworkManager/dispatcher.d/kube-vip | 0 .../tree => node}/etc/containerd/config.toml | 0 .../tree => node}/etc/haproxy/haproxy.cfg | 0 node/etc/kubeadm.conf.d/00-base.yaml | 18 +++ node/etc/kubeadm.conf.d/70-proxy.yaml | 4 + .../tree => node}/etc/modules-load.d/k8s.conf | 0 .../tree => node}/etc/profile.d/crictl.sh | 0 .../tree => node}/etc/profile.d/kubectl.sh | 0 .../tree => node}/etc/sysctl.d/k8s.conf | 0 .../etc/yum.repos.d/kubernetes.repo | 0 .../etc/yum.repos.d/tailscale.repo | 0 .../zincati/config.d/55-update-strategy.toml | 0 .../lib/systemd/system/etc-kubernetes.mount | 8 ++ .../lib/systemd/system/tailscale-init.service | 20 +++ 19 files changed, 248 insertions(+), 165 deletions(-) create mode 100644 .github/workflows/node-image.yml create mode 100644 control-plane/ignition-files.enc/tree/etc/kubeadm.conf.d/90-control-plane.yaml delete mode 100644 control-plane/ignition-files.enc/tree/etc/kubeadm.yaml create mode 100644 node/Dockerfile rename {control-plane/ignition-files/tree => node}/etc/NetworkManager/dispatcher.d/kube-vip (100%) rename {control-plane/ignition-files/tree => node}/etc/containerd/config.toml (100%) rename {control-plane/ignition-files/tree => node}/etc/haproxy/haproxy.cfg (100%) create mode 100644 node/etc/kubeadm.conf.d/00-base.yaml create mode 100644 node/etc/kubeadm.conf.d/70-proxy.yaml rename {control-plane/ignition-files/tree => node}/etc/modules-load.d/k8s.conf (100%) rename {control-plane/ignition-files/tree => node}/etc/profile.d/crictl.sh (100%) rename {control-plane/ignition-files/tree => node}/etc/profile.d/kubectl.sh (100%) rename {control-plane/ignition-files/tree => node}/etc/sysctl.d/k8s.conf (100%) rename {control-plane/ignition-files/tree => node}/etc/yum.repos.d/kubernetes.repo (100%) rename {control-plane/ignition-files/tree => node}/etc/yum.repos.d/tailscale.repo (100%) rename {control-plane/ignition-files/tree => node}/etc/zincati/config.d/55-update-strategy.toml (100%) create mode 100644 node/usr/lib/systemd/system/etc-kubernetes.mount create mode 100644 node/usr/lib/systemd/system/tailscale-init.service diff --git a/.github/workflows/node-image.yml b/.github/workflows/node-image.yml new file mode 100644 index 00000000..f2f24b25 --- /dev/null +++ b/.github/workflows/node-image.yml @@ -0,0 +1,94 @@ +name: node/ Image + +on: + schedule: + - cron: '22 9 * * *' + push: + branches: [ main ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + pull_request: + branches: [ main ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}-node + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 + with: + cosign-release: 'v2.2.4' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: node/ + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/control-plane/config.bu b/control-plane/config.bu index 68f8753d..93a0908e 100644 --- a/control-plane/config.bu +++ b/control-plane/config.bu @@ -24,9 +24,6 @@ passwd: - systemd-journal ssh_authorized_keys: [ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFwawprQXEkGl38Q7T0PNseL0vpoyr4TbATMkEaZJTWQ] storage: - directories: - - path: /etc/haproxy - - path: /etc/haproxy/conf.d disks: - device: /dev/disk/by-id/coreos-boot-disk wipe_table: false @@ -38,10 +35,6 @@ storage: - size_mib: 0 label: var files: - - path: /etc/containerd/config.toml - overwrite: true - - path: /etc/kubeadm.yaml - mode: 0600 - path: /etc/ssh/ssh_host_ecdsa_key mode: 0600 - path: /etc/ssh/ssh_host_ed25519_key @@ -68,43 +61,16 @@ storage: - local: tree systemd: units: - - name: containerd.service - enabled: true - - name: etc-kubernetes.mount - enabled: true - contents: | - [Mount] - What=/var/lib/kubernetes - Where=/etc/kubernetes - Type=none - Options=bind - - [Install] - WantedBy=multi-user.target - - name: init.service + - name: rebase-custom.service enabled: true contents: | [Unit] - Wants=network-online.target + ConditionFirstBoot=true After=network-online.target - Before=zincati.service - ConditionPathExists=!/etc/%N.stamp - [Service] - LoadCredential=authkey:/etc/tailscale-auth.key - RemainAfterExit=yes + After=coreos-ignition-firstboot-complete.service Type=oneshot - # Install stuff. - ExecStart=/usr/bin/rpm-ostree install --apply-live --allow-inactive dmidecode haproxy kubeadm kubectl kubelet man-db man-pages tailscale tcpdump usbutils vim - - # Start stuff just installed. - ExecStart=systemctl enable --now haproxy.service kubelet.service tailscaled.service - - # Wait until tailscale is up. - ExecStart=bash -c 'until tailscale status --json; do sleep 0.5; done' - ExecStart=bash -c 'while [[ "$(tailscale status --json | jq -r .BackendState)" == "NoState" ]]; do sleep 0.5; done' - ExecStart=bash -c '[[ "$(tailscale status --json | jq -r .BackendState)" == "NeedsLogin" ]] && tailscale up --auth-key=file:${CREDENTIALS_DIRECTORY}/authkey --accept-dns=false --advertise-routes=172.29.0.1/32 || exit 0' - ExecStart=/bin/touch /etc/%N.stamp - + RemainAfterExit=yes + ExecStart=rpm-ostree rebase --reboot ostree-unverified-registry:ghcr.io/samcday/home-cluster-node:latest [Install] WantedBy=multi-user.target diff --git a/control-plane/ignition-files.enc/tree/etc/kubeadm.conf.d/90-control-plane.yaml b/control-plane/ignition-files.enc/tree/etc/kubeadm.conf.d/90-control-plane.yaml new file mode 100644 index 00000000..2e54c30d --- /dev/null +++ b/control-plane/ignition-files.enc/tree/etc/kubeadm.conf.d/90-control-plane.yaml @@ -0,0 +1,63 @@ +apiVersion: kubeadm.k8s.io/v1beta3 +kind: JoinConfiguration +controlPlane: + certificateKey: ENC[AES256_GCM,data:mGVlFWlaxfBhgTzQF3UKP9QExlvcC0cOHweXPAfYc6v3a+iUiYrQ9ZsoL5Eeb5h4Mac5XrrrQwvffb7mkIIwKg==,iv:QGmR9P0fWfFtYKVgzzYD+rigg9gmr82+WGvPT7qLun8=,tag:oaMcsc6xvNxPPnW3qM3CTw==,type:str] +nodeRegistration: + taints: [] +discovery: + bootstrapToken: + apiServerEndpoint: 172.29.0.1:6444 + token: ENC[AES256_GCM,data:iQ1Yszkh5MoWb0CcLhe46XqA0BBSwI0=,iv:HOmkqDz+aDRJ3s3WTi+NzuuojGzm5BpFt5+HZWBGiwQ=,tag:OTas+BeLeOgwIlBPzw1pZg==,type:str] + caCertHashes: + - sha256:115e5a84bdb260f15defdbd836acf108769cff1e18bc9e2c473eb6f208051c58 +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age10c9vvuvfkflc7zypu6zm8dtw0gdn028nlr3gslt35df8vdqrap5q36xav4 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIV05yUmI2WDA3dkd6Nmt5 + amdwUFU0NHl0MElNOG8zQnMwYzFFekNoZUVVCktpaXUyMGw4Ung0SjZYRDlpQi9F + amY1WngrMnZIWXdHanREbWpKbGN0d2sKLS0tIEwzbjM1emE0ZFJKRloxTDZkeFVy + Uk9rRndTVmNKS0h5N3E3QkFNbVVrdlEK+SIz9paPmHB7Cfyv080/H4I9fWKEbhyk + wyd01z7xNBq29HI3UoOnuGmVXHP8ZCz3bfRKpdv96Th6BZn9sL5ZqQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-06-20T13:48:40Z" + mac: ENC[AES256_GCM,data:g+BrOR2vUK+NMIzwLKlfJQZj737aOpd5fDZ330kzLm3BhTumQL+qLC8D2AqYKv30NvLR3qnFEQJ54FNBzeCauXiIwN7IuwAGTLP60lKSxl/Up7zRHoumfuCDw6TwVjOIS2NHqZ13htuSWC9JRKr4ds1iCzcBLkYeUU2yzjIoyiQ=,iv:TSjiSx8nLhQk45ZdmeQO26P00yZIazOKbl6DBJd0KvI=,tag:Ll/3GmactsVwW+3nZV0etg==,type:str] + pgp: [] + encrypted_regex: token|certificateKey + version: 3.8.1 +--- +apiVersion: kubeadm.k8s.io/v1beta3 +kind: InitConfiguration +bootstrapTokens: + - token: ENC[AES256_GCM,data:nUO2q0UL0BhNI4wKwKbIO4buRGEnjwo=,iv:QfYQuQEDATIagygqINw32GN2C145SCZiAH5u8104rK8=,tag:VaGflIjLkc2GygGEFDYejA==,type:str] + ttl: "0" +certificateKey: ENC[AES256_GCM,data:LhTFvyPlA/haZ/G24Op1Uy/IL1os8aeuAjqeqxRZv6wUJILm3KUKTAmNBdKzWn6Eo7fVKSZFwkUtju/dWoeh7Q==,iv:vkEwjBh1z+De4aLCWwjlSFBFN39uifXZ804ZfLje5Ro=,tag:JF1D9w/pUtACHCmDQbde2g==,type:str] +nodeRegistration: + kubeletExtraArgs: + volume-plugin-dir: /var/lib/kubelet/volume-plugins + taints: [] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age10c9vvuvfkflc7zypu6zm8dtw0gdn028nlr3gslt35df8vdqrap5q36xav4 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIV05yUmI2WDA3dkd6Nmt5 + amdwUFU0NHl0MElNOG8zQnMwYzFFekNoZUVVCktpaXUyMGw4Ung0SjZYRDlpQi9F + amY1WngrMnZIWXdHanREbWpKbGN0d2sKLS0tIEwzbjM1emE0ZFJKRloxTDZkeFVy + Uk9rRndTVmNKS0h5N3E3QkFNbVVrdlEK+SIz9paPmHB7Cfyv080/H4I9fWKEbhyk + wyd01z7xNBq29HI3UoOnuGmVXHP8ZCz3bfRKpdv96Th6BZn9sL5ZqQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-06-20T13:48:40Z" + mac: ENC[AES256_GCM,data:g+BrOR2vUK+NMIzwLKlfJQZj737aOpd5fDZ330kzLm3BhTumQL+qLC8D2AqYKv30NvLR3qnFEQJ54FNBzeCauXiIwN7IuwAGTLP60lKSxl/Up7zRHoumfuCDw6TwVjOIS2NHqZ13htuSWC9JRKr4ds1iCzcBLkYeUU2yzjIoyiQ=,iv:TSjiSx8nLhQk45ZdmeQO26P00yZIazOKbl6DBJd0KvI=,tag:Ll/3GmactsVwW+3nZV0etg==,type:str] + pgp: [] + encrypted_regex: token|certificateKey + version: 3.8.1 diff --git a/control-plane/ignition-files.enc/tree/etc/kubeadm.yaml b/control-plane/ignition-files.enc/tree/etc/kubeadm.yaml deleted file mode 100644 index 7a56264c..00000000 --- a/control-plane/ignition-files.enc/tree/etc/kubeadm.yaml +++ /dev/null @@ -1,126 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta3 -kind: InitConfiguration -bootstrapTokens: - - token: ENC[AES256_GCM,data:yKizp2iXQNahoMLiebF0TO1cNjSkGso=,iv:pAz0QINX8iCnkV35a0uXVR9P4Mea0ik8hGBGbhYi6aw=,tag:huySqV1ZVOzqbukrco3f7g==,type:str] - ttl: "0" -certificateKey: ENC[AES256_GCM,data:k6bbmF+eLmS89t/e/WhKGlGoDTYpCI3+zht6vyY0WBIrLH8/Q5bVOuHT8mni9osyn3HTQGKVXg6T3PBxwATSQg==,iv:HS0/55tGM/4nQSQmW5QaEPubbEvOQGzxmWXDObQUlmk=,tag:HHAo9HdvfBw5hlsbhjSnkw==,type:str] -nodeRegistration: - kubeletExtraArgs: - volume-plugin-dir: /var/lib/kubelet/volume-plugins - taints: [] -sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] - age: - - recipient: age10c9vvuvfkflc7zypu6zm8dtw0gdn028nlr3gslt35df8vdqrap5q36xav4 - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIV05yUmI2WDA3dkd6Nmt5 - amdwUFU0NHl0MElNOG8zQnMwYzFFekNoZUVVCktpaXUyMGw4Ung0SjZYRDlpQi9F - amY1WngrMnZIWXdHanREbWpKbGN0d2sKLS0tIEwzbjM1emE0ZFJKRloxTDZkeFVy - Uk9rRndTVmNKS0h5N3E3QkFNbVVrdlEK+SIz9paPmHB7Cfyv080/H4I9fWKEbhyk - wyd01z7xNBq29HI3UoOnuGmVXHP8ZCz3bfRKpdv96Th6BZn9sL5ZqQ== - -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-05-10T08:19:31Z" - mac: ENC[AES256_GCM,data:y85Kf2M4fVQSUSdBqbBKroVV/HQ0ac7PWaeCDY1nqKjwyEWEr/IVt+rFPrMqOtMQ0DF1JCHqr3aquX/ChbdhhSeTZsb1XyxfXk3Ee51uhbGsqXZNb4PKBdmbpQHMTOpuf0ELiDkRk+QHo15jvyspCig8bq4rjEmVnDk5P2neGos=,iv:pVaQNe4pxK7jtdvUWtSwkUVfC1maRIYaKeLwrHyn1bs=,tag:V4e6gitnjz1jbq6sDsDIaw==,type:str] - pgp: [] - encrypted_regex: token|certificateKey - version: 3.8.1 ---- -apiVersion: kubeadm.k8s.io/v1beta3 -kind: ClusterConfiguration -controlPlaneEndpoint: 172.29.0.1:6444 -controllerManager: - extraArgs: - bind-address: 0.0.0.0 - flex-volume-plugin-dir: /var/lib/kubelet/volume-plugins -etcd: - local: - extraArgs: - listen-metrics-urls: http://0.0.0.0:2381 -networking: - dnsDomain: home-cluster.local - podSubnet: 172.30.0.0/16 - serviceSubnet: 172.31.0.0/16 -scheduler: - extraArgs: - bind-address: 0.0.0.0 -sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] - age: - - recipient: age10c9vvuvfkflc7zypu6zm8dtw0gdn028nlr3gslt35df8vdqrap5q36xav4 - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIV05yUmI2WDA3dkd6Nmt5 - amdwUFU0NHl0MElNOG8zQnMwYzFFekNoZUVVCktpaXUyMGw4Ung0SjZYRDlpQi9F - amY1WngrMnZIWXdHanREbWpKbGN0d2sKLS0tIEwzbjM1emE0ZFJKRloxTDZkeFVy - Uk9rRndTVmNKS0h5N3E3QkFNbVVrdlEK+SIz9paPmHB7Cfyv080/H4I9fWKEbhyk - wyd01z7xNBq29HI3UoOnuGmVXHP8ZCz3bfRKpdv96Th6BZn9sL5ZqQ== - -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-05-10T08:19:31Z" - mac: ENC[AES256_GCM,data:y85Kf2M4fVQSUSdBqbBKroVV/HQ0ac7PWaeCDY1nqKjwyEWEr/IVt+rFPrMqOtMQ0DF1JCHqr3aquX/ChbdhhSeTZsb1XyxfXk3Ee51uhbGsqXZNb4PKBdmbpQHMTOpuf0ELiDkRk+QHo15jvyspCig8bq4rjEmVnDk5P2neGos=,iv:pVaQNe4pxK7jtdvUWtSwkUVfC1maRIYaKeLwrHyn1bs=,tag:V4e6gitnjz1jbq6sDsDIaw==,type:str] - pgp: [] - encrypted_regex: token|certificateKey - version: 3.8.1 ---- -apiVersion: kubeadm.k8s.io/v1beta3 -kind: JoinConfiguration -controlPlane: - certificateKey: ENC[AES256_GCM,data:QrYYJnbNWoM4WRY84e68oB/2IF3rqURmaLA+HMtpJNyeIxHEQgL1NFSiJwIEUFJaT4/SgV/87KawwK68FqxwxA==,iv:zNpJnN+CivQWiXkyO9MXM1ZW4ItDeJegTWui8knMCZA=,tag:1RF7QyQ4iP2aBGvFuWRCpQ==,type:str] -nodeRegistration: - taints: [] -discovery: - bootstrapToken: - apiServerEndpoint: 172.29.0.1:6444 - token: ENC[AES256_GCM,data:E3FursuMS3bPrzEffkYcLvhpx5mnUzg=,iv:2T+GZgsjbAHt5stqHmaLngsc50MToj351p+u9sK498k=,tag:9tLADAbwKWL/KoYmuR868Q==,type:str] - caCertHashes: - - sha256:115e5a84bdb260f15defdbd836acf108769cff1e18bc9e2c473eb6f208051c58 -sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] - age: - - recipient: age10c9vvuvfkflc7zypu6zm8dtw0gdn028nlr3gslt35df8vdqrap5q36xav4 - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIV05yUmI2WDA3dkd6Nmt5 - amdwUFU0NHl0MElNOG8zQnMwYzFFekNoZUVVCktpaXUyMGw4Ung0SjZYRDlpQi9F - amY1WngrMnZIWXdHanREbWpKbGN0d2sKLS0tIEwzbjM1emE0ZFJKRloxTDZkeFVy - Uk9rRndTVmNKS0h5N3E3QkFNbVVrdlEK+SIz9paPmHB7Cfyv080/H4I9fWKEbhyk - wyd01z7xNBq29HI3UoOnuGmVXHP8ZCz3bfRKpdv96Th6BZn9sL5ZqQ== - -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-05-10T08:19:31Z" - mac: ENC[AES256_GCM,data:y85Kf2M4fVQSUSdBqbBKroVV/HQ0ac7PWaeCDY1nqKjwyEWEr/IVt+rFPrMqOtMQ0DF1JCHqr3aquX/ChbdhhSeTZsb1XyxfXk3Ee51uhbGsqXZNb4PKBdmbpQHMTOpuf0ELiDkRk+QHo15jvyspCig8bq4rjEmVnDk5P2neGos=,iv:pVaQNe4pxK7jtdvUWtSwkUVfC1maRIYaKeLwrHyn1bs=,tag:V4e6gitnjz1jbq6sDsDIaw==,type:str] - pgp: [] - encrypted_regex: token|certificateKey - version: 3.8.1 ---- -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -metricsBindAddress: 0.0.0.0:10249 -sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] - age: - - recipient: age10c9vvuvfkflc7zypu6zm8dtw0gdn028nlr3gslt35df8vdqrap5q36xav4 - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIV05yUmI2WDA3dkd6Nmt5 - amdwUFU0NHl0MElNOG8zQnMwYzFFekNoZUVVCktpaXUyMGw4Ung0SjZYRDlpQi9F - amY1WngrMnZIWXdHanREbWpKbGN0d2sKLS0tIEwzbjM1emE0ZFJKRloxTDZkeFVy - Uk9rRndTVmNKS0h5N3E3QkFNbVVrdlEK+SIz9paPmHB7Cfyv080/H4I9fWKEbhyk - wyd01z7xNBq29HI3UoOnuGmVXHP8ZCz3bfRKpdv96Th6BZn9sL5ZqQ== - -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-05-10T08:19:31Z" - mac: ENC[AES256_GCM,data:y85Kf2M4fVQSUSdBqbBKroVV/HQ0ac7PWaeCDY1nqKjwyEWEr/IVt+rFPrMqOtMQ0DF1JCHqr3aquX/ChbdhhSeTZsb1XyxfXk3Ee51uhbGsqXZNb4PKBdmbpQHMTOpuf0ELiDkRk+QHo15jvyspCig8bq4rjEmVnDk5P2neGos=,iv:pVaQNe4pxK7jtdvUWtSwkUVfC1maRIYaKeLwrHyn1bs=,tag:V4e6gitnjz1jbq6sDsDIaw==,type:str] - pgp: [] - encrypted_regex: token|certificateKey - version: 3.8.1 diff --git a/node/Dockerfile b/node/Dockerfile new file mode 100644 index 00000000..de7f45a4 --- /dev/null +++ b/node/Dockerfile @@ -0,0 +1,36 @@ +FROM quay.io/fedora/fedora-coreos:stable + +# extra packages +COPY etc/yum.repos.d/ /etc/yum.repos.d +RUN rpm-ostree install \ + dmidecode \ + etcd \ + haproxy \ + kubeadm \ + kubectl \ + kubelet \ + man-db \ + man-pages \ + tailscale \ + tcpdump \ + usbutils \ + vim \ + && \ + ostree container commit + +COPY etc/ /etc +COPY usr/ /usr +RUN mkdir -p /etc/haproxy/conf.d && \ + chmod 700 /etc/kubeadm.conf.d && \ + ostree container commit + +# enable services +RUN systemctl enable \ + containerd.service \ + etc-kubernetes.mount \ + haproxy.service \ + kubelet.service \ + tailscale-init.service \ + tailscaled.service \ + && \ + ostree container commit diff --git a/control-plane/ignition-files/tree/etc/NetworkManager/dispatcher.d/kube-vip b/node/etc/NetworkManager/dispatcher.d/kube-vip similarity index 100% rename from control-plane/ignition-files/tree/etc/NetworkManager/dispatcher.d/kube-vip rename to node/etc/NetworkManager/dispatcher.d/kube-vip diff --git a/control-plane/ignition-files/tree/etc/containerd/config.toml b/node/etc/containerd/config.toml similarity index 100% rename from control-plane/ignition-files/tree/etc/containerd/config.toml rename to node/etc/containerd/config.toml diff --git a/control-plane/ignition-files/tree/etc/haproxy/haproxy.cfg b/node/etc/haproxy/haproxy.cfg similarity index 100% rename from control-plane/ignition-files/tree/etc/haproxy/haproxy.cfg rename to node/etc/haproxy/haproxy.cfg diff --git a/node/etc/kubeadm.conf.d/00-base.yaml b/node/etc/kubeadm.conf.d/00-base.yaml new file mode 100644 index 00000000..6ba1df78 --- /dev/null +++ b/node/etc/kubeadm.conf.d/00-base.yaml @@ -0,0 +1,18 @@ +apiVersion: kubeadm.k8s.io/v1beta3 +kind: ClusterConfiguration +controlPlaneEndpoint: 172.29.0.1:6444 +controllerManager: + extraArgs: + bind-address: 0.0.0.0 + flex-volume-plugin-dir: /var/lib/kubelet/volume-plugins +etcd: + local: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2381 +networking: + dnsDomain: home-cluster.local + podSubnet: 172.30.0.0/16 + serviceSubnet: 172.31.0.0/16 +scheduler: + extraArgs: + bind-address: 0.0.0.0 diff --git a/node/etc/kubeadm.conf.d/70-proxy.yaml b/node/etc/kubeadm.conf.d/70-proxy.yaml new file mode 100644 index 00000000..e1a82727 --- /dev/null +++ b/node/etc/kubeadm.conf.d/70-proxy.yaml @@ -0,0 +1,4 @@ +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +metricsBindAddress: 0.0.0.0:10249 diff --git a/control-plane/ignition-files/tree/etc/modules-load.d/k8s.conf b/node/etc/modules-load.d/k8s.conf similarity index 100% rename from control-plane/ignition-files/tree/etc/modules-load.d/k8s.conf rename to node/etc/modules-load.d/k8s.conf diff --git a/control-plane/ignition-files/tree/etc/profile.d/crictl.sh b/node/etc/profile.d/crictl.sh similarity index 100% rename from control-plane/ignition-files/tree/etc/profile.d/crictl.sh rename to node/etc/profile.d/crictl.sh diff --git a/control-plane/ignition-files/tree/etc/profile.d/kubectl.sh b/node/etc/profile.d/kubectl.sh similarity index 100% rename from control-plane/ignition-files/tree/etc/profile.d/kubectl.sh rename to node/etc/profile.d/kubectl.sh diff --git a/control-plane/ignition-files/tree/etc/sysctl.d/k8s.conf b/node/etc/sysctl.d/k8s.conf similarity index 100% rename from control-plane/ignition-files/tree/etc/sysctl.d/k8s.conf rename to node/etc/sysctl.d/k8s.conf diff --git a/control-plane/ignition-files/tree/etc/yum.repos.d/kubernetes.repo b/node/etc/yum.repos.d/kubernetes.repo similarity index 100% rename from control-plane/ignition-files/tree/etc/yum.repos.d/kubernetes.repo rename to node/etc/yum.repos.d/kubernetes.repo diff --git a/control-plane/ignition-files/tree/etc/yum.repos.d/tailscale.repo b/node/etc/yum.repos.d/tailscale.repo similarity index 100% rename from control-plane/ignition-files/tree/etc/yum.repos.d/tailscale.repo rename to node/etc/yum.repos.d/tailscale.repo diff --git a/control-plane/ignition-files/tree/etc/zincati/config.d/55-update-strategy.toml b/node/etc/zincati/config.d/55-update-strategy.toml similarity index 100% rename from control-plane/ignition-files/tree/etc/zincati/config.d/55-update-strategy.toml rename to node/etc/zincati/config.d/55-update-strategy.toml diff --git a/node/usr/lib/systemd/system/etc-kubernetes.mount b/node/usr/lib/systemd/system/etc-kubernetes.mount new file mode 100644 index 00000000..92ebf28f --- /dev/null +++ b/node/usr/lib/systemd/system/etc-kubernetes.mount @@ -0,0 +1,8 @@ +[Mount] +What=/var/lib/kubernetes +Where=/etc/kubernetes +Type=none +Options=bind + +[Install] +WantedBy=multi-user.target diff --git a/node/usr/lib/systemd/system/tailscale-init.service b/node/usr/lib/systemd/system/tailscale-init.service new file mode 100644 index 00000000..ad50e45e --- /dev/null +++ b/node/usr/lib/systemd/system/tailscale-init.service @@ -0,0 +1,20 @@ +[Unit] +Wants=network-online.target +After=network-online.target +Before=zincati.service +ConditionPathExists=!/etc/%N.stamp +ConditionPathExists=/etc/tailscale-auth.key + +[Service] +LoadCredential=authkey:/etc/tailscale-auth.key +RemainAfterExit=yes +Type=oneshot + +# Wait until tailscale is up. +ExecStart=bash -c 'until tailscale status --json; do sleep 0.5; done' +ExecStart=bash -c 'while [[ "$(tailscale status --json | jq -r .BackendState)" == "NoState" ]]; do sleep 0.5; done' +ExecStart=bash -c '[[ "$(tailscale status --json | jq -r .BackendState)" == "NeedsLogin" ]] && tailscale up --auth-key=file:${CREDENTIALS_DIRECTORY}/authkey --accept-dns=false --advertise-routes=172.29.0.1/32 || exit 0' +ExecStart=/bin/touch /etc/%N.stamp + +[Install] +WantedBy=multi-user.target