feat: gcp support, and dockerfiles with nix support now

- kloudlite CRDs are now applied with `--server-side` flag

.dockerignore

@@ -1,6 +1,13 @@
 **
+# !terraform
+# !build-scripts
+# !infrastructure-templates
+# infrastructure-templates/**/.terraform.d
+# !.ci
+# !flake.nix
+# !flake.lock
+# !context.tar
 !terraform
-!build-scripts
 !infrastructure-templates
-!.ci
-!.terraform.d.zip
+infrastructure-templates/**/.terraform.d
+!context.tar

.gitignore

@@ -8,3 +8,4 @@
 .direnv
 .terraform.d
 .dump
+.buildx-cache

Dockerfile

@@ -1,28 +1,57 @@
 #syntax=docker/dockerfile:1.4
 FROM alpine:3.16
-RUN apk add bash curl gettext jq lz4 helm kubectl --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community --no-cache 
+RUN apk add bash curl gettext jq lz4 helm kubectl zstd --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community --no-cache 
 RUN curl -L0 https://releases.hashicorp.com/terraform/1.5.7/terraform_1.5.7_linux_amd64.zip > tf.zip && unzip tf.zip && mv terraform /usr/local/bin && rm tf.zip
-RUN adduser --disabled-password --home="/app" --uid 1717 nonroot
-USER nonroot
 WORKDIR /app
 COPY --chown=nonroot ./terraform ./terraform
 RUN mkdir -p infrastructure-templates
-COPY --chown=nonroot ./infrastructure-templates ./infrastructure-templates
+# COPY --chown=nonroot ./infrastructure-templates ./infrastructure-templates
 ENV TF_PLUGIN_CACHE_DIR="/app/.terraform.d/plugin-cache"
 # COPY .terraform.d.zip /app/terraform.zip
 RUN mkdir -p $TF_PLUGIN_CACHE_DIR
-SHELL ["/bin/bash", "-c"]
-RUN <<'EOF'
-  for dir in $(ls -d ./infrastructure-templates/*); do
-    pushd $dir
-    terraform init -backend=false &
-    popd
-  done
+# SHELL ["/bin/bash", "-c"]
+# RUN <<'EOF'
+#   for dir in $(ls -d ./infrastructure-templates/{aws,gcp}/*); do
+#     pushd $dir
+#     terraform init -backend=false &
+#     popd
+#   done
+#
+#   wait
+#
+#   tdir=$(basename $(dirname $TF_PLUGIN_CACHE_DIR))
+#   tar cf - $tdir | lz4 -v -5 > tf.lz4 && rm -rf $tdir
+# EOF
+# ENV DECOMPRESS_CMD="lz4 -d tf.lz4 | tar xf -"
+# ENV TEMPLATES_DIR="/app/infrastructure-templates"
+
+WORKDIR /app
+ENV TF_PLUGIN_CACHE_DIR="/app/.terraform.d/plugin-cache"
+RUN mkdir -p $TF_PLUGIN_CACHE_DIR
+RUN cat > script.sh <<EOF
+  #!/usr/bin/env bash
+  echo "hi" >> log.file
+  ls -d ./infrastructure-templates/{gcp,aws}/* | tee log.file | xargs -I{} bash -c "echo name is {}; $(terraform init chdir={} -backend=false &)"
+  # for dir in $(ls -d ./infrastructure-templates/{gcp,aws}/*); do
+  #   echo $dir >> log.file
+  #   pushd $dir
+  #   terraform init -backend=false &
+  #   popd
+  # done
 
   wait
 
   tdir=$(basename $(dirname $TF_PLUGIN_CACHE_DIR))
   tar cf - $tdir | lz4 -v -5 > tf.lz4 && rm -rf $tdir
 EOF
+
+COPY --chown=nonroot ./terraform ./terraform
+COPY --chown=nonroot ./infrastructure-templates ./infrastructure-templates
+# RUN --mount=type=bind,source=./infrastructure-templates,target=infrastructure-templates \
+#   --mount=type=bind,source=./terraform,target=terraform \
+RUN --mount=type=cache,id=sample,target=/app/.terraform.d/plugin-cache \
+  chmod +x /app/script.sh && bash /app/script.sh
+RUN adduser --disabled-password --home="/app" --uid 1717 nonroot
+USER nonroot
 ENV DECOMPRESS_CMD="lz4 -d tf.lz4 | tar xf -"
 ENV TEMPLATES_DIR="/app/infrastructure-templates"

DockerfileNIX

@@ -0,0 +1,107 @@
+# vim: set ft=dockerfile:
+FROM nixos/nix:latest AS builder
+
+WORKDIR /app
+
+# COPY flake.nix flake.lock ./
+ENV NIX_STORE_DIR=/nix/store2
+
+RUN --mount=type=bind,source=flake.nix,target=flake.nix \
+  --mount=type=bind,source=flake.lock,target=flake.lock \
+  --mount=type=cache,target=/nix/store2 \
+  cp -R /nix/store /nix/store2 && nix \
+  --extra-experimental-features "nix-command flakes" \
+  --option filter-syscalls false \
+  build .#container
+
+# Copy the Nix store closure into a directory. The Nix store closure is the
+# entire set of Nix store values that we need for our build.
+RUN mkdir /tmp/nix-store-closure
+RUN cp -R $(nix-store -qR result/) /tmp/nix-store-closure
+
+# Set up the environment to use the packages defined in the flake
+# ENV PATH /root/.nix-profile/bin:$PATH
+
+# Your application's setup continues here...
+# FROM ubuntu:latest
+# FROM cgr.dev/chainguard/busybox:latest
+# FROM debian:12-slim
+# FROM busybox:latest
+FROM cgr.dev/chainguard/bash:latest
+# FROM gcr.io/distroless/static:latest
+COPY --from=builder /tmp/nix-store-closure /nix/store
+RUN mkdir -p /usr/local/bin
+COPY --from=builder /app/result/bin/* /usr/local/bin
+
+RUN mkdir -p /etc/ssl/certs
+COPY --from=builder /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
+
+WORKDIR /app
+COPY ./terraform ./terraform
+ARG CLOUD_PROVIDER
+RUN mkdir -p infrastructure-templates
+COPY ./infrastructure-templates/${CLOUD_PROVIDER} ./infrastructure-templates/${CLOUD_PROVIDER}
+ENV TF_PLUGIN_CACHE_DIR="/app/.terraform.d/plugin-cache"
+# COPY .terraform.d.zip /app/terraform.zip
+RUN mkdir -p $TF_PLUGIN_CACHE_DIR
+SHELL ["bash", "-c"]
+RUN <<'EOF'
+  # for dir in $(ls -d ./infrastructure-templates/{gcp,aws}/*); do
+  for dir in $(ls -d ./infrastructure-templates/${CLOUD_PROVIDER}/*); do
+    pushd $dir
+    terraform init -backend=false &
+    popd
+  done
+
+  wait
+
+  tdir=$(basename $(dirname $TF_PLUGIN_CACHE_DIR))
+  # tar cf - $tdir | lz4 -v -5 > tf.lz4 && rm -rf $tdir
+  tar cf - $tdir | zstd -12 --compress > tf.zst && rm -rf $tdir
+EOF
+# ENV DECOMPRESS_CMD="lz4 -d tf.lz4 | tar xf -"
+ENV DECOMPRESS_CMD="zstd --decompress tf.zst --stdout | tar xf -"
+ENV TEMPLATES_DIR="/app/infrastructure-templates"
+
+# # Your application's setup continues here...
+# FROM ubuntu:latest
+# # FROM debian:12-slim
+# RUN mkdir -p /etc/ssl/certs
+# COPY --from=builder /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
+#
+# COPY --from=builder /tmp/nix-store-closure /nix/store
+# COPY --from=builder /app/result/bin/* /usr/local/bin
+#
+# WORKDIR /app
+# # COPY ./terraform ./terraform
+# # RUN mkdir -p infrastructure-templates
+# # COPY ./infrastructure-templates ./infrastructure-templates
+# ENV TF_PLUGIN_CACHE_DIR="/app/.terraform.d/plugin-cache"
+# # COPY .terraform.d.zip /app/terraform.zip
+# RUN mkdir -p $TF_PLUGIN_CACHE_DIR
+# RUN cat > script.sh <<EOF
+#   #!/usr/bin/env bash
+#   echo "hi"
+#   # ls -d ./infrastructure-templates/{gcp,aws}/* | tee log.file | xargs -I{} bash -c "echo name is {}; pushd {}; terraform init -backend=false ;popd"
+#   item=$(ls -d infrastructure-templates/{gcp,aws}/* | xargs -I{} printf "%s " {})
+#   echo "$item"
+#   exit 1
+#   # for dir in $(ls -d infrastructure-templates/{gcp,aws}/*); do
+#   #   echo $dir >> log.file
+#   #   pushd $dir
+#   #   terraform init -backend=false &
+#   #   popd
+#   # done
+#
+#   wait
+#
+#   # exit 1
+#   tdir=$(basename $(dirname $TF_PLUGIN_CACHE_DIR))
+#   tar cf - $tdir | lz4 -v -5 > tf.lz4 && rm -rf $tdir
+# EOF
+# RUN --mount=type=bind,source=infrastructure-templates,target=infrastructure-templates,readwrite \
+#   --mount=type=bind,source=terraform,target=terraform \
+#   --mount=type=cache,target=/app/.terraform.d/plugin-cache \
+#   chmod +x ./script.sh && bash ./script.sh && cat log.file && exit 1
+# ENV DECOMPRESS_CMD="lz4 -d tf.lz4 | tar xf -"
+# ENV TEMPLATES_DIR="/app/infrastructure-templates"

DockerfileNIX2

@@ -0,0 +1,32 @@
+# vim: set ft=dockerfile:
+FROM nixos/nix:latest AS nix
+
+FROM busybox:latest
+
+RUN mkdir -p /etc/ssl/certs
+COPY --from=nix /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
+
+WORKDIR /app
+RUN --mount=type=bind,source=context.tar,target=context.tar \
+  tar xf context.tar && \
+  mkdir /nix && mv nixstore /nix/store && \
+  mkdir -p /usr/local/bin && mv result/bin/* /usr/local/bin/ && rm -rf result && \
+  mv tf.zst /app/tf.zst
+
+RUN adduser --disabled-password --home="/app" --uid 1717 nonroot
+USER nonroot
+COPY --chown=nonroot ./terraform ./terraform
+COPY --chown=nonroot ./infrastructure-templates ./infrastructure-templates
+# COPY --from=nixstore ./ /nix/store
+# RUN mkdir -p /usr/local/bin
+# COPY --from=builder ./bin/* /usr/local/bin
+# RUN --mount=type=bind,source=result,target=/result cp -r /result/bin/* /usr/local/bin
+# COPY ./bin/* /usr/local/bin
+# WORKDIR /app
+# COPY ./terraform ./terraform
+# ARG CLOUD_PROVIDER
+# COPY ./infrastructure-templates/${CLOUD_PROVIDER} ./infrastructure-templates/${CLOUD_PROVIDER}
+ENV TF_PLUGIN_CACHE_DIR="/app/.terraform.d/plugin-cache"
+# # COPY ./tf.zst ./tf.zst
+ENV DECOMPRESS_CMD="zstd --decompress tf.zst --stdout | tar xf -"
+ENV TEMPLATES_DIR="/app/infrastructure-templates"

Taskfile.yml

@@ -11,16 +11,57 @@ tasks:
     preconditions:
       - sh: '[[ -n "{{.Image}}" ]]'
         msg: 'var Image must have a value'
+      - sh: '[[ -n "{{.cloudprovider}}" ]]'
+        msg: 'var cloudprovider must have a value'
     vars:
-      Push: "{{.Push | default false}}"
+      push: "{{.push | default false}}"
     # silent: true
     cmds:
-      - nerdctl build -t {{.Image}} .
+      - nerdctl build -f DockerfileNIX --build-arg CLOUD_PROVIDER={{.cloudprovider}} -t {{.Image}} . --cache-from=type=local,src=$PWD/.buildx-cache --cache-to=type=local,dest=$PWD/.buildx-cache,mode=max
       - |+
-        if [ "{{.Push}}" == "true" ]; then
+        if [ "{{.push}}" == "true" ]; then
           nerdctl push {{.Image}}
         fi
 
+  local:build:iac-job:
+    preconditions:
+      - sh: '[[ -n "{{.cloudprovider}}" ]]'
+        msg: 'var cloudprovider must have a value'
+    vars:
+      nix_store_closure: /tmp/nix-store-closure
+    cmds:
+      - sudo rm -rf result
+      - nix build .#container
+      - sudo rm -rf {{.nix_store_closure}}
+      - mkdir {{.nix_store_closure}}
+      - cp -R $(nix-store -qR result/) {{.nix_store_closure}}
+      - |+
+        export TF_PLUGIN_CACHE_DIR="$PWD/.terraform.d/plugin-cache"
+        # for dir in $(ls -d ./infrastructure-templates/{{.cloudprovider}}/*); do
+        for dir in $(ls -d ./infrastructure-templates/{gcp,aws}/*); do
+          terraform -chdir=$dir init -backend=false -upgrade &
+        done
+
+        wait
+
+        echo "compressing"
+        tdir=$(basename $(dirname $TF_PLUGIN_CACHE_DIR))
+        # tar cf - $tdir | zstd -12 --compress > tf.zst
+        tar cf - $tdir | zstd --compress > tf.zst
+
+      # - docker buildx build -f DockerfileNIX2 --build-arg CLOUD_PROVIDER={{.cloudprovider}} --build-context builder=result --build-context nixstore={{.nix_store_closure}} -t {{.Image}} --output type=image,oci-mediatypes=true,compression=zstd,compression-level=10,force-compression=true,push=true . --no-cache
+      - |+
+        dir=$(mktemp -d)
+        rm -rf context.tar
+        mv tf.zst result $dir
+        mv /tmp/nix-store-closure $dir/nixstore
+        pushd $dir
+        tar cf context.tar .
+        popd
+        mv $dir/context.tar .
+      - nerdctl build -f DockerfileNIX2 --build-arg CLOUD_PROVIDER={{.cloudprovider}} -t {{.Image}} . --no-cache
+      # - rm ./context.tar
+
   container:build-and-push:
     preconditions:
       - sh: '[[ -n "{{.Image}}" ]]'

cmd/gcp-setup/setup.sh

@@ -0,0 +1,26 @@
+#! /usr/bin/env bash
+
+role="kloudlite_custom_role"
+project_id="rich-wavelet-412321"
+
+service_account="kloudlite-sa"
+# export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="/home/nxtcoder17/Downloads/rich-wavelet-412321-adc26c13a544.json"
+# export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="/tmp/gcp-auth.json"
+
+# gcloud auth login
+
+gcloud iam roles create $role \
+	--project=$project_id \
+	--title="Kloudlite Role" \
+	--description="kloudlite admin role" \
+	--permissions=compute.instances.start,compute.instances.stop \
+	--stage=GA | tee -a /tmp/output.json
+
+gcloud iam service-accounts create $service_account \
+	--description="service account json" \
+	--display-name="Kloudlite Account" \
+	--project="$project_id"
+
+gcloud projects add-iam-policy-binding "$project_id" \
+	--member="serviceAccount:$service_account@" \
+	--role="projects/$project_id/roles/$role"

cmd/vm-setup/main.sh

@@ -31,4 +31,4 @@ ExecStart=$PWD/runner --config $PWD/runner-config.yml
 WantedBy=multi-user.target
 EOF
 
-systemctl enable --now kloudlite-k3s.service
+systemctl enable --now kloudlite-k3s.service

examples-infra/gcp/masters-and-workers/main.tf

@@ -34,7 +34,7 @@ module "worker-nodes" {
   nodes                       = each.value.nodes
   node_labels                 = each.value.node_labels
   provision_mode              = each.value.provision_mode
-  nodepool_name             = each.key
+  nodepool_name               = each.key
   bootvolume_type             = each.value.bootvolume_type
   bootvolume_size             = each.value.bootvolume_size
   additional_disk             = each.value.additional_disk

examples-infra/gcp/masters-and-workers/providers.tf

@@ -20,7 +20,7 @@ provider "google" {
   # Configuration options
   project     = var.gcp_project_id
   region      = var.gcp_region
-  credentials = var.gcp_credentials_json
+  credentials = base64decode(var.gcp_credentials_json)
 }
 
 provider "ssh" {