tinkerbell/charts: introduce showcase chart

tinkerbell/showcase/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: showcase
+description: Generates Tinkerbell CR's for a plethora of standard and exotic hardware; downloads & prepares Hook and OS images for provisioning
+type: application
+version: "0.0.2"
+

tinkerbell/showcase/README.md

@@ -0,0 +1,31 @@
+#### tinkerbell/charts: introduce `showcase` chart
+
+- `showcase` is a chart that, based on values.yaml dictionaries:
+    - generates Tinkerbell CRs (Hardware/Template/Workflow) for both standard (UEFI) & exotic (supported by Armbian) devices
+    - generates download/process jobs for multiple Hook flavors (see https://github.com/tinkerbell/hook/pull/205)
+    - generates download/process jobs for a few OS images (Ubuntu Cloud Images, Armbian, etc)
+    - should be independent of how one deployed Tinkerbell itself (stack chart, individual components, etc)
+- A few features:
+    - validates values.yaml for common mistakes; arch must match, etc.
+    - validates & handles rootDisk differences (re-invents "formatPartition()" a bit)
+    - avoids re-downloading Hooks and Images that are already on disk, even if Job re-runs
+    - allows easy way to use
+        - custom Hooks
+        - custom Kernel cmdline parameters at both the Hook & device level
+            - for example `acpi=off` at Hook level and `console=ttyS0` at board level
+        - custom OS images for deployment
+        - reboot or kexec to finish deployment
+        - different partition numbers for OS image's rootfs (some images have ESP, some have a separate `/boot`, etc)
+        - control if growpart and/or ssh/user setup is done during provisioning or not
+        - conversion of OS images (`qemu-to-raw-gzip` and `xz-to-gz`)
+    - has a "merge" mechanism with a common way to set parameters like net gateway, UEFI, etc (also easy to override per-device)
+    - default values have everything `enabled: false` thus showcase should produce nothing by default.
+        - Hooks & Images can be forced `enabled: true` in values.yaml, or
+            - `enabled: true` Devices automatically enable their Hook & Image
+- Probably missing:
+    - More validations
+    - Currently pointing to my Tinkerbell Actions, which I haven't PR'ed yet
+- How to use:
+    - Clone it, edit the values.yaml to your liking, and deploy.
+
+Signed-off-by: Ricardo Pardini <[email protected]>

tinkerbell/showcase/templates/hooks.yaml

@@ -0,0 +1,103 @@
+{{- range $hookId, $hook := .Values.provision.hook }}
+{{- $common := $.Values.provision.common -}}
+{{- $hook = merge $hook $common }}
+---
+# --> HookId: {{$hookId}}
+  # downloadId: {{ $hook.hookDownloadId }}
+  # downloadFile: {{ $hook.downloadFile }}
+  # hookDownloadBaseUrl: {{ $hook.hookDownloadBaseUrl }}
+{{- $hash := trunc 8 (sha1sum (printf "%s-%s-%s" $hook.hookDownloadId $hook.hookDownloadBaseUrl $hook.downloadFile)) }}
+# Loop over $.Values.hardware.devices (a dictionary) and for each value, check if it's hookRef matches this $hookId - if so, enable this hook.
+{{- $enabledByDeviceUsage := false }}
+{{- range $device := $.Values.hardware.devices }}
+# Testing {{$device.hookRef}} against {{$hookId}}...
+{{- if and $device.enabled (eq $device.hookRef $hookId) -}}
+  # YES!
+{{- $enabledByDeviceUsage = true -}}
+{{- else -}}
+  # NO!
+{{- end -}}
+{{- end }}
+
+{{- if or $enabledByDeviceUsage $hook.enabled }}
+# Enabled hook: {{$hookId}} - force enabled? {{$hook.enabled}} -- enabled by device usage? {{$enabledByDeviceUsage}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "download-hook-{{$hookId}}-{{$hash}}"
+  namespace: "{{ $.Release.Namespace }}"
+  labels:
+    "app.kubernetes.io/instance": "{{ $.Release.Name }}"
+    "app.kubernetes.io/part-of": "tinkerbell-showcase"
+data:
+  entrypoint.sh: |
+    #!/usr/bin/env bash
+    # This script is designed to download the Hook artifacts.
+    set -euxo pipefail
+    if [[ -f "/output/download_hook_{{$hookId}}_{{$hash}}.done" ]]; then
+    	echo "Hook already downloaded ({{$hookId}} - hash: {{$hash}}), skipping."
+    	exit 0
+    fi
+
+    {{
+    if $hook.skipDownload 
+    }}
+    echo "Skipping download for hook {{$hookId}}."
+    ls -lah "/output/{{$hook.kernel}}"
+    ls -lah "/output/{{$hook.initrd}}"
+    touch "/output/download_hook_{{$hookId}}_{{$hash}}.done"
+    {{ else }}
+    mkdir -p "/output/download_hook_{{$hookId}}"
+    cd "/output/download_hook_{{$hookId}}"
+    declare down_url="{{ $hook.hookDownloadBaseURL }}{{$hook.downloadFile}}"
+    declare down_file="/output/download_hook_{{$hookId}}/{{$hook.downloadFile}}"
+    wget -O "${down_file}.tmp" "${down_url}"
+    mv -v "${down_file}.tmp" "${down_file}"
+    mkdir -p "/output/download_hook_{{$hookId}}/extract"
+    cd "/output/download_hook_{{$hookId}}/extract"
+    tar -xvzf "${down_file}"
+    mv -v "/output/download_hook_{{$hookId}}/extract/"* /output/
+    rm -rf "/output/download_hook_{{$hookId}}"
+    ls -lah "/output/{{$hook.kernel}}"
+    ls -lah "/output/{{$hook.initrd}}"
+    touch "/output/download_hook_{{$hookId}}_{{$hash}}.done"    {{ end }}
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "download-hook-{{$hookId}}-{{$hash}}"
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    "app.kubernetes.io/instance": "{{ $.Release.Name }}"
+    "app.kubernetes.io/part-of": "tinkerbell-showcase"
+spec:
+  backoffLimit: 50
+  template:
+    metadata:
+      labels:
+        app: download-hook
+    spec:
+      containers:
+        - name: download-hook-{{$hookId}}
+          image: bash:5
+          command: [ "/script/entrypoint.sh" ]
+          volumeMounts:
+            - mountPath: /output
+              name: hook-artifacts
+            - mountPath: /script
+              name: configmap-volume
+      restartPolicy: OnFailure
+      volumes:
+        - name: hook-artifacts
+          hostPath:
+            path: {{ $.Values.tinkerbell.hostDirectory | quote }}
+            type: DirectoryOrCreate
+        - name: configmap-volume
+          configMap:
+            defaultMode: 0700
+            name: "download-hook-{{$hookId}}-{{$hash}}"
+{{- else }}
+# Disabled Hook: {{$hookId}}
+{{- end }}
+{{- end }}

tinkerbell/showcase/templates/images.yaml

@@ -0,0 +1,182 @@
+{{- range $imageId, $image := .Values.provision.images }}
+---
+# --> ImageId: {{ $imageId }}
+  # downloadUrl: "{{ $image.downloadURL }}"
+  # image: "{{ $image.image }}"
+  # conversion: "{{ $image.conversion }}"
+{{- $hash :=  trunc 8 (sha1sum (printf "%s-%s-%s" $image.downloadURL $image.image $image.conversion )) }}
+# Loop over $.Values.hardware.devices (a dictionary) and for each value, check if it's imageRef matches this $imageId - if so, enable this image.
+{{- $enabledByDeviceUsage := false }}
+{{- range $device := $.Values.hardware.devices }}
+# Testing {{$device.imageRef}} against {{$imageId}}...
+{{- if and $device.enabled (eq $device.imageRef $imageId) -}}
+# YES!
+{{- $enabledByDeviceUsage = true -}}
+{{- else -}}
+# NO!
+{{- end -}}
+{{- end }}
+{{- if or $enabledByDeviceUsage $image.enabled }}
+# ImageId: {{ $imageId }} - force enabled? {{ $image.enabled }} -- enabled by device usage? {{ $enabledByDeviceUsage }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "download-image-{{$imageId}}-{{$hash}}"
+  labels:
+    "app.kubernetes.io/instance": "{{ $.Release.Name }}"
+    "app.kubernetes.io/part-of": "tinkerbell-showcase"
+data:
+  entrypoint.sh: |-
+    #!/usr/bin/env bash
+    set -euxo pipefail
+    cat > /etc/apk/repositories << EOF; $(echo)
+    https://dl-cdn.alpinelinux.org/alpine/v$(cut -d'.' -f1,2 /etc/alpine-release)/main/
+    https://dl-cdn.alpinelinux.org/alpine/v$(cut -d'.' -f1,2 /etc/alpine-release)/community/
+    https://dl-cdn.alpinelinux.org/alpine/edge/community/
+    EOF
+    {{- if eq "qemu-to-raw-gzip" $image.conversion }}
+    # This script is designed to download a cloud image file (.img) and then convert it to a .raw.gz file.
+    # This is purpose built so non-raw cloud image files can be used with the "image2disk" action.
+    # See https://artifacthub.io/packages/tbaction/tinkerbell-community/image2disk.
+    image_url=$1
+    file=$2/${image_url##*/}
+    file=${file%.*}.raw.gz
+    if [[ ! -f "$file" ]]; then
+        echo "Image file $file does not exist. Downloading...."
+        apk add --update pigz qemu-img
+        wget "$image_url" -O image.img
+        qemu-img convert -O raw image.img image.raw
+        pigz < image.raw > "$file"
+        rm -f image.img image.raw
+    else
+        echo "File $file already exists, skipping download and conversion."
+    fi
+    {{- end }}
+
+    {{- if eq "none" $image.conversion }}
+    # No conversion, just a simple download; image must be in raw format, possibly compressed.
+    image_url="{{ $image.downloadURL }}"
+    file="/output/{{ $image.image }}"
+    if [[ ! -f "$file" ]]; then
+      echo "Image file $file does not exist. Downloading...."
+      wget "$image_url" -O "${file}.tmp"
+      mv -v "${file}.tmp" "$file"
+      echo "Done downloading image."  
+    else
+      echo "Image file $file already exists. Skipping download."
+    fi
+    {{- end }}
+
+    {{- if eq "xz-to-gz" $image.conversion }}
+    # Download, then produce a .gz file from a .xz file, using pigz and pixz, and using a pipe.
+    image_url="{{ $image.downloadURL }}"
+    file="/output/{{ $image.image }}"
+    if [[ ! -f "$file" ]]; then
+      echo "Image file $file does not exist. Downloading...."
+      apk add --update pigz pixz
+      wget "$image_url" -O "${file}.tmp.xz"
+      echo "Done downloading image. Converting from xz to gz..."  
+      pixz -d < "${file}.tmp.xz" | pigz > "${file}.tmp"  
+      mv -v "${file}.tmp" "$file"
+      echo "Done converting image."
+      rm -f "${file}.tmp.xz"
+    else
+      echo "Image file $file already exists. Skipping download."
+    fi
+    {{- end }}
+
+    {{- if eq "xz-qcow2-to-img-gz" $image.conversion }}
+    # Download, decompress xz, convert qcow2 to img, compress img to gz.
+    image_url="{{ $image.downloadURL }}"
+    file="/output/{{ $image.image }}"
+    if [[ ! -f "$file" ]]; then
+      echo "Image file $file does not exist. Downloading...."
+      apk add --update pigz pixz qemu-img
+      wget "$image_url" -O "${file}.tmp.xz"
+      echo "Done downloading image. Decompressing xz..."  
+      pixz -d < "${file}.tmp.xz" > "${file}.tmp.unxz"
+      rm -v "${file}.tmp.xz"
+      echo "Done decompressing xz, converting qcow2 to img..."
+      qemu-img convert -O raw "${file}.tmp.unxz" "${file}.tmp.raw"
+      rm -v "${file}.tmp.unxz"
+      echo "Done converting qcow2 to img, compressing img to gz..."
+      pigz < "${file}.tmp.raw" > "${file}"
+      rm -v "${file}.tmp.raw"
+      echo "Done converting image."
+    else
+      echo "Image file $file already exists. Skipping download."
+    fi
+    {{- end }}
+
+    {{- if eq "download-only" $image.conversion }}
+    # Download only. Image is already in format necessary.
+    image_url="{{ $image.downloadURL }}"
+    file="/output/{{ $image.image }}"
+    if [[ ! -f "$file" ]]; then
+      echo "Download-only Image file $file does not exist. Downloading...."
+      wget "$image_url" -O "${file}.tmp"
+      echo "Done downloading image. Moving..."
+      mv -v "${file}.tmp" "$file"  
+      echo "Done moving image."
+      rm -f "${file}.tmp.xz"
+    else
+      echo "Download-only image file $file already exists. Skipping download."
+    fi
+    {{- end }}
+
+    {{- if eq "local" $image.conversion }}
+    # Not even download anything. Image should be in there already somehow.
+    image_url="{{ $image.downloadURL }}"
+    file="/output/{{ $image.image }}"
+    if [[ ! -f "$file" ]]; then
+      echo "Image file $file does not exist. Please deploy image file manually, or use a different conversion than none. Sleeping 10s and exiting with error 66."
+      sleep 10
+      exit 66
+    else
+      echo "No-conversion image file $file already exists. Ready to use."
+    fi
+    {{- end }}
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "download-image-{{$imageId}}-{{$hash}}"
+  labels:
+    "app.kubernetes.io/instance": "{{ $.Release.Name }}"
+    "app.kubernetes.io/part-of": "tinkerbell-showcase"
+spec:
+  template:
+    metadata:
+      labels:
+        "app.kubernetes.io/instance": "{{ $.Release.Name }}"
+        "app.kubernetes.io/part-of": "tinkerbell-showcase"
+    spec:
+      containers:
+        - name: download-{{$imageId}}
+          image: bash:5
+          command: [ "/script/entrypoint.sh" ]
+          args:
+            [
+              "{{$image.downloadURL}}",
+              "/output",
+            ]
+          volumeMounts:
+            - mountPath: /output
+              name: image-artifacts
+            - mountPath: /script
+              name: configmap-volume
+      restartPolicy: OnFailure
+      volumes:
+        - name: image-artifacts
+          hostPath:
+            path: {{ $.Values.tinkerbell.hostDirectory }}
+            type: DirectoryOrCreate
+        - name: configmap-volume
+          configMap:
+            defaultMode: 0700
+            name: "download-image-{{$imageId}}-{{$hash}}"
+{{- else }}
+# ImageId: {{ $imageId }} DISABLED - force enabled? {{ $image.enabled }} -- enabled by device usage? {{ $enabledByDeviceUsage }}
+{{- end }}
+{{- end }}