Run ciux-based CI

- Hack to install go module in v3 - Increase wait time for topics retrieval in CI - Use ciux to install itest deps * Use latest finkctl in gha
astrolabsoftware · Dec 12, 2023 · 8040a5f · 8040a5f
1 parent fb8926f
commit 8040a5f
Show file tree

Hide file tree

Showing 6 changed files with 77 additions and 93 deletions.
diff --git a/.ciux b/.ciux
@@ -0,0 +1,11 @@
+registry: gitlab-registry.in2p3.fr/astrolabsoftware/fink
+dependencies:
+  - url: https://github.com/astrolabsoftware/fink-alert-simulator
+    # If true repository will be locally cloned
+    clone: true
+    # If true, container image will be pulled
+    pull: true
+  - url: https://github.com/astrolabsoftware/finkctl
+    clone: true
+  - package: github.com/k8s-school/[email protected]
+
diff --git a/.github/workflows/itest-gha.yml b/.github/workflows/itest-gha.yml
@@ -5,10 +5,11 @@ on:
     branches:
       - master
 env:
-  FINKCTL_VERSION: v1.1.0-rc2
+  CIUXCONFIG: /tmp/ciux.sh
+  CIUX_VERSION: v0.0.1-rc8
+  GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   NOSCIENCE: true
   MINIMAL: true
-
 jobs:
   build:
     name: Build image
@@ -18,24 +19,24 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Declare Version Variables
-        id: vars
-        shell: bash
-        run: |
-          DIR=$(pwd)
-          . ./conf.sh
-          echo "IMAGE=$IMAGE" >> $GITHUB_ENV
-          echo "IMAGE=${IMAGE}"
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21.4'
+      - name: Install ciux
+        run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
+      - name: Check dependencies consistency
+        run: ciux ignite $PWD
       - name: Build fink-broker image for k8s
         run: |
           ./build.sh
       - name: Export fink-broker image
         run: |
           docker images
-          echo "${{ env.IMAGE }}"
+          . "$CIUXCONFIG"
+          . ./conf.sh
           mkdir -p artifacts
-          docker save "${{ env.IMAGE }}" > artifacts/image.tar
-          echo "${{ env.IMAGE }}" > artifacts/image-tag
+          docker save "$IMAGE" > artifacts/image.tar
+          echo "$IMAGE" > artifacts/image-tag
       - uses: actions/upload-artifact@v2
         with:
           name: docker-artifact
@@ -49,21 +50,22 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Declare Version Variables
-        id: vars
-        shell: bash
-        run: |
-          echo "FINK_ALERT_SIMULATOR_DIR=$(. ./conf.sh && echo $FINK_ALERT_SIMULATOR_DIR)" >> $GITHUB_ENV
-      - name: Clone fink-alert-simulator code
-        run: ./itest/clone-fink-alert-simulator.sh
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21.4'
+      - name: Install ciux
+        run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
+      - name: Ciux project ignition
+        run: ciux ignite $PWD --itest --branch="$GHA_BRANCH_NAME"
       - name: Create k8s/kind cluster
         run: |
-          git clone --depth 1 -b "k8s-1.25.0" --single-branch https://github.com/k8s-school/kind-helper.git
-          ./kind-helper/k8s-create.sh -s
+          ktbx install kind
+          ktbx install kubectl
+          ktbx create -s
       - name: Load fink-alert-simulator image inside kind
         run: |
-          DIR="${{ env.FINK_ALERT_SIMULATOR_DIR }}"
-          .  "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/conf.sh
+          . "$CIUXCONFIG"
+          .  "$FINK_ALERT_SIMULATOR_DIR"/conf.sh
           if docker exec -t -- kind-control-plane crictl pull "$IMAGE"; then
             echo "::notice Succeeded to pull $IMAGE"
           else
@@ -81,7 +83,8 @@ jobs:
           docker exec -- kind-control-plane crictl image
       - name: Install fink-alert-simulator pre-requisites (argoCD)
         run: |
-          "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/prereq-install.sh
+          . "$CIUXCONFIG"
+          .  "$FINK_ALERT_SIMULATOR_DIR"/prereq-install.sh
       - name: Install strimzi (kafka-operator)
         run: ./itest/strimzi-install.sh
       - name: Setup kafka
@@ -90,14 +93,10 @@ jobs:
         run: ./itest/prereq-install.sh
       - name: Install MinIO
         run: ./itest/minio-install.sh
-      - uses: actions/setup-go@v4
-        with:
-          go-version: '1.19.2'
-      - name: Install finkctl
-        run: go install github.com/astrolabsoftware/finkctl@"${{ env.FINKCTL_VERSION }}"
       - name: Run fink-alert-simulator
         run: |
-          "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/argo-submit.sh
+          . "$CIUXCONFIG"
+          "$FINK_ALERT_SIMULATOR_DIR"/argo-submit.sh
           argo watch @latest
       # - name: Setup tmate session
       # uses: mxschmitt/action-tmate@v3

diff --git a/.github/workflows/itest.yml b/.github/workflows/itest.yml
@@ -16,7 +16,9 @@ on:
         required: true
 
 env:
-  FINKCTL_VERSION: v1.1.0-rc2
+  CIUXCONFIG: /tmp/ciux.sh
+  CIUX_VERSION: v0.0.1-rc8
+  GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   MINIMAL: ${{ inputs.minimal }}
   NOSCIENCE: ${{ inputs.noscience }}
   # Override the self-hosted runner value
@@ -34,13 +36,20 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21.4'
+      - name: Install ciux
+        run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
+      - name: Check dependencies consistency
+        run: ciux ignite $PWD
       - name: Build fink-broker image for k8s
         run: |
           ./build.sh
       - name: Push image to local registry
         id: push
         run: |
-          DIR=$(pwd)
+          . "$CIUXCONFIG"
           . ./conf.sh
           docker push $IMAGE
           echo "IMAGE=$IMAGE" >> "$GITHUB_OUTPUT"
@@ -54,19 +63,16 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Declare Version Variables
-        id: vars
-        shell: bash
-        run: |
-          DIR=$(pwd)
-          . ./conf.sh
-          echo "FINK_ALERT_SIMULATOR_DIR=$FINK_ALERT_SIMULATOR_DIR" >> $GITHUB_ENV
-          echo "IMAGE=$IMAGE" >> $GITHUB_ENV
-      - name: Clone fink-alert-simulator code
-        run: ./itest/clone-fink-alert-simulator.sh
-      - name: Create kind-helper configuration
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21.4'
+      - name: Install ciux
+        run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
+      - name: Ciux project ignition
+        run: ciux ignite $PWD --itest --branch="$GHA_BRANCH_NAME"
+      - name: Create ktbx configuration
         run: |
-          cat <<EOF > $HOME/.kind-helper
+          cat <<EOF > $HOME/.ktbx
           kind:
             # Supported only for clusters with one node
             # Use host directory to share data between host and kind node
@@ -89,13 +95,16 @@ jobs:
           EOF
       - name: Create k8s/kind cluster
         run: |
-          VERSION="v1.0.2-rc1"
-          curl -sfL https://raw.githubusercontent.com/k8s-school/kind-helper/$VERSION/install.sh | bash
-          kind-helper create
+          ktbx install kind
+          ktbx install kubectl
+          kind version
+          kubectl version --client
+          ktbx version
+          ktbx create -s
       - name: Load fink-alert-simulator image inside kind
         run: |
-          DIR="${{ env.FINK_ALERT_SIMULATOR_DIR }}"
-          .  "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/conf.sh
+          . "$CIUXCONFIG"
+          .  "$FINK_ALERT_SIMULATOR_DIR"/conf.sh
           if docker exec -t -- kind-control-plane crictl pull "$IMAGE"; then
             echo "::notice Succeeded to pull $IMAGE"
           else
@@ -112,7 +121,8 @@ jobs:
           fi
       - name: Install fink-alert-simulator pre-requisites (argo-workflows)
         run: |
-          "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/prereq-install.sh
+          . "$CIUXCONFIG"
+          "$FINK_ALERT_SIMULATOR_DIR"/prereq-install.sh
       - name: Install strimzi (kafka-operator)
         run: ./itest/strimzi-install.sh
       - name: Setup kafka
@@ -124,14 +134,10 @@ jobs:
           ./itest/prereq-install.sh
       - name: Install MinIO
         run: ./itest/minio-install.sh
-      - uses: actions/setup-go@v4
-        with:
-          go-version: '1.19.2'
-      - name: Install finkctl
-        run: go install github.com/astrolabsoftware/finkctl@"${{ env.FINKCTL_VERSION }}"
       - name: Run fink-alert-simulator
         run: |
-          "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/argo-submit.sh
+          . "$CIUXCONFIG"
+          "$FINK_ALERT_SIMULATOR_DIR"/argo-submit.sh
           argo watch @latest
       # - name: Setup tmate session
       # uses: mxschmitt/action-tmate@v3

diff --git a/conf.sh b/conf.sh
@@ -1,6 +1,5 @@
-# Set variable below to empty string in order to
-# build and install current development version
-FINK_BROKER_RELEASE=''
+
+. "$CIUXCONFIG"
 
 # Do not launch science pipeline if true
 NOSCIENCE="${NOSCIENCE:-false}"
@@ -15,7 +14,7 @@ MINIMAL="${MINIMAL:-false}"
 REPO="gitlab-registry.in2p3.fr/astrolabsoftware/fink"
 CI_REPO="${CI_REPO:-$REPO}"
 # Tag to apply to the built image, or to identify the image to be pushed
-TAG=${FINK_BROKER_RELEASE:-$(git -C $DIR describe --dirty --always)}
+TAG="$FINK_BROKER_VERSION"
 # WARNING "spark-py" is hard-coded in spark build script
 
 # Disable science pipeline
@@ -59,5 +58,3 @@ KAFKA_CLUSTER="kafka-cluster"
 # Default values are the ones set in fink-alert-simulator CI environment
 KAFKA_SOCKET=${KAFKA_SOCKET:-"kafka-cluster-kafka-external-bootstrap.kafka:9094"}
 KAFKA_TOPIC=${KAFKA_TOPIC:-"ztf-stream-sim"}
-
-FINK_ALERT_SIMULATOR_DIR="/tmp/fink-alert-simulator"
diff --git a/itest/check-results.sh b/itest/check-results.sh
@@ -26,5 +26,5 @@ DIR=$(cd "$(dirname "$0")"; pwd -P)
 
 . $DIR/../conf.sh
 
-finkctl wait topics --expected 10 --timeout 240s
+finkctl wait topics --expected 10 --timeout 600s -v1
 finkctl get topics
diff --git a/itest/clone-fink-alert-simulator.sh b/itest/clone-fink-alert-simulator.sh