diff --git a/.ciux b/.ciux new file mode 100644 index 00000000..585d96a1 --- /dev/null +++ b/.ciux @@ -0,0 +1,11 @@ +registry: gitlab-registry.in2p3.fr/astrolabsoftware/fink +dependencies: + - url: https://github.com/astrolabsoftware/fink-alert-simulator + # If true repository will be locally cloned + clone: true + # If true, container image will be pulled + pull: true + - url: https://github.com/astrolabsoftware/finkctl + clone: true + - package: github.com/k8s-school/ktbx@v1.1.1-rc6 + diff --git a/.github/workflows/itest-gha.yml b/.github/workflows/itest-gha.yml index 10f550bf..6d08bc87 100644 --- a/.github/workflows/itest-gha.yml +++ b/.github/workflows/itest-gha.yml @@ -5,10 +5,10 @@ on: branches: - master env: - FINKCTL_VERSION: v1.1.0-rc2 + CIUXCONFIG: /tmp/ciux.sh + CIUX_VERSION: v0.0.1-rc7 NOSCIENCE: true MINIMAL: true - jobs: build: name: Build image @@ -18,24 +18,24 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Declare Version Variables - id: vars - shell: bash - run: | - DIR=$(pwd) - . ./conf.sh - echo "IMAGE=$IMAGE" >> $GITHUB_ENV - echo "IMAGE=${IMAGE}" + - uses: actions/setup-go@v4 + with: + go-version: '1.21.4' + - name: Install ciux + run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" + - name: Check dependencies consistency + run: ciux ignite $PWD - name: Build fink-broker image for k8s run: | ./build.sh - name: Export fink-broker image run: | docker images - echo "${{ env.IMAGE }}" + . "$CIUXCONFIG" + . ./conf.sh mkdir -p artifacts - docker save "${{ env.IMAGE }}" > artifacts/image.tar - echo "${{ env.IMAGE }}" > artifacts/image-tag + docker save "$IMAGE" > artifacts/image.tar + echo "$IMAGE" > artifacts/image-tag - uses: actions/upload-artifact@v2 with: name: docker-artifact @@ -49,21 +49,22 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Declare Version Variables - id: vars - shell: bash - run: | - echo "FINK_ALERT_SIMULATOR_DIR=$(. ./conf.sh && echo $FINK_ALERT_SIMULATOR_DIR)" >> $GITHUB_ENV - - name: Clone fink-alert-simulator code - run: ./itest/clone-fink-alert-simulator.sh + - uses: actions/setup-go@v4 + with: + go-version: '1.21.4' + - name: Install ciux + run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" + - name: Ciux project ignition + run: ciux ignite $PWD --itest - name: Create k8s/kind cluster run: | - git clone --depth 1 -b "k8s-1.25.0" --single-branch https://github.com/k8s-school/kind-helper.git - ./kind-helper/k8s-create.sh -s + ktbx install kind + ktbx install kubectl + ktbx create -s - name: Load fink-alert-simulator image inside kind run: | - DIR="${{ env.FINK_ALERT_SIMULATOR_DIR }}" - . "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/conf.sh + . "$CIUXCONFIG" + . "$FINK_ALERT_SIMULATOR_DIR"/conf.sh if docker exec -t -- kind-control-plane crictl pull "$IMAGE"; then echo "::notice Succeeded to pull $IMAGE" else @@ -81,7 +82,8 @@ jobs: docker exec -- kind-control-plane crictl image - name: Install fink-alert-simulator pre-requisites (argoCD) run: | - "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/prereq-install.sh + . "$CIUXCONFIG" + . "$FINK_ALERT_SIMULATOR_DIR"/prereq-install.sh - name: Install strimzi (kafka-operator) run: ./itest/strimzi-install.sh - name: Setup kafka @@ -90,14 +92,10 @@ jobs: run: ./itest/prereq-install.sh - name: Install MinIO run: ./itest/minio-install.sh - - uses: actions/setup-go@v4 - with: - go-version: '1.19.2' - - name: Install finkctl - run: go install github.com/astrolabsoftware/finkctl@"${{ env.FINKCTL_VERSION }}" - name: Run fink-alert-simulator run: | - "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/argo-submit.sh + . "$CIUXCONFIG" + "$FINK_ALERT_SIMULATOR_DIR"/argo-submit.sh argo watch @latest # - name: Setup tmate session # uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/itest.yml b/.github/workflows/itest.yml index 48793a32..bca3b981 100644 --- a/.github/workflows/itest.yml +++ b/.github/workflows/itest.yml @@ -16,7 +16,8 @@ on: required: true env: - FINKCTL_VERSION: v1.1.0-rc2 + CIUXCONFIG: /tmp/ciux.sh + CIUX_VERSION: v0.0.1-rc7 MINIMAL: ${{ inputs.minimal }} NOSCIENCE: ${{ inputs.noscience }} # Override the self-hosted runner value @@ -34,13 +35,20 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 + - uses: actions/setup-go@v4 + with: + go-version: '1.21.4' + - name: Install ciux + run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" + - name: Check dependencies consistency + run: ciux ignite $PWD - name: Build fink-broker image for k8s run: | ./build.sh - name: Push image to local registry id: push run: | - DIR=$(pwd) + . "$CIUXCONFIG" . ./conf.sh docker push $IMAGE echo "IMAGE=$IMAGE" >> "$GITHUB_OUTPUT" @@ -54,19 +62,18 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Declare Version Variables - id: vars - shell: bash - run: | - DIR=$(pwd) - . ./conf.sh - echo "FINK_ALERT_SIMULATOR_DIR=$FINK_ALERT_SIMULATOR_DIR" >> $GITHUB_ENV - echo "IMAGE=$IMAGE" >> $GITHUB_ENV + - uses: actions/setup-go@v4 + with: + go-version: '1.21.4' + - name: Install ciux + run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" + - name: Check dependencies consistency + run: ciux ignite $PWD --itest - name: Clone fink-alert-simulator code run: ./itest/clone-fink-alert-simulator.sh - - name: Create kind-helper configuration + - name: Create ktbx configuration run: | - cat < $HOME/.kind-helper + cat < $HOME/.ktbx kind: # Supported only for clusters with one node # Use host directory to share data between host and kind node @@ -89,13 +96,13 @@ jobs: EOF - name: Create k8s/kind cluster run: | - VERSION="v1.0.2-rc1" - curl -sfL https://raw.githubusercontent.com/k8s-school/kind-helper/$VERSION/install.sh | bash - kind-helper create + ktbx install kind + ktbx install kubectl + ktbx create -s - name: Load fink-alert-simulator image inside kind run: | - DIR="${{ env.FINK_ALERT_SIMULATOR_DIR }}" - . "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/conf.sh + . "$CIUXCONFIG" + . "$FINK_ALERT_SIMULATOR_DIR"/conf.sh if docker exec -t -- kind-control-plane crictl pull "$IMAGE"; then echo "::notice Succeeded to pull $IMAGE" else @@ -112,7 +119,8 @@ jobs: fi - name: Install fink-alert-simulator pre-requisites (argo-workflows) run: | - "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/prereq-install.sh + . "$CIUXCONFIG" + "$FINK_ALERT_SIMULATOR_DIR"/prereq-install.sh - name: Install strimzi (kafka-operator) run: ./itest/strimzi-install.sh - name: Setup kafka @@ -124,14 +132,14 @@ jobs: ./itest/prereq-install.sh - name: Install MinIO run: ./itest/minio-install.sh - - uses: actions/setup-go@v4 - with: - go-version: '1.19.2' - name: Install finkctl - run: go install github.com/astrolabsoftware/finkctl@"${{ env.FINKCTL_VERSION }}" + run: | + . "$CIUXCONFIG" + go install github.com/astrolabsoftware/finkctl@"$FINKCTL_VERSION" - name: Run fink-alert-simulator run: | - "${{ env.FINK_ALERT_SIMULATOR_DIR }}"/argo-submit.sh + . "$CIUXCONFIG" + "$FINK_ALERT_SIMULATOR_DIR"/argo-submit.sh argo watch @latest # - name: Setup tmate session # uses: mxschmitt/action-tmate@v3 diff --git a/conf.sh b/conf.sh index cf1d48ca..67ffd715 100755 --- a/conf.sh +++ b/conf.sh @@ -1,6 +1,5 @@ -# Set variable below to empty string in order to -# build and install current development version -FINK_BROKER_RELEASE='' + +. "$CIUXCONFIG" # Do not launch science pipeline if true NOSCIENCE="${NOSCIENCE:-false}" @@ -15,7 +14,7 @@ MINIMAL="${MINIMAL:-false}" REPO="gitlab-registry.in2p3.fr/astrolabsoftware/fink" CI_REPO="${CI_REPO:-$REPO}" # Tag to apply to the built image, or to identify the image to be pushed -TAG=${FINK_BROKER_RELEASE:-$(git -C $DIR describe --dirty --always)} +TAG="$FINK_BROKER_VERSION" # WARNING "spark-py" is hard-coded in spark build script # Disable science pipeline @@ -59,5 +58,3 @@ KAFKA_CLUSTER="kafka-cluster" # Default values are the ones set in fink-alert-simulator CI environment KAFKA_SOCKET=${KAFKA_SOCKET:-"kafka-cluster-kafka-external-bootstrap.kafka:9094"} KAFKA_TOPIC=${KAFKA_TOPIC:-"ztf-stream-sim"} - -FINK_ALERT_SIMULATOR_DIR="/tmp/fink-alert-simulator" diff --git a/itest/check-results.sh b/itest/check-results.sh index af1825dd..d236fca4 100755 --- a/itest/check-results.sh +++ b/itest/check-results.sh @@ -26,5 +26,5 @@ DIR=$(cd "$(dirname "$0")"; pwd -P) . $DIR/../conf.sh -finkctl wait topics --expected 10 --timeout 240s +finkctl wait topics --expected 10 --timeout 600s finkctl get topics diff --git a/itest/clone-fink-alert-simulator.sh b/itest/clone-fink-alert-simulator.sh deleted file mode 100755 index af05812f..00000000 --- a/itest/clone-fink-alert-simulator.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Clone fink-alert-simulator code - -# @author Fabrice Jammes SLAC/IN2P3 - -set -euxo pipefail - -DIR=$(cd "$(dirname "$0")"; pwd -P) -. $DIR/../conf.sh - -if [ -d "$FINK_ALERT_SIMULATOR_DIR" ]; then - rm -rf "$FINK_ALERT_SIMULATOR_DIR" -fi - -REPO_URL="https://github.com/astrolabsoftware/fink-alert-simulator" - -GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) -BRANCH=${GHA_BRANCH_NAME:-$GIT_BRANCH} -# Retrieve same fink-alert-simulator branch if it exists, else use latest version -if git ls-remote --exit-code --heads "$REPO_URL" "$BRANCH" -then - FINK_ALERT_SIMULATOR_VERSION="$BRANCH" -else - FINK_ALERT_SIMULATOR_VERSION="v0.2.2" -fi - -git clone "$REPO_URL" --branch "$FINK_ALERT_SIMULATOR_VERSION" \ - --single-branch "$FINK_ALERT_SIMULATOR_DIR"