diff --git a/.ciux b/.ciux index 3fcac83b..b2f82ee2 100644 --- a/.ciux +++ b/.ciux @@ -7,17 +7,28 @@ dependencies: # If true, container image will be pulled pull: true labels: + dev: "true" itest: "true" ci: "true" - url: https://github.com/astrolabsoftware/finkctl clone: true labels: + dev: "true" itest: "true" ci: "true" + - url: https://github.com/astrolabsoftware/fink-cd + labels: + dev: "true" + itest: "true" + ci: "true" + - url: https://github.com/k8s-school/ktbx + clone: true + labels: + dev: "true" - image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:k8s-3.4.1 labels: build: "true" - - package: github.com/k8s-school/ktbx@v1.1.1-rc7 + - package: github.com/k8s-school/ktbx@v1.1.1-rc9 labels: itest: "optional" ci: "true" diff --git a/.github/workflows/itest-gha.yml b/.github/workflows/itest-gha.yml index 08727f51..ea6e56bd 100644 --- a/.github/workflows/itest-gha.yml +++ b/.github/workflows/itest-gha.yml @@ -6,7 +6,7 @@ on: - master env: CIUXCONFIG: /tmp/ciux.sh - CIUX_VERSION: v0.0.1-rc11 + CIUX_VERSION: v0.0.1-rc12 GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} NOSCIENCE: true MINIMAL: true @@ -57,21 +57,18 @@ jobs: - name: Ciux project ignition run: | ciux ignite --selector ci --branch="$GHA_BRANCH_NAME" $PWD - - name: Create k8s/kind cluster + - name: Create k8s (kind) cluster run: | ktbx install kind ktbx install kubectl ktbx create -s - - name: Load fink-alert-simulator image inside kind + - name: Install olm and argocd operators run: | - . "$CIUXCONFIG" - . "$FINK_ALERT_SIMULATOR_DIR"/conf.sh - if docker exec -t -- kind-control-plane crictl pull "$IMAGE"; then - echo "::notice Succeeded to pull $IMAGE" - else - echo "::error Failed to pull $IMAGE" - exit 1 - fi + ktbx install olm + ktbx install argocd + - name: Run argoCD + run: | + ./itest/argocd.sh - name: Download image uses: actions/download-artifact@v3 with: @@ -81,18 +78,10 @@ jobs: run: | kind load image-archive artifacts/image.tar docker exec -- kind-control-plane crictl image - - name: Install fink-alert-simulator pre-requisites (argoCD) + - name: Install fink-alert-simulator pre-requisites (argo-workflows) run: | . "$CIUXCONFIG" . "$FINK_ALERT_SIMULATOR_DIR"/prereq-install.sh - - name: Install strimzi (kafka-operator) - run: ./itest/strimzi-install.sh - - name: Setup kafka - run: ./itest/strimzi-setup.sh - - name: Install fink-broker pre-requisites (Spark) - run: ./itest/prereq-install.sh - - name: Install MinIO - run: ./itest/minio-install.sh - name: Run fink-alert-simulator run: | . "$CIUXCONFIG" @@ -100,6 +89,11 @@ jobs: argo watch @latest # - name: Setup tmate session # uses: mxschmitt/action-tmate@v3 + - name: Install fink-broker pre-requisites (JDK, Spark) + run: | + sudo apt-get -y update + sudo apt-get -y install openjdk-8-jdk-headless + ./itest/prereq-install.sh - name: Run fink-broker run: | ./itest/fink-start.sh diff --git a/.github/workflows/itest.yml b/.github/workflows/itest.yml index d053e852..c586f975 100644 --- a/.github/workflows/itest.yml +++ b/.github/workflows/itest.yml @@ -16,7 +16,7 @@ on: required: true env: CIUXCONFIG: /tmp/ciux.sh - CIUX_VERSION: v0.0.1-rc11 + CIUX_VERSION: v0.0.1-rc12 GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} MINIMAL: ${{ inputs.minimal }} NOSCIENCE: ${{ inputs.noscience }} @@ -95,7 +95,8 @@ jobs: EOF - name: Create k8s/kind cluster run: | - ktbx install kind + echo "Installing kind in version v0.15.0 because kind latest version is not supported by self-hosted runners" + ktbx install kind --kind-version="v0.15.0" ktbx install kubectl kind version kubectl version --client @@ -127,7 +128,7 @@ jobs: run: ./itest/strimzi-install.sh - name: Setup kafka run: ./itest/strimzi-setup.sh - - name: Install fink-broker pre-requisites (Spark) + - name: Install fink-broker pre-requisites (JDK, Spark) run: | sudo apt-get -y update sudo apt-get -y install openjdk-8-jdk-headless diff --git a/TODO.org b/TODO.org index 5d2459bf..49e7fc56 100644 --- a/TODO.org +++ b/TODO.org @@ -16,7 +16,7 @@ Tu crois qu'on pourrais pré-créer les topic pour éviter ce problème * TODO Enable authZ in kafka (require authN setup in fink-alert-simulator) * TODO [#B] distribute should wait for data to appear instead of crashing in connect_to_raw_database() * TODO move nodeport to internal for svc kafka-cluster-kafka-external-bootstrap -* TODO improve final test in CI (check Kafka with fink-client https://github.com/astrolabsoftware/fink-client) +* DONE improve final test in CI (check Kafka with fink-client https://github.com/astrolabsoftware/fink-client) * TODO run code-check.sh in CI * DONE add unit test for schema_converter * TODO https://stackoverflow.com/questions/30385981/how-to-access-s3a-files-from-apache-spark @@ -41,7 +41,7 @@ cf. Julien are they required? 5 curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o $HOME/minio-binaries/mc 6 chmod +x $HOME/minio-binaries/mc 15 export PATH=$PATH:$HOME/minio-binaries/ - 17 mc alias set s3 http://minio.minio-dev:9000 minioadmin minioadmin + 17 mc alias set s3 http://minio.minio:9000 minioadmin minioadmin 19 mc ls s3 27 mc mb s3/fink-broker-online mc ls f1 --recursive fink-broker-online/ diff --git a/itest/argocd.sh b/itest/argocd.sh new file mode 100755 index 00000000..a567d552 --- /dev/null +++ b/itest/argocd.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Install fink-broker stack (kafka+minio) +# Based on https://min.io/docs/minio/kubernetes/upstream/index.html + +# @author Fabrice Jammes + +set -euxo pipefail + +. $CIUXCONFIG + +function retry { + local n=1 + local max=5 + local delay=5 + while true; do + "$@" && break || { + if [[ $n -lt $max ]]; then + ((n++)) + echo "Command failed. Attempt $n/$max:" + sleep $delay; + else + echo "The command has failed after $n attempts." >&2 + exit 1 + fi + } + done +} + + +NS=argocd + +argocd login --core +kubectl config set-context --current --namespace="$NS" + +# Create fink app +argocd app create fink --dest-server https://kubernetes.default.svc \ + --dest-namespace "$NS" \ + --repo https://github.com/astrolabsoftware/fink-cd.git \ + --path apps --revision "$FINK_CD_WORKBRANCH" \ + +# Sync fink app-of-apps +argocd app sync fink + +# Synk fink apps +argocd app sync strimzi +# TODO Try to make it simpler, try a sync-wave on Strimzi Application? +# see https://github.com/argoproj/argo-cd/discussions/16729 +# and https://stackoverflow.com/questions/77750481/argocd-app-of-apps-ensuring-strimzi-child-app-health-before-kafka-app-sync +retry kubectl wait --for condition=established --timeout=60s crd/kafkas.kafka.strimzi.io crd/kafkatopics.kafka.strimzi.io +argocd app sync -l app.kubernetes.io/instance=fink diff --git a/itest/fink-start.sh b/itest/fink-start.sh index 33c19bf8..6477ecb8 100755 --- a/itest/fink-start.sh +++ b/itest/fink-start.sh @@ -28,8 +28,15 @@ DIR=$(cd "$(dirname "$0")"; pwd -P) echo $IMAGE +NS=spark +echo "Create $NS namespace" +kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - +kubectl config set-context --current --namespace="$NS" + echo "Create S3 bucket" -kubectl port-forward -n minio-dev svc/minio 9000 & +kubectl port-forward -n minio svc/minio 9000 & +# Wait to port-forward to start +sleep 2 export FINKCONFIG="$DIR" finkctl --endpoint=localhost:9000 s3 makebucket diff --git a/itest/finkctl.yaml b/itest/finkctl.yaml index 894d96bb..5e89fd65 100644 --- a/itest/finkctl.yaml +++ b/itest/finkctl.yaml @@ -1,5 +1,5 @@ s3: - endpoint: http://minio.minio-dev:9000 + endpoint: http://minio.minio:9000 use_ssl: "false" bucket: fink-broker-online # General parameter used to run fink tasks diff --git a/itest/manifests/minio.yaml b/itest/manifests/minio.yaml index 1413dd02..38436c22 100644 --- a/itest/manifests/minio.yaml +++ b/itest/manifests/minio.yaml @@ -2,13 +2,13 @@ apiVersion: v1 kind: Namespace metadata: - name: minio-dev + name: minio --- apiVersion: v1 kind: Service metadata: name: minio - namespace: minio-dev + namespace: minio spec: selector: app.kubernetes.io/name: minio @@ -19,8 +19,8 @@ spec: apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: minio-dev - namespace: minio-dev + name: minio + namespace: minio spec: accessModes: - ReadWriteOnce @@ -34,7 +34,7 @@ metadata: labels: app.kubernetes.io/name: minio name: minio - namespace: minio-dev + namespace: minio spec: replicas: 1 selector: @@ -59,4 +59,4 @@ spec: volumes: - name: data persistentVolumeClaim: - claimName: minio-dev + claimName: minio diff --git a/itest/minio-install.sh b/itest/minio-install.sh index 3618652f..0a077589 100755 --- a/itest/minio-install.sh +++ b/itest/minio-install.sh @@ -9,7 +9,7 @@ set -euxo pipefail DIR=$(cd "$(dirname "$0")"; pwd -P) -NS="minio-dev" +NS="minio" if kubectl apply -f $DIR/manifests/minio.yaml then @@ -19,4 +19,4 @@ else exit 1 fi -kubectl rollout status deployment minio -n "$NS" --timeout=90s \ No newline at end of file +kubectl rollout status deployment minio -n "$NS" --timeout=90s