Improve e2e science, self-hosted test

Configure fine-grained resource management
Fix fink startup script error management
Use 3GB memory for raw2science
Configure number of expected topics for e2e test output

.github/workflows/itest-gha.yml → .github/workflows/e2e-gha.yml

@@ -9,7 +9,6 @@ env:
   CIUX_VERSION: v0.0.1-rc12
   GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   NOSCIENCE: true
-  MINIMAL: true
 jobs:
   build:
     name: Build image
@@ -68,7 +67,7 @@ jobs:
           ktbx install argocd
       - name: Run argoCD
         run: |
-          ./itest/argocd.sh
+          ./e2e/argocd.sh
       - name: Download image
         uses: actions/download-artifact@v3
         with:
@@ -93,13 +92,13 @@ jobs:
         run: |
           sudo apt-get -y update
           sudo apt-get -y install openjdk-8-jdk-headless
-          ./itest/prereq-install.sh
+          ./e2e/prereq-install.sh
       - name: Run fink-broker
         run: |
-          ./itest/fink-start.sh
+          ./e2e/fink-start.sh
       - name: Check results
         run: |
-          ./itest/check-results.sh
+          ./e2e/check-results.sh
   image-analysis:
     name: Analyze image
     runs-on: ubuntu-22.04

.github/workflows/itest-noscience.yml → .github/workflows/e2e-noscience.yml

@@ -7,9 +7,8 @@ on:
 
 jobs:
   call-workflow-passing-data:
-    uses: ./.github/workflows/itest.yml
+    uses: ./.github/workflows/e2e.yml
     with:
-      minimal: true
       noscience: true
     secrets:
       registry_username: ${{ secrets.REGISTRY_USERNAME }}

.github/workflows/itest-science.yml → .github/workflows/e2e-science.yml

@@ -12,9 +12,8 @@ on:
 
 jobs:
   call-workflow-passing-data:
-    uses: ./.github/workflows/itest.yml
+    uses: ./.github/workflows/e2e.yml
     with:
-      minimal: false
       noscience: false
     secrets:
       registry_username: ${{ secrets.REGISTRY_USERNAME }}

.github/workflows/itest.yml → .github/workflows/e2e.yml

@@ -3,9 +3,6 @@ name: "Reusable workflow for Fink self-hosted e2e tests"
 on:
   workflow_call:
     inputs:
-      minimal:
-        required: true
-        type: string
       noscience:
         required: true
         type: string
@@ -18,7 +15,6 @@ env:
   CIUXCONFIG: /tmp/ciux.sh
   CIUX_VERSION: v0.0.1-rc12
   GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
-  MINIMAL: ${{ inputs.minimal }}
   NOSCIENCE: ${{ inputs.noscience }}
   # Override the self-hosted runner value
   POD_NAMESPACE: default
@@ -108,7 +104,7 @@ jobs:
           ktbx install argocd
       - name: Run argoCD
         run: |
-          ./itest/argocd.sh
+          ./e2e/argocd.sh
       - name: Load fink-broker image inside kind
         run: |
           IMAGE=${{ needs.build.outputs.image }}
@@ -125,7 +121,7 @@ jobs:
         run: |
           sudo apt-get -y update
           sudo apt-get -y install openjdk-8-jdk-headless
-          ./itest/prereq-install.sh
+          ./e2e/prereq-install.sh
       - name: Run fink-alert-simulator
         run: |
           . "$CIUXCONFIG"
@@ -135,10 +131,10 @@ jobs:
       # uses: mxschmitt/action-tmate@v3
       - name: Run fink-broker
         run: |
-          ./itest/fink-start.sh
+          ./e2e/fink-start.sh
       - name: Check ouput topics are created
         run: |
-          ./itest/check-results.sh
+          ./e2e/check-results.sh
   image-analysis:
     name: Analyze image
     runs-on: [self-hosted, v3]

TODO.org

@@ -1,4 +1,4 @@
-
+* DONE trigger ci for OOMkill
 * 729
 ** DONE use "kubectl get kafkatopics.kafka.strimzi.io  -n kafka" to check success of integration tests, maybe in fnkctl?
 ** TODO DELAYED BECAUSE IT NOT BLOCKS BUT WARN create topic in distribute before sending alerts in order to avoid error below: https://fink-broker.slack.com/archives/D03KJ390F17/p1692008729660549

conf.sh

@@ -2,9 +2,9 @@
 . "$CIUXCONFIG"
 
 # Do not launch science pipeline if true
+# Set minimal limits/requests for Spark driver and executor if true
 NOSCIENCE="${NOSCIENCE:-false}"
 
-# Set minimal limits/requests for Spark driver and executor if true
 MINIMAL="${MINIMAL:-false}"
 
 

itest/check-results.sh → e2e/check-results.sh

@@ -26,5 +26,25 @@ DIR=$(cd "$(dirname "$0")"; pwd -P)
 
 . $DIR/../conf.sh
 
-finkctl wait topics --expected 11 --timeout 600s -v1
+# TODO improve management of expected topics
+# for example in the argo workflow job witch launch the alert simulator
+if [ $NOSCIENCE = true ]
+then
+  EXPECTED_TOPICS="11"
+else
+  EXPECTED_TOPICS="1"
+fi
+
+count=0
+while ! finkctl wait topics --expected "$EXPECTED_TOPICS" --timeout 60s -v1
+do
+    echo "Waiting for topics to be created"
+    sleep 5
+    kubectl get pods
+    count=$((count+1))
+    if [ $count -eq 10 ]; then
+        echo "Timeout waiting for topics to be created"
+        exit 1
+    fi
+done
 finkctl get topics

itest/fink-start.sh → e2e/fink-start.sh

@@ -37,7 +37,17 @@ echo "Create S3 bucket"
 kubectl port-forward -n minio svc/minio 9000 &
 # Wait to port-forward to start
 sleep 2
-export FINKCONFIG="$DIR"
+
+if [ "$NOSCIENCE" = true ];
+then
+  NOSCIENCE_OPT="--noscience"
+  export FINKCONFIG="$DIR/finkconfig_noscience"
+else
+  NOSCIENCE_OPT=""
+  export FINKCONFIG="$DIR/finkconfig"
+fi
+
+
 finkctl --endpoint=localhost:9000 s3 makebucket
 
 echo "Create spark ServiceAccount"
@@ -48,33 +58,24 @@ NS=$(kubectl get sa -o=jsonpath='{.items[0]..metadata.namespace}')
 kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=$NS:spark \
   --namespace=default --dry-run=client -o yaml | kubectl apply -f -
 
-if [ "$NOSCIENCE" = true ];
-then
-  NOSCIENCE_OPT="--noscience"
-else
-  NOSCIENCE_OPT=""
-fi
-
-if [ "$MINIMAL" = true ];
-then
-  MINIMAL_OPT="--minimal"
-else
-  MINIMAL_OPT=""
-fi
-
 tasks="stream2raw raw2science distribution"
 for task in $tasks; do
-  finkctl run $MINIMAL_OPT $NOSCIENCE_OPT $task --image $IMAGE >& "/tmp/$task.log" &
+  finkctl run $NOSCIENCE_OPT $task --image $IMAGE >& "/tmp/$task.log" &
 done
 
 # Wait for Spark pods to be created and warm up
 # Debug in case of not expected behaviour
-if ! finkctl wait tasks --timeout=180s
+if ! finkctl wait tasks --timeout=300s
 then
   for task in $tasks; do
     echo "--------- $task log file ---------"
     cat "/tmp/$task.log"
   done
   kubectl describe pods -l "spark-role in (executor, driver)"
   kubectl get pods
+  echo "ERROR: unable to start fink-broker"
+  exit 1
 fi
+
+kubectl describe pods -l "spark-role in (executor, driver)"
+kubectl get pods

e2e/finkconfig/finkctl.secret.yaml

@@ -0,0 +1 @@
+../finkconfig_noscience/finkctl.secret.yaml

itest/finkctl.yaml → e2e/finkconfig/finkctl.yaml

@@ -1,3 +1,4 @@
+apiVersion: v1alpha1
 s3:
   endpoint: http://minio.minio:9000
   use_ssl: "false"
@@ -17,6 +18,7 @@ stream2raw:
   kafka_starting_offset: earliest
   kafka_topic: ztf-stream-sim
 raw2science:
+  memory: 3000m
   night: "20200101"
 distribution:
   # Comma-separated list of kafka servers, default to stream2raw.kafka_socket

e2e/finkconfig_noscience/finkctl.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1alpha1
+s3:
+  endpoint: http://minio.minio:9000
+  use_ssl: "false"
+  bucket: fink-broker-online
+# General parameter used to run fink tasks
+run:
+  cpu: 0
+  memory: 466m
+  fink_trigger_update: "2"
+  # Can be set using --image option
+  # image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/fink-broker:2.7.1-33-ge27a2aa-dirty
+  # Default to s3a://<s3.bucket>
+  # online_data_prefix: s3a://fink-broker-online
+  producer: sims
+  log_level: INFO
+stream2raw:
+  fink_alert_schema: /home/fink/fink-alert-schemas/ztf/ztf_public_20190903.schema.avro
+  kafka_socket: kafka-cluster-kafka-bootstrap.kafka:9092
+  kafka_starting_offset: earliest
+  kafka_topic: ztf-stream-sim
+raw2science:
+  night: "20200101"
+distribution:
+  # Comma-separated list of kafka servers, default to stream2raw.kafka_socket
+  distribution_servers: "kafka-cluster-kafka-external-bootstrap.kafka:9094"
+  distribution_schema: "/home/fink/fink-alert-schemas/ztf/distribution_schema_0p2.avsc"
+  substream_prefix: "fink_"
+  # Default to <stream2raw.night>
+  # night: "20200101"
+