diff --git a/.github/workflows/itest-gha.yml b/.github/workflows/e2e-gha.yml similarity index 96% rename from .github/workflows/itest-gha.yml rename to .github/workflows/e2e-gha.yml index ea6e56bd..b8848b11 100644 --- a/.github/workflows/itest-gha.yml +++ b/.github/workflows/e2e-gha.yml @@ -9,7 +9,6 @@ env: CIUX_VERSION: v0.0.1-rc12 GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} NOSCIENCE: true - MINIMAL: true jobs: build: name: Build image @@ -68,7 +67,7 @@ jobs: ktbx install argocd - name: Run argoCD run: | - ./itest/argocd.sh + ./e2e/argocd.sh - name: Download image uses: actions/download-artifact@v3 with: @@ -93,13 +92,13 @@ jobs: run: | sudo apt-get -y update sudo apt-get -y install openjdk-8-jdk-headless - ./itest/prereq-install.sh + ./e2e/prereq-install.sh - name: Run fink-broker run: | - ./itest/fink-start.sh + ./e2e/fink-start.sh - name: Check results run: | - ./itest/check-results.sh + ./e2e/check-results.sh image-analysis: name: Analyze image runs-on: ubuntu-22.04 diff --git a/.github/workflows/itest-noscience.yml b/.github/workflows/e2e-noscience.yml similarity index 85% rename from .github/workflows/itest-noscience.yml rename to .github/workflows/e2e-noscience.yml index d9f2aa9a..e68755bf 100644 --- a/.github/workflows/itest-noscience.yml +++ b/.github/workflows/e2e-noscience.yml @@ -7,9 +7,8 @@ on: jobs: call-workflow-passing-data: - uses: ./.github/workflows/itest.yml + uses: ./.github/workflows/e2e.yml with: - minimal: true noscience: true secrets: registry_username: ${{ secrets.REGISTRY_USERNAME }} diff --git a/.github/workflows/itest-science.yml b/.github/workflows/e2e-science.yml similarity index 87% rename from .github/workflows/itest-science.yml rename to .github/workflows/e2e-science.yml index 529652e8..9c7cd9dd 100644 --- a/.github/workflows/itest-science.yml +++ b/.github/workflows/e2e-science.yml @@ -12,9 +12,8 @@ on: jobs: call-workflow-passing-data: - uses: ./.github/workflows/itest.yml + uses: ./.github/workflows/e2e.yml with: - minimal: false noscience: false secrets: registry_username: ${{ secrets.REGISTRY_USERNAME }} diff --git a/.github/workflows/itest.yml b/.github/workflows/e2e.yml similarity index 96% rename from .github/workflows/itest.yml rename to .github/workflows/e2e.yml index 3856fe28..36e81178 100644 --- a/.github/workflows/itest.yml +++ b/.github/workflows/e2e.yml @@ -3,9 +3,6 @@ name: "Reusable workflow for Fink self-hosted e2e tests" on: workflow_call: inputs: - minimal: - required: true - type: string noscience: required: true type: string @@ -18,7 +15,6 @@ env: CIUXCONFIG: /tmp/ciux.sh CIUX_VERSION: v0.0.1-rc12 GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - MINIMAL: ${{ inputs.minimal }} NOSCIENCE: ${{ inputs.noscience }} # Override the self-hosted runner value POD_NAMESPACE: default @@ -108,7 +104,7 @@ jobs: ktbx install argocd - name: Run argoCD run: | - ./itest/argocd.sh + ./e2e/argocd.sh - name: Load fink-broker image inside kind run: | IMAGE=${{ needs.build.outputs.image }} @@ -125,7 +121,7 @@ jobs: run: | sudo apt-get -y update sudo apt-get -y install openjdk-8-jdk-headless - ./itest/prereq-install.sh + ./e2e/prereq-install.sh - name: Run fink-alert-simulator run: | . "$CIUXCONFIG" @@ -135,10 +131,10 @@ jobs: # uses: mxschmitt/action-tmate@v3 - name: Run fink-broker run: | - ./itest/fink-start.sh + ./e2e/fink-start.sh - name: Check ouput topics are created run: | - ./itest/check-results.sh + ./e2e/check-results.sh image-analysis: name: Analyze image runs-on: [self-hosted, v3] diff --git a/TODO.org b/TODO.org index 49e7fc56..e0c34345 100644 --- a/TODO.org +++ b/TODO.org @@ -1,4 +1,4 @@ - +* DONE trigger ci for OOMkill * 729 ** DONE use "kubectl get kafkatopics.kafka.strimzi.io -n kafka" to check success of integration tests, maybe in fnkctl? ** TODO DELAYED BECAUSE IT NOT BLOCKS BUT WARN create topic in distribute before sending alerts in order to avoid error below: https://fink-broker.slack.com/archives/D03KJ390F17/p1692008729660549 diff --git a/conf.sh b/conf.sh index 3dbd293c..da4b99aa 100755 --- a/conf.sh +++ b/conf.sh @@ -2,9 +2,9 @@ . "$CIUXCONFIG" # Do not launch science pipeline if true +# Set minimal limits/requests for Spark driver and executor if true NOSCIENCE="${NOSCIENCE:-false}" -# Set minimal limits/requests for Spark driver and executor if true MINIMAL="${MINIMAL:-false}" diff --git a/itest/argocd.sh b/e2e/argocd.sh similarity index 100% rename from itest/argocd.sh rename to e2e/argocd.sh diff --git a/itest/check-results.sh b/e2e/check-results.sh similarity index 65% rename from itest/check-results.sh rename to e2e/check-results.sh index 521c9098..ab6cfc59 100755 --- a/itest/check-results.sh +++ b/e2e/check-results.sh @@ -26,5 +26,25 @@ DIR=$(cd "$(dirname "$0")"; pwd -P) . $DIR/../conf.sh -finkctl wait topics --expected 11 --timeout 600s -v1 +# TODO improve management of expected topics +# for example in the argo workflow job witch launch the alert simulator +if [ $NOSCIENCE = true ] +then + EXPECTED_TOPICS="11" +else + EXPECTED_TOPICS="1" +fi + +count=0 +while ! finkctl wait topics --expected "$EXPECTED_TOPICS" --timeout 60s -v1 +do + echo "Waiting for topics to be created" + sleep 5 + kubectl get pods + count=$((count+1)) + if [ $count -eq 10 ]; then + echo "Timeout waiting for topics to be created" + exit 1 + fi +done finkctl get topics diff --git a/itest/fink-start.sh b/e2e/fink-start.sh similarity index 86% rename from itest/fink-start.sh rename to e2e/fink-start.sh index 6477ecb8..431b9667 100755 --- a/itest/fink-start.sh +++ b/e2e/fink-start.sh @@ -37,7 +37,17 @@ echo "Create S3 bucket" kubectl port-forward -n minio svc/minio 9000 & # Wait to port-forward to start sleep 2 -export FINKCONFIG="$DIR" + +if [ "$NOSCIENCE" = true ]; +then + NOSCIENCE_OPT="--noscience" + export FINKCONFIG="$DIR/finkconfig_noscience" +else + NOSCIENCE_OPT="" + export FINKCONFIG="$DIR/finkconfig" +fi + + finkctl --endpoint=localhost:9000 s3 makebucket echo "Create spark ServiceAccount" @@ -48,28 +58,14 @@ NS=$(kubectl get sa -o=jsonpath='{.items[0]..metadata.namespace}') kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=$NS:spark \ --namespace=default --dry-run=client -o yaml | kubectl apply -f - -if [ "$NOSCIENCE" = true ]; -then - NOSCIENCE_OPT="--noscience" -else - NOSCIENCE_OPT="" -fi - -if [ "$MINIMAL" = true ]; -then - MINIMAL_OPT="--minimal" -else - MINIMAL_OPT="" -fi - tasks="stream2raw raw2science distribution" for task in $tasks; do - finkctl run $MINIMAL_OPT $NOSCIENCE_OPT $task --image $IMAGE >& "/tmp/$task.log" & + finkctl run $NOSCIENCE_OPT $task --image $IMAGE >& "/tmp/$task.log" & done # Wait for Spark pods to be created and warm up # Debug in case of not expected behaviour -if ! finkctl wait tasks --timeout=180s +if ! finkctl wait tasks --timeout=300s then for task in $tasks; do echo "--------- $task log file ---------" @@ -77,4 +73,9 @@ then done kubectl describe pods -l "spark-role in (executor, driver)" kubectl get pods + echo "ERROR: unable to start fink-broker" + exit 1 fi + +kubectl describe pods -l "spark-role in (executor, driver)" +kubectl get pods diff --git a/e2e/finkconfig/finkctl.secret.yaml b/e2e/finkconfig/finkctl.secret.yaml new file mode 120000 index 00000000..8d5963c3 --- /dev/null +++ b/e2e/finkconfig/finkctl.secret.yaml @@ -0,0 +1 @@ +../finkconfig_noscience/finkctl.secret.yaml \ No newline at end of file diff --git a/itest/finkctl.yaml b/e2e/finkconfig/finkctl.yaml similarity index 96% rename from itest/finkctl.yaml rename to e2e/finkconfig/finkctl.yaml index 5e89fd65..c366bd56 100644 --- a/itest/finkctl.yaml +++ b/e2e/finkconfig/finkctl.yaml @@ -1,3 +1,4 @@ +apiVersion: v1alpha1 s3: endpoint: http://minio.minio:9000 use_ssl: "false" @@ -17,6 +18,7 @@ stream2raw: kafka_starting_offset: earliest kafka_topic: ztf-stream-sim raw2science: + memory: 3000m night: "20200101" distribution: # Comma-separated list of kafka servers, default to stream2raw.kafka_socket diff --git a/itest/finkctl.secret.yaml b/e2e/finkconfig_noscience/finkctl.secret.yaml similarity index 100% rename from itest/finkctl.secret.yaml rename to e2e/finkconfig_noscience/finkctl.secret.yaml diff --git a/e2e/finkconfig_noscience/finkctl.yaml b/e2e/finkconfig_noscience/finkctl.yaml new file mode 100644 index 00000000..c5a192b9 --- /dev/null +++ b/e2e/finkconfig_noscience/finkctl.yaml @@ -0,0 +1,31 @@ +apiVersion: v1alpha1 +s3: + endpoint: http://minio.minio:9000 + use_ssl: "false" + bucket: fink-broker-online +# General parameter used to run fink tasks +run: + cpu: 0 + memory: 466m + fink_trigger_update: "2" + # Can be set using --image option + # image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/fink-broker:2.7.1-33-ge27a2aa-dirty + # Default to s3a:// + # online_data_prefix: s3a://fink-broker-online + producer: sims + log_level: INFO +stream2raw: + fink_alert_schema: /home/fink/fink-alert-schemas/ztf/ztf_public_20190903.schema.avro + kafka_socket: kafka-cluster-kafka-bootstrap.kafka:9092 + kafka_starting_offset: earliest + kafka_topic: ztf-stream-sim +raw2science: + night: "20200101" +distribution: + # Comma-separated list of kafka servers, default to stream2raw.kafka_socket + distribution_servers: "kafka-cluster-kafka-external-bootstrap.kafka:9094" + distribution_schema: "/home/fink/fink-alert-schemas/ztf/distribution_schema_0p2.avsc" + substream_prefix: "fink_" + # Default to + # night: "20200101" + diff --git a/itest/manifests/minio.yaml b/e2e/manifests/minio.yaml similarity index 100% rename from itest/manifests/minio.yaml rename to e2e/manifests/minio.yaml diff --git a/itest/minio-install.sh b/e2e/minio-install.sh similarity index 100% rename from itest/minio-install.sh rename to e2e/minio-install.sh diff --git a/itest/prereq-install.sh b/e2e/prereq-install.sh similarity index 100% rename from itest/prereq-install.sh rename to e2e/prereq-install.sh diff --git a/itest/run-minikube.sh b/e2e/run-minikube.sh similarity index 100% rename from itest/run-minikube.sh rename to e2e/run-minikube.sh diff --git a/itest/strimzi-install.sh b/e2e/strimzi-install.sh similarity index 100% rename from itest/strimzi-install.sh rename to e2e/strimzi-install.sh diff --git a/itest/strimzi-setup.sh b/e2e/strimzi-setup.sh similarity index 100% rename from itest/strimzi-setup.sh rename to e2e/strimzi-setup.sh