diff --git a/.github/workflows/itest-noscience.yml b/.github/workflows/e2e-noscience.yaml similarity index 90% rename from .github/workflows/itest-noscience.yml rename to .github/workflows/e2e-noscience.yaml index d9f2aa9a..07050065 100644 --- a/.github/workflows/itest-noscience.yml +++ b/.github/workflows/e2e-noscience.yaml @@ -7,7 +7,7 @@ on: jobs: call-workflow-passing-data: - uses: ./.github/workflows/itest.yml + uses: ./.github/workflows/e2e.yml with: minimal: true noscience: true diff --git a/.github/workflows/itest-science.yml b/.github/workflows/e2e-science.yaml similarity index 91% rename from .github/workflows/itest-science.yml rename to .github/workflows/e2e-science.yaml index 529652e8..61ac8904 100644 --- a/.github/workflows/itest-science.yml +++ b/.github/workflows/e2e-science.yaml @@ -12,7 +12,7 @@ on: jobs: call-workflow-passing-data: - uses: ./.github/workflows/itest.yml + uses: ./.github/workflows/e2e.yml with: minimal: false noscience: false diff --git a/.github/workflows/itest-gha.yml b/.github/workflows/itest-gha.yml index ea6e56bd..a4c6ec6a 100644 --- a/.github/workflows/itest-gha.yml +++ b/.github/workflows/itest-gha.yml @@ -68,7 +68,7 @@ jobs: ktbx install argocd - name: Run argoCD run: | - ./itest/argocd.sh + ./e2e/argocd.sh - name: Download image uses: actions/download-artifact@v3 with: @@ -93,13 +93,13 @@ jobs: run: | sudo apt-get -y update sudo apt-get -y install openjdk-8-jdk-headless - ./itest/prereq-install.sh + ./e2e/prereq-install.sh - name: Run fink-broker run: | - ./itest/fink-start.sh + ./e2e/fink-start.sh - name: Check results run: | - ./itest/check-results.sh + ./e2e/check-results.sh image-analysis: name: Analyze image runs-on: ubuntu-22.04 diff --git a/.github/workflows/itest.yml b/.github/workflows/itest.yml index 3856fe28..e8260a37 100644 --- a/.github/workflows/itest.yml +++ b/.github/workflows/itest.yml @@ -108,7 +108,7 @@ jobs: ktbx install argocd - name: Run argoCD run: | - ./itest/argocd.sh + ./e2e/argocd.sh - name: Load fink-broker image inside kind run: | IMAGE=${{ needs.build.outputs.image }} @@ -125,7 +125,7 @@ jobs: run: | sudo apt-get -y update sudo apt-get -y install openjdk-8-jdk-headless - ./itest/prereq-install.sh + ./e2e/prereq-install.sh - name: Run fink-alert-simulator run: | . "$CIUXCONFIG" @@ -135,10 +135,10 @@ jobs: # uses: mxschmitt/action-tmate@v3 - name: Run fink-broker run: | - ./itest/fink-start.sh + ./e2e/fink-start.sh - name: Check ouput topics are created run: | - ./itest/check-results.sh + ./e2e/check-results.sh image-analysis: name: Analyze image runs-on: [self-hosted, v3] diff --git a/itest/argocd.sh b/e2e/argocd.sh similarity index 100% rename from itest/argocd.sh rename to e2e/argocd.sh diff --git a/itest/check-results.sh b/e2e/check-results.sh similarity index 100% rename from itest/check-results.sh rename to e2e/check-results.sh diff --git a/itest/fink-start.sh b/e2e/fink-start.sh similarity index 91% rename from itest/fink-start.sh rename to e2e/fink-start.sh index 6477ecb8..6201dd71 100755 --- a/itest/fink-start.sh +++ b/e2e/fink-start.sh @@ -37,7 +37,17 @@ echo "Create S3 bucket" kubectl port-forward -n minio svc/minio 9000 & # Wait to port-forward to start sleep 2 -export FINKCONFIG="$DIR" + +if [ "$NOSCIENCE" = true ]; +then + NOSCIENCE_OPT="--noscience" + export FINKCONFIG="$DIR/finkconfig_noscience" +else + NOSCIENCE_OPT="" + export FINKCONFIG="$DIR/finkconfig" +fi + + finkctl --endpoint=localhost:9000 s3 makebucket echo "Create spark ServiceAccount" @@ -48,23 +58,9 @@ NS=$(kubectl get sa -o=jsonpath='{.items[0]..metadata.namespace}') kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=$NS:spark \ --namespace=default --dry-run=client -o yaml | kubectl apply -f - -if [ "$NOSCIENCE" = true ]; -then - NOSCIENCE_OPT="--noscience" -else - NOSCIENCE_OPT="" -fi - -if [ "$MINIMAL" = true ]; -then - MINIMAL_OPT="--minimal" -else - MINIMAL_OPT="" -fi - tasks="stream2raw raw2science distribution" for task in $tasks; do - finkctl run $MINIMAL_OPT $NOSCIENCE_OPT $task --image $IMAGE >& "/tmp/$task.log" & + finkctl run $NOSCIENCE_OPT $task --image $IMAGE >& "/tmp/$task.log" & done # Wait for Spark pods to be created and warm up diff --git a/e2e/finkconfig/finkctl.secret.yaml b/e2e/finkconfig/finkctl.secret.yaml new file mode 120000 index 00000000..8d5963c3 --- /dev/null +++ b/e2e/finkconfig/finkctl.secret.yaml @@ -0,0 +1 @@ +../finkconfig_noscience/finkctl.secret.yaml \ No newline at end of file diff --git a/itest/finkctl.yaml b/e2e/finkconfig/finkctl.yaml similarity index 96% rename from itest/finkctl.yaml rename to e2e/finkconfig/finkctl.yaml index 5e89fd65..2cab7730 100644 --- a/itest/finkctl.yaml +++ b/e2e/finkconfig/finkctl.yaml @@ -1,3 +1,4 @@ +apiVersion: v1alpha1 s3: endpoint: http://minio.minio:9000 use_ssl: "false" @@ -19,6 +20,7 @@ stream2raw: raw2science: night: "20200101" distribution: + memory: 3000m # Comma-separated list of kafka servers, default to stream2raw.kafka_socket distribution_servers: "kafka-cluster-kafka-external-bootstrap.kafka:9094" distribution_schema: "/home/fink/fink-alert-schemas/ztf/distribution_schema_0p2.avsc" diff --git a/itest/finkctl.secret.yaml b/e2e/finkconfig_noscience/finkctl.secret.yaml similarity index 100% rename from itest/finkctl.secret.yaml rename to e2e/finkconfig_noscience/finkctl.secret.yaml diff --git a/e2e/finkconfig_noscience/finkctl.yaml b/e2e/finkconfig_noscience/finkctl.yaml new file mode 100644 index 00000000..c5a192b9 --- /dev/null +++ b/e2e/finkconfig_noscience/finkctl.yaml @@ -0,0 +1,31 @@ +apiVersion: v1alpha1 +s3: + endpoint: http://minio.minio:9000 + use_ssl: "false" + bucket: fink-broker-online +# General parameter used to run fink tasks +run: + cpu: 0 + memory: 466m + fink_trigger_update: "2" + # Can be set using --image option + # image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/fink-broker:2.7.1-33-ge27a2aa-dirty + # Default to s3a:// + # online_data_prefix: s3a://fink-broker-online + producer: sims + log_level: INFO +stream2raw: + fink_alert_schema: /home/fink/fink-alert-schemas/ztf/ztf_public_20190903.schema.avro + kafka_socket: kafka-cluster-kafka-bootstrap.kafka:9092 + kafka_starting_offset: earliest + kafka_topic: ztf-stream-sim +raw2science: + night: "20200101" +distribution: + # Comma-separated list of kafka servers, default to stream2raw.kafka_socket + distribution_servers: "kafka-cluster-kafka-external-bootstrap.kafka:9094" + distribution_schema: "/home/fink/fink-alert-schemas/ztf/distribution_schema_0p2.avsc" + substream_prefix: "fink_" + # Default to + # night: "20200101" + diff --git a/e2e/finkctl.secret.yaml b/e2e/finkctl.secret.yaml new file mode 100644 index 00000000..465e75a1 --- /dev/null +++ b/e2e/finkctl.secret.yaml @@ -0,0 +1,10 @@ +s3: + id: "minioadmin" + secret: "minioadmin" +distribution: + kafka: + username: "fink-producer" + # If empty, password is set to "kubectl get -n kafka secrets/fink-producer --template={{.data.password}} | base64 --decode" + # this is used for integration tests and CI which use a local kafka cluster + password: "" + diff --git a/e2e/finkctl.yaml b/e2e/finkctl.yaml new file mode 100644 index 00000000..aeb8e7fd --- /dev/null +++ b/e2e/finkctl.yaml @@ -0,0 +1,34 @@ +apiVersion: v1alpha1 +s3: + endpoint: http://minio.minio:9000 + use_ssl: "false" + bucket: fink-broker-online +# General parameter used to run fink tasks +run: + cpu: 1 + memory: 1Gi + fink_trigger_update: "2" + # Can be set using --image option + # image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/fink-broker:2.7.1-33-ge27a2aa-dirty + # Default to s3a:// + # online_data_prefix: s3a://fink-broker-online + producer: sims + log_level: INFO +stream2raw: + fink_alert_schema: /home/fink/fink-alert-schemas/ztf/ztf_public_20190903.schema.avro + kafka_socket: kafka-cluster-kafka-bootstrap.kafka:9092 + kafka_starting_offset: earliest + kafka_topic: ztf-stream-sim + memory: 3Gi +raw2science: + night: "20200101" +distribution: + cpu: 2 + memory: 2Gi + # Comma-separated list of kafka servers, default to stream2raw.kafka_socket + distribution_servers: "kafka-cluster-kafka-external-bootstrap.kafka:9094" + distribution_schema: "/home/fink/fink-alert-schemas/ztf/distribution_schema_0p2.avsc" + substream_prefix: "fink_" + # Default to + # night: "20200101" + diff --git a/itest/manifests/minio.yaml b/e2e/manifests/minio.yaml similarity index 100% rename from itest/manifests/minio.yaml rename to e2e/manifests/minio.yaml diff --git a/itest/minio-install.sh b/e2e/minio-install.sh similarity index 100% rename from itest/minio-install.sh rename to e2e/minio-install.sh diff --git a/itest/prereq-install.sh b/e2e/prereq-install.sh similarity index 100% rename from itest/prereq-install.sh rename to e2e/prereq-install.sh diff --git a/itest/run-minikube.sh b/e2e/run-minikube.sh similarity index 100% rename from itest/run-minikube.sh rename to e2e/run-minikube.sh diff --git a/itest/strimzi-install.sh b/e2e/strimzi-install.sh similarity index 100% rename from itest/strimzi-install.sh rename to e2e/strimzi-install.sh diff --git a/itest/strimzi-setup.sh b/e2e/strimzi-setup.sh similarity index 100% rename from itest/strimzi-setup.sh rename to e2e/strimzi-setup.sh