diff --git a/conf.sh b/conf.sh deleted file mode 100755 index dd93c761..00000000 --- a/conf.sh +++ /dev/null @@ -1,16 +0,0 @@ -# Spark parameters -# ---------------- - -# TODO manage spark inside container (or with ciux)? - -# Spark version -SPARK_VERSION="3.4.1" - -# Name for the Spark archive -SPARK_NAME="spark-${SPARK_VERSION}-bin-hadoop3" - -# Spark install location -SPARK_INSTALL_DIR="${HOME}/fink-k8s-tmp" - -export SPARK_HOME="${SPARK_INSTALL_DIR}/${SPARK_NAME}" -export PATH="$SPARK_HOME/bin:$PATH" diff --git a/e2e/argocd.sh b/e2e/argocd.sh index a5048154..fbcc7693 100755 --- a/e2e/argocd.sh +++ b/e2e/argocd.sh @@ -10,13 +10,12 @@ set -euxo pipefail DIR=$(cd "$(dirname "$0")"; pwd -P) CIUXCONFIG=${CIUXCONFIG:-"$HOME/.ciux/ciux.sh"} -echo "CIUXCONFIG=${CIUXCONFIG}" . $CIUXCONFIG function retry { local n=1 - local max=5 - local delay=5 + local max=10 + local delay=15 while true; do "$@" && break || { if [[ $n -lt $max ]]; then @@ -51,7 +50,7 @@ argocd app sync fink argocd app sync strimzi minio-operator spark-operator # TODO Try to make it simpler, try a sync-wave on Strimzi Application? -# see https://github.com/argoproj/argo-cd/discussions/16729argocd app set +# see https://github.com/argoproj/argo-cd/discussions/16729 # and https://stackoverflow.com/questions/77750481/argocd-app-of-apps-ensuring-strimzi-child-app-health-before-kafka-app-sync retry kubectl wait --for condition=established --timeout=60s crd/kafkas.kafka.strimzi.io \ crd/kafkatopics.kafka.strimzi.io \ @@ -63,7 +62,7 @@ retry kubectl wait --for condition=established --timeout=60s crd/kafkas.kafka.st # Set fink-broker parameters echo "Use fink-broker image: $CIUX_IMAGE_URL" -if [[ "$$CIUX_IMAGE_URL" =~ "-noscience" ]]; +if [[ "$CIUX_IMAGE_URL" =~ "-noscience" ]]; then valueFile=values-ci-noscience.yaml else @@ -80,5 +79,25 @@ argocd app set fink-broker -p image.repository="$CIUX_IMAGE_REGISTRY" \ argocd app sync -l app.kubernetes.io/instance=fink -# TODO Wait for kafkatopic to exist -retry kubectl wait --for condition=ready kafkatopics -n kafka ztf-stream-sim +kafka_topic="ztf-stream-sim" +echo "Wait for kafkatopic $kafka_topic to exist" +retry kubectl wait --for condition=ready kafkatopics -n kafka "$kafka_topic" + +# Check if kafka namespace exists, +# if yes, it means that the e2e tests are running +if kubectl get namespace kafka; then + echo "Retrieve kafka secrets for e2e tests" + if [[ "$CIUX_IMAGE_URL" =~ "-noscience" ]]; + then + FINKCONFIG="$DIR/finkconfig_noscience" + else + FINKCONFIG="$DIR/finkconfig" + fi + while ! kubectl get secret fink-producer --namespace kafka + do + echo "Waiting for secret/fink-producer in ns kafka" + sleep 10 + done + kubectl config set-context --current --namespace="spark" + finkctl createsecrets +fi diff --git a/e2e/fink-start.sh b/e2e/fink-start.sh index a69bd378..c2d811bd 100755 --- a/e2e/fink-start.sh +++ b/e2e/fink-start.sh @@ -24,8 +24,8 @@ set -euxo pipefail DIR=$(cd "$(dirname "$0")"; pwd -P) -# Used only to set path to spark-submit.sh -. $DIR/../conf.sh +CIUXCONFIG=${CIUXCONFIG:-"$HOME/.ciux/ciux.sh"} +. $CIUXCONFIG usage() { echo "Usage: $0 [-e] [-f finkconfig] [-i image]" @@ -42,26 +42,9 @@ done NS=spark -# Prepare e2e tests -. $CIUXCONFIG -IMAGE="$CIUX_IMAGE_URL" -echo "Use CIUX_IMAGE_URL to set fink-broker image: $CIUX_IMAGE_URL" -if [[ "$IMAGE" =~ "-noscience" ]]; -then - FINKCONFIG="$DIR/finkconfig_noscience" -else - FINKCONFIG="$DIR/finkconfig" -fi -kubectl config set-context --current --namespace="$NS" -echo "Create secrets" -while ! kubectl get secret fink-producer --namespace kafka -do - echo "Waiting for secret/fink-producer in ns kafka" - sleep 10 -done -finkctl createsecrets +kubectl config set-context --current --namespace="$NS" # Wait for Spark pods to be created and warm up # Debug in case of not expected behaviour