Add support for fine-grained resource management

.github/workflows/itest-noscience.yml → .github/workflows/e2e-noscience.yaml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-workflow-passing-data:
-    uses: ./.github/workflows/itest.yml
+    uses: ./.github/workflows/e2e.yml
     with:
       minimal: true
       noscience: true

.github/workflows/itest-science.yml → .github/workflows/e2e-science.yaml

@@ -12,7 +12,7 @@ on:
 
 jobs:
   call-workflow-passing-data:
-    uses: ./.github/workflows/itest.yml
+    uses: ./.github/workflows/e2e.yml
     with:
       minimal: false
       noscience: false

.github/workflows/itest-gha.yml

@@ -68,7 +68,7 @@ jobs:
           ktbx install argocd
       - name: Run argoCD
         run: |
-          ./itest/argocd.sh
+          ./e2e/argocd.sh
       - name: Download image
         uses: actions/download-artifact@v3
         with:
@@ -93,13 +93,13 @@ jobs:
         run: |
           sudo apt-get -y update
           sudo apt-get -y install openjdk-8-jdk-headless
-          ./itest/prereq-install.sh
+          ./e2e/prereq-install.sh
       - name: Run fink-broker
         run: |
-          ./itest/fink-start.sh
+          ./e2e/fink-start.sh
       - name: Check results
         run: |
-          ./itest/check-results.sh
+          ./e2e/check-results.sh
   image-analysis:
     name: Analyze image
     runs-on: ubuntu-22.04

.github/workflows/itest.yml

@@ -108,7 +108,7 @@ jobs:
           ktbx install argocd
       - name: Run argoCD
         run: |
-          ./itest/argocd.sh
+          ./e2e/argocd.sh
       - name: Load fink-broker image inside kind
         run: |
           IMAGE=${{ needs.build.outputs.image }}
@@ -125,7 +125,7 @@ jobs:
         run: |
           sudo apt-get -y update
           sudo apt-get -y install openjdk-8-jdk-headless
-          ./itest/prereq-install.sh
+          ./e2e/prereq-install.sh
       - name: Run fink-alert-simulator
         run: |
           . "$CIUXCONFIG"
@@ -135,10 +135,10 @@ jobs:
       # uses: mxschmitt/action-tmate@v3
       - name: Run fink-broker
         run: |
-          ./itest/fink-start.sh
+          ./e2e/fink-start.sh
       - name: Check ouput topics are created
         run: |
-          ./itest/check-results.sh
+          ./e2e/check-results.sh
   image-analysis:
     name: Analyze image
     runs-on: [self-hosted, v3]

itest/fink-start.sh → e2e/fink-start.sh

@@ -37,7 +37,17 @@ echo "Create S3 bucket"
 kubectl port-forward -n minio svc/minio 9000 &
 # Wait to port-forward to start
 sleep 2
-export FINKCONFIG="$DIR"
+
+if [ "$NOSCIENCE" = true ];
+then
+  NOSCIENCE_OPT="--noscience"
+  export FINKCONFIG="$DIR/finkconfig_noscience"
+else
+  NOSCIENCE_OPT=""
+  export FINKCONFIG="$DIR/finkconfig"
+fi
+
+
 finkctl --endpoint=localhost:9000 s3 makebucket
 
 echo "Create spark ServiceAccount"
@@ -48,23 +58,9 @@ NS=$(kubectl get sa -o=jsonpath='{.items[0]..metadata.namespace}')
 kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=$NS:spark \
   --namespace=default --dry-run=client -o yaml | kubectl apply -f -
 
-if [ "$NOSCIENCE" = true ];
-then
-  NOSCIENCE_OPT="--noscience"
-else
-  NOSCIENCE_OPT=""
-fi
-
-if [ "$MINIMAL" = true ];
-then
-  MINIMAL_OPT="--minimal"
-else
-  MINIMAL_OPT=""
-fi
-
 tasks="stream2raw raw2science distribution"
 for task in $tasks; do
-  finkctl run $MINIMAL_OPT $NOSCIENCE_OPT $task --image $IMAGE >& "/tmp/$task.log" &
+  finkctl run $NOSCIENCE_OPT $task --image $IMAGE >& "/tmp/$task.log" &
 done
 
 # Wait for Spark pods to be created and warm up

e2e/finkconfig/finkctl.secret.yaml

@@ -0,0 +1 @@
+../finkconfig_noscience/finkctl.secret.yaml

itest/finkctl.yaml → e2e/finkconfig/finkctl.yaml

@@ -1,3 +1,4 @@
+apiVersion: v1alpha1
 s3:
   endpoint: http://minio.minio:9000
   use_ssl: "false"
@@ -19,6 +20,7 @@ stream2raw:
 raw2science:
   night: "20200101"
 distribution:
+  memory: 3000m
   # Comma-separated list of kafka servers, default to stream2raw.kafka_socket
   distribution_servers: "kafka-cluster-kafka-external-bootstrap.kafka:9094"
   distribution_schema: "/home/fink/fink-alert-schemas/ztf/distribution_schema_0p2.avsc"

e2e/finkconfig_noscience/finkctl.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1alpha1
+s3:
+  endpoint: http://minio.minio:9000
+  use_ssl: "false"
+  bucket: fink-broker-online
+# General parameter used to run fink tasks
+run:
+  cpu: 0
+  memory: 466m
+  fink_trigger_update: "2"
+  # Can be set using --image option
+  # image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/fink-broker:2.7.1-33-ge27a2aa-dirty
+  # Default to s3a://<s3.bucket>
+  # online_data_prefix: s3a://fink-broker-online
+  producer: sims
+  log_level: INFO
+stream2raw:
+  fink_alert_schema: /home/fink/fink-alert-schemas/ztf/ztf_public_20190903.schema.avro
+  kafka_socket: kafka-cluster-kafka-bootstrap.kafka:9092
+  kafka_starting_offset: earliest
+  kafka_topic: ztf-stream-sim
+raw2science:
+  night: "20200101"
+distribution:
+  # Comma-separated list of kafka servers, default to stream2raw.kafka_socket
+  distribution_servers: "kafka-cluster-kafka-external-bootstrap.kafka:9094"
+  distribution_schema: "/home/fink/fink-alert-schemas/ztf/distribution_schema_0p2.avsc"
+  substream_prefix: "fink_"
+  # Default to <stream2raw.night>
+  # night: "20200101"
+

e2e/finkctl.secret.yaml

@@ -0,0 +1,10 @@
+s3:
+  id: "minioadmin"
+  secret: "minioadmin"
+distribution:
+  kafka:
+    username: "fink-producer"
+    # If empty, password is set to "kubectl get -n kafka secrets/fink-producer --template={{.data.password}} | base64 --decode"
+    # this is used for integration tests and CI which use a local kafka cluster
+    password: ""
+

e2e/finkctl.yaml

@@ -0,0 +1,34 @@
+apiVersion: v1alpha1
+s3:
+  endpoint: http://minio.minio:9000
+  use_ssl: "false"
+  bucket: fink-broker-online
+# General parameter used to run fink tasks
+run:
+  cpu: 1
+  memory: 1Gi
+  fink_trigger_update: "2"
+  # Can be set using --image option
+  # image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/fink-broker:2.7.1-33-ge27a2aa-dirty
+  # Default to s3a://<s3.bucket>
+  # online_data_prefix: s3a://fink-broker-online
+  producer: sims
+  log_level: INFO
+stream2raw:
+  fink_alert_schema: /home/fink/fink-alert-schemas/ztf/ztf_public_20190903.schema.avro
+  kafka_socket: kafka-cluster-kafka-bootstrap.kafka:9092
+  kafka_starting_offset: earliest
+  kafka_topic: ztf-stream-sim
+  memory: 3Gi
+raw2science:
+  night: "20200101"
+distribution:
+  cpu: 2
+  memory: 2Gi
+  # Comma-separated list of kafka servers, default to stream2raw.kafka_socket
+  distribution_servers: "kafka-cluster-kafka-external-bootstrap.kafka:9094"
+  distribution_schema: "/home/fink/fink-alert-schemas/ztf/distribution_schema_0p2.avsc"
+  substream_prefix: "fink_"
+  # Default to <stream2raw.night>
+  # night: "20200101"
+