Improve pip dependencies management

Add Dockerfile to ciux source pathes
Increase parameters management
Add separate log level for spark
Improve build script configuration

.ciux

@@ -1,6 +1,7 @@
 apiVersion: v1alpha1
 registry: gitlab-registry.in2p3.fr/astrolabsoftware/fink
 sourcePathes:
+  - Dockerfile
   - fink_broker
   - bin
   - deps

.github/workflows/e2e-common.yml

@@ -105,7 +105,7 @@ jobs:
           df -h
           sudo rm -rf /opt/hostedtoolcache/CodeQL
           df -h
-          sudo docker image prune --all --force
+          docker image prune --all --force
           df -h
       - uses: actions/setup-go@v4
         with:

Dockerfile

@@ -66,13 +66,16 @@ ENV FINK_JARS ""
 ENV FINK_PACKAGES ""
 # pytest requirements
 ADD deps/requirements-test.txt $FINK_HOME/deps
-RUN pip install -r $FINK_HOME/deps/requirements-test.txt
+# Listing all requirements helps pip in computing a correct dependencies tree
+# See additional explanation in https://github.com/astrolabsoftware/fink-broker/issues/865
+RUN pip install -r $FINK_HOME/deps/requirements.txt -r $FINK_HOME/deps/requirements-test.txt
 
 ADD --chown=${spark_uid} . $FINK_HOME/
 
 FROM noscience AS full
 
 ADD deps/requirements-science.txt $FINK_HOME/
-RUN pip install -r $FINK_HOME/requirements-science.txt
+# Listing all requirements helps pip in computing a correct dependencies tree
+RUN pip install -r $FINK_HOME/deps/requirements.txt -r $FINK_HOME/deps/requirements-test.txt -r $FINK_HOME/requirements-science.txt
 ADD deps/requirements-science-no-deps.txt $FINK_HOME/
 RUN pip install -r $FINK_HOME/requirements-science-no-deps.txt --no-deps

bin/distribute.py

@@ -58,7 +58,9 @@ def main():
 
     # Initialise Spark session
     spark = init_sparksession(
-        name="distribute_{}_{}".format(args.producer, args.night), shuffle_partitions=2
+        name="distribute_{}_{}".format(args.producer, args.night),
+        shuffle_partitions=2,
+        log_level=args.spark_log_level,
     )
 
     # The level here should be controlled by an argument.

bin/raw2science.py

@@ -55,6 +55,7 @@ def main():
         name="raw2science_{}_{}".format(args.producer, args.night),
         shuffle_partitions=2,
         tz=tz,
+        log_level=args.spark_log_level,
     )
 
     # Logger to print useful debug statements

bin/stream2raw.py

@@ -60,7 +60,7 @@ def main():
         name="stream2raw_{}_{}".format(args.producer, args.night),
         shuffle_partitions=2,
         tz=tz,
-        log_level=args.log_level,
+        log_level=args.spark_log_level,
     )
 
     logger = init_logger(args.log_level)

build.sh

@@ -25,7 +25,7 @@ set -euxo pipefail
 DIR=$(cd "$(dirname "$0")"; pwd -P)
 
 # This will avoid overriding user ciuxconfig during a build
-export CIUXCONFIG=/tmp/ciux.build.sh
+export CIUXCONFIG=$HOME/.ciux/ciux.build.sh
 
 usage() {
   cat << EOD

chart/templates/spark-fink-raw2science.yaml

@@ -10,18 +10,18 @@ spec:
     - '{{ .Values.night }}'
   sparkConf: {{- include "fink.s3config" . | nindent 4 }}
   driver:
-    cores: {{ tpl .Values.distribution.cores . }}
-    coreRequest: "{{ tpl .Values.distribution.coreRequest . }}"
-    memory: "{{ tpl .Values.distribution.memory . }}"
+    cores: {{ tpl .Values.raw2science.cores . }}
+    coreRequest: "{{ tpl .Values.raw2science.coreRequest . }}"
+    memory: "{{ tpl .Values.raw2science.memory . }}"
     javaOptions: "-Divy.cache.dir=/tmp -Divy.home=/tmp -Dcom.amazonaws.sdk.disableCertChecking=true"
     labels:
       version: 3.4.1
     serviceAccount: spark
   executor:
-    cores: {{ tpl .Values.distribution.cores . }}
-    coreRequest: "{{ tpl .Values.distribution.coreRequest . }}"
-    memory: "{{ tpl .Values.distribution.memory . }}"
+    cores: {{ tpl .Values.raw2science.cores . }}
+    coreRequest: "{{ tpl .Values.raw2science.coreRequest . }}"
+    memory: "{{ tpl .Values.raw2science.memory . }}"
     javaOptions: "-Dcom.amazonaws.sdk.disableCertChecking=true"
-    instances: {{ tpl .Values.distribution.instances . }}
+    instances: {{ tpl .Values.raw2science.instances . }}
     labels:
       version: 3.4.1

chart/templates/spark-fink-stream2raw.yaml

@@ -18,16 +18,16 @@ spec:
   sparkConf: {{- include "fink.s3config" . | nindent 4 }}
   driver:
     cores: {{ tpl .Values.distribution.cores . }}
-    coreRequest: "{{ tpl .Values.distribution.coreRequest . }}"
-    memory: "{{ tpl .Values.distribution.memory . }}"
+    coreRequest: "{{ tpl .Values.stream2raw.coreRequest . }}"
+    memory: "{{ tpl .Values.stream2raw.memory . }}"
     labels:
       version: 3.4.1
     serviceAccount: spark
     javaOptions: "-Divy.cache.dir=/tmp -Divy.home=/tmp -Dcom.amazonaws.sdk.disableCertChecking=true"
   executor:
     cores: {{ tpl .Values.distribution.cores . }}
-    coreRequest: "{{ tpl .Values.distribution.coreRequest . }}"
-    memory: "{{ tpl .Values.distribution.memory . }}"
+    coreRequest: "{{ tpl .Values.stream2raw.coreRequest . }}"
+    memory: "{{ tpl .Values.stream2raw.memory . }}"
     instances: {{ tpl .Values.distribution.instances . }}
     javaOptions: "-Dcom.amazonaws.sdk.disableCertChecking=true"
     memory: "512m"

chart/values-ci-noscience.yaml

@@ -1,21 +1,53 @@
-# Default values for chart.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
+# Can be overriden in stream2raw, raw2science and distribution sections
+cores: 1
+coreRequest: 0
+instances: 1
+memory: 1g
+# instances: 1
 
-night: "20200101"
+fink_trigger_update: "2"
 
-image:
-  name: fink-broker-noscience
+# Can be overriden using --image option
+
+# Default to s3a://<s3.bucket>
+# online_data_prefix: s3a://fink-broker-online
+producer: sims
+
+log_level: INFO
 
 #
 # Parameters used to run the stream2raw task
 #
 stream2raw:
+  cores: "{{.Values.cores}}"
+  coreRequest: "{{.Values.coreRequest}}"
+  memory: "{{.Values.memory}}"
+  instances: "{{.Values.instances}}"
+  fink_alert_schema: /home/fink/fink-alert-schemas/ztf/ztf_public_20190903.schema.avro
   kafka:
-    topic: "ztf-stream-sim"
+    in_sockets: kafka-cluster-kafka-bootstrap.kafka:9092
+    starting_offset: earliest
+    topic: ztf-stream-sim
 
 #
-# Parameters used to access the S3 bucket
+# Parameters used to run the raw2science task
+#
+raw2science:
+  cores: "{{.Values.cores}}"
+  coreRequest: "{{.Values.coreRequest}}"
+  memory: "{{.Values.memory}}"
+  instances: "{{.Values.instances}}"
+
 #
-s3:
-  bucket: "fink-broker-online"
+# Parameters used to run the distribution task
+#
+distribution:
+  cores: "{{.Values.cores}}"
+  coreRequest: "{{.Values.coreRequest}}"
+  memory: "{{.Values.memory}}"
+  instances: "{{.Values.instances}}"
+  kafka:
+    out_sockets: "kafka-cluster-kafka-external-bootstrap.kafka:9094"
+    schema: "/home/fink/fink-alert-schemas/ztf/distribution_schema_0p2.avsc"
+    substream_prefix: "fink_"
+

chart/values-ci-science.yaml

@@ -0,0 +1,53 @@
+# Can be overriden in stream2raw, raw2science and distribution sections
+cores: 1
+coreRequest: 0
+instances: 1
+memory: 1g
+# instances: 1
+
+fink_trigger_update: "2"
+
+# Can be overriden using --image option
+
+# Default to s3a://<s3.bucket>
+# online_data_prefix: s3a://fink-broker-online
+producer: sims
+
+log_level: INFO
+
+#
+# Parameters used to run the stream2raw task
+#
+stream2raw:
+  cores: "{{.Values.cores}}"
+  coreRequest: "{{.Values.coreRequest}}"
+  memory: "{{.Values.memory}}"
+  instances: "{{.Values.instances}}"
+  fink_alert_schema: /home/fink/fink-alert-schemas/ztf/ztf_public_20190903.schema.avro
+  kafka:
+    in_sockets: kafka-cluster-kafka-bootstrap.kafka:9092
+    starting_offset: earliest
+    topic: ztf-stream-sim
+
+#
+# Parameters used to run the raw2science task
+#
+raw2science:
+  cores: "2"
+  coreRequest: "0"
+  memory: "3000m"
+  instances: "2"
+
+#
+# Parameters used to run the distribution task
+#
+distribution:
+  cores: "{{.Values.cores}}"
+  coreRequest: "{{.Values.coreRequest}}"
+  memory: "{{.Values.memory}}"
+  instances: "{{.Values.instances}}"
+  kafka:
+    out_sockets: "kafka-cluster-kafka-external-bootstrap.kafka:9094"
+    schema: "/home/fink/fink-alert-schemas/ztf/distribution_schema_0p2.avsc"
+    substream_prefix: "fink_"
+

e2e/argocd.sh

@@ -7,7 +7,9 @@
 
 set -euxo pipefail
 
-CIUXCONFIG=${CIUXCONFIG:-"$HOME/.ciuxconfig"}
+DIR=$(cd "$(dirname "$0")"; pwd -P)
+
+CIUXCONFIG=${CIUXCONFIG:-"$HOME/.ciux/ciux.sh"}
 echo "CIUXCONFIG=${CIUXCONFIG}"
 . $CIUXCONFIG
 
@@ -35,10 +37,20 @@ argocd login --core
 kubectl config set-context --current --namespace="$NS"
 
 # Create fink app
+IMAGE="$CIUX_IMAGE_URL"
+echo "Use CIUX_IMAGE_URL to set fink-broker image: $CIUX_IMAGE_URL"
+if [[ "$IMAGE" =~ "-noscience" ]];
+then
+  valueFile=values-ci-noscience.yaml
+else
+  valueFile=values-ci-science.yaml
+fi
 argocd app create fink --dest-server https://kubernetes.default.svc \
     --dest-namespace "$NS" \
     --repo https://github.com/astrolabsoftware/fink-cd.git \
-    --path apps --revision "$FINK_CD_WORKBRANCH"
+    --path apps --revision "$FINK_CD_WORKBRANCH" \
+    -p finkbroker.revision="$FINK_BROKER_WORKBRANCH" \
+    -p finkbroker.valueFile="$valueFile" \
 
 # Sync fink app-of-apps
 argocd app sync fink
@@ -47,7 +59,7 @@ argocd app sync fink
 argocd app sync strimzi minio-operator spark-operator
 
 # TODO Try to make it simpler, try a sync-wave on Strimzi Application?
-# see https://github.com/argoproj/argo-cd/discussions/16729
+# see https://github.com/argoproj/argo-cd/discussions/16729argocd app set
 # and https://stackoverflow.com/questions/77750481/argocd-app-of-apps-ensuring-strimzi-child-app-health-before-kafka-app-sync
 retry kubectl wait --for condition=established --timeout=60s crd/kafkas.kafka.strimzi.io \
   crd/kafkatopics.kafka.strimzi.io \
@@ -61,7 +73,11 @@ retry kubectl wait --for condition=established --timeout=60s crd/kafkas.kafka.st
 argocd app set fink-broker -p image.repository="$CIUX_IMAGE_REGISTRY" \
     -p image.name="$CIUX_IMAGE_NAME" \
     -p image.tag="$CIUX_IMAGE_TAG" \
+    -p log_level="DEBUG" \
     -p night="20200101"
+# TODO pass parameters using a valuefile here, and not in 'argocd app create fink'
+# see https://argo-cd.readthedocs.io/en/stable/user-guide/commands/argocd_app_set/
+
 argocd app sync -l app.kubernetes.io/instance=fink
 
 # TODO Wait for kafkatopic to exist

e2e/fink-start.sh

@@ -48,10 +48,8 @@ IMAGE="$CIUX_IMAGE_URL"
 echo "Use CIUX_IMAGE_URL to set fink-broker image: $CIUX_IMAGE_URL"
 if [[ "$IMAGE" =~ "-noscience" ]];
 then
-  VALUE_FILE="$DIR/../chart/values-ci-noscience.yaml"
   FINKCONFIG="$DIR/finkconfig_noscience"
 else
-  VALUE_FILE="$DIR/../chart/values-ci.yaml"
   FINKCONFIG="$DIR/finkconfig"
 fi
 

fink_broker/parser.py

@@ -126,6 +126,15 @@ def getargs(parser: argparse.ArgumentParser) -> argparse.Namespace:
         [LOG_LEVEL]
         """,
     )
+    parser.add_argument(
+        "-spark_log_level",
+        type=str,
+        default="WARN",
+        help="""
+        The minimum level of log for the Spark framework: OFF, DEBUG, INFO, WARN, ERROR, CRITICAL
+        [LOG_LEVEL]
+        """,
+    )
     parser.add_argument(
         "-finkwebpath",
         type=str,

push-image.sh

@@ -7,6 +7,8 @@
 set -euxo pipefail
 
 DIR=$(cd "$(dirname "$0")"; pwd -P)
+
+export CIUXCONFIG=$HOME/.ciux/ciux.build.sh
 . "$CIUXCONFIG"
 
 set -e