865 incompatible pyarrow version causing issues with parquet file reading in fink science library

.ciux

@@ -1,6 +1,7 @@
 apiVersion: v1alpha1
 registry: gitlab-registry.in2p3.fr/astrolabsoftware/fink
 sourcePathes:
+  - Dockerfile
   - fink_broker
   - bin
   - deps

.github/workflows/e2e-common.yml

@@ -24,7 +24,6 @@ on:
       private_registry_token:
         required: true
 env:
-  CIUXCONFIG: /tmp/ciux.sh
   CIUX_VERSION: v0.0.4-rc2
   GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   SUFFIX: ${{ inputs.suffix }}
@@ -49,6 +48,7 @@ jobs:
         run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
       - name: Build fink-broker image for k8s
         run: |
+          mkdir -p $HOME/.ciux
           ./build.sh -s "${{ env.SUFFIX }}" -r "${{ env.CI_REPO }}"
       - name: Export fink-broker image
         id: export
@@ -89,6 +89,10 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+      # - name: Setup tmate session
+      #   uses: mxschmitt/action-tmate@v3
+      #   with:
+      #     detached: true
       - name: Maximize build space
         run: |
           echo "Removing unwanted software... "
@@ -110,6 +114,7 @@ jobs:
         run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
       - name: Ciux project ignition
         run: |
+          mkdir -p $HOME/.ciux
           ciux ignite --selector ci --branch="$GHA_BRANCH_NAME" $PWD --suffix "${{ env.SUFFIX }}" --tmp-registry "${{ env.CI_REPO }}"
       - name: Create k8s (kind) cluster
         run: |
@@ -137,7 +142,7 @@ jobs:
           path: artifacts
       - name: Load container image inside kind
         run: |
-          . "$CIUXCONFIG"
+          . "$HOME/.ciux/ciux.sh"
           if [ -f artifacts/image.tar ]; then
             echo "Loading image from archive"
             kind load image-archive artifacts/image.tar
@@ -147,10 +152,10 @@ jobs:
           fi
       - name: Run fink-alert-simulator
         run: |
+          export CIUXCONFIG="$HOME/.ciux/ciux.sh"
           . "$CIUXCONFIG"
-          kubectl config set-context --current --namespace=argo
           "$FINK_ALERT_SIMULATOR_DIR"/argo-submit.sh
-          argo watch @latest
+          argo watch -n argo @latest
       - name: Run fink-broker
         run: |
           ./e2e/fink-start.sh
@@ -160,7 +165,7 @@ jobs:
       - name: Promote fink-broker image
         id: promote
         run: |
-          . "$CIUXCONFIG"
+          . "$HOME/.ciux/ciux.sh"
           echo "PROMOTED_IMAGE=$CIUX_IMAGE_REGISTRY/$CIUX_IMAGE_NAME:$CIUX_IMAGE_TAG" >> "$GITHUB_OUTPUT"
           echo "NEW_IMAGE=$CIUX_BUILD" >> "$GITHUB_OUTPUT"
   image-analysis:

Dockerfile

@@ -66,13 +66,16 @@ ENV FINK_JARS ""
 ENV FINK_PACKAGES ""
 # pytest requirements
 ADD deps/requirements-test.txt $FINK_HOME/deps
-RUN pip install -r $FINK_HOME/deps/requirements-test.txt
+# Listing all requirements helps pip in computing a correct dependencies tree
+# See additional explanation in https://github.com/astrolabsoftware/fink-broker/issues/865
+RUN pip install -r $FINK_HOME/deps/requirements.txt -r $FINK_HOME/deps/requirements-test.txt
 
 ADD --chown=${spark_uid} . $FINK_HOME/
 
 FROM noscience AS full
 
 ADD deps/requirements-science.txt $FINK_HOME/
-RUN pip install -r $FINK_HOME/requirements-science.txt
+# Listing all requirements helps pip in computing a correct dependencies tree
+RUN pip install -r $FINK_HOME/deps/requirements.txt -r $FINK_HOME/deps/requirements-test.txt -r $FINK_HOME/requirements-science.txt
 ADD deps/requirements-science-no-deps.txt $FINK_HOME/
 RUN pip install -r $FINK_HOME/requirements-science-no-deps.txt --no-deps

bin/distribute.py

@@ -58,7 +58,9 @@ def main():
 
     # Initialise Spark session
     spark = init_sparksession(
-        name="distribute_{}_{}".format(args.producer, args.night), shuffle_partitions=2
+        name="distribute_{}_{}".format(args.producer, args.night),
+        shuffle_partitions=2,
+        log_level=args.spark_log_level,
     )
 
     # The level here should be controlled by an argument.

bin/raw2science.py

@@ -55,6 +55,7 @@ def main():
         name="raw2science_{}_{}".format(args.producer, args.night),
         shuffle_partitions=2,
         tz=tz,
+        log_level=args.spark_log_level,
     )
 
     # Logger to print useful debug statements

bin/stream2raw.py

@@ -60,7 +60,7 @@ def main():
         name="stream2raw_{}_{}".format(args.producer, args.night),
         shuffle_partitions=2,
         tz=tz,
-        log_level=args.log_level,
+        log_level=args.spark_log_level,
     )
 
     logger = init_logger(args.log_level)

build.sh

@@ -25,7 +25,7 @@ set -euxo pipefail
 DIR=$(cd "$(dirname "$0")"; pwd -P)
 
 # This will avoid overriding user ciuxconfig during a build
-export CIUXCONFIG=/tmp/ciux.build.sh
+export CIUXCONFIG=$HOME/.ciux/ciux.build.sh
 
 usage() {
   cat << EOD

chart/templates/spark-fink-raw2science.yaml

@@ -10,18 +10,18 @@ spec:
     - '{{ .Values.night }}'
   sparkConf: {{- include "fink.s3config" . | nindent 4 }}
   driver:
-    cores: {{ tpl .Values.distribution.cores . }}
-    coreRequest: "{{ tpl .Values.distribution.coreRequest . }}"
-    memory: "{{ tpl .Values.distribution.memory . }}"
+    cores: {{ tpl .Values.raw2science.cores . }}
+    coreRequest: "{{ tpl .Values.raw2science.coreRequest . }}"
+    memory: "{{ tpl .Values.raw2science.memory . }}"
     javaOptions: "-Divy.cache.dir=/tmp -Divy.home=/tmp -Dcom.amazonaws.sdk.disableCertChecking=true"
     labels:
       version: 3.4.1
     serviceAccount: spark
   executor:
-    cores: {{ tpl .Values.distribution.cores . }}
-    coreRequest: "{{ tpl .Values.distribution.coreRequest . }}"
-    memory: "{{ tpl .Values.distribution.memory . }}"
+    cores: {{ tpl .Values.raw2science.cores . }}
+    coreRequest: "{{ tpl .Values.raw2science.coreRequest . }}"
+    memory: "{{ tpl .Values.raw2science.memory . }}"
     javaOptions: "-Dcom.amazonaws.sdk.disableCertChecking=true"
-    instances: {{ tpl .Values.distribution.instances . }}
+    instances: {{ tpl .Values.raw2science.instances . }}
     labels:
       version: 3.4.1

chart/templates/spark-fink-stream2raw.yaml

@@ -18,16 +18,16 @@ spec:
   sparkConf: {{- include "fink.s3config" . | nindent 4 }}
   driver:
     cores: {{ tpl .Values.distribution.cores . }}
-    coreRequest: "{{ tpl .Values.distribution.coreRequest . }}"
-    memory: "{{ tpl .Values.distribution.memory . }}"
+    coreRequest: "{{ tpl .Values.stream2raw.coreRequest . }}"
+    memory: "{{ tpl .Values.stream2raw.memory . }}"
     labels:
       version: 3.4.1
     serviceAccount: spark
     javaOptions: "-Divy.cache.dir=/tmp -Divy.home=/tmp -Dcom.amazonaws.sdk.disableCertChecking=true"
   executor:
     cores: {{ tpl .Values.distribution.cores . }}
-    coreRequest: "{{ tpl .Values.distribution.coreRequest . }}"
-    memory: "{{ tpl .Values.distribution.memory . }}"
+    coreRequest: "{{ tpl .Values.stream2raw.coreRequest . }}"
+    memory: "{{ tpl .Values.stream2raw.memory . }}"
     instances: {{ tpl .Values.distribution.instances . }}
     javaOptions: "-Dcom.amazonaws.sdk.disableCertChecking=true"
     memory: "512m"

chart/values-ci-noscience.yaml

@@ -1,21 +1,53 @@
-# Default values for chart.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
+# Can be overriden in stream2raw, raw2science and distribution sections
+cores: 1
+coreRequest: 0
+instances: 1
+memory: 1g
+# instances: 1
 
-night: "20200101"
+fink_trigger_update: "2"
 
-image:
-  name: fink-broker-noscience
+# Can be overriden using --image option
+
+# Default to s3a://<s3.bucket>
+# online_data_prefix: s3a://fink-broker-online
+producer: sims
+
+log_level: INFO
 
 #
 # Parameters used to run the stream2raw task
 #
 stream2raw:
+  cores: "{{.Values.cores}}"
+  coreRequest: "{{.Values.coreRequest}}"
+  memory: "{{.Values.memory}}"
+  instances: "{{.Values.instances}}"
+  fink_alert_schema: /home/fink/fink-alert-schemas/ztf/ztf_public_20190903.schema.avro
   kafka:
-    topic: "ztf-stream-sim"
+    in_sockets: kafka-cluster-kafka-bootstrap.kafka:9092
+    starting_offset: earliest
+    topic: ztf-stream-sim
 
 #
-# Parameters used to access the S3 bucket
+# Parameters used to run the raw2science task
+#
+raw2science:
+  cores: "{{.Values.cores}}"
+  coreRequest: "{{.Values.coreRequest}}"
+  memory: "{{.Values.memory}}"
+  instances: "{{.Values.instances}}"
+
 #
-s3:
-  bucket: "fink-broker-online"
+# Parameters used to run the distribution task
+#
+distribution:
+  cores: "{{.Values.cores}}"
+  coreRequest: "{{.Values.coreRequest}}"
+  memory: "{{.Values.memory}}"
+  instances: "{{.Values.instances}}"
+  kafka:
+    out_sockets: "kafka-cluster-kafka-external-bootstrap.kafka:9094"
+    schema: "/home/fink/fink-alert-schemas/ztf/distribution_schema_0p2.avsc"
+    substream_prefix: "fink_"
+

chart/values-ci-science.yaml

@@ -0,0 +1,53 @@
+# Can be overriden in stream2raw, raw2science and distribution sections
+cores: 1
+coreRequest: 0
+instances: 1
+memory: 1g
+# instances: 1
+
+fink_trigger_update: "2"
+
+# Can be overriden using --image option
+
+# Default to s3a://<s3.bucket>
+# online_data_prefix: s3a://fink-broker-online
+producer: sims
+
+log_level: INFO
+
+#
+# Parameters used to run the stream2raw task
+#
+stream2raw:
+  cores: "{{.Values.cores}}"
+  coreRequest: "{{.Values.coreRequest}}"
+  memory: "{{.Values.memory}}"
+  instances: "{{.Values.instances}}"
+  fink_alert_schema: /home/fink/fink-alert-schemas/ztf/ztf_public_20190903.schema.avro
+  kafka:
+    in_sockets: kafka-cluster-kafka-bootstrap.kafka:9092
+    starting_offset: earliest
+    topic: ztf-stream-sim
+
+#
+# Parameters used to run the raw2science task
+#
+raw2science:
+  cores: "2"
+  coreRequest: "0"
+  memory: "3000m"
+  instances: "2"
+
+#
+# Parameters used to run the distribution task
+#
+distribution:
+  cores: "{{.Values.cores}}"
+  coreRequest: "{{.Values.coreRequest}}"
+  memory: "{{.Values.memory}}"
+  instances: "{{.Values.instances}}"
+  kafka:
+    out_sockets: "kafka-cluster-kafka-external-bootstrap.kafka:9094"
+    schema: "/home/fink/fink-alert-schemas/ztf/distribution_schema_0p2.avsc"
+    substream_prefix: "fink_"
+