Upgrade fink-broker libraries

Downgrade kind
Update sparks libs to 3.4.1
Check output topics are created on self-hosted CI
Bump common-pool jar to 2-2.11.1
Manage spark_py image with ciux
Bump ciux to rc11
Use ciux labels

.ciux

@@ -1,11 +1,24 @@
+apiVersion: v1alpha1
 registry: gitlab-registry.in2p3.fr/astrolabsoftware/fink
 dependencies:
   - url: https://github.com/astrolabsoftware/fink-alert-simulator
     # If true repository will be locally cloned
     clone: true
     # If true, container image will be pulled
     pull: true
+    labels:
+      itest: "true"
+      ci: "true"
   - url: https://github.com/astrolabsoftware/finkctl
     clone: true
-  - package: github.com/k8s-school/[email protected]
+    labels:
+      itest: "true"
+      ci: "true"
+  - image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:k8s-3.4.1
+    labels:
+      build: "true"
+  - package: github.com/k8s-school/[email protected]
+    labels:
+      itest: "optional"
+      ci: "true"
 

.github/workflows/itest-gha.yml

@@ -6,7 +6,7 @@ on:
       - master
 env:
   CIUXCONFIG: /tmp/ciux.sh
-  CIUX_VERSION: v0.0.1-rc8
+  CIUX_VERSION: v0.0.1-rc11
   GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   NOSCIENCE: true
   MINIMAL: true
@@ -24,15 +24,14 @@ jobs:
           go-version: '1.21.4'
       - name: Install ciux
         run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
-      - name: Check dependencies consistency
+      - name: Ciux project ignition
         run: ciux ignite $PWD
       - name: Build fink-broker image for k8s
         run: |
           ./build.sh
       - name: Export fink-broker image
         run: |
           docker images
-          . "$CIUXCONFIG"
           . ./conf.sh
           mkdir -p artifacts
           docker save "$IMAGE" > artifacts/image.tar
@@ -56,7 +55,8 @@ jobs:
       - name: Install ciux
         run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
       - name: Ciux project ignition
-        run: ciux ignite $PWD --itest --branch="$GHA_BRANCH_NAME"
+        run: |
+          ciux ignite --selector ci --branch="$GHA_BRANCH_NAME" $PWD
       - name: Create k8s/kind cluster
         run: |
           ktbx install kind

.github/workflows/itest-noscience.yml

@@ -1,5 +1,6 @@
 name: "Fink-broker e2e workflow (noscience, self-hosted)"
 on:
+  push:
   schedule:
     # At 03:00 UTC on every day-of-week from Monday through Friday.
     - cron:  '0 3 * * 1-5'

.github/workflows/itest.yml

@@ -14,10 +14,9 @@ on:
         required: true
       registry_token:
         required: true
-
 env:
   CIUXCONFIG: /tmp/ciux.sh
-  CIUX_VERSION: v0.0.1-rc8
+  CIUX_VERSION: v0.0.1-rc11
   GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
   MINIMAL: ${{ inputs.minimal }}
   NOSCIENCE: ${{ inputs.noscience }}
@@ -41,8 +40,8 @@ jobs:
           go-version: '1.21.4'
       - name: Install ciux
         run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
-      - name: Check dependencies consistency
-        run: ciux ignite $PWD
+      - name: Ciux project ignition
+        run: ciux ignite --selector build $PWD
       - name: Build fink-broker image for k8s
         run: |
           ./build.sh
@@ -69,10 +68,11 @@ jobs:
       - name: Install ciux
         run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}"
       - name: Ciux project ignition
-        run: ciux ignite $PWD --itest --branch="$GHA_BRANCH_NAME"
+        run: ciux ignite --selector ci --branch="$GHA_BRANCH_NAME" $PWD
       - name: Create ktbx configuration
         run: |
-          cat <<EOF > $HOME/.ktbx
+          mkdir -p $HOME/.ktbx
+          cat <<EOF > $HOME/.ktbx/config
           kind:
             # Supported only for clusters with one node
             # Use host directory to share data between host and kind node
@@ -144,6 +144,9 @@ jobs:
       - name: Run fink-broker
         run: |
           ./itest/fink-start.sh
+      - name: Check ouput topics are created
+        run: |
+          ./itest/check-results.sh
   image-analysis:
     name: Analyze image
     runs-on: [self-hosted, v3]

Dockerfile

@@ -14,8 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-ARG spark_image_tag
-FROM gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:${spark_image_tag} as noscience
+ARG spark_py_image
+FROM ${spark_py_image} as noscience
 
 ARG spark_uid=185
 ENV spark_uid ${spark_uid}
@@ -75,4 +75,4 @@ FROM noscience AS full
 ADD deps/requirements-science.txt $FINK_HOME/
 RUN pip install -r $FINK_HOME/requirements-science.txt
 ADD deps/requirements-science-no-deps.txt $FINK_HOME/
-RUN pip install -r $FINK_HOME/requirements-science-no-deps.txt --no-deps
+RUN pip install -r $FINK_HOME/requirements-science-no-deps.txt --no-deps

TODO.org

@@ -1,3 +1,4 @@
+
 * 729
 ** DONE use "kubectl get kafkatopics.kafka.strimzi.io  -n kafka" to check success of integration tests, maybe in fnkctl?
 ** TODO DELAYED BECAUSE IT NOT BLOCKS BUT WARN create topic in distribute before sending alerts in order to avoid error below: https://fink-broker.slack.com/archives/D03KJ390F17/p1692008729660549

build.sh

@@ -22,8 +22,10 @@
 
 set -euxo pipefail
 
+
 DIR=$(cd "$(dirname "$0")"; pwd -P)
 
+ciux ignite refresh "$DIR"
 . $DIR/conf.sh
 
 if $NOSCIENCE
@@ -34,5 +36,5 @@ else
 fi
 
 # Build image
-docker image build --tag "$IMAGE" --build-arg spark_image_tag="$SPARK_IMAGE_TAG" "$DIR" --target $TARGET
+docker image build --tag "$IMAGE" --build-arg spark_py_image="$ASTROLABSOFTWARE_FINK_SPARK_PY_IMAGE" "$DIR" --target $TARGET
 

conf.sh

@@ -30,9 +30,8 @@ fi
 # Spark parameters
 # ----------------
 
-# Spark image tag
-# Spark image is built here: https://github.com/astrolabsoftware/k8s-spark-py/
-SPARK_IMAGE_TAG="k8s-3.4.1"
+# TODO remove and manage with ciux
+SPARK_PY_IMAGE="gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:k8s-3.4.1"
 
 # Spark version
 SPARK_VERSION="3.4.1"

deps/jars-urls.txt

@@ -1,13 +1,13 @@
-https://repo1.maven.org/maven2/org/apache/spark/spark-streaming-kafka-0-10-assembly_2.12/3.2.3/spark-streaming-kafka-0-10-assembly_2.12-3.2.3.jar
-https://repo1.maven.org/maven2/org/apache/spark/spark-sql-kafka-0-10_2.12/3.2.3/spark-sql-kafka-0-10_2.12-3.2.3.jar
-https://repo1.maven.org/maven2/org/apache/spark/spark-avro_2.12/3.2.3/spark-avro_2.12-3.2.3.jar
-https://repo1.maven.org/maven2/org/apache/spark/spark-token-provider-kafka-0-10_2.12/3.2.3/spark-token-provider-kafka-0-10_2.12-3.2.3.jar
+https://repo1.maven.org/maven2/org/apache/spark/spark-streaming-kafka-0-10-assembly_2.12/3.4.1/spark-streaming-kafka-0-10-assembly_2.12-3.4.1.jar
+https://repo1.maven.org/maven2/org/apache/spark/spark-sql-kafka-0-10_2.12/3.4.1/spark-sql-kafka-0-10_2.12-3.4.1.jar
+https://repo1.maven.org/maven2/org/apache/spark/spark-avro_2.12/3.4.1/spark-avro_2.12-3.4.1.jar
+https://repo1.maven.org/maven2/org/apache/spark/spark-token-provider-kafka-0-10_2.12/3.4.1/spark-token-provider-kafka-0-10_2.12-3.4.1.jar
 https://repo1.maven.org/maven2/org/apache/hbase/hbase-shaded-mapreduce/2.2.7/hbase-shaded-mapreduce-2.2.7.jar
-https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.3/hadoop-aws-3.2.3.jar
+https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.1/hadoop-aws-3.3.1.jar
 https://repo1.maven.org/maven2/org/spark-project/spark/unused/1.0.0/unused-1.0.0.jar
 https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/2.8.1/kafka-clients-2.8.1.jar
 https://repo1.maven.org/maven2/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar
-https://repo1.maven.org/maven2/org/apache/commons/commons-pool2/2.6.2/commons-pool2-2.6.2.jar
+https://repo1.maven.org/maven2/org/apache/commons/commons-pool2/2.11.1/commons-pool2-2.11.1.jar
 https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-runtime/3.3.1/hadoop-client-runtime-3.3.1.jar
 https://repo1.maven.org/maven2/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar
 https://repo1.maven.org/maven2/org/xerial/snappy/snappy-java/1.1.8.4/snappy-java-1.1.8.4.jar