diff --git a/.ciux b/.ciux index 585d96a1..3fcac83b 100644 --- a/.ciux +++ b/.ciux @@ -1,3 +1,4 @@ +apiVersion: v1alpha1 registry: gitlab-registry.in2p3.fr/astrolabsoftware/fink dependencies: - url: https://github.com/astrolabsoftware/fink-alert-simulator @@ -5,7 +6,19 @@ dependencies: clone: true # If true, container image will be pulled pull: true + labels: + itest: "true" + ci: "true" - url: https://github.com/astrolabsoftware/finkctl clone: true - - package: github.com/k8s-school/ktbx@v1.1.1-rc6 + labels: + itest: "true" + ci: "true" + - image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:k8s-3.4.1 + labels: + build: "true" + - package: github.com/k8s-school/ktbx@v1.1.1-rc7 + labels: + itest: "optional" + ci: "true" diff --git a/.github/workflows/itest-gha.yml b/.github/workflows/itest-gha.yml index 6097d49e..24a1b273 100644 --- a/.github/workflows/itest-gha.yml +++ b/.github/workflows/itest-gha.yml @@ -6,7 +6,7 @@ on: - master env: CIUXCONFIG: /tmp/ciux.sh - CIUX_VERSION: v0.0.1-rc8 + CIUX_VERSION: v0.0.1-rc11 GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} NOSCIENCE: true MINIMAL: true @@ -24,7 +24,7 @@ jobs: go-version: '1.21.4' - name: Install ciux run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" - - name: Check dependencies consistency + - name: Ciux project ignition run: ciux ignite $PWD - name: Build fink-broker image for k8s run: | @@ -32,7 +32,6 @@ jobs: - name: Export fink-broker image run: | docker images - . "$CIUXCONFIG" . ./conf.sh mkdir -p artifacts docker save "$IMAGE" > artifacts/image.tar @@ -56,7 +55,8 @@ jobs: - name: Install ciux run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" - name: Ciux project ignition - run: ciux ignite $PWD --itest --branch="$GHA_BRANCH_NAME" + run: | + ciux ignite --selector ci --branch="$GHA_BRANCH_NAME" $PWD - name: Create k8s/kind cluster run: | ktbx install kind diff --git a/.github/workflows/itest-noscience.yml b/.github/workflows/itest-noscience.yml index f2dcdca7..06ffa050 100644 --- a/.github/workflows/itest-noscience.yml +++ b/.github/workflows/itest-noscience.yml @@ -1,5 +1,6 @@ name: "Fink-broker e2e workflow (noscience, self-hosted)" on: + push: schedule: # At 03:00 UTC on every day-of-week from Monday through Friday. - cron: '0 3 * * 1-5' diff --git a/.github/workflows/itest.yml b/.github/workflows/itest.yml index 7d8ad60f..d053e852 100644 --- a/.github/workflows/itest.yml +++ b/.github/workflows/itest.yml @@ -14,10 +14,9 @@ on: required: true registry_token: required: true - env: CIUXCONFIG: /tmp/ciux.sh - CIUX_VERSION: v0.0.1-rc8 + CIUX_VERSION: v0.0.1-rc11 GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} MINIMAL: ${{ inputs.minimal }} NOSCIENCE: ${{ inputs.noscience }} @@ -41,8 +40,8 @@ jobs: go-version: '1.21.4' - name: Install ciux run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" - - name: Check dependencies consistency - run: ciux ignite $PWD + - name: Ciux project ignition + run: ciux ignite --selector build $PWD - name: Build fink-broker image for k8s run: | ./build.sh @@ -69,10 +68,11 @@ jobs: - name: Install ciux run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" - name: Ciux project ignition - run: ciux ignite $PWD --itest --branch="$GHA_BRANCH_NAME" + run: ciux ignite --selector ci --branch="$GHA_BRANCH_NAME" $PWD - name: Create ktbx configuration run: | - cat < $HOME/.ktbx + mkdir -p $HOME/.ktbx + cat < $HOME/.ktbx/config kind: # Supported only for clusters with one node # Use host directory to share data between host and kind node @@ -144,6 +144,9 @@ jobs: - name: Run fink-broker run: | ./itest/fink-start.sh + - name: Check ouput topics are created + run: | + ./itest/check-results.sh image-analysis: name: Analyze image runs-on: [self-hosted, v3] diff --git a/Dockerfile b/Dockerfile index 7500cd68..c587b1b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -ARG spark_image_tag -FROM gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:${spark_image_tag} as noscience +ARG spark_py_image +FROM ${spark_py_image} as noscience ARG spark_uid=185 ENV spark_uid ${spark_uid} @@ -75,4 +75,4 @@ FROM noscience AS full ADD deps/requirements-science.txt $FINK_HOME/ RUN pip install -r $FINK_HOME/requirements-science.txt ADD deps/requirements-science-no-deps.txt $FINK_HOME/ -RUN pip install -r $FINK_HOME/requirements-science-no-deps.txt --no-deps \ No newline at end of file +RUN pip install -r $FINK_HOME/requirements-science-no-deps.txt --no-deps diff --git a/TODO.org b/TODO.org index 2e1b59a6..5d2459bf 100644 --- a/TODO.org +++ b/TODO.org @@ -1,3 +1,4 @@ + * 729 ** DONE use "kubectl get kafkatopics.kafka.strimzi.io -n kafka" to check success of integration tests, maybe in fnkctl? ** TODO DELAYED BECAUSE IT NOT BLOCKS BUT WARN create topic in distribute before sending alerts in order to avoid error below: https://fink-broker.slack.com/archives/D03KJ390F17/p1692008729660549 diff --git a/build.sh b/build.sh index 31adee7b..36e43b27 100755 --- a/build.sh +++ b/build.sh @@ -22,8 +22,10 @@ set -euxo pipefail + DIR=$(cd "$(dirname "$0")"; pwd -P) +ciux ignite refresh "$DIR" . $DIR/conf.sh if $NOSCIENCE @@ -34,5 +36,5 @@ else fi # Build image -docker image build --tag "$IMAGE" --build-arg spark_image_tag="$SPARK_IMAGE_TAG" "$DIR" --target $TARGET +docker image build --tag "$IMAGE" --build-arg spark_py_image="$ASTROLABSOFTWARE_FINK_SPARK_PY_IMAGE" "$DIR" --target $TARGET diff --git a/conf.sh b/conf.sh index 67ffd715..3dbd293c 100755 --- a/conf.sh +++ b/conf.sh @@ -30,9 +30,8 @@ fi # Spark parameters # ---------------- -# Spark image tag -# Spark image is built here: https://github.com/astrolabsoftware/k8s-spark-py/ -SPARK_IMAGE_TAG="k8s-3.4.1" +# TODO remove and manage with ciux +SPARK_PY_IMAGE="gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:k8s-3.4.1" # Spark version SPARK_VERSION="3.4.1" diff --git a/deps/jars-urls.txt b/deps/jars-urls.txt index 44ae7b09..3944cb91 100644 --- a/deps/jars-urls.txt +++ b/deps/jars-urls.txt @@ -1,13 +1,13 @@ -https://repo1.maven.org/maven2/org/apache/spark/spark-streaming-kafka-0-10-assembly_2.12/3.2.3/spark-streaming-kafka-0-10-assembly_2.12-3.2.3.jar -https://repo1.maven.org/maven2/org/apache/spark/spark-sql-kafka-0-10_2.12/3.2.3/spark-sql-kafka-0-10_2.12-3.2.3.jar -https://repo1.maven.org/maven2/org/apache/spark/spark-avro_2.12/3.2.3/spark-avro_2.12-3.2.3.jar -https://repo1.maven.org/maven2/org/apache/spark/spark-token-provider-kafka-0-10_2.12/3.2.3/spark-token-provider-kafka-0-10_2.12-3.2.3.jar +https://repo1.maven.org/maven2/org/apache/spark/spark-streaming-kafka-0-10-assembly_2.12/3.4.1/spark-streaming-kafka-0-10-assembly_2.12-3.4.1.jar +https://repo1.maven.org/maven2/org/apache/spark/spark-sql-kafka-0-10_2.12/3.4.1/spark-sql-kafka-0-10_2.12-3.4.1.jar +https://repo1.maven.org/maven2/org/apache/spark/spark-avro_2.12/3.4.1/spark-avro_2.12-3.4.1.jar +https://repo1.maven.org/maven2/org/apache/spark/spark-token-provider-kafka-0-10_2.12/3.4.1/spark-token-provider-kafka-0-10_2.12-3.4.1.jar https://repo1.maven.org/maven2/org/apache/hbase/hbase-shaded-mapreduce/2.2.7/hbase-shaded-mapreduce-2.2.7.jar -https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.3/hadoop-aws-3.2.3.jar +https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.1/hadoop-aws-3.3.1.jar https://repo1.maven.org/maven2/org/spark-project/spark/unused/1.0.0/unused-1.0.0.jar https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/2.8.1/kafka-clients-2.8.1.jar https://repo1.maven.org/maven2/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar -https://repo1.maven.org/maven2/org/apache/commons/commons-pool2/2.6.2/commons-pool2-2.6.2.jar +https://repo1.maven.org/maven2/org/apache/commons/commons-pool2/2.11.1/commons-pool2-2.11.1.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-runtime/3.3.1/hadoop-client-runtime-3.3.1.jar https://repo1.maven.org/maven2/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar https://repo1.maven.org/maven2/org/xerial/snappy/snappy-java/1.1.8.4/snappy-java-1.1.8.4.jar