diff --git a/.ciux b/.ciux index 598f604e..d3b0776e 100644 --- a/.ciux +++ b/.ciux @@ -1,5 +1,9 @@ apiVersion: v1alpha1 registry: gitlab-registry.in2p3.fr/astrolabsoftware/fink +sourcePathes: + - fink_broker + - bin + - deps dependencies: - url: https://github.com/astrolabsoftware/fink-alert-simulator # If true repository will be locally cloned @@ -28,7 +32,7 @@ dependencies: - image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:k8s-3.4.1 labels: build: "true" - - package: github.com/k8s-school/ktbx@v1.1.1-rc11 + - package: github.com/k8s-school/ktbx@v1.1.1-rc17 labels: itest: "optional" ci: "true" diff --git a/.github/workflows/e2e-common.yml b/.github/workflows/e2e-common.yml new file mode 100644 index 00000000..e27bece2 --- /dev/null +++ b/.github/workflows/e2e-common.yml @@ -0,0 +1,242 @@ +name: "Reusable workflow for Fink self-hosted e2e tests" +on: + workflow_call: + inputs: + suffix: + required: true + type: string + ci_repo: + required: true + type: string + runner: + required: true + type: string + kind_version: + required: true + type: string + secrets: + registry_username: + required: true + registry_token: + required: true +env: + CIUXCONFIG: /tmp/ciux.sh + CIUX_VERSION: v0.0.1-rc13 + GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + SUFFIX: ${{ inputs.suffix }} + CI_REPO: ${{ inputs.ci_repo }} + # Override the self-hosted runner value + POD_NAMESPACE: default +jobs: + build: + name: Build image + runs-on: ${{ fromJSON(inputs.runner) }} + outputs: + image: ${{ steps.export.outputs.IMAGE }} + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-go@v4 + with: + go-version: '1.21.4' + - name: Install ciux + run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" + - name: Build fink-broker image for k8s + run: | + ./build.sh -s "${{ env.SUFFIX }}" -r "${{ env.CI_REPO }}" + - name: Export fink-broker image + id: export + run: | + # Cannot use CIUXCONFIG because it may not have been created yet + # TODO make it simpler! + $(ciux get image --check $PWD --suffix "${{ env.SUFFIX }}" --tmp-registry "${{ env.CI_REPO }}" --env) + mkdir -p artifacts + if [ $CIUX_BUILD = true ]; then + if [ -n "${{ env.CI_REPO }}" ]; then + echo "Pushing image $CIUX_IMAGE_URL to CI internal registry" + docker push $CIUX_IMAGE_URL + touch artifacts/empty + else + echo "Export $CIUX_IMAGE_URL to Github artifact store" + docker save "$CIUX_IMAGE_URL" > artifacts/image.tar + fi + else + echo "Using existing image $CIUX_IMAGE_URL" + touch artifacts/empty + fi + echo "IMAGE=$CIUX_IMAGE_URL" >> "$GITHUB_OUTPUT" + - uses: actions/upload-artifact@v2 + with: + name: docker-artifact + path: artifacts + integration-tests: + name: Run integration tests + runs-on: ${{ fromJSON(inputs.runner) }} + outputs: + new_image: ${{ steps.promote.outputs.NEW_IMAGE }} + promoted_image: ${{ steps.promote.outputs.PROMOTED_IMAGE }} + needs: build + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-go@v4 + with: + go-version: '1.21.4' + - name: Install ciux + run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" + - name: Ciux project ignition + run: | + ciux ignite --selector ci --branch="$GHA_BRANCH_NAME" $PWD --suffix "${{ env.SUFFIX }}" --tmp-registry "${{ env.CI_REPO }}" + - name: Create k8s (kind) cluster + run: | + # v0.20.0 does not work on self-hosted runners + ktbx install kind --kind-version=${{ inputs.kind_version }} + ktbx install kubectl + # Configure private registry if needed + ./e2e/kind-config.sh -r "${{ env.CI_REPO }}" + ktbx create -s + - name: Install olm and argocd operators + run: | + ktbx install olm + ktbx install argocd + - name: Install argo-workflows (fink-alert-simulator pre-requisite) + run: | + ktbx install argowf + - name: Run argoCD + run: | + ./e2e/argocd.sh + - name: Download image + uses: actions/download-artifact@v3 + with: + name: docker-artifact + path: artifacts + - name: Load container image inside kind + run: | + . "$CIUXCONFIG" + if [ -f artifacts/image.tar ]; then + echo "Loading image from archive" + kind load image-archive artifacts/image.tar + docker exec -- kind-control-plane crictl image + else + echo "Using existing image: $CIUX_IMAGE_URL" + fi + - name: Run fink-alert-simulator + run: | + . "$CIUXCONFIG" + "$FINK_ALERT_SIMULATOR_DIR"/argo-submit.sh + argo watch @latest + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + - name: Install fink-broker pre-requisites (JDK, Spark) + run: | + sudo apt-get -y update + sudo apt-get -y install openjdk-8-jdk-headless + ./e2e/prereq-install.sh + - name: Run fink-broker + run: | + ./e2e/fink-start.sh + - name: Check results + run: | + ./e2e/check-results.sh + - name: Promote fink-broker image + id: promote + run: | + . "$CIUXCONFIG" + echo "PROMOTED_IMAGE=$CIUX_IMAGE_REGISTRY/$CIUX_IMAGE_NAME/$FINKCTL_VERSION" >> "$GITHUB_OUTPUT" + echo "NEW_IMAGE=$CIUX_BUILD" >> "$GITHUB_OUTPUT" + image-analysis: + name: Analyze image + runs-on: ${{ fromJSON(inputs.runner) }} + permissions: + security-events: write + needs: build + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Download image + uses: actions/download-artifact@v3 + with: + name: docker-artifact + path: artifacts + - name: Load image in local registry + run: | + if [ -f artifacts/image.tar ]; then + echo "Loading image ${{ needs.build.outputs.image }} from archive" + docker load --input artifacts/image.tar + else + echo "Using existing image ${{ needs.build.outputs.image }}" + fi + - name: Scan fink-broker image + uses: anchore/scan-action@v3 + id: scan + with: + image: "${{ needs.build.outputs.image }}" + fail-build: false + - name: Display SARIF report + run: | + cat ${{ steps.scan.outputs.sarif }} + - name: upload Anchore scan SARIF report + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: ${{ steps.scan.outputs.sarif }} + push: + env: + NEW_IMAGE: ${{ needs.integration-tests.outputs.new_image }} + IMAGE: ${{ needs.build.outputs.image }} + PROMOTED_IMAGE: ${{ needs.integration-tests.outputs.promoted_image }} + name: Push fink-broker image to IN2P3 registry + runs-on: ${{ fromJSON(inputs.runner) }} + needs: [build, integration-tests] + steps: + - name: Download image + uses: actions/download-artifact@v3 + with: + name: docker-artifact + path: artifacts + - name: Load image in local registry + run: | + if [ $NEW_IMAGE = true ]; then + # GHA setup + if [ -f artifacts/image.tar ]; then + echo "Loading image "$IMAGE" from archive" + docker load --input artifacts/image.tar + # Self-hosted runners, new image is stored in the local registry + elif [ -n "$CI_REPO" ]; then + echo "Pulling image "$IMAGE" from $CI_REPO" + docker pull "$IMAGE" + else + echo "Error: no image found" + exit 1 + fi + else + echo "Using existing image $IMAGE" + fi + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + registry: gitlab-registry.in2p3.fr + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + - name: Push image to official registry + run: | + if [ $NEW_IMAGE = true ]; then + echo "Push image $PROMOTED_IMAGE" + docker tag "$IMAGE" "$PROMOTED_IMAGE" + docker push "$PROMOTED_IMAGE" + else + if which skopeo; then + echo "skopeo is already installed" + else + echo "Install skopeo" + sudo apt-get update -y + sudo apt-get install -y skopeo + fi + echo "Add image tag $PROMOTED_IMAGE to $IMAGE" + skopeo copy docker://$IMAGE docker://$PROMOTED_IMAGE + fi + + diff --git a/.github/workflows/e2e-gha.yml b/.github/workflows/e2e-gha.yml index b8848b11..b0fc0e10 100644 --- a/.github/workflows/e2e-gha.yml +++ b/.github/workflows/e2e-gha.yml @@ -4,152 +4,14 @@ on: pull_request: branches: - master -env: - CIUXCONFIG: /tmp/ciux.sh - CIUX_VERSION: v0.0.1-rc12 - GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - NOSCIENCE: true jobs: - build: - name: Build image - runs-on: ubuntu-22.04 - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - uses: actions/setup-go@v4 - with: - go-version: '1.21.4' - - name: Install ciux - run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" - - name: Ciux project ignition - run: ciux ignite $PWD - - name: Build fink-broker image for k8s - run: | - ./build.sh - - name: Export fink-broker image - run: | - docker images - . ./conf.sh - mkdir -p artifacts - docker save "$IMAGE" > artifacts/image.tar - echo "$IMAGE" > artifacts/image-tag - - uses: actions/upload-artifact@v2 - with: - name: docker-artifact - path: artifacts - integration-tests: - name: Run integration tests - runs-on: ubuntu-22.04 - needs: build - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - uses: actions/setup-go@v4 - with: - go-version: '1.21.4' - - name: Install ciux - run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" - - name: Ciux project ignition - run: | - ciux ignite --selector ci --branch="$GHA_BRANCH_NAME" $PWD - - name: Create k8s (kind) cluster - run: | - ktbx install kind - ktbx install kubectl - ktbx create -s - - name: Install olm and argocd operators - run: | - ktbx install olm - ktbx install argocd - - name: Run argoCD - run: | - ./e2e/argocd.sh - - name: Download image - uses: actions/download-artifact@v3 - with: - name: docker-artifact - path: artifacts - - name: Load container image inside kind - run: | - kind load image-archive artifacts/image.tar - docker exec -- kind-control-plane crictl image - - name: Install fink-alert-simulator pre-requisites (argo-workflows) - run: | - . "$CIUXCONFIG" - . "$FINK_ALERT_SIMULATOR_DIR"/prereq-install.sh - - name: Run fink-alert-simulator - run: | - . "$CIUXCONFIG" - "$FINK_ALERT_SIMULATOR_DIR"/argo-submit.sh - argo watch @latest - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - - name: Install fink-broker pre-requisites (JDK, Spark) - run: | - sudo apt-get -y update - sudo apt-get -y install openjdk-8-jdk-headless - ./e2e/prereq-install.sh - - name: Run fink-broker - run: | - ./e2e/fink-start.sh - - name: Check results - run: | - ./e2e/check-results.sh - image-analysis: - name: Analyze image - runs-on: ubuntu-22.04 - permissions: - security-events: write - needs: build - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Download image - uses: actions/download-artifact@v3 - with: - name: docker-artifact - path: artifacts - - name: Load image in local registry - run: | - docker load --input artifacts/image.tar - echo "IMAGE=$(cat artifacts/artifacts/image-tag)" >> $GITHUB_ENV - - name: Scan fink-broker image - uses: anchore/scan-action@v3 - id: scan - with: - image: "${{ env.IMAGE }}" - fail-build: false - - name: Display SARIF report - run: | - cat ${{ steps.scan.outputs.sarif }} - - name: upload Anchore scan SARIF report - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: ${{ steps.scan.outputs.sarif }} - push: - name: Push fink-broker image to IN2P3 registry - runs-on: ubuntu-22.04 - needs: integration-tests - steps: - - name: Download image - uses: actions/download-artifact@v3 - with: - name: docker-artifact - path: artifacts - - name: Load image in local registry - run: | - docker load --input artifacts/image.tar - echo "IMAGE=$(cat artifacts/image-tag)" >> $GITHUB_ENV - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - registry: gitlab-registry.in2p3.fr - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_TOKEN }} - - name: Push image to IN2P3 registry - run: | - docker push ${{ env.IMAGE }} + call-workflow-passing-data: + uses: ./.github/workflows/e2e-common.yml + with: + suffix: "noscience" + ci_repo: "" + runner: "['ubuntu-latest']" + kind_version: "v0.20.0" + secrets: + registry_username: ${{ secrets.REGISTRY_USERNAME }} + registry_token: ${{ secrets.REGISTRY_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/e2e-noscience.yml b/.github/workflows/e2e-noscience.yml index e68755bf..151ee01d 100644 --- a/.github/workflows/e2e-noscience.yml +++ b/.github/workflows/e2e-noscience.yml @@ -4,12 +4,14 @@ on: schedule: # At 03:00 UTC on every day-of-week from Monday through Friday. - cron: '0 3 * * 1-5' - jobs: call-workflow-passing-data: - uses: ./.github/workflows/e2e.yml + uses: ./.github/workflows/e2e-common.yml with: - noscience: true + suffix: "noscience" + ci_repo: "docker-registry.docker-registry:5000" + runner: "['arc-runners']" + kind_version: "v0.15.0" secrets: registry_username: ${{ secrets.REGISTRY_USERNAME }} registry_token: ${{ secrets.REGISTRY_TOKEN }} diff --git a/.github/workflows/e2e-science.yml b/.github/workflows/e2e-science.yml index 9c7cd9dd..1156623c 100644 --- a/.github/workflows/e2e-science.yml +++ b/.github/workflows/e2e-science.yml @@ -4,17 +4,20 @@ on: # At 05:00 UTC on every day-of-week from Monday through Friday. - cron: '0 5 * * 1-5' push: - branches: - - master - pull_request: - branches: - - master - + # ciux does not support pull_request when trying not to build image + # branches: + # - master + # pull_request: + # branches: + # - master jobs: call-workflow-passing-data: - uses: ./.github/workflows/e2e.yml + uses: ./.github/workflows/e2e-common.yml with: - noscience: false + suffix: "" + ci_repo: "docker-registry.docker-registry:5000" + runner: "['arc-runners']" + kind_version: "v0.15.0" secrets: registry_username: ${{ secrets.REGISTRY_USERNAME }} registry_token: ${{ secrets.REGISTRY_TOKEN }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml deleted file mode 100644 index 36e81178..00000000 --- a/.github/workflows/e2e.yml +++ /dev/null @@ -1,179 +0,0 @@ -name: "Reusable workflow for Fink self-hosted e2e tests" - -on: - workflow_call: - inputs: - noscience: - required: true - type: string - secrets: - registry_username: - required: true - registry_token: - required: true -env: - CIUXCONFIG: /tmp/ciux.sh - CIUX_VERSION: v0.0.1-rc12 - GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - NOSCIENCE: ${{ inputs.noscience }} - # Override the self-hosted runner value - POD_NAMESPACE: default - CI_REPO: docker-registry.docker-registry:5000 -jobs: - build: - name: Build image - runs-on: [self-hosted, v3] - outputs: - image: ${{ steps.push.outputs.IMAGE }} - promoted_image: ${{ steps.push.outputs.PROMOTED_IMAGE }} - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - uses: actions/setup-go@v4 - with: - go-version: '1.21.4' - - name: Install ciux - run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" - - name: Ciux project ignition - run: ciux ignite --selector build $PWD - - name: Build fink-broker image for k8s - run: | - ./build.sh - - name: Push image to local registry - id: push - run: | - . "$CIUXCONFIG" - . ./conf.sh - docker push $IMAGE - echo "IMAGE=$IMAGE" >> "$GITHUB_OUTPUT" - echo "PROMOTED_IMAGE=$PROMOTED_IMAGE" >> "$GITHUB_OUTPUT" - integration-tests: - name: Run integration tests - runs-on: [self-hosted, v3] - needs: build - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - uses: actions/setup-go@v4 - with: - go-version: '1.21.4' - - name: Install ciux - run: go install github.com/k8s-school/ciux@"${{ env.CIUX_VERSION }}" - - name: Ciux project ignition - run: ciux ignite --selector ci --branch="$GHA_BRANCH_NAME" $PWD - - name: Create ktbx configuration - run: | - mkdir -p $HOME/.ktbx - cat < $HOME/.ktbx/config - kind: - # Supported only for clusters with one node - # Use host directory to share data between host and kind node - # host directory will be mounted on /mnt/extra on each node - # extraMountPath: /path/to/host/directory - - # Sets "127.0.0.1" as an extra Subject Alternative Names (SANs) for the API Server signing certificate. - # See https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta3/#kubeadm-k8s-io-v1beta3-APIServer - # Usefull to access API server through a ssh tunnel - localcertsans: false - - # Use calico CNI instead of kindnet - # useCalico: true - # Number of worker nodes - workers: 0 - - # Supported only for clusters with one node - # Certificates must be available on kind host at "/etc/docker/certs.d/{{ .PrivateRegistry }}" - privateRegistry: "docker-registry.docker-registry:5000" - EOF - - name: Create k8s/kind cluster - run: | - echo "Installing kind in version v0.15.0 because kind latest version is not supported by self-hosted runners" - ktbx install kind --kind-version="v0.15.0" - ktbx install kubectl - kind version - kubectl version --client - ktbx version - ktbx create -s - - name: Install olm and argocd operators - run: | - ktbx install olm - ktbx install argocd - - name: Run argoCD - run: | - ./e2e/argocd.sh - - name: Load fink-broker image inside kind - run: | - IMAGE=${{ needs.build.outputs.image }} - if docker exec -t -- kind-control-plane crictl pull "$IMAGE"; then - echo "::notice Succeeded to pull $IMAGE" - else - echo "::error Failed to pull $IMAGE" - fi - - name: Install fink-alert-simulator pre-requisites (argo-workflows) - run: | - . "$CIUXCONFIG" - "$FINK_ALERT_SIMULATOR_DIR"/prereq-install.sh - - name: Install fink-broker pre-requisites (JDK, Spark) - run: | - sudo apt-get -y update - sudo apt-get -y install openjdk-8-jdk-headless - ./e2e/prereq-install.sh - - name: Run fink-alert-simulator - run: | - . "$CIUXCONFIG" - "$FINK_ALERT_SIMULATOR_DIR"/argo-submit.sh - argo watch @latest - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - - name: Run fink-broker - run: | - ./e2e/fink-start.sh - - name: Check ouput topics are created - run: | - ./e2e/check-results.sh - image-analysis: - name: Analyze image - runs-on: [self-hosted, v3] - permissions: - security-events: write - needs: build - steps: - - name: Pull image from local registry - run: | - IMAGE=${{ needs.build.outputs.image }} - docker pull "$IMAGE" - - name: Scan fink-broker image - uses: anchore/scan-action@v3 - id: scan - with: - image: "${{ needs.build.outputs.image }}" - fail-build: false - - name: Display SARIF report - run: | - cat ${{ steps.scan.outputs.sarif }} - - name: upload Anchore scan SARIF report - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: ${{ steps.scan.outputs.sarif }} - push: - name: Push fink-broker image to IN2P3 registry - runs-on: [self-hosted, v3] - needs: [build, integration-tests, image-analysis] - steps: - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - registry: gitlab-registry.in2p3.fr - username: ${{ secrets.registry_username }} - password: ${{ secrets.registry_token }} - - name: Push image to IN2P3 registry - run: | - IMAGE="${{ needs.build.outputs.image }}" - PROMOTED_IMAGE="${{ needs.build.outputs.promoted_image }}" - docker pull "$IMAGE" - docker tag "$IMAGE" "$PROMOTED_IMAGE" - docker push "$PROMOTED_IMAGE" diff --git a/bin/DEBUG-TMP b/bin/DEBUG-TMP new file mode 100644 index 00000000..e69de29b diff --git a/build.sh b/build.sh index 36e43b27..a4ff24ee 100755 --- a/build.sh +++ b/build.sh @@ -22,19 +22,54 @@ set -euxo pipefail - DIR=$(cd "$(dirname "$0")"; pwd -P) -ciux ignite refresh "$DIR" -. $DIR/conf.sh -if $NOSCIENCE +usage() { + cat << EOD + +Usage: `basename $0` [options] + + Available options: + -h this message + -s image suffix, default to none, only 'noscience' is supported + +Build image containing fink-broker for k8s +EOD +} + +suffix="" +tmp_registry="" + +# get the options +while getopts hr:s: c ; do + case $c in + h) usage ; exit 0 ;; + r) tmp_registry=$OPTARG ;; + s) suffix=$OPTARG ;; + \?) usage ; exit 2 ;; + esac +done +shift `expr $OPTIND - 1` + +# This command avoid retrieving build dependencies if not needed +$(ciux get image --check $DIR --suffix "$suffix" --tmp-registry "$tmp_registry" --env) + +if [ $CIUX_BUILD = false ]; then - TARGET=noscience + echo "Build cancelled, image $CIUX_IMAGE_URL already exists and contains current source code" + exit 0 +fi + +ciux ignite --selector build $DIR --suffix "$suffix" --tmp-registry "$tmp_registry" +. $DIR/conf.sh + +if [[ $suffix =~ ^noscience* ]]; then + target="noscience" else - TARGET=full + target="full" fi # Build image -docker image build --tag "$IMAGE" --build-arg spark_py_image="$ASTROLABSOFTWARE_FINK_SPARK_PY_IMAGE" "$DIR" --target $TARGET +docker image build --tag "$CIUX_IMAGE_URL" --build-arg spark_py_image="$ASTROLABSOFTWARE_FINK_SPARK_PY_IMAGE" "$DIR" --target $target diff --git a/conf.sh b/conf.sh index da4b99aa..98b99574 100755 --- a/conf.sh +++ b/conf.sh @@ -1,37 +1,10 @@ . "$CIUXCONFIG" -# Do not launch science pipeline if true -# Set minimal limits/requests for Spark driver and executor if true -NOSCIENCE="${NOSCIENCE:-false}" - -MINIMAL="${MINIMAL:-false}" - - -# Build parameters -# ---------------- -# Repository address -REPO="gitlab-registry.in2p3.fr/astrolabsoftware/fink" -CI_REPO="${CI_REPO:-$REPO}" -# Tag to apply to the built image, or to identify the image to be pushed -TAG="$FINK_BROKER_VERSION" -# WARNING "spark-py" is hard-coded in spark build script - -# Disable science pipeline -if [ "$NOSCIENCE" = true ]; -then - IMAGE="$CI_REPO/fink-broker-noscience:$TAG" - PROMOTED_IMAGE="$REPO/fink-broker-noscience:$TAG" -else - IMAGE="$CI_REPO/fink-broker:$TAG" - PROMOTED_IMAGE="$REPO/fink-broker:$TAG" -fi - # Spark parameters # ---------------- -# TODO remove and manage with ciux -SPARK_PY_IMAGE="gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:k8s-3.4.1" +# TODO manage spark inside container (or with ciux)? # Spark version SPARK_VERSION="3.4.1" @@ -44,16 +17,3 @@ SPARK_INSTALL_DIR="${HOME}/fink-k8s-tmp" export SPARK_HOME="${SPARK_INSTALL_DIR}/${SPARK_NAME}" export PATH="$SPARK_HOME/bin:$PATH" - -# Kafka cluster parameters -# ------------------------ -# Name for Kafka cluster -KAFKA_NS="kafka" -KAFKA_CLUSTER="kafka-cluster" - - -# Spark job 'stream2raw' parameters -# --------------------------------- -# Default values are the ones set in fink-alert-simulator CI environment -KAFKA_SOCKET=${KAFKA_SOCKET:-"kafka-cluster-kafka-external-bootstrap.kafka:9094"} -KAFKA_TOPIC=${KAFKA_TOPIC:-"ztf-stream-sim"} diff --git a/doc/devel.adoc b/doc/devel.adoc index be163cf5..156bf568 100644 --- a/doc/devel.adoc +++ b/doc/devel.adoc @@ -21,7 +21,6 @@ cd fink-broker [,shell] ---- # Work with minimal cpu/memory requirements and no science code -export MINIMAL=true export NOSCIENCE=true ---- diff --git a/e2e/check-results.sh b/e2e/check-results.sh index 8dc39cb2..382bb2bc 100755 --- a/e2e/check-results.sh +++ b/e2e/check-results.sh @@ -27,18 +27,18 @@ DIR=$(cd "$(dirname "$0")"; pwd -P) . $DIR/../conf.sh # TODO improve management of expected topics -# for example in the argo workflow job witch launch the alert simulator -if [ $NOSCIENCE = true ] +# for example in finkctl.yaml +if [ "$SUFFIX" = "noscience" ]; then - EXPECTED_TOPICS="11" + expected_topics="11" else - EXPECTED_TOPICS="1" + expected_topics="1" fi count=0 -while ! finkctl wait topics --expected "$EXPECTED_TOPICS" --timeout 60s -v1 +while ! finkctl wait topics --expected "$expected_topics" --timeout 60s -v1 do - echo "Waiting for expected topics: $EXPECTED_TOPICS" + echo "Waiting for expected topics: $expected_topics" sleep 5 kubectl get pods count=$((count+1)) diff --git a/e2e/fink-start.sh b/e2e/fink-start.sh index 2b29c0ee..135c7ae2 100755 --- a/e2e/fink-start.sh +++ b/e2e/fink-start.sh @@ -24,9 +24,18 @@ set -euxo pipefail DIR=$(cd "$(dirname "$0")"; pwd -P) +. $CIUXCONFIG + +# Used only to set path to spark-submit.sh . $DIR/../conf.sh -echo $IMAGE +if [ -n $CIUX_IMAGE_URL ]; +then + IMAGE="$CIUX_IMAGE_URL" +else + echo "ERROR: CIUX_IMAGE_URL is not set" + exit 1 +fi NS=spark echo "Create $NS namespace" @@ -38,7 +47,7 @@ kubectl port-forward -n minio svc/minio 9000 & # Wait to port-forward to start sleep 2 -if [ "$NOSCIENCE" = true ]; +if [[ "$IMAGE" =~ "-noscience" ]]; then NOSCIENCE_OPT="--noscience" export FINKCONFIG="$DIR/finkconfig_noscience" @@ -47,7 +56,6 @@ else export FINKCONFIG="$DIR/finkconfig" fi - finkctl --endpoint=localhost:9000 s3 makebucket echo "Create spark ServiceAccount" @@ -65,18 +73,36 @@ done # Wait for Spark pods to be created and warm up # Debug in case of not expected behaviour -timeout="300s" -if ! finkctl wait tasks --timeout="$timeout" -then - for task in $tasks; do - echo "--------- $task log file ---------" - cat "/tmp/$task.log" - done - kubectl describe pods -l "spark-role in (executor, driver)" - kubectl get pods - echo "ERROR: unable to start fink-broker in $timeout" - exit 1 -fi +# Science setup is VERY slow to start, because of raw2science-exec pod +timeout="600s" + +counter=0 +max_retries=3 +# Sometimes spark pods crashes and finktctl wait may fail +# even if Spark pod will be running after a while +# TODO implement the retry in "finkctl wait" +while ! finkctl wait tasks --timeout="$timeout"; do + if [ $counter -gt $max_retries ]; then + echo "Spark log files" + echo "---------------" + for task in $tasks; do + echo "--------- $task log file ---------" + cat "/tmp/$task.log" + done + echo "Pods description" + echo "----------------" + kubectl describe pods -l "spark-role in (executor, driver)" + kubectl get pods + echo "ERROR: unable to start fink-broker in $timeout" + # For interactive access for debugging purpose + sleep 7200 + exit 1 + fi + echo "ERROR: Spark pods are not running after $timeout, retry $counter/$max_retries" + sleep 60 + counter=$((counter+1)) +done kubectl describe pods -l "spark-role in (executor, driver)" kubectl get pods +echo "SUCCESS: fink-broker is running" diff --git a/e2e/finkconfig_noscience/finkctl.yaml b/e2e/finkconfig_noscience/finkctl.yaml index c5a192b9..e4ed91ee 100644 --- a/e2e/finkconfig_noscience/finkctl.yaml +++ b/e2e/finkconfig_noscience/finkctl.yaml @@ -6,7 +6,7 @@ s3: # General parameter used to run fink tasks run: cpu: 0 - memory: 466m + memory: 1000m fink_trigger_update: "2" # Can be set using --image option # image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/fink-broker:2.7.1-33-ge27a2aa-dirty diff --git a/e2e/strimzi-setup.sh b/e2e/kind-config.sh similarity index 56% rename from e2e/strimzi-setup.sh rename to e2e/kind-config.sh index 24bdf3bb..d7631d44 100755 --- a/e2e/strimzi-setup.sh +++ b/e2e/kind-config.sh @@ -16,7 +16,7 @@ # limitations under the License. # -# Setup Kafka for Fink +# Create docker image containing Fink packaged for k8s # @author Fabrice Jammes @@ -24,30 +24,29 @@ set -euxo pipefail DIR=$(cd "$(dirname "$0")"; pwd -P) -readonly FINKKUB=$(readlink -f "${DIR}/..") -. $FINKKUB/conf.sh - -cat << EOF | kubectl create -n $KAFKA_NS -f - -apiVersion: kafka.strimzi.io/v1beta2 -kind: KafkaUser -metadata: - name: fink-producer - labels: - strimzi.io/cluster: "$KAFKA_CLUSTER" -spec: - authentication: - type: scram-sha-512 -EOF - -cat << EOF | kubectl create -n $KAFKA_NS -f - -apiVersion: kafka.strimzi.io/v1beta2 -kind: KafkaTopic -metadata: - name: ztf-stream-sim - labels: - strimzi.io/cluster: "$KAFKA_CLUSTER" -spec: - partitions: 3 - replicas: 1 -EOF - +private_registry="" +# Get option -r for tmp-registry +while getopts ":r:" opt; do + case $opt in + r) private_registry="$OPTARG" + ;; + \?) echo "Invalid option -$OPTARG" >&2 + ;; + esac +done + +if [ -z "$private_registry" ]; then + echo "Option -r not set. Using default kind configuration" + exit 0 +fi + +echo "Using private registry: $private_registry" +mkdir -p $HOME/.ktbx +cat < $HOME/.ktbx/config +kind: + workers: 0 + + # Supported only for clusters with one node + # Certificates must be available on kind host at "/etc/docker/certs.d/{{ .PrivateRegistry }}" + privateRegistry: "$private_registry" +EOF \ No newline at end of file diff --git a/e2e/manifests/minio.yaml b/e2e/manifests/minio.yaml deleted file mode 100644 index 38436c22..00000000 --- a/e2e/manifests/minio.yaml +++ /dev/null @@ -1,62 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: minio ---- -apiVersion: v1 -kind: Service -metadata: - name: minio - namespace: minio -spec: - selector: - app.kubernetes.io/name: minio - ports: - - protocol: TCP - port: 9000 ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: minio - namespace: minio -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/name: minio - name: minio - namespace: minio -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: minio - template: - metadata: - labels: - app.kubernetes.io/name: minio - spec: - containers: - - name: minio - image: quay.io/minio/minio:latest - command: - - /bin/bash - - -c - args: - - minio server /data --console-address :9090 - volumeMounts: - - mountPath: /data - name: data - volumes: - - name: data - persistentVolumeClaim: - claimName: minio diff --git a/e2e/minio-install.sh b/e2e/minio-install.sh deleted file mode 100755 index 0a077589..00000000 --- a/e2e/minio-install.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Install MinIO inside k8s -# Based on https://min.io/docs/minio/kubernetes/upstream/index.html - -# @author Fabrice Jammes - -set -euxo pipefail - -DIR=$(cd "$(dirname "$0")"; pwd -P) - -NS="minio" - -if kubectl apply -f $DIR/manifests/minio.yaml -then - echo "Deploy minio" -else - >&2 echo "ERROR: No able to deploy minio" - exit 1 -fi - -kubectl rollout status deployment minio -n "$NS" --timeout=90s diff --git a/e2e/strimzi-install.sh b/e2e/strimzi-install.sh deleted file mode 100755 index f53d9394..00000000 --- a/e2e/strimzi-install.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Install Strimzi inside k8s - -# @author Fabrice Jammes - -set -euxo pipefail - -DIR=$(cd "$(dirname "$0")"; pwd -P) - -readonly FINKKUB=$(readlink -f "${DIR}/..") -. $FINKKUB/conf.sh - -# Strimzi version -readonly STRIMZI_VERSION="0.31.1" - -# Name for the Strimzi archive -readonly STRIMZI_NAME="strimzi-${STRIMZI_VERSION}" - -# Strimzi install location -readonly STRIMZI_INSTALL_DIR="${HOME}/strimzi-tmp" - -STRIMZI_HOME="${STRIMZI_INSTALL_DIR}/${STRIMZI_NAME}" - -mkdir -p "$STRIMZI_INSTALL_DIR" - -if [ ! -d "${STRIMZI_HOME}" ] -then - readonly STRIMZI_ARCHIVE="${STRIMZI_NAME}.tar.gz" - echo "Download and extract Strimzi ($STRIMZI_NAME)" - curl -Lo "${STRIMZI_INSTALL_DIR}/${STRIMZI_ARCHIVE}" "https://github.com/strimzi/strimzi-kafka-operator/releases/download/${STRIMZI_VERSION}/${STRIMZI_ARCHIVE}" - tar -C ${STRIMZI_INSTALL_DIR} -zxvf "${STRIMZI_INSTALL_DIR}/${STRIMZI_ARCHIVE}" -fi - -sed -i 's/namespace: .*/namespace: kafka/' "${STRIMZI_HOME}"/install/cluster-operator/*RoleBinding*.yaml - - -# Install operator and CR in the same namespace -kubectl create namespace "$KAFKA_NS" --dry-run=client -o yaml | kubectl apply -f - -kubectl apply -n "$KAFKA_NS" -f "${STRIMZI_HOME}"/install/cluster-operator/020-RoleBinding-strimzi-cluster-operator.yaml -kubectl apply -n "$KAFKA_NS" -f "${STRIMZI_HOME}"/install/cluster-operator/031-RoleBinding-strimzi-cluster-operator-entity-operator-delegation.yaml -kubectl apply -n "$KAFKA_NS" -f "${STRIMZI_HOME}"/install/cluster-operator/ - -cat << EOF | kubectl apply -n "$KAFKA_NS" -f - -apiVersion: kafka.strimzi.io/v1beta2 -kind: Kafka -metadata: - name: $KAFKA_CLUSTER -spec: - kafka: - replicas: 1 - listeners: - - name: plain - port: 9092 - type: internal - tls: false - - name: tls - port: 9093 - type: internal - tls: true - authentication: - type: tls - - name: external - port: 9094 - type: nodeport - tls: false - authentication: - type: scram-sha-512 - storage: - type: jbod - volumes: - - id: 0 - type: persistent-claim - size: 1Gi - deleteClaim: false - config: - offsets.topic.replication.factor: 1 - transaction.state.log.replication.factor: 1 - transaction.state.log.min.isr: 1 - default.replication.factor: 1 - min.insync.replicas: 1 - zookeeper: - replicas: 1 - storage: - type: persistent-claim - size: 1Gi - deleteClaim: false - entityOperator: - topicOperator: {} - userOperator: {} -EOF - -kubectl wait kafka/kafka-cluster --for=condition=Ready --timeout=300s -n "$KAFKA_NS" - - diff --git a/examples/aliases-fink.zsh b/examples/aliases-fink.zsh index 68bdc2ba..242f5bb3 100644 --- a/examples/aliases-fink.zsh +++ b/examples/aliases-fink.zsh @@ -15,13 +15,13 @@ FINK_BROKER_SRC_DIR="$ASTROLAB_DIR/fink-broker" FINK_ALERT_SIM_SRC_DIR="$ASTROLAB_DIR/fink-alert-simulator" FINKCTL_SRC_DIR="$ASTROLAB_DIR/finkctl" -export FINKCONFIG=$FINK_BROKER_SRC_DIR/itest +export FINKCONFIG=$FINK_BROKER_SRC_DIR/e2e/finkconfig_noscience alias cdfa="cd $FINK_ALERT_SIM_SRC_DIR" alias cdfb="cd $FINK_BROKER_SRC_DIR" alias cdfc="cd $FINKCTL_SRC_DIR" -alias fns="export MINIMAL=true NOSCIENCE=true" +alias fns="export NOSCIENCE=true" alias fbp="$FINK_BROKER_SRC_DIR/build.sh && $FINK_BROKER_SRC_DIR/push-image.sh" alias fadel="kubectl delete pod -l workflows.argoproj.io/completed" diff --git a/push-image.sh b/push-image.sh index 145dd796..22ea7f61 100755 --- a/push-image.sh +++ b/push-image.sh @@ -19,7 +19,7 @@ Usage: `basename $0` [options] path host [host ...] Available options: -h this message -k development mode: load image in kind - -d do not push image to remote registry + -d do not push image to remote registry Push image to remote registry and/or load it inside kind EOD @@ -45,8 +45,8 @@ if [ $# -ne 0 ] ; then fi if [ $kind = true ]; then - kind load docker-image "$IMAGE" + kind load docker-image "$CIUX_IMAGE_URL" fi if [ $registry = true ]; then - docker push "$IMAGE" + docker push "$CIUX_IMAGE_URL" fi