From 98effdea305ddafc17ffff735101fafcb73c0b10 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 31 Jan 2024 12:26:03 +0100 Subject: [PATCH] Add support for GHA build WIP --- .github/workflows/e2e-gha.yml | 26 +++++++++++++++++--------- .github/workflows/e2e.yml | 1 - build.sh | 9 +++++---- conf.sh | 23 ----------------------- e2e/fink-start.sh | 13 ++++++++----- examples/aliases-fink.zsh | 2 +- 6 files changed, 31 insertions(+), 43 deletions(-) diff --git a/.github/workflows/e2e-gha.yml b/.github/workflows/e2e-gha.yml index 8936a70f..29d3f157 100644 --- a/.github/workflows/e2e-gha.yml +++ b/.github/workflows/e2e-gha.yml @@ -8,7 +8,7 @@ env: CIUXCONFIG: /tmp/ciux.sh CIUX_VERSION: 769944575a GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - NOSCIENCE: true + SUFFIX: noscience jobs: build: name: Build image @@ -27,14 +27,18 @@ jobs: run: ciux ignite $PWD - name: Build fink-broker image for k8s run: | - ./build.sh + ./build.sh --suffix "${{ env.SUFFIX }}" --tmp-registry "${{ env.CI_REPO }}" - name: Export fink-broker image run: | - docker images - . ./conf.sh + $(ciux get image --check $PWD --suffix "${{ env.SUFFIX }}" --env) mkdir -p artifacts - docker save "$IMAGE" > artifacts/image.tar - echo "$IMAGE" > artifacts/image-tag + if [ $CIUX_BUILD = true ]; then + echo "Export $CIUX_IMAGE_URL to archive" + docker save "$CIUX_IMAGE_URL" > artifacts/image.tar + else + echo "Using existing image $CIUX_IMAGE_URL" + fi + echo "IMAGE=$CIUX_IMAGE_URL" >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@v2 with: name: docker-artifact @@ -75,8 +79,12 @@ jobs: path: artifacts - name: Load container image inside kind run: | - kind load image-archive artifacts/image.tar - docker exec -- kind-control-plane crictl image + if [ -f artifacts/image.tar ]; then + echo "Loading image from archive" + docker load --input artifacts/image.tar + else + echo "Using existing image" + fi - name: Install fink-alert-simulator pre-requisites (argo-workflows) run: | . "$CIUXCONFIG" @@ -152,4 +160,4 @@ jobs: password: ${{ secrets.REGISTRY_TOKEN }} - name: Push image to IN2P3 registry run: | - docker push ${{ env.IMAGE }} + docker push ${{ env.IMAGE }} \ No newline at end of file diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f8cee9a4..e68ad2c1 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -15,7 +15,6 @@ env: CIUXCONFIG: /tmp/ciux.sh CIUX_VERSION: 769944575a GHA_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - MINIMAL: ${{ inputs.minimal }} NOSCIENCE: ${{ inputs.noscience }} # Override the self-hosted runner value POD_NAMESPACE: default diff --git a/build.sh b/build.sh index a27f1ec7..bccbcc60 100755 --- a/build.sh +++ b/build.sh @@ -50,13 +50,14 @@ while getopts hs: c ; do done shift `expr $OPTIND - 1` -image=$(ciux get image $PWD --suffix "$suffix") # TODO use error code for anormal exit # --check should return true or false -if image=$(ciux get image --check $PWD --suffix "$suffix") +$(ciux get image --check $PWD --suffix "$suffix" --tmp-registry "$tmp_registry" --env) + +if [ $CIUX_BUILD = false ]; then - echo "Build cancelled, image $image already exists with current source code" + echo "Build cancelled, image $CIUX_IMAGE_URL already exists and contains current source code" exit 0 fi @@ -70,5 +71,5 @@ else fi # Build image -docker image build --tag "$IMAGE" --build-arg spark_py_image="$ASTROLABSOFTWARE_FINK_SPARK_PY_IMAGE" "$DIR" --target $target +docker image build --tag "$CIUX_IMAGE_URL" --build-arg spark_py_image="$ASTROLABSOFTWARE_FINK_SPARK_PY_IMAGE" "$DIR" --target $target diff --git a/conf.sh b/conf.sh index 82ae638e..98b99574 100755 --- a/conf.sh +++ b/conf.sh @@ -1,29 +1,6 @@ . "$CIUXCONFIG" -# Do not launch science pipeline if true -NOSCIENCE="${NOSCIENCE:-false}" - -# Build parameters -# ---------------- - -# TODO make it simpler!! -# TODO add this code to 'ciux get image --check --build-registry' -if [ "$FINK_BROKER_VERSION" = "$CIUX_IMAGE_TAG" ] -then - # We need to build the image - # and eventually set repository address on self-hosted runner - REGISTRY="${CI_REPO:-$CIUX_IMAGE_REGISTRY}" -else - # We can use an existing image - REGISTRY="$CIUX_IMAGE_REGISTRY" -fi - -IMAGE="$REGISTRY/$CIUX_IMAGE_NAME:$CIUX_IMAGE_TAG" - -# TODO Implement ciux get image --current ? -PROMOTED_IMAGE="$CIUX_IMAGE_REGISTRY/$CIUX_IMAGE_NAME:$FINK_BROKER_VERSION" - # Spark parameters # ---------------- diff --git a/e2e/fink-start.sh b/e2e/fink-start.sh index 2b29c0ee..cda6d557 100755 --- a/e2e/fink-start.sh +++ b/e2e/fink-start.sh @@ -24,9 +24,13 @@ set -euxo pipefail DIR=$(cd "$(dirname "$0")"; pwd -P) -. $DIR/../conf.sh - -echo $IMAGE +if [ -n $CIUX_IMAGE_URL ]; +then + IMAGE="$CIUX_IMAGE_URL" +else + echo "ERROR: CIUX_IMAGE_URL is not set" + exit 1 +fi NS=spark echo "Create $NS namespace" @@ -38,7 +42,7 @@ kubectl port-forward -n minio svc/minio 9000 & # Wait to port-forward to start sleep 2 -if [ "$NOSCIENCE" = true ]; +if [ "$IMAGE" =~ "-noscience" ]; then NOSCIENCE_OPT="--noscience" export FINKCONFIG="$DIR/finkconfig_noscience" @@ -47,7 +51,6 @@ else export FINKCONFIG="$DIR/finkconfig" fi - finkctl --endpoint=localhost:9000 s3 makebucket echo "Create spark ServiceAccount" diff --git a/examples/aliases-fink.zsh b/examples/aliases-fink.zsh index 13fcbc4a..242f5bb3 100644 --- a/examples/aliases-fink.zsh +++ b/examples/aliases-fink.zsh @@ -15,7 +15,7 @@ FINK_BROKER_SRC_DIR="$ASTROLAB_DIR/fink-broker" FINK_ALERT_SIM_SRC_DIR="$ASTROLAB_DIR/fink-alert-simulator" FINKCTL_SRC_DIR="$ASTROLAB_DIR/finkctl" -export FINKCONFIG=$FINK_BROKER_SRC_DIR/itest +export FINKCONFIG=$FINK_BROKER_SRC_DIR/e2e/finkconfig_noscience alias cdfa="cd $FINK_ALERT_SIM_SRC_DIR" alias cdfb="cd $FINK_BROKER_SRC_DIR"