From 2ccf5a83b3b041faff2b1c5b509732336ecbf629 Mon Sep 17 00:00:00 2001 From: Vincent Lohse Date: Thu, 9 Nov 2023 16:20:24 +0000 Subject: [PATCH 1/3] Created release 0.4 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index de3bfe6ba..bd73f4707 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4-SNAPSHOT +0.4 From eb6b741fc13a6b949ff240713c80dfc0a463e20d Mon Sep 17 00:00:00 2001 From: Vincent Lohse Date: Thu, 9 Nov 2023 17:22:10 +0000 Subject: [PATCH 2/3] Fixed release tags for AMD --- .github/workflows/test_and_deploy.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml index e398cfa15..d967fbb3e 100644 --- a/.github/workflows/test_and_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -139,10 +139,10 @@ jobs: - name: Build and push ARM64 image for RONDB_VERSION_LTS run: | if [[ "${{ startsWith(github.ref_name, 'release-') }}" == "true" ]]; then - TAG="hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_LTS-latest" - else VERSION="$(< "./VERSION" sed -e 's/^[[:space:]]*//')" TAG="hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION" + else + TAG="hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_LTS-latest" fi docker buildx build . \ --tag $TAG \ @@ -157,11 +157,11 @@ jobs: - name: Build and push ARM64 image for RONDB_VERSION_STABLE run: | if [[ "${{ startsWith(github.ref_name, 'release-') }}" == "true" ]]; then - TAGS="--tag hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_STABLE-latest" - TAGS+=" --tag hopsworks/$ARM_IMAGE_NAME:latest" - else VERSION="$(< "./VERSION" sed -e 's/^[[:space:]]*//')" TAGS="--tag hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION" + else + TAGS="--tag hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_STABLE-latest" + TAGS+=" --tag hopsworks/$ARM_IMAGE_NAME:latest" fi docker buildx build . \ $TAGS \ From 9a5a2514a034a4d6c9f18a35bcee11e2f2fc16a1 Mon Sep 17 00:00:00 2001 From: Vincent Lohse Date: Thu, 23 May 2024 11:22:00 +0200 Subject: [PATCH 3/3] RONDB-662: Backport fixes in CI (#65) * Bumped RonDB 21.04.15 version * Backported CI fixes --- .github/workflows/test_and_deploy.yml | 109 +++++++++++++++----------- Dockerfile | 2 +- README.md | 12 +-- managed_rondb/docker-compose.yml | 4 +- 4 files changed, 71 insertions(+), 56 deletions(-) diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml index d967fbb3e..81070f1a8 100644 --- a/.github/workflows/test_and_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -1,5 +1,7 @@ name: Building Image, Running Benchmarks and Deploying to Dockerhub on: + # Launch on manual trigger + workflow_dispatch: # Launch on any push push: pull_request: @@ -11,21 +13,28 @@ on: types: [opened, synchronize, reopened] env: - RONDB_VERSION_LTS: 21.04.15 - RONDB_VERSION_STABLE: 22.10.1 + BASE_DEPLOY_URL: https://repo.hops.works/master ARM_IMAGE_NAME: rondb-standalone-arm64 - AMD_IMAGE_NAME: rondb-standalone-amd64 + X86_IMAGE_NAME: rondb-standalone-amd64 + + RONDB_VERSION_LTS: 21.04.16 + LTS_X86_TARBALL_NAME: rondb-21.04.16-linux-glibc2.17-x86_64.tar.gz + LTS_ARM_TARBALL_NAME: rondb-21.04.16-linux-glibc2.35-arm64_v8.tar.gz + + RONDB_VERSION_STABLE: 22.10.1 + STABLE_X86_TARBALL_NAME: rondb-22.10.1-linux-glibc2.28-x86_64.tar.gz + STABLE_ARM_TARBALL_NAME: rondb-22.10.1-linux-glibc2.35-arm64_v8.tar.gz jobs: integration-test-and-package: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build and run Docker Compose cluster with benchmarking for RONDB_VERSION_LTS run: | ./run.sh -lv \ - --rondb-tarball-url https://repo.hops.works/master/rondb-$RONDB_VERSION_LTS-linux-glibc2.28-x86_64.tar.gz \ + --rondb-tarball-url $BASE_DEPLOY_URL/$LTS_X86_TARBALL_NAME \ --rondb-version $RONDB_VERSION_LTS \ --size mini \ --run-benchmark sysbench_single \ @@ -84,7 +93,7 @@ jobs: username: hopsworks password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build AMD64 image for RONDB_VERSION_STABLE + - name: Build X86 image for RONDB_VERSION_STABLE if: github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main') run: | VERSION="$(< "./VERSION" sed -e 's/^[[:space:]]*//')" @@ -92,34 +101,38 @@ jobs: --tag rondb-standalone:$RONDB_VERSION_STABLE-$VERSION \ --build-arg RONDB_VERSION=$RONDB_VERSION_STABLE \ --build-arg RONDB_TARBALL_LOCAL_REMOTE=remote \ - --build-arg RONDB_TARBALL_URI=https://repo.hops.works/master/rondb-$RONDB_VERSION_STABLE-linux-glibc2.28-x86_64.tar.gz + --build-arg RONDB_TARBALL_URI=$BASE_DEPLOY_URL/$STABLE_X86_TARBALL_NAME # In the main branch, we work with "-SNAPSHOT", which might as well be called "latest" - - name: Push AMD64 *latest* images to Dockerhub + - name: Push X86 *latest* images to Dockerhub if: github.repository == 'logicalclocks/rondb-docker' && github.ref_name == 'main' run: | VERSION="$(< "./VERSION" sed -e 's/^[[:space:]]*//')" - docker tag rondb-standalone:$RONDB_VERSION_LTS-$VERSION hopsworks/$AMD_IMAGE_NAME:$RONDB_VERSION_LTS-latest - docker tag rondb-standalone:$RONDB_VERSION_STABLE-$VERSION hopsworks/$AMD_IMAGE_NAME:$RONDB_VERSION_STABLE-latest - docker tag rondb-standalone:$RONDB_VERSION_STABLE-$VERSION hopsworks/$AMD_IMAGE_NAME:latest - docker push hopsworks/$AMD_IMAGE_NAME:$RONDB_VERSION_LTS-latest - docker push hopsworks/$AMD_IMAGE_NAME:$RONDB_VERSION_STABLE-latest - docker push hopsworks/$AMD_IMAGE_NAME:latest - - - name: Push AMD64 *release* images to Dockerhub + docker tag rondb-standalone:$RONDB_VERSION_LTS-$VERSION hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION + docker tag rondb-standalone:$RONDB_VERSION_STABLE-$VERSION hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION + docker tag rondb-standalone:$RONDB_VERSION_LTS-$VERSION hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_LTS-latest + docker tag rondb-standalone:$RONDB_VERSION_STABLE-$VERSION hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_STABLE-latest + docker tag rondb-standalone:$RONDB_VERSION_STABLE-$VERSION hopsworks/$X86_IMAGE_NAME:latest + docker push hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION + docker push hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION + docker push hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_LTS-latest + docker push hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_STABLE-latest + docker push hopsworks/$X86_IMAGE_NAME:latest + + - name: Push X86 *release* images to Dockerhub if: github.repository == 'logicalclocks/rondb-docker' && startsWith(github.ref_name, 'release-') run: | VERSION="$(< "./VERSION" sed -e 's/^[[:space:]]*//')" - docker tag rondb-standalone:$RONDB_VERSION_LTS-$VERSION hopsworks/$AMD_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION - docker tag rondb-standalone:$RONDB_VERSION_STABLE-$VERSION hopsworks/$AMD_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION - docker push hopsworks/$AMD_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION - docker push hopsworks/$AMD_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION + docker tag rondb-standalone:$RONDB_VERSION_LTS-$VERSION hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION + docker tag rondb-standalone:$RONDB_VERSION_STABLE-$VERSION hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION + docker push hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION + docker push hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION build-and-push-ARM64: runs-on: ubuntu-latest if: github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main') steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -138,29 +151,27 @@ jobs: # all types of benchmarking to the tests. - name: Build and push ARM64 image for RONDB_VERSION_LTS run: | - if [[ "${{ startsWith(github.ref_name, 'release-') }}" == "true" ]]; then - VERSION="$(< "./VERSION" sed -e 's/^[[:space:]]*//')" - TAG="hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION" - else - TAG="hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_LTS-latest" + VERSION="$(< "./VERSION" sed -e 's/^[[:space:]]*//')" + TAGS="hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION" + if [[ "${{ github.ref_name == 'main' }}" == "true" ]]; then + TAGS+=" --tag hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_LTS-latest" fi docker buildx build . \ - --tag $TAG \ + --tag $TAGS \ --platform=linux/arm64 \ --output type=registry \ --build-arg RONDB_VERSION=$RONDB_VERSION_LTS \ --build-arg RONDB_TARBALL_LOCAL_REMOTE=remote \ - --build-arg RONDB_TARBALL_URI=https://repo.hops.works/master/rondb-$RONDB_VERSION_LTS-linux-glibc2.35-arm64_v8.tar.gz \ + --build-arg RONDB_TARBALL_URI=$BASE_DEPLOY_URL/$LTS_ARM_TARBALL_NAME \ --cache-to type=registry,ref=hopsworks/rondb-standalone-cache,mode=max \ --cache-from type=registry,ref=hopsworks/rondb-standalone-cache,mode=max - name: Build and push ARM64 image for RONDB_VERSION_STABLE run: | - if [[ "${{ startsWith(github.ref_name, 'release-') }}" == "true" ]]; then - VERSION="$(< "./VERSION" sed -e 's/^[[:space:]]*//')" - TAGS="--tag hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION" - else - TAGS="--tag hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_STABLE-latest" + VERSION="$(< "./VERSION" sed -e 's/^[[:space:]]*//')" + TAGS="--tag hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION" + if [[ "${{ github.ref_name == 'main' }}" == "true" ]]; then + TAGS+=" --tag hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_STABLE-latest" TAGS+=" --tag hopsworks/$ARM_IMAGE_NAME:latest" fi docker buildx build . \ @@ -169,7 +180,7 @@ jobs: --output type=registry \ --build-arg RONDB_VERSION=$RONDB_VERSION_STABLE \ --build-arg RONDB_TARBALL_LOCAL_REMOTE=remote \ - --build-arg RONDB_TARBALL_URI=https://repo.hops.works/master/rondb-$RONDB_VERSION_STABLE-linux-glibc2.35-arm64_v8.tar.gz \ + --build-arg RONDB_TARBALL_URI=$BASE_DEPLOY_URL/$STABLE_ARM_TARBALL_NAME \ --cache-to type=registry,ref=hopsworks/rondb-standalone-cache,mode=max \ --cache-from type=registry,ref=hopsworks/rondb-standalone-cache,mode=max @@ -178,7 +189,7 @@ jobs: if: github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main') needs: [integration-test-and-package, build-and-push-ARM64] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -191,22 +202,26 @@ jobs: - name: Create and push multi-platform image run: | - if [[ "${{ startsWith(github.ref_name, 'release-') }}" == "true" ]]; then - VERSION="$(< "./VERSION" sed -e 's/^[[:space:]]*//')" - docker buildx imagetools create -t hopsworks/rondb-standalone:$RONDB_VERSION_LTS-$VERSION \ - hopsworks/$AMD_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION \ - hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION - docker buildx imagetools create -t hopsworks/rondb-standalone:$RONDB_VERSION_STABLE-$VERSION \ - hopsworks/$AMD_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION \ - hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION - else + VERSION="$(< "./VERSION" sed -e 's/^[[:space:]]*//')" + + # Using $VERSION will create weird `-SNAPSHOT` tags, when in main, but we need + # those Dockerhub images in order to run the main branch. + + docker buildx imagetools create -t hopsworks/rondb-standalone:$RONDB_VERSION_LTS-$VERSION \ + hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION \ + hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_LTS-$VERSION + docker buildx imagetools create -t hopsworks/rondb-standalone:$RONDB_VERSION_STABLE-$VERSION \ + hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION \ + hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_STABLE-$VERSION + + if [[ "${{ github.ref_name == 'main' }}" == "true" ]]; then docker buildx imagetools create -t hopsworks/rondb-standalone:$RONDB_VERSION_LTS-latest \ - hopsworks/$AMD_IMAGE_NAME:$RONDB_VERSION_LTS-latest \ + hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_LTS-latest \ hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_LTS-latest docker buildx imagetools create -t hopsworks/rondb-standalone:$RONDB_VERSION_STABLE-latest \ - hopsworks/$AMD_IMAGE_NAME:$RONDB_VERSION_STABLE-latest \ + hopsworks/$X86_IMAGE_NAME:$RONDB_VERSION_STABLE-latest \ hopsworks/$ARM_IMAGE_NAME:$RONDB_VERSION_STABLE-latest docker buildx imagetools create -t hopsworks/rondb-standalone:latest \ - hopsworks/$AMD_IMAGE_NAME:latest \ + hopsworks/$X86_IMAGE_NAME:latest \ hopsworks/$ARM_IMAGE_NAME:latest fi diff --git a/Dockerfile b/Dockerfile index b32d37815..d5d0d708b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ RUN mkdir $DOWNLOADS_CACHE_DIR # Copying bare minimum of Hopsworks cloud environment for now FROM rondb_runtime_dependencies as cloud_preparation -ARG RONDB_VERSION=21.04.15 +ARG RONDB_VERSION=21.04.16 RUN groupadd mysql && adduser mysql --ingroup mysql ENV HOPSWORK_DIR=/srv/hops ENV RONDB_BIN_DIR=$HOPSWORK_DIR/mysql-$RONDB_VERSION diff --git a/README.md b/README.md index 574b363b5..d11de19a6 100644 --- a/README.md +++ b/README.md @@ -91,8 +91,8 @@ Commands to run: # Build and run image **for local platform** in docker-compose using local RonDB tarball (download it first!) # Beware that the local platform is linux/arm64 in this case ./build_run_docker.sh \ - --rondb-tarball-path ./rondb-21.04.15-linux-glibc2.35-arm64_v8.tar.gz \ - --rondb-version 21.04.15 \ + --rondb-tarball-path ./rondb-21.04.16-linux-glibc2.35-arm64_v8.tar.gz \ + --rondb-version 21.04.16 \ --num-mgm-nodes 1 \ --node-groups 1 \ --replication-factor 2 \ @@ -101,13 +101,13 @@ Commands to run: --num-benchmarking-nodes 1 # Build cross-platform image (linux/arm64 here) -docker buildx build . --platform=linux/arm64 -t rondb-standalone:21.04.15 \ - --build-arg RONDB_VERSION=21.04.15 \ +docker buildx build . --platform=linux/arm64 -t rondb-standalone:21.04.16 \ + --build-arg RONDB_VERSION=21.04.16 \ --build-arg RONDB_TARBALL_LOCAL_REMOTE=remote \ # alternatively "local" - --build-arg RONDB_TARBALL_URI=https://repo.hops.works/master/rondb-21.04.15-linux-glibc2.35-arm64_v8.tar.gz # alternatively a local file path + --build-arg RONDB_TARBALL_URI=https://repo.hops.works/master/rondb-21.04.16-linux-glibc2.35-arm64_v8.tar.gz # alternatively a local file path # Explore image -docker run --rm -it --entrypoint=/bin/bash rondb-standalone:21.04.15 +docker run --rm -it --entrypoint=/bin/bash rondb-standalone:21.04.16 ``` Exemplatory commands to run with running docker-compose cluster: diff --git a/managed_rondb/docker-compose.yml b/managed_rondb/docker-compose.yml index 3949b2796..eb9593661 100644 --- a/managed_rondb/docker-compose.yml +++ b/managed_rondb/docker-compose.yml @@ -6,7 +6,7 @@ services: container_name: flask-server command: ["--no-active-test-worker"] environment: - IMAGE_TO_SPAWN: hopsworks/rondb-managed:21.04.15-1.0 + IMAGE_TO_SPAWN: hopsworks/rondb-managed:21.04.16-1.0 CLOUD_BASE_URL: "http://flask-server:8000" ports: # Host:Docker @@ -32,7 +32,7 @@ services: # This container can be killed once the cluster is running bootstrap_agent: - image: hopsworks/rondb-managed:21.04.15-1.0 + image: hopsworks/rondb-managed:21.04.16-1.0 container_name: bootstrap_agent environment: IS_BOOTSTRAP_HOST: "true"