build(images): append branch name to the tag of each build-env-* imag…

…e to support building image for each branch (cherry-pick #1763) (#1764) Since build-env-* images may vary for different releases, they should be built for released and master branches. We need to append branch name to the tag to create image for each branch. This PR is to cherry-pick #1763 into v2.5.
apache · Dec 12, 2023 · f857187 · f857187
1 parent 00bc001
commit f857187
Show file tree

Hide file tree

Showing 9 changed files with 34 additions and 29 deletions.
diff --git a/.github/workflows/build-push-env-docker.yml b/.github/workflows/build-push-env-docker.yml
@@ -27,7 +27,6 @@ on:
     paths:
       - 'docker/pegasus-build-env/**'
       - '.github/workflows/build-push-env-docker.yml'
-      - 'thirdparty/**'
 
   # for manually triggering workflow
   workflow_dispatch:
@@ -62,6 +61,6 @@ jobs:
           file: ./docker/pegasus-build-env/${{ matrix.dockertag }}/Dockerfile
           push: true
           tags: |
-            apache/pegasus:build-env-${{ matrix.dockertag }}
+            apache/pegasus:build-env-${{ matrix.dockertag }}-${{ github.ref_name }}
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml
@@ -25,8 +25,10 @@ on:
       - ci-test # testing branch for github action
       - '*dev' # developing branch
     paths:
+      - 'docker/pegasus-build-env/**'
       - 'docker/thirdparties-src/**'
       - 'docker/thirdparties-bin/**'
+      - '.github/workflows/build-push-env-docker.yml'
       - '.github/workflows/thirdparty-regular-push.yml'
       - 'thirdparty/**'
 

diff --git a/docker/README.md b/docker/README.md
@@ -35,10 +35,10 @@ Building environment for Pegasus compilation.
 
 Github Actions automatically rebuilds and publishes build-env for every commit.
 
-- `apache/pegasus:build-env-centos7`
-- `apache/pegasus:build-env-ubuntu1804`
-- `apache/pegasus:build-env-ubuntu2004`
-- `apache/pegasus:build-env-ubuntu2204`
+- `apache/pegasus:build-env-centos7-<branch>`
+- `apache/pegasus:build-env-ubuntu1804-<branch>`
+- `apache/pegasus:build-env-ubuntu2004-<branch>`
+- `apache/pegasus:build-env-ubuntu2204-<branch>`
 
 DockerHub: https://hub.docker.com/r/apache/pegasus
 
@@ -58,7 +58,7 @@ without downloading from the cloud object storage.
 This is a Docker image for Pegasus unit-testing. It prebuilts the thirdparty libraries,
 so jobs based on this image can skip building third-parties.
 
-- `apache/pegasus:thirdparties-bin-centos7-master`
-- `apache/pegasus:thirdparties-bin-ubuntu1804-master`
-- `apache/pegasus:thirdparties-bin-ubuntu2004-master`
-- `apache/pegasus:thirdparties-bin-ubuntu2204-master`
+- `apache/pegasus:thirdparties-bin-centos7-<branch>`
+- `apache/pegasus:thirdparties-bin-ubuntu1804-<branch>`
+- `apache/pegasus:thirdparties-bin-ubuntu2004-<branch>`
+- `apache/pegasus:thirdparties-bin-ubuntu2204-<branch>`
diff --git a/docker/ci-env/Dockerfile b/docker/ci-env/Dockerfile
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM apache/pegasus:build-env-centos7
+FROM apache/pegasus:build-env-centos7-master
 
 WORKDIR /root
 

diff --git a/docker/pegasus-docker-compose/README.md b/docker/pegasus-docker-compose/README.md
@@ -39,7 +39,7 @@ Then build the image:
 
 ```
 cd /your/local/pegasus-docker/pegasus-docker-compose
-./build_docker.sh /your/local/apache-pegasus-source
+./build_docker.sh /your/local/apache-pegasus-source github-branch-for-build-env-image(default: master)
 ```
 
 You will have a docker image called "pegasus:latest" right now built on you machine. Check it out:

diff --git a/docker/pegasus-docker-compose/build_docker.sh b/docker/pegasus-docker-compose/build_docker.sh
@@ -18,14 +18,11 @@
 
 #!/bin/bash
 
-# Usage:
-# 	./build_docker.sh /your/local/apache-pegasus-source
-
 set -e
 
-if [[ $# -ne 1 ]]; then
-	echo "ERROR: must specify /your/local/apache-pegasus-source"
-	exit 1
+if [[ $# -lt 1 ]]; then
+    echo "USAGE: $0 /your/local/apache-pegasus-source [github-branch-for-build-env-image(default: master)]"
+    exit 1
 fi
 
 # ROOT is where the script is.
@@ -34,15 +31,20 @@ ROOT=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")
 # PEGASUS_ROOT is the absolute path of apache-pegasus-source
 PEGASUS_ROOT="$(readlink -f "$1")"
 if [[ ! -f "${PEGASUS_ROOT}"/PACKAGE ]]; then
-	echo "ERROR: no such file ${PEGASUS_ROOT}/PACKAGE"
-	exit 1
+    echo "ERROR: no such file ${PEGASUS_ROOT}/PACKAGE"
+    exit 1
 fi
 SERVER_PKG_NAME=$(cat "${PEGASUS_ROOT}"/PACKAGE)
 if [[ ! -f "${PEGASUS_ROOT}/${SERVER_PKG_NAME}.tar.gz" ]]; then
-	echo "Failed to find package ${SERVER_PKG_NAME}.tar.gz in ${PEGASUS_ROOT}"
-	exit 1
+    echo "Failed to find package ${SERVER_PKG_NAME}.tar.gz in ${PEGASUS_ROOT}"
+    exit 1
 else
-	echo "Found package ${PEGASUS_ROOT}/${SERVER_PKG_NAME}.tar.gz"
+    echo "Found package ${PEGASUS_ROOT}/${SERVER_PKG_NAME}.tar.gz"
+fi
+
+GITHUB_BRANCH=master
+if [[ $# -ge 2 ]]; then
+    GITHUB_BRANCH=$2
 fi
 
 ###
@@ -54,5 +56,5 @@ echo "Building image ${IMAGE_NAME}"
 cd "${ROOT}"/image_for_prebuilt_bin || exit 1
 
 cp "${PEGASUS_ROOT}/${SERVER_PKG_NAME}.tar.gz" .
-docker build --build-arg SERVER_PKG_NAME="${SERVER_PKG_NAME}" -t ${IMAGE_NAME} .
+docker build --build-arg SERVER_PKG_NAME="${SERVER_PKG_NAME}" --build-arg GITHUB_BRANCH="{GITHUB_BRANCH}" -t ${IMAGE_NAME} .
 rm "${SERVER_PKG_NAME}.tar.gz"
diff --git a/docker/pegasus-docker-compose/image_for_prebuilt_bin/Dockerfile b/docker/pegasus-docker-compose/image_for_prebuilt_bin/Dockerfile
@@ -15,7 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM apache/pegasus:build-env-ubuntu2004
+ARG GITHUB_BRANCH=master
+FROM apache/pegasus:build-env-ubuntu2004-${GITHUB_BRANCH}
 
 ARG SERVER_PKG_NAME
 

diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile
@@ -18,13 +18,13 @@
 ARG GITHUB_BRANCH=master
 ARG OS_VERSION=centos7
 FROM apache/pegasus:thirdparties-src-${GITHUB_BRANCH} as builder
-FROM apache/pegasus:build-env-${OS_VERSION}
+FROM apache/pegasus:build-env-${OS_VERSION}-${GITHUB_BRANCH}
 
 WORKDIR /root
 
 COPY --from=builder /root/thirdparties-src.zip /root/thirdparties-src.zip
 
-ARG GITHUB_BRANCH=master
+ARG GITHUB_BRANCH
 ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
 ARG ROCKSDB_PORTABLE=OFF
 ARG USE_JEMALLOC=OFF

diff --git a/docker/thirdparties-src/Dockerfile b/docker/thirdparties-src/Dockerfile
@@ -15,11 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM apache/pegasus:build-env-centos7 as builder
+ARG GITHUB_BRANCH=master
+FROM apache/pegasus:build-env-centos7-${GITHUB_BRANCH} as builder
 
 WORKDIR /root
 
-ARG GITHUB_BRANCH=master
+ARG GITHUB_BRANCH
 ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
 RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL}