Moves packaging image creation into Github Actions (#745)

.github/workflows/build-citus-community-nightlies.yml

@@ -66,8 +66,7 @@ jobs:
           --secret_key "${PACKAGING_SECRET_KEY}" \
           --passphrase "${PACKAGING_PASSPHRASE}" \
           --output_dir "$(pwd)/packages/" \
-          --input_files_dir "$(pwd)/packaging" \
-          --output_validation
+          --input_files_dir "$(pwd)/packaging"
 
       - name: Publish packages
         run: |

.github/workflows/build-citus-enterprise-nightlies.yml

@@ -67,8 +67,7 @@ jobs:
           --secret_key "${PACKAGING_SECRET_KEY}" \
           --passphrase "${PACKAGING_PASSPHRASE}" \
           --output_dir "$(pwd)/packages/" \
-          --input_files_dir "$(pwd)/packaging" \
-          --output_validation
+          --input_files_dir "$(pwd)/packaging"
 
       - name: Publish packages
         run: |

.github/workflows/build-package.yml

@@ -0,0 +1,78 @@
+name: Build Package
+
+env:
+  GH_TOKEN: "${{ secrets.GH_TOKEN }}"
+  PACKAGING_SECRET_KEY: "${{ secrets.PACKAGING_SECRET_KEY }}"
+  PACKAGE_ENCRYPTION_KEY: "${{ secrets.PACKAGE_ENCRYPTION_KEY }}"
+  PACKAGING_PASSPHRASE: "${{ secrets.PACKAGING_PASSPHRASE }}"
+  DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_PASSWORD }}"
+  DOCKERHUB_USER_NAME: "${{ secrets.DOCKERHUB_USER_NAME }}"
+  CURRENT_BRANCH: ${GITHUB_REF##*/}
+on:
+  push:
+    branches: "**"
+
+  workflow_dispatch:
+
+jobs:
+  check_docker_files_integrity:
+    name: Check if docker files are compliant with templates
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Update and check dockerfiles
+        run: |
+          ./update_dockerfiles
+          git add --intent-to-add dockerfiles
+          git diff --exit-code dockerfiles
+
+  build_package:
+    name: Build package
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        TARGET_PLATFORM:
+          - centos,8
+          - centos,7
+          - debian,bullseye
+          - debian,buster
+          - debian,stretch
+          - oraclelinux,8
+          - oraclelinux,7
+          - ubuntu,focal
+          - ubuntu,bionic
+          - ubuntu,xenial
+          - pgxn
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install package dependencies
+        run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources
+
+      - name: Install wheel for el/8
+        if: matrix.TARGET_PLATFORM  == 'centos,8'
+        run: python -m pip install wheel
+
+      - name: Build image for the target platform
+        run: |
+          git checkout -- dockerfiles
+          ./update_image
+        env:
+          TARGET_PLATFORM: ${{ matrix.TARGET_PLATFORM }}
+
+      - name: Clone tools repo for test
+        run: git clone -b v0.8.8 --depth=1  https://github.com/citusdata/tools.git tools
+
+      - name: Execute packaging tests
+        run: |
+          python -m pip install -r tools/packaging_automation/requirements.txt
+          python -m pytest -q tools/packaging_automation/tests/test_citus_package.py -k 'test_build_packages'
+        env:
+          PACKAGING_IMAGE_PLATFORM: "${{matrix.TARGET_PLATFORM}}"
+
+      - name: Push images
+        run: ./ci/push_images

.github/workflows/build-pgaf-community-nightlies.yml

@@ -65,8 +65,7 @@ jobs:
           --secret_key "${PACKAGING_SECRET_KEY}" \
           --passphrase "${PACKAGING_PASSPHRASE}" \
           --output_dir "$(pwd)/packages/" \
-          --input_files_dir "$(pwd)/packaging" \
-          --output_validation
+          --input_files_dir "$(pwd)/packaging"
 
       - name: Publish packages
         run: |

.github/workflows/build-pgaf-enterprise-nightlies.yml

@@ -67,8 +67,7 @@ jobs:
           --secret_key "${PACKAGING_SECRET_KEY}" \
           --passphrase "${PACKAGING_PASSPHRASE}" \
           --output_dir "$(pwd)/packages/" \
-          --input_files_dir "$(pwd)/packaging" \
-          --output_validation
+          --input_files_dir "$(pwd)/packaging"
 
       - name: Publish packages
         run: |

ci/push_images

@@ -12,12 +12,12 @@ badusage=64
 
 nprocs="${1:-1}"
 
-if [[ $TRAVIS_PULL_REQUEST != "false" ]] || [[ $TRAVIS_BRANCH != "develop" ]] ; then
+if [[ ${CURRENT_BRANCH} != "develop" ]] ; then
     echo "PR is not merged to develop, not pushing to DockerHub"
     exit 0;
 fi
 
-echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+echo "${DOCKERHUB_PASSWORD}" | docker login -u "${DOCKERHUB_USER_NAME}" --password-stdin
 
 declare args
 

update_image

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# This script builds a docker image that is used for packaging.
+#
+# This script requires $TARGET_PLATFORM env to contain a supported platform name.
+# Sample platform names 'debian,buster', 'centos,7' etc.
+
+# make bash behave
+set -euo pipefail
+IFS=$'\n\t'
+
+pgversions='10 11 12 13 14'
+topdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+dockerfiles_dir="${topdir}/dockerfiles"
+
+badusage=64
+
+
+nprocs="${1:-1}"
+
+declare args
+
+
+IFS=',' read -r os release <<< "${TARGET_PLATFORM}"
+
+if [[ "${os}" = 'debian' ]] || [[ "${os}" = 'ubuntu' ]]; then
+    tag="${os}-${release}-all"
+    args+="build -t citus/packaging:${tag} -f ${dockerfiles_dir}/${tag}/Dockerfile .\n"
+elif [[ "${os}" = 'centos' ]] || [[ "${os}" = 'oraclelinux' ]]; then
+    # redhat variants need an image for each PostgreSQL version
+    IFS=' '
+    for pgversion in ${pgversions}; do
+        if { [[ "${os}" = 'centos' ]] || [[ "${os}" = 'oraclelinux' ]]; } && \
+             [[ "${release}" = '6' ]] && [[ "${pgversion}" = '13' ]]; then
+            # CentOS and OracleLinux 6 doesn't have pg13 packages yet.
+            # So skip building docker images for them.
+            continue
+        fi
+        pgshort=${pgversion//./}
+        tag="${os}-${release}-pg${pgshort}"
+        args+="build -t citus/packaging:${tag} -f ${dockerfiles_dir}/${tag}/Dockerfile .\n"
+    done
+elif [[ "${os}" = 'pgxn' ]]; then
+    tag="${os}-all"
+    args+="build -t citus/packaging:${tag} -f ${dockerfiles_dir}/${tag}/Dockerfile .\n"
+else
+    echo "$0: unrecognized OS -- ${os}" >&2
+    exit $badusage
+fi
+
+
+echo -e "${args}" | xargs -t -L1 -P "${nprocs}" docker