WIP rebase on 9.4.x

.github/workflows/build-and-publish-debug.yml

@@ -0,0 +1,31 @@
+name: Build and publish debug
+# This workflow is triggered on new tags of Community Edition 7.8.x or later,
+# or any of the weekly tag names starting 'master'
+on:
+  push:
+    tags:
+    - 'master*'
+    - 'community_*'
+    - '!community_7.6.*'
+    - '!community_7.4.*'
+    - '!community_7.2.*'
+    - '!community_7.0.*'
+    - '!community_6.*'
+
+jobs:
+  build:
+    name: "Build and publish debug container"
+    runs-on: ubuntu-20.04
+    if: github.repository == 'hpcc-systems/HPCC-Platform'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Build
+        uses: ./dockerfiles
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          build_type: Debug
+          github_actor: ${{ secrets.LNB_ACTOR }}
+          github_token: ${{ secrets.LNB_TOKEN }}
+

.github/workflows/build-and-publish.yml

@@ -0,0 +1,80 @@
+name: Build and publish
+# This workflow is triggered on new tags of Community Edition 7.8.x or later,
+# or any of the weekly tag names starting 'master'
+on:
+  push:
+    tags:
+    - 'master*'
+    - 'community_*'
+    - '!community_7.6.*'
+    - '!community_7.4.*'
+    - '!community_7.2.*'
+    - '!community_7.0.*'
+    - '!community_6.*'
+
+jobs:
+  build:
+    name: "Build and publish release container"
+    runs-on: ubuntu-20.04
+    if: github.repository == 'hpcc-systems/HPCC-Platform'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Build
+        uses: ./dockerfiles
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          latest: 1   # this should only be set on the current minor branch
+          github_actor: ${{ secrets.LNB_ACTOR }}
+          github_token: ${{ secrets.LNB_TOKEN }}
+          sign_modules: 1
+          signing_secret: ${{ secrets.SIGNING_SECRET }}
+          signing_keyid: ${{ secrets.SIGNING_KEYID }}
+          signing_passphrase: ${{ secrets.SIGNING_PASSPHRASE }}
+
+  ml-builds:
+    needs: build
+    runs-on: ubuntu-20.04
+    if: github.repository == 'hpcc-systems/HPCC-Platform'
+    strategy:
+      matrix:
+        engine: ['ml', 'gnn', 'gnn-gpu']
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Build
+      uses: ./dockerfiles
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        latest: 1   # this should only be set on the current minor branch
+        build_ml: ${{ matrix.engine }}
+        github_actor: ${{ secrets.LNB_ACTOR }}
+        github_token: ${{ secrets.LNB_TOKEN }}
+
+  ln-builds:
+    needs: build
+    runs-on: ubuntu-20.04
+    if: github.repository == 'hpcc-systems/HPCC-Platform'
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Build
+      uses: ./dockerfiles
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        ln_username: ${{ secrets.JFROG_USERNAME }}
+        ln_password: ${{ secrets.JFROG_PASSWORD }}
+        ln_registry: ${{ secrets.JFROG_REGISTRY }}
+        ln_docker_repo: ${{ secrets.JFROG_REPOSITORY }}
+        lnb_token: ${{ secrets.LNB_TOKEN }}
+        latest: 1   # this should only be set on the current minor branch
+        build_ln: 1
+        github_actor: ${{ secrets.LNB_ACTOR }}
+        github_token: ${{ secrets.LNB_TOKEN }}
+        sign_modules: 1
+        signing_secret: ${{ secrets.SIGNING_SECRET }}
+        signing_keyid: ${{ secrets.SIGNING_KEYID }}
+        signing_passphrase: ${{ secrets.SIGNING_PASSPHRASE }}

.github/workflows/build-containers-pr.yml

@@ -0,0 +1,141 @@
+name: Docker smoketest build
+on:
+  pull_request:
+    branches:
+      - "master"
+      - "candidate-*.x"
+      - "!candidate-8.2.*"
+      - "!candidate-8.0.*"
+      - "!candidate-7.12.*"
+      - "!candidate-7.10.*"
+      - "!candidate-7.8.*"
+      - "!candidate-7.6.*"
+      - "!candidate-7.4.*"
+      - "!candidate-7.2.*"
+      - "!candidate-7.0.*"
+      - "!candidate-6.*"
+
+jobs:
+  check-skip:
+    # continue-on-error: true # Uncomment once integration is finished
+    runs-on: ubuntu-20.04
+    # Map a step output to a job output
+    outputs:
+      changed: ${{ steps.skip_check.outputs.dockerfiles || steps.skip_check.outputs.platform }}
+    steps:
+      - id: skip_check
+        uses: hpcc-systems/github-actions/changed-modules@main
+        with:
+          github_token: ${{ github.token }}
+
+  build-images:
+    needs: check-skip
+    if: ${{ needs.check-skip.outputs.changed }}
+    runs-on: ubuntu-20.04
+    steps:
+      - name: vars
+        id: vars
+        run: |
+          # echo ::set-output name=container_registry::ghcr.io
+          # echo ::set-output name=cr_user::${{ github.repository_owner }}
+          echo ::set-output name=container_registry::docker.io
+          echo ::set-output name=cr_user::hpccbuilds
+          echo ::set-output name=build_prbase_sha::${{ github.event.pull_request.base.sha }}
+          echo ::set-output name=build_prbase_label::${{ github.base_ref }}
+          echo ::set-output name=build_user::${{ github.actor }}
+          echo ::set-output name=build_type::RelWithDebInfo
+          echo ::set-output name=use_cppunit::1
+          echo ::set-output name=platform_build::smoketest-platform-build-vcpkg
+
+      - name: tracing
+        run: |
+          echo "Base ref   = ${{ github.ref }}"
+          echo "Action     = ${{ github.action }}"
+          echo "Event      = ${{ github.event_name }}"
+          echo "Actor      = ${{ github.actor }}"
+          echo "Ref        = ${{ github.ref }}"
+          echo "base sha   = ${{ github.event.pull_request.base.sha }}"
+          echo "Sha        = ${{ github.sha }}"
+          echo "github.repository = ${{ github.repository }}"
+          echo "repository_owner = ${{ github.repository_owner }}"
+          echo "github.workspace = ${{ github.workspace }}"
+          echo "runner.workspace = ${{ runner.workspace }}"
+          echo "github.event.pull_request.head.repo.owner.login = ${{ github.event.pull_request.head.repo.owner.login }}"
+          echo "build_prbase_label = ${{ steps.vars.outputs.build_prbase_label }}"
+          echo "platform_build = ${{ steps.vars.outputs.platform_build }}"
+
+      - name: Checkout PR
+        uses: actions/checkout@v3
+
+      - name: Calculate vcpkg vars
+        id: vcpkg_vars
+        run: |
+          echo "base_ver=$(git submodule status vcpkg | cut -c2-9)" >> $GITHUB_OUTPUT
+
+      - name: tracing (vcpkg_vars)
+        run: |
+          echo "vcpkg_vars.base_ver = ${{ steps.vcpkg_vars.outputs.base_ver }}"
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver: docker
+
+      - name: Check if PR-Base prebuilt
+        id: check-images
+        run: |
+          prbase_missing=$(docker manifest inspect ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-${{ github.event.pull_request.base.sha }} > /dev/null ; echo $?)
+          echo prbase_missing=${prbase_missing}
+          if [[ "${prbase_missing}" -eq 1 ]]
+          then
+            echo "Current PR target branch image cannot be found, using latest"
+            prbase_missing=$(docker manifest inspect ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-latest > /dev/null ; echo $?)
+            if [[ "${prbase_missing}" -eq 1 ]]
+            then
+              echo "Cannot find the 'latest' target branch image"
+              echo ::set-output name=prbase_missing::${prbase_missing}
+            fi
+            echo ::set-output name=platform_prbase_ver::${{ steps.vars.outputs.build_prbase_label }}-latest
+          else
+            echo ::set-output name=platform_prbase_ver::${{ steps.vars.outputs.build_prbase_label }}-${{ github.event.pull_request.base.sha }}
+          fi
+
+      # Normal expectation is that the following step would normally be skipped
+      # NB: This is only for the case where the target branch image doesn't already exist.
+      # The build-containers-target-branch.yml action would normally have built/published this image,
+      # or they'll be a "latest" which will have been picked up by the check-images step.
+      - name: branch image
+        if: ${{ steps.check-images.outputs.prbase_missing == '1' }}
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./dockerfiles/platform-build/Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          tags: |
+            ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-${{ steps.vars.outputs.build_prbase_sha }}
+            ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-latest
+          build-args: |
+            CR_REPO=${{ steps.vars.outputs.container_registry }}
+            BASE_VER=${{ steps.vcpkg_vars.outputs.base_ver }}
+            BUILD_USER=${{ github.repository_owner }}
+            GITHUB_ACTOR=${{ github.actor }}
+            BUILD_TAG=${{ steps.vars.outputs.build_prbase_sha }}
+            BUILD_TYPE=${{ steps.vars.outputs.build_type }}
+            GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
+            USE_CPPUNIT=${{ steps.vars.outputs.use_cppunit }}
+            BUILD_THREADS=${{ steps.vars.outputs.build_threads }}
+
+      - name: PR image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./dockerfiles/platform-build-incremental-container/Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          build-args: |
+            CR_USER=${{ steps.vars.outputs.cr_user }}
+            CR_REPO=${{ steps.vars.outputs.container_registry }}
+            PLATFORM_PRBASE_VER=${{ steps.check-images.outputs.platform_prbase_ver }}
+            GITHUB_REPO=${{ github.repository }}
+            GITHUB_PRREF=${{ github.ref }}
+            BUILD_THREADS=${{ steps.vars.outputs.build_threads }}

.github/workflows/build-containers-target-branch.yml

@@ -0,0 +1,112 @@
+name: Docker target branch build
+on:
+  push:
+    branches:
+      - "master"
+      - "candidate-*.x"
+      - "!candidate-8.2.*"
+      - "!candidate-8.0.*"
+      - "!candidate-7.12.*"
+      - "!candidate-7.10.*"
+      - "!candidate-7.8.*"
+      - "!candidate-7.6.*"
+      - "!candidate-7.4.*"
+      - "!candidate-7.2.*"
+      - "!candidate-7.0.*"
+      - "!candidate-6.*"
+
+jobs:
+  check-skip:
+    # continue-on-error: true # Uncomment once integration is finished
+    runs-on: ubuntu-20.04
+    # Map a step output to a job output
+    outputs:
+      changed: ${{ steps.skip_check.outputs.dockerfiles || steps.skip_check.outputs.platform }}
+    steps:
+      - id: skip_check
+        uses: hpcc-systems/github-actions/changed-modules@main
+        with:
+          github_token: ${{ github.token }}
+
+  build-images:
+    needs: check-skip
+    if: ${{ needs.check-skip.outputs.changed }}
+    runs-on: ubuntu-20.04
+    steps:
+      - name: vars
+        id: vars
+        run: |
+          # echo ::set-output name=container_registry::ghcr.io
+          # echo ::set-output name=cr_user::${{ github.repository_owner }}
+          echo ::set-output name=container_registry::docker.io
+          echo ::set-output name=cr_user::hpccbuilds
+          echo ::set-output name=build_base_sha::${{ github.sha }}
+          echo ::set-output name=build_base_label::${GITHUB_REF##*/}
+          echo ::set-output name=build_user::${{ github.actor }}
+          echo ::set-output name=build_type::RelWithDebInfo
+          echo ::set-output name=use_cppunit::1
+          echo ::set-output name=platform_build::smoketest-platform-build-vcpkg
+
+      - name: tracing
+        run: |
+          echo "Action     = ${{ github.action }}"
+          echo "Event      = ${{ github.event_name }}"
+          echo "Actor      = ${{ github.actor }}"
+          echo "Ref        = ${{ github.ref }}"
+          echo "Sha        = ${{ github.sha }}"
+          echo "github.repository = ${{ github.repository }}"
+          echo "repository_owner = ${{ github.repository_owner }}"
+          echo "github.workspace = ${{ github.workspace }}"
+          echo "runner.workspace = ${{ runner.workspace }}"
+          echo "build_base_sha = ${{ steps.vars.outputs.build_base_sha }}"
+          echo "build_base_label = ${{ steps.vars.outputs.build_base_label }}"
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Calculate vcpkg vars
+        id: vcpkg_vars
+        run: |
+          echo "base_ver=$(git submodule status vcpkg | cut -c2-9)" >> $GITHUB_OUTPUT
+
+      - name: tracing (vcpkg_vars)
+        run: |
+          echo "vcpkg_vars.base_ver = ${{ steps.vcpkg_vars.outputs.base_ver }}"
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      # - name: Login to GitHub Container Registry
+      #   uses: docker/login-action@v1
+      #   with:
+      #     registry: ghcr.io
+      #     username: ${{ github.repository_owner }}
+      #     password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      # build branch image
+      - name: branch image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./dockerfiles/platform-build/Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          tags: |
+            ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_base_label }}-${{ steps.vars.outputs.build_base_sha }}
+            ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_base_label }}-latest
+          push: true
+          build-args: |
+            CR_REPO=${{ steps.vars.outputs.container_registry }}
+            BASE_VER=${{ steps.vcpkg_vars.outputs.base_ver }}
+            BUILD_USER=${{ github.repository_owner }}
+            BUILD_TAG=${{ steps.vars.outputs.build_base_sha }}
+            BUILD_TYPE=${{ steps.vars.outputs.build_type }}
+            GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
+            USE_CPPUNIT=${{ steps.vars.outputs.use_cppunit }}
+            BUILD_THREADS=${{ steps.vars.outputs.build_threads }}