diff --git a/.github/workflows/build-and-publish-debug.yml b/.github/workflows/build-and-publish-debug.yml new file mode 100644 index 00000000000..3b32b2ebe44 --- /dev/null +++ b/.github/workflows/build-and-publish-debug.yml @@ -0,0 +1,31 @@ +name: Build and publish debug +# This workflow is triggered on new tags of Community Edition 7.8.x or later, +# or any of the weekly tag names starting 'master' +on: + push: + tags: + - 'master*' + - 'community_*' + - '!community_7.6.*' + - '!community_7.4.*' + - '!community_7.2.*' + - '!community_7.0.*' + - '!community_6.*' + +jobs: + build: + name: "Build and publish debug container" + runs-on: ubuntu-20.04 + if: github.repository == 'hpcc-systems/HPCC-Platform' + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Build + uses: ./dockerfiles + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + build_type: Debug + github_actor: ${{ secrets.LNB_ACTOR }} + github_token: ${{ secrets.LNB_TOKEN }} + diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml new file mode 100644 index 00000000000..2d09d008f2e --- /dev/null +++ b/.github/workflows/build-and-publish.yml @@ -0,0 +1,80 @@ +name: Build and publish +# This workflow is triggered on new tags of Community Edition 7.8.x or later, +# or any of the weekly tag names starting 'master' +on: + push: + tags: + - 'master*' + - 'community_*' + - '!community_7.6.*' + - '!community_7.4.*' + - '!community_7.2.*' + - '!community_7.0.*' + - '!community_6.*' + +jobs: + build: + name: "Build and publish release container" + runs-on: ubuntu-20.04 + if: github.repository == 'hpcc-systems/HPCC-Platform' + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Build + uses: ./dockerfiles + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + latest: 1 # this should only be set on the current minor branch + github_actor: ${{ secrets.LNB_ACTOR }} + github_token: ${{ secrets.LNB_TOKEN }} + sign_modules: 1 + signing_secret: ${{ secrets.SIGNING_SECRET }} + signing_keyid: ${{ secrets.SIGNING_KEYID }} + signing_passphrase: ${{ secrets.SIGNING_PASSPHRASE }} + + ml-builds: + needs: build + runs-on: ubuntu-20.04 + if: github.repository == 'hpcc-systems/HPCC-Platform' + strategy: + matrix: + engine: ['ml', 'gnn', 'gnn-gpu'] + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Build + uses: ./dockerfiles + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + latest: 1 # this should only be set on the current minor branch + build_ml: ${{ matrix.engine }} + github_actor: ${{ secrets.LNB_ACTOR }} + github_token: ${{ secrets.LNB_TOKEN }} + + ln-builds: + needs: build + runs-on: ubuntu-20.04 + if: github.repository == 'hpcc-systems/HPCC-Platform' + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Build + uses: ./dockerfiles + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + ln_username: ${{ secrets.JFROG_USERNAME }} + ln_password: ${{ secrets.JFROG_PASSWORD }} + ln_registry: ${{ secrets.JFROG_REGISTRY }} + ln_docker_repo: ${{ secrets.JFROG_REPOSITORY }} + lnb_token: ${{ secrets.LNB_TOKEN }} + latest: 1 # this should only be set on the current minor branch + build_ln: 1 + github_actor: ${{ secrets.LNB_ACTOR }} + github_token: ${{ secrets.LNB_TOKEN }} + sign_modules: 1 + signing_secret: ${{ secrets.SIGNING_SECRET }} + signing_keyid: ${{ secrets.SIGNING_KEYID }} + signing_passphrase: ${{ secrets.SIGNING_PASSPHRASE }} diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index ab97927677a..21dd61c309a 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -34,7 +34,8 @@ jobs: internal_ref: ${{ steps.vars.outputs.internal_ref }} community_tag: ${{ steps.vars.outputs.community_tag }} internal_tag: ${{ steps.vars.outputs.internal_tag }} - community_branch: ${{ steps.vars.outputs.community_branch }} + candidate_base_branch: ${{ steps.vars.outputs.candidate_base_branch }} + candidate_branch: ${{ steps.vars.outputs.candidate_branch }} cmake_docker_config: ${{ steps.vars.outputs.cmake_docker_config }} gpg_import: ${{ steps.vars.outputs.gpg_import }} steps: @@ -54,8 +55,10 @@ jobs: echo "community_tag=$community_tag" >> $GITHUB_OUTPUT echo "internal_tag=$(echo $community_tag | sed 's/community/internal/')" >> $GITHUB_OUTPUT community_base_ref=${{ github.event.base_ref || github.ref }} - echo "community_branch=$(echo $community_base_ref | cut -d'/' -f3)" >> $GITHUB_OUTPUT - echo "cmake_docker_config=-DCMAKE_BUILD_TYPE=RelWithDebInfo -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DSIGN_MODULES=ON" >> $GITHUB_OUTPUT + candidate_branch=$(echo $community_base_ref | cut -d'/' -f3) + echo "candidate_branch=$candidate_branch" >> $GITHUB_OUTPUT + echo "candidate_base_branch=$(echo $candidate_branch | awk -F'.' -v OFS='.' '{ $3="x"; print }')" >> $GITHUB_OUTPUT + echo "cmake_docker_config=-DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DSIGN_MODULES=${{ github.repository_owner == 'hpcc-systems' && 'ON' || 'OFF' }}" >> $GITHUB_OUTPUT echo 'gpg_import=gpg --batch --import /hpcc-dev/build/private.key' >> $GITHUB_OUTPUT - name: Print vars @@ -82,29 +85,40 @@ jobs: build-docker: name: Build Docker - # if: github.repository == 'hpcc-systems/HPCC-Platform' needs: preamble runs-on: ubuntu-22.04 strategy: matrix: include: - - os: ubuntu-22.10 + - os: ubuntu-23.04 - os: ubuntu-22.04 - container: true - os: ubuntu-22.04 + name: k8s + container: true - os: ubuntu-22.04 + name: docs documentation: true - os: ubuntu-20.04 - os: centos-8 - os: centos-7 - os: amazonlinux + - os: ubuntu-22.04 + name: LN k8s + ln: true + container: true - os: ubuntu-20.04 + name: LN ln: true - os: centos-7 + name: LN ln: true fail-fast: false steps: + - name: Free additional disk space (remove Android SDK + Tools) + run: | + sudo rm -rf /usr/local/lib/android + - name: Checkout HPCC-Platform uses: actions/checkout@v3 with: @@ -126,8 +140,12 @@ jobs: id: vars working-directory: ${{ needs.preamble.outputs.folder_platform }}/vcpkg run: | - echo "vcpkg_sha_short=$(git rev-parse --short=8 HEAD)" >> $GITHUB_OUTPUT - echo "docker_label=hpccsystems/platform-build-${{ matrix.os }}:$(git rev-parse --short=8 HEAD)" >> $GITHUB_OUTPUT + vcpkg_sha_short=$(git rev-parse --short=8 HEAD) + echo "vcpkg_sha_short=$vcpkg_sha_short" >> $GITHUB_OUTPUT + docker_build_label=hpccsystems/platform-build-${{ matrix.os }} + echo "docker_build_label=$docker_build_label" >> $GITHUB_OUTPUT + echo "docker_tag=$docker_build_label:$vcpkg_sha_short" >> $GITHUB_OUTPUT + echo "docker_tag_candidate_base=$docker_build_label:${{ needs.preamble.outputs.candidate_base_branch }}" >> $GITHUB_OUTPUT - name: Print vars run: | @@ -150,87 +168,136 @@ jobs: builder: ${{ steps.buildx.outputs.name }} file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/${{ matrix.os }}.dockerfile context: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg - push: true + push: ${{ github.repository_owner == 'hpcc-systems' }} + load: ${{ github.repository_owner != 'hpcc-systems' }} build-args: | VCPKG_REF=${{ steps.vars.outputs.vcpkg_sha_short }} tags: | - ${{ steps.vars.outputs.docker_label }} - cache-from: type=registry,ref=${{ steps.vars.outputs.docker_label }} + ${{ steps.vars.outputs.docker_tag_candidate_base }} + cache-from: | + type=registry,ref=${{ steps.vars.outputs.docker_tag_candidate_base }} + type=registry,ref=${{ steps.vars.outputs.docker_tag }} cache-to: type=inline - # - name: Update Docker Image (vcpkg_overlays) - # run: | - # mkdir -p ${{ needs.preamble.outputs.folder_build }} - # echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key - # CONTAINER=$(docker run -d --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "tail -f /dev/null") - # docker exec $CONTAINER /bin/bash -c "${{ needs.preamble.outputs.gpg_import }} && cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF" - # docker commit $CONTAINER ${{ steps.vars.outputs.docker_label }} - # docker push $CONTAINER ${{ steps.vars.outputs.docker_label }} - # docker rm -f $CONTAINER - - - name: Prepare build folder + # Communtiy Build + - name: CMake Packages (community) + if: ${{ !matrix.ln && !matrix.container && !matrix.documentation }} run: | mkdir -p ${{ needs.preamble.outputs.folder_build }} - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cp /hpcc-dev/vcpkg_installed /hpcc-dev/build" - - - name: CMake Packages - if: ${{ !matrix.container && !matrix.ln && !matrix.documentation }} - run: | echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key - plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED" "PLATFORM") + plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM" "CLIENTTOOLS_ONLY") for plugin in "${plugins[@]}"; do sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "${{ needs.preamble.outputs.gpg_import }} && cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF" - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "${{ needs.preamble.outputs.gpg_import }} && cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \ + cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF && \ + cmake --build /hpcc-dev/build --parallel $(nproc) --target package" # Disabled as not currently needed --- - # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON" - # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON" + # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" done - - name: CMake Containerized Packages - if: ${{ matrix.container }} + - name: CMake Containerized Packages (community) + if: ${{ !matrix.ln && matrix.container && !matrix.documentation }} run: | + mkdir -p ${{ needs.preamble.outputs.folder_build }} echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "${{ needs.preamble.outputs.gpg_import }} && cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF" - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "${{ needs.preamble.outputs.gpg_import }} && cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "${{ needs.preamble.outputs.gpg_import }} && cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON" - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "${{ needs.preamble.outputs.gpg_import }} && cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - - - name: CMake documentation - if: ${{ matrix.documentation }} + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \ + cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF && \ + cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \ + cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON && \ + cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + + - name: CMake documentation (community) + if: ${{ !matrix.ln && !matrix.container && matrix.documentation }} run: | mkdir -p {${{needs.preamble.outputs.folder_build }},EN_US,PT_BR} sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL" - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "cmake --build /hpcc-dev/build --parallel $(nproc) --target all" - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-${{ needs.preamble.outputs.community_tag }}.zip *.pdf" - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-${{ needs.preamble.outputs.community_tag }}.zip *.pdf" + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\ + cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \ + cmake --build /hpcc-dev/build --parallel $(nproc) --target all" + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-${{ needs.preamble.outputs.community_tag }}.zip *.pdf" + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-${{ needs.preamble.outputs.community_tag }}.zip *.pdf" - - name: Upload Assets - uses: ncipollo/release-action@v1.12.0 + - name: Upload Assets (community) if: ${{ !matrix.ln }} + uses: ncipollo/release-action@v1.12.0 with: allowUpdates: true generateReleaseNotes: false prerelease: ${{ contains(github.ref, '-rc') }} artifacts: "${{ needs.preamble.outputs.folder_build }}/*.deb,${{ needs.preamble.outputs.folder_build }}/*.rpm,${{ needs.preamble.outputs.folder_build }}/Release/docs/*.zip,${{ needs.preamble.outputs.folder_build }}/Release/docs/EN_US/*.zip,${{ needs.preamble.outputs.folder_build }}/Release/docs/PT_BR/*.zip,${{ needs.preamble.outputs.folder_build }}/docs/EN_US/EclipseHelp/*.zip,${{ needs.preamble.outputs.folder_build }}/docs/EN_US/HTMLHelp/*.zip,${{ needs.preamble.outputs.folder_build }}/docs/PT_BR/HTMLHelp/*.zip" - - name: CMake LN Packages - if: ${{ matrix.ln }} + - name: Locate k8s deb file (community) + if: ${{ !matrix.ln && matrix.container && !matrix.documentation }} + id: container + run: | + k8s_pkg_path=$(ls -t ${{ needs.preamble.outputs.folder_build }}/*64_k8s.deb 2>/dev/null | head -1) + k8s_pkg_file=$(basename "$k8s_pkg_path") + echo "k8s_pkg_file=$k8s_pkg_file" >> $GITHUB_OUTPUT + echo "$k8s_pkg_file" + + - name: Create Docker Image (community) + uses: docker/build-push-action@v4 + if: ${{ !matrix.ln && matrix.container && !matrix.documentation }} + with: + builder: ${{ steps.buildx.outputs.name }} + file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/platform-core-${{ matrix.os }}/Dockerfile + context: ${{ needs.preamble.outputs.folder_build }} + push: ${{ github.repository_owner == 'hpcc-systems' }} + build-args: | + PKG_FILE=${{ steps.container.outputs.k8s_pkg_file }} + tags: | + hpccsystems/platform-core-${{ matrix.os }}:${{ needs.preamble.outputs.community_tag }} + hpccsystems/platform-core-${{ matrix.os }}:${{ needs.preamble.outputs.candidate_base_branch }} + cache-from: | + type=registry,ref=hpccsystems/platform-core-${{ matrix.os }}:${{ needs.preamble.outputs.candidate_base_branch }} + + # Internal Build --- + - name: Login to JFrog (internal) + if: ${{ matrix.ln && matrix.container && github.repository_owner == 'hpcc-systems' }} + uses: docker/login-action@v2 + with: + registry: ${{ secrets.JFROG_REGISTRY }} + username: ${{ secrets.JFROG_USERNAME }} + password: ${{ secrets.JFROG_PASSWORD }} + + - name: CMake Packages (internal) + if: ${{ matrix.ln && !matrix.container && !matrix.documentation }} + run: | + mkdir -p ${{ needs.preamble.outputs.folder_build }} + echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key + plugins=("PLATFORM" "CLIENTTOOLS_ONLY") + for plugin in "${plugins[@]}"; do + sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt + sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \ + cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -D$plugin=ON -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF && \ + cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + # Disabled as not currently needed --- + # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON" + # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + done + + - name: CMake Containerized Packages (internal) + if: ${{ matrix.ln && matrix.container && !matrix.documentation }} run: | + mkdir -p ${{ needs.preamble.outputs.folder_build }} echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "${{ needs.preamble.outputs.gpg_import }} && cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF" - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "${{ needs.preamble.outputs.gpg_import }} && cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON" - # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_label }} "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - - - name: Upload LN Assets + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \ + cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF && \ + cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \ + cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON && \ + cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + + - name: Upload Assets (internal) if: ${{ matrix.ln }} uses: ncipollo/release-action@v1.12.0 with: @@ -241,8 +308,34 @@ jobs: repo: LN token: ${{ secrets.LNB_TOKEN }} tag: ${{ needs.preamble.outputs.internal_tag }} - artifacts: "${{ needs.preamble.outputs.folder_build }}/hpccsystems-platform-internal*.deb,${{ needs.preamble.outputs.folder_build }}/hpccsystems-platform-internal*.rpm" + artifacts: "${{ needs.preamble.outputs.folder_build }}/hpccsystems-*-internal*.deb,${{ needs.preamble.outputs.folder_build }}/hpccsystems-*-internal*.rpm" + + - name: Locate k8s deb file (internal) + if: ${{ matrix.ln && matrix.container && !matrix.documentation }} + id: ln-container + run: | + k8s_pkg_path=$(ls -t ${{ needs.preamble.outputs.folder_build }}/*64_k8s.deb 2>/dev/null | head -1) + k8s_pkg_file=$(basename "$k8s_pkg_path") + echo "k8s_pkg_file=$k8s_pkg_file" >> $GITHUB_OUTPUT + echo "$k8s_pkg_file" + + - name: Create Docker Image (internal) + uses: docker/build-push-action@v4 + if: ${{ matrix.ln && matrix.container && !matrix.documentation }} + with: + builder: ${{ steps.buildx.outputs.name }} + file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/platform-core-${{ matrix.os }}/Dockerfile + context: ${{ needs.preamble.outputs.folder_build }} + push: ${{ github.repository_owner == 'hpcc-systems' }} + build-args: | + PKG_FILE=${{ steps.ln-container.outputs.k8s_pkg_file }} + tags: | + ${{ secrets.JFROG_REGISTRY || 'dummy.io' }}/ln/platform-core-${{ matrix.os }}:${{ needs.preamble.outputs.internal_tag }} + ${{ secrets.JFROG_REGISTRY || 'dummy.io' }}/ln/platform-core-${{ matrix.os }}:${{ needs.preamble.outputs.candidate_base_branch }} + cache-from: | + type=registry,ref=hpccsystems/platform-core-${{ matrix.os }}:${{ needs.preamble.outputs.candidate_base_branch }} + # Common --- - name: Cleanup Environment if: always() run: rm -f ${{ needs.preamble.outputs.folder_build }}/private.key @@ -251,7 +344,7 @@ jobs: if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v3 with: - name: ${{ matrix.os }}-${{ matrix.package }}-logs + name: ${{ matrix.os }}-${{ matrix.ln }}-${{ matrix.container }}-${{ matrix.documentation }}-logs path: ${{ needs.preamble.outputs.folder_build }}/**/*.log build-bare-metal: @@ -364,5 +457,5 @@ jobs: if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v3 with: - name: ${{ matrix.os }}-${{ matrix.package }}-logs + name: ${{ matrix.os }}-logs path: ./build/**/*.log diff --git a/.github/workflows/build-containers-pr.yml b/.github/workflows/build-containers-pr.yml new file mode 100644 index 00000000000..c5cca9badd6 --- /dev/null +++ b/.github/workflows/build-containers-pr.yml @@ -0,0 +1,141 @@ +name: Docker smoketest build +on: + pull_request: + branches: + - "master" + - "candidate-*.x" + - "!candidate-8.2.*" + - "!candidate-8.0.*" + - "!candidate-7.12.*" + - "!candidate-7.10.*" + - "!candidate-7.8.*" + - "!candidate-7.6.*" + - "!candidate-7.4.*" + - "!candidate-7.2.*" + - "!candidate-7.0.*" + - "!candidate-6.*" + +jobs: + check-skip: + # continue-on-error: true # Uncomment once integration is finished + runs-on: ubuntu-20.04 + # Map a step output to a job output + outputs: + changed: ${{ steps.skip_check.outputs.dockerfiles || steps.skip_check.outputs.platform }} + steps: + - id: skip_check + uses: hpcc-systems/github-actions/changed-modules@main + with: + github_token: ${{ github.token }} + + build-images: + needs: check-skip + if: ${{ needs.check-skip.outputs.changed }} + runs-on: ubuntu-20.04 + steps: + - name: vars + id: vars + run: | + # echo ::set-output name=container_registry::ghcr.io + # echo ::set-output name=cr_user::${{ github.repository_owner }} + echo ::set-output name=container_registry::docker.io + echo ::set-output name=cr_user::hpccbuilds + echo ::set-output name=build_prbase_sha::${{ github.event.pull_request.base.sha }} + echo ::set-output name=build_prbase_label::${{ github.base_ref }} + echo ::set-output name=build_user::${{ github.actor }} + echo ::set-output name=build_type::RelWithDebInfo + echo ::set-output name=use_cppunit::1 + echo ::set-output name=platform_build::smoketest-platform-build-vcpkg + + - name: tracing + run: | + echo "Base ref = ${{ github.ref }}" + echo "Action = ${{ github.action }}" + echo "Event = ${{ github.event_name }}" + echo "Actor = ${{ github.actor }}" + echo "Ref = ${{ github.ref }}" + echo "base sha = ${{ github.event.pull_request.base.sha }}" + echo "Sha = ${{ github.sha }}" + echo "github.repository = ${{ github.repository }}" + echo "repository_owner = ${{ github.repository_owner }}" + echo "github.workspace = ${{ github.workspace }}" + echo "runner.workspace = ${{ runner.workspace }}" + echo "github.event.pull_request.head.repo.owner.login = ${{ github.event.pull_request.head.repo.owner.login }}" + echo "build_prbase_label = ${{ steps.vars.outputs.build_prbase_label }}" + echo "platform_build = ${{ steps.vars.outputs.platform_build }}" + + - name: Checkout PR + uses: actions/checkout@v3 + + - name: Calculate vcpkg vars + id: vcpkg_vars + run: | + echo "base_ver=$(git submodule status vcpkg | cut -c2-9)" >> $GITHUB_OUTPUT + + - name: tracing (vcpkg_vars) + run: | + echo "vcpkg_vars.base_ver = ${{ steps.vcpkg_vars.outputs.base_ver }}" + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + with: + driver: docker + + - name: Check if PR-Base prebuilt + id: check-images + run: | + prbase_missing=$(docker manifest inspect ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-${{ github.event.pull_request.base.sha }} > /dev/null ; echo $?) + echo prbase_missing=${prbase_missing} + if [[ "${prbase_missing}" -eq 1 ]] + then + echo "Current PR target branch image cannot be found, using latest" + prbase_missing=$(docker manifest inspect ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-latest > /dev/null ; echo $?) + if [[ "${prbase_missing}" -eq 1 ]] + then + echo "Cannot find the 'latest' target branch image" + echo ::set-output name=prbase_missing::${prbase_missing} + fi + echo ::set-output name=platform_prbase_ver::${{ steps.vars.outputs.build_prbase_label }}-latest + else + echo ::set-output name=platform_prbase_ver::${{ steps.vars.outputs.build_prbase_label }}-${{ github.event.pull_request.base.sha }} + fi + + # Normal expectation is that the following step would normally be skipped + # NB: This is only for the case where the target branch image doesn't already exist. + # The build-containers-target-branch.yml action would normally have built/published this image, + # or they'll be a "latest" which will have been picked up by the check-images step. + - name: branch image + if: ${{ steps.check-images.outputs.prbase_missing == '1' }} + uses: docker/build-push-action@v2 + with: + context: . + file: ./dockerfiles/platform-build/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + tags: | + ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-${{ steps.vars.outputs.build_prbase_sha }} + ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-latest + build-args: | + CR_REPO=${{ steps.vars.outputs.container_registry }} + BASE_VER=${{ steps.vcpkg_vars.outputs.base_ver }} + BUILD_USER=${{ github.repository_owner }} + GITHUB_ACTOR=${{ github.actor }} + BUILD_TAG=${{ steps.vars.outputs.build_prbase_sha }} + BUILD_TYPE=${{ steps.vars.outputs.build_type }} + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + USE_CPPUNIT=${{ steps.vars.outputs.use_cppunit }} + BUILD_THREADS=${{ steps.vars.outputs.build_threads }} + + - name: PR image + uses: docker/build-push-action@v2 + with: + context: . + file: ./dockerfiles/platform-build-incremental-container/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + build-args: | + CR_USER=${{ steps.vars.outputs.cr_user }} + CR_REPO=${{ steps.vars.outputs.container_registry }} + PLATFORM_PRBASE_VER=${{ steps.check-images.outputs.platform_prbase_ver }} + GITHUB_REPO=${{ github.repository }} + GITHUB_PRREF=${{ github.ref }} + BUILD_THREADS=${{ steps.vars.outputs.build_threads }} diff --git a/.github/workflows/build-containers-target-branch.yml b/.github/workflows/build-containers-target-branch.yml new file mode 100644 index 00000000000..dafe221a23e --- /dev/null +++ b/.github/workflows/build-containers-target-branch.yml @@ -0,0 +1,112 @@ +name: Docker target branch build +on: + push: + branches: + - "master" + - "candidate-*.x" + - "!candidate-8.2.*" + - "!candidate-8.0.*" + - "!candidate-7.12.*" + - "!candidate-7.10.*" + - "!candidate-7.8.*" + - "!candidate-7.6.*" + - "!candidate-7.4.*" + - "!candidate-7.2.*" + - "!candidate-7.0.*" + - "!candidate-6.*" + +jobs: + check-skip: + # continue-on-error: true # Uncomment once integration is finished + runs-on: ubuntu-20.04 + # Map a step output to a job output + outputs: + changed: ${{ steps.skip_check.outputs.dockerfiles || steps.skip_check.outputs.platform }} + steps: + - id: skip_check + uses: hpcc-systems/github-actions/changed-modules@main + with: + github_token: ${{ github.token }} + + build-images: + needs: check-skip + if: ${{ needs.check-skip.outputs.changed }} + runs-on: ubuntu-20.04 + steps: + - name: vars + id: vars + run: | + # echo ::set-output name=container_registry::ghcr.io + # echo ::set-output name=cr_user::${{ github.repository_owner }} + echo ::set-output name=container_registry::docker.io + echo ::set-output name=cr_user::hpccbuilds + echo ::set-output name=build_base_sha::${{ github.sha }} + echo ::set-output name=build_base_label::${GITHUB_REF##*/} + echo ::set-output name=build_user::${{ github.actor }} + echo ::set-output name=build_type::RelWithDebInfo + echo ::set-output name=use_cppunit::1 + echo ::set-output name=platform_build::smoketest-platform-build-vcpkg + + - name: tracing + run: | + echo "Action = ${{ github.action }}" + echo "Event = ${{ github.event_name }}" + echo "Actor = ${{ github.actor }}" + echo "Ref = ${{ github.ref }}" + echo "Sha = ${{ github.sha }}" + echo "github.repository = ${{ github.repository }}" + echo "repository_owner = ${{ github.repository_owner }}" + echo "github.workspace = ${{ github.workspace }}" + echo "runner.workspace = ${{ runner.workspace }}" + echo "build_base_sha = ${{ steps.vars.outputs.build_base_sha }}" + echo "build_base_label = ${{ steps.vars.outputs.build_base_label }}" + + - name: Checkout + uses: actions/checkout@v2 + + - name: Calculate vcpkg vars + id: vcpkg_vars + run: | + echo "base_ver=$(git submodule status vcpkg | cut -c2-9)" >> $GITHUB_OUTPUT + + - name: tracing (vcpkg_vars) + run: | + echo "vcpkg_vars.base_ver = ${{ steps.vcpkg_vars.outputs.base_ver }}" + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + # - name: Login to GitHub Container Registry + # uses: docker/login-action@v1 + # with: + # registry: ghcr.io + # username: ${{ github.repository_owner }} + # password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + # build branch image + - name: branch image + uses: docker/build-push-action@v2 + with: + context: . + file: ./dockerfiles/platform-build/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + tags: | + ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_base_label }}-${{ steps.vars.outputs.build_base_sha }} + ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_base_label }}-latest + push: true + build-args: | + CR_REPO=${{ steps.vars.outputs.container_registry }} + BASE_VER=${{ steps.vcpkg_vars.outputs.base_ver }} + BUILD_USER=${{ github.repository_owner }} + BUILD_TAG=${{ steps.vars.outputs.build_base_sha }} + BUILD_TYPE=${{ steps.vars.outputs.build_type }} + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + USE_CPPUNIT=${{ steps.vars.outputs.use_cppunit }} + BUILD_THREADS=${{ steps.vars.outputs.build_threads }} diff --git a/.github/workflows/build-test-eclwatch.yml b/.github/workflows/build-test-eclwatch.yml new file mode 100644 index 00000000000..13eee0bfa2a --- /dev/null +++ b/.github/workflows/build-test-eclwatch.yml @@ -0,0 +1,56 @@ +name: Build Test ECL Watch + +on: + pull_request: + branches: + - "master" + - "candidate-*" + - "!candidate-7.6.*" + - "!candidate-7.4.*" + - "!candidate-7.2.*" + - "!candidate-7.0.*" + - "!candidate-6.*" + +jobs: + pre_job: + runs-on: ubuntu-20.04 + outputs: + eclwatch: ${{ steps.skip_check.outputs.eclwatch }} + steps: + - id: skip_check + uses: hpcc-systems/github-actions/changed-modules@main + with: + github_token: ${{ github.token }} + + build: + strategy: + matrix: + node: ["18", "16"] + fail-fast: false + name: "Check eclwatch and npm" + needs: pre_job + if: ${{ needs.pre_job.outputs.eclwatch }} + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + submodules: "recursive" + - uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node }} + - name: Cache node modules + uses: actions/cache@v2 + with: + path: ./esp/src/node_modules + key: npm-deps-${{ hashFiles('./esp/src/package-lock.json') }} + restore-keys: | + npm-deps-${{ hashFiles('./esp/src/package-lock.json') }} + - name: Install Dependencies + working-directory: ./esp/src + run: npm ci + - name: Build + working-directory: ./esp/src + run: npm run build + - name: Test + working-directory: ./esp/src + run: npm run test diff --git a/.github/workflows/build-vcpkg.yml b/.github/workflows/build-vcpkg.yml index a3839432fcc..70a4fa5f179 100644 --- a/.github/workflows/build-vcpkg.yml +++ b/.github/workflows/build-vcpkg.yml @@ -35,7 +35,7 @@ jobs: internal_ref: ${{ steps.vars.outputs.internal_ref }} community_tag: ${{ steps.vars.outputs.community_tag }} internal_tag: ${{ steps.vars.outputs.internal_tag }} - community_branch: ${{ steps.vars.outputs.community_branch }} + candidate_base_branch: ${{ steps.vars.outputs.candidate_base_branch }} cmake_docker_config: ${{ steps.vars.outputs.cmake_docker_config }} platform: "1" platform_testing_do_not_release: ${{ steps.skip_check.outputs.platform }} @@ -56,9 +56,8 @@ jobs: community_tag=$(echo $community_ref | cut -d'/' -f3) echo "community_tag=$community_tag" >> $GITHUB_OUTPUT echo "internal_tag=$(echo $community_tag | sed 's/community/internal/')" >> $GITHUB_OUTPUT - community_base_ref=${{ github.event.base_ref || github.ref }} - echo "community_branch=$(echo $community_base_ref | cut -d'/' -f3)" >> $GITHUB_OUTPUT - echo "cmake_docker_config=-DCMAKE_BUILD_TYPE=RelWithDebInfo -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF" >> $GITHUB_OUTPUT + echo "candidate_base_branch=${{ github.base_ref }}" >> $GITHUB_OUTPUT + echo "cmake_docker_config=-DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF" >> $GITHUB_OUTPUT - id: skip_check uses: hpcc-systems/github-actions/changed-modules@main @@ -72,15 +71,15 @@ jobs: build-docker: name: Build Docker - # if: github.repository == 'hpcc-systems/HPCC-Platform' needs: preamble runs-on: ubuntu-22.04 strategy: matrix: include: - - os: ubuntu-22.10 + - os: ubuntu-23.04 event_name: "pull_request" - os: ubuntu-22.04 + name: k8s container: true event_name: "schedule" - os: ubuntu-22.04 @@ -94,11 +93,16 @@ jobs: - os: amazonlinux event_name: "schedule" - os: centos-7 + name: LN ln: true event_name: "schedule" fail-fast: false steps: + - name: Free additional disk space (remove Android SDK + Tools) + run: | + sudo rm -rf /usr/local/lib/android + - name: Checkout HPCC-Platform if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} uses: actions/checkout@v3 @@ -122,7 +126,12 @@ jobs: id: vars working-directory: ${{ needs.preamble.outputs.folder_platform }}/vcpkg run: | - echo "vcpkg_sha_short=$(git rev-parse --short=8 HEAD)" >> $GITHUB_OUTPUT + vcpkg_sha_short=$(git rev-parse --short=8 HEAD) + echo "vcpkg_sha_short=$vcpkg_sha_short" >> $GITHUB_OUTPUT + docker_build_label=hpccsystems/platform-build-${{ matrix.os }} + echo "docker_build_label=$docker_build_label" >> $GITHUB_OUTPUT + echo "docker_tag=$docker_build_label:$vcpkg_sha_short" >> $GITHUB_OUTPUT + echo "docker_tag_candidate_base=$docker_build_label:${{ needs.preamble.outputs.candidate_base_branch }}" >> $GITHUB_OUTPUT - name: Print vars run: | @@ -136,7 +145,7 @@ jobs: - name: Create Build Image if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: builder: ${{ steps.buildx.outputs.name }} file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/${{ matrix.os }}.dockerfile @@ -145,27 +154,32 @@ jobs: build-args: | VCPKG_REF=${{ steps.vars.outputs.vcpkg_sha_short }} tags: | - build-${{ matrix.os }}:latest + ${{ steps.vars.outputs.docker_tag_candidate_base }} + cache-from: | + type=registry,ref=${{ steps.vars.outputs.docker_tag_candidate_base }} + type=registry,ref=${{ steps.vars.outputs.docker_tag }} + cache-to: type=inline + # Communtiy Build - name: CMake Packages - if: ${{ !matrix.container && !matrix.ln && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} + if: ${{ !matrix.ln && !matrix.container && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} run: | mkdir -p ${{ needs.preamble.outputs.folder_build }} declare -a plugins if [ ${{ needs.preamble.outputs.include_plugins }} == "ON" ]; then - plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED" "PLATFORM") + plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM") else plugins=("PLATFORM") fi for plugin in "${plugins[@]}"; do sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles - docker_label=build-${{ matrix.os }}:latest - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF" - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\ + cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF && \ + cmake --build /hpcc-dev/build --parallel $(nproc) --target package" # Disabled as not currently needed --- - # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON" - # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON" + # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" done - name: CMake Containerized Packages @@ -174,29 +188,30 @@ jobs: mkdir -p ${{ needs.preamble.outputs.folder_build }} sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles - docker_label=build-${{ matrix.os }}:latest - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF" - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\ + cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF && \ + cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + # Internal Build --- - name: CMake LN Packages - if: ${{ matrix.ln && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} + if: ${{ matrix.ln && !matrix.container && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} run: | mkdir -p ${{ needs.preamble.outputs.folder_build }} sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles - docker_label=build-${{ matrix.os }}:latest - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF" - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\ + cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF && \ + cmake --build /hpcc-dev/build --parallel $(nproc) --target package" # Disabled as not currently needed --- - # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON" - # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON" + # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + # Common --- - name: Upload error logs if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v3 with: - name: ${{ matrix.os }}-${{ matrix.package }}-logs + name: ${{ matrix.os }}-${{ matrix.ln }}-${{ matrix.container }}-logs path: ${{ needs.preamble.outputs.folder_build }}/**/*.log build-bare-metal: @@ -341,5 +356,5 @@ jobs: if: ${{ failure() || cancelled() }} uses: actions/upload-artifact@v3 with: - name: ${{ matrix.os }}-${{ matrix.package }}-logs + name: ${{ matrix.os }}--${{ matrix.ln }}-${{ matrix.container }}-logs path: ./build/**/*.log diff --git a/.github/workflows/codeql-eclwatch.yml b/.github/workflows/codeql-eclwatch.yml new file mode 100644 index 00000000000..14a0635f3d4 --- /dev/null +++ b/.github/workflows/codeql-eclwatch.yml @@ -0,0 +1,83 @@ +name: CodeQL ECL Watch + +on: + push: + branches: + - "master" + - "candidate-*" + - "!candidate-7.6.*" + - "!candidate-7.4.*" + - "!candidate-7.2.*" + - "!candidate-7.0.*" + - "!candidate-6.*" + pull_request: + branches: + - "master" + - "candidate-*" + - "!candidate-7.6.*" + - "!candidate-7.4.*" + - "!candidate-7.2.*" + - "!candidate-7.0.*" + - "!candidate-6.*" + schedule: + - cron: "0 7 * * 1" + +jobs: + pre_job: + runs-on: ubuntu-20.04 + # Map a step output to a job output + outputs: + eclwatch: ${{ steps.skip_check.outputs.eclwatch }} + steps: + - id: skip_check + uses: hpcc-systems/github-actions/changed-modules@main + with: + github_token: ${{ github.token }} + + analyze: + name: Analyze + + strategy: + fail-fast: false + matrix: + language: ["javascript"] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + needs: pre_job + if: ${{ needs.pre_job.outputs.eclwatch && github.repository == 'hpcc-systems/HPCC-Platform' }} + runs-on: ubuntu-20.04 + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + # - name: Autobuild + # uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 00000000000..f5a8dfd8bf9 --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,35 @@ +name: Deploy vitepress content to Pages + +on: + push: + branches: + - master + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + # Build job + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: 16 + + - run: npm ci + - run: npm run build + + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./devdoc/.vitepress/dist + force_orphan: true + # cname: example.com # if wanna deploy to custom domain + env: + CI: true diff --git a/.github/workflows/jirabot.yml b/.github/workflows/jirabot.yml new file mode 100644 index 00000000000..7ee726192de --- /dev/null +++ b/.github/workflows/jirabot.yml @@ -0,0 +1,103 @@ +# JiraBot github action +# ===================== +# +name: jirabot + +on: + pull_request_target: + types: [opened, reopened] + branches: + - "master" + - "candidate-*" + +jobs: + jirabot: + runs-on: ubuntu-20.04 + steps: + - uses: "actions/setup-python@v2" + with: + python-version: "3.8" + - name: "Install dependencies" + run: | + set -xe + python -VV + python -m site + python -m pip install --upgrade pip setuptools wheel + python -m pip install --upgrade jira + - name: "Run" + env: + JIRABOT_USERNAME : ${{ secrets.JIRABOT_USERNAME }} + JIRABOT_PASSWORD : ${{ secrets.JIRABOT_PASSWORD }} + JIRA_URL : ${{ secrets.JIRA_URL }} + PULL_REQUEST_NUMBER : ${{ github.event.pull_request.number }} + PULL_REQUEST_TITLE : ${{ github.event.pull_request.title }} + PULL_REQUEST_AUTHOR_NAME : ${{ github.event.pull_request.user.login }} + PULL_URL: ${{ github.event.pull_request.html_url }} + COMMENTS_URL: ${{ github.event.pull_request.comments_url }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + run: | + import os + import re + from jira.client import JIRA + + jirabot_user = os.environ['JIRABOT_USERNAME'] + jirabot_pass = os.environ['JIRABOT_PASSWORD'] + jira_url = os.environ['JIRA_URL'] + pr = os.environ['PULL_REQUEST_NUMBER'] + title = os.environ['PULL_REQUEST_TITLE'] + user = os.environ['PULL_REQUEST_AUTHOR_NAME'] + comments_url = os.environ['COMMENTS_URL'] + pull_url = os.environ['PULL_URL'] + github_token = os.environ['GITHUB_TOKEN'] + + print("%s %s %s" % (title, user, comments_url)) + status = '' + issuem = re.search("(HPCC|HH|IDE|EPE|ML|JAPI)-[0-9]+", title) + if issuem: + issue_name = issuem.group() + if user == 'dehilsterlexis': + user = 'dehilster' + if user == 'kunalaswani': + user = 'kunal.aswani' + if user == 'timothyklemm': + user = 'klemti01' + if user == 'jpmcmu': + user = 'mcmuja01' + if user == 'asselitx': + user = 'terrenceasselin' + if user == 'jeclrsg': + user = 'clemje01' + if user == 'jackdelv': + user = 'delvecja' + options = { + 'server': jira_url + } + jira = JIRA(options=options, basic_auth=(jirabot_user, jirabot_pass)) + issue = jira.issue(issue_name) + status = jira_url + '/browse/' + issue_name + '\\n' + if False and issue.fields.status.name != 'Active' and issue.fields.status.name != 'Open' and issue.fields.status.name != 'New' and issue.fields.status.name != 'Discussing' and issue.fields.status.name != 'Awaiting Information': + status += 'Jira not updated (state was not active or new)' + elif issue.fields.customfield_10010 != None: + if issue.fields.customfield_10010 != pull_url: + status += 'Jira not updated (pull request "%s" already registered)' % issue.fields.customfield_10010 + else: + status += 'This pull request is already registered' + elif issue.fields.assignee is not None and issue.fields.assignee.name.lower() != user.lower(): + status += 'Jira not updated (user does not match)' + else: + if issue.fields.assignee is None: + jira.assign_issue(issue, user) + issue.update(fields={'customfield_10010': pull_url}) + issue = jira.issue(issue_name) + try: + transitions = jira.transitions(issue) + jira.transition_issue(issue, '291') # Attach Pull Request + except: + status += 'Failed to set to merge pending: transitions=%s' % transitions + status += 'Jira updated' + print('curl -X POST %s -H "Content-Type: application/json" -H "Authorization: token %s" --data \'{ "body": "%s" }\'' % ( comments_url, github_token, status )) + os.system('curl -X POST %s -H "Content-Type: application/json" -H "Authorization: token %s" --data \'{ "body": "%s" }\'' % ( comments_url, github_token, status )) + + print(status) + shell: python diff --git a/.github/workflows/nightly-publish.yml b/.github/workflows/nightly-publish.yml new file mode 100644 index 00000000000..3aab3451a45 --- /dev/null +++ b/.github/workflows/nightly-publish.yml @@ -0,0 +1,26 @@ +name: Nightly master build and publish +on: + schedule: + - cron: '0 0 * * *' + +jobs: + build: + name: "Nightly build and publish release container" + runs-on: ubuntu-20.04 + if: github.repository == 'hpcc-systems/HPCC-Platform' + steps: + - name: vars + id: vars + run: | + echo "::set-output name=date::$(date +'%Y-%m-%d')" + - name: Checkout + uses: actions/checkout@v2 + - name: Build + uses: ./dockerfiles + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + build_ver: "master" + build_label: "master-${{ steps.vars.outputs.date }}" + + diff --git a/.github/workflows/smoketest.yml b/.github/workflows/smoketest.yml new file mode 100644 index 00000000000..0428925cbaf --- /dev/null +++ b/.github/workflows/smoketest.yml @@ -0,0 +1,592 @@ +# Smoketest github action +# ======================= +# +# Uses cached instances of previous builds to perform partial (and therefore quicker) incremental builds. +# Once the platform is built and installed (make install), +# it runs the regression suite setup stage within the same github job ('build-and-setup'). +# +# On success, an artifact is built, that contains the install binaries and +# the HPCC binaries that have accumulated from the setup stage. +# +# Once the 'build-and-setup' job is complete, the dependent regression suite jobs and unittest job are launched in parallel. +# The regression suite queries are manually sub-divided into chunks (alphabeticaly), and run via a job matrix for parallism. +# If anything fails, all jobs are aborted (fail-fast=true), and the logs are captured into a published artifact. +# +# NOTES: +# + pre-requisite build dependencies (and runtime dependencies) are listed manually, and must be kept up to date with the +# requirements of the platform. MORE: a list of required build dependencies could be kept within the platform source and picked up +# + 'cacheversion' is purely in case it is necessary to force a cache-miss, i.e. all cached items are based on this version +# + Caching is via github's actions/cache, and is limited to 5GB total per repository, with oldest ejected first, and/or >7 days +# cached builds are tagged with: +# 1) base+ref+SHA (exact match, e.g. hpccbuild-1-7.12.10-6c981c48ae6e35b62d86d8e59e42799c5cd14812) +# 2) base_ref (branch match, e.g. hpccbuild-1-7.12.10) +# 3) base-ref major-minor only (e.g. hpccbuild-1-7.12.) +# 4) base-ref major only (e.g. hpccbuild-1-7.) +# 5) generic cacheversion only (e.g. hpccbuild-1-) +# The cache will initially try to match an exact key match (only true if rerunning PR and SHA same), +# and will then failover to trying to find a match of 2-5, in that order. + + +name: smoketest +env: + cacheversion: 3 + VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,read" + OS_DEPENDENCIES: "bison flex build-essential binutils-dev curl lsb-release libcppunit-dev python3-dev default-jdk + r-base-dev r-cran-rcpp r-cran-rinside r-cran-inline pkg-config libtool autotools-dev automake git cmake" + +on: + pull_request: + branches: + - "master" + - "newbuild" + - "candidate-*" + - "!candidate-7.6.*" + - "!candidate-7.4.*" + - "!candidate-7.2.*" + - "!candidate-7.0.*" + - "!candidate-6.*" + +# NB: this fails to cancel in-flight actions, with error 'Resource not accessible by integration', +# but it's non-fatal. Appears to be dependent on what permissions the PR owner has, i.e. the github token used +# does not have permission to cancel actions. +jobs: + check-skip: + # continue-on-error: true # Uncomment once integration is finished + runs-on: ubuntu-22.04 + # Map a step output to a job output + outputs: + platform: ${{ contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') || steps.skip_check.outputs.platform }} + eclwatch: ${{ contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') || steps.skip_check.outputs.eclwatch }} + + steps: + - id: skip_check + uses: hpcc-systems/github-actions/changed-modules@main + with: + github_token: ${{ github.token }} + + build-and-setup: + name: Build platform and regress setup + needs: check-skip + if: ${{ needs.check-skip.outputs.platform || needs.check-skip.outputs.eclwatch }} + runs-on: ubuntu-22.04 + timeout-minutes: 150 # the build step has its own timeout, however the job can sometimes take time to download dependencies + outputs: + cache-state: ${{ steps.check-cache.outputs.state }} + + steps: + - name: Setup and git clone + id: vars + run: | + if [[ "${{ github.event_name }}" = 'pull_request' ]] + then + echo ::set-output name=branch::${{ github.head_ref }} + echo ::set-output name=base_ref::${{ github.base_ref }} + versioned=$(echo ${{ github.base_ref }} | sed -E -n 's/^(.*)-([0-9]+)\.([0-9]+)\.([0-9]+|x)(-[0-9]+|).*$/\1 \2 \3 \4 \5/p') + if [[ "${versioned}" ]] + then + echo "matched version = ${versioned}" + IFS=' ' read prefix major minor point cand <<< ${versioned} + echo "name=base-majorminor-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}.${minor}." + echo "name=base-majoronly-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}." + echo ::set-output name=base-majorminor-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}.${minor}. + echo ::set-output name=base-majoronly-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}. + fi + else + echo ::set-output name=branch::${GITHUB_REF#refs/heads/} + echo ::set-output name=base_ref::${GITHUB_REF#refs/heads/} + fi + if [[ "${{ github.event.pull_request.head.repo.owner.login }}" = "${{ github.repository_owner }}" ]] + then + repository=${{ github.repository }} + else + repository=$(echo -n ${{ github.event.pull_request.head.repo.owner.login }}/ ; echo -n ${{ github.repository }} | sed -E -n 's@^[^/]+/(.+)$@\1@p') + fi + echo ::set-output name=repository::${repository} + echo "Cloning https://github.com/${repository}" + git clone -n https://github.com/${repository} src + if [[ "${{ github.event_name }}" = 'pull_request' ]] + then + cd src + git remote add upstream https://github.com/${{ github.repository }} + git fetch upstream +${{ github.ref }} + fetch_sha=$(git rev-parse --verify FETCH_HEAD) + git fetch upstream ${{ github.base_ref}} + base_ref_sha=$(git rev-parse upstream/${{ github.base_ref }}) + echo ::set-output name=base_ref_sha::${base_ref_sha} + echo ::set-output name=sha::${fetch_sha} + echo "base_ref_sha = ${base_ref_sha}" + echo "fetch_sha = ${fetch_sha}" + else + echo ::set-output name=sha::${{ github.sha }} + echo ::set-output name=base_ref_sha::${{ github.sha }} + fi + + # NB: restore-keys set to e.g. hpccbuild-2-7.12.10, hpccbuild-2-7.12, hpccbuild-2-7, hpccbuild-2- + - name: Fetch cache + id: cache + uses: actions/cache@v2 + with: + path: | + build-cache + merge-patch + src-cache + key: hpccbuild-${{ env.cacheversion }}-${{ steps.vars.outputs.base_ref }}-${{ steps.vars.outputs.sha }} + restore-keys: | + hpccbuild-${{ env.cacheversion }}-${{ steps.vars.outputs.base_ref }} + ${{ steps.vars.outputs.base-majorminor-restore-key }} + ${{ steps.vars.outputs.base-majoronly-restore-key }} + hpccbuild-${{ env.cacheversion }}- + + - name: tracing + run: | + echo "Branch = ${{ steps.vars.outputs.branch }}" + echo "Base ref = ${{ steps.vars.outputs.base_ref }}" + echo "SHA = ${{ steps.vars.outputs.sha}}" + echo "Action = ${{ github.action }}" + echo "Event = ${{ github.event_name }}" + echo "Actor = ${{ github.actor }}" + echo "Ref = ${{ github.ref }}" + echo "Sha = ${{ github.sha }}" + echo "github.repository = ${{ github.repository }}" + echo "repository = ${{ steps.vars.outputs.repository }}" + echo "repository_owner = ${{ github.repository_owner }}" + echo "github.workspace = ${{ github.workspace }}" + echo "runner.workspace = ${{ runner.workspace }}" + echo "github.event.pull_request.head.repo.owner.login = ${{ github.event.pull_request.head.repo.owner.login }}" + + if [[ "${{ github.event_name }}" = 'pull_request' ]] + then + echo "PR base_ref = ${{ github.base_ref }}" + echo "PR head_ref = ${{ github.head_ref }}" + echo "PR base SHA = ${{ github.event.pull_request.base.sha }}" + fi + echo "restore-key1: hpccbuild-${{ env.cacheversion }}-${{ steps.vars.outputs.base_ref }}" + echo "restore-key2: ${{ steps.vars.outputs.base-majorminor-restore-key }}" + echo "restore-key3: ${{ steps.vars.outputs.base-majoronly-restore-key }}" + echo "restore-key4: hpccbuild-${{ env.cacheversion }}-" + + # NB: actions/cache does not set cache-hit to true if restore-keys used. + # So here we: + # 1) check if a cache was restored + # 2) check that the SHA stored in the cache can be found in git + # 3) mv cache into place (build-cache->build) + # 4) checkout source @ base_ref + # 5) get submodules + # 6) apply git merge-patch from cache, bringing source up-to-date with the source used in the cached binary build + # 7) copy src-cache files into place + # 8) touch all files so that 'old' [before original make timestamp], i.e. so behind objects in cache + # 9) checkout pr branch - ensuring only pr files that differ are updated (and hence will rebuild) + # 10) list updated files (for tracing only) + # 11) set 'state' output variable to 'miss' if not absolute match, + # and set to 'hit' if cache is usable. i.e. if set at all, run build and tests. + - name: Check cache + id: check-cache + if: ${{ (steps.cache.outputs.cache-hit != 'true') || contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') }} + run: | + if [[ ("${{ steps.cache.outputs.cache-hit }}" == 'true') && ("${{ contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') }}" == 'true') ]] + then + echo "Exact cache match, but 'smoketest-force-rerun' label forced rerun" + fi + echo ::set-output name=state::miss + if [[ "${{ contains(github.event.pull_request.labels.*.name, 'smoketest-ignore-cache') }}" == 'true' ]] + then + echo "Cache found, but ignored because 'smoketest-ignore-cache' label set" + rm -rf build-cache merge-patch src-cache + else + if [[ -d build-cache ]] + then + echo "Attempting to use existing cache" + stat build-cache merge-patch src-cache + du -sh build-cache merge-patch src-cache + cd src + if [[ "commit" == "$(git cat-file -t $(cat ../build-cache/cache-sha))" ]] + then + mv ../build-cache ../build + echo "Checking out cache SHA (cached github.base_ref SHA): $(cat ../build/cache-sha)" + git checkout $(cat ../build/cache-sha) + git submodule update --init --recursive --jobs 4 + echo "Applying merge patch" + git apply ../merge-patch + # Add all mods that came from merge-patch into a commit, + # so that the final git checkout will not clash with working tree files + git add -A + git -c user.name='Smoketest' -c user.email='smoke@test.com' commit -m 'merge-patch' + cd ../src-cache + echo "Cached source tree files:" + find . -type f + echo "================" + find . -type f | cpio -p -dum ../src + cd ../src + find . -type f -exec touch -r ../build/cache-timestamp {} + + git checkout ${{ steps.vars.outputs.sha }} + git submodule update --init --recursive --jobs 4 + echo "Changed files (from SHA: $(cat ../build/cache-sha)):" + find -name '.git*' -prune -o -newer ../build/cache-timestamp -type f -print + echo ::set-output name=state::hit + else + echo "Cached SHA $(cat ../build-cache/cache-sha) could not be found). Skipping cache." + fi + fi + fi + + - name: Dependencies22.04 + if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set then step needed + run: | + sudo apt-get update + sudo apt-get install -y ${{ env.OS_DEPENDENCIES }} + sudo apt-get install -y xmlstarlet + + curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - + sudo apt-get install -y nodejs + + echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" + + - name: Cache miss prep + if: ${{ steps.check-cache.outputs.state == 'miss' }} + run: | + rm -rf build build-cache merge-patch src-cache + mkdir build + mkdir src-cache + cd src + git checkout ${{ steps.vars.outputs.sha }} + git submodule update --init --recursive --jobs 4 + + - name: "Setup NuGet credentials" + if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set then step needed + run: | + mono `vcpkg fetch nuget | tail -n 1` \ + sources add \ + -source "https://nuget.pkg.github.com/hpcc-systems/index.json" \ + -storepasswordincleartext \ + -name "GitHub" \ + -username "${{ github.actor }}" \ + -password "${{ secrets.GITHUB_TOKEN }}" + + - name: "Remove builtin vcpkg" + if: ${{ github.event_name == matrix.event_name && needs.check-skip.outputs.platform }} + shell: "bash" + run: | + sudo rm -rf "$VCPKG_INSTALLATION_ROOT" + + - name: "vcpkg Bootstrap" + if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set then step needed + working-directory: ./src/vcpkg + run: | + ./bootstrap-vcpkg.sh + + # Subsequent cache hit builds use cache-timestamp to ensure all sources are marked 'old', + # except those changed between the cache SHA and the SHA being built. + # NB: BUILD_TAG is overridden to a constant "smoketest", this is to prevent the auto-generated build tag being + # used, which would cause it to change per PR (per SHA), and because it's in a common header, cause a lot of undesirable rebuilding. + - name: Build + timeout-minutes: 120 # Generous, typical build time from clean is < 60 + if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set to anything, then step needed + run: | + cd build + echo "removing CMakeCache.txt" + rm -f CMakeCache.txt + if [[ ! -f cache-timestamp ]]; then touch cache-timestamp; fi + touch ../cmake-timestamp + `../src/vcpkg/vcpkg fetch cmake | tail -n 1` ../src -Wno-dev -DRUNTIME_USER=$(whoami) -DRUNTIME_GROUP=$(id -gn) -DDESTDIR=$(realpath ..)/install -DINCLUDE_PLUGINS=1 -DCMAKE_BUILD_TYPE=Release -DUSE_PYTHON2=0 -DUSE_PYTHON3=1 -DSUPPRESS_SPARK=1 -DUSE_CPPUNIT=1 -DUSE_CASSANDRA=Off -DCMAKE_CXX_FLAGS="-DBUILD_TAG=\\\"dummytag\\\" -DBUILD_VERSION_MAJOR=1 -DBUILD_VERSION_MINOR=1 -DBUILD_VERSION_POINT=0" -DBUILD_TAG="smoketest" -DSTRIP_RELEASE_SYMBOLS=0 -DCMAKE_CXX_FLAGS="-DTRACE_GLOBAL_GROUP=1" + make -j$(nproc) install + + - name: build-logs-artifact + if: ${{ failure() }} + uses: actions/upload-artifact@v2 + with: + name: build-logs + path: | + build/CMakeCache.txt + build/CMakeFiles/CMakeOutput.log + build/CMakeFiles/CMakeError.log + + # Cache contains: + # - make binaries (build-cache) + # - Any sources written since cache-timestamp within the source dir (src-cache) (e.g. AWS writes a config header) + # - cache-timestamp, timestamp of cache-miss build. All files except diffs set to this time, ensuring no existing binaries are rebuilt + # - cache-sha, the SHA of the base_ref used to build. On a cache hit, used to find changed files and ensure they are only ones rebuilt. + # - merge-path, the diffs between the base_ref and the PR commit at the time of PR/cache build. + - name: Prepare cache + if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set to anything, then step needed + run: | + echo ${{ steps.vars.outputs.base_ref_sha }} > build/cache-sha + mv build build-cache + cd src + find -name '.git*' -prune -o -newer ../cmake-timestamp -type f -print | cpio -p -dum ../src-cache + echo "Merge patch files:" + git diff --name-only ${{ steps.vars.outputs.base_ref_sha }} ${{ steps.vars.outputs.sha }} + git diff --binary ${{ steps.vars.outputs.base_ref_sha }} ${{ steps.vars.outputs.sha }} > ../merge-patch + + # - alter stock ecl-test.json, to point to install dir (using jq) + # - create a clean copy of the install directory (for later reuse) + # - rm hpcc-data from copy (the post-setup hpcc-data will be moved in when done) + - name: Prepare install artifact + if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }} + run: | + # configure ecl-test.json with correct paths + jq --arg rootdir "${{ github.workspace }}" \ + '.Regress.dropzonePath = $rootdir + "/install" + .Regress.dropzonePath | .Regress.setupExtraParams.OriginalTextFilesOsPath = $rootdir + "/install" + .Regress.setupExtraParams.OriginalTextFilesOsPath | .Regress.setupExtraDParams.HPCCBaseDir = $rootdir + "/install" + .Regress.setupExtraDParams.HPCCBaseDir | .Regress.regressionDir = $rootdir + "/regress" | .Regress.maxAttemptCount = "1" | .Regress.logDir = $rootdir + "/regress/log"' \ + install/opt/HPCCSystems/testing/regress/ecl-test.json > ecl-test.json + mv -f ecl-test.json install/opt/HPCCSystems/testing/regress/ecl-test.json + # configure environment.xml to slavesPerNode=2, channelsPerNode=1 + xmlstarlet ed -L -u 'Environment/Software/ThorCluster/@slavesPerNode' -v 2 -u 'Environment/Software/ThorCluster/@channelsPerSlave' -v 1 install/etc/HPCCSystems/environment.xml + cp ${{ github.workspace }}/src/.github/workflows/timeoutcmd install/opt/HPCCSystems/bin/ + cp ${{ github.workspace }}/src/.github/workflows/smoketest-preabort.sh install/opt/HPCCSystems/bin/ + mkdir copy + cp -rp install copy/ + rm -rf copy/install/var/lib/HPCCSystems/hpcc-data + rm -rf copy/install/var/lib/HPCCSystems/queries + + - name: Run regression suite setup + if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }} + timeout-minutes: 10 + run: | + export LANG="en_US.UTF-8" + sudo update-locale + source install/opt/HPCCSystems/sbin/hpcc_setenv + install/opt/HPCCSystems/etc/init.d/hpcc-init start + mkdir -p regress + cd install/opt/HPCCSystems/testing/regress + # force regression suite to timeout after 8 minutes, so it captures ZAP report of any inflight hung queries + timeoutcmd $((8 * 60)) \ + ./ecl-test setup --preAbort 'smoketest-preabort.sh' --pq 2 --generateStackTrace + grep Failure: ${{ github.workspace }}/regress/log/setup_*.log + if [[ "$(grep -ohP '(?<=^Failure: )[0-9]+$' ${{ github.workspace }}/regress/log/setup_*.log | awk '{s+=$1} END {print s}')" -gt 0 ]]; then exit 1; fi + ${{ github.workspace }}/install/opt/HPCCSystems/etc/init.d/hpcc-init stop + + - name: regression-setup-logs-artifact + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v2 + with: + name: regress-setup-logs + path: | + install/var/log/HPCCSystems + regress/ + + # - mv regression suite setup created data from hpcc-data and queries into copy of install + # - create tarball of install for artifact uploading + - name: Finalize install artifact + if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }} + run: | + mv install/var/lib/HPCCSystems/hpcc-data copy/install/var/lib/HPCCSystems/ + mv install/var/lib/HPCCSystems/queries copy/install/var/lib/HPCCSystems/ + [[ -d ${{ github.workspace }}/src/esp/src/test-ui ]] && cp -r ${{ github.workspace }}/src/esp/src/test-ui copy/install/ + cd copy + tar --zstd -cf ../install.tgz install + + - name: Upload install artifact + if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }} + uses: actions/upload-artifact@v2 + with: + name: installsetup-artifact + path: | + install.tgz + + # Matrix of jobs run in parallel once build+setup above completes successfully. + # All use the post-setup install.tgz artifact, that contains binaries and setup state + # Break each engine run into sets for speed + # NB: each regression suite job, runs these steps: + # - installs dependencies (probably too many for runtime) + # - Starts HPCC + # - Runs regression suite with params { engine, match[pattern,exclude] } + # TODO: needs to process results, capture report, to be assembled by workflow when all jobs done + regression-jobs: + needs: build-and-setup + if: ${{ needs.build-and-setup.outputs.cache-state != '' && github.event_name == 'pull_request' && needs.check-skip.outputs.platform }} + timeout-minutes: 60 # each matrix step has its own timeout, however the job can sometimes take time to download dependencies + runs-on: ubuntu-22.04 + strategy: + fail-fast: true + matrix: + engine: ['hthor', 'thor', 'roxie'] + match: + - pattern: '[a-d]*.ecl' + - pattern: '[e-h]*.ecl' + - pattern: '[i-l]*.ecl' + - pattern: '[m-q]*.ecl' + - pattern: '[r-u]*.ecl' + exclude: teststdlibrary.ecl + - pattern: '[v-z]*.ecl' + include: + - engine: 'hthor' + match: + pattern: teststdlibrary.ecl + - engine: 'thor' + match: + pattern: teststdlibrary.ecl + - engine: 'roxie' + match: + pattern: teststdlibrary.ecl + + steps: + - uses: actions/download-artifact@v2 + with: + name: installsetup-artifact + + - name: vars + id: vars + run: | + echo ::set-output name=matrix-setname::$(echo -n ${{ matrix.match.pattern }} | tr -c "[:alnum:]" _) + + - name: Prerequisites + run: | + sudo apt-get update + sudo apt-get install -y ${{ env.OS_DEPENDENCIES }} + sudo apt-get install -y gdb + + curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - + sudo apt-get install -y nodejs + + echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" + + - name: run + timeout-minutes: 30 # generous, each matrix step shouldn't take more than 10 mins + run: | + export LANG="en_US.UTF-8" + sudo update-locale + tar --zstd -xf install.tgz + source install/opt/HPCCSystems/sbin/hpcc_setenv + install/opt/HPCCSystems/etc/init.d/hpcc-init start + cd install/opt/HPCCSystems/testing/regress + + (cd ecl; ls ${{ matrix.match.pattern }}) > matches.tmp + echo queries are: + cat matches.tmp + if [[ -z "${{ matrix.match.exclude }}" ]] + then + queries="$(cat matches.tmp | tr '\n' ' ')" + else + queries="$(cd ecl; ls ${{ matrix.match.exclude }} | grep -v -f - ../matches.tmp | tr '\n' ' ')" + fi + echo queries after exclude: ${queries} + + # force regression suite to timeout after 28 minutes, so it captures ZAP report of any inflight hung queries + timeoutcmd $((28 * 60)) \ + ./ecl-test query --preAbort 'smoketest-preabort.sh' --pq 2 --target ${{ matrix.engine }} --excludeclass python2,embedded-r,embedded-js,3rdpartyservice,mongodb --generateStackTrace ${queries} + grep Failure: ${{ github.workspace }}/regress/log/${{ matrix.engine }}.*.log + if [[ "$(grep -oP '(?<=^Failure: )[0-9]+$' ${{ github.workspace }}/regress/log/${{ matrix.engine }}.*.log)" -gt 0 ]]; then exit 1; fi + + - name: regression-run-logs-artifact + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v2 + with: + name: regression-run-logs-${{ matrix.engine }}-${{ steps.vars.outputs.matrix-setname }} + path: | + install/var/log/HPCCSystems + regress/ + if-no-files-found: error + + + # NB: this doesn't really need the post-setup data files included in install.tgz + # but as this is relatively quick and in parallel with others, it probably doesn't matter + unittests: + needs: build-and-setup + if: ${{ needs.build-and-setup.outputs.cache-state != '' && github.event_name == 'pull_request' }} + timeout-minutes: 30 # the unittests run step has its own timeout, however the job can sometimes take time to download dependencies + runs-on: ubuntu-22.04 + steps: + - uses: actions/download-artifact@v2 + with: + name: installsetup-artifact + + - name: Prerequisites + run: | + sudo apt-get update + sudo apt-get install -y ${{ env.OS_DEPENDENCIES }} + sudo apt-get install -y gdb + + curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - + sudo apt-get install -y nodejs + + echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" + + - name: run + timeout-minutes: 10 # generous, expected time is approx 1 min. + run: | + export LANG="en_US.UTF-8" + sudo update-locale + tar --zstd -xf install.tgz + install/opt/HPCCSystems/bin/unittests + + # NB: this doesn't really need the post-setup data files included in install.tgz + # but as this is relatively quick and in parallel with others, it probably doesn't matter + eclwatch-ui-tests: + needs: build-and-setup + if: ${{ needs.build-and-setup.outputs.cache-state != '' && github.event_name == 'pull_request' && needs.check-skip.outputs.eclwatch }} + timeout-minutes: 30 # the ui-tests run step has its own timeout, however the job can sometimes take time to download dependencies + runs-on: ubuntu-22.04 + steps: + - uses: actions/download-artifact@v2 + with: + name: installsetup-artifact + + - name: Check-ECLWatch-UI-Test-Directory + id: check + run: | + tar --zstd -xf install.tgz + if [[ ! -d install/test-ui/tests ]] + then + echo "ECLWatch UI test-ui/tests directory missing." + else + javaFilesCount=$(find install/test-ui/tests/ -iname '*.java' -type f -print | wc -l ) + echo "Number of test java files is $javaFilesCount" + if [[ ${javaFilesCount} -eq 0 ]] + then + echo "No java files, do nothing." + else + echo ::set-output name=runtests::true + fi + fi + + - name: Prerequisites + if: steps.check.outputs.runtests + run: | + sudo apt-get update + sudo apt-get install -y git wget net-tools + sudo apt-get install -y tzdata unzip xvfb libxi6 + sudo apt-get install -y default-jdk + sudo apt-get install -y ${{ env.OS_DEPENDENCIES }} + sudo apt-get install -y gdb + + curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - + sudo apt-get install -y nodejs + + echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" + + - name: UI-Test-Prerequisites + if: steps.check.outputs.runtests + run: | + wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb + sudo apt-get install -y ./google-chrome-stable_current_amd64.deb + wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip + unzip chromedriver_linux64.zip + sudo mv chromedriver /usr/bin/chromedriver + sudo chown root:root /usr/bin/chromedriver + sudo chmod +x /usr/bin/chromedriver + wget https://selenium-release.storage.googleapis.com/3.141/selenium-server-standalone-3.141.59.jar + wget http://www.java2s.com/Code/JarDownload/testng/testng-6.8.7.jar.zip + unzip testng-6.8.7.jar.zip + + - name: run + timeout-minutes: 10 # generous, current runtime is ~1min, this should be increased if new tests are added + if: steps.check.outputs.runtests + run: | + export LANG="en_US.UTF-8" + sudo update-locale + source install/opt/HPCCSystems/sbin/hpcc_setenv + install/opt/HPCCSystems/etc/init.d/hpcc-init start + export CLASSPATH=".:$(realpath selenium-server-standalone-3.141.59.jar):$(realpath testng-6.8.7.jar)" + pushd install/test-ui + ./run.sh tests http://localhost:8010 > eclWatchUiTest.log 2>&1 + retCode=$? + echo "UI test done" + [[ $retCode -ne 0 ]] && exit 1 + popd + + - name: eclwatch-ui-test-logs-artifact + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v2 + with: + name: ECLWatchUiTest + path: install/test-ui/eclWatchUiTest.log + if-no-files-found: error diff --git a/.github/workflows/test-eclhelper.yml b/.github/workflows/test-eclhelper.yml new file mode 100644 index 00000000000..4098beaa64d --- /dev/null +++ b/.github/workflows/test-eclhelper.yml @@ -0,0 +1,23 @@ +name: Check that eclhelper interface has not changed + +on: + pull_request: + branches: + - "master" + - "candidate-*" + +jobs: + build: + name: "Check eclhelper interface" + runs-on: ubuntu-20.04 + steps: + - name: Checkout pr + uses: actions/checkout@v2 + with: + fetch-depth: 1 + - name: Fetch base + run: | + git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin ${{ github.event.pull_request.base.sha }} + - name: Check eclhelper + run: | + git diff ${{ github.event.pull_request.base.sha }} --exit-code -- rtl/include/eclhelper.hpp diff --git a/.github/workflows/test-helm.yml b/.github/workflows/test-helm.yml new file mode 100644 index 00000000000..f25bb5c33db --- /dev/null +++ b/.github/workflows/test-helm.yml @@ -0,0 +1,64 @@ +name: Run helm chart tests + +on: + push: + branches: + - "master" + - "candidate-*" + - "!candidate-7.8.*" + - "!candidate-7.6.*" + - "!candidate-7.4.*" + - "!candidate-7.2.*" + - "!candidate-7.0.*" + - "!candidate-6.*" + pull_request: + branches: + - "master" + - "candidate-*" + - "!candidate-7.8.*" + - "!candidate-7.6.*" + - "!candidate-7.4.*" + - "!candidate-7.2.*" + - "!candidate-7.0.*" + - "!candidate-6.*" + +jobs: + pre_job: + # continue-on-error: true # Uncomment once integration is finished + runs-on: ubuntu-20.04 + # Map a step output to a job output + outputs: + helm: ${{ steps.skip_check.outputs.helm }} + steps: + - id: skip_check + uses: hpcc-systems/github-actions/changed-modules@main + with: + github_token: ${{ github.token }} + build: + name: "Check helm chart lint" + runs-on: ubuntu-20.04 + needs: pre_job + if: ${{ github.repository == 'hpcc-systems/HPCC-Platform' && needs.pre_job.outputs.helm }} + steps: + - uses: actions/checkout@v2 + with: + submodules: false + - uses: azure/setup-helm@v3 + with: + version: 'latest' + token: ${{ secrets.GITHUB_TOKEN }} + id: install + - name: Install additional tools + working-directory: . + run: wget https://github.com/zegl/kube-score/releases/download/v1.8.1/kube-score_1.8.1_linux_amd64 && sudo mv kube-score_1.8.1_linux_amd64 /usr/bin/kube-score && sudo chmod +x /usr/bin/kube-score + - name: Validate Helm Templates + working-directory: ./testing/helm + run: ./run.sh + - name: Check for changes in helm output for default values + working-directory: . + run: | + helm template helm/hpcc > ${{ runner.temp }}/newoutput.txt + git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin ${{ github.base_ref }} + git checkout ${{ github.base_ref }} + helm template helm/hpcc > ${{ runner.temp }}/oldoutput.txt + diff ${{ runner.temp }}/oldoutput.txt ${{ runner.temp }}/newoutput.txt diff --git a/vcpkg b/vcpkg index b9cba616c4f..19a480bbf1f 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit b9cba616c4f25daa93ef3880f1d5d6f6c4788a87 +Subproject commit 19a480bbf1f8a8f29f03fb5c719b560fd7ed7f65