diff --git a/.github/workflows/build-ce-platform.yml b/.github/workflows/build-ce-platform.yml index d8cb9a2..8b749b6 100644 --- a/.github/workflows/build-ce-platform.yml +++ b/.github/workflows/build-ce-platform.yml @@ -5,8 +5,8 @@ env: VCPKG_NUGET_REPOSITORY: "https://github.com/hpcc-systems/vcpkg" on: - #workflow_dispatch - push: + workflow_dispatch + #push: jobs: preamble: @@ -61,8 +61,8 @@ jobs: matrix: include: - os: ubuntu-22.04 - name: docs - documentation: true + #- os: centos-7 + # cmake_options_extra: "-DVCPKG_TARGET_TRIPLET=x64-centos-7-dynamic" fail-fast: false steps: @@ -138,12 +138,6 @@ jobs: # Communtiy Build - name: CMake Packages (community) if: ${{ !matrix.ln && !matrix.ee && !matrix.container && !matrix.documentation }} - run: | - mkdir -p ${{ needs.preamble.outputs.folder_build }} - #echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key - - - name: CMake documentation (community) - if: ${{ !matrix.ln && !matrix.ee && !matrix.container && matrix.documentation }} run: | mkdir -p ${{needs.preamble.outputs.folder_build }} echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key @@ -154,9 +148,9 @@ jobs: docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \ cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF ${{ matrix.cmake_options_extra }} && \ cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \ - cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON ${{ matrix.cmake_options_extra }} && \ - cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + #docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \ + # cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON ${{ matrix.cmake_options_extra }} && \ + # cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - name: Upload Artifacts CE Platform uses: actions/upload-artifact@v4 diff --git a/.github/workflows/build-ce-plugins.yml b/.github/workflows/build-ce-plugins.yml new file mode 100644 index 0000000..25f129b --- /dev/null +++ b/.github/workflows/build-ce-plugins.yml @@ -0,0 +1,161 @@ +name: Build CE Plugins + +env: + VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,read" + VCPKG_NUGET_REPOSITORY: "https://github.com/hpcc-systems/vcpkg" + +on: + #workflow_dispatch + push: + +jobs: + preamble: + name: Publish Release + env: + COMMUNITY_REF: "dc1cdacdbdda6e5ca1c5b836e130c2deb2d2fe62" + #COMMUNITY_REF: + runs-on: ubuntu-22.04 + permissions: + contents: write + outputs: + folder_platform: ${{ steps.vars.outputs.folder_platform }} + mount_platform: ${{ steps.vars.outputs.mount_platform }} + folder_build: ${{ steps.vars.outputs.folder_build }} + mount_build: ${{ steps.vars.outputs.mount_build }} + community_ref: ${{ steps.vars.outputs.community_ref }} + community_tag: ${{ steps.vars.outputs.community_tag }} + hpcc_version: ${{ steps.vars.outputs.hpcc_version }} + candidate_base_branch: ${{ steps.vars.outputs.candidate_base_branch }} + candidate_branch: ${{ steps.vars.outputs.candidate_branch }} + cmake_docker_config: ${{ steps.vars.outputs.cmake_docker_config }} + gpg_import: ${{ steps.vars.outputs.gpg_import }} + steps: + - name: Calculate vars + id: vars + run: | + echo "folder_platform=${{ github.workspace }}/HPCC-Platform" >> $GITHUB_OUTPUT + echo 'mount_platform=source="${{ github.workspace }}/HPCC-Platform",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached' >> $GITHUB_OUTPUT + echo "folder_build=${{ github.workspace }}/build" >> $GITHUB_OUTPUT + echo 'mount_build=source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=cached' >> $GITHUB_OUTPUT + community_ref=${{ env.COMMUNITY_REF }} + echo "community_ref=$community_ref" >> $GITHUB_OUTPUT + community_tag=$(echo $community_ref | cut -d'/' -f3) + echo "community_tag=$community_tag" >> $GITHUB_OUTPUT + echo "hpcc_version=$(echo $community_tag | sed 's/community_//' | sed 's/-[0-9]$//')" >> $GITHUB_OUTPUT + community_base_ref=${{ github.event.base_ref || github.ref }} + candidate_branch=$(echo $community_base_ref | cut -d'/' -f3) + echo "candidate_branch=$candidate_branch" >> $GITHUB_OUTPUT + echo "candidate_base_branch=$(echo $candidate_branch | awk -F'.' -v OFS='.' '{ $3="x"; print }')" >> $GITHUB_OUTPUT + echo "cmake_docker_config=-DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DSIGN_MODULES=${{ github.repository_owner == 'hpcc-systems' && 'ON' || 'OFF' }}" >> $GITHUB_OUTPUT + echo 'gpg_import=gpg --batch --import /hpcc-dev/build/private.key' >> $GITHUB_OUTPUT + + - name: Print vars + run: | + echo "${{ toJSON(steps.vars.outputs) }})" + + build-docker: + name: Build Docker + needs: preamble + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - os: ubuntu-22.04 + #- os: centos-7 + # cmake_options_extra: "-DVCPKG_TARGET_TRIPLET=x64-centos-7-dynamic" + fail-fast: false + + steps: + - name: Free additional disk space (remove Android SDK + Tools) + continue-on-error: true + run: | + df -h + sudo rm -rf ${ANDROID_HOME} + sudo rm -rf ${JAVA_HOME_8_X64} + sudo rm -rf ${JAVA_HOME_11_X64} + sudo rm -rf ${JAVA_HOME_17_X64} + sudo rm -rf ${JAVA_HOME_21_X64} + sudo rm -rf ${CHROMEWEBDRIVER} + sudo rm -rf ${EDGEWEBDRIVER} + sudo rm -rf ${GECKOWEBDRIVER} + sudo rm -rf ${SELENIUM_JAR_PATH} + + - name: Display disk space (post free) + run: | + df -h + + - name: Checkout HPCC-Platform + uses: actions/checkout@v3 + with: + repository: ${{ github.repository_owner }}/HPCC-Platform + ref: ${{ needs.preamble.outputs.community_ref }} + submodules: recursive + path: ${{ needs.preamble.outputs.folder_platform }} + + - name: Calculate vars + id: vars + working-directory: ${{ needs.preamble.outputs.folder_platform }}/vcpkg + run: | + vcpkg_sha_short=$(git rev-parse --short=8 HEAD) + echo "vcpkg_sha_short=$vcpkg_sha_short" >> $GITHUB_OUTPUT + docker_build_label=hpccsystems/platform-build-${{ matrix.os }} + echo "docker_build_label=$docker_build_label" >> $GITHUB_OUTPUT + echo "docker_tag=$docker_build_label:$vcpkg_sha_short" >> $GITHUB_OUTPUT + echo "docker_tag_candidate_base=$docker_build_label:${{ needs.preamble.outputs.candidate_base_branch }}" >> $GITHUB_OUTPUT + + - name: Print vars + run: | + echo "${{ toJSON(needs.preamble.outputs) }})" + echo "${{ toJSON(steps.vars.outputs) }})" + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Create Build Image + uses: docker/build-push-action@v4 + with: + builder: ${{ steps.buildx.outputs.name }} + file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/${{ matrix.os }}.dockerfile + context: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg + push: ${{ github.repository_owner == 'hpcc-systems' }} + load: ${{ github.repository_owner != 'hpcc-systems' }} + build-args: | + VCPKG_REF=${{ steps.vars.outputs.vcpkg_sha_short }} + tags: | + ${{ steps.vars.outputs.docker_tag_candidate_base }} + cache-from: | + type=registry,ref=${{ steps.vars.outputs.docker_tag_candidate_base }} + type=registry,ref=${{ steps.vars.outputs.docker_tag }} + cache-to: type=inline + + # Communtiy Build + - name: CMake Packages (community) + if: ${{ !matrix.ln && !matrix.ee && !matrix.container && !matrix.documentation }} + run: | + mkdir -p ${{needs.preamble.outputs.folder_build }} + ls -l + echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key + plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED" "PLATFORM" "CLIENTTOOLS_ONLY") + for plugin in "${plugins[@]}"; do + sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt + sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles + docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "${{ needs.preamble.outputs.gpg_import }} && \ + cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF ${{ matrix.cmake_options_extra }} && \ + cmake --build /hpcc-dev/build --parallel $(nproc) --target package" + done + + - name: Upload Artifacts CE Plugins + uses: actions/upload-artifact@v4 + with: + name: CE-HPCC-Plugins + path: | + ${{ needs.preamble.outputs.folder_build }}/*.deb + ${{ needs.preamble.outputs.folder_build }}/*.rpm + compression-level: 0 diff --git a/README.md b/README.md index 91c95f5..893fcf1 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,25 @@ Get the first line first column and replace in .github/workflows/build COMMUNITY_REF: ``` The default artifact file name: html-help-documents.zip + +## CE Platform +```console +git show-ref --head +``` +Get the first line first column and replace in .github/workflows/build-ce-platform.yml +```code +COMMUNITY_REF: +``` +The default artifact file name: CE-HPCC-Platfor.mzip + +## CE Plugins +```console +git show-ref --head +``` +Get the first line first column and replace in .github/workflows/build-ce-plugins.yml +```code +COMMUNITY_REF: +``` +The default artifact file name: CE-HPCC-Plugins.mzip + +