Merge pull request #3235 from autonomys/fix-non-skylake-builds #446
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This action enables building container images and executables for farmer and node, can be triggered manually or by | |
| # release creation. | |
| # | |
| # Container images are only pushed to GitHub Container Registry for releases. | |
| # Executables are built both for releases and for manually triggered runs, uploaded to artifacts and assets. | |
| name: Snapshot build | |
| on: | |
| workflow_dispatch: | |
| push: | |
| tags: | |
| - "snapshot-*" | |
| - "taurus-*" | |
| - "mainnet-*" | |
| # Incremental compilation here isn't helpful | |
| env: | |
| CARGO_INCREMENTAL: 0 | |
| jobs: | |
| # This will build container images and then extract executables out of them | |
| ubuntu: | |
| runs-on: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-22.04-x86-64"]' || '"ubuntu-22.04"') }} | |
| permissions: | |
| contents: write | |
| packages: write | |
| strategy: | |
| matrix: | |
| build: | |
| - image: farmer | |
| base-artifact: subspace-farmer | |
| upload-executables: true | |
| - image: node | |
| base-artifact: subspace-node | |
| upload-executables: true | |
| - image: bootstrap-node | |
| base-artifact: subspace-bootstrap-node | |
| upload-executables: false | |
| fail-fast: false | |
| steps: | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 | |
| if: github.repository_owner == 'autonomys' | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 | |
| with: | |
| # Limit concurrency so it can complete with small official runners | |
| buildkitd-config-inline: | | |
| [worker.oci] | |
| max-parallelism = 1 | |
| if: github.repository_owner != 'autonomys' | |
| - name: Log into registry | |
| uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ github.token }} | |
| - name: Extract Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 | |
| with: | |
| images: | | |
| ghcr.io/${{ github.repository_owner }}/${{ matrix.build.image }} | |
| tags: | | |
| type=ref,event=tag | |
| type=ref,event=branch | |
| type=sha,format=long | |
| flavor: | | |
| latest=false | |
| suffix= | |
| - name: Build and push ${{ matrix.build.image }} image | |
| uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 | |
| with: | |
| file: docker/${{ matrix.build.image }}.Dockerfile | |
| # TODO: Add `linux/riscv64` once https://github.com/paritytech/polkadot-sdk/issues/5996 is resolved and ring | |
| # 0.16.x is no longer in dependencies | |
| # TODO: Add linux/amd64/v4 when runner supports it | |
| platforms: linux/amd64,linux/amd64/v2,linux/amd64/v3,linux/arm64 | |
| pull: true | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-args: | | |
| SUBSTRATE_CLI_GIT_COMMIT_HASH=${{ github.sha }} | |
| - name: Trigger trivy-security-scan Workflow | |
| uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # @v3.0.0 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| event-type: trivy-scan-dispatch | |
| client-payload: '{"image": "${{ fromJSON(steps.meta.outputs.json).tags[0] }}"}' | |
| - name: Prepare executables for uploading | |
| run: | | |
| mkdir executables | |
| cd executables | |
| IMAGE="${{ fromJSON(steps.meta.outputs.json).tags[0] }}" | |
| ARTIFACT="${{ matrix.build.base-artifact }}" | |
| docker run --rm --platform linux/amd64 --entrypoint /bin/cat $IMAGE /$ARTIFACT > $ARTIFACT-ubuntu-x86_64-skylake-${{ github.ref_name }} | |
| # TODO: Pull is a workaround for https://github.com/moby/moby/issues/48197#issuecomment-2472265028 | |
| docker pull --platform linux/amd64/v2 $IMAGE | |
| docker run --rm --platform linux/amd64/v2 --entrypoint /bin/cat $IMAGE /$ARTIFACT > $ARTIFACT-ubuntu-x86_64-v2-${{ github.ref_name }} | |
| if [ "${{ matrix.build.image }}" == "farmer" ]; then | |
| docker run --rm --platform linux/amd64 --entrypoint /bin/cat $IMAGE /$ARTIFACT-rocm > $ARTIFACT-rocm-ubuntu-x86_64-skylake-${{ github.ref_name }} | |
| # TODO: Pull is a workaround for https://github.com/moby/moby/issues/48197#issuecomment-2472265028 | |
| docker pull --platform linux/amd64/v2 $IMAGE | |
| docker run --rm --platform linux/amd64/v2 --entrypoint /bin/cat $IMAGE /$ARTIFACT-rocm > $ARTIFACT-rocm-ubuntu-x86_64-v2-${{ github.ref_name }} | |
| fi | |
| docker run --rm --platform linux/arm64 --entrypoint /bin/cat $IMAGE /$ARTIFACT > $ARTIFACT-ubuntu-aarch64-${{ github.ref_name }} | |
| if: matrix.build.upload-executables | |
| - name: Upload node and farmer executables to artifacts | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.1.3 | |
| with: | |
| name: executables-ubuntu-${{ matrix.build.image }}-${{ github.ref_name }} | |
| path: | | |
| executables/* | |
| if-no-files-found: error | |
| if: matrix.build.upload-executables | |
| - name: Upload node and farmer executables to assets | |
| uses: alexellis/upload-assets@13926a61cdb2cb35f5fdef1c06b8b591523236d3 # 0.4.1 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| with: | |
| asset_paths: '["executables/*"]' | |
| if: matrix.build.upload-executables && github.event_name == 'push' && github.ref_type == 'tag' | |
| executables: | |
| strategy: | |
| matrix: | |
| build: | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "macos-14-arm64"]' || '"macos-14"') }} | |
| target: aarch64-apple-darwin | |
| suffix: macos-aarch64-${{ github.ref_name }} | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }} | |
| target: x86_64-pc-windows-msvc | |
| suffix: windows-x86_64-skylake-${{ github.ref_name }} | |
| rustflags: "-C target-cpu=skylake" | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }} | |
| target: x86_64-pc-windows-msvc | |
| suffix: windows-x86_64-v2-${{ github.ref_name }} | |
| rustflags: "-C target-cpu=x86-64-v2" | |
| fail-fast: false | |
| runs-on: ${{ matrix.build.os }} | |
| env: | |
| PRODUCTION_TARGET: target/${{ matrix.build.target }}/production | |
| RUSTFLAGS: ${{ matrix.build.rustflags }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| # On macOS, we need a proper Clang version, not Apple's custom version without wasm32 support | |
| - name: Install LLVM and Clang for macOS | |
| uses: KyleMayes/install-llvm-action@dec985c8d7b46a2f363ea1a78f660c946a3349ea # v2.0.1 | |
| with: | |
| env: true | |
| version: 17 | |
| if: runner.os == 'macOS' | |
| # Because macOS, see https://andreasfertig.blog/2021/02/clang-and-gcc-on-macos-catalina-finding-the-include-paths/ | |
| - name: Configure C compiler macOS | |
| run: | | |
| echo "SDKROOT=$(xcrun --show-sdk-path)" >> $GITHUB_ENV | |
| if: runner.os == 'macOS' | |
| - name: Install glibtoolize (macOS) | |
| run: brew install libtool | |
| if: runner.os == 'macOS' | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Needed for hwloc | |
| - name: Install automake (macOS) | |
| run: brew install automake | |
| if: runner.os == 'macOS' | |
| - name: CUDA toolchain | |
| uses: Jimver/cuda-toolkit@9b295696791d75d658d8de64c4a85097ad8abeaf # v0.2.16 | |
| with: | |
| cuda: '12.4.1' | |
| method: network | |
| sub-packages: '["nvcc", "cudart"]' | |
| if: runner.os == 'Windows' | |
| - name: Configure ROCm cache (Windows) | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| id: rocm-cache | |
| with: | |
| path: C:\Program Files\AMD\ROCm | |
| key: ${{ runner.os }}-rocm | |
| if: runner.os == 'Windows' | |
| - name: ROCm toolchain | |
| run: | | |
| $ErrorActionPreference = "Stop" | |
| Invoke-WebRequest -Uri https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q3-WinSvr2022-For-HIP.exe -OutFile "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" | |
| Start-Process "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" -ArgumentList '-install' -NoNewWindow -Wait | |
| Remove-Item "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" | |
| if: runner.os == 'Windows' && steps.rocm-cache.outputs.cache-hit != 'true' | |
| - name: Configure cache | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - name: Build farmer | |
| run: | | |
| cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer | |
| if: runner.os == 'macOS' || !startsWith(matrix.build.target, 'x86_64') | |
| # ROCm can't be enabled together with CUDA for now | |
| - name: Build farmer (ROCm, Windows) | |
| env: | |
| # Why `PROGRA~1` instead of `Program Files`? Because Windows! | |
| HIPCC: C:\PROGRA~1\AMD\ROCm\6.1\bin\hipcc.bin.exe | |
| run: | | |
| cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features rocm | |
| move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe | |
| if: runner.os == 'Windows' && startsWith(matrix.build.target, 'x86_64') | |
| - name: Build farmer | |
| run: | | |
| cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features cuda | |
| # TODO: We don't configure CUDA for cross-compilation purposes, hence only x86-64 for now | |
| if: runner.os == 'Windows' && startsWith(matrix.build.target, 'x86_64') | |
| - name: Build node | |
| run: | | |
| cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-node | |
| - name: Sign Application (macOS) | |
| run: | | |
| echo "Importing certificate" | |
| echo "${{ secrets.MACOS_CERTIFICATE }}" | base64 --decode > certificate.p12 | |
| security create-keychain -p "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain | |
| security default-keychain -s build.keychain | |
| security unlock-keychain -p "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain | |
| security import certificate.p12 -k build.keychain -P "${{ secrets.MACOS_CERTIFICATE_PW }}" -T /usr/bin/codesign | |
| security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain | |
| echo "Signing farmer" | |
| codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-farmer | |
| echo "Signing node" | |
| codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-node | |
| echo "Creating a ZIP archive" | |
| mkdir -p ${{ env.PRODUCTION_TARGET }}/macos-binaries | |
| cp ${{ env.PRODUCTION_TARGET }}/subspace-farmer ${{ env.PRODUCTION_TARGET }}/subspace-node ${{ env.PRODUCTION_TARGET }}/macos-binaries | |
| ditto -c -k --rsrc ${{ env.PRODUCTION_TARGET }}/macos-binaries subspace-binaries.zip | |
| echo "Notarizing ZIP archive file" | |
| xcrun notarytool submit subspace-binaries.zip --apple-id "${{ secrets.MACOS_APPLE_ID }}" --password "${{ secrets.MACOS_APP_PW }}" --team-id "${{ secrets.MACOS_TEAM_ID }}" --wait | |
| # stapling does not work for .zip archives only .app bundles and .dmg files. Commenting this for now! | |
| # echo "Stapling notarization to ZIP file" | |
| # xcrun stapler staple subspace-binaries.zip | |
| echo "Done!" | |
| # Allow code signing to fail on non-release builds and in non-subspace repos (forks) | |
| continue-on-error: ${{ github.repository_owner != 'autonomys' || github.event_name != 'push' || github.ref_type != 'tag' }} | |
| if: runner.os == 'macOS' | |
| - name: Sign Application (Windows) | |
| run: | | |
| AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe" | |
| AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe" | |
| AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-node.exe" | |
| # Allow code signing to fail on non-release builds and in non-subspace repos (forks) | |
| continue-on-error: ${{ github.repository_owner != 'autonomys' || github.event_name != 'push' || github.ref_type != 'tag' }} | |
| if: runner.os == 'Windows' | |
| - name: Prepare executables for uploading (macOS) | |
| run: | | |
| mkdir executables | |
| mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer executables/subspace-farmer-${{ matrix.build.suffix }} | |
| mv ${{ env.PRODUCTION_TARGET }}/subspace-node executables/subspace-node-${{ matrix.build.suffix }} | |
| # Zip it so that signature is not lost | |
| ditto -c -k --rsrc executables/subspace-farmer-${{ matrix.build.suffix }} executables/subspace-farmer-${{ matrix.build.suffix }}.zip | |
| ditto -c -k --rsrc executables/subspace-node-${{ matrix.build.suffix }} executables/subspace-node-${{ matrix.build.suffix }}.zip | |
| rm executables/subspace-farmer-${{ matrix.build.suffix }} | |
| rm executables/subspace-node-${{ matrix.build.suffix }} | |
| if: runner.os == 'macOS' | |
| - name: Prepare executables for uploading (Windows) | |
| run: | | |
| mkdir executables | |
| move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe executables/subspace-farmer-${{ matrix.build.suffix }}.exe | |
| move ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe executables/subspace-farmer-rocm-${{ matrix.build.suffix }}.exe | |
| move ${{ env.PRODUCTION_TARGET }}/subspace-node.exe executables/subspace-node-${{ matrix.build.suffix }}.exe | |
| if: runner.os == 'Windows' | |
| - name: Upload node and farmer executables to artifacts | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.1.3 | |
| with: | |
| name: executables-${{ matrix.build.suffix }} | |
| path: | | |
| executables/* | |
| if-no-files-found: error | |
| - name: Upload node and farmer executables to assets | |
| uses: alexellis/upload-assets@13926a61cdb2cb35f5fdef1c06b8b591523236d3 # 0.4.1 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| with: | |
| asset_paths: '["executables/*"]' | |
| if: github.event_name == 'push' && github.ref_type == 'tag' |