Skip to content

Commit

Permalink
[Release|CI/CD] Add pipeline to promote release candidate from rcX to…
Browse files Browse the repository at this point in the history
… final in S3 (#6748)

This PR adds the pipeline, that moves release candidate artefacts from
`polkadot-stableYYMM-rcX` bucket to the one that is going to be the
final `polkadot-stableYYMM` (bucket name matches the tag name). So that
it could be used for publishing later without a need to re-build it
again.
  • Loading branch information
EgorPopelyaev authored Dec 4, 2024
1 parent a2ffae3 commit 377bc3f
Show file tree
Hide file tree
Showing 6 changed files with 253 additions and 19 deletions.
2 changes: 1 addition & 1 deletion .github/scripts/common/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ fetch_release_artifacts_from_s3() {
pwd
ls -al --color
popd > /dev/null

unset OUTPUT_DIR
}

# Pass the name of the binary as input, it will
Expand Down
44 changes: 39 additions & 5 deletions .github/scripts/release/release_lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -129,15 +129,17 @@ upload_s3_release() {
echo "Working on version: $version "
echo "Working on platform: $target "

URL_BASE=$(get_s3_url_base $product)

echo "Current content, should be empty on new uploads:"
aws s3 ls "s3://releases.parity.io/${product}/${version}/${target}" --recursive --human-readable --summarize || true
aws s3 ls "s3://${URL_BASE}/${version}/${target}" --recursive --human-readable --summarize || true
echo "Content to be uploaded:"
artifacts="artifacts/$product/"
artifacts="release-artifacts/$target/$product/"
ls "$artifacts"
aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/${product}/${version}/${target}"
aws s3 sync --acl public-read "$artifacts" "s3://${URL_BASE}/${version}/${target}"
echo "Uploaded files:"
aws s3 ls "s3://releases.parity.io/${product}/${version}/${target}" --recursive --human-readable --summarize
echo "✅ The release should be at https://releases.parity.io/${product}/${version}/${target}"
aws s3 ls "s3://${URL_BASE}/${version}/${target}" --recursive --human-readable --summarize
echo "✅ The release should be at https://${URL_BASE}/${version}/${target}"
}

# Upload runtimes artifacts to s3 release bucket
Expand All @@ -161,3 +163,35 @@ upload_s3_runtimes_release_artifacts() {
aws s3 ls "s3://releases.parity.io/polkadot/runtimes/${version}/" --recursive --human-readable --summarize
echo "✅ The release should be at https://releases.parity.io/polkadot/runtimes/${version}"
}


# Pass the name of the binary as input, it will
# return the s3 base url
function get_s3_url_base() {
name=$1
case $name in
polkadot | polkadot-execute-worker | polkadot-prepare-worker )
printf "releases.parity.io/polkadot"
;;

polkadot-parachain)
printf "releases.parity.io/polkadot-parachain"
;;

polkadot-omni-node)
printf "releases.parity.io/polkadot-omni-node"
;;

chain-spec-builder)
printf "releases.parity.io/chain-spec-builder"
;;

frame-omni-bencher)
printf "releases.parity.io/frame-omni-bencher"
;;
*)
printf "UNSUPPORTED BINARY $name"
exit 1
;;
esac
}
125 changes: 125 additions & 0 deletions .github/workflows/release-31_promote-rc-to-final.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
name: Release - Promote RC to final candidate on S3

on:
workflow_dispatch:
inputs:
binary:
description: Binary to be build for the release
default: all
type: choice
options:
- polkadot
- polkadot-parachain
- polkadot-omni-node
- frame-omni-bencher
- chain-spec-builder
- all
release_tag:
description: Tag matching the actual release candidate with the format polkadot-stableYYMM(-X)-rcX
type: string


jobs:

check-synchronization:
uses: paritytech-release/sync-workflows/.github/workflows/check-syncronization.yml@main

validate-inputs:
needs: [ check-synchronization ]
if: ${{ needs.check-synchronization.outputs.checks_passed }} == 'true'
runs-on: ubuntu-latest
outputs:
release_tag: ${{ steps.validate_inputs.outputs.release_tag }}
final_tag: ${{ steps.validate_inputs.outputs.final_tag }}

steps:
- name: Checkout sources
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

- name: Validate inputs
id: validate_inputs
run: |
. ./.github/scripts/common/lib.sh
RELEASE_TAG=$(validate_stable_tag ${{ inputs.release_tag }})
echo "release_tag=${RELEASE_TAG}" >> $GITHUB_OUTPUT
promote-polkadot-rc-to-final:
if: ${{ inputs.binary == 'polkadot' || inputs.binary == 'all' }}
needs: [ validate-inputs ]
uses: ./.github/workflows/release-reusable-promote-to-final.yml
strategy:
matrix:
target: [ x86_64-unknown-linux-gnu, aarch64-apple-darwin ]
with:
package: polkadot
release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
target: ${{ matrix.target }}
secrets:
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}

promote-polkadot-parachain-rc-to-final:
if: ${{ inputs.binary == 'polkadot-parachain' || inputs.binary == 'all' }}
needs: [ validate-inputs ]
uses: ./.github/workflows/release-reusable-promote-to-final.yml
strategy:
matrix:
target: [ x86_64-unknown-linux-gnu, aarch64-apple-darwin ]
with:
package: polkadot-parachain
release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
target: ${{ matrix.target }}
secrets:
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}

promote-polkadot-omni-node-rc-to-final:
if: ${{ inputs.binary == 'polkadot-omni-node' || inputs.binary == 'all' }}
needs: [ validate-inputs ]
uses: ./.github/workflows/release-reusable-promote-to-final.yml
strategy:
matrix:
target: [ x86_64-unknown-linux-gnu, aarch64-apple-darwin ]
with:
package: polkadot-omni-node
release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
target: ${{ matrix.target }}
secrets:
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}

promote-frame-omni-bencher-rc-to-final:
if: ${{ inputs.binary == 'frame-omni-bencher' || inputs.binary == 'all' }}
needs: [ validate-inputs ]
uses: ./.github/workflows/release-reusable-promote-to-final.yml
strategy:
matrix:
target: [ x86_64-unknown-linux-gnu, aarch64-apple-darwin ]
with:
package: frame-omni-bencher
release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
target: ${{ matrix.target }}
secrets:
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}

promote-chain-spec-builder-rc-to-final:
if: ${{ inputs.binary == 'chain-spec-builder' || inputs.binary == 'all' }}
needs: [ validate-inputs ]
uses: ./.github/workflows/release-reusable-promote-to-final.yml
strategy:
matrix:
target: [ x86_64-unknown-linux-gnu, aarch64-apple-darwin ]
with:
package: chain-spec-builder
release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
target: ${{ matrix.target }}
secrets:
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
83 changes: 83 additions & 0 deletions .github/workflows/release-reusable-promote-to-final.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
name: Promote rc to final

on:
workflow_call:
inputs:
package:
description: Package to be promoted
required: true
type: string

release_tag:
description: Tag matching the actual release candidate with the format polkadot-stableYYMM(-X)-rcX taht will be changed to final in form of polkadot-stableYYMM(-X)
required: true
type: string

target:
description: Target triple for which the artifacts are being uploaded (e.g aarch64-apple-darwin)
required: true
type: string

secrets:
AWS_DEFAULT_REGION:
required: true
AWS_RELEASE_ACCESS_KEY_ID:
required: true
AWS_RELEASE_SECRET_ACCESS_KEY:
required: true

jobs:

promote-release-artifacts:
environment: release
runs-on: ubuntu-latest
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

steps:
- name: Checkout sources
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

- name: Prepare final tag
id: prepare_final_tag
shell: bash
run: |
tag="$(echo ${{ inputs.release_tag }} | sed 's/-rc[0-9]*$//')"
echo $tag
echo "FINAL_TAG=${tag}" >> $GITHUB_OUTPUT
- name: Fetch binaries from s3 based on version
run: |
. ./.github/scripts/common/lib.sh
VERSION="${{ inputs.release_tag }}"
if [[ ${{ inputs.package }} == 'polkadot' ]]; then
packages=(polkadot polkadot-prepare-worker polkadot-execute-worker)
for package in "${packages[@]}"; do
fetch_release_artifacts_from_s3 $package ${{ inputs.target }}
done
else
fetch_release_artifacts_from_s3 ${{ inputs.package }} ${{ inputs.target }}
fi
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}

- name: Upload ${{ inputs.package }} ${{ inputs.target }} artifacts to s3
run: |
. ./.github/scripts/release/release_lib.sh
if [[ ${{ inputs.package }} == 'polkadot' ]]; then
packages=(polkadot polkadot-prepare-worker polkadot-execute-worker)
for package in "${packages[@]}"; do
upload_s3_release $package ${{ steps.prepare_final_tag.outputs.final_tag }} ${{ inputs.target }}
done
else
upload_s3_release ${{ inputs.package }} ${{ steps.prepare_final_tag.outputs.final_tag }} ${{ inputs.target }}
fi
4 changes: 2 additions & 2 deletions .github/workflows/release-reusable-rc-buid.yml
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ jobs:
- name: Upload ${{ matrix.binaries }} artifacts
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: ${{ matrix.binaries }}
name: ${{ matrix.binaries }}_${{ inputs.target }}
path: /artifacts/${{ matrix.binaries }}

build-macos-rc:
Expand Down Expand Up @@ -285,7 +285,7 @@ jobs:
- name: Upload ${{inputs.package }} artifacts
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: ${{ inputs.package }}
name: ${{ inputs.package }}_${{ inputs.target }}
path: target/production
overwrite: true

Expand Down
14 changes: 3 additions & 11 deletions .github/workflows/release-reusable-s3-upload.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ on:
type: string

release_tag:
description: Tag matching the actual release candidate with the format stableYYMM-rcX or stableYYMM-rcX
description: Tag matching the actual release candidate with the format polkadot-stableYYMM(-X)-rcX or polkadot-stableYYMM-rcX
required: true
type: string

Expand Down Expand Up @@ -40,18 +40,10 @@ jobs:
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0

- name: Download amd64 artifacts
if: ${{ inputs.target == 'x86_64-unknown-linux-gnu' }}
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: ${{ inputs.package }}
path: artifacts/${{ inputs.package }}

- name: Download arm artifacts
if: ${{ inputs.target == 'aarch64-apple-darwin' }}
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: ${{ inputs.package }}_aarch64-apple-darwin
path: artifacts/${{ inputs.package }}
name: ${{ inputs.package }}_${{ inputs.target }}
path: release-artifacts/${{ inputs.target }}/${{ inputs.package }}

- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
Expand Down

0 comments on commit 377bc3f

Please sign in to comment.