Merge branch 'master' into Migrating-polkadot-runtime-parachains-para…

…s_inherent-to-Benchmarking-V2

.config/nextest.toml

@@ -21,7 +21,6 @@ retries = 5
 # The number of threads to run tests with. Supported values are either an integer or
 # the string "num-cpus". Can be overridden through the `--test-threads` option.
 # test-threads = "num-cpus"
-
 test-threads = 20
 
 # The number of threads required for each test. This is generally used in overrides to

.github/env

@@ -1 +1 @@
-IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-09-11-v202409111034"
+IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-11-19-v202411281558"

.github/scripts/common/lib.sh

@@ -237,24 +237,52 @@ fetch_release_artifacts() {
   popd > /dev/null
 }
 
-# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
+# Fetch deb package from S3. Assumes the ENV are set:
 # - RELEASE_ID
 # - GITHUB_TOKEN
 # - REPO in the form paritytech/polkadot
-fetch_release_artifacts_from_s3() {
+fetch_debian_package_from_s3() {
   BINARY=$1
   echo "Version    : $VERSION"
   echo "Repo       : $REPO"
   echo "Binary     : $BINARY"
+  echo "Tag        : $RELEASE_TAG"
   OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
   echo "OUTPUT_DIR : $OUTPUT_DIR"
 
   URL_BASE=$(get_s3_url_base $BINARY)
   echo "URL_BASE=$URL_BASE"
 
-  URL_BINARY=$URL_BASE/$VERSION/$BINARY
-  URL_SHA=$URL_BASE/$VERSION/$BINARY.sha256
-  URL_ASC=$URL_BASE/$VERSION/$BINARY.asc
+  URL=$URL_BASE/$RELEASE_TAG/x86_64-unknown-linux-gnu/${BINARY}_${VERSION}_amd64.deb
+
+  mkdir -p "$OUTPUT_DIR"
+  pushd "$OUTPUT_DIR" > /dev/null
+
+  echo "Fetching deb package..."
+
+  echo "Fetching %s" "$URL"
+  curl --progress-bar -LO "$URL" || echo "Missing $URL"
+
+  pwd
+  ls -al --color
+  popd > /dev/null
+
+}
+
+# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
+# inputs: binary (polkadot), target(aarch64-apple-darwin)
+fetch_release_artifacts_from_s3() {
+  BINARY=$1
+  TARGET=$2
+  OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${TARGET}/${BINARY}"}
+  echo "OUTPUT_DIR : $OUTPUT_DIR"
+
+  URL_BASE=$(get_s3_url_base $BINARY)
+  echo "URL_BASE=$URL_BASE"
+
+  URL_BINARY=$URL_BASE/$VERSION/$TARGET/$BINARY
+  URL_SHA=$URL_BASE/$VERSION/$TARGET/$BINARY.sha256
+  URL_ASC=$URL_BASE/$VERSION/$TARGET/$BINARY.asc
 
   # Fetch artifacts
   mkdir -p "$OUTPUT_DIR"
@@ -277,15 +305,26 @@ fetch_release_artifacts_from_s3() {
 function get_s3_url_base() {
     name=$1
     case $name in
-    polkadot | polkadot-execute-worker | polkadot-prepare-worker | staking-miner)
+      polkadot | polkadot-execute-worker | polkadot-prepare-worker )
         printf "https://releases.parity.io/polkadot"
         ;;
 
-    polkadot-parachain)
-        printf "https://releases.parity.io/cumulus"
+      polkadot-parachain)
+        printf "https://releases.parity.io/polkadot-parachain"
+        ;;
+
+      polkadot-omni-node)
+        printf "https://releases.parity.io/polkadot-omni-node"
         ;;
 
-    *)
+      chain-spec-builder)
+        printf "https://releases.parity.io/chain-spec-builder"
+        ;;
+
+      frame-omni-bencher)
+        printf "https://releases.parity.io/frame-omni-bencher"
+        ;;
+      *)
         printf "UNSUPPORTED BINARY $name"
         exit 1
         ;;
@@ -468,3 +507,16 @@ validate_stable_tag() {
         exit 1
     fi
 }
+
+# Prepare docker stable tag form the polkadot stable tag
+# input: tag (polkaodot-stableYYMM(-X) or polkadot-stableYYMM(-X)-rcX)
+# output: stableYYMM(-X) or stableYYMM(-X)-rcX
+prepare_docker_stable_tag() {
+  tag="$1"
+  if [[ "$tag" =~ stable[0-9]{4}(-[0-9]+)?(-rc[0-9]+)? ]]; then
+      echo "${BASH_REMATCH[0]}"
+  else
+      echo "Tag is invalid: $tag"
+      exit 1
+  fi
+}

.github/scripts/release/build-linux-release.sh

@@ -3,6 +3,8 @@
 # This is used to build our binaries:
 # - polkadot
 # - polkadot-parachain
+# - polkadot-omni-node 
+#
 # set -e
 
 BIN=$1
@@ -21,7 +23,7 @@ time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PAC
 echo "Artifact target: $ARTIFACTS"
 
 cp ./target/$PROFILE/$BIN "$ARTIFACTS"
-pushd "$ARTIFACTS" > /dev/nul
+pushd "$ARTIFACTS" > /dev/null
 sha256sum "$BIN" | tee "$BIN.sha256"
 
 EXTRATAG="$($ARTIFACTS/$BIN --version |

.github/scripts/release/build-macos-release.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# This is used to build our binaries:
+# - polkadot
+# - polkadot-parachain
+# - polkadot-omni-node
+# set -e
+
+BIN=$1
+PACKAGE=${2:-$BIN}
+
+PROFILE=${PROFILE:-production}
+# parity-macos runner needs a path where it can
+# write, so make it relative to github workspace.
+ARTIFACTS=$GITHUB_WORKSPACE/artifacts/$BIN
+VERSION=$(git tag -l --contains HEAD | grep -E "^v.*")
+
+echo "Artifacts will be copied into $ARTIFACTS"
+mkdir -p "$ARTIFACTS"
+
+git log --pretty=oneline -n 1
+time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PACKAGE
+
+echo "Artifact target: $ARTIFACTS"
+
+cp ./target/$PROFILE/$BIN "$ARTIFACTS"
+pushd "$ARTIFACTS" > /dev/null
+sha256sum "$BIN" | tee "$BIN.sha256"
+
+EXTRATAG="$($ARTIFACTS/$BIN --version |
+    sed -n -r 's/^'$BIN' ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
+
+EXTRATAG="${VERSION}-${EXTRATAG}-$(cut -c 1-8 $ARTIFACTS/$BIN.sha256)"
+
+echo "$BIN version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
+echo -n ${VERSION} > "$ARTIFACTS/VERSION"
+echo -n ${EXTRATAG} > "$ARTIFACTS/EXTRATAG"

.github/scripts/release/distributions

@@ -0,0 +1,39 @@
+Origin: Parity
+Label: Parity
+Codename: release
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity Staging
+Codename: staging
+Architectures: amd64
+Components: main
+Description: Staging distribution for Parity Technologies Ltd. packages
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity stable2407
+Codename: stable2407
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity stable2409
+Codename: stable2409
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE
+
+Origin: Parity
+Label: Parity stable2412
+Codename: stable2412
+Architectures: amd64
+Components: main
+Description: Apt repository for software made by Parity Technologies Ltd.
+SignWith: 90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE

.github/scripts/release/release_lib.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Set the new version by replacing the value of the constant given as patetrn
+# Set the new version by replacing the value of the constant given as pattern
 # in the file.
 #
 # input: pattern, version, file
@@ -119,21 +119,45 @@ set_polkadot_parachain_binary_version() {
 
 
 upload_s3_release() {
+    alias aws='podman run --rm -it docker.io/paritytech/awscli -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_BUCKET aws'
+
+    product=$1
+    version=$2
+    target=$3
+
+    echo "Working on product:  $product "
+    echo "Working on version:  $version "
+    echo "Working on platform: $target "
+
+    echo "Current content, should be empty on new uploads:"
+    aws s3 ls "s3://releases.parity.io/${product}/${version}/${target}" --recursive --human-readable --summarize || true
+    echo "Content to be uploaded:"
+    artifacts="artifacts/$product/"
+    ls "$artifacts"
+    aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/${product}/${version}/${target}"
+    echo "Uploaded files:"
+    aws s3 ls "s3://releases.parity.io/${product}/${version}/${target}" --recursive --human-readable --summarize
+    echo "✅ The release should be at https://releases.parity.io/${product}/${version}/${target}"
+}
+
+# Upload runtimes artifacts to s3 release bucket
+#
+# input: version (stable release tage.g. polkadot-stable2412 or polkadot-stable2412-rc1)
+# output: none
+upload_s3_runtimes_release_artifacts() {
   alias aws='podman run --rm -it docker.io/paritytech/awscli -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_BUCKET aws'
 
-  product=$1
-  version=$2
+  version=$1
 
-  echo "Working on product: $product "
   echo "Working on version: $version "
 
   echo "Current content, should be empty on new uploads:"
-  aws s3 ls "s3://releases.parity.io/polkadot/${version}/" --recursive --human-readable --summarize || true
+  aws s3 ls "s3://releases.parity.io/polkadot/runtimes/${version}/" --recursive --human-readable --summarize || true
   echo "Content to be uploaded:"
-  artifacts="artifacts/$product/"
+  artifacts="artifacts/runtimes/"
   ls "$artifacts"
-  aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/polkadot/${version}/"
+  aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/polkadot/runtimes/${version}/"
   echo "Uploaded files:"
-  aws s3 ls "s3://releases.parity.io/polkadot/${version}/" --recursive --human-readable --summarize
-  echo "✅ The release should be at https://releases.parity.io/polkadot/${version}"
+  aws s3 ls "s3://releases.parity.io/polkadot/runtimes/${version}/" --recursive --human-readable --summarize
+  echo "✅ The release should be at https://releases.parity.io/polkadot/runtimes/${version}"
 }

.github/workflows/build-misc.yml

@@ -44,7 +44,7 @@ jobs:
           forklift cargo check -p rococo-runtime
           forklift cargo check -p polkadot-test-runtime
       - name: Stop all workflows if failed
-        if: ${{ failure() && steps.required.conclusion == 'failure' }}
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper
         with:
           app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
@@ -73,7 +73,7 @@ jobs:
           cd ./substrate/bin/utils/subkey
           forklift cargo build --locked --release
       - name: Stop all workflows if failed
-        if: ${{ failure() && steps.required.conclusion == 'failure' }}
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper
         with:
           app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}

.github/workflows/check-frame-omni-bencher.yml

@@ -41,7 +41,7 @@ jobs:
           forklift cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks
           forklift cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet
       - name: Stop all workflows if failed
-        if: ${{ failure() && steps.required.conclusion == 'failure' }}
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper
         with:
           app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
@@ -99,7 +99,7 @@ jobs:
           echo "Running command: $cmd"
           eval "$cmd"
       - name: Stop all workflows if failed
-        if: ${{ failure() && steps.required.conclusion == 'failure' }}
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper
         with:
           app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}

.github/workflows/check-links.yml

@@ -33,7 +33,7 @@ jobs:
       - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.0 (22. Sep 2023)
 
       - name: Lychee link checker
-        uses: lycheeverse/lychee-action@7cd0af4c74a61395d455af97419279d86aafaede # for v1.9.1 (10. Jan 2024)
+        uses: lycheeverse/lychee-action@f81112d0d2814ded911bd23e3beaa9dda9093915 # for v1.9.1 (10. Jan 2024)
         with:
           args: >-
             --config .config/lychee.toml

.github/workflows/check-semver.yml

@@ -11,7 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  TOOLCHAIN: nightly-2024-06-01
+  TOOLCHAIN: nightly-2024-11-19
 
 jobs:
   preflight:
@@ -74,7 +74,7 @@ jobs:
 
       - name: install parity-publish
         # Set the target dir to cache the build.
-        run: CARGO_TARGET_DIR=./target/ cargo install parity-publish@0.8.0 --locked -q
+        run: CARGO_TARGET_DIR=./target/ cargo install parity-publish@0.10.2 --locked -q
 
       - name: check semver
         run: |

.github/workflows/checks-quick.yml

@@ -30,7 +30,7 @@ jobs:
         id: required
         run: cargo +nightly fmt --all -- --check
       - name: Stop all workflows if failed
-        if: ${{ failure() && steps.required.conclusion == 'failure' }}
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper
         with:
           app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
@@ -97,7 +97,6 @@ jobs:
           --exclude
           "substrate/frame/contracts/fixtures/build"
           "substrate/frame/contracts/fixtures/contracts/common"
-          "substrate/frame/revive/fixtures/build"
           "substrate/frame/revive/fixtures/contracts/common"
       - name: deny git deps
         run: python3 .github/scripts/deny-git-deps.py .

.github/workflows/checks.yml

@@ -36,7 +36,7 @@ jobs:
           cargo clippy --all-targets --locked --workspace --quiet
           cargo clippy --all-targets --all-features --locked --workspace --quiet
       - name: Stop all workflows if failed
-        if: ${{ failure() && steps.required.conclusion == 'failure' }}
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper
         with:
           app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
@@ -62,7 +62,7 @@ jobs:
           # experimental code may rely on try-runtime and vice-versa
           forklift cargo check --locked --all --features try-runtime,experimental --quiet
       - name: Stop all workflows if failed
-        if: ${{ failure() && steps.required.conclusion == 'failure' }}
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper
         with:
           app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}
@@ -91,7 +91,7 @@ jobs:
           ./check-features-variants.sh
           cd -
       - name: Stop all workflows if failed
-        if: ${{ failure() && steps.required.conclusion == 'failure' }}
+        if: ${{ failure() && steps.required.conclusion == 'failure' && !github.event.pull_request.head.repo.fork }}
         uses: ./.github/actions/workflow-stopper
         with:
           app-id: ${{ secrets.WORKFLOW_STOPPER_RUNNER_APP_ID }}

.github/workflows/command-backport.yml

@@ -40,7 +40,7 @@ jobs:
         uses: korthout/backport-action@v3
         id: backport
         with:
-          target_branches: stable2407 stable2409
+          target_branches: stable2407 stable2409 stable2412
           merge_commits: skip
           github_token: ${{ steps.generate_token.outputs.token }}
           pull_description: |