Short benchmarking in CI with frame-omni-bencher + extract `chain-s…

…pec-builder` stuff to `get_preset` (#379)

This PR introduces a new CI pipeline for checking and running runtime
benchmarks as part of the test pipelines. This means we will now know if
any changes break the benchmarks.

The pipeline is based on downloading `frame-omni-bencher` (building it
in-place is too expensive for every matrix run) and running it with a
minimal setup: `--steps 2 --repeat 1`.

Another part of this PR splits the default genesis setups from
`chain-spec-generator` and moves them to the
`sp_genesis_builder::GenesisBuilder::get_preset` runtime API
implementation for every runtime, which is required by
`frame-omni-bencher`.


Closes: #197
Relates to: #298
Relates to: #324

<!-- Remember that you can run `/merge` to enable auto-merge in the PR
-->

<!-- Remember to modify the changelog. If you don't need to modify it,
you can check the following box.
Instead, if you have already modified it, simply delete the following
line. -->

- [X] Does not require a CHANGELOG entry

## Future works

When [this issue](paritytech/polkadot-sdk#5083)
is fixed, I will rewrite
[weight-generation.md](https://github.com/polkadot-fellows/runtimes/blob/main/docs/weight-generation.md).
The new version will be significantly simplified, with instructions to
simply download `frame-omni-bencher`, build the runtime WASM, and run
it—no other steps will be necessary.

```
2024-07-18 14:45:51 Using the chain spec instead of the runtime to generate the genesis state is deprecated. Please remove the `--chain`/`--dev`/`--local` argument, point `--runtime` to your runtime blob and set `--genesis-builder=runtime`. This warning may become a hard error any time after December 2024.    
```

---------

Co-authored-by: Bastian Köcher <[email protected]>

.github/workflows/test.yml

@@ -9,6 +9,9 @@ on:
   pull_request:
   workflow_dispatch:
 
+env:
+  FRAME_OMNI_BENCHER_RELEASE_VERSION: polkadot-v1.13.0
+
 # cancel previous runs
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -82,6 +85,13 @@ jobs:
         with:
           shared-key: "fellowship-cache-tests"
 
+      - name: Download frame-omni-bencher
+        run: |
+          curl -sL https://github.com/paritytech/polkadot-sdk/releases/download/$FRAME_OMNI_BENCHER_RELEASE_VERSION/frame-omni-bencher -o frame-omni-bencher
+          chmod +x ./frame-omni-bencher
+          ./frame-omni-bencher --version
+        shell: bash
+
       - name: Test ${{ matrix.runtime.name }}
         run: cargo test -p ${{ matrix.runtime.package }} --release --locked -q
         env:
@@ -93,6 +103,20 @@ jobs:
           RUSTFLAGS: "-C debug-assertions -D warnings"
           SKIP_WASM_BUILD: 1
 
+      - name: Test benchmarks ${{ matrix.runtime.name }}
+        run: |
+          PACKAGE_NAME=${{ matrix.runtime.package }}
+          RUNTIME_BLOB_NAME=$(echo $PACKAGE_NAME | sed 's/-/_/g').compact.compressed.wasm
+          RUNTIME_BLOB_PATH=./target/production/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME
+          # build wasm
+          echo "Preparing wasm for benchmarking RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH"
+          cargo build --profile production -p ${{ matrix.runtime.package }} --features=runtime-benchmarks -q --locked
+          # run benchmarking
+          echo "Running benchmarking for RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH"
+          ./frame-omni-bencher v1 benchmark pallet --runtime $RUNTIME_BLOB_PATH --all --steps 2 --repeat 1
+        env:
+          RUSTFLAGS: "-C debug-assertions -D warnings"
+
   # Job required by "confirmTestPassed"
   integration-test:
     needs: [integration-test-matrix]

Cargo.toml

@@ -191,7 +191,7 @@ sc-chain-spec = { version = "34.0.0" }
 scale-info = { version = "2.10.0", default-features = false }
 separator = { version = "0.4.1" }
 serde = { version = "1.0.196" }
-serde_json = { version = "1.0.113" }
+serde_json = { version = "1.0.113", default-features = false }
 smallvec = { version = "1.13.1" }
 snowbridge-beacon-primitives = { version = "0.7.0", default-features = false }
 snowbridge-core = { version = "0.7.0", default-features = false }
@@ -271,6 +271,7 @@ members = [
 	"integration-tests/emulated/tests/coretime/coretime-kusama",
 	"integration-tests/emulated/tests/people/people-kusama",
 	"integration-tests/emulated/tests/people/people-polkadot",
+	"integration-tests/zombienet",
 	"relay/kusama",
 	"relay/kusama/constants",
 	"relay/polkadot",
@@ -291,7 +292,6 @@ members = [
 	"system-parachains/gluttons/glutton-kusama",
 	"system-parachains/people/people-kusama",
 	"system-parachains/people/people-polkadot",
-	"integration-tests/zombienet",
 ]
 
 [profile.release]

chain-spec-generator/Cargo.toml

@@ -12,23 +12,9 @@ serde_json = { workspace = true }
 serde = { features = ["derive"], workspace = true }
 
 polkadot-runtime = { workspace = true }
-polkadot-runtime-constants = { workspace = true, default-features = true }
 kusama-runtime = { workspace = true }
-kusama-runtime-constants = { workspace = true, default-features = true }
 
 sc-chain-spec = { workspace = true }
-runtime-parachains = { default-features = true, workspace = true }
-polkadot-primitives = { workspace = true, default-features = true }
-babe-primitives = { workspace = true, default-features = true }
-authority-discovery-primitives = { workspace = true, default-features = true }
-sp-core = { workspace = true, default-features = true }
-pallet-staking = { workspace = true, default-features = true }
-grandpa = { workspace = true }
-sp-runtime = { workspace = true, default-features = true }
-beefy-primitives = { workspace = true, default-features = true }
-xcm = { workspace = true, default-features = true }
-parachains-common = { workspace = true, default-features = true }
-cumulus-primitives-core = { workspace = true, default-features = true }
 
 asset-hub-polkadot-runtime = { workspace = true }
 asset-hub-kusama-runtime = { workspace = true }
@@ -50,17 +36,10 @@ runtime-benchmarks = [
 	"bridge-hub-polkadot-runtime/runtime-benchmarks",
 	"collectives-polkadot-runtime/runtime-benchmarks",
 	"coretime-kusama-runtime/runtime-benchmarks",
-	"cumulus-primitives-core/runtime-benchmarks",
 	"encointer-kusama-runtime/runtime-benchmarks",
 	"glutton-kusama-runtime/runtime-benchmarks",
 	"kusama-runtime/runtime-benchmarks",
-	"pallet-staking/runtime-benchmarks",
-	"parachains-common/runtime-benchmarks",
 	"people-kusama-runtime/runtime-benchmarks",
 	"people-polkadot-runtime/runtime-benchmarks",
-	"polkadot-primitives/runtime-benchmarks",
 	"polkadot-runtime/runtime-benchmarks",
-	"runtime-parachains/runtime-benchmarks",
-	"sp-runtime/runtime-benchmarks",
-	"encointer-kusama-runtime/runtime-benchmarks"
 ]

chain-spec-generator/src/common.rs

@@ -19,46 +19,12 @@ use crate::{
 	relay_chain_specs::{KusamaChainSpec, PolkadotChainSpec},
 	system_parachains_specs::{
 		AssetHubKusamaChainSpec, AssetHubPolkadotChainSpec, BridgeHubKusamaChainSpec,
-		BridgeHubPolkadotChainSpec, CollectivesPolkadotChainSpec, EncointerKusamaChainSpec,
-		GluttonKusamaChainSpec, PeopleKusamaChainSpec, PeoplePolkadotChainSpec,
+		BridgeHubPolkadotChainSpec, CollectivesPolkadotChainSpec, CoretimeKusamaChainSpec,
+		EncointerKusamaChainSpec, GluttonKusamaChainSpec, PeopleKusamaChainSpec,
+		PeoplePolkadotChainSpec,
 	},
 	ChainSpec,
 };
-use polkadot_primitives::{AccountId, AccountPublic};
-use sp_core::{sr25519, Pair, Public};
-use sp_runtime::traits::IdentifyAccount;
-
-pub fn testnet_accounts() -> Vec<AccountId> {
-	vec![
-		get_account_id_from_seed::<sr25519::Public>("Alice"),
-		get_account_id_from_seed::<sr25519::Public>("Bob"),
-		get_account_id_from_seed::<sr25519::Public>("Charlie"),
-		get_account_id_from_seed::<sr25519::Public>("Dave"),
-		get_account_id_from_seed::<sr25519::Public>("Eve"),
-		get_account_id_from_seed::<sr25519::Public>("Ferdie"),
-		get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
-		get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
-		get_account_id_from_seed::<sr25519::Public>("Charlie//stash"),
-		get_account_id_from_seed::<sr25519::Public>("Dave//stash"),
-		get_account_id_from_seed::<sr25519::Public>("Eve//stash"),
-		get_account_id_from_seed::<sr25519::Public>("Ferdie//stash"),
-	]
-}
-
-/// Helper function to generate a crypto pair from seed
-pub fn get_from_seed<TPublic: Public>(seed: &str) -> <TPublic::Pair as Pair>::Public {
-	TPublic::Pair::from_string(&format!("//{}", seed), None)
-		.expect("static values are valid; qed")
-		.public()
-}
-
-/// Helper function to generate an account ID from seed
-pub fn get_account_id_from_seed<TPublic: Public>(seed: &str) -> AccountId
-where
-	AccountPublic: From<<TPublic::Pair as Pair>::Public>,
-{
-	AccountPublic::from(get_from_seed::<TPublic>(seed)).into_account()
-}
 
 #[derive(Debug, serde::Deserialize)]
 struct EmptyChainSpecWithId {
@@ -86,6 +52,8 @@ pub fn from_json_file(filepath: &str, supported: String) -> Result<Box<dyn Chain
 			Ok(Box::new(BridgeHubPolkadotChainSpec::from_json_file(path)?)),
 		x if x.starts_with("bridge-hub-kusama") =>
 			Ok(Box::new(BridgeHubKusamaChainSpec::from_json_file(path)?)),
+		x if x.starts_with("coretime-kusama") =>
+			Ok(Box::new(CoretimeKusamaChainSpec::from_json_file(path)?)),
 		x if x.starts_with("glutton-kusama") =>
 			Ok(Box::new(GluttonKusamaChainSpec::from_json_file(path)?)),
 		x if x.starts_with("encointer-kusama") =>