From 3f13f2bf167bbaf308706a39178e92869986bc7e Mon Sep 17 00:00:00 2001 From: eseiker Date: Tue, 12 Nov 2024 19:43:48 +0900 Subject: [PATCH] Add thor to github workflows and scripts --- .github/scripts/reset-snapshot.sh | 25 ++++---- .github/scripts/validate-charts.sh | 2 +- .github/workflows/apv.yaml | 7 ++- .github/workflows/reset-bridge-service.yaml | 11 ++-- .github/workflows/snapshot-reset.yaml | 68 ++++++++++++++++++++- .github/workflows/test-internal-chain.yaml | 1 + .github/workflows/update-paev.yaml | 1 + .github/workflows/update-values.yaml | 1 + scripts/app/constants.py | 7 ++- scripts/app/test_internal_chain.py | 37 ++++------- scripts/app/types.py | 3 - scripts/app/update_apv.py | 8 +-- scripts/app/update_bridge_service.py | 39 ++++-------- scripts/app/update_paev.py | 20 +++--- scripts/app/utils/converter.py | 8 +-- scripts/cli.py | 2 +- 16 files changed, 132 insertions(+), 108 deletions(-) delete mode 100644 scripts/app/types.py diff --git a/.github/scripts/reset-snapshot.sh b/.github/scripts/reset-snapshot.sh index 42dda7730..666678b1e 100644 --- a/.github/scripts/reset-snapshot.sh +++ b/.github/scripts/reset-snapshot.sh @@ -4,10 +4,12 @@ set -ex BASEDIR=$(dirname "$0") echo "$BASEDIR" +BUCKET="s3://9c-snapshots-v2" + # AWS configuration must be already set on your environment reset_snapshot() { CHAIN=$3 - PREFIX="s3://9c-snapshots-v2/" + PREFIX="$BUCKET/" PREVIOUS_MAINNET_EPOCH_PATH="${1#$PREFIX}" BASE_URL_PATH="${2#$PREFIX}" NEW_SNAPSHOT_TIP=0 @@ -90,14 +92,13 @@ reset_snapshot() { fi - BUCKET="s3://9c-snapshots-v2" BUCKET_PREFIX=$(echo $BUCKET | awk '{gsub(/\//,"\\/");print}') CF_PATH=$(echo $1/ | sed -e "s/^$BUCKET_PREFIX//" | sed "s/.*/&*/") # reset cf path CF_DISTRIBUTION_ID="EAU4XRUZSBUD5" aws cloudfront create-invalidation --distribution-id "$CF_DISTRIBUTION_ID" --paths "$CF_PATH" - if [ "$CHAIN" = "odin" ] || [ "$CHAIN" = "heimdall" ]; then + if [[ $CHAIN != *-preview ]]; then curl --data "[9C-INFRA] Internal $CHAIN snapshot file reset complete. New tip: \`#$NEW_SNAPSHOT_TIP\`." "https://planetariumhq.slack.com/services/hooks/slackbot?token=$SLACK_TOKEN&channel=%239c-internal" else curl --data "[9C-INFRA] Preview $CHAIN snapshot file reset complete. New tip: \`#$NEW_SNAPSHOT_TIP\`." "https://planetariumhq.slack.com/services/hooks/slackbot?token=$SLACK_TOKEN&channel=%239c-previewnet" @@ -107,20 +108,16 @@ reset_snapshot() { # Type "y" to reset the cluster with a new snapshot and "n" to just deploy the cluster. echo "Do you want to reset the cluster with a new snapshot(y/n)?" read response -CHAIN_NAME=$1 +CHAIN=$1 if [ "$response" = "y" ]; then echo "Reset cluster with a new snapshot" - - if [ "$CHAIN_NAME" = "odin" ]; then - reset_snapshot "s3://9c-snapshots-v2/internal" "s3://9c-snapshots-v2/main/partition/internal" "$CHAIN_NAME" || true - elif [ "$CHAIN_NAME" = "heimdall" ]; then - reset_snapshot "s3://9c-snapshots-v2/internal/heimdall" "s3://9c-snapshots-v2/main/heimdall/partition/internal" "$CHAIN_NAME" || true - elif [ "$CHAIN_NAME" = "odin-preview" ]; then - reset_snapshot "s3://9c-snapshots-v2/preview" "s3://9c-snapshots-v2/main/partition/internal" "$CHAIN_NAME" || true - else - reset_snapshot "s3://9c-snapshots-v2/preview/heimdall" "s3://9c-snapshots-v2/main/heimdall/partition/internal" "$CHAIN_NAME" || true - fi + + INTERNAL_OR_PREVIEW=$([[ $CHAIN != *-preview ]] && echo "internal" || echo "preview") + CHAIN_NAME=${CHAIN%-preview} + CHAIN_PATH=$([[ $CHAIN_NAME = odin ]] && echo "" || echo "/$CHAIN_NAME") + + reset_snapshot "$BUCKET/$INTERNAL_OR_PREVIEW$CHAIN_PATH" "$BUCKET/main$CHAIN_PATH/partition/internal" "$CHAIN_NAME" || true else echo "Reset cluster without resetting snapshot." diff --git a/.github/scripts/validate-charts.sh b/.github/scripts/validate-charts.sh index 277eb5e16..23a534e6f 100755 --- a/.github/scripts/validate-charts.sh +++ b/.github/scripts/validate-charts.sh @@ -10,7 +10,7 @@ done # Check multiplanetary networks manually CLUSTERS=("9c-main" "9c-internal") -NETWORKS=("9c-network" "heimdall" "idun") +NETWORKS=("9c-network" "heimdall" "thor" "idun") for cluster in "${CLUSTERS[@]}"; do for network in "${NETWORKS[@]}"; do helm template --values "$cluster/multiplanetary/network/$network.yaml" \ diff --git a/.github/workflows/apv.yaml b/.github/workflows/apv.yaml index ceee64abe..40d3bf07b 100644 --- a/.github/workflows/apv.yaml +++ b/.github/workflows/apv.yaml @@ -8,10 +8,12 @@ on: default: 'main' type: choice options: - - main-heimdall - main-odin - - internal-heimdall + - main-heimdall + - main-thor - internal-odin + - internal-heimdall + - internal-thor number: required: true description: 'Apv number (e.g. 100310)' @@ -32,6 +34,7 @@ on: - general - 9c-network - heimdall + - thor jobs: manage-apv: diff --git a/.github/workflows/reset-bridge-service.yaml b/.github/workflows/reset-bridge-service.yaml index e3584fbd1..a0912d0bb 100644 --- a/.github/workflows/reset-bridge-service.yaml +++ b/.github/workflows/reset-bridge-service.yaml @@ -9,8 +9,9 @@ on: default: 'internal' type: choice options: - - internal - - preview + - heimdall-internal + - thor-internal + - heimdall-preview jobs: reset-bridge-service: @@ -32,11 +33,7 @@ jobs: - name: Set namespace based on input run: | - if [ "${{ github.event.inputs.network }}" = "internal" ]; then - echo "NAMESPACE=heimdall" >> $GITHUB_ENV - else - echo "NAMESPACE=heimdall-preview" >> $GITHUB_ENV - fi + echo ${${{ github.event.inputs.network }}%"-internal"} >> $GITHUB_ENV - uses: actions-hub/kubectl@master env: diff --git a/.github/workflows/snapshot-reset.yaml b/.github/workflows/snapshot-reset.yaml index d2c630f41..363aa02ce 100644 --- a/.github/workflows/snapshot-reset.yaml +++ b/.github/workflows/snapshot-reset.yaml @@ -8,12 +8,13 @@ on: default: 'odin' type: choice options: + - internal-all - odin - heimdall + - thor + - preview-all - odin-preview - heimdall-preview - - internal-all - - preview-all concurrency: group: release @@ -51,6 +52,22 @@ jobs: AWS_REGION: ${{ secrets.AWS_REGION }} SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} + release-thor: + if: ${{ github.event.inputs.chain == 'thor' || github.event.inputs.chain == 'internal-all' }} + runs-on: ubuntu-latest + environment: + name: internal + steps: + - uses: actions/checkout@v3 + - name: reset internal snapshot + run: | + echo "y" | bash .github/scripts/reset-snapshot.sh thor + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} + release-odin-preview: if: ${{ github.event.inputs.chain == 'odin-preview' || github.event.inputs.chain == 'preview-all' }} runs-on: ubuntu-latest @@ -83,7 +100,7 @@ jobs: AWS_REGION: ${{ secrets.AWS_REGION }} SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} - reset-bridge-service: + reset-bridge-service-heimdall: if: ${{ github.event.inputs.chain == 'internal-all' || github.event.inputs.chain == 'odin' || github.event.inputs.chain == 'heimdall' }} runs-on: ubuntu-latest needs: [release-odin, release-heimdall] @@ -128,6 +145,51 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.P_GITHUB_TOKEN }} + reset-bridge-service-thor: + if: ${{ github.event.inputs.chain == 'internal-all' || github.event.inputs.chain == 'odin' || github.event.inputs.chain == 'thor' }} + runs-on: ubuntu-latest + needs: [release-odin, release-thor] + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + - name: Set up kubectl + uses: azure/setup-kubectl@v1 + with: + version: 'v1.29.0' + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y jq + - uses: actions-hub/kubectl@master + env: + KUBE_CONFIG: ${{ secrets.KUBECONFIG }} + with: + args: scale --replicas=0 statefulset/bridge-service -n thor + - uses: actions-hub/kubectl@master + env: + KUBE_CONFIG: ${{ secrets.KUBECONFIG }} + with: + args: scale --replicas=0 statefulset/bridge-service-db -n thor + - uses: actions-hub/kubectl@master + env: + KUBE_CONFIG: ${{ secrets.KUBECONFIG }} + with: + args: delete pvc/bridge-service-db-data-bridge-service-db-0 --namespace=thor + - uses: actions/setup-python@v2.2.2 + with: + python-version: 3.10.13 + - run: | + python -m pip install -r requirements.txt + flit install + name: install dependencies + working-directory: ./scripts + - name: Update 'bridgeService.rdb.defaultStartBlockIndex' + run: | + python cli.py update-bridge-service 9c-internal thor + working-directory: ./scripts + env: + GITHUB_TOKEN: ${{ secrets.P_GITHUB_TOKEN }} + reset-bridge-service-preview: if: ${{ github.event.inputs.chain == 'preview-all' || github.event.inputs.chain == 'odin-preview' || github.event.inputs.chain == 'heimdall-preview' }} runs-on: ubuntu-latest diff --git a/.github/workflows/test-internal-chain.yaml b/.github/workflows/test-internal-chain.yaml index f488eb3d2..61fa25cff 100644 --- a/.github/workflows/test-internal-chain.yaml +++ b/.github/workflows/test-internal-chain.yaml @@ -11,6 +11,7 @@ on: options: - odin-internal - heimdall-internal + - thor-internal offset: required: true description: 'Insert the snapshot tip index of the internal network (e.g. 1115268)' diff --git a/.github/workflows/update-paev.yaml b/.github/workflows/update-paev.yaml index 507da1c9a..4e1ebb7b9 100644 --- a/.github/workflows/update-paev.yaml +++ b/.github/workflows/update-paev.yaml @@ -10,6 +10,7 @@ on: options: - odin - heimdall + - thor description: 'Choose the mainnet network to update the appsettings.json' previous_version_block_index: required: true diff --git a/.github/workflows/update-values.yaml b/.github/workflows/update-values.yaml index 42ea4e3e3..3ea5bba08 100644 --- a/.github/workflows/update-values.yaml +++ b/.github/workflows/update-values.yaml @@ -19,6 +19,7 @@ on: - general - 9c-network - heimdall + - thor - idun headless: description: 'Dockerhub tag for headless (e.g. 30, git-5db812731f7fef1156b00a90ff0267e95dd31820)' diff --git a/scripts/app/constants.py b/scripts/app/constants.py index 27527c9f6..279f9c2c7 100644 --- a/scripts/app/constants.py +++ b/scripts/app/constants.py @@ -7,5 +7,8 @@ RELEASE_BASE_URL = "https://release.nine-chronicles.com" RELEASE_BUCKET = "9c-release.planetariumhq.com" -MAIN_ODIN_SIGNER="0xAB2da648b9154F2cCcAFBD85e0Bc3d51f97330Fc" -MAIN_HEIMDALL_SIGNER="0xeE394bb942fa7c2d807C170C7Db7F26cb3EA037F" +MAIN_SIGNERS = { + "odin": "0xAB2da648b9154F2cCcAFBD85e0Bc3d51f97330Fc", + "heimdall": "0xeE394bb942fa7c2d807C170C7Db7F26cb3EA037F", + "thor": "0xC6553c8e634bEE685F264F4C5720d65919dc9c9c", +} diff --git a/scripts/app/test_internal_chain.py b/scripts/app/test_internal_chain.py index c4da02739..b62b2f834 100644 --- a/scripts/app/test_internal_chain.py +++ b/scripts/app/test_internal_chain.py @@ -1,38 +1,25 @@ import aiohttp import asyncio -import sys -import os import requests from urllib.parse import urljoin from app.client import GithubClient from app.config import config -Network = str -Offset = int -Limit = int -Delay = int - class InternalChainTester: def __init__(self) -> None: self.github_client = GithubClient( config.github_token, org="planetarium", repo="TEMP" ) - - def test( - self, - network: Network, - offset: Offset, - limit: Limit, - delay: Delay - ): + + def test(self, network: str, offset: int, limit: int, delay: int): # mainnet headless URL and initial setup - mainnet_headless = "odin-full-state.nine-chronicles.com" - internal_target_validator = "odin-internal-validator-5.nine-chronicles.com" + stripped_network = network.removesuffix("-internal") + validator_target = 5 if stripped_network == "odin" else 1 + + mainnet_headless = f"{stripped_network}-full-state.nine-chronicles.com" + internal_target_validator = f"{stripped_network}-internal-validator-{validator_target}.nine-chronicles.com" sleep_time = delay - if network == "heimdall-internal": - mainnet_headless = "heimdall-full-state.nine-chronicles.com" - internal_target_validator = "heimdall-internal-validator-1.nine-chronicles.com" mainnet_headless_url = urljoin(f"http://{mainnet_headless}", "graphql") target_validator_url = urljoin(f"http://{internal_target_validator}", "graphql") @@ -65,7 +52,7 @@ def test( response = requests.post(mainnet_headless_url, json={'query': mainnet_headless_query}, headers={'content-type': 'application/json'}) data = response.json() blocks = data['data']['chainQuery']['blockQuery']['blocks'] - + # Process the blocks here (e.g., print them out or handle them as needed) print(f"Fetched {len(blocks)} blocks starting from offset {offset}") @@ -91,7 +78,7 @@ async def process_block(blocks): offset += current_limit if hasattr(config, 'slack_token') and config.slack_token: - url = f'https://planetariumhq.slack.com/services/hooks/slackbot?token={config.slack_token}&channel=%23{config.slack_channel}' - data = f"[9C-INFRA] Finished testing `{network}` network from `#{original_offset}` to `#{tip_index}`." - headers = {'Content-Type': 'text/plain'} - response = requests.post(url, data=data, headers=headers) + url = f"https://planetariumhq.slack.com/services/hooks/slackbot?token={config.slack_token}&channel=%23{config.slack_channel}" + data = f"[9C-INFRA] Finished testing `{network}` network from `#{original_offset}` to `#{tip_index}`." + headers = {"Content-Type": "text/plain"} + response = requests.post(url, data=data, headers=headers) diff --git a/scripts/app/types.py b/scripts/app/types.py deleted file mode 100644 index e006652de..000000000 --- a/scripts/app/types.py +++ /dev/null @@ -1,3 +0,0 @@ -from typing import Literal - -Network = Literal["main", "internal"] diff --git a/scripts/app/update_apv.py b/scripts/app/update_apv.py index 1fc5d1bb2..fd9edb5c3 100644 --- a/scripts/app/update_apv.py +++ b/scripts/app/update_apv.py @@ -4,7 +4,7 @@ from app.config import config from app.tools.planet import Apv, Planet -from app.constants import MAIN_HEIMDALL_SIGNER, MAIN_ODIN_SIGNER +from app.constants import MAIN_SIGNERS logger = structlog.get_logger(__name__) from tempfile import TemporaryFile @@ -124,10 +124,8 @@ def update_apv(contents: str, apv: Apv): return new_doc def check_correct_signer(dir_name:str, file_name: str): - if "9c-main" in dir_name and file_name == "heimdall": - assert config.key_address == MAIN_HEIMDALL_SIGNER - if "9c-main" in dir_name and file_name == "odin": - assert config.key_address == MAIN_ODIN_SIGNER + if "9c-main" in dir_name: + assert config.key_address == MAIN_SIGNERS[file_name] def generate_apv(planet: Planet, number: int) -> Apv: timestamp = datetime.utcnow().strftime("%Y-%m-%d") diff --git a/scripts/app/update_bridge_service.py b/scripts/app/update_bridge_service.py index 9283d26dd..0e2f8c4e0 100644 --- a/scripts/app/update_bridge_service.py +++ b/scripts/app/update_bridge_service.py @@ -8,36 +8,18 @@ from app.client import GithubClient from app.config import config -from app.dockerhub.image import check_image_exists logger = structlog.get_logger(__name__) GQL_QUERY = {"query": "{ nodeStatus { tip { index } } }"} -Network = Literal["9c-internal"] -Planet = Literal["heimdall"] - -SNAPSHOT_METADATA_URL_MAP = { - "9c-internal": { - "odin": "https://9c-snapshots-v2.s3.us-east-2.amazonaws.com/internal/latest.json", - "heimdall": "https://9c-snapshots-v2.s3.us-east-2.amazonaws.com/internal/heimdall/latest.json", - "odin-preview": "https://9c-snapshots-v2.s3.us-east-2.amazonaws.com/preview/latest.json", - "heimdall-preview": "https://9c-snapshots-v2.s3.us-east-2.amazonaws.com/preview/heimdall/latest.json", - }, -} - - class BridgeServiceUpdater: def __init__(self) -> None: self.github_client = GithubClient( config.github_token, org="planetarium", repo="TEMP" ) - def update( - self, - dir_name: Network, - file_name: Planet - ): + def update(self, dir_name, file_name): new_branch = f"update-bridge-service-{int(time())}" file_path = f"{dir_name}/multiplanetary/network/{file_name}.yaml" @@ -151,11 +133,16 @@ def update_index_recursively(data): return new_doc -def get_metadata_url_pair(network: Network, planet: Planet) -> Tuple[str, str]: - match (network, planet): - case ("9c-internal", "heimdall"): - return (SNAPSHOT_METADATA_URL_MAP["9c-internal"]["odin"], SNAPSHOT_METADATA_URL_MAP["9c-internal"]["heimdall"]) - case ("9c-internal", "heimdall-preview"): - return (SNAPSHOT_METADATA_URL_MAP["9c-internal"]["odin-preview"], SNAPSHOT_METADATA_URL_MAP["9c-internal"]["heimdall-preview"]) +def get_metadata_url(network: str, planet: str) -> str: + if network != "9c-internal": + raise TypeError(f"Not supported network and planet: {network}, {planet}") + + stripped_planet = planet.removesuffix("-preview") + internal_or_preview = "preview" if planet.endswith("-preview") else "internal" + chain_path = "" if stripped_planet is "odin" else f"/{stripped_planet}" + + return f"https://9c-snapshots-v2.s3.us-east-2.amazonaws.com/{internal_or_preview}{chain_path}/latest.json" + - raise TypeError(f"Not supported network and planet: {network}, {planet}") +def get_metadata_url_pair(network, planet) -> Tuple[str, str]: + return (get_metadata_url(network, "odin"), get_metadata_url(network, planet)) diff --git a/scripts/app/update_paev.py b/scripts/app/update_paev.py index 118f3088d..2ba9825da 100644 --- a/scripts/app/update_paev.py +++ b/scripts/app/update_paev.py @@ -1,4 +1,4 @@ -import boto3 +import boto3 # type: ignore import requests import json @@ -24,16 +24,10 @@ def prep_update( x64_suffix = "/linux-x64.zip" # Define URLs based on network type - if network_type == "odin": - paev_urls = [ - "https://9c-cluster-config.s3.us-east-2.amazonaws.com/9c-main/odin/appsettings.json", - "https://9c-cluster-config.s3.us-east-2.amazonaws.com/9c-main/odin/appsettings-nodeinfra.json" - ] - else: - paev_urls = [ - "https://9c-cluster-config.s3.us-east-2.amazonaws.com/9c-main/heimdall/appsettings.json", - "https://9c-cluster-config.s3.us-east-2.amazonaws.com/9c-main/heimdall/appsettings-nodeinfra.json" - ] + paev_urls = [ + f"https://9c-cluster-config.s3.us-east-2.amazonaws.com/9c-main/{network_type}/appsettings.json", + f"https://9c-cluster-config.s3.us-east-2.amazonaws.com/9c-main/{network_type}/appsettings-nodeinfra.json", + ] # Iterate over each URL in the list for url in paev_urls: @@ -42,7 +36,7 @@ def prep_update( # Construct the plugin URL plugin_url = f"{base_url}{previous_version_lib9c_commit}{suffix}" - + # Call update function self.update(url, previous_version_block_index, plugin_url) @@ -89,7 +83,7 @@ def update( pairs.insert(insert_index, value_to_append) else: raise Exception("The specified range was not found in the pairs.") - + else: raise Exception("The specified structure does not exist in the provided JSON data.") diff --git a/scripts/app/utils/converter.py b/scripts/app/utils/converter.py index c82798d62..1d5ae0457 100644 --- a/scripts/app/utils/converter.py +++ b/scripts/app/utils/converter.py @@ -1,9 +1,5 @@ -from typing import Dict - -from app.types import Network - -def infra_dir2network(dir: str) -> Network: - infra_dir2network_map: Dict[str, Network] = { +def infra_dir2network(dir: str): + infra_dir2network_map = { "9c-main": "main", "9c-internal": "internal", } diff --git a/scripts/cli.py b/scripts/cli.py index 4e54c6bc5..22dfb089a 100644 --- a/scripts/cli.py +++ b/scripts/cli.py @@ -48,7 +48,7 @@ def update_bridge_service( def test_internal_chain( network: str = typer.Argument( ..., - help="odin-internal or heimdall-internal", + help="odin-internal, heimdall-internal, ...", ), offset: int = typer.Argument( ...,