From e7a2ce4951ed26d6bb9c76e0f51a4a04847b9c7e Mon Sep 17 00:00:00 2001 From: James Bayly Date: Tue, 7 May 2024 08:22:11 +0800 Subject: [PATCH] Remove db snapshots --- docs/.vuepress/config.ts | 2 - docs/.vuepress/sidebar.ts | 1 - .../indexers/dictionary-restore.md | 102 ------------------ .../node_operators/indexers/index-project.md | 42 -------- .../setup/becoming-a-node-operator.md | 4 - 5 files changed, 151 deletions(-) delete mode 100644 docs/subquery_network/node_operators/indexers/dictionary-restore.md diff --git a/docs/.vuepress/config.ts b/docs/.vuepress/config.ts index 79fdfaa86ea..949c5fc281c 100644 --- a/docs/.vuepress/config.ts +++ b/docs/.vuepress/config.ts @@ -177,8 +177,6 @@ export default defineUserConfig({ "/subquery_network/node_operators/indexers/install-indexer-linux.html", "/subquery_network/kepler/indexers/index-project.html": "/subquery_network/node_operators/indexers/index-project.html", - "/subquery_network/kepler/indexers/dictionary-restore.html": - "/subquery_network/node_operators/indexers/dictionary-restore.html", "/subquery_network/kepler/indexers/plans.html": "/subquery_network/node_operators/indexers/plans.html", "/subquery_network/kepler/indexers/troubleshooting-indexers.html": diff --git a/docs/.vuepress/sidebar.ts b/docs/.vuepress/sidebar.ts index bcd6b6fed1a..c0b254af34a 100644 --- a/docs/.vuepress/sidebar.ts +++ b/docs/.vuepress/sidebar.ts @@ -633,7 +633,6 @@ export const getSidebar = (locale: string) => children: [ `${locale}/subquery_network/node_operators/indexers/introduction.md`, `${locale}/subquery_network/node_operators/indexers/index-project.md`, - `${locale}/subquery_network/node_operators/indexers/dictionary-restore.md`, ], }, { diff --git a/docs/subquery_network/node_operators/indexers/dictionary-restore.md b/docs/subquery_network/node_operators/indexers/dictionary-restore.md deleted file mode 100644 index 7bba14310cd..00000000000 --- a/docs/subquery_network/node_operators/indexers/dictionary-restore.md +++ /dev/null @@ -1,102 +0,0 @@ -# Restoring Dictionary Projects - -Below are the [Dictionary Projects](../../../indexer/academy/tutorials_examples/dictionary.md) supported on Mainnet. These projects are commonly used, are simple to index and create, but create large datasets. - -In order to speed up the onboarding of Indexers, we are providing database snapshots for most of these dictionaries - -## Downloading Database Snapshots - -| Network | Deployment ID | Database Size | S3 Bucket URL | SHA256 | -| --------- | ------------------------------------------------ | ------------- | ---------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | -| Polkadot | `QmZGAZQ7e1oZgfuK4V29Fa5gveYK3G2zEwvUzTZKNvSBsm` | ~220GB | [S3 URL](https://kepler-dictionary-projects-prod.s3.amazonaws.com/polkadot/polkadot.tar) | `c1a1f67e2a205dc9fdd90f738686f3ee57052fcc7bc383a054574ab81e17584f` | -| Kusama | `QmXwfCF8858YY924VHgNLsxRQfBLosVbB31mygRLhgJbWn` | ~260GB | [S3 URL](https://kepler-dictionary-projects-prod.s3.amazonaws.com/kusama/kusama.tar) | `65f6fc3dd0410b296f651369690fd866070dbba8781a61454fc00cc11676452c` | -| Nodle | `QmQtmsHoJEYUcxKE4tBqr9Z8kudcgkczQPfhkAxVExQX5y` | ~15GB | [S3 URL](https://kepler-dictionary-projects-prod.s3.amazonaws.com/nodle/nodle.tar) | `71b52ef798f96c86214213e26a1960477d11f0f80916914159fd2feec2ba17fe` | -| Moonbeam | `QmeeqBHdVu7iYnhVE9ZiYEKTWe4jXVUD5pVoGXT6LbCP2t` | ~123GB | [S3 URL](https://kepler-dictionary-projects-prod.s3.amazonaws.com/moonbeam/schema_qmeeqbhdvu7iynh.tar.bz2) | `508a47bf84476e222d7ce72d4ca870a41da46d7c2284abdb3db915964d0a69c6` | -| Moonriver | `QmXCr6uZFdY1YcGTa4u6ZieQPXK4VHE1Pjy7CBr7ubFwKR` | ~130GB | [S3 URL](https://kepler-dictionary-projects-prod.s3.amazonaws.com/moonriver/moonriver.tar) | `db22b7565d8fea385a9e69636eb805079bf6708e898296f27673bc2b4d7a476d` | -| Acala | `Qmarrhgrpqw5VK71rMtb4GARpPvq8ajMjAqnjnWZFLV61N` | ~10G | [S3 URL](https://kepler-dictionary-projects-prod.s3.amazonaws.com/acala/acala.tar) | `4f0d8105f45ca856c57fa5f87d102e398e1e99403612c077750cb07bc9839c0d` | -| Karura | `QmWumrabg4k6t4EUMhQg19xWwcxGq1hWbcmfmRYiy2Bod5` | ~10G | [S3 URL](https://kepler-dictionary-projects-prod.s3.amazonaws.com/karura/karura.tar) | `046674efb30cdc7ab61b1e834201ac125548e0fafb5f6b69e321a9ddf7b06ae9` | -| Khala | `QmYCAns2cunZKJFU85KNK8CvL2ATAmCFVZRdBf963GqWYs` | ~78G | [S3 URL](https://kepler-dictionary-projects-prod.s3.amazonaws.com/khala/khala.tar) | `1b18b40345b7473fb4d8219f1da60381126ec8bbbe064158d2ed5b1b3ad532cd` | -| Westend | `Qma6BeSQGHrhP5aydmkQcJCR25TEwuNMogS5boovBBwoeW` | ~35G | [S3 URL](https://kepler-dictionary-projects-prod.s3.amazonaws.com/westend/westend.tar) | `72c94be8187a1298a81a7039900566a80447996899b047f6b4fe3f3066a89bef` | -| Astar | `QmUmnKPhcE6JwGMYvY3Yitb5j8qxbQBMxgpkHpVQuXqxDH` | ~65G | [S3 URL](https://kepler-dictionary-projects-prod.s3.amazonaws.com/astar/astar.tar) | `db2c8be67d18e7401290d67b3d7f457dc1881ef0505eb22807487d03b5031e81` | - -You can download the snapshot either from the s3 bucket URL or the BitTorrent magnet link: - -### Download Snapshot - -Downloading via Bittorrent will be faster, you can use your favourite bittorent client or install Aria2 - -```bash -sudo apt update -sudo apt install aria2 - -aria2c -``` - -To download from an S3 bucket - -```bash -curl -o dictionary.tar -``` - -## Restoring the Database Snapshot - -This assumes that you have an indexer [running locally](../../../indexer/run_publish/run.md) with admin access to a PostgresQL database (you will be using the `pg_restore` command). - -1. First extract the downloaded snapshot and then extract it using the following command: - -```bash -tar -xvf dictionary.tar -``` - -You will now have a pg dump file called: `schema_xxxxxxx.dump` - -2. Copy the `schema_xxxxxxx.dump` to `.data/postgres/` and then use this command: - -```bash -docker exec -it indexer_db pg_restore -v -j 2 -h localhost -p 5432 -U postgres -d postgres /var/lib/postgresql/data/schema_xxxxxxx.dump > restore.log 2>&1 & -``` - -:::note - -- please Make sure your `indexer_db` container is running with healthy status before starting restore process - -- We use the `-j` parameter to update the number of jobs running concurrently. Depending on your machine size, you may want to increase this number to speed up the restore process. [Read more](https://www.postgresql.org/docs/current/app-pgrestore.html) - -::: - -The restore process will start and take quite a long time (like 2 days), please make sure you run this cmd in the background (use tools like tmux/screen/nohup). Here is an example of the output log. - -``` -pg_restore: creating SCHEMA "schema_qmzj9whrhrmvn2h" -pg_restore: creating FUNCTION "schema_qmzj9whrhrmvn2h.schema_notification()" -pg_restore: creating TABLE "schema_qmzj9whrhrmvn2h._metadata" -pg_restore: creating TABLE "schema_qmzj9whrhrmvn2h._poi" -pg_restore: creating TABLE "schema_qmzj9whrhrmvn2h.events" -pg_restore: creating TABLE "schema_qmzj9whrhrmvn2h.extrinsics" -pg_restore: creating TABLE "schema_qmzj9whrhrmvn2h.spec_versions" -pg_restore: processing data for table "schema_qmzj9whrhrmvn2h._metadata" -pg_restore: processing data for table "schema_qmzj9whrhrmvn2h._poi" -pg_restore: processing data for table "schema_qmzj9whrhrmvn2h.events" -pg_restore: processing data for table "schema_qmzj9whrhrmvn2h.extrinsics" -pg_restore: processing data for table "schema_qmzj9whrhrmvn2h.spec_versions" -pg_restore: creating CONSTRAINT "schema_qmzj9whrhrmvn2h._metadata _metadata_pkey" -pg_restore: creating CONSTRAINT "schema_qmzj9whrhrmvn2h._poi _poi_chainBlockHash_key" -pg_restore: creating CONSTRAINT "schema_qmzj9whrhrmvn2h._poi _poi_hash_key" -pg_restore: creating CONSTRAINT "schema_qmzj9whrhrmvn2h._poi _poi_mmrRoot_key" -pg_restore: creating CONSTRAINT "schema_qmzj9whrhrmvn2h._poi _poi_parentHash_key" -pg_restore: creating CONSTRAINT "schema_qmzj9whrhrmvn2h._poi _poi_pkey" -pg_restore: creating CONSTRAINT "schema_qmzj9whrhrmvn2h.events events_pkey" -pg_restore: creating CONSTRAINT "schema_qmzj9whrhrmvn2h.extrinsics extrinsics_pkey" -pg_restore: creating CONSTRAINT "schema_qmzj9whrhrmvn2h.spec_versions spec_versions_pkey" -pg_restore: creating INDEX "schema_qmzj9whrhrmvn2h.events_block_height__block_range" -pg_restore: creating INDEX "schema_qmzj9whrhrmvn2h.events_event__block_range" -pg_restore: creating INDEX "schema_qmzj9whrhrmvn2h.events_module__block_range" -pg_restore: creating INDEX "schema_qmzj9whrhrmvn2h.extrinsics_block_height__block_range" -pg_restore: creating INDEX "schema_qmzj9whrhrmvn2h.extrinsics_call__block_range" -pg_restore: creating INDEX "schema_qmzj9whrhrmvn2h.extrinsics_module__block_range" -pg_restore: creating INDEX "schema_qmzj9whrhrmvn2h.extrinsics_tx_hash__block_range" -pg_restore: creating INDEX "schema_qmzj9whrhrmvn2h.poi_hash" -pg_restore: creating TRIGGER "schema_qmzj9whrhrmvn2h._metadata 0xc1aaf8b4176d0f02" -``` - -After the data restored, you can start adding the specific project to your service inside admin app, and start indexing the project, the indexing will start basing on the restored data and continue indexing the project. diff --git a/docs/subquery_network/node_operators/indexers/index-project.md b/docs/subquery_network/node_operators/indexers/index-project.md index 46e293d888a..1192cc426ca 100644 --- a/docs/subquery_network/node_operators/indexers/index-project.md +++ b/docs/subquery_network/node_operators/indexers/index-project.md @@ -69,48 +69,6 @@ Once the Controller Account has been added, a new Projects tab appears at the to Before clicking **`Add Project`,** you will need to obtain the project's deployment ID. If the project has already been deployed and indexed by other indexers, you may be able to find it listed in the SubQuery Explorer as the obtain the project’s Deployment ID. -The following projects are available for indexing on the Mainnet network. Some projects have a backup restore provided. - -| Network | Project Name | Deployment ID | Database Size | Dictionary Endpoint | Has Restore File | -| --------- | ----------------------- | ------------------------------------------------ | ------------- | ------------------------------------------------------------------------------ | ------------------------------ | -| Polkadot | Polkadot Dictionary | `QmZGAZQ7e1oZgfuK4V29Fa5gveYK3G2zEwvUzTZKNvSBsm` | ~220GB | N/A | [Yes](./dictionary-restore.md) | -| Kusama | Kusama Dictionary | `QmXwfCF8858YY924VHgNLsxRQfBLosVbB31mygRLhgJbWn` | ~260GB | N/A | [Yes](./dictionary-restore.md) | -| Moonbeam | Moonbeam Dictionary | `QmeeqBHdVu7iYnhVE9ZiYEKTWe4jXVUD5pVoGXT6LbCP2t` | ~123GB | N/A | [Yes](./dictionary-restore.md) | -| Moonriver | Moonriver Dictionary | `QmXCr6uZFdY1YcGTa4u6ZieQPXK4VHE1Pjy7CBr7ubFwKR` | ~130GB | N/A | [Yes](./dictionary-restore.md) | -| Moonbase | Moonbase Alpha Dictionary | `QmUgn2eP1nvAECSe9HE9zTHTHwkQMDwSN7rpB1aXcsthfe` | N/A | N/A | N/A | -| Acala | Acala Dictionary | `Qmarrhgrpqw5VK71rMtb4GARpPvq8ajMjAqnjnWZFLV61N` | ~10G | N/A | [Yes](./dictionary-restore.md) | -| Karura | Karura Dictionary | `QmWumrabg4k6t4EUMhQg19xWwcxGq1hWbcmfmRYiy2Bod5` | ~10G | N/A | [Yes](./dictionary-restore.md) | -| Astar | Astar Dictionary | `QmUmnKPhcE6JwGMYvY3Yitb5j8qxbQBMxgpkHpVQuXqxDH` | ~65G | N/A | [Yes](./dictionary-restore.md) | -| Shiden | Shiden Dictionary | `QmV25WVPgdmAgRCqkbGUU49xdeg9td3CK5LbtBjeQEMxTW` | N/A | N/A | N/A | -| Khala | Khala Dictionary | `QmYCAns2cunZKJFU85KNK8CvL2ATAmCFVZRdBf963GqWYs` | ~78G | N/A | [Yes](./dictionary-restore.md) | -| Nodle | Nodle Parachain Dictionary | `QmQtmsHoJEYUcxKE4tBqr9Z8kudcgkczQPfhkAxVExQX5y` | ~15G | N/A | [Yes](./dictionary-restore.md) | -| Westend | Westend Dictionary | `Qma6BeSQGHrhP5aydmkQcJCR25TEwuNMogS5boovBBwoeW` | ~35G | N/A | [Yes](./dictionary-restore.md) | -| Algorand | Algorand Dictionary | `QmYNRtrcD2QKftkff2UpjV3fr3ubPZuYahTNDAct4Ad2NW` | N/A | N/A | N/A | -| Asset Hub | Asset Hub Dictionary | `QmckGGY1AhrB75MwPPzR9orgWjwDVF4kXfwkZehZSZxmdE` | N/A | N/A | N/A | -| Aleph Zero | Aleph Zero Dictionary | `QmXp3MdCjZyUsmXhFXJTisxQiP1P96sm81WGmu2ew7v8WN` | N/A | N/A | N/A | -| Kilt Spiritnet | Kilt Spiritnet Dictionary | `Qmc9svij5SxCEGApMZzV9MwWgy8TuMTtGgsrWxR1yaUqZ9` | N/A | N/A | N/A | -| Fetch AI | Fetch AI Dictionary | `QmbtSt8USCUTBWeAqevN1AwmUhKzqmtvhSdFYHYA1BviC8` | N/A | N/A | N/A | -| Bifrost | Bifrost (Kusama) Dictionary | `QmUWd1o3BJb5qSR1ZaAhSw1duVgQ5bsczdfRNakNUL5cJy` | N/A | N/A | N/A | -| Calamari | Calamari Dictionary | `QmUpvkmvTRkMDCGDXnAVjCBZLzZEv9UCVKHH2s3gj3hYQK` | N/A | N/A | N/A | -| Near | Near Dictionary | `QmSKrk3BpzjWzKfS8sZRS5vyjmtXvkJnK8nHUVBhiCmz41` | N/A | N/A | N/A | -| Juno | Juno Dictionary | `QmPjq55mgUt9S8S491Q3wEbb87fXyEkdxymT6Gwe2xe1Z1` | N/A | N/A | N/A | -| SQ | SQ Mainnet Project | `QmTMphPvRg143xWWKsdgZZgNc1HgDxH9fK9ZYpzLiQPet1` | N/A | Yes | N/A | -| Nova | Nova wallet - Polkadot | `QmaTy1aG5uZfeyUXRu8bDci1P6AzbTYBEzM57yEYk3MPEt` | N/A | Yes | N/A | -| Nova | Nova Wallet - Kusama | `QmWS4bvLU9Y1YBkrcDBq3Z7enZf8LykeyjSvgVKjB7FSVz` | N/A | Yes | N/A | -| Nova | Nova Wallet - Moonriver | `QmPzEH1Juo7RQB2X37DvYATQdCQ7oBV8V1yX92DHD71ma5` | N/A | Yes | N/A | -| Nova | Nova Wallet - Moonbeam | `QmegTE8BimTw5iTpBqtJSMC2jWApU4g2q5ojGZAL3iU1fr` | N/A | Yes | N/A | -| Nova | Nova Wallet - Astar | `QmRonzFGNrsmpG2NrVhcVC6rCtCBqYFupX6MEECReWXWZT` | N/A | Yes | N/A | -| Nova | Nova Wallet - Aleph Zero | `QmZNNYQqBs3c9f3t7UUybt1Unr49F5gTEdvU3Byv6DntKo` | N/A | Yes | N/A | -| Nova | Nova Wallet - Polkadex | `QmdzL852vGNgmdmk4UdvpPWeVsTtYJiMsjJd6ZxnSbQsfP` | N/A | Yes | N/A | -| RPC | Eth Mainnet Rpc - Archive Node | `Qmf6uZkxuNzpcNvnhReXrz1BTzMWgmtkdFQrSNByPytkuk` | N/A | N/A | N/A | -| RPC | Eth Mainnet Rpc - Full Node | `QmNa36oZ4zRS1i2wQhiFznU5DjEuNP3wopV6U3VcUWMUKu` | N/A | N/A | N/A | -| RPC | Base Rpc - Archive Node | `QmTfhYrb3wusYS715KvHfaL56R8M1SrM8vwhuLyYVehfKB` | N/A | N/A | N/A | -| RPC | Base Rpc - Full Node | `QmbReTnhCweQHmbXxgffkDqkkedo7ojjsUWTKopP1auuTp` | N/A | N/A | N/A | - -::: info Note -If you are wanting to index a Dictionary, then you may be able to restore your project from our dictionary snapshots to save a lot of time. Instructions are [here](./dictionary-restore.md) -::: - Enter the project deployment ID when adding the new project. ![Add a New Project Using Deploy ID](/assets/img/network/indexer_project_add.png) diff --git a/docs/subquery_network/node_operators/setup/becoming-a-node-operator.md b/docs/subquery_network/node_operators/setup/becoming-a-node-operator.md index 79f153c058b..1e7868fe9e0 100644 --- a/docs/subquery_network/node_operators/setup/becoming-a-node-operator.md +++ b/docs/subquery_network/node_operators/setup/becoming-a-node-operator.md @@ -96,10 +96,6 @@ Once your Indexing Service is all set and running successfully, you should open To index a data indexing project, please follow the instructions [here](../indexers/index-project.md). -::: info Note -If you are wanting to index a Dictionary, then you may be able to restore your project from our dictionary snapshots to save a lot of time. Instructions are [here](../indexers/dictionary-restore.md) -::: - Guides to setup RPC endpoints will come shortly. ## 5. Create a Plan from a Plan Template