feat(indexer): introduce indexer (#191)

Co-authored-by: sergey filyanin <[email protected]>

.github/workflows/indexer-ci.yml

@@ -0,0 +1,39 @@
+name: CI indexer
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - indexer/**
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Node.js 20.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.x
+
+      - name: Install dependencies
+        working-directory: indexer
+        run: npm install
+
+      - name: Copy api files
+        run: |
+          mkdir -p indexer/assets
+          node indexer/scripts/abi.js api/ethereum/IERC20Manager.json indexer/assets
+          node indexer/scripts/abi.js api/ethereum/IMessageQueue.json indexer/assets
+
+      - name: Build
+        working-directory: indexer
+        run: npm run build

.gitignore

@@ -13,3 +13,8 @@ out/
 ethereum/broadcast
 
 .env
+.DS_Store
+
+node_modules/
+indexer/lib/
+indexer/assets/

.dockerignore → Dockerfile.dockerignore

@@ -1,19 +1,19 @@
 # Rust build directory
 target
 
-# Resource files 
+# Resource files
 audits
 images
 LICENSE
 *.md
 api
 
-# Git 
+# Git
 .git
 .gitignore
 .github
 
-# IDE-generated files 
+# IDE-generated files
 .vscode
 .idea
 
@@ -28,5 +28,8 @@ ethereum/cache
 ethereum/out
 ethereum/broadcast
 
+# Indexer-related
+indexer/
+
 # Frontend
-frontend
+frontend/

indexer/.dockerignore

@@ -0,0 +1,8 @@
+/.git
+/node_modules
+/lib
+/*Versions.json
+npm-debug.log
+
+# OS Files
+.DS_Store

indexer/.squidignore

@@ -0,0 +1,7 @@
+node_modules
+builds
+lib
+Dockerfile
+.git
+.github
+.idea

indexer/Dockerfile

@@ -0,0 +1,46 @@
+FROM node:20-alpine AS node
+
+FROM node AS api
+WORKDIR /assets
+COPY api/gear/vft_manager.idl .
+COPY api/gear/erc20_relay.idl .
+COPY api/ethereum/IERC20Manager.json .
+COPY api/ethereum/IMessageQueue.json .
+COPY indexer/scripts/abi.js .
+RUN node abi.js IERC20Manager.json .
+RUN node abi.js IMessageQueue.json .
+
+FROM node AS node-with-gyp
+RUN apk add g++ make python3
+FROM node-with-gyp AS builder
+WORKDIR /squid
+COPY indexer/package.json .
+COPY indexer/package-lock.json .
+COPY indexer/db db
+COPY indexer/schema.graphql .
+RUN npm i -g @subsquid/cli && npm ci
+COPY indexer/tsconfig.json .
+COPY indexer/src src
+COPY --from=api /assets assets
+RUN npm run build
+RUN ls -ltr
+
+FROM node-with-gyp AS deps
+WORKDIR /squid
+COPY indexer/package.json .
+COPY indexer/package-lock.json .
+RUN npm ci --production
+
+FROM node AS squid
+WORKDIR /squid
+COPY --from=deps /squid/package.json .
+COPY --from=deps /squid/package-lock.json .
+COPY --from=deps /squid/node_modules node_modules
+COPY --from=builder /squid/lib lib
+COPY --from=builder /squid/db db
+COPY --from=builder /squid/schema.graphql schema.graphql
+COPY --from=api /assets assets
+COPY indexer/commands.json .
+RUN echo -e "loglevel=silent\\nupdate-notifier=false" >/squid/.npmrc
+RUN npm i -g @subsquid/commands && mv $(which squid-commands) /usr/local/bin/sqd
+ENV PROCESSOR_PROMETHEUS_PORT=3000

indexer/commands.json

@@ -0,0 +1,111 @@
+{
+  "$schema": "https://cdn.subsquid.io/schemas/commands.json",
+  "commands": {
+    "clean": {
+      "description": "delete all build artifacts",
+      "cmd": ["npx", "--yes", "rimraf", "lib"]
+    },
+    "build": {
+      "description": "Build the squid project",
+      "deps": ["clean"],
+      "cmd": ["tsc"]
+    },
+    "up": {
+      "description": "Start a PG database",
+      "cmd": ["docker", "compose", "up", "-d"]
+    },
+    "down": {
+      "description": "Drop a PG database",
+      "cmd": ["docker", "compose", "down"]
+    },
+    "migration:apply": {
+      "description": "Apply the DB migrations",
+      "cmd": ["squid-typeorm-migration", "apply"]
+    },
+    "migration:generate": {
+      "description": "Generate a DB migration matching the TypeORM entities",
+      "deps": ["build", "migration:clean"],
+      "cmd": ["squid-typeorm-migration", "generate"]
+    },
+    "migration:clean": {
+      "description": "Clean the migrations folder",
+      "cmd": ["npx", "--yes", "rimraf", "./db/migrations"]
+    },
+    "migration": {
+      "deps": ["build"],
+      "cmd": ["squid-typeorm-migration", "generate"],
+      "hidden": true
+    },
+    "codegen": {
+      "description": "Generate TypeORM entities from the schema file",
+      "cmd": ["squid-typeorm-codegen"]
+    },
+    "typegen": {
+      "description": "Generate data access classes for an ABI file(s) in the ./abi folder",
+      "cmd": [
+        "squid-evm-typegen",
+        "./src/abi",
+        { "glob": "./abi/*.json" },
+        "--multicall"
+      ]
+    },
+    "process:eth": {
+      "description": "Load .env and start the ETH squid processor",
+      "deps": ["build", "migration:apply"],
+      "cmd": ["node", "--require=dotenv/config", "lib/eth/main.js"]
+    },
+    "process:gear": {
+      "description": "Load .env and start the gear squid processor",
+      "deps": ["build", "migration:apply"],
+      "cmd": ["node", "--require=dotenv/config", "lib/gear/main.js"]
+    },
+    "process:prod:eth": {
+      "description": "Start the squid processor",
+      "deps": ["migration:apply"],
+      "cmd": ["node", "lib/eth/main.js"],
+      "hidden": true
+    },
+    "process:prod:gear": {
+      "description": "Start the squid processor",
+      "cmd": ["node", "lib/gear/main.js"],
+      "hidden": true
+    },
+    "serve": {
+      "description": "Start the GraphQL API server",
+      "cmd": ["squid-graphql-server"]
+    },
+    "serve:prod": {
+      "description": "Start the GraphQL API server with caching and limits",
+      "cmd": [
+        "squid-graphql-server",
+        "--dumb-cache",
+        "in-memory",
+        "--dumb-cache-ttl",
+        "1000",
+        "--dumb-cache-size",
+        "100",
+        "--dumb-cache-max-age",
+        "1000"
+      ]
+    },
+    "check-updates": {
+      "cmd": [
+        "npx",
+        "--yes",
+        "npm-check-updates",
+        "--filter=/subsquid/",
+        "--upgrade"
+      ],
+      "hidden": true
+    },
+    "bump": {
+      "description": "Bump @subsquid packages to the latest versions",
+      "deps": ["check-updates"],
+      "cmd": ["npm", "i", "-f"]
+    },
+    "open": {
+      "description": "Open a local browser window",
+      "cmd": ["npx", "--yes", "opener"]
+    }
+  }
+}

indexer/db/migrations/1730394854267-Data.js

@@ -0,0 +1,29 @@
+module.exports = class Data1730394854267 {
+    name = 'Data1730394854267'
+
+    async up(db) {
+        await db.query(`CREATE TABLE "transfer" ("id" character varying NOT NULL, "tx_hash" text NOT NULL, "block_number" text NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "nonce" text NOT NULL, "source_network" character varying(8) NOT NULL, "source" text NOT NULL, "dest_network" character varying(8) NOT NULL, "destination" text NOT NULL, "status" character varying(10) NOT NULL, "sender" text NOT NULL, "receiver" text NOT NULL, "amount" numeric NOT NULL, CONSTRAINT "PK_fd9ddbdd49a17afcbe014401295" PRIMARY KEY ("id"))`)
+        await db.query(`CREATE INDEX "IDX_70ff8b624c3118ac3a4862d22c" ON "transfer" ("timestamp") `)
+        await db.query(`CREATE INDEX "IDX_5662ca6334321160c607988dc2" ON "transfer" ("nonce") `)
+        await db.query(`CREATE INDEX "IDX_1aa446c2e82f2abbb358ab5248" ON "transfer" ("source") `)
+        await db.query(`CREATE INDEX "IDX_329c2ee277e5c977d4c5fbb22f" ON "transfer" ("destination") `)
+        await db.query(`CREATE INDEX "IDX_9a4ceb5c3899b95c695eb5b112" ON "transfer" ("sender") `)
+        await db.query(`CREATE INDEX "IDX_e95f070ab35073a24097069e6d" ON "transfer" ("receiver") `)
+        await db.query(`CREATE TABLE "pair" ("id" character varying NOT NULL, "gear_token" text NOT NULL, "eth_token" text NOT NULL, CONSTRAINT "PK_3eaf216329c5c50aedb94fa797e" PRIMARY KEY ("id"))`)
+        await db.query(`CREATE TABLE "completed_transfer" ("id" character varying NOT NULL, "nonce" text NOT NULL, "dest_network" character varying(8) NOT NULL, CONSTRAINT "PK_c966d1eba60d5625faf13b457a4" PRIMARY KEY ("id"))`)
+        await db.query(`CREATE UNIQUE INDEX "IDX_ab14e0c37eabeb5ba0dc3f2f78" ON "completed_transfer" ("nonce") `)
+    }
+
+    async down(db) {
+        await db.query(`DROP TABLE "transfer"`)
+        await db.query(`DROP INDEX "public"."IDX_70ff8b624c3118ac3a4862d22c"`)
+        await db.query(`DROP INDEX "public"."IDX_5662ca6334321160c607988dc2"`)
+        await db.query(`DROP INDEX "public"."IDX_1aa446c2e82f2abbb358ab5248"`)
+        await db.query(`DROP INDEX "public"."IDX_329c2ee277e5c977d4c5fbb22f"`)
+        await db.query(`DROP INDEX "public"."IDX_9a4ceb5c3899b95c695eb5b112"`)
+        await db.query(`DROP INDEX "public"."IDX_e95f070ab35073a24097069e6d"`)
+        await db.query(`DROP TABLE "pair"`)
+        await db.query(`DROP TABLE "completed_transfer"`)
+        await db.query(`DROP INDEX "public"."IDX_ab14e0c37eabeb5ba0dc3f2f78"`)
+    }
+}