feat: compute registry data

package.json

@@ -16,7 +16,10 @@
   },
   "prettier": "@snapshot-labs/prettier-config",
   "dependencies": {
+    "@apollo/client": "^3.11.4",
+    "@ethersproject/units": "^5.7.0",
     "@snapshot-labs/checkpoint": "^0.1.0-beta.39",
+    "@snapshot-labs/snapshot.js": "^0.12.7",
     "@types/node": "^18.11.6",
     "async-mutex": "^0.5.0",
     "cors": "^2.8.5",

src/compute.ts

@@ -1,10 +1,87 @@
-// @ts-ignore
+import { formatUnits } from '@ethersproject/units';
 import { register } from '@snapshot-labs/checkpoint/dist/src/register';
+import snapshotjs from '@snapshot-labs/snapshot.js';
 import { Mutex } from 'async-mutex';
-import { Governance } from '../.checkpoint/models';
+import {
+  NETWORK_COMPUTE_DELAY_SECONDS,
+  SCORE_API_URL,
+  SPACE_COMPUTE_DELAY_SECONDS
+} from './constants';
+import { getSpace, Space } from './hub';
+import { Delegate, Governance } from '../.checkpoint/models';
 
+type NetworkCache = {
+  timestamp: number;
+  data: Awaited<ReturnType<typeof snapshotjs.utils.getDelegatesBySpace>>;
+};
+
+const DECIMALS = 18;
+const DELEGATION_STRATEGIES = [
+  'delegation',
+  'erc20-balance-of-delegation',
+  'delegation-with-cap',
+  'delegation-with-overrides'
+];
+
+const networkDelegationsCache = new Map<string, NetworkCache>();
+const lastSpaceCompute = new Map<string, number>();
 const mutex = new Mutex();
 
+async function getNetworkDelegations(network: string) {
+  const cache = networkDelegationsCache.get(network);
+  const now = Math.floor(Date.now() / 1000);
+
+  if (cache && now - cache.timestamp < NETWORK_COMPUTE_DELAY_SECONDS) {
+    return cache.data;
+  }
+
+  const delegations = await snapshotjs.utils.getDelegatesBySpace(
+    network,
+    null,
+    'latest'
+  );
+
+  networkDelegationsCache.set(network, {
+    timestamp: now,
+    data: delegations
+  });
+
+  return delegations;
+}
+
+async function getScores(
+  network: string,
+  governance: string,
+  strategies: Space['strategies'],
+  delegatesAddresses: string[]
+): Promise<Record<string, number>> {
+  const chunks = delegatesAddresses.reduce((acc, address, i) => {
+    const chunkIndex = Math.floor(i / 1000);
+    if (!acc[chunkIndex]) acc[chunkIndex] = [];
+    acc[chunkIndex].push(address);
+    return acc;
+  }, [] as string[][]);
+
+  let scores: Record<string, number> = {};
+  for (const chunk of chunks) {
+    const result = await snapshotjs.utils.getScores(
+      governance,
+      strategies,
+      network,
+      chunk,
+      'latest',
+      `${SCORE_API_URL}/api/scores`
+    );
+
+    scores = {
+      ...scores,
+      ...result[0]
+    };
+  }
+
+  return scores;
+}
+
 // This function is called everytime governance information is queried from GraphQL API.
 // It receives array of governances that we want to update information about.
 //
@@ -18,16 +95,86 @@ export async function compute(governances: string[]) {
   try {
     register.setCurrentBlock(register.getCurrentBlock() + 1n);
 
-    console.log('compute governances for', governances);
-
     for (const governance of governances) {
+      console.log('computing', governance);
+
+      const computedAt = lastSpaceCompute.get(governance) ?? 0;
+      const now = Math.floor(Date.now() / 1000);
+      if (now - computedAt < SPACE_COMPUTE_DELAY_SECONDS) {
+        console.log('ignoring because of recent compute');
+        continue;
+      }
+      lastSpaceCompute.set(governance, now);
+
+      const space = await getSpace(governance);
+      const delegations = await getNetworkDelegations(space.network);
+
+      const delegatorCounter = {};
+      for (const delegation of delegations) {
+        if (!delegatorCounter[delegation.delegate]) {
+          delegatorCounter[delegation.delegate] = 0;
+        }
+
+        delegatorCounter[delegation.delegate] += 1;
+      }
+
+      const delegationsMap = Object.fromEntries(
+        delegations.map(d => [d.delegate, d])
+      );
+      const delegatesAddresses = Object.keys(delegationsMap);
+      const uniqueDelegates = Object.values(delegationsMap);
+
+      const strategies = space.strategies.filter(strategy =>
+        DELEGATION_STRATEGIES.includes(strategy.name)
+      );
+
+      const scores = await getScores(
+        space.network,
+        governance,
+        strategies,
+        delegatesAddresses
+      );
+
+      const delegates = uniqueDelegates.map(delegate => ({
+        ...delegate,
+        score: BigInt(
+          Math.floor((scores[delegate.delegate] ?? 0) * 10 ** DECIMALS)
+        )
+      }));
+
+      const sortedDelegates = delegates
+        .filter(delegate => delegate.score > 0n)
+        .sort((a, b) => (b.score > a.score ? 1 : -1));
+
+      const totalVotes = sortedDelegates.reduce(
+        (acc, delegate) => acc + delegate.score,
+        0n
+      );
+
       const governanceEntity = new Governance(governance);
-      governanceEntity.currentDelegates = 0;
-      governanceEntity.totalDelegates = 0;
-      governanceEntity.delegatedVotesRaw = '0';
-      governanceEntity.delegatedVotes = '0';
+      governanceEntity.currentDelegates = sortedDelegates.length;
+      governanceEntity.totalDelegates = sortedDelegates.length;
+      governanceEntity.delegatedVotesRaw = totalVotes.toString();
+      governanceEntity.delegatedVotes = formatUnits(totalVotes, DECIMALS);
       await governanceEntity.save();
+
+      for (const delegate of sortedDelegates) {
+        const delegateEntity = new Delegate(
+          `${governance}/${delegate.delegate}`
+        );
+        delegateEntity.governance = governance;
+        delegateEntity.user = delegate.delegate;
+        delegateEntity.delegatedVotesRaw = delegate.score.toString();
+        delegateEntity.delegatedVotes = formatUnits(delegate.score, DECIMALS);
+        delegateEntity.tokenHoldersRepresentedAmount =
+          delegatorCounter[delegate.delegate];
+        await delegateEntity.save();
+      }
+
+      console.log('finished compute', governance);
     }
+  } catch (e) {
+    console.error('compute error', e);
   } finally {
     release();
   }

src/constants.ts

@@ -0,0 +1,6 @@
+export const HUB_URL = process.env.HUB_URL ?? 'https://hub.snapshot.org';
+export const SCORE_API_URL =
+  process.env.SCORE_API_URL ?? 'https://score.snapshot.org';
+
+export const NETWORK_COMPUTE_DELAY_SECONDS = 24 * 60 * 60; // 24 hours
+export const SPACE_COMPUTE_DELAY_SECONDS = 24 * 60 * 60; // 24 hours

src/hub.ts

@@ -0,0 +1,40 @@
+import { ApolloClient, gql, InMemoryCache } from '@apollo/client/core';
+import { HUB_URL } from './constants';
+
+export const SPACE_QUERY = gql`
+  query Space($id: String!) {
+    space(id: $id) {
+      id
+      network
+      strategies {
+        name
+        network
+        params
+      }
+    }
+  }
+`;
+
+export type Space = {
+  id: string;
+  network: string;
+  strategies: {
+    name: string;
+    network: string;
+    params: any;
+  }[];
+};
+
+const client = new ApolloClient({
+  uri: `${HUB_URL}/graphql`,
+  cache: new InMemoryCache()
+});
+
+export async function getSpace(space: string): Promise<Space> {
+  const { data } = await client.query({
+    query: SPACE_QUERY,
+    variables: { id: space }
+  });
+
+  return data.space;
+}

src/index.ts

@@ -12,6 +12,10 @@ const dir = __dirname.endsWith('dist/src') ? '../' : '';
 const schemaFile = path.join(__dirname, `${dir}../src/schema.gql`);
 const schema = fs.readFileSync(schemaFile, 'utf8');
 
+if (process.env.CA_CERT) {
+  process.env.CA_CERT = process.env.CA_CERT.replace(/\\n/g, '\n');
+}
+
 const indexer = new NoopIndexer();
 const checkpoint = new Checkpoint(config, indexer, schema, {
   logLevel: LogLevel.Info,