From d23ce0a9f1d1ec31fd5cf790fd5306d1884e1a33 Mon Sep 17 00:00:00 2001 From: Tian Qin Date: Thu, 3 Oct 2024 17:41:09 -0400 Subject: [PATCH] update add markets to send market map proposal first and validation flow to use v7.x commit --- .../workflows/validate-other-market-data.yml | 9 +- scripts/markets/add-markets.ts | 62 ++++-- scripts/markets/help.ts | 183 +++++++++++++++++- scripts/markets/validate-other-market-data.ts | 34 +++- 4 files changed, 259 insertions(+), 29 deletions(-) diff --git a/.github/workflows/validate-other-market-data.yml b/.github/workflows/validate-other-market-data.yml index 19f13b0dd..b034fc54f 100644 --- a/.github/workflows/validate-other-market-data.yml +++ b/.github/workflows/validate-other-market-data.yml @@ -4,7 +4,7 @@ on: pull_request: paths: - 'public/configs/otherMarketData.json' - - 'scripts/validate-other-market-data.ts' + - 'scripts/markets/validate-other-market-data.ts' jobs: validate: @@ -37,7 +37,7 @@ jobs: uses: actions/checkout@v3 with: repository: 'dydxprotocol/v4-chain' - ref: '725ad716cbe9e9aebf2472e877fe2a71e8a63d87' + ref: '18c72e1b2d9bf9849c1062ac022caba4fa9ecf60' path: 'v4-chain' - name: Start v4 localnet @@ -49,11 +49,6 @@ jobs: echo "Starting localnet..." DOCKER_BUILDKIT=1 make localnet-init DOCKER_BUILDKIT=1 make localnet-compose-upd -e RAYDIUM_URL=${{ secrets.RAYDIUM_URL }} - - - name: Get diff of otherMarketData.json - run: | - git fetch origin - git diff remotes/origin/main -- public/configs/otherMarketData.json > otherMarketDiff.txt - name: Checkout main branch uses: actions/checkout@v3 diff --git a/scripts/markets/add-markets.ts b/scripts/markets/add-markets.ts index b1aafa681..846557fad 100644 --- a/scripts/markets/add-markets.ts +++ b/scripts/markets/add-markets.ts @@ -4,22 +4,27 @@ This script adds markets to a dYdX chain. Markets are read from public/config/ot Supported environments: local, dev, dev2, dev3, dev4, dev5, staging. Usage: - $ pnpx tsx scripts/markets/add-markets.ts + $ pnpx tsx scripts/markets/add-markets.ts Example (add 10 markets on staging): - $ pnpx tsx scripts/markets/add-markets.ts staging 10 + $ pnpx tsx scripts/markets/add-markets.ts staging 10 /Users/alice/v4-chain/protocol/build/dydxprotocold */ import { CompositeClient, IndexerConfig, LocalWallet as LocalWalletType, Network, - ValidatorConfig, + ValidatorConfig } from '@dydxprotocol/v4-client-js'; -import { PerpetualMarketType } from '@dydxprotocol/v4-proto/src/codegen/dydxprotocol/perpetuals/perpetual'; import { readFileSync } from 'fs'; import Long from 'long'; - -import { Proposal, retry, sleep, voteOnProposals } from './help'; +import { + createAndSendMarketMapProposal, + PerpetualMarketType, + Proposal, + retry, + sleep, + voteOnProposals +} from './help'; const LocalWalletModule = await import( '@dydxprotocol/v4-client-js/src/clients/modules/local-wallet' @@ -132,7 +137,12 @@ const ENV_CONFIG = { }, }; -async function addMarkets(env: Env, numMarkets: number, proposals: Proposal[]): Promise { +async function addMarkets( + env: Env, + numMarkets: number, + proposals: Proposal[], + binary: string +): Promise { // Initialize client and wallets. const config = ENV_CONFIG[env]; const indexerConfig = new IndexerConfig(config.indexerRestEndpoint, config.indexerWsEndpoint); @@ -150,6 +160,7 @@ async function addMarkets(env: Env, numMarkets: number, proposals: Proposal[]): 'Client Example' ); const network = new Network(env, indexerConfig, validatorConfig); + const sleepMsBtwTxs = 3.5 * config.blockTimeSeconds * 1000; const client = await CompositeClient.connect(network); const wallets: LocalWalletType[] = await Promise.all( @@ -158,16 +169,37 @@ async function addMarkets(env: Env, numMarkets: number, proposals: Proposal[]): }) ); - // Send proposals to add all markets (skip markets that already exist). + // Filter out markets that already exist on-chain. const allPerps = await client.validatorClient.get.getAllPerpetuals(); const allTickers = allPerps.perpetual.map((perp) => perp.params!.ticker); const filteredProposals = proposals.filter( (proposal) => !allTickers.includes(proposal.params.ticker) ); - console.log(`Adding ${numMarkets} new markets to ${env}...`); + // Add markets to market map first. + console.log("Submitting market map proposal..."); + await createAndSendMarketMapProposal( + filteredProposals.slice(0, numMarkets), + config.validatorEndpoint, + config.chainId, + binary, + ); + await sleep(sleepMsBtwTxs); + console.log("Submitted market map proposal"); + + // Get latest gov proposal ID. + const allProposalsResp = await client.validatorClient.get.getAllGovProposals(); + let latestProposalId = allProposalsResp.proposals.reduce( + (max, proposal) => (proposal.id.toNumber() > max ? proposal.id.toNumber() : max), + 0 + ); + + for (const wallet of wallets) { + retry(() => voteOnProposals([latestProposalId], client, wallet)); + } + console.log(`Voted on market map proposal with id ${latestProposalId}`); + await sleep(sleepMsBtwTxs); - const sleepMsBtwTxs = 3.5 * config.blockTimeSeconds * 1000; let numProposalsToSend = Math.min(numMarkets, filteredProposals.length); let numProposalsSent = 0; const numExistingMarkets = allPerps.perpetual.reduce( @@ -184,8 +216,7 @@ async function addMarkets(env: Env, numMarkets: number, proposals: Proposal[]): break; } const proposal = proposalsToSend[j]; - const proposalId: number = i + j + 1; - const marketId: number = numExistingMarkets + proposalId; + const marketId: number = numExistingMarkets + numProposalsSent + 1; // Send proposal. const exchangeConfigString = `{"exchanges":${JSON.stringify( @@ -220,7 +251,7 @@ async function addMarkets(env: Env, numMarkets: number, proposals: Proposal[]): console.log(`Proposed market ${marketId} with ticker ${proposal.params.ticker}`); // Record proposed market. - proposalIds.push(proposalId); + proposalIds.push(++latestProposalId); numProposalsSent++; } @@ -242,12 +273,15 @@ async function main(): Promise { const args = process.argv.slice(2); const env = args[0] as Env; const numMarkets = parseInt(args[1], 10); + const binary = args[2]; // Validate inputs. if (!Object.values(Env).includes(env)) { throw new Error(`Invalid environment: ${env}`); } else if (isNaN(numMarkets) || numMarkets <= 0) { throw new Error(`Invalid number of markets: ${numMarkets}`); + } else if (!binary) { + throw new Error(`dydx binary path not provided`); } // Read proposals. @@ -256,7 +290,7 @@ async function main(): Promise { ); // Add markets. - await addMarkets(env, numMarkets, Object.values(proposals)); + await addMarkets(env, numMarkets, Object.values(proposals), binary); } main() diff --git a/scripts/markets/help.ts b/scripts/markets/help.ts index 6a1104cb5..da1428469 100644 --- a/scripts/markets/help.ts +++ b/scripts/markets/help.ts @@ -9,8 +9,11 @@ import { VoteOption, } from '@dydxprotocol/v4-client-js'; import { MsgVote } from '@dydxprotocol/v4-proto/src/codegen/cosmos/gov/v1/tx'; +import { spawn } from 'child_process'; +import * as fs from 'fs'; import Long from 'long'; + const VOTE_FEE: StdFee = { amount: [ { @@ -25,6 +28,7 @@ export interface Exchange { exchangeName: ExchangeName; ticker: string; adjustByMarket?: string; + invert?: boolean; } export enum ExchangeName { @@ -68,6 +72,61 @@ export interface Proposal { params: Params; } + +export enum PerpetualMarketType { + /** PERPETUAL_MARKET_TYPE_UNSPECIFIED - Unspecified market type. */ + PERPETUAL_MARKET_TYPE_UNSPECIFIED = 0, + + /** PERPETUAL_MARKET_TYPE_CROSS - Market type for cross margin perpetual markets. */ + PERPETUAL_MARKET_TYPE_CROSS = 1, + + /** PERPETUAL_MARKET_TYPE_ISOLATED - Market type for isolated margin perpetual markets. */ + PERPETUAL_MARKET_TYPE_ISOLATED = 2, + UNRECOGNIZED = -1, +} + +interface Market { + ticker: { + currency_pair: { + Base: string; + Quote: string; + }; + decimals: string; + enabled: boolean; + min_provider_count: string; + metadata_JSON: string; + }; + provider_configs: ProviderConfig[]; +} + +interface ProviderConfig { + name: string; + normalize_by_pair: { + Base: string; + Quote: string; + } | null; + off_chain_ticker: string; + invert: boolean; + metadata_JSON: string; +} + +const exchangeNameToMarketMapProviderName: Record = { + [ExchangeName.Binance]: 'binance_ws', + [ExchangeName.BinanceUS]: 'binance_ws', + [ExchangeName.Bitfinex]: 'bitfinex_ws', + [ExchangeName.Bitstamp]: 'bitstamp_api', + [ExchangeName.Bybit]: 'bybit_ws', + [ExchangeName.CoinbasePro]: 'coinbase_ws', + [ExchangeName.CryptoCom]: 'crypto_dot_com_ws', + [ExchangeName.Gate]: 'gate_ws', + [ExchangeName.Huobi]: 'huobi_ws', + [ExchangeName.Kraken]: 'kraken_api', + [ExchangeName.Kucoin]: 'kucoin_ws', + [ExchangeName.Mexc]: 'mexc_ws', + [ExchangeName.Okx]: 'okx_ws', + [ExchangeName.Raydium]: 'raydium_api', +}; + export async function sleep(ms: number): Promise { return new Promise((resolve) => { setTimeout(resolve, ms); @@ -130,4 +189,126 @@ export async function voteOnProposals( } else { console.log(`Voted on proposals ${proposalIds} with wallet ${wallet.address}`); } -} \ No newline at end of file +} + +export async function createAndSendMarketMapProposal( + proposals: Proposal[], + validatorEndpoint: string, + chainId: string, + binary: string, +) { + const markets: Market[] = proposals.map((proposal) => { + const { ticker, priceExponent, minExchanges, exchangeConfigJson } = proposal.params; + + const providerConfigs: ProviderConfig[] = exchangeConfigJson.map((config) => { + let normalize_by_pair: { Base: string; Quote: string } | null = null; + + if (config.adjustByMarket) { + const [Base, Quote] = config.adjustByMarket.split('-'); + normalize_by_pair = { Base, Quote }; + } + + return { + name: `${exchangeNameToMarketMapProviderName[config.exchangeName as ExchangeName]}`, + normalize_by_pair, + off_chain_ticker: config.ticker, + invert: config.invert || false, + metadata_JSON: '', + }; + }); + + return { + ticker: { + currency_pair: { + Base: ticker.split('-')[0], + Quote: ticker.split('-')[1], + }, + decimals: Math.abs(priceExponent).toString(), + enabled: true, + min_provider_count: minExchanges.toString(), + metadata_JSON: '', + }, + provider_configs: providerConfigs, + }; + }); + + const proposal = { + "title": "Add markets to market map", + "summary":"Add markets to market map", + "messages": [ + { + "@type": "/slinky.marketmap.v1.MsgCreateMarkets", + "authority": "dydx10d07y265gmmuvt4z0w9aw880jnsr700jnmapky", + "create_markets": markets, + }, + ], + "deposit":"5000000000000000000adv4tnt", + "expedited": true, + }; + + const proposalFile = 'marketMapProposal.json'; + fs.writeFileSync(proposalFile, JSON.stringify(proposal, null, 2), 'utf-8'); + + try { + await execCLI( + binary, + ['keys', 'show', 'alice'], + ) + } catch (error) { + await execCLI( + binary, + ['keys', 'add', 'tom', '--recover'], + 'merge panther lobster crazy road hollow amused security before critic about cliff exhibit cause coyote talent happy where lion river tobacco option coconut small', + ) + } + + await execCLI( + binary, + [ + '--node', validatorEndpoint, + 'tx', 'gov', 'submit-proposal', 'marketMapProposal.json', + '--from', 'alice', + '--fees', '300000000000000000adv4tnt', + '--chain-id', chainId, + '--gas', 'auto' + ], + 'y', + ) + + fs.unlinkSync(proposalFile); +} + +export function execCLI( + command: string, + args?: string[], + input?: string, +): Promise { + return new Promise((resolve, reject) => { + const process = spawn(command, args); + + let output = ''; + + process.stdout.on('data', (data) => { + console.log(`stdout: ${data}`); + output += data.toString(); + }); + + process.stderr.on('data', (data) => { + console.error(`stderr: ${data}`); + output += data.toString(); + }); + + process.on('close', (code) => { + if (code === 0) { + resolve(output); + } else { + reject(`Process exited with code: ${code}`); + } + }); + + if (input) { + process.stdin.write(`${input}\n`); + } + process.stdin.end(); + }); +} diff --git a/scripts/markets/validate-other-market-data.ts b/scripts/markets/validate-other-market-data.ts index def279e6a..07020d31f 100644 --- a/scripts/markets/validate-other-market-data.ts +++ b/scripts/markets/validate-other-market-data.ts @@ -15,10 +15,7 @@ import { ProposalStatus, } from '@dydxprotocol/v4-client-js'; import { ClobPair } from '@dydxprotocol/v4-proto/src/codegen/dydxprotocol/clob/clob_pair'; -import { - Perpetual, - PerpetualMarketType, -} from '@dydxprotocol/v4-proto/src/codegen/dydxprotocol/perpetuals/perpetual'; +import { Perpetual } from '@dydxprotocol/v4-proto/src/codegen/dydxprotocol/perpetuals/perpetual'; import { MarketPrice } from '@dydxprotocol/v4-proto/src/codegen/dydxprotocol/prices/market_price'; import Ajv from 'ajv'; import axios from 'axios'; @@ -26,7 +23,16 @@ import { readFileSync } from 'fs'; import Long from 'long'; import { PrometheusDriver } from 'prometheus-query'; -import { Exchange, ExchangeName, Proposal, retry, sleep, voteOnProposals } from './help'; +import { + createAndSendMarketMapProposal, + Exchange, + ExchangeName, + PerpetualMarketType, + Proposal, + retry, + sleep, + voteOnProposals, +} from './help'; const LocalWalletModule = await import( '@dydxprotocol/v4-client-js/src/clients/modules/local-wallet' @@ -318,12 +324,27 @@ async function validateAgainstLocalnet(proposals: Proposal[]): Promise { (proposal) => !allTickers.includes(proposal.params.ticker) ); + // Send market map proposal first. + await createAndSendMarketMapProposal( + filteredProposals, + network.validatorConfig.restEndpoint, + network.validatorConfig.chainId, + 'v4-chain/protocol/build/dydxprotocold', + ); + console.log("Submitted market map proposal"); + await sleep(5000); + for (const wallet of wallets) { + retry(() => voteOnProposals([1], client, wallet)); + } + await sleep(5000); + const numExistingMarkets = allPerps.perpetual.reduce( (max, perp) => (perp.params!.id > max ? perp.params!.id : max), 0 ); const marketsProposed = new Map(); // marketId -> Proposal + let proposalId: number = 2; for (let i = 0; i < filteredProposals.length; i += 4) { // Send out proposals in groups of 4 or fewer. const proposalsToSend = filteredProposals.slice(i, i + 4); @@ -331,7 +352,6 @@ async function validateAgainstLocalnet(proposals: Proposal[]): Promise { for (let j = 0; j < proposalsToSend.length; j++) { // Use wallets[j] to send out proposalsToSend[j] const proposal = proposalsToSend[j]; - const proposalId: number = i + j + 1; const marketId: number = numExistingMarkets + proposalId; // Send proposal. @@ -372,7 +392,7 @@ async function validateAgainstLocalnet(proposals: Proposal[]): Promise { // Record proposed market. marketsProposed.set(marketId, { ...proposal, id: Long.fromNumber(proposalId) }); - proposalIds.push(proposalId); + proposalIds.push(proposalId++); } // Wait 5 seconds for proposals to be processed.