From faae0e2b385520414433def08e88bbe8a78b4a11 Mon Sep 17 00:00:00 2001 From: anonymoussprocket Date: Sat, 20 Mar 2021 03:18:02 -0400 Subject: [PATCH 1/4] - add throttling - add and disable profiling --- conseilUtil.js | 20 ++++++++++++-- package.json | 74 ++++++++++++++++++++++++++------------------------ 2 files changed, 56 insertions(+), 38 deletions(-) diff --git a/conseilUtil.js b/conseilUtil.js index b022581..874e22b 100644 --- a/conseilUtil.js +++ b/conseilUtil.js @@ -2,6 +2,8 @@ const conseiljs = require('conseiljs') const fetch = require('node-fetch') const log = require('loglevel') const BigNumber = require('bignumber.js') +const pThrottle = require('p-throttle') +// const { performance } = require('perf_hooks'); // NOTE: disabled for prod const logger = log.getLogger('conseiljs') logger.setLevel('error', false) @@ -11,6 +13,8 @@ const conseilServer = 'https://conseil-prod.cryptonomic-infra.tech' const conseilApiKey = 'aa73fa8a-8626-4f43-a605-ff63130f37b1' // signup at nautilus.cloud const tezosNode = '' +const throttleConseil = pThrottle({ limit: 15, interval: 1200 }) + const mainnet = require('./config').networkConfig /** @@ -275,12 +279,13 @@ const getArtisticUniverse = async (max_time) => { const objectQueries = queryChunks.map(c => makeObjectQuery(c)) + // const a = performance.now() let universe = [] await Promise.all( objectQueries.map(async (q) => { - const r = [] + let r = [] try { - r = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q) + r = await throttleConseilQuery('big_map_contents', q) .then(result => result.map(row => { const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1') const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') @@ -288,10 +293,15 @@ const getArtisticUniverse = async (max_time) => { universe.push({ objectId, ipfsHash, minter: artistMap[objectId], swaps: swapMap[objectId] !== undefined ? swapMap[objectId] : []}) })) // NOTE: it's a work in progress, this will drop failed requests and return a smaller set than expected + } catch (error) { + console.log('failed at query', q, 'with error', error) } finally { return r }})) + // const b = performance.now() + // console.log(`time ${b - a}`) + return universe } @@ -353,6 +363,12 @@ const chunkArray = (arr, len) => { // TODO: move to util.js return chunks; } +const throttleConseilQuery = throttleConseil(async (table, query) => { + const result = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', table, query) + + return Promise.resolve(result) +}) + module.exports = { getCollectionForAddress, gethDaoBalanceForAddress, diff --git a/package.json b/package.json index 04d9450..301e8ef 100644 --- a/package.json +++ b/package.json @@ -1,38 +1,40 @@ { - "name": "hicetnunc-apiv2", - "version": "1.0.0", - "description": "", - "main": "index.js", - "scripts": { - "start": "node index.js" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/hicetnunc2000/hicetnunc.git" - }, - "author": "@hicetnunc2000", - "license": "MIT", - "bugs": { - "url": "https://github.com/hicetnunc2000/hicetnunc/issues" - }, - "homepage": "https://github.com/hicetnunc2000/hicetnunc#readme", - "dependencies": { - "axios": "^0.21.1", - "bignumber.js": "9.0.1", - "cloud-local-storage": "0.0.11", - "conseiljs": "5.0.7-2", - "cors": "^2.8.5", - "dotenv": "^8.2.0", - "express": "^4.17.1", - "fetch": "^1.1.0", - "lodash": "^4.17.21", - "loglevel": "1.7.1", - "node-fetch": "2.6.1", - "serverless-dotenv-plugin": "^3.8.1", - "serverless-http": "^2.7.0" - }, - "engines": { - "node": "12.20.1", - "npm": "6.14.10" - } + "name": "hicetnunc-apiv2", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "start": "node index.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/hicetnunc2000/hicetnunc-api.git" + }, + "author": "@hicetnunc2000", + "license": "MIT", + "bugs": { + "url": "https://github.com/hicetnunc2000/hicetnunc-api/issues" + }, + "homepage": "https://github.com/hicetnunc2000/hicetnunc-api#readme", + "dependencies": { + "axios": "^0.21.1", + "bignumber.js": "9.0.1", + "cloud-local-storage": "0.0.11", + "conseiljs": "5.0.7-2", + "cors": "^2.8.5", + "dotenv": "^8.2.0", + "express": "^4.17.1", + "fetch": "^1.1.0", + "lodash": "^4.17.21", + "loglevel": "1.7.1", + "node-fetch": "2.6.1", + "perf_hooks": "0.0.1", + "p-throttle": "4.1.1", + "serverless-dotenv-plugin": "^3.8.1", + "serverless-http": "^2.7.0" + }, + "engines": { + "node": "12.20.1", + "npm": "6.14.10" + } } From b9c42f3610e11018a1935cbe63a310b800269abc Mon Sep 17 00:00:00 2001 From: anonymoussprocket Date: Thu, 25 Mar 2021 12:19:50 -0400 Subject: [PATCH 2/4] - added getObjectOwnersById --- conseilUtil.js | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/conseilUtil.js b/conseilUtil.js index 874e22b..bed70a6 100644 --- a/conseilUtil.js +++ b/conseilUtil.js @@ -351,6 +351,25 @@ const getObjectById = async (objectId) => { return { objectId, ipfsHash, swaps } } +const getObjectOwnersById = async (objectId) => { + let objectQuery = conseiljs.ConseilQueryBuilder.blankQuery(); + objectQuery = conseiljs.ConseilQueryBuilder.addFields(objectQuery, 'key', 'value'); + objectQuery = conseiljs.ConseilQueryBuilder.addPredicate(objectQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftLedger]) + objectQuery = conseiljs.ConseilQueryBuilder.addPredicate(objectQuery, 'key', conseiljs.ConseilOperator.ENDSWITH, [` ${objectId}`]) + objectQuery = conseiljs.ConseilQueryBuilder.setLimit(objectQuery, 20_000) + + const objectResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', objectQuery) + + const ownerMap = objectResult.map(r => { + const address = conseiljs.TezosMessageUtils.readAddress(r['key'].replace(/Pair 0x([0-9a-z]{1,}) [0-9]+$/, '$1')) + const amount = parseInt(r['value']) + + return { address, amount } + }) + + return ownerMap +} + const chunkArray = (arr, len) => { // TODO: move to util.js let chunks = [], i = 0, @@ -374,5 +393,6 @@ module.exports = { gethDaoBalanceForAddress, getArtisticOutputForAddress, getObjectById, - getArtisticUniverse + getObjectOwnersById, + getArtisticUniverse, } From 8ca26518d250aebc4291705386b60216bb890ae6 Mon Sep 17 00:00:00 2001 From: anonymoussprocket Date: Sat, 3 Apr 2021 00:55:31 -0400 Subject: [PATCH 3/4] - add swapList --- conseilUtil.js | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/conseilUtil.js b/conseilUtil.js index bed70a6..806fb50 100644 --- a/conseilUtil.js +++ b/conseilUtil.js @@ -370,6 +370,73 @@ const getObjectOwnersById = async (objectId) => { return ownerMap } +const listSwaps = async () => { + let swapsQuery = conseiljs.ConseilQueryBuilder.blankQuery() + swapsQuery = conseiljs.ConseilQueryBuilder.addFields(swapsQuery, 'key', 'value') + swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate(swapsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftSwapMap]) + swapsQuery = conseiljs.ConseilQueryBuilder. + swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 50_000) + + const swapsResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', swapsQuery) + + const swapValuePattern = new RegExp('Pair [(]Pair 0x([0-9a-z]{44}) ([0-9]+)[)] [(]Pair ([0-9]+) ([0-9]+)[)]'); + const swapList = swapsResult + .filter(row => row['value'] && row['value'].length > 0) + .map(row => { + const swapDefinition = row['value'] + + if (swapValuePattern.test(swapDefinition)) { + match = swapDefinition.match(swapValuePattern); + + return { + swap_id: row['key'], + issuer: conseiljs.TezosMessageUtils.readAddress(match[1]), + objkt_id: match[3], + objkt_amount: match[2], + xtz_per_objkt: (new BigNumber(match[4])).dividedBy(1_000_000).toFixed(6) + } + } + + return undefined + }).filter(row => row) + + const distinctIds = [... new Set(swapList.map(swap => swap['swap_id']))] + const ipfsMap = await getObjktInfoUrl(distinctIds) + + swapList.forEach(row => { + row['ipfsHash'] = ipfsMap[row['objkt_id']] || '' + }) + + return swapList.filter(row => row['ipfsHash'].length) +} + +const getObjktInfoUrl = async (objktList) => { + const queryChunks = chunkArray(objktList, 100) + + const makeObjectQuery = (keys) => { + let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery(); + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields(mintedObjectsQuery, 'key_hash', 'value'); + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftMetadataMap]) + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'key', (keys.length > 1 ? conseiljs.ConseilOperator.IN : conseiljs.ConseilOperator.EQ), keys) + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.setLimit(mintedObjectsQuery, keys.length) + + return mintedObjectsQuery + } + + const objectQueries = queryChunks.map(c => makeObjectQuery(c)) + const objectIpfsMap = {} + await Promise.all(objectQueries.map(async (q) => await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q) + .then(result => result.map(row => { + const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1') + const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') + const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7); + + objectIpfsMap[objectId] = ipfsHash + })))) + + return objectIpfsMap +} + const chunkArray = (arr, len) => { // TODO: move to util.js let chunks = [], i = 0, @@ -395,4 +462,5 @@ module.exports = { getObjectById, getObjectOwnersById, getArtisticUniverse, + listSwaps } From 04969650e37b14c673cdff93e61acea8a6c4cee2 Mon Sep 17 00:00:00 2001 From: anonymoussprocket Date: Sat, 3 Apr 2021 01:13:44 -0400 Subject: [PATCH 4/4] - missing line --- conseilUtil.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conseilUtil.js b/conseilUtil.js index 806fb50..4c62df2 100644 --- a/conseilUtil.js +++ b/conseilUtil.js @@ -374,7 +374,7 @@ const listSwaps = async () => { let swapsQuery = conseiljs.ConseilQueryBuilder.blankQuery() swapsQuery = conseiljs.ConseilQueryBuilder.addFields(swapsQuery, 'key', 'value') swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate(swapsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftSwapMap]) - swapsQuery = conseiljs.ConseilQueryBuilder. + swapsQuery = conseiljs.ConseilQueryBuilder.addOrdering(swapsQuery, 'block_level', conseiljs.ConseilSortDirection.DESC) swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 50_000) const swapsResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', swapsQuery)