Skip to content
This repository has been archived by the owner on Nov 10, 2021. It is now read-only.

added getObjectOwnersById #17

Open
wants to merge 4 commits into
base: develop
Choose a base branch
from
Open

added getObjectOwnersById #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
faae0e2
- add throttling
anonymoussprocket Mar 20, 2021
b9c42f3
- added getObjectOwnersById
anonymoussprocket Mar 25, 2021
8ca2651
- add swapList
anonymoussprocket Apr 3, 2021
0496965
- missing line
anonymoussprocket Apr 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 107 additions & 3 deletions conseilUtil.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ const conseiljs = require('conseiljs')
const fetch = require('node-fetch')
const log = require('loglevel')
const BigNumber = require('bignumber.js')
const pThrottle = require('p-throttle')
// const { performance } = require('perf_hooks'); // NOTE: disabled for prod

const logger = log.getLogger('conseiljs')
logger.setLevel('error', false)
Expand All @@ -11,6 +13,8 @@ const conseilServer = 'https://conseil-prod.cryptonomic-infra.tech'
const conseilApiKey = 'aa73fa8a-8626-4f43-a605-ff63130f37b1' // signup at nautilus.cloud
const tezosNode = ''

const throttleConseil = pThrottle({ limit: 15, interval: 1200 })

const mainnet = require('./config').networkConfig

/**
Expand Down Expand Up @@ -275,23 +279,29 @@ const getArtisticUniverse = async (max_time) => {

const objectQueries = queryChunks.map(c => makeObjectQuery(c))

// const a = performance.now()
let universe = []
await Promise.all(
objectQueries.map(async (q) => {
const r = []
let r = []
try {
r = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q)
r = await throttleConseilQuery('big_map_contents', q)
.then(result => result.map(row => {
const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1')
const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1')
const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7);

universe.push({ objectId, ipfsHash, minter: artistMap[objectId], swaps: swapMap[objectId] !== undefined ? swapMap[objectId] : []})
})) // NOTE: it's a work in progress, this will drop failed requests and return a smaller set than expected
} catch (error) {
console.log('failed at query', q, 'with error', error)
} finally {
return r
}}))

// const b = performance.now()
// console.log(`time ${b - a}`)

return universe
}

Expand Down Expand Up @@ -341,6 +351,92 @@ const getObjectById = async (objectId) => {
return { objectId, ipfsHash, swaps }
}

const getObjectOwnersById = async (objectId) => {
let objectQuery = conseiljs.ConseilQueryBuilder.blankQuery();
objectQuery = conseiljs.ConseilQueryBuilder.addFields(objectQuery, 'key', 'value');
objectQuery = conseiljs.ConseilQueryBuilder.addPredicate(objectQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftLedger])
objectQuery = conseiljs.ConseilQueryBuilder.addPredicate(objectQuery, 'key', conseiljs.ConseilOperator.ENDSWITH, [` ${objectId}`])
objectQuery = conseiljs.ConseilQueryBuilder.setLimit(objectQuery, 20_000)

const objectResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', objectQuery)

const ownerMap = objectResult.map(r => {
const address = conseiljs.TezosMessageUtils.readAddress(r['key'].replace(/Pair 0x([0-9a-z]{1,}) [0-9]+$/, '$1'))
const amount = parseInt(r['value'])

return { address, amount }
})

return ownerMap
}

const listSwaps = async () => {
let swapsQuery = conseiljs.ConseilQueryBuilder.blankQuery()
swapsQuery = conseiljs.ConseilQueryBuilder.addFields(swapsQuery, 'key', 'value')
swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate(swapsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftSwapMap])
swapsQuery = conseiljs.ConseilQueryBuilder.addOrdering(swapsQuery, 'block_level', conseiljs.ConseilSortDirection.DESC)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is what you talked about right, being able to filter swaps on block_level?

swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 50_000)

const swapsResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', swapsQuery)

const swapValuePattern = new RegExp('Pair [(]Pair 0x([0-9a-z]{44}) ([0-9]+)[)] [(]Pair ([0-9]+) ([0-9]+)[)]');
const swapList = swapsResult
.filter(row => row['value'] && row['value'].length > 0)
.map(row => {
const swapDefinition = row['value']

if (swapValuePattern.test(swapDefinition)) {
match = swapDefinition.match(swapValuePattern);

return {
swap_id: row['key'],
issuer: conseiljs.TezosMessageUtils.readAddress(match[1]),
objkt_id: match[3],
objkt_amount: match[2],
xtz_per_objkt: (new BigNumber(match[4])).dividedBy(1_000_000).toFixed(6)
}
}

return undefined
}).filter(row => row)

const distinctIds = [... new Set(swapList.map(swap => swap['swap_id']))]
const ipfsMap = await getObjktInfoUrl(distinctIds)

swapList.forEach(row => {
row['ipfsHash'] = ipfsMap[row['objkt_id']] || ''
})

return swapList.filter(row => row['ipfsHash'].length)
}

const getObjktInfoUrl = async (objktList) => {
const queryChunks = chunkArray(objktList, 100)

const makeObjectQuery = (keys) => {
let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery();
mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields(mintedObjectsQuery, 'key_hash', 'value');
mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftMetadataMap])
mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'key', (keys.length > 1 ? conseiljs.ConseilOperator.IN : conseiljs.ConseilOperator.EQ), keys)
mintedObjectsQuery = conseiljs.ConseilQueryBuilder.setLimit(mintedObjectsQuery, keys.length)

return mintedObjectsQuery
}

const objectQueries = queryChunks.map(c => makeObjectQuery(c))
const objectIpfsMap = {}
await Promise.all(objectQueries.map(async (q) => await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q)
.then(result => result.map(row => {
const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1')
const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1')
const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7);

objectIpfsMap[objectId] = ipfsHash
}))))

return objectIpfsMap
}

const chunkArray = (arr, len) => { // TODO: move to util.js
let chunks = [],
i = 0,
Expand All @@ -353,10 +449,18 @@ const chunkArray = (arr, len) => { // TODO: move to util.js
return chunks;
}

const throttleConseilQuery = throttleConseil(async (table, query) => {
const result = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', table, query)

return Promise.resolve(result)
})

module.exports = {
getCollectionForAddress,
gethDaoBalanceForAddress,
getArtisticOutputForAddress,
getObjectById,
getArtisticUniverse
getObjectOwnersById,
getArtisticUniverse,
listSwaps
}
74 changes: 38 additions & 36 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,38 +1,40 @@
{
"name": "hicetnunc-apiv2",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"start": "node index.js"
},
"repository": {
"type": "git",
"url": "git+https://github.com/hicetnunc2000/hicetnunc.git"
},
"author": "@hicetnunc2000",
"license": "MIT",
"bugs": {
"url": "https://github.com/hicetnunc2000/hicetnunc/issues"
},
"homepage": "https://github.com/hicetnunc2000/hicetnunc#readme",
"dependencies": {
"axios": "^0.21.1",
"bignumber.js": "9.0.1",
"cloud-local-storage": "0.0.11",
"conseiljs": "5.0.7-2",
"cors": "^2.8.5",
"dotenv": "^8.2.0",
"express": "^4.17.1",
"fetch": "^1.1.0",
"lodash": "^4.17.21",
"loglevel": "1.7.1",
"node-fetch": "2.6.1",
"serverless-dotenv-plugin": "^3.8.1",
"serverless-http": "^2.7.0"
},
"engines": {
"node": "12.20.1",
"npm": "6.14.10"
}
"name": "hicetnunc-apiv2",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"start": "node index.js"
},
"repository": {
"type": "git",
"url": "git+https://github.com/hicetnunc2000/hicetnunc-api.git"
},
"author": "@hicetnunc2000",
"license": "MIT",
"bugs": {
"url": "https://github.com/hicetnunc2000/hicetnunc-api/issues"
},
"homepage": "https://github.com/hicetnunc2000/hicetnunc-api#readme",
"dependencies": {
"axios": "^0.21.1",
"bignumber.js": "9.0.1",
"cloud-local-storage": "0.0.11",
"conseiljs": "5.0.7-2",
"cors": "^2.8.5",
"dotenv": "^8.2.0",
"express": "^4.17.1",
"fetch": "^1.1.0",
"lodash": "^4.17.21",
"loglevel": "1.7.1",
"node-fetch": "2.6.1",
"perf_hooks": "0.0.1",
"p-throttle": "4.1.1",
"serverless-dotenv-plugin": "^3.8.1",
"serverless-http": "^2.7.0"
},
"engines": {
"node": "12.20.1",
"npm": "6.14.10"
}
}