feat: NFTIO-2406 attribute rarity ranker (#1122)

* attribute scoring NFTIO-2406

* improve

* add migration

* add job to the end of traits job

* wrap up

* add relations

* fixes

* rarity

db/migrations/1717977154451-Data.js

@@ -0,0 +1,19 @@
+module.exports = class Data1717977154451 {
+    name = 'Data1717977154451'
+
+    async up(db) {
+        await db.query(`CREATE TABLE "token_rarity" ("id" character varying NOT NULL, "score" numeric NOT NULL, "rank" integer NOT NULL, "collection_id" character varying, "token_id" character varying, CONSTRAINT "PK_aa34e9209b73d6ac33b006b2f6d" PRIMARY KEY ("id"))`)
+        await db.query(`CREATE INDEX "IDX_2983370aa94af81928e41175e0" ON "token_rarity" ("collection_id") `)
+        await db.query(`CREATE INDEX "IDX_a708c8b728c92f895762b5d027" ON "token_rarity" ("token_id") `)
+        await db.query(`ALTER TABLE "token_rarity" ADD CONSTRAINT "FK_2983370aa94af81928e41175e0e" FOREIGN KEY ("collection_id") REFERENCES "collection"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`)
+        await db.query(`ALTER TABLE "token_rarity" ADD CONSTRAINT "FK_a708c8b728c92f895762b5d0278" FOREIGN KEY ("token_id") REFERENCES "token"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`)
+    }
+
+    async down(db) {
+        await db.query(`DROP TABLE "token_rarity"`)
+        await db.query(`DROP INDEX "public"."IDX_2983370aa94af81928e41175e0"`)
+        await db.query(`DROP INDEX "public"."IDX_a708c8b728c92f895762b5d027"`)
+        await db.query(`ALTER TABLE "token_rarity" DROP CONSTRAINT "FK_2983370aa94af81928e41175e0e"`)
+        await db.query(`ALTER TABLE "token_rarity" DROP CONSTRAINT "FK_a708c8b728c92f895762b5d0278"`)
+    }
+}

db/migrations/1718013477503-Data.js

@@ -0,0 +1,19 @@
+module.exports = class Data1718013477503 {
+    name = 'Data1718013477503'
+
+    async up(db) {
+        await db.query(`DROP INDEX "public"."IDX_a708c8b728c92f895762b5d027"`)
+        await db.query(`ALTER TABLE "token_rarity" DROP CONSTRAINT "FK_a708c8b728c92f895762b5d0278"`)
+        await db.query(`ALTER TABLE "token_rarity" ADD CONSTRAINT "UQ_a708c8b728c92f895762b5d0278" UNIQUE ("token_id")`)
+        await db.query(`CREATE UNIQUE INDEX "IDX_a708c8b728c92f895762b5d027" ON "token_rarity" ("token_id") `)
+        await db.query(`ALTER TABLE "token_rarity" ADD CONSTRAINT "FK_a708c8b728c92f895762b5d0278" FOREIGN KEY ("token_id") REFERENCES "token"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`)
+    }
+
+    async down(db) {
+        await db.query(`CREATE INDEX "IDX_a708c8b728c92f895762b5d027" ON "token_rarity" ("token_id") `)
+        await db.query(`ALTER TABLE "token_rarity" ADD CONSTRAINT "FK_a708c8b728c92f895762b5d0278" FOREIGN KEY ("token_id") REFERENCES "token"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`)
+        await db.query(`ALTER TABLE "token_rarity" DROP CONSTRAINT "UQ_a708c8b728c92f895762b5d0278"`)
+        await db.query(`DROP INDEX "public"."IDX_a708c8b728c92f895762b5d027"`)
+        await db.query(`ALTER TABLE "token_rarity" DROP CONSTRAINT "FK_a708c8b728c92f895762b5d0278"`)
+    }
+}

package.json

@@ -39,6 +39,7 @@
     "dotenv": "^16.4.5",
     "express": "^4.19.2",
     "lodash": "^4.17.21",
+    "mathjs": "^13.0.0",
     "mime-types": "^2.1.35",
     "node-cache": "^5.1.2",
     "pg": "^8.12.0",

schema.graphql

@@ -480,6 +480,7 @@ type Collection @entity {
   tokenAccounts: [TokenAccount] @derivedFrom(field: "collection")
   attributes: [Attribute] @derivedFrom(field: "collection")
   traits: [Trait] @derivedFrom(field: "collection")
+  rarity: [TokenRarity] @derivedFrom(field: "collection")
 
   # Extras
   metadata: Metadata
@@ -587,13 +588,22 @@ type Token @entity {
   bestListing: Listing 
   recentListing: Listing
   lastSale: ListingSale
+  rarity: TokenRarity @derivedFrom(field: "token")
 
   # Extras
   nonFungible: Boolean!
   metadata: Metadata
   createdAt: DateTime! @index
 }
 
+type TokenRarity @entity {
+  id: ID!
+  collection: Collection!
+  token: Token! @unique
+  score: Float!
+  rank: Int!
+}
+
 type TokenApproval {
   account: String!
   amount: BigInt!

src/job-handlers/compute-traits.ts

@@ -4,6 +4,7 @@ import Queue from 'bull'
 import connection from '../connection'
 import { Collection, Token, Trait, TraitToken } from '../model'
 import { JobData } from '../jobs/compute-traits'
+import { computeRarityRank, rarityQueue } from '../jobs/rarity-ranker'
 
 type TraitValueMap = Map<string, bigint>
 
@@ -119,5 +120,7 @@ export default async (job: Queue.Job<JobData>, done: Queue.DoneCallback) => {
         await em.save(TraitToken, traitTokensToSave as any, { chunk: 1000 })
     }
 
+    computeRarityRank(collectionId)
+
     done(null, { timeElapsed: new Date().getTime() - start.getTime() })
 }

src/job-handlers/rarity-ranker.ts

@@ -0,0 +1,86 @@
+import Queue from 'bull'
+import * as mathjs from 'mathjs'
+import { informationContentScoring } from '../open-rarity/handlers/information-content-scoring'
+import connection from '../connection'
+import { Collection, Token, TokenRarity } from '../model'
+import { JobData } from '../jobs/rarity-ranker'
+
+export default async (job: Queue.Job<JobData>, done: Queue.DoneCallback) => {
+    if (!job.data.collectionId) {
+        throw new Error('Collection ID not provided.')
+    }
+
+    if (!connection.isInitialized) {
+        await connection.initialize().catch(() => {
+            done(new Error('Failed to initialize connection'))
+        })
+    }
+
+    const em = connection.manager
+
+    console.time('rarity-ranker')
+
+    const [collection, tokens] = await Promise.all([
+        em.findOneOrFail(Collection, {
+            relations: {
+                traits: true,
+            },
+            where: { id: job.data.collectionId },
+        }),
+        em.find(Token, {
+            relations: {
+                traits: {
+                    trait: true,
+                },
+            },
+            where: { collection: { id: job.data.collectionId } },
+        }),
+    ])
+
+    const totalSupply = collection.stats.supply
+
+    // check if total supply is greater than 0
+    if (!totalSupply || totalSupply <= 0) {
+        return done(new Error('Total supply is 0'))
+    }
+
+    const entropy = informationContentScoring.collectionEntropy(totalSupply, collection.traits)
+
+    if (!entropy) {
+        return done(new Error('Collection entropy is 0'))
+    }
+
+    const tokenRarities = tokens.map((token) => {
+        return { score: informationContentScoring.scoreToken(totalSupply, entropy, token), token }
+    })
+
+    tokenRarities.sort((a, b) => Number(mathjs.compare(b.score, a.score)))
+
+    // sort by token.id if two tokens have the same score
+    tokenRarities.sort((a, b) => {
+        if (Number(mathjs.compare(a.score, b.score)) === 0) {
+            return Number(mathjs.compare(mathjs.bignumber(a.token.tokenId), mathjs.bignumber(b.token.tokenId)))
+        }
+        return 0
+    })
+
+    const tokenRanks = tokenRarities.map((tokenRarity, index) => {
+        return new TokenRarity({
+            id: `${tokenRarity.token.id}`,
+            collection,
+            token: tokenRarity.token,
+            score: tokenRarity.score.toNumber(),
+            rank: index + 1,
+        })
+    })
+
+    // delete existing token rarities
+    await em.delete(TokenRarity, { collection: { id: job.data.collectionId } })
+
+    // save new token rarities
+    await em.save(tokenRanks)
+
+    console.timeEnd('rarity-ranker')
+
+    return done()
+}

src/job-handlers/worker.ts

@@ -11,6 +11,7 @@ import { fetchBalanceQueue } from '../jobs/fetch-balance'
 import { traitsQueue } from '../jobs/compute-traits'
 import { fetchCollectionExtraQueue } from '../jobs/fetch-collection-extra'
 import { invalidateExpiredListings } from '../jobs/invalidate-expired-listings'
+import { rarityQueue } from '../jobs/rarity-ranker'
 
 async function main() {
     if (!connection.isInitialized) {
@@ -23,11 +24,13 @@ async function main() {
     console.info('handling jobs...')
 
     traitsQueue.process(2, `${__dirname}/compute-traits.js`)
+    rarityQueue.process(2, `${__dirname}/rarity-ranker.js`)
     metadataQueue.process(
         process.env.MAX_WORKER_CONCURRENCY ? parseInt(process.env.MAX_WORKER_CONCURRENCY, 10) : 50,
         `${__dirname}/process-metadata.js`
     )
-    collectionStatsQueue.process(10, `${__dirname}/collection-stats.js`)
+    collectionStatsQueue.process(2, `${__dirname}/collection-stats.js`)
+
     fetchAccountQueue.process(5, `${__dirname}/fetch-account.js`)
     fetchBalanceQueue.process(5, `${__dirname}/fetch-balance.js`)
     fetchCollectionExtraQueue.process(5, `${__dirname}/fetch-collection-extra.js`)
@@ -51,6 +54,7 @@ async function main() {
             new BullAdapter(fetchAccountQueue),
             new BullAdapter(fetchBalanceQueue),
             new BullAdapter(traitsQueue),
+            new BullAdapter(rarityQueue),
             new BullAdapter(fetchCollectionExtraQueue),
             new BullAdapter(invalidateExpiredListings),
         ],

src/jobs/process-metadata.ts

@@ -9,7 +9,7 @@ export const metadataQueue = new Queue<JobData>('metadataQueue', {
         attempts: 3,
         backoff: {
             type: 'exponential',
-            delay: 3000,
+            delay: 4000,
         },
         removeOnComplete: true,
     },

src/jobs/rarity-ranker.ts

@@ -0,0 +1,28 @@
+import Queue from 'bull'
+import { redisConfig } from './common'
+
+export type JobData = { collectionId: string }
+
+export const rarityQueue = new Queue<JobData>('rarityQueue', {
+    defaultJobOptions: { delay: 1000, attempts: 2, removeOnComplete: true },
+    redis: redisConfig,
+    settings: {
+        maxStalledCount: 3,
+    },
+})
+
+export const computeRarityRank = async (collectionId: string) => {
+    if (!collectionId) {
+        throw new Error('Collection ID not provided.')
+    }
+
+    if (collectionId === '0') {
+        return
+    }
+
+    rarityQueue.add({ collectionId }, { jobId: collectionId }).catch(() => {
+        // eslint-disable-next-line no-console
+        console.log('Closing connection as Redis is not available')
+        rarityQueue.close(true)
+    })
+}

src/model/generated/collection.model.ts

@@ -10,6 +10,7 @@ import {CollectionAccount} from "./collectionAccount.model"
 import {TokenAccount} from "./tokenAccount.model"
 import {Attribute} from "./attribute.model"
 import {Trait} from "./trait.model"
+import {TokenRarity} from "./tokenRarity.model"
 import {Metadata} from "./_metadata"
 import {CollectionFlags} from "./_collectionFlags"
 import {CollectionSocials} from "./_collectionSocials"
@@ -71,6 +72,9 @@ export class Collection {
     @OneToMany_(() => Trait, e => e.collection)
     traits!: Trait[]
 
+    @OneToMany_(() => TokenRarity, e => e.collection)
+    rarity!: TokenRarity[]
+
     @Column_("jsonb", {transformer: {to: obj => obj == null ? undefined : obj.toJSON(), from: obj => obj == null ? undefined : new Metadata(undefined, obj)}, nullable: true})
     metadata!: Metadata | undefined | null
 

src/model/generated/index.ts

@@ -74,6 +74,7 @@ export * from "./_tokenBehavior"
 export * from "./_tokenBehaviorHasRoyalty"
 export * from "./_tokenBehaviorType"
 export * from "./_tokenBehaviorIsCurrency"
+export * from "./tokenRarity.model"
 export * from "./tokenAccount.model"
 export * from "./_tokenNamedReserve"
 export * from "./_tokenLock"

src/model/generated/token.model.ts

@@ -1,4 +1,4 @@
-import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, BigIntColumn as BigIntColumn_, Index as Index_, BooleanColumn as BooleanColumn_, IntColumn as IntColumn_, ManyToOne as ManyToOne_, OneToMany as OneToMany_, DateTimeColumn as DateTimeColumn_} from "@subsquid/typeorm-store"
+import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, BigIntColumn as BigIntColumn_, Index as Index_, BooleanColumn as BooleanColumn_, IntColumn as IntColumn_, ManyToOne as ManyToOne_, OneToMany as OneToMany_, OneToOne as OneToOne_, DateTimeColumn as DateTimeColumn_} from "@subsquid/typeorm-store"
 import * as marshal from "./marshal"
 import {FreezeState} from "./_freezeState"
 import {TokenCap, fromJsonTokenCap} from "./_tokenCap"
@@ -9,6 +9,7 @@ import {Attribute} from "./attribute.model"
 import {Listing} from "./listing.model"
 import {TraitToken} from "./traitToken.model"
 import {ListingSale} from "./listingSale.model"
+import {TokenRarity} from "./tokenRarity.model"
 import {Metadata} from "./_metadata"
 
 @Entity_()
@@ -85,6 +86,9 @@ export class Token {
     @ManyToOne_(() => ListingSale, {nullable: true})
     lastSale!: ListingSale | undefined | null
 
+    @OneToOne_(() => TokenRarity, e => e.token)
+    rarity!: TokenRarity | undefined | null
+
     @BooleanColumn_({nullable: false})
     nonFungible!: boolean
 

src/model/generated/tokenRarity.model.ts

@@ -0,0 +1,28 @@
+import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, ManyToOne as ManyToOne_, Index as Index_, OneToOne as OneToOne_, JoinColumn as JoinColumn_, FloatColumn as FloatColumn_, IntColumn as IntColumn_} from "@subsquid/typeorm-store"
+import {Collection} from "./collection.model"
+import {Token} from "./token.model"
+
+@Entity_()
+export class TokenRarity {
+    constructor(props?: Partial<TokenRarity>) {
+        Object.assign(this, props)
+    }
+
+    @PrimaryColumn_()
+    id!: string
+
+    @Index_()
+    @ManyToOne_(() => Collection, {nullable: true})
+    collection!: Collection
+
+    @Index_({unique: true})
+    @OneToOne_(() => Token, {nullable: true})
+    @JoinColumn_()
+    token!: Token
+
+    @FloatColumn_({nullable: false})
+    score!: number
+
+    @IntColumn_({nullable: false})
+    rank!: number
+}

src/open-rarity/handlers/information-content-scoring.ts

@@ -0,0 +1,36 @@
+import * as mathjs from 'mathjs'
+import { Trait, Token } from '../../model'
+
+export const informationContentScoring = {
+    scoreToken(totalSupply: bigint, entropy: number, token: Token) {
+        const traits = token.traits.map((trait) => trait.trait)
+
+        if (traits.length === 0) {
+            return mathjs.bignumber(0)
+        }
+
+        const tokenAttributesScore = this.getTokenAttributesScore(totalSupply, traits)
+
+        return mathjs.bignumber(tokenAttributesScore).div(entropy)
+    },
+
+    getTokenAttributesScore(totalSupply: bigint, tokenTraits: Trait[]) {
+        const scores: mathjs.BigNumber[] = []
+
+        tokenTraits.forEach((trait) => {
+            scores.push(mathjs.bignumber(trait.count).div(mathjs.bignumber(totalSupply)))
+        })
+
+        return mathjs.sum(mathjs.log2(scores))
+    },
+
+    collectionEntropy(totalSupply: bigint, traits: Trait[]) {
+        const collectionProbabilities: mathjs.BigNumber[] = []
+
+        traits.forEach((trait) => {
+            collectionProbabilities.push(mathjs.bignumber(trait.count).div(mathjs.bignumber(totalSupply)))
+        })
+
+        return mathjs.dot(collectionProbabilities, mathjs.log2(collectionProbabilities))
+    },
+}