feat(processing): make listing TTL configurable

prenaissance · Nov 23, 2023 · f2c1c8c · f2c1c8c
1 parent f32ad34
commit f2c1c8c
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -20,6 +20,7 @@ Environment variables:
 | MongoDbUrl       | The url of the mongo database         | mongodb://localhost:27017 |
 | MongoDbName      | The name of the mongo database        | backpack-tf-replica       |
 | BackpackTfCookie | The cookie from a backpack.tf session | **REQUIRED**              |
+| ListingsTtlHours | How long to keep stale listings in db | 6                         |
 
 The following collections are aggregated in the provided mongo database:
 

diff --git a/src/processing/Program.fs b/src/processing/Program.fs
@@ -85,7 +85,8 @@ let getExamplePricingEvent () =
 type Config =
     { MongoDbUrl: string
       MongoDbName: string
-      BackpackTfCookie: string }
+      BackpackTfCookie: string
+      ListingsTtlHours: int }
 
 let configuration =
     let builder = ConfigurationBuilder()
@@ -101,7 +102,12 @@ let configuration =
 
     { config with
         MongoDbUrl = config.MongoDbUrl |> StringUtils.defaultIfEmpty "mongodb://localhost:27017"
-        MongoDbName = config.MongoDbName |> StringUtils.defaultIfEmpty "backpack-tf-replica" }
+        MongoDbName = config.MongoDbName |> StringUtils.defaultIfEmpty "backpack-tf-replica"
+        ListingsTtlHours =
+            if config.ListingsTtlHours > 0 then
+                config.ListingsTtlHours
+            else
+                6 }
 
 [<Literal>]
 let wsUrl = "wss://ws.backpack.tf/events"
@@ -117,7 +123,9 @@ let db = Db.connectToMongoDb configuration.MongoDbUrl configuration.MongoDbName
 
 
 let tradeListingsCollection =
-    db |> Db.TradeListings.getCollection |> Async.RunSynchronously
+    db
+    |> Db.TradeListings.getCollection configuration.ListingsTtlHours
+    |> Async.RunSynchronously
 
 
 let getWsEventStream (url: string) =

diff --git a/src/processing/Services/Db.fs b/src/processing/Services/Db.fs
@@ -14,7 +14,7 @@ module Db =
         database
 
     module TradeListings =
-        open PricingTf.Common.Models
+        open MongoDB.Driver
         open MongoDB.Bson
         open System
         open PricingTf.Common.Configuration
@@ -40,26 +40,41 @@ module Db =
 
             CreateIndexModel(indexKey, options)
 
-        let private bumpedAtIndexModel =
+        let private getBumpedAtIndex listingsTtlHours =
             // ttl index
             let indexKey = IndexKeysDefinitionBuilder<TradeListing>().Ascending("bumpedAt")
 
             let options = new CreateIndexOptions()
-            options.ExpireAfter <- TimeSpan.FromDays(1.)
+            options.ExpireAfter <- TimeSpan.FromHours(listingsTtlHours)
 
             CreateIndexModel(indexKey, options)
 
-        let private indices =
-            [ nameIndexModel
-              nameAndIntentIndexModel
-              listingIdIndexModel
-              bumpedAtIndexModel ]
-
-        let getCollection (database: IMongoDatabase) =
+        let getCollection listingsTtlHours (database: IMongoDatabase) =
             async {
                 let collection =
                     database.GetCollection<TradeListing>(PricingCollection.TradeListings)
 
+                // drop existing bumpedAt index if the ttl is different
+                let! existingIndexes = collection.Indexes.List().ToListAsync() |> Async.AwaitTask
+
+                let bumpedAtIndex =
+                    existingIndexes
+                    |> Seq.tryFind (fun x -> x.GetElement("name").Value.AsString = "bumpedAt_1")
+
+                match bumpedAtIndex with
+                | Some index ->
+                    let ttlHours = index.GetElement("expireAfterSeconds").Value.AsInt32 / 3600
+
+                    if ttlHours <> listingsTtlHours then
+                        do! collection.Indexes.DropOneAsync("bumpedAt_1") |> Async.AwaitTask |> Async.Ignore
+                | None -> ()
+
+                let indices =
+                    [ nameIndexModel
+                      nameAndIntentIndexModel
+                      listingIdIndexModel
+                      getBumpedAtIndex listingsTtlHours ]
+
                 do! collection.Indexes.CreateManyAsync(indices) |> Async.AwaitTask |> Async.Ignore
 
                 return collection