keep some failed/successful jobs and update controlflow

sandwichfarm · Mar 16, 2024 · c6eb3c0 · c6eb3c0
1 parent 88bb2f3
commit c6eb3c0
Show file tree

Hide file tree

Showing 7 changed files with 36 additions and 32 deletions.
diff --git a/apps/nocapd/docker-compose.yaml b/apps/nocapd/docker-compose.yaml
@@ -16,32 +16,32 @@ services:
     environment:
       DOCKER: 'yes'
     env_file: .env.docker
-    depends_on: 
-      - cache
-    networks:
-      - nocapdnet
-  cache:
-    image: redis:latest
-    restart: always
-    ports:
-      - '6379:6379'
-    volumes: 
-      - ./.redis:/.data:rw
-    env_file: .env.docker
-    command: redis-server
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 10s
-      timeout: 5s
-      retries: 3
+    # depends_on: 
+    #   - cache
     networks:
       - nocapdnet
+  # cache:
+  #   image: redis:latest
+  #   restart: always
+  #   ports:
+  #     - '6379:6379'
+  #   volumes: 
+  #     - ./.redis:/.data:rw
+  #   env_file: .env.docker
+  #   command: redis-server
+  #   healthcheck:
+  #     test: ["CMD", "redis-cli", "ping"]
+  #     interval: 10s
+  #     timeout: 5s
+  #     retries: 3
+  #   networks:
+  #     - nocapdnet
 
 volumes: 
   lmdb:
     driver: local
-  cache: 
-    driver: local
+  # cache: 
+  #   driver: local
 
 networks:
   nocapdnet:

diff --git a/apps/nocapd/package.json b/apps/nocapd/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@nostrwatch/nocapd",
   "type": "module",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "main": "index.js",
   "license": "MIT",
   "dependencies": {

diff --git a/apps/nocapd/src/classes/Worker.js b/apps/nocapd/src/classes/Worker.js
@@ -52,14 +52,13 @@ export class NWWorker {
     this.nocapOpts = { 
       timeout: this.timeout,
       checked_by: this.pubkey
-      // rejectOnConnectFailure: true
     }
   }
 
   setupJobOpts(){
-    this.jobOpts = {
-      removeOnComplete: true,
-      removeOnFail: true
+    this.jobOpts ={
+      removeOnComplete: 100,
+      removeOnFail: 100
     }
   }
 

diff --git a/apps/nocapd/src/daemon.js b/apps/nocapd/src/daemon.js
@@ -70,6 +70,12 @@ const syncRelaysIn = async () => {
     return persisted
 }
 
+const queueOpts = () => {
+  return {
+    lockDuration: 2*60*1000
+  }
+}
+
 const initWorker = async () => {
   const connection = RedisConnectionDetails()
   const concurrency = config?.nocapd?.bullmq?.worker?.concurrency? config.nocapd.bullmq.worker.concurrency: 1
@@ -79,7 +85,7 @@ const initWorker = async () => {
     .set( 'queue'  , ncdq.$Queue )
     .set( 'events' , ncdq.$QueueEvents )
     .set( 'checker', new NWWorker(PUBKEY, $q, rcache, {...config, logger: new Logger('@nostrwatch/nocapd:worker'), pubkey: PUBKEY }) )
-    .set( 'worker' , new BullMQ.Worker($q.queue.name, $q.route_work.bind($q), { concurrency, connection, lockDuration: 2*60*1000 } ) )
+    .set( 'worker' , new BullMQ.Worker($q.queue.name, $q.route_work.bind($q), { concurrency, connection, ...queueOpts() } ) )
   await $q.checker.populator()
 
   schedulePopulator($q.checker)
@@ -93,7 +99,7 @@ const initWorker = async () => {
 }
 
 const maybeBootstrap = async () => {
-  log.info(`Boostrapping...`)
+  log.info(`Bootstrapping...`)
   if(rcache.relay.count.all() === 0){
     const persisted = await syncRelaysIn()
     log.info(`Boostrapped ${persisted.length} relays`)

diff --git a/package.json b/package.json
@@ -3,10 +3,11 @@
   "private": true,
   "version": "0.0.1",
   "scripts": {
-    "deploy:nocapd@all": "ansible-playbook .ansible/nocapd/deploy.yaml -i .ansible/inventories/amsterdam -i .ansible/inventories/johannesburg -i .ansible/inventories/mumbai -i .ansible/inventories/newyork -i .ansible/inventories/saopaulo -i .ansible/inventories/seoul -i .ansible/inventories/siliconvalley -i .ansible/inventories/sydney",
+    "deploy:nocapd@all": "ansible-playbook .ansible/nocapd/deploy.yaml -i .ansible/inventories/amsterdam -i .ansible/inventories/mumbai -i .ansible/inventories/newyork -i .ansible/inventories/saopaulo -i .ansible/inventories/seoul -i .ansible/inventories/siliconvalley -i .ansible/inventories/sydney",
     "deploy:nocapd@theship": "ansible-playbook .ansible/nocapd/deploy.yaml -i .ansible/inventories/theship --ask-become-pass",
     "deploy:nocapd@frankfurt": "ansible-playbook .ansible/nocapd/deploy.yaml -i .ansible/inventories/frankfurt --ask-become-pass",
-    "deploy:nocapd@amsterdam": "ansible-playbook .ansible/nocapd/deploy.yaml -i .ansible/inventories/amsterdam"
+    "deploy:nocapd@amsterdam": "ansible-playbook .ansible/nocapd/deploy.yaml -i .ansible/inventories/amsterdam",
+    "deploy:nocapd@johannesburg": "ansible-playbook .ansible/nocapd/deploy.yaml -i .ansible/inventories/johannesburg"
 
   },
   "packageManager": "[email protected]^",

diff --git a/packages/controlflow/package.json b/packages/controlflow/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@nostrwatch/controlflow",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "description": "Provides exports for application control flow",
   "main": "index.js",
   "type": "module",

diff --git a/packages/controlflow/src/queues.js b/packages/controlflow/src/queues.js
@@ -29,8 +29,6 @@ export const RestApiQueue = (qopts={}) => {
 export const QueueInit = (key, qopts={}) => {
   if($?.[key]) return $[key]
   const connection = RedisConnectionDetails()
-  log.debug(connection)
-  log.debug(qopts)
   qopts = { connection, ...qopts }
   const $Queue = new Queue(key, qopts)
   const $QueueEvents = new QueueEvents($Queue.name, { connection } )