safety increase performance

apps/nocapd/src/classes/WorkerManager.js

@@ -73,6 +73,8 @@ export class WorkerManager {
     this.processed = 0
 
     this.total = 0
+
+    this.relayMeta = new Map()
   }
 
   calculateProgress() {
@@ -86,11 +88,11 @@ export class WorkerManager {
     const success = chalk.bold.green;
     const mute = chalk.gray
     this.log.info(
-      `[${chalk.bgBlack(this.calculateProgress())}]`, 
-        `${mute(this.processed)}/${mute(this.total)}`,
-      `${url}:`, 
-      result?.connect?.data? success("online"): failure("offline")),
-      error? chalk.gray.italic('error'): ''
+      `[${chalk.bgBlack(this.calculateProgress())}] `+
+      `${mute(this.processed)}/${mute(this.total)}  `+
+      `${url}: ${result?.connect?.data? success("online"): failure("offline")}  `+
+      `${(result?.connect?.duration+result?.read?.duration+result?.write?.duration)/1000} seconds  `+
+      `${error? chalk.gray.italic('error'): ''}`)
   }
 
   siblingKeys(){
@@ -175,15 +177,40 @@ export class WorkerManager {
   }
 
   async addRelayJob(jdata){
+    const priority = this.getPriority(jdata.relay)
     const jobOpts = {
-      priority: this.priority,
+      priority: priority,
       removeOnComplete: true,
       removeOnFail: true
     }
     this.log.debug(`Adding job for ${this.slug()}: ${JSON.stringify(jdata)}`)
     return this.$.queue.add( this.id(), jdata, { jobId: this.jobId(jdata.relay), ...jobOpts})
   }
 
+  getPriority(relay){
+    const {group, retries} = this.relayMeta.get(relay)
+    if(group === 'online')
+      return 1
+    if(group === 'unchecked')
+      return 10 
+    if(group === 'expired'){
+      if(!retries)
+        return 50
+      if(retries > 16)
+        return 100
+      else if(retries > 12)
+        return 80
+      else if(retries > 8)
+        return 70
+      else if(retries > 6)
+       return 65
+      else if( retries > 3)
+        return 55
+      else 
+        return 50
+    }
+  }
+
   cacheId(url){
     return lastCheckedId(this.shortname, url)
   }
@@ -230,32 +257,46 @@ export class WorkerManager {
     const allRelays = await this.rcache.relay.get.all();
     const onlineRelays = [];
     const uncheckedRelays = [];
-    const expiredRelays = [];
+    let expiredRelays = [];
+
+    this.relayMeta = new Map()
+
     for (const relay of allRelays) {
       const lastChecked = await this.rcache.cachetime.get.one(this.cacheId(relay.url));
+      const retries = await this.retry.getRetries(relay.url);
       const isExpired = await this.isExpired(relay.url, lastChecked);
-      const isOnline = relay?.online === true
-
+      const isOnline = relay?.online === true;
+
+      let group = '';
       if (isOnline && isExpired) {
         onlineRelays.push(relay.url);
+        group = 'online';
       } else if (!lastChecked) {
         uncheckedRelays.push(relay.url);
+        group = 'unchecked';
       } else if (isExpired) {
-        expiredRelays.push({ url: relay.url, lastChecked });
+        expiredRelays.push({ url: relay.url, lastChecked, retries });
+        group = 'expired';
       }
+
+      this.relayMeta.set(relay.url, { group, retries: retries > 0 ? retries : undefined });
     }
-    expiredRelays.sort((a, b) => a.lastChecked - b.lastChecked);
+
+    expiredRelays = expiredRelays.sort((a, b) => a.retries - b.retries);
+
     this.log.info(`online: ${await this.rcache.relay.get.online()?.length}, \
     online & expired: ${onlineRelays.length}, \
     expired: ${expiredRelays.length}, \
     unchecked: ${uncheckedRelays.filter(this.qualifyNetwork.bind(this)).length}, \
-    total: ${allRelays.length}`)
-    const deduped = [...onlineRelays, ...uncheckedRelays, ...expiredRelays.map(r => r.url)];
-
-    const relaysFiltered = deduped.filter(this.qualifyNetwork.bind(this))
-    const truncateLength = this.get_truncate_length(allRelays)
-    return relaysFiltered.slice(0, truncateLength)
+    total: ${allRelays.length}`);
+
+    const deduped = [...new Set([...onlineRelays, ...uncheckedRelays, ...expiredRelays.map(r => r.url)])];
+    const relaysFiltered = deduped.filter(this.qualifyNetwork.bind(this));
+    const truncateLength = this.get_truncate_length(allRelays);
+
+    return relaysFiltered.slice(0, truncateLength);
   }
+
 
   get_truncate_length(relays){
     let length = relays.length

apps/nocapd/src/daemon.js

@@ -75,7 +75,7 @@ const initWorker = async () => {
   await $NocapdQueue.pause()
   await $NocapdQueue.drain()
 
-  const $worker = new Worker($NocapdQueue.name, $q.route.bind($q), { concurrency: 1 } )
+  const $worker = new Worker($NocapdQueue.name, $q.route.bind($q), { concurrency: 10 } )
   await $worker.pause()
 
   $q.queue = $NocapdQueue

packages/nocap/src/classes/Base.js

@@ -59,14 +59,6 @@ export default class {
     this.logger.debug(`constructor(${url}, ${JSON.stringify(config)})`)
   }
 
-  isActive(){
-    return this.current === null? false: true 
-  }
-
-  async checkAll(){
-    return this.check('all')
-  }
-
   /**
    * check
    * Public method for dataprep and routing a check request
@@ -127,6 +119,18 @@ export default class {
     return result
   } 
 
+
+  /**
+   * can_check
+   * Determines if a check can be performed for a given key
+   *
+   * This method checks whether the current environment is a browser and the key is 'ssl'.
+   * If so, it logs an error indicating SSL checks cannot be performed from the browser.
+   * Otherwise, it returns true, allowing the check to proceed.
+   *
+   * @param {string} key - The key to verify if a check can be performed on
+   * @returns {boolean} - True if the check can be performed, otherwise false
+   */
   can_check(key){
     if(this.is_browser() && key === 'ssl') {
       this.logger.err('Cannot check SSL from browser')
@@ -135,6 +139,34 @@ export default class {
     return true 
   }
 
+  /**
+   * isActive
+   * Checks if the current object is active
+   *
+   * This method evaluates the 'current' property of the instance.
+   * If 'current' is null, the method returns false, indicating the object is not active.
+   * Otherwise, it returns true, indicating the object is active.
+   *
+   * @returns {boolean} - True if the current object is active, otherwise false
+   */
+  isActive(){
+    return this.current === null? false: true 
+  }
+
+  /**
+   * checkAll
+   * Asynchronously initiates a check for all items
+   *
+   * This method is a convenience wrapper that asynchronously triggers a check for 'all' items.
+   * It delegates the actual checking process to the 'check' method with 'all' as the argument.
+   *
+   * @async
+   * @returns {Promise<*>} - The result of the check for all items
+   */
+  async checkAll(){
+    return this.check('all')
+  }
+
   /**
    * maybeTimeoutReject
    * Creates a reject function for a timeout scenario