From 0e2d398fb7e281cc52117705a9c2353571934ec7 Mon Sep 17 00:00:00 2001 From: dskvr Date: Mon, 1 Jan 2024 15:09:53 +0100 Subject: [PATCH 1/3] Update to NIP-66 draft4 --- packages/controlflow/src/retry.js | 4 + .../default/SslAdapterDefault/index.js | 1 + packages/nocapd/.gitignore | 3 +- packages/nocapd/src/classes/NocapdQueues.js | 8 -- packages/nocapd/src/classes/WorkerManager.js | 2 +- packages/nocapd/src/daemon.js | 3 - packages/nocapd/src/managers/AllManager.js | 73 ++++++++++++------- packages/publisher/src/Publisher.js | 19 ++++- packages/publisher/src/kinds/Kind30066.js | 34 ++++----- packages/trawler/src/daemon.js | 1 + packages/trawler/src/queue.js | 6 +- 11 files changed, 96 insertions(+), 58 deletions(-) diff --git a/packages/controlflow/src/retry.js b/packages/controlflow/src/retry.js index aff705f4..edf62a60 100644 --- a/packages/controlflow/src/retry.js +++ b/packages/controlflow/src/retry.js @@ -40,6 +40,10 @@ export class RetryManager { return await rcache.retry.get(this.cacheId(url)) } + async getExpiry( url ){ + return this.expiry( await this.getRetries(url) ) + } + async setRetries( url, success ){ let id if(success) { diff --git a/packages/nocap/adapters/default/SslAdapterDefault/index.js b/packages/nocap/adapters/default/SslAdapterDefault/index.js index dfeefb05..718870d9 100644 --- a/packages/nocap/adapters/default/SslAdapterDefault/index.js +++ b/packages/nocap/adapters/default/SslAdapterDefault/index.js @@ -21,6 +21,7 @@ class SslAdapterDefault { result = { status: "success", data } this.$.finish('ssl', result) } + sslCheckerOptions(port){ return { method: "GET", port: port || 443 } } diff --git a/packages/nocapd/.gitignore b/packages/nocapd/.gitignore index 3c00e74e..b5f7ca43 100644 --- a/packages/nocapd/.gitignore +++ b/packages/nocapd/.gitignore @@ -1,3 +1,4 @@ node_modules config.y*ml -yarn.lock \ No newline at end of file +yarn.lock +.events \ No newline at end of file diff --git a/packages/nocapd/src/classes/NocapdQueues.js b/packages/nocapd/src/classes/NocapdQueues.js index f63b9b44..5df154af 100644 --- a/packages/nocapd/src/classes/NocapdQueues.js +++ b/packages/nocapd/src/classes/NocapdQueues.js @@ -48,12 +48,6 @@ export class NocapdQueues { else if (typeof job === 'string') name = job.split(':')[0] - // if(event === 'drained') - // console.log(`${event}!`, name, job?.id) - - // if(event === 'completed') - // console.log(`${event}!`, name, job?.id) - if(name) { const daemonManager = name.split('@')[0] const daemonPubkey = name.split('@')[1] @@ -84,9 +78,7 @@ export class NocapdQueues { } bind_events(){ - // if(!this.bindEvents) return this.worker_events.forEach(handler => { - // console.log(`bind on_${handler} event handler on ${this.worker.name}:${this.constructor.name}`) this.worker.on(handler, (...args) => this.route_event(handler, ...args)) }) } diff --git a/packages/nocapd/src/classes/WorkerManager.js b/packages/nocapd/src/classes/WorkerManager.js index d895e98a..c7df6703 100644 --- a/packages/nocapd/src/classes/WorkerManager.js +++ b/packages/nocapd/src/classes/WorkerManager.js @@ -274,7 +274,7 @@ export class WorkerManager { async isExpired(url, lastChecked) { const retries = await this.retry.getRetries(url); - const expiry = retries > 0 ? this.retry.expiry(retries) : this.expires; + const expiry = retries > 0 ? await this.retry.getExpiry(url) : this.expires; return lastChecked < Date.now() - expiry; } diff --git a/packages/nocapd/src/daemon.js b/packages/nocapd/src/daemon.js index 0b2f98da..286bb3db 100644 --- a/packages/nocapd/src/daemon.js +++ b/packages/nocapd/src/daemon.js @@ -109,9 +109,6 @@ export const Nocapd = async () => { await maybeBootstrap() const $q = await initWorker() return { - // stop: () => { - // log.warn('Stopping nocapd') - // }, $q } } \ No newline at end of file diff --git a/packages/nocapd/src/managers/AllManager.js b/packages/nocapd/src/managers/AllManager.js index 93bb23bc..16aadd40 100644 --- a/packages/nocapd/src/managers/AllManager.js +++ b/packages/nocapd/src/managers/AllManager.js @@ -1,13 +1,13 @@ import mapper from 'object-mapper' import ngeotags from 'nostr-geotags' - - import Publish from '@nostrwatch/publisher' import { WorkerManager } from '../classes/WorkerManager.js' const publish30066 = new Publish.Kind30066() + + export class AllManager extends WorkerManager { constructor($, rcache, opts){ super($, rcache, opts) @@ -23,7 +23,6 @@ export class AllManager extends WorkerManager { try { this.log.debug(`Running comprehensive check for ${job.data.relay}`) const { relay:url } = job.data - // console.log(await this.getLastChecked(url), Date.now(), (Date.now()-await this.getLastChecked(url))/1000/60) const dpubkey = this.pubkey const nocapOpts = { timeout: { @@ -56,7 +55,7 @@ export class AllManager extends WorkerManager { const { relay:url } = job.data const { result } = rvalue - if(!result) + if(!result || !result?.connect?.data) return this.on_failed(job, new Error(`Nocap complete (all) check failed for ${url}`)) const { checked_at } = result @@ -68,22 +67,17 @@ export class AllManager extends WorkerManager { const retry_id = await this.retry.setRetries( url, true ) const lastChecked_id = await this.setLastChecked( url, Date.now() ) - // console.log('success', await this.rcache.$.get(relay_id)?.online, await this.rcache.$.get(retry_id)?.v, await this.rcache.$.get(lastChecked_id)?.v) - const event30066Data = event30066DataFromResult( result ) - await publish30066.one(event30066Data) + await publish30066.one(event30066Data) } async on_failed(job, err){ const { relay:url } = job.data - // console.log('url:onfailed', url) this.log?.debug(`Websocket check failed for ${job.data.relay}: ${JSON.stringify(err)}`) const retry_id = await this.retry.setRetries(url, false) const lastChecked_id = await this.setLastChecked( url, Date.now() ) - // console.log('failed', await this.rcache.$.get(retry_id)?.v, await this.rcache.$.get(lastChecked_id)?.v) this.progressMessage(url, null, true) this.processed++ - } } @@ -106,7 +100,11 @@ const truncatedResult = (result, type) => { const event30066DataFromResult = result => { const eventData = {} - const attributes = [] + const labels = [] + const nips = [] + + const dns = result.dns?.data || {} + const isDns = Object.keys(dns)?.length > 0 const geo = transformGeoResult(result.geo?.data) || {} const isGeo = Object.keys(geo)?.length > 0 @@ -122,41 +120,65 @@ const event30066DataFromResult = result => { eventData.rtt = [] + if(result?.network) + eventData.network = result.network + if(result?.connect?.duration > 0) - eventData.rtt.push({ type: 'open', rtt: result.connect.duration }) + eventData.rtt.push([ 'open', result.connect.duration ]) if(result?.read?.duration > 0) - eventData.rtt.push({ type: 'subscribe', rtt: result.read.duration }) + eventData.rtt.push([ 'subscribe', result.read.duration ]) if(result?.write?.duration > 0) - eventData.rtt.push({ type: 'publish', rtt: result.write.duration }) + eventData.rtt.push([ 'publish', result.write.duration ]) if(eventData.retries > 0) eventData.retries = result.retries if(isGeo) - eventData.geo = ngeotags(geo, { iso31662: true }) + eventData.geo = ngeotags(geo, { iso31662: true, iso3163: true }) if(isInfo){ - if(info?.limitations?.payment_required === true) - attributes.push('payment-required') - if(info?.limitations?.auth_required === true) - attributes.push('auth-required') + if(info?.limitation?.payment_required === true) + labels.push(['nip11.limitation', 'payment-required']) + if(info?.limitation?.auth_required === true) + labels.push(['nip11.limitation', 'auth-required']) + if(info?.pubkey) + labels.push(['nip11.pubkey', info.pubkey]) + if(info?.contact) + labels.push(['nip11.contact', info.contact]) + if(info?.name) + labels.push(['nip11.name', info.name]) + if(info?.software) + labels.push(['nip11.software', info.software]) + if(info?.version) + labels.push(['nip11.version', info.version]) if(info?.supported_nips instanceof Array) - info.supported_nips.forEach(nip => attributes.push(`nip-${nip}`)) + info.supported_nips.forEach(nip => { + nips.push(`${nip}`) + }) + if(info?.tags) + labels.push(['nip11.tags', ...info.tags]) + if(info?.language_tags) + labels.push(['nip11.language_tags', ...info.language_tags ]) } if(isSsl) - attributes.push(ssl?.valid === true? 'ssl-valid' :'ssl-invalid') + eventData.ssltag = [ 'ssl', ssl?.valid === true? 'valid': 'invalid', `${new Date(ssl.valid_from).getTime()}`, `${new Date(ssl.valid_to).getTime()}` ] if(isGeo) if(geo?.as) - attributes.push(geo.as) + labels.push(['as', geo.as]) + if(geo?.asn) + labels.push(['asn', geo.asn]) if(geo?.ip) - attributes.push(geo.ip) + labels.push([`ipv4`, geo.ip]) + + if(labels.length) + eventData.labels = labels - if(attributes.length) - eventData.attributes = attributes + if(nips.length) + eventData.nips = nips return eventData } @@ -164,6 +186,7 @@ const event30066DataFromResult = result => { const transformGeoResult = geo => { const map = { "as": "as", + "asn": "asn", "city": "cityName", "countryCode": "countryCode", "regionName": "regionName", diff --git a/packages/publisher/src/Publisher.js b/packages/publisher/src/Publisher.js index 1b10d7c4..51dbe445 100644 --- a/packages/publisher/src/Publisher.js +++ b/packages/publisher/src/Publisher.js @@ -4,6 +4,23 @@ import { loadConfig, chunkArray } from '@nostrwatch/utils' const config = await loadConfig() +import fs from 'fs/promises'; + +async function writeObjectToFile(obj) { + + const filename = obj.tags.find( tag => tag[0] === 'd' )[1].replace('wss://', '').replace('ws://', '').replace('http://', '').replace('https://', '').replace('/', '') + + const fileName = `./.events/${filename}.json`; + const jsonContent = JSON.stringify(obj, null, 2); // pretty-print JSON + + try { + await fs.writeFile(fileName, jsonContent); + // console.log(`File ${fileName} has been written.`); + } catch (error) { + console.error('Error writing file:', error); + } +} + export class Publisher { constructor(){ @@ -81,6 +98,7 @@ export class Publisher { } async publishEvent(signedEvent){ + writeObjectToFile(signedEvent); const pool = new SimplePool(); const relays = config.publisher.to_relays let pubs = pool.publish(relays, signedEvent) @@ -93,6 +111,5 @@ export class Publisher { publishes.push( await this.publishEvent(signedEvent) ) } return publishes - } } \ No newline at end of file diff --git a/packages/publisher/src/kinds/Kind30066.js b/packages/publisher/src/kinds/Kind30066.js index 1042dfbc..d915526e 100644 --- a/packages/publisher/src/kinds/Kind30066.js +++ b/packages/publisher/src/kinds/Kind30066.js @@ -15,28 +15,28 @@ export class Kind30066 extends Publisher { tags.push(['d', relay?.url]) - if(typeof relay?.online === 'boolean') - tags.push(['s', relay?.online? 'online' : 'offline']) + if(relay?.network) + tags.push(['n', relay?.network]) - if(relay?.network){ - tags.push(['L', 'network']) - tags.push(['l', relay?.network, 'network']) - } + if(relay?.rtt && relay?.rtt instanceof Array) + relay.rtt.forEach( rtt => tags.push(['rtt', rtt.shift(), ...rtt.map(String)]) ) - if(relay?.rtt && relay?.rtt?.type && relay?.rtt?.data instanceof Array) - relay.rtt.forEach( rtt => tags.push([`rtt_${rtt.type}`, ...rtt.data]) ) + if(relay?.ssltag) + tags.push(relay.ssltag) if(relay?.geo) - if(typeof relay?.geo === 'string') - tags.push(['g', relay?.geo]) - else if(typeof relay?.geo === 'array') - relay?.geo.forEach( geo => tags.push(['g', geo]) ) - - if(relay?.attributes) - relay?.attributes.forEach( attribute => tags.push(['t', attribute]) ) + relay?.geo.forEach( geo => tags.push(geo) ) + + if(relay?.labels){ + relay.labels.forEach(labels => { + const key = labels.shift() + tags.push(['L', key]) + labels.forEach(label => tags.push(['l', label, key])) + }) + } - if(relay?.retries) - tags.push(['retries', relay?.retries]) + if(relay?.nips && relay?.nips instanceof Array) + relay.nips.forEach( nip => tags.push(['N', `${nip}`]) ) const event = { ...eventTpl, diff --git a/packages/trawler/src/daemon.js b/packages/trawler/src/daemon.js index e8fb3d22..75277099 100644 --- a/packages/trawler/src/daemon.js +++ b/packages/trawler/src/daemon.js @@ -49,6 +49,7 @@ export default async () => { const relays = seed[0] const updatedAt = seed[1] await populateTrawler( relays ) + trawlWorker.on('drained', () => populateTrawler( relays ) ) resolve({ queues: { trawlQueue }, watcher: null }) }) } \ No newline at end of file diff --git a/packages/trawler/src/queue.js b/packages/trawler/src/queue.js index cd1f4912..4bd951fe 100644 --- a/packages/trawler/src/queue.js +++ b/packages/trawler/src/queue.js @@ -33,11 +33,13 @@ export const configureQueues = async function(){ } } - const trawlQueueDrained = () => {} + const trawlQueueDrained = () => { + + } trawler.$Queue.drain() - trawler.$Queue.on('drained', trawlQueueDrained) + // trawler.$Queue.on('drained', trawlQueueDrained) // trawler.$QueueEvents.on('progress', trawlJobProgress) const trawlJobCompleted = async ($job, foundRelays) => { From 58824a2a59b855d721c380ef8fd9b1b00a5dd257 Mon Sep 17 00:00:00 2001 From: dskvr Date: Tue, 2 Jan 2024 10:06:33 +0100 Subject: [PATCH 2/3] fix unhandled error --- packages/nocap/adapters/default/GeoAdapterDefault/index.js | 2 +- packages/nocapd/package.json | 1 - packages/publisher/src/Publisher.js | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/nocap/adapters/default/GeoAdapterDefault/index.js b/packages/nocap/adapters/default/GeoAdapterDefault/index.js index 7733ff52..d49b6cb7 100644 --- a/packages/nocap/adapters/default/GeoAdapterDefault/index.js +++ b/packages/nocap/adapters/default/GeoAdapterDefault/index.js @@ -19,7 +19,7 @@ import { fetch } from 'cross-fetch' else endpoint = `http://ip-api.com/json/${ip}?fields=${fields}` const headers = { 'accept': 'application/json' } - const response = await fetch(endpoint, { headers }).catch(e => err=e) + const response = await fetch(endpoint, { headers }).catch(this.$.logger.warn) delete response.query delete response.status const result = { status: "success", data: await response.json() } diff --git a/packages/nocapd/package.json b/packages/nocapd/package.json index 0a0b38e5..6b4374bb 100644 --- a/packages/nocapd/package.json +++ b/packages/nocapd/package.json @@ -8,7 +8,6 @@ "@nostr-fetch/adapter-nostr-tools": "0.14.1", "chalk": "5.3.0", "nostr-fetch": "0.14.1", - "nostr-geotags": "0.0.7", "object-mapper": "6.2.0" }, "scripts": { diff --git a/packages/publisher/src/Publisher.js b/packages/publisher/src/Publisher.js index 51dbe445..4bcf2aa0 100644 --- a/packages/publisher/src/Publisher.js +++ b/packages/publisher/src/Publisher.js @@ -98,7 +98,7 @@ export class Publisher { } async publishEvent(signedEvent){ - writeObjectToFile(signedEvent); + // writeObjectToFile(signedEvent); const pool = new SimplePool(); const relays = config.publisher.to_relays let pubs = pool.publish(relays, signedEvent) From 824690de2c235b6b1112a1a61b55ebf2ec2df6a8 Mon Sep 17 00:00:00 2001 From: sandwich <299465+dskvr@users.noreply.github.com> Date: Thu, 11 Jan 2024 18:28:15 +0100 Subject: [PATCH 3/3] move apps, update publish --- .github/workflows/publish-package.yml | 17 +++++++++-- .gitignore | 2 +- {packages => apps}/nocapd/.gitignore | 0 {packages => apps}/nocapd/.gitkeep | 0 {packages => apps}/nocapd/docker-compose.yaml | 0 {packages => apps}/nocapd/package.json | 0 .../nocapd/scripts/genkeypair.js | 0 .../nocapd/src/classes/NocapWrapper.js | 0 .../nocapd/src/classes/NocapdQueues.js | 0 .../nocapd/src/classes/WorkerManager.js | 0 {packages => apps}/nocapd/src/daemon.js | 0 {packages => apps}/nocapd/src/index.js | 0 .../nocapd/src/managers/AllManager.js | 5 ++-- {packages => apps}/nocapd/src/managers/dns.js | 0 {packages => apps}/nocapd/src/managers/geo.js | 0 .../nocapd/src/managers/info.js | 0 {packages => apps}/nocapd/src/managers/ssl.js | 0 .../nocapd/src/managers/websocket.js | 0 .../nocapd/src/managers/welcome.js | 0 {packages => apps}/trawler/.dockerignore | 0 {packages => apps}/trawler/.env.sample | 0 {packages => apps}/trawler/.gitignore | 0 {packages => apps}/trawler/Dockerfile | 0 {packages => apps}/trawler/README.md | 0 {packages => apps}/trawler/config.yaml | 0 {packages => apps}/trawler/nginx.conf | 0 {packages => apps}/trawler/package.json | 0 .../scripts/gen-keypair-save-to-env.js | 0 {packages => apps}/trawler/src/check-cache.js | 0 {packages => apps}/trawler/src/config.js | 0 {packages => apps}/trawler/src/daemon.js | 0 {packages => apps}/trawler/src/index.js | 0 {packages => apps}/trawler/src/parsers.js | 0 {packages => apps}/trawler/src/publish.js | 0 {packages => apps}/trawler/src/queue.js | 0 {packages => apps}/trawler/src/relaydb.js | 0 {packages => apps}/trawler/src/sanitizers.js | 0 {packages => apps}/trawler/src/sync.js | 0 {packages => apps}/trawler/src/trawler.js | 0 {packages => apps}/trawler/src/utils.js | 0 packages/history-relay/.gitignore | 4 --- packages/history-relay/docker-compose.yaml | 11 ------- packages/history-relay/whitelist.js | 29 ------------------- packages/publisher/src/kinds/Kind10066.js | 4 +-- packages/publisher/src/kinds/Kind30066.js | 6 ++-- 45 files changed, 23 insertions(+), 55 deletions(-) rename {packages => apps}/nocapd/.gitignore (100%) rename {packages => apps}/nocapd/.gitkeep (100%) rename {packages => apps}/nocapd/docker-compose.yaml (100%) rename {packages => apps}/nocapd/package.json (100%) rename {packages => apps}/nocapd/scripts/genkeypair.js (100%) rename {packages => apps}/nocapd/src/classes/NocapWrapper.js (100%) rename {packages => apps}/nocapd/src/classes/NocapdQueues.js (100%) rename {packages => apps}/nocapd/src/classes/WorkerManager.js (100%) rename {packages => apps}/nocapd/src/daemon.js (100%) rename {packages => apps}/nocapd/src/index.js (100%) rename {packages => apps}/nocapd/src/managers/AllManager.js (97%) rename {packages => apps}/nocapd/src/managers/dns.js (100%) rename {packages => apps}/nocapd/src/managers/geo.js (100%) rename {packages => apps}/nocapd/src/managers/info.js (100%) rename {packages => apps}/nocapd/src/managers/ssl.js (100%) rename {packages => apps}/nocapd/src/managers/websocket.js (100%) rename {packages => apps}/nocapd/src/managers/welcome.js (100%) rename {packages => apps}/trawler/.dockerignore (100%) rename {packages => apps}/trawler/.env.sample (100%) rename {packages => apps}/trawler/.gitignore (100%) rename {packages => apps}/trawler/Dockerfile (100%) rename {packages => apps}/trawler/README.md (100%) rename {packages => apps}/trawler/config.yaml (100%) rename {packages => apps}/trawler/nginx.conf (100%) rename {packages => apps}/trawler/package.json (100%) rename {packages => apps}/trawler/scripts/gen-keypair-save-to-env.js (100%) rename {packages => apps}/trawler/src/check-cache.js (100%) rename {packages => apps}/trawler/src/config.js (100%) rename {packages => apps}/trawler/src/daemon.js (100%) rename {packages => apps}/trawler/src/index.js (100%) rename {packages => apps}/trawler/src/parsers.js (100%) rename {packages => apps}/trawler/src/publish.js (100%) rename {packages => apps}/trawler/src/queue.js (100%) rename {packages => apps}/trawler/src/relaydb.js (100%) rename {packages => apps}/trawler/src/sanitizers.js (100%) rename {packages => apps}/trawler/src/sync.js (100%) rename {packages => apps}/trawler/src/trawler.js (100%) rename {packages => apps}/trawler/src/utils.js (100%) delete mode 100644 packages/history-relay/.gitignore delete mode 100644 packages/history-relay/docker-compose.yaml delete mode 100644 packages/history-relay/whitelist.js diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index 01e2498a..e3ecbcff 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -12,8 +12,19 @@ jobs: strategy: matrix: include: + - package: 'logger' + nodeVersion: '20' - package: 'nocap' - nodeVersion: '18' + nodeVersion: '20' + - package: 'seed' + nodeVersion: '20' + - package: 'utils' + nodeVersion: '20' + - package: 'nwcache' + nodeVersion: '20' + - package: 'controlflow' + nodeVersion: '20' + steps: - uses: actions/checkout@v3 @@ -24,7 +35,7 @@ jobs: - name: Change to package directory run: cd packages/${{ matrix.package }} - + - run: yarn install - run: yarn test @@ -45,4 +56,4 @@ jobs: release_name: v${{ matrix.package }}-${{ steps.publish.outputs.version }} body: "" draft: false - prerelease: true + prerelease: true \ No newline at end of file diff --git a/.gitignore b/.gitignore index 49eb55b5..add95169 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,4 @@ ours.sh .pg packages/synx packages/kinds -.npmrc \ No newline at end of file +.npmrc diff --git a/packages/nocapd/.gitignore b/apps/nocapd/.gitignore similarity index 100% rename from packages/nocapd/.gitignore rename to apps/nocapd/.gitignore diff --git a/packages/nocapd/.gitkeep b/apps/nocapd/.gitkeep similarity index 100% rename from packages/nocapd/.gitkeep rename to apps/nocapd/.gitkeep diff --git a/packages/nocapd/docker-compose.yaml b/apps/nocapd/docker-compose.yaml similarity index 100% rename from packages/nocapd/docker-compose.yaml rename to apps/nocapd/docker-compose.yaml diff --git a/packages/nocapd/package.json b/apps/nocapd/package.json similarity index 100% rename from packages/nocapd/package.json rename to apps/nocapd/package.json diff --git a/packages/nocapd/scripts/genkeypair.js b/apps/nocapd/scripts/genkeypair.js similarity index 100% rename from packages/nocapd/scripts/genkeypair.js rename to apps/nocapd/scripts/genkeypair.js diff --git a/packages/nocapd/src/classes/NocapWrapper.js b/apps/nocapd/src/classes/NocapWrapper.js similarity index 100% rename from packages/nocapd/src/classes/NocapWrapper.js rename to apps/nocapd/src/classes/NocapWrapper.js diff --git a/packages/nocapd/src/classes/NocapdQueues.js b/apps/nocapd/src/classes/NocapdQueues.js similarity index 100% rename from packages/nocapd/src/classes/NocapdQueues.js rename to apps/nocapd/src/classes/NocapdQueues.js diff --git a/packages/nocapd/src/classes/WorkerManager.js b/apps/nocapd/src/classes/WorkerManager.js similarity index 100% rename from packages/nocapd/src/classes/WorkerManager.js rename to apps/nocapd/src/classes/WorkerManager.js diff --git a/packages/nocapd/src/daemon.js b/apps/nocapd/src/daemon.js similarity index 100% rename from packages/nocapd/src/daemon.js rename to apps/nocapd/src/daemon.js diff --git a/packages/nocapd/src/index.js b/apps/nocapd/src/index.js similarity index 100% rename from packages/nocapd/src/index.js rename to apps/nocapd/src/index.js diff --git a/packages/nocapd/src/managers/AllManager.js b/apps/nocapd/src/managers/AllManager.js similarity index 97% rename from packages/nocapd/src/managers/AllManager.js rename to apps/nocapd/src/managers/AllManager.js index 16aadd40..3ba2a31c 100644 --- a/packages/nocapd/src/managers/AllManager.js +++ b/apps/nocapd/src/managers/AllManager.js @@ -12,7 +12,7 @@ export class AllManager extends WorkerManager { constructor($, rcache, opts){ super($, rcache, opts) this.interval = 60*1000 //checks for expired items every... - this.timeout = 6*1000 + this.timeout = 9*1000 this.timeoutBuffer = 1000 } @@ -74,8 +74,9 @@ export class AllManager extends WorkerManager { async on_failed(job, err){ const { relay:url } = job.data this.log?.debug(`Websocket check failed for ${job.data.relay}: ${JSON.stringify(err)}`) - const retry_id = await this.retry.setRetries(url, false) + const retry_id = await this.retry.setRetries( url, false ) const lastChecked_id = await this.setLastChecked( url, Date.now() ) + const relay_id = await this.updateRelayCache({ url, connect: { data: false }} ) this.progressMessage(url, null, true) this.processed++ } diff --git a/packages/nocapd/src/managers/dns.js b/apps/nocapd/src/managers/dns.js similarity index 100% rename from packages/nocapd/src/managers/dns.js rename to apps/nocapd/src/managers/dns.js diff --git a/packages/nocapd/src/managers/geo.js b/apps/nocapd/src/managers/geo.js similarity index 100% rename from packages/nocapd/src/managers/geo.js rename to apps/nocapd/src/managers/geo.js diff --git a/packages/nocapd/src/managers/info.js b/apps/nocapd/src/managers/info.js similarity index 100% rename from packages/nocapd/src/managers/info.js rename to apps/nocapd/src/managers/info.js diff --git a/packages/nocapd/src/managers/ssl.js b/apps/nocapd/src/managers/ssl.js similarity index 100% rename from packages/nocapd/src/managers/ssl.js rename to apps/nocapd/src/managers/ssl.js diff --git a/packages/nocapd/src/managers/websocket.js b/apps/nocapd/src/managers/websocket.js similarity index 100% rename from packages/nocapd/src/managers/websocket.js rename to apps/nocapd/src/managers/websocket.js diff --git a/packages/nocapd/src/managers/welcome.js b/apps/nocapd/src/managers/welcome.js similarity index 100% rename from packages/nocapd/src/managers/welcome.js rename to apps/nocapd/src/managers/welcome.js diff --git a/packages/trawler/.dockerignore b/apps/trawler/.dockerignore similarity index 100% rename from packages/trawler/.dockerignore rename to apps/trawler/.dockerignore diff --git a/packages/trawler/.env.sample b/apps/trawler/.env.sample similarity index 100% rename from packages/trawler/.env.sample rename to apps/trawler/.env.sample diff --git a/packages/trawler/.gitignore b/apps/trawler/.gitignore similarity index 100% rename from packages/trawler/.gitignore rename to apps/trawler/.gitignore diff --git a/packages/trawler/Dockerfile b/apps/trawler/Dockerfile similarity index 100% rename from packages/trawler/Dockerfile rename to apps/trawler/Dockerfile diff --git a/packages/trawler/README.md b/apps/trawler/README.md similarity index 100% rename from packages/trawler/README.md rename to apps/trawler/README.md diff --git a/packages/trawler/config.yaml b/apps/trawler/config.yaml similarity index 100% rename from packages/trawler/config.yaml rename to apps/trawler/config.yaml diff --git a/packages/trawler/nginx.conf b/apps/trawler/nginx.conf similarity index 100% rename from packages/trawler/nginx.conf rename to apps/trawler/nginx.conf diff --git a/packages/trawler/package.json b/apps/trawler/package.json similarity index 100% rename from packages/trawler/package.json rename to apps/trawler/package.json diff --git a/packages/trawler/scripts/gen-keypair-save-to-env.js b/apps/trawler/scripts/gen-keypair-save-to-env.js similarity index 100% rename from packages/trawler/scripts/gen-keypair-save-to-env.js rename to apps/trawler/scripts/gen-keypair-save-to-env.js diff --git a/packages/trawler/src/check-cache.js b/apps/trawler/src/check-cache.js similarity index 100% rename from packages/trawler/src/check-cache.js rename to apps/trawler/src/check-cache.js diff --git a/packages/trawler/src/config.js b/apps/trawler/src/config.js similarity index 100% rename from packages/trawler/src/config.js rename to apps/trawler/src/config.js diff --git a/packages/trawler/src/daemon.js b/apps/trawler/src/daemon.js similarity index 100% rename from packages/trawler/src/daemon.js rename to apps/trawler/src/daemon.js diff --git a/packages/trawler/src/index.js b/apps/trawler/src/index.js similarity index 100% rename from packages/trawler/src/index.js rename to apps/trawler/src/index.js diff --git a/packages/trawler/src/parsers.js b/apps/trawler/src/parsers.js similarity index 100% rename from packages/trawler/src/parsers.js rename to apps/trawler/src/parsers.js diff --git a/packages/trawler/src/publish.js b/apps/trawler/src/publish.js similarity index 100% rename from packages/trawler/src/publish.js rename to apps/trawler/src/publish.js diff --git a/packages/trawler/src/queue.js b/apps/trawler/src/queue.js similarity index 100% rename from packages/trawler/src/queue.js rename to apps/trawler/src/queue.js diff --git a/packages/trawler/src/relaydb.js b/apps/trawler/src/relaydb.js similarity index 100% rename from packages/trawler/src/relaydb.js rename to apps/trawler/src/relaydb.js diff --git a/packages/trawler/src/sanitizers.js b/apps/trawler/src/sanitizers.js similarity index 100% rename from packages/trawler/src/sanitizers.js rename to apps/trawler/src/sanitizers.js diff --git a/packages/trawler/src/sync.js b/apps/trawler/src/sync.js similarity index 100% rename from packages/trawler/src/sync.js rename to apps/trawler/src/sync.js diff --git a/packages/trawler/src/trawler.js b/apps/trawler/src/trawler.js similarity index 100% rename from packages/trawler/src/trawler.js rename to apps/trawler/src/trawler.js diff --git a/packages/trawler/src/utils.js b/apps/trawler/src/utils.js similarity index 100% rename from packages/trawler/src/utils.js rename to apps/trawler/src/utils.js diff --git a/packages/history-relay/.gitignore b/packages/history-relay/.gitignore deleted file mode 100644 index e75e72c6..00000000 --- a/packages/history-relay/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.env -node_modules -strfry.conf -strfry-db diff --git a/packages/history-relay/docker-compose.yaml b/packages/history-relay/docker-compose.yaml deleted file mode 100644 index 8f43b162..00000000 --- a/packages/history-relay/docker-compose.yaml +++ /dev/null @@ -1,11 +0,0 @@ -version: "3" -services: - strfry: - container_name: strfry - image: dockurr/strfry:latest - ports: - - 7777:7777 - volumes: - - ./strfry-db:/app/strfry-db - - ./strfry.conf:/etc/strfry.conf - - ./whitelist.js:/app/whitelist.js \ No newline at end of file diff --git a/packages/history-relay/whitelist.js b/packages/history-relay/whitelist.js deleted file mode 100644 index e3bd6d79..00000000 --- a/packages/history-relay/whitelist.js +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env node - -const kinds = { - '10002': true -}; - -const rl = require('readline').createInterface({ - input: process.stdin, - output: process.stdout, - terminal: false -}); - -rl.on('line', (line) => { - let req = JSON.parse(line); - - if (req.type !== 'new') { - console.error("unexpected request type"); // will appear in strfry logs - return; - } - - let res = { id: req.event.id }; // must echo the event's id - - if (kinds?.[req.event.kind]) { - res.action = 'accept'; - } else { - res.action = 'reject'; - res.msg = 'blocked: not kind 10002'; - } -}); \ No newline at end of file diff --git a/packages/publisher/src/kinds/Kind10066.js b/packages/publisher/src/kinds/Kind10066.js index bac7c2d4..7925d82f 100644 --- a/packages/publisher/src/kinds/Kind10066.js +++ b/packages/publisher/src/kinds/Kind10066.js @@ -6,11 +6,11 @@ export class Kind10066 extends Publisher { this.kind = 10066 } - generateEvent(publisher){ + generateEvent(relay){ const eventTpl = this.tpl() const tags = [] - const { url, kinds, parameters, geo, } = publisher + // const { url, kinds, parameters, geo, } = publisher if(kinds) kinds.forEach( kind => tags.push(['kind', kind]) ) diff --git a/packages/publisher/src/kinds/Kind30066.js b/packages/publisher/src/kinds/Kind30066.js index d915526e..29d0526d 100644 --- a/packages/publisher/src/kinds/Kind30066.js +++ b/packages/publisher/src/kinds/Kind30066.js @@ -24,9 +24,6 @@ export class Kind30066 extends Publisher { if(relay?.ssltag) tags.push(relay.ssltag) - if(relay?.geo) - relay?.geo.forEach( geo => tags.push(geo) ) - if(relay?.labels){ relay.labels.forEach(labels => { const key = labels.shift() @@ -35,6 +32,9 @@ export class Kind30066 extends Publisher { }) } + if(relay?.geo) + relay?.geo.forEach( geo => tags.push(geo) ) + if(relay?.nips && relay?.nips instanceof Array) relay.nips.forEach( nip => tags.push(['N', `${nip}`]) )