From 3f7f25a944b306e182cded75d45a56e84fa03616 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 2 May 2024 11:59:15 -0400 Subject: [PATCH] Remove log changes in backend/src/tasks --- backend/src/tasks/amass.ts | 9 ++-- backend/src/tasks/bastion.ts | 13 +++--- backend/src/tasks/censys.ts | 11 ++--- backend/src/tasks/censysCertificates.ts | 46 ++++++++----------- backend/src/tasks/censysIpv4.ts | 27 +++++------ backend/src/tasks/cve.ts | 26 ++++------- backend/src/tasks/dnstwist.ts | 9 ++-- backend/src/tasks/dotgov.ts | 8 ++-- backend/src/tasks/ecs-client.ts | 5 +- backend/src/tasks/es-client.ts | 29 ++++-------- backend/src/tasks/findomain.ts | 18 ++------ backend/src/tasks/functions.yml | 9 ++++ .../tasks/helpers/fetchPublicSuffixList.ts | 5 +- backend/src/tasks/hibp.ts | 7 ++- backend/src/tasks/intrigue-ident.ts | 11 ++--- backend/src/tasks/lambda-client.ts | 3 +- backend/src/tasks/lookingGlass.ts | 7 ++- backend/src/tasks/makeGlobalAdmin.ts | 7 +-- backend/src/tasks/portscanner.ts | 7 ++- backend/src/tasks/rootDomainSync.ts | 7 ++- backend/src/tasks/s3-client.ts | 10 ++-- backend/src/tasks/saved-search.ts | 7 ++- backend/src/tasks/scheduler.ts | 36 ++++++++------- backend/src/tasks/search-sync.ts | 11 ++--- backend/src/tasks/shodan.ts | 11 ++--- backend/src/tasks/sslyze.ts | 9 ++-- backend/src/tasks/syncdb.ts | 33 +++++++------ backend/src/tasks/trustymail.ts | 41 +++++++++-------- backend/src/tasks/updateScanTaskStatus.ts | 3 +- backend/src/tasks/wappalyzer.ts | 15 +++--- backend/src/tasks/webscraper.ts | 21 ++++----- 31 files changed, 214 insertions(+), 247 deletions(-) diff --git a/backend/src/tasks/amass.ts b/backend/src/tasks/amass.ts index 26422c09..34017d22 100644 --- a/backend/src/tasks/amass.ts +++ b/backend/src/tasks/amass.ts @@ -6,14 +6,13 @@ import { CommandOptions } from './ecs-client'; import getRootDomains from './helpers/getRootDomains'; import saveDomainsToDb from './helpers/saveDomainsToDb'; import * as path from 'path'; -import logger from '../tools/lambda-logger'; const OUT_PATH = path.join(__dirname, 'out-' + Math.random() + '.txt'); export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName, scanId } = commandOptions; - logger.info(`Running amass on organization ${organizationName}`); + console.log('Running amass on organization', organizationName); const rootDomains = await getRootDomains(organizationId!); @@ -28,7 +27,7 @@ export const handler = async (commandOptions: CommandOptions) => { '-json', OUT_PATH ]; - logger.info(`Running amass with args ${JSON.stringify(args)}`); + console.log('Running amass with args', args); spawnSync('amass', args, { stdio: 'pipe' }); const output = String(readFileSync(OUT_PATH)); const lines = output.split('\n'); @@ -49,9 +48,9 @@ export const handler = async (commandOptions: CommandOptions) => { ); } await saveDomainsToDb(domains); - logger.info(`amass created/updated ${domains.length} new domains`); + console.log(`amass created/updated ${domains.length} new domains`); } catch (e) { - logger.error(JSON.stringify(e)); + console.error(e); continue; } } diff --git a/backend/src/tasks/bastion.ts b/backend/src/tasks/bastion.ts index 0aa73645..ab6c0556 100644 --- a/backend/src/tasks/bastion.ts +++ b/backend/src/tasks/bastion.ts @@ -1,22 +1,21 @@ import { Handler } from 'aws-lambda'; import { connectToDatabase, User } from '../models'; import ESClient from '../tasks/es-client'; -import logger from '../tools/lambda-logger'; -export const handler: Handler = async (event, context) => { +export const handler: Handler = async (event) => { if (event.mode === 'db') { - const connection = await connectToDatabase(); + const connection = await connectToDatabase(true); const res = await connection.query(event.query); - logger.info(JSON.stringify(res), { context }); + console.log(res); } else if (event.mode === 'es') { if (event.query === 'delete') { const client = new ESClient(); await client.deleteAll(); - logger.info('Index successfully deleted', { context }); + console.log('Index successfully deleted'); } else { - logger.info(`Query not found: ${event.query}`, { context }); + console.log('Query not found: ' + event.query); } } else { - logger.info(`Mode not found: ${event.mode}`, { context }); + console.log('Mode not found: ' + event.mode); } }; diff --git a/backend/src/tasks/censys.ts b/backend/src/tasks/censys.ts index f610c248..83e92a92 100644 --- a/backend/src/tasks/censys.ts +++ b/backend/src/tasks/censys.ts @@ -5,7 +5,6 @@ import * as dns from 'dns'; import saveDomainsToDb from './helpers/saveDomainsToDb'; import { CommandOptions } from './ecs-client'; import getRootDomains from './helpers/getRootDomains'; -import logger from '../tools/lambda-logger'; interface CensysAPIResponse { result: { @@ -26,9 +25,9 @@ const sleep = (milliseconds: number) => { }; const fetchCensysData = async (rootDomain: string) => { - logger.info(`Fetching certificates for ${rootDomain}`); + console.log(`Fetching certificates for ${rootDomain}`); const data = await fetchPage(rootDomain); - logger.info( + console.log( `Censys found ${data.result.total} certificates for ${rootDomain} Fetching ${Math.min(data.result.total, resultLimit)} of them...` ); @@ -67,7 +66,7 @@ const fetchPage = async (rootDomain: string, nextToken?: string) => { export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName, scanId } = commandOptions; - logger.info(`Running Censys on: ${organizationName}`); + console.log(`Running Censys on: ${organizationName}`); const rootDomains = await getRootDomains(organizationId!); const uniqueNames = new Set(); //used to dedupe domain names @@ -108,7 +107,7 @@ export const handler = async (commandOptions: CommandOptions) => { // Project Sonar has both forward & reverse DNS for finding subdomains // Save domains to database - logger.info(`Saving ${organizationName} subdomains to database...`); + console.log(`Saving ${organizationName} subdomains to database...`); const domains: Domain[] = []; for (const domain of foundDomains) { let ip: string | null; @@ -130,7 +129,7 @@ export const handler = async (commandOptions: CommandOptions) => { ); } await saveDomainsToDb(domains); - logger.info( + console.log( `Censys saved or updated ${domains.length} subdomains for ${organizationName}` ); }; diff --git a/backend/src/tasks/censysCertificates.ts b/backend/src/tasks/censysCertificates.ts index 3b838332..f2bd369e 100644 --- a/backend/src/tasks/censysCertificates.ts +++ b/backend/src/tasks/censysCertificates.ts @@ -1,16 +1,10 @@ -import { - connectToDatabase, - Domain, - Organization, - Scan, - Service -} from '../models'; +import { connectToDatabase, Domain, Scan } from '../models'; import { plainToClass } from 'class-transformer'; import saveDomainsToDb from './helpers/saveDomainsToDb'; import { CommandOptions } from './ecs-client'; import { CensysCertificatesData } from '../models/generated/censysCertificates'; -import saveServicesToDb from './helpers/saveServicesToDb'; import getAllDomains from './helpers/getAllDomains'; +import sanitizeChunkValues from './helpers/sanitizeChunkValues'; import * as zlib from 'zlib'; import * as readline from 'readline'; import got from 'got'; @@ -18,7 +12,6 @@ import PQueue from 'p-queue'; import pRetry from 'p-retry'; import axios from 'axios'; import getScanOrganizations from './helpers/getScanOrganizations'; -import logger from '../tools/lambda-logger'; interface CommonNameToDomainsMap { [commonName: string]: Domain[]; @@ -42,10 +35,12 @@ const downloadPath = async ( path: string, commonNameToDomainsMap: CommonNameToDomainsMap, i: number, - numFiles: number, - commandOptions: CommandOptions + numFiles: number ): Promise => { - logger.info(`i: ${i} of ${numFiles}: starting download of url ${path}`); + if (i >= 100) { + throw new Error('Invalid chunk number.'); + } + console.log(`i: ${i} of ${numFiles}: starting download of url ${path}`); const domains: Domain[] = []; const gunzip = zlib.createGunzip(); @@ -105,11 +100,11 @@ const downloadPath = async ( readInterface.on('SIGTSTP', reject); }); if (!domains.length) { - logger.info( + console.log( `censysCertificates - processed file ${i} of ${numFiles}: got no results` ); } else { - logger.info( + console.log( `censysCertificates - processed file ${i} of ${numFiles}: got some results: ${domains.length} domains` ); } @@ -118,16 +113,14 @@ const downloadPath = async ( }; export const handler = async (commandOptions: CommandOptions) => { - const { chunkNumber, numChunks, organizationId } = commandOptions; + const { organizationId } = commandOptions; - if (chunkNumber === undefined || numChunks === undefined) { - throw new Error('Chunks not specified.'); - } + const { chunkNumber, numChunks } = await sanitizeChunkValues(commandOptions); const { data: { results } } = await pRetry(() => axios.get(CENSYS_CERTIFICATES_ENDPOINT, { auth }), { - // Perform less retries on jest to make tests faster + // Perform fewer retries on jest to make tests faster retries: typeof jest === 'undefined' ? 5 : 2, randomize: true }); @@ -135,7 +128,7 @@ export const handler = async (commandOptions: CommandOptions) => { const { data: { files } } = await pRetry(() => axios.get(results.latest.details_url, { auth }), { - // Perform less retries on jest to make tests faster + // Perform fewer retries on jest to make tests faster retries: typeof jest === 'undefined' ? 5 : 2, randomize: true }); @@ -162,9 +155,9 @@ export const handler = async (commandOptions: CommandOptions) => { const fileNames = Object.keys(files).sort(); const jobs: Promise[] = []; - let startIndex = Math.floor(((1.0 * chunkNumber) / numChunks) * numFiles); + let startIndex = Math.floor(((1.0 * chunkNumber!) / numChunks!) * numFiles); let endIndex = - Math.floor(((1.0 * (chunkNumber + 1)) / numChunks) * numFiles) - 1; + Math.floor(((1.0 * (chunkNumber! + 1)) / numChunks!) * numFiles) - 1; if (process.env.IS_LOCAL && typeof jest === 'undefined') { // For local testing. @@ -200,11 +193,10 @@ export const handler = async (commandOptions: CommandOptions) => { files[fileName].download_path, commonNameToDomainsMap, idx, - numFiles, - commandOptions + numFiles ), { - // Perform less retries on jest to make tests faster + // Perform fewer retries on jest to make tests faster retries: typeof jest === 'undefined' ? 5 : 2, randomize: true } @@ -212,8 +204,8 @@ export const handler = async (commandOptions: CommandOptions) => { ) ); } - logger.info(`censysCertificates: scheduled all tasks`); + console.log(`censysCertificates: scheduled all tasks`); await Promise.all(jobs); - logger.info(`censysCertificates done`); + console.log(`censysCertificates done`); }; diff --git a/backend/src/tasks/censysIpv4.ts b/backend/src/tasks/censysIpv4.ts index 51808451..6972d38c 100644 --- a/backend/src/tasks/censysIpv4.ts +++ b/backend/src/tasks/censysIpv4.ts @@ -13,7 +13,7 @@ import PQueue from 'p-queue'; import pRetry from 'p-retry'; import axios from 'axios'; import getScanOrganizations from './helpers/getScanOrganizations'; -import logger from '../tools/lambda-logger'; +import sanitizeChunkValues from './helpers/sanitizeChunkValues'; export interface IpToDomainsMap { [ip: string]: Domain[]; @@ -39,7 +39,10 @@ const downloadPath = async ( numFiles: number, commandOptions: CommandOptions ): Promise => { - logger.info(`i: ${i} of ${numFiles}: starting download of url ${path}`); + if (i >= 100) { + throw new Error('Invalid chunk number.'); + } + console.log(`i: ${i} of ${numFiles}: starting download of url ${path}`); const domains: Domain[] = []; const services: Service[] = []; @@ -106,11 +109,11 @@ const downloadPath = async ( readInterface.on('SIGTSTP', reject); }); if (!domains.length) { - logger.info( + console.log( `censysipv4 - processed file ${i} of ${numFiles}: got no results` ); } else { - logger.info( + console.log( `censysipv4 - processed file ${i} of ${numFiles}: got some results: ${domains.length} domains and ${services.length} services` ); } @@ -120,11 +123,9 @@ const downloadPath = async ( }; export const handler = async (commandOptions: CommandOptions) => { - const { chunkNumber, numChunks, organizationId } = commandOptions; + const { organizationId } = commandOptions; - if (chunkNumber === undefined || numChunks === undefined) { - throw new Error('Chunks not specified.'); - } + const { chunkNumber, numChunks } = await sanitizeChunkValues(commandOptions); const { data: { results } @@ -156,9 +157,9 @@ export const handler = async (commandOptions: CommandOptions) => { const fileNames = Object.keys(files).sort(); const jobs: Promise[] = []; - let startIndex = Math.floor(((1.0 * chunkNumber) / numChunks) * numFiles); + let startIndex = Math.floor(((1.0 * chunkNumber!) / numChunks!) * numFiles); let endIndex = - Math.floor(((1.0 * (chunkNumber + 1)) / numChunks) * numFiles) - 1; + Math.floor(((1.0 * (chunkNumber! + 1)) / numChunks!) * numFiles) - 1; if (process.env.IS_LOCAL && typeof jest === 'undefined') { // For local testing. @@ -192,7 +193,7 @@ export const handler = async (commandOptions: CommandOptions) => { commandOptions ), { - // Perform less retries on jest to make tests faster + // Perform fewer retries on jest to make tests faster retries: typeof jest === 'undefined' ? 5 : 2, randomize: true } @@ -200,8 +201,8 @@ export const handler = async (commandOptions: CommandOptions) => { ) ); } - logger.info(`censysipv4: scheduled all tasks`); + console.log(`censysipv4: scheduled all tasks`); await Promise.all(jobs); - logger.info(`censysipv4 done`); + console.log(`censysipv4 done`); }; diff --git a/backend/src/tasks/cve.ts b/backend/src/tasks/cve.ts index 1c6aac6c..bf1b2bf2 100644 --- a/backend/src/tasks/cve.ts +++ b/backend/src/tasks/cve.ts @@ -10,19 +10,11 @@ import { plainToClass } from 'class-transformer'; import { CommandOptions } from './ecs-client'; import * as buffer from 'buffer'; import saveVulnerabilitiesToDb from './helpers/saveVulnerabilitiesToDb'; -import { - LessThan, - MoreThan, - FindOperator, - In, - MoreThanOrEqual, - Not -} from 'typeorm'; +import { LessThan, MoreThan, In, MoreThanOrEqual, Not } from 'typeorm'; import * as fs from 'fs'; import * as zlib from 'zlib'; import axios, { AxiosResponse } from 'axios'; import { CISACatalogOfKnownExploitedVulnerabilities } from 'src/models/generated/kev'; -import logger from '../tools/lambda-logger'; /** * The CVE scan creates vulnerabilities based on existing @@ -134,21 +126,21 @@ const identifyPassiveCVEsFromCPEs = async (allDomains: Domain[]) => { }); if (hostsToCheck.length === 0) { - logger.warn('No hosts to check - no domains with CPEs found.'); + console.warn('No hosts to check - no domains with CPEs found.'); return; } const numBatches = hostsToCheck.length / CPE2CVE_BATCH_SIZE; for (let batch = 0; batch < numBatches; batch++) { let input = ''; - logger.info(`\tcpe2cve: starting batch ${batch} / ${numBatches}`); + console.log(`\tcpe2cve: starting batch ${batch} / ${numBatches}`); for ( let index = CPE2CVE_BATCH_SIZE * batch; index < Math.min(CPE2CVE_BATCH_SIZE * (batch + 1), hostsToCheck.length); index++ ) { input += `${index} ${hostsToCheck[index].cpes.join(',')}\n`; - logger.info(`\t${index} ${hostsToCheck[index].cpes.join(',')}`); + console.log(`\t${index} ${hostsToCheck[index].cpes.join(',')}`); } let res: Buffer; try { @@ -157,7 +149,7 @@ const identifyPassiveCVEsFromCPEs = async (allDomains: Domain[]) => { { input: input, maxBuffer: buffer.constants.MAX_LENGTH } ); } catch (e) { - logger.error(JSON.stringify(e)); + console.error(e); continue; } const split = String(res).split('\n'); @@ -307,7 +299,7 @@ const populateVulnerabilitiesFromNVD = async () => { for (const item of parsed.CVE_Items) { const cve = item.cve.CVE_data_meta.ID; if (vulnerabilitiesMap[cve]) { - logger.info(cve); + console.log(cve); const vulns = vulnerabilitiesMap[cve]; const description = item.cve.description.description_data.find( (desc) => desc.lang === 'en' @@ -347,7 +339,7 @@ const populateVulnerabilitiesFromKEV = async () => { } ); if (affected > 0) { - logger.info(`KEV ${kevVuln.cveID}: updated ${affected} vulns`); + console.log(`KEV ${kevVuln.cveID}: updated ${affected} vulns`); } } }; @@ -388,7 +380,7 @@ const reopenClosedVulnerabilities = async () => { }; export const handler = async (commandOptions: CommandOptions) => { - logger.info('Running cve detection globally'); + console.log('Running cve detection globally'); // CVE is a global scan; organizationId is used only for testing. const { organizationId } = commandOptions; @@ -412,7 +404,7 @@ export const handler = async (commandOptions: CommandOptions) => { break; } - logger.info(`Running batch ${batchNum}`); + console.log(`Running batch ${batchNum}`); await identifyPassiveCVEsFromCPEs(domains); diff --git a/backend/src/tasks/dnstwist.ts b/backend/src/tasks/dnstwist.ts index 391bbabc..bf8bc1f2 100644 --- a/backend/src/tasks/dnstwist.ts +++ b/backend/src/tasks/dnstwist.ts @@ -6,7 +6,6 @@ import saveVulnerabilitiesToDb from './helpers/saveVulnerabilitiesToDb'; import { spawnSync } from 'child_process'; import saveDomainsToDb from './helpers/saveDomainsToDb'; import * as dns from 'dns'; -import logger from '../tools/lambda-logger'; async function runDNSTwist(domain: string) { const child = spawnSync( @@ -19,7 +18,7 @@ async function runDNSTwist(domain: string) { ); const savedOutput = child.stdout; const finalResults = JSON.parse(savedOutput); - logger.info( + console.log( `Got ${ Object.keys(finalResults).length } similar domains for root domain ${domain}` @@ -31,10 +30,10 @@ export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName } = commandOptions; await connectToDatabase(); const dateNow = new Date(Date.now()); - logger.info(`Running dnstwist on organization ${organizationName}`); + console.log('Running dnstwist on organization', organizationName); const root_domains = await getRootDomains(organizationId!); const vulns: Vulnerability[] = []; - logger.info(`Root domains: ${root_domains}`); + console.log('Root domains:', root_domains); for (const root_domain of root_domains) { try { const results = await runDNSTwist(root_domain); @@ -105,7 +104,7 @@ export const handler = async (commandOptions: CommandOptions) => { continue; } } catch (e) { - logger.error(JSON.stringify(e)); + console.error(e); continue; } } diff --git a/backend/src/tasks/dotgov.ts b/backend/src/tasks/dotgov.ts index 1f7d8e70..fd686aca 100644 --- a/backend/src/tasks/dotgov.ts +++ b/backend/src/tasks/dotgov.ts @@ -2,7 +2,6 @@ import { connectToDatabase, Organization, OrganizationTag } from '../models'; import { CommandOptions } from './ecs-client'; import axios from 'axios'; import * as Papa from 'papaparse'; -import logger from '../tools/lambda-logger'; interface ParsedRow { 'Domain Name': string; @@ -61,7 +60,8 @@ export const handler = async (commandOptions: CommandOptions) => { orgsToRootDomainMap )) { // Create a new organization if needed; else, create the same one. - logger.info(orgName); + console.log(orgName); + console.log(row); const organization = (await Organization.findOne({ name: orgName @@ -77,8 +77,8 @@ export const handler = async (commandOptions: CommandOptions) => { // Replace existing rootDomains with new rootDomains. organization.rootDomains = [...rootDomains]; await organization.save(); - logger.info(`Finished org ${orgName}`); + console.log('Finished org', orgName); } - logger.info(`dotgov done`); + console.log(`dotgov done`); }; diff --git a/backend/src/tasks/ecs-client.ts b/backend/src/tasks/ecs-client.ts index 82879bb3..b628fb01 100644 --- a/backend/src/tasks/ecs-client.ts +++ b/backend/src/tasks/ecs-client.ts @@ -1,6 +1,5 @@ import { ECS, CloudWatchLogs } from 'aws-sdk'; import { SCAN_SCHEMA } from '../api/scans'; -import logger from '../tools/lambda-logger'; export interface CommandOptions { /** A list of organizations (id and name) that this @@ -87,6 +86,8 @@ class ECSClient { }, Env: [ `CROSSFEED_COMMAND_OPTIONS=${JSON.stringify(commandOptions)}`, + `CF_API_KEY=${process.env.CF_API_KEY}`, + `PE_API_KEY=${process.env.PE_API_KEY}`, `DB_DIALECT=${process.env.DB_DIALECT}`, `DB_HOST=${process.env.DB_HOST}`, `IS_LOCAL=true`, @@ -129,7 +130,7 @@ class ECSClient { failures: [] }; } catch (e) { - logger.error(e); + console.error(e); return { tasks: [], failures: [{}] diff --git a/backend/src/tasks/es-client.ts b/backend/src/tasks/es-client.ts index 0e157004..f470dd0f 100644 --- a/backend/src/tasks/es-client.ts +++ b/backend/src/tasks/es-client.ts @@ -1,9 +1,7 @@ import { Client } from '@elastic/elasticsearch'; import { Domain, Webpage } from '../models'; -import logger from '../tools/lambda-logger'; export const DOMAINS_INDEX = 'domains-5'; -const httpProxy = require('https-proxy-agent'); interface DomainRecord extends Domain { suggest: { input: string | string[]; weight: number }[]; @@ -39,16 +37,7 @@ class ESClient { client: Client; constructor(isLocal?: boolean) { - process.env.HTTPS_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - process.env.HTTP_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - - const proxyAgent = process.env.HTTPS_PROXY || process.env.HTTP_PROXY; - const agent = proxyAgent ? httpProxy(proxyAgent) : undefined; - - this.client = new Client({ - node: process.env.ELASTICSEARCH_ENDPOINT!, - agent: agent - }); + this.client = new Client({ node: process.env.ELASTICSEARCH_ENDPOINT! }); } /** @@ -81,10 +70,10 @@ class ESClient { index: DOMAINS_INDEX, body: { properties: mapping } }); - logger.info(`Index ${DOMAINS_INDEX} updated.`); + console.log(`Index ${DOMAINS_INDEX} updated.`); } catch (e) { if (e.meta?.body?.error.type !== 'index_not_found_exception') { - logger.error(e.meta?.body); + console.error(e.meta?.body); throw e; } await this.client.indices.create({ @@ -104,7 +93,7 @@ class ESClient { } } }); - logger.info(`Created index ${DOMAINS_INDEX}.`); + console.log(`Created index ${DOMAINS_INDEX}.`); } await this.client.indices.putSettings({ index: DOMAINS_INDEX, @@ -112,7 +101,7 @@ class ESClient { settings: { refresh_interval: '1800s' } } }); - logger.info(`Updated settings for index ${DOMAINS_INDEX}.`); + console.log(`Updated settings for index ${DOMAINS_INDEX}.`); } excludeFields = (domain: Domain) => { @@ -151,13 +140,13 @@ class ESClient { ]; }, onDrop(doc) { - logger.error(`${doc.error} ${doc.document}`); + console.error(doc.error, doc.document); b.abort(); } }); const result = await b; if (result.aborted) { - logger.error(result); + console.error(result); throw new Error('Bulk operation aborted'); } return result; @@ -191,13 +180,13 @@ class ESClient { ]; }, onDrop(doc) { - logger.error(`${doc.error} ${doc.document}`); + console.error(doc.error, doc.document); b.abort(); } }); const result = await b; if (result.aborted) { - logger.error(result); + console.error(result); throw new Error('Bulk operation aborted'); } return result; diff --git a/backend/src/tasks/findomain.ts b/backend/src/tasks/findomain.ts index c4d0ad78..440d8a56 100644 --- a/backend/src/tasks/findomain.ts +++ b/backend/src/tasks/findomain.ts @@ -6,14 +6,13 @@ import { CommandOptions } from './ecs-client'; import getRootDomains from './helpers/getRootDomains'; import saveDomainsToDb from './helpers/saveDomainsToDb'; import * as path from 'path'; -import logger from '../tools/lambda-logger'; const OUT_PATH = path.join(__dirname, 'out-' + Math.random() + '.txt'); export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName, scanId } = commandOptions; - logger.info(`Running findomain on organization ${organizationName}`); + console.log('Running findomain on organization', organizationName); const rootDomains = await getRootDomains(organizationId!); @@ -27,15 +26,8 @@ export const handler = async (commandOptions: CommandOptions) => { '-u', OUT_PATH ]; - logger.info(`Running findomain with args ${args}`); - spawnSync('findomain', args, { - stdio: 'pipe', - env: { - ...process.env, - HTTP_PROXY: process.env.HTTPS_PROXY, - HTTPS_PROXY: process.env.HTTPS_PROXY - } - }); + console.log('Running findomain with args', args); + spawnSync('findomain', args, { stdio: 'pipe' }); const output = String(readFileSync(OUT_PATH)); const lines = output.split('\n'); const domains: Domain[] = []; @@ -54,9 +46,9 @@ export const handler = async (commandOptions: CommandOptions) => { ); } await saveDomainsToDb(domains); - logger.info(`Findomain created/updated ${domains.length} new domains`); + console.log(`Findomain created/updated ${domains.length} new domains`); } catch (e) { - logger.error(JSON.stringify(e)); + console.error(e); continue; } } diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index c7f6c2f0..bd5ee23b 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -1,3 +1,12 @@ +--- +cloudwatchToS3: + handler: src/tasks/cloudwatchToS3.handler + timeout: 900 + events: + - schedule: rate(4 hours) + reservedConcurrency: 1 + memorySize: 4096 + scheduler: handler: src/tasks/scheduler.handler timeout: 900 diff --git a/backend/src/tasks/helpers/fetchPublicSuffixList.ts b/backend/src/tasks/helpers/fetchPublicSuffixList.ts index 7292d98b..07c8f1df 100644 --- a/backend/src/tasks/helpers/fetchPublicSuffixList.ts +++ b/backend/src/tasks/helpers/fetchPublicSuffixList.ts @@ -1,6 +1,5 @@ import * as fs from 'fs'; import fetch from 'node-fetch'; -import logger from '../../tools/lambda-logger'; // Checks for PSL at ${path} and fetches a new one if missing or out of date export default async (path = 'public_suffix_list.dat'): Promise => { @@ -13,7 +12,7 @@ export default async (path = 'public_suffix_list.dat'): Promise => { try { fs.writeFileSync(fd, await fetchPsl()); } catch (e) { - logger.error(e); + console.error(e); } finally { fs.closeSync(fd); } @@ -27,7 +26,7 @@ async function handleExistingPsl(path) { fs.writeFileSync(fd, await fetchPsl()); } } catch (e) { - logger.error(e); + console.error(e); } finally { fs.closeSync(fd); } diff --git a/backend/src/tasks/hibp.ts b/backend/src/tasks/hibp.ts index 4a595484..f9519a1a 100644 --- a/backend/src/tasks/hibp.ts +++ b/backend/src/tasks/hibp.ts @@ -3,7 +3,6 @@ import { CommandOptions } from './ecs-client'; import { plainToClass } from 'class-transformer'; import saveVulnerabilitiesToDb from './helpers/saveVulnerabilitiesToDb'; import axios from 'axios'; -import logger from '../tools/lambda-logger'; /** * The hibp scan looks up emails from a particular .gov domain @@ -91,7 +90,7 @@ async function lookupEmails( finalResults['Breaches'] = breachResults; return finalResults; } catch (error) { - logger.error( + console.error( `An error occured when trying to access the HIPB API using the domain: ${domain.name}: ${error} ` ); return null; @@ -101,7 +100,7 @@ async function lookupEmails( export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName } = commandOptions; - logger.info(`Running hibp on organization ${organizationName}`); + console.log('Running hibp on organization', organizationName); const domainsWithIPs = await getIps(organizationId); const { data } = await axios.get( 'https://haveibeenpwned.com/api/v2/breaches', @@ -122,7 +121,7 @@ export const handler = async (commandOptions: CommandOptions) => { const results = await lookupEmails(breachesDict, domain); if (results) { - logger.info( + console.log( `Got ${Object.keys(results['Emails']).length} emails for domain ${ domain.name }` diff --git a/backend/src/tasks/intrigue-ident.ts b/backend/src/tasks/intrigue-ident.ts index 48934b97..fc074d3d 100644 --- a/backend/src/tasks/intrigue-ident.ts +++ b/backend/src/tasks/intrigue-ident.ts @@ -3,10 +3,9 @@ import { getLiveWebsites, LiveDomain } from './helpers/getLiveWebsites'; import PQueue from 'p-queue'; import * as buffer from 'buffer'; import { spawnSync } from 'child_process'; -import logger from '../tools/lambda-logger'; const intrigueIdent = async (domain: LiveDomain): Promise => { - logger.info(`Domain ${domain.name}`); + console.log('Domain', domain.name); const { stdout, stderr, status } = spawnSync( 'intrigue-ident', ['--uri', domain.url, '--json'], @@ -20,10 +19,10 @@ const intrigueIdent = async (domain: LiveDomain): Promise => { } ); if (stderr?.toString()) { - logger.error(`stderr ${stderr.toString()}`); + console.error('stderr', stderr.toString()); } if (status !== 0) { - logger.error('IntrigueIdent failed'); + console.error('IntrigueIdent failed'); return; } const output = stdout.toString(); @@ -37,7 +36,7 @@ const intrigueIdent = async (domain: LiveDomain): Promise => { export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName } = commandOptions; - logger.info(`Running intrigueIdent on organization ${organizationName}`); + console.log('Running intrigueIdent on organization', organizationName); const liveWebsites = await getLiveWebsites(organizationId!); const queue = new PQueue({ concurrency: 5 }); @@ -45,5 +44,5 @@ export const handler = async (commandOptions: CommandOptions) => { liveWebsites.map((site) => queue.add(() => intrigueIdent(site))) ); - logger.info(`IntrigueIdent finished for ${liveWebsites.length} domains`); + console.log(`IntrigueIdent finished for ${liveWebsites.length} domains`); }; diff --git a/backend/src/tasks/lambda-client.ts b/backend/src/tasks/lambda-client.ts index 9ac849b2..81e727ec 100644 --- a/backend/src/tasks/lambda-client.ts +++ b/backend/src/tasks/lambda-client.ts @@ -1,6 +1,5 @@ import { Lambda } from 'aws-sdk'; import { handler as scheduler } from './scheduler'; -import logger from '../tools/lambda-logger'; /** * Lambda Client used to invoke lambda functions. @@ -25,7 +24,7 @@ class LambdaClient { }: { name: string; }): Promise { - logger.info(`Invoking lambda function ${name}`); + console.log('Invoking lambda function ', name); if (this.isLocal) { scheduler({}, {} as any, () => null); return { StatusCode: 202, Payload: '' }; diff --git a/backend/src/tasks/lookingGlass.ts b/backend/src/tasks/lookingGlass.ts index 4a5f626c..da37636b 100644 --- a/backend/src/tasks/lookingGlass.ts +++ b/backend/src/tasks/lookingGlass.ts @@ -10,7 +10,6 @@ import { plainToClass } from 'class-transformer'; import saveVulnerabilitiesToDb from './helpers/saveVulnerabilitiesToDb'; import { exit } from 'process'; import saveDomainsToDb from './helpers/saveDomainsToDb'; -import logger from '../tools/lambda-logger'; interface LGCollectionsResponse { children: any[]; @@ -202,7 +201,7 @@ async function saveAndPullDomains( export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName, scanId } = commandOptions; - logger.info(`Running lookingGlass on organization ${organizationName}`); + console.log('Running lookingGlass on organization', organizationName); const collections: LGCollectionsResponse[] = await getCollectionForCurrentWorkspace(); @@ -224,7 +223,7 @@ export const handler = async (commandOptions: CommandOptions) => { // Only pull domains that have been seen in the last two months. const twoMonthsAgo = new Date(); twoMonthsAgo.setMonth(twoMonthsAgo.getMonth() - 2); - logger.info(`Pulling domains seen since ${twoMonthsAgo}`); + console.log(`Pulling domains seen since ${twoMonthsAgo}`); for (const l of data.results) { // Create a dictionary of relevant fields from the API request. const lastSeen = new Date(l.lastSeen); @@ -292,7 +291,7 @@ export const handler = async (commandOptions: CommandOptions) => { } await saveVulnerabilitiesToDb(vulnerabilities, false); - logger.info('Vulnerabilities saved to db'); + console.log('Vulnerabilities saved to db'); } } }; diff --git a/backend/src/tasks/makeGlobalAdmin.ts b/backend/src/tasks/makeGlobalAdmin.ts index 8862cb1e..b1067311 100644 --- a/backend/src/tasks/makeGlobalAdmin.ts +++ b/backend/src/tasks/makeGlobalAdmin.ts @@ -1,10 +1,8 @@ import { Handler } from 'aws-lambda'; import { connectToDatabase, User, UserType } from '../models'; -import logger from '../tools/lambda-logger'; -export const handler: Handler = async (event, context) => { - logger.info('makeGlobalAdmin function started', { context }); - await connectToDatabase(); +export const handler: Handler = async (event) => { + await connectToDatabase(true); if (event.email) { const user = await User.findOne({ email: event.email @@ -14,5 +12,4 @@ export const handler: Handler = async (event, context) => { await User.save(user); } } - logger.info('Success', { context }); }; diff --git a/backend/src/tasks/portscanner.ts b/backend/src/tasks/portscanner.ts index af1be4f9..1521e2b2 100644 --- a/backend/src/tasks/portscanner.ts +++ b/backend/src/tasks/portscanner.ts @@ -7,12 +7,11 @@ import * as portscanner from 'portscanner'; import getIps from './helpers/getIps'; import { CommandOptions } from './ecs-client'; import saveServicesToDb from './helpers/saveServicesToDb'; -import logger from '../tools/lambda-logger'; export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName } = commandOptions; - logger.info(`Running portscanner on organization ${organizationName}`); + console.log('Running portscanner on organization', organizationName); const domainsWithIPs = await getIps(organizationId); @@ -32,7 +31,7 @@ export const handler = async (commandOptions: CommandOptions) => { ); } } catch (e) { - logger.error(JSON.stringify(e)); + console.error(e); continue; } } @@ -40,5 +39,5 @@ export const handler = async (commandOptions: CommandOptions) => { await saveServicesToDb(services); - logger.info(`Portscan finished for ${services.length} services`); + console.log(`Portscan finished for ${services.length} services`); }; diff --git a/backend/src/tasks/rootDomainSync.ts b/backend/src/tasks/rootDomainSync.ts index 216d0fce..d60fb9eb 100644 --- a/backend/src/tasks/rootDomainSync.ts +++ b/backend/src/tasks/rootDomainSync.ts @@ -4,17 +4,16 @@ import { Domain } from '../models'; import { CommandOptions } from './ecs-client'; import saveDomainsToDb from './helpers/saveDomainsToDb'; import * as dns from 'dns'; -import logger from '../tools/lambda-logger'; export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName } = commandOptions; - logger.info(`Syncing domains from ${organizationName}`); + console.log('Syncing domains from', organizationName); const rootDomains = await getRootDomains(organizationId!); const domains: Domain[] = []; for (const rootDomain of rootDomains) { - logger.info(rootDomain); + console.log(rootDomain); let ipAddress; try { ipAddress = (await dns.promises.lookup(rootDomain)).address; @@ -31,5 +30,5 @@ export const handler = async (commandOptions: CommandOptions) => { ); } await saveDomainsToDb(domains); - logger.info(`Scan created/updated ${domains.length} new domains`); + console.log(`Scan created/updated ${domains.length} new domains`); }; diff --git a/backend/src/tasks/s3-client.ts b/backend/src/tasks/s3-client.ts index 3fd863a2..923713c8 100644 --- a/backend/src/tasks/s3-client.ts +++ b/backend/src/tasks/s3-client.ts @@ -1,5 +1,4 @@ import { S3 } from 'aws-sdk'; -import logger from '../tools/lambda-logger'; /** * S3 Client. Normally, interacts with S3. @@ -47,12 +46,12 @@ class S3Client { // Do this so exports are accessible when running locally. if (this.isLocal) { - logger.info(url.replace('minio:9000', 'localhost:9000')); + console.log(url.replace('minio:9000', 'localhost:9000')); return url.replace('minio:9000', 'localhost:9000'); } return url; } catch (e) { - logger.error(e); + console.error(e); throw e; } } @@ -71,10 +70,11 @@ class S3Client { } return url; } catch (e) { - logger.error(e); + console.error(e); throw e; } } + async listReports(orgId: string) { try { const params = { @@ -90,7 +90,7 @@ class S3Client { .promise(); return data.Contents; } catch (e) { - logger.error(e); + console.error(e); throw e; } } diff --git a/backend/src/tasks/saved-search.ts b/backend/src/tasks/saved-search.ts index 2f4a805c..cc63911b 100644 --- a/backend/src/tasks/saved-search.ts +++ b/backend/src/tasks/saved-search.ts @@ -10,10 +10,9 @@ import { userTokenBody } from '../api/auth'; import { fetchAllResults } from '../api/search'; -import logger from '../tools/lambda-logger'; export const handler = async (commandOptions: CommandOptions) => { - logger.info('Running saved search'); + console.log('Running saved search'); await connectToDatabase(); const savedSearches = await SavedSearch.find(); @@ -45,7 +44,7 @@ export const handler = async (commandOptions: CommandOptions) => { const client = new ESClient(); searchResults = await client.searchDomains(request); } catch (e) { - logger.error(e.meta.body.error); + console.error(e.meta.body.error); continue; } const hits: number = searchResults.body.hits.total.value; @@ -68,5 +67,5 @@ export const handler = async (commandOptions: CommandOptions) => { } } - logger.info(`Saved search finished for ${savedSearches.length} searches`); + console.log(`Saved search finished for ${savedSearches.length} searches`); }; diff --git a/backend/src/tasks/scheduler.ts b/backend/src/tasks/scheduler.ts index ee392121..b38ec833 100644 --- a/backend/src/tasks/scheduler.ts +++ b/backend/src/tasks/scheduler.ts @@ -5,7 +5,6 @@ import { SCAN_SCHEMA } from '../api/scans'; import { In, IsNull, Not } from 'typeorm'; import getScanOrganizations from './helpers/getScanOrganizations'; import { chunk } from 'lodash'; -import logger from '../tools/lambda-logger'; class Scheduler { ecs: ECSClient; @@ -39,8 +38,8 @@ class Scheduler { this.maxConcurrentTasks = Number(process.env.FARGATE_MAX_CONCURRENCY!); this.orgsPerScanTask = orgsPerScanTask; - logger.info(`Number of running Fargate tasks: ${this.numExistingTasks}`); - logger.info(`Number of queued scan tasks: ${this.queuedScanTasks.length}`); + console.log('Number of running Fargate tasks: ', this.numExistingTasks); + console.log('Number of queued scan tasks: ', this.queuedScanTasks.length); } launchSingleScanTask = async ({ @@ -87,8 +86,9 @@ class Scheduler { if (!scanTask.queuedAt) { scanTask.queuedAt = new Date(); } - logger.info( - `Reached maximum concurrency, queueing scantask ${scanTask.id}` + console.log( + 'Reached maximum concurrency, queueing scantask', + scanTask.id ); await scanTask.save(); return; @@ -98,7 +98,7 @@ class Scheduler { if (type === 'fargate') { const result = await ecsClient.runCommand(commandOptions); if (result.tasks!.length === 0) { - logger.error(result.failures); + console.error(result.failures); throw new Error( `Failed to start fargate task for scan ${scan.name} -- got ${ result.failures!.length @@ -108,7 +108,7 @@ class Scheduler { const taskArn = result.tasks![0].taskArn; scanTask.fargateTaskArn = taskArn; if (typeof jest === 'undefined') { - logger.info( + console.log( `Successfully invoked ${scan.name} scan with fargate on ${organizations.length} organizations, with ECS task ARN ${taskArn}` + (commandOptions.numChunks ? `, Chunk ${commandOptions.chunkNumber}/${commandOptions.numChunks}` @@ -122,8 +122,8 @@ class Scheduler { scanTask.requestedAt = new Date(); this.numLaunchedTasks++; } catch (error) { - logger.error(`Error invoking ${scan.name} scan.`); - logger.error(error); + console.error(`Error invoking ${scan.name} scan.`); + console.error(error); scanTask.output = JSON.stringify(error); scanTask.status = 'failed'; scanTask.finishedAt = new Date(); @@ -144,6 +144,8 @@ class Scheduler { // For running server on localhost -- doesn't apply in jest tests, though. numChunks = 1; } + // Sanitizes numChunks to protect against arbitrarily large numbers + numChunks = numChunks > 100 ? 100 : numChunks; for (let chunkNumber = 0; chunkNumber < numChunks; chunkNumber++) { await this.launchSingleScanTask({ organizations, @@ -168,7 +170,7 @@ class Scheduler { const prev_numLaunchedTasks = this.numLaunchedTasks; if (!SCAN_SCHEMA[scan.name]) { - logger.error(`Invalid scan name ${scan.name}`); + console.error('Invalid scan name ', scan.name); continue; } const { global } = SCAN_SCHEMA[scan.name]; @@ -195,8 +197,11 @@ class Scheduler { await this.launchScanTask({ organizations: orgs, scan }); } } - logger.info( - `Launched ${this.numLaunchedTasks}scanTasks for scan ${scan.name}` + console.log( + 'Launched', + this.numLaunchedTasks, + 'scanTasks for scan', + scan.name ); // If at least 1 new scan task was launched for this scan, update the scan if (this.numLaunchedTasks > prev_numLaunchedTasks) { @@ -314,9 +319,9 @@ interface Event { orgsPerScanTask?: number; } -export const handler: Handler = async (event, context) => { +export const handler: Handler = async (event) => { await connectToDatabase(); - logger.info('Running scheduler...', { context }); + console.log('Running scheduler...'); const scanIds = event.scanIds || []; if (event.scanId) { @@ -345,7 +350,6 @@ export const handler: Handler = async (event, context) => { relations: ['scan'] }); - logger.info('Initializing...', { context }); const scheduler = new Scheduler(); await scheduler.initialize({ scans, @@ -358,5 +362,5 @@ export const handler: Handler = async (event, context) => { }); await scheduler.runQueued(); await scheduler.run(); - logger.info('Finished running scheduler.', { context }); + console.log('Finished running scheduler.'); }; diff --git a/backend/src/tasks/search-sync.ts b/backend/src/tasks/search-sync.ts index 217f0438..3d674db8 100644 --- a/backend/src/tasks/search-sync.ts +++ b/backend/src/tasks/search-sync.ts @@ -4,7 +4,6 @@ import { In } from 'typeorm'; import ESClient from './es-client'; import { chunk } from 'lodash'; import pRetry from 'p-retry'; -import logger from '../tools/lambda-logger'; /** * Chunk sizes. These values are small during testing to facilitate testing. @@ -14,7 +13,7 @@ export const DOMAIN_CHUNK_SIZE = typeof jest === 'undefined' ? 50 : 10; export const handler = async (commandOptions: CommandOptions) => { const { organizationId, domainId } = commandOptions; - logger.info('Running searchSync'); + console.log('Running searchSync'); await connectToDatabase(); const client = new ESClient(); @@ -41,7 +40,7 @@ export const handler = async (commandOptions: CommandOptions) => { } const domainIds = (await qs.getMany()).map((e) => e.id); - logger.info(`Got ${domainIds.length} domains.`); + console.log(`Got ${domainIds.length} domains.`); if (domainIds.length) { const domainIdChunks = chunk(domainIds, DOMAIN_CHUNK_SIZE); for (const domainIdChunk of domainIdChunks) { @@ -49,7 +48,7 @@ export const handler = async (commandOptions: CommandOptions) => { where: { id: In(domainIdChunk) }, relations: ['services', 'organization', 'vulnerabilities'] }); - logger.info(`Syncing ${domains.length} domains...`); + console.log(`Syncing ${domains.length} domains...`); await pRetry(() => client.updateDomains(domains), { retries: 3, randomize: true @@ -61,8 +60,8 @@ export const handler = async (commandOptions: CommandOptions) => { .where({ id: In(domains.map((e) => e.id)) }) .execute(); } - logger.info('Domain sync complete.'); + console.log('Domain sync complete.'); } else { - logger.info('Not syncing any domains.'); + console.log('Not syncing any domains.'); } }; diff --git a/backend/src/tasks/shodan.ts b/backend/src/tasks/shodan.ts index 640fd67a..1c72b27d 100644 --- a/backend/src/tasks/shodan.ts +++ b/backend/src/tasks/shodan.ts @@ -8,7 +8,6 @@ import axios from 'axios'; import pRetry from 'p-retry'; import { IpToDomainsMap, sanitizeStringField } from './censysIpv4'; import saveVulnerabilitiesToDb from './helpers/saveVulnerabilitiesToDb'; -import logger from '../tools/lambda-logger'; // Shodan allows searching up to 100 IPs at once const CHUNK_SIZE = 100; @@ -48,7 +47,7 @@ const sleep = (milliseconds) => { export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName } = commandOptions; - logger.info(`Running shodan on organization ${organizationName}`); + console.log('Running shodan on organization', organizationName); const domainsWithIPs = await getIps(organizationId); @@ -66,7 +65,7 @@ export const handler = async (commandOptions: CommandOptions) => { const chunks = chunk(domainsWithIPs, CHUNK_SIZE); for (const domainChunk of chunks) { - logger.info( + console.log( `Scanning ${domainChunk.length} domains beginning with ${domainChunk[0].name}` ); try { @@ -110,7 +109,7 @@ export const handler = async (commandOptions: CommandOptions) => { if (service.vulns) { const vulns: Vulnerability[] = []; for (const cve in service.vulns) { - // logger.info('Creating vulnerability', cve); + // console.log('Creating vulnerability', cve); vulns.push( plainToClass(Vulnerability, { domain: domain, @@ -137,11 +136,11 @@ export const handler = async (commandOptions: CommandOptions) => { } } } catch (e) { - logger.error(JSON.stringify(e)); + console.error(e); } await sleep(1000); // Wait for Shodan rate limit of 1 request / second } - logger.info(`Shodan finished for ${domainsWithIPs.length} domains`); + console.log(`Shodan finished for ${domainsWithIPs.length} domains`); }; diff --git a/backend/src/tasks/sslyze.ts b/backend/src/tasks/sslyze.ts index 4f5ef6ab..c6c13b7b 100644 --- a/backend/src/tasks/sslyze.ts +++ b/backend/src/tasks/sslyze.ts @@ -2,7 +2,6 @@ import { CommandOptions } from './ecs-client'; import { getLiveWebsites, LiveDomain } from './helpers/getLiveWebsites'; import PQueue from 'p-queue'; import * as sslChecker from 'ssl-checker'; -import logger from '../tools/lambda-logger'; interface IResolvedValues { valid: boolean; @@ -22,7 +21,7 @@ const sslyze = async (domain: LiveDomain): Promise => { validTo: response.validTo, altNames: response.validFor }; - logger.info( + console.log( domain.name, ': valid', response.valid, @@ -33,18 +32,18 @@ const sslyze = async (domain: LiveDomain): Promise => { ); await domain.save(); } catch (e) { - logger.error(domain.name, JSON.stringify(e)); + console.error(domain.name, e); } }; export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName } = commandOptions; - logger.info(`Running sslyze on organization ${organizationName}`); + console.log('Running sslyze on organization', organizationName); const liveWebsites = await getLiveWebsites(organizationId!, [], true); const queue = new PQueue({ concurrency: 5 }); await Promise.all(liveWebsites.map((site) => queue.add(() => sslyze(site)))); - logger.info(`sslyze finished for ${liveWebsites.length} domains`); + console.log(`sslyze finished for ${liveWebsites.length} domains`); }; diff --git a/backend/src/tasks/syncdb.ts b/backend/src/tasks/syncdb.ts index 1c9d124a..27738a5d 100644 --- a/backend/src/tasks/syncdb.ts +++ b/backend/src/tasks/syncdb.ts @@ -5,52 +5,53 @@ import { Domain, Organization, OrganizationTag, - Scan, Vulnerability } from '../models'; import ESClient from './es-client'; import * as Sentencer from 'sentencer'; import * as services from './sample_data/services.json'; import * as cpes from './sample_data/cpes.json'; +import * as cves from './sample_data/cves.json'; import * as vulnerabilities from './sample_data/vulnerabilities.json'; import * as nouns from './sample_data/nouns.json'; import * as adjectives from './sample_data/adjectives.json'; +import { saveToDb } from './cve-sync'; import { sample } from 'lodash'; import { handler as searchSync } from './search-sync'; import { In } from 'typeorm'; -import logger from '../tools/lambda-logger'; const SAMPLE_TAG_NAME = 'Sample Data'; // Tag name for sample data const NUM_SAMPLE_ORGS = 10; // Number of sample orgs const NUM_SAMPLE_DOMAINS = 10; // Number of sample domains per org const PROB_SAMPLE_SERVICES = 0.5; // Higher number means more services per domain const PROB_SAMPLE_VULNERABILITIES = 0.5; // Higher number means more vulnerabilities per domain +const SAMPLE_STATES = ['VA', 'CA', 'CO']; +const SAMPLE_REGIONIDS = ['1', '2', '3']; -export const handler: Handler = async (event, context) => { +export const handler: Handler = async (event) => { const connection = await connectToDatabase(false); const type = event?.type || event; const dangerouslyforce = type === 'dangerouslyforce'; if (connection) { await connection.synchronize(dangerouslyforce); } else { - logger.error('Error: could not sync', { context }); + console.error('Error: could not sync'); } if (process.env.NODE_ENV !== 'test') { // Create indices on elasticsearch only when not using tests. const client = new ESClient(); if (dangerouslyforce) { - logger.info('Deleting all data in elasticsearch...', { context }); + console.log('Deleting all data in elasticsearch...'); await client.deleteAll(); - logger.info('Done.', { context }); + console.log('Done.'); } await client.syncDomainsIndex(); } if (type === 'populate') { - logger.info('Populating the database with some sample data...', { - context - }); + console.log('Populating the database with some sample data...'); + await saveToDb(cves); Sentencer.configure({ nounList: nouns, adjectiveList: adjectives, @@ -74,6 +75,7 @@ export const handler: Handler = async (event, context) => { } for (let i = 0; i <= NUM_SAMPLE_ORGS; i++) { const organization = await Organization.create({ + acronym: Math.random().toString(36).slice(2, 7), name: Sentencer.make('{{ adjective }} {{ entity }}').replace( /\b\w/g, (l) => l.toUpperCase() @@ -81,9 +83,12 @@ export const handler: Handler = async (event, context) => { rootDomains: ['crossfeed.local'], ipBlocks: [], isPassive: false, - tags: [tag] + tags: [tag], + state: SAMPLE_STATES[Math.floor(Math.random() * SAMPLE_STATES.length)], + regionId: + SAMPLE_REGIONIDS[Math.floor(Math.random() * SAMPLE_REGIONIDS.length)] }).save(); - logger.info(organization.name, { context }); + console.log(organization.name); organizationIds.push(organization.id); for (let i = 0; i <= NUM_SAMPLE_DOMAINS; i++) { const randomNum = () => Math.floor(Math.random() * 256); @@ -94,7 +99,7 @@ export const handler: Handler = async (event, context) => { subdomainSource: 'findomain', organization }).save(); - logger.info(`\t${domain.name}`, { context }); + console.log(`\t${domain.name}`); let service; for (const serviceData of services) { if (service && Math.random() < PROB_SAMPLE_SERVICES) continue; @@ -131,7 +136,7 @@ export const handler: Handler = async (event, context) => { } } - logger.info('Done. Running search sync...', { context }); + console.log('Done. Running search sync...'); for (const organizationId of organizationIds) { await searchSync({ organizationId, @@ -141,6 +146,6 @@ export const handler: Handler = async (event, context) => { scanTaskId: 'scanTaskId' }); } - logger.info('Done.', { context }); + console.log('Done.'); } }; diff --git a/backend/src/tasks/trustymail.ts b/backend/src/tasks/trustymail.ts index 7fa86da1..35650893 100644 --- a/backend/src/tasks/trustymail.ts +++ b/backend/src/tasks/trustymail.ts @@ -3,31 +3,30 @@ import { ChildProcessWithoutNullStreams, spawn } from 'child_process'; import { CommandOptions } from './ecs-client'; import saveTrustymailResultsToDb from './helpers/saveTrustymailResultsToDb'; import * as chokidar from 'chokidar'; -import logger from '../tools/lambda-logger'; // TODO: Retry failed domains // TODO: PSL is downloaded once per container (number of orgs / 2, rounded up); would like to download only once per day export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName } = commandOptions; - logger.info(`Running Trustymail on organization ${organizationName}`); + console.log('Running Trustymail on organization', organizationName); const domains = await getAllDomains([organizationId!]); const queue = new Set(domains.map((domain) => `${domain.id}.json`)); const failedDomains = new Set(); - logger.info( + console.log( `${organizationName} domains to be scanned:`, domains.map((domain) => domain.name) ); - logger.info(`queue ${queue}`); + console.log(`queue`, queue); for (const domain of domains) { const path = `${domain.id}.json`; // Event listener detects Trustymail results file at creation and syncs it to db chokidar.watch(path).on('add', (path) => { - logger.info(`Trustymail saved results to ${path}`); - logger.info(`Syncing ${path} to db...`); + console.log('Trustymail saved results to', path); + console.log(`Syncing ${path} to db...`); saveTrustymailResultsToDb(path).then(() => { - logger.info(`Deleting ${path} and removing it from queue...`); + console.log(`Deleting ${path} and removing it from queue...`); queue.delete(path); - logger.info(`Items left in queue: ${queue}`); + console.log(`Items left in queue:`, queue); }); }); try { @@ -37,7 +36,7 @@ export const handler = async (commandOptions: CommandOptions) => { `--output=${domain.id}`, '--psl-filename=public_suffix_list.dat' ]; - logger.info(`Running Trustymail with args: ${args}`); + console.log('Running Trustymail with args:', args); const child = spawn('trustymail', args, { stdio: 'pipe' }); addEventListenersToChildProcess( child, @@ -47,20 +46,21 @@ export const handler = async (commandOptions: CommandOptions) => { failedDomains ); } catch (e) { - logger.error(JSON.stringify(e)); + console.error(e); } } // Keep container alive while syncing results to db; timeout after 10 minutes (60 * 10 seconds) let dbSyncTimeout = 0; while (queue.size > 0 && dbSyncTimeout < 60) { - logger.info('Syncing...'); + console.log('Syncing...'); await delay(1000 * 2); dbSyncTimeout++; } - logger.info( + console.log( `Trustymail successfully scanned ${ domains.length - failedDomains.size - } out of ${domains.length} domains \n failed domains: ${failedDomains}` + } out of ${domains.length} domains \n failed domains:`, + failedDomains ); }; @@ -76,22 +76,25 @@ function addEventListenersToChildProcess( failedDomains: Set ) { child.stdout.on('data', (data) => { - logger.info(`${domain.name} (${domain.id}) stdout: ${data}`); + console.log(`${domain.name} (${domain.id}) stdout: ${data}`); }); child.on('spawn', () => { - logger.info( + console.log( `Spawned ${domain.name} (${domain.id}) Trustymail child process` ); }); child.on('error', (err) => - logger.error( + console.error( `Error with Trustymail ${domain.name} (${domain.id}) child process:`, - JSON.stringify(err) + err ) ); child.on('exit', (code, signal) => { - logger.info( - `Trustymail child process ${domain.name} (${domain.id}) exited with code ${code} and signal ${signal}` + console.log( + `Trustymail child process ${domain.name} (${domain.id}) exited with code`, + code, + 'and signal', + signal ); if (code !== 0) { queue.delete(path); diff --git a/backend/src/tasks/updateScanTaskStatus.ts b/backend/src/tasks/updateScanTaskStatus.ts index a4b302cc..6d5e195b 100644 --- a/backend/src/tasks/updateScanTaskStatus.ts +++ b/backend/src/tasks/updateScanTaskStatus.ts @@ -2,7 +2,6 @@ import { Handler } from 'aws-lambda'; import { connectToDatabase, ScanTask } from '../models'; import { Task } from 'aws-sdk/clients/ecs'; import pRetry from 'p-retry'; -import logger from '../tools/lambda-logger'; export type EventBridgeEvent = { detail: Task & { @@ -52,7 +51,7 @@ export const handler: Handler = async ( } else { return; } - logger.info( + console.log( `Updating status of ScanTask ${scanTask.id} from ${oldStatus} to ${scanTask.status}.` ); await scanTask.save(); diff --git a/backend/src/tasks/wappalyzer.ts b/backend/src/tasks/wappalyzer.ts index 03ba1dc4..ce3ce901 100644 --- a/backend/src/tasks/wappalyzer.ts +++ b/backend/src/tasks/wappalyzer.ts @@ -3,11 +3,10 @@ import { CommandOptions } from './ecs-client'; import { getLiveWebsites, LiveDomain } from './helpers/getLiveWebsites'; import PQueue from 'p-queue'; import { wappalyzer } from './helpers/simple-wappalyzer'; -import logger from '../tools/lambda-logger'; const wappalyze = async (domain: LiveDomain): Promise => { try { - logger.info(`Executing wapplyzer on ${domain.url}`); + console.log(`Executing wapplyzer on ${domain.url}`); const { data, status, headers } = await axios.get(domain.url, { timeout: 10000, @@ -24,17 +23,17 @@ const wappalyze = async (domain: LiveDomain): Promise => { if (filteredResults.length > 0) { domain.service.wappalyzerResults = filteredResults; await domain.service.save(); - logger.info( + console.log( `${domain.url} - ${filteredResults.length} technology matches saved.` ); } else { - logger.info( + console.log( `${domain.url} - there were no technology matches for domain.` ); } } } else { - logger.info( + console.log( `${domain.url} returned and HTTP error code. HTTPStatusCode: ${status}` ); } @@ -48,7 +47,7 @@ const wappalyze = async (domain: LiveDomain): Promise => { Response: ${error.response}`); } else if (error.code === 'ECONNABORTED') { // request timed out - logger.error( + console.error( `${domain.url} domain did not respond in a timely manner: ${error.message}` ); } else { @@ -72,7 +71,7 @@ const wappalyze = async (domain: LiveDomain): Promise => { export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName } = commandOptions; - logger.info(`Running wappalyzer on organization ${organizationName}`); + console.log(`Running wappalyzer on organization ${organizationName}`); const liveWebsites = await getLiveWebsites(organizationId!); const queue = new PQueue({ concurrency: 5 }); @@ -80,7 +79,7 @@ export const handler = async (commandOptions: CommandOptions) => { liveWebsites.map((site) => queue.add(() => wappalyze(site))) ); - logger.info( + console.log( `Wappalyzer finished for organization ${organizationName} on ${liveWebsites.length} domains` ); }; diff --git a/backend/src/tasks/webscraper.ts b/backend/src/tasks/webscraper.ts index fa296276..d1a2240a 100644 --- a/backend/src/tasks/webscraper.ts +++ b/backend/src/tasks/webscraper.ts @@ -16,7 +16,6 @@ import PQueue from 'p-queue'; import { chunk } from 'lodash'; import { In } from 'typeorm'; import getScanOrganizations from './helpers/getScanOrganizations'; -import logger from '../tools/lambda-logger'; const WEBSCRAPER_DIRECTORY = '/app/worker/webscraper'; const INPUT_PATH = path.join(WEBSCRAPER_DIRECTORY, 'domains.txt'); @@ -59,13 +58,13 @@ export const handler = async (commandOptions: CommandOptions) => { const organizationIds = ( chunk(organizations, numChunks)[chunkNumber!] || [] ).map((e) => e.id); - logger.info(`Running webscraper on organizations ${organizationIds}`); + console.log('Running webscraper on organizations ', organizationIds); const liveWebsites = await getLiveWebsites(undefined, organizationIds); const urls = liveWebsites.map((domain) => domain.url); - logger.info(`input urls ${urls}`); + console.log('input urls', urls); if (urls.length === 0) { - logger.info('no urls, returning'); + console.log('no urls, returning'); return; } @@ -102,14 +101,14 @@ export const handler = async (commandOptions: CommandOptions) => { }); await new Promise((resolve, reject) => { - logger.info('Going to save webpages to the database...'); + console.log('Going to save webpages to the database...'); let scrapedWebpages: ScraperItem[] = []; readInterfaceStderr.on('line', (line) => - logger.error(line?.substring(0, 999)) + console.error(line?.substring(0, 999)) ); readInterface.on('line', async (line) => { if (!line?.trim() || line.indexOf('database_output: ') === -1) { - logger.info(line); + console.log(line); return; } const item: ScraperItem = JSON.parse( @@ -119,7 +118,7 @@ export const handler = async (commandOptions: CommandOptions) => { ); const domain = liveWebsitesMap[item.domain_name]; if (!domain) { - logger.error( + console.error( `No corresponding domain found for item with domain_name ${item.domain_name}.` ); return; @@ -135,7 +134,7 @@ export const handler = async (commandOptions: CommandOptions) => { if (scrapedWebpages.length === 0) { return; } - logger.info( + console.log( `Saving ${scrapedWebpages.length} webpages, starting with ${scrapedWebpages[0].url}...` ); await saveWebpagesToDb(scrapedWebpages); @@ -148,7 +147,7 @@ export const handler = async (commandOptions: CommandOptions) => { readInterface.on('close', async () => { await queue.onIdle(); if (scrapedWebpages.length > 0) { - logger.info( + console.log( `Saving ${scrapedWebpages.length} webpages to the database...` ); await saveWebpagesToDb(scrapedWebpages); @@ -161,7 +160,7 @@ export const handler = async (commandOptions: CommandOptions) => { readInterface.on('SIGCONT', reject); readInterface.on('SIGTSTP', reject); }); - logger.info( + console.log( `Webscraper finished for ${liveWebsites.length} domains, saved ${totalNumWebpages} webpages in total` ); };