From facac52d2b7f6eb5753d26ea2d02bfd7346f643e Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Thu, 11 Apr 2024 11:04:39 -0500 Subject: [PATCH] Add rscSync to worker enntrypoint and scanSchema; add associated handler and fix questionDictionary. --- backend/src/api/scans.ts | 190 ++++++++++++++++++----------------- backend/src/tasks/rscSync.ts | 180 ++++++++++++++++++++------------- backend/src/worker.ts | 62 ++++++------ 3 files changed, 239 insertions(+), 193 deletions(-) diff --git a/backend/src/api/scans.ts b/backend/src/api/scans.ts index 33423b2d..91875c33 100644 --- a/backend/src/api/scans.ts +++ b/backend/src/api/scans.ts @@ -10,11 +10,10 @@ import { IsArray } from 'class-validator'; import { - Scan, connectToDatabase, Organization, - ScanTask, - OrganizationTag + OrganizationTag, + Scan } from '../models'; import { validateBody, wrapHandler, NotFound, Unauthorized } from './helpers'; import { isGlobalWriteAdmin, isGlobalViewAdmin } from './auth'; @@ -49,47 +48,70 @@ interface ScanSchema { } export const SCAN_SCHEMA: ScanSchema = { - vulnSync: { + amass: { + type: 'fargate', + isPassive: false, + global: false, + description: + 'Open source tool that integrates passive APIs and active subdomain enumeration in order to discover target subdomains' + }, + censys: { + type: 'fargate', + isPassive: true, + global: false, + description: 'Passive discovery of subdomains from public certificates' + }, + censysCertificates: { type: 'fargate', isPassive: true, global: true, - description: 'Pull in vulnerability data from PEs Vulnerability database', - cpu: '1024', - memory: '8192' + cpu: '2048', + memory: '6144', + numChunks: 20, + description: 'Fetch TLS certificate data from censys certificates dataset' }, - cveSync: { + censysIpv4: { type: 'fargate', isPassive: true, global: true, - description: - "Matches detected software versions to CVEs from NIST NVD and CISA's Known Exploited Vulnerabilities Catalog.", - cpu: '1024', - memory: '8192' + cpu: '2048', + memory: '6144', + numChunks: 20, + description: 'Fetch passive port and banner data from censys ipv4 dataset' }, - testProxy: { + cve: { type: 'fargate', - isPassive: false, + isPassive: true, global: true, - description: 'Not a real scan, used to test proxy' + cpu: '1024', + memory: '8192', + description: + "Matches detected software versions to CVEs from NIST NVD and CISA's Known Exploited Vulnerabilities Catalog." }, - test: { + cveSync: { type: 'fargate', - isPassive: false, + isPassive: true, global: true, - description: 'Not a real scan, used to test' + description: + "Matches detected software versions to CVEs from NIST NVD and CISA's Known Exploited Vulnerabilities Catalog.", + cpu: '1024', + memory: '8192' }, - censys: { + dnstwist: { type: 'fargate', isPassive: true, global: false, - description: 'Passive discovery of subdomains from public certificates' + cpu: '2048', + memory: '16384', + description: + 'Domain name permutation engine for detecting similar registered domains.' }, - amass: { + dotgov: { type: 'fargate', - isPassive: false, - global: false, + isPassive: true, + global: true, description: - 'Open source tool that integrates passive APIs and active subdomain enumeration in order to discover target subdomains' + 'Create organizations based on root domains from the dotgov registrar dataset. All organizations are created with the "dotgov" tag and have a " (dotgov)" suffix added to their name.' }, findomain: { type: 'fargate', @@ -98,13 +120,16 @@ export const SCAN_SCHEMA: ScanSchema = { description: 'Open source tool that integrates passive APIs in order to discover target subdomains' }, - portscanner: { + hibp: { type: 'fargate', - isPassive: false, + isPassive: true, global: false, - description: 'Active port scan of common ports' + cpu: '2048', + memory: '16384', + description: + 'Finds emails that have appeared in breaches related to a given domain' }, - wappalyzer: { + intrigueIdent: { type: 'fargate', isPassive: true, global: false, @@ -113,54 +138,37 @@ export const SCAN_SCHEMA: ScanSchema = { description: 'Open source tool that fingerprints web technologies based on HTTP responses' }, - shodan: { + lookingGlass: { type: 'fargate', isPassive: true, global: false, - description: - 'Fetch passive port, banner, and vulnerability data from shodan', - cpu: '1024', - memory: '8192' + description: 'Finds vulnerabilities and malware from the LookingGlass API' }, - sslyze: { + portscanner: { type: 'fargate', - isPassive: true, + isPassive: false, global: false, - description: 'SSL certificate inspection' - }, - censysIpv4: { - type: 'fargate', - isPassive: true, - global: true, - cpu: '2048', - memory: '6144', - numChunks: 20, - description: 'Fetch passive port and banner data from censys ipv4 dataset' + description: 'Active port scan of common ports' }, - censysCertificates: { + rootDomainSync: { type: 'fargate', isPassive: true, - global: true, - cpu: '2048', - memory: '6144', - numChunks: 20, - description: 'Fetch TLS certificate data from censys certificates dataset' + global: false, + description: + 'Creates domains from root domains by doing a single DNS lookup for each root domain.' }, - cve: { + rscSync: { type: 'fargate', isPassive: true, global: true, - cpu: '1024', - memory: '8192', description: - "Matches detected software versions to CVEs from NIST NVD and CISA's Known Exploited Vulnerabilities Catalog." + 'Retrieves and saves assessments from ReadySetCyber mission instance.' }, - dotgov: { + savedSearch: { type: 'fargate', isPassive: true, global: true, - description: - 'Create organizations based on root domains from the dotgov registrar dataset. All organizations are created with the "dotgov" tag and have a " (dotgov)" suffix added to their name.' + description: 'Performs saved searches to update their search results' }, searchSync: { type: 'fargate', @@ -171,67 +179,65 @@ export const SCAN_SCHEMA: ScanSchema = { description: 'Syncs records with Elasticsearch so that they appear in search results.' }, - intrigueIdent: { + shodan: { type: 'fargate', isPassive: true, global: false, - cpu: '1024', - memory: '4096', description: - 'Open source tool that fingerprints web technologies based on HTTP responses' - }, - webscraper: { - type: 'fargate', - isPassive: true, - global: true, - numChunks: 3, + 'Fetch passive port, banner, and vulnerability data from shodan', cpu: '1024', - memory: '4096', - description: 'Scrapes all webpages on a given domain, respecting robots.txt' + memory: '8192' }, - hibp: { + sslyze: { type: 'fargate', isPassive: true, global: false, - cpu: '2048', - memory: '16384', - description: - 'Finds emails that have appeared in breaches related to a given domain' + description: 'SSL certificate inspection' }, - lookingGlass: { + test: { type: 'fargate', - isPassive: true, - global: false, - description: 'Finds vulnerabilities and malware from the LookingGlass API' + isPassive: false, + global: true, + description: 'Not a real scan, used to test' }, - dnstwist: { + testProxy: { type: 'fargate', - isPassive: true, - global: false, - cpu: '2048', - memory: '16384', - description: - 'Domain name permutation engine for detecting similar registered domains.' + isPassive: false, + global: true, + description: 'Not a real scan, used to test proxy' }, - rootDomainSync: { + trustymail: { type: 'fargate', isPassive: true, global: false, description: - 'Creates domains from root domains by doing a single DNS lookup for each root domain.' + 'Evaluates SPF/DMARC records and checks MX records for STARTTLS support' }, - savedSearch: { + vulnSync: { type: 'fargate', isPassive: true, global: true, - description: 'Performs saved searches to update their search results' + description: 'Pull in vulnerability data from PEs Vulnerability database', + cpu: '1024', + memory: '8192' }, - trustymail: { + wappalyzer: { type: 'fargate', isPassive: true, global: false, + cpu: '1024', + memory: '4096', description: - 'Evaluates SPF/DMARC records and checks MX records for STARTTLS support' + 'Open source tool that fingerprints web technologies based on HTTP responses' + }, + webscraper: { + type: 'fargate', + isPassive: true, + global: true, + numChunks: 3, + cpu: '1024', + memory: '4096', + description: 'Scrapes all webpages on a given domain, respecting robots.txt' } }; diff --git a/backend/src/tasks/rscSync.ts b/backend/src/tasks/rscSync.ts index 9c9d9541..ac5255f8 100644 --- a/backend/src/tasks/rscSync.ts +++ b/backend/src/tasks/rscSync.ts @@ -1,8 +1,15 @@ import axios from 'axios'; import { Buffer } from 'buffer'; import { plainToClass } from 'class-transformer'; -import { Assessment, Question, Response, User } from '../models'; +import { + Assessment, + connectToDatabase, + Question, + Response, + User +} from '../models'; import { getRepository } from 'typeorm'; +import * as console from 'console'; interface AssessmentEntry { metadata: Metadata; @@ -14,73 +21,91 @@ interface Metadata { u_customer_email: string; } interface Questions { - u_3rd_party_cyber_validation?: string | null; - u_annual_cyber_training?: string | null; - u_asset_inventory?: string | null; - u_asset_recovery_plan?: string | null; - u_backup_access?: string | null; - u_backup_auto?: string | null; - u_backup_data?: string | null; - u_backup_method?: string | null; - u_basline_config_documents?: string | null; - u_change_default_pw?: string | null; - u_connect_deny_default?: string | null; - u_cyber_incident_plan?: string | null; - u_departing_employee_return?: string | null; - u_device_config?: string | null; - u_device_type?: string | null; - u_disable_svcs?: string | null; - u_email_tls_dkim?: string | null; - u_first_name?: string | null; - u_fw_av?: string | null; - u_governance_training?: string | null; - u_hw_software_firmware_approval?: string | null; - u_iam_security?: string | null; - u_inc_reporting_policy?: string | null; - u_inc_response_plan?: string | null; - u_it_org_size?: string | null; - u_kev_mitagation?: string | null; - u_last_name?: string | null; - u_log_storage?: string | null; - u_log_storage_detect_resp?: string | null; - u_log_unsuccessful_login?: string | null; - u_macros_disabled?: string | null; - u_mfa_enabled?: string | null; - u_named_cyber_role?: string | null; - u_network_diagrams?: string | null; - u_no_public_devices?: string | null; - u_ot_cyber_training?: string | null; - u_prohibit_unauth_devices?: string | null; - u_public_services_disabled?: string | null; - u_pw_different?: string | null; - u_pw_length?: string | null; - u_pw_strong?: string | null; - u_regular_backups?: string | null; - u_reputable_software?: string | null; - u_secure_credential_storage?: string | null; - u_security_features?: string | null; - u_security_inc_sla?: string | null; - u_security_research_contact?: string | null; - u_security_vuln_sla?: string | null; - u_separate_admin?: string | null; - u_software_updates?: string | null; - u_strong_mfa?: string | null; - u_supply_chain_risk?: string | null; - u_tls_enabled?: string | null; - u_ttp_list?: string | null; - u_unique_svc_accounts?: string | null; - u_vendor_eval_docs?: string | null; - u_vul_management?: string | null; + [key: string]: string | null; } +function castApiResponseToAssessmentInterface(data: any[]): AssessmentEntry[] { + return data.map((q) => ({ + metadata: { + sys_created_on: new Date(q.sys_created_on), + sys_id: q.sys_id, + u_customer_email: q.u_customer_email + }, + questions: { + u_3rd_party_cyber_validation: q.u_3rd_party_cyber_validation || null, + u_annual_cyber_training: q.u_annual_cyber_training || null, + u_asset_inventory: q.u_asset_inventory || null, + u_asset_recovery_plan: q.u_asset_recovery_plan || null, + u_backup_access: q.u_backup_access || null, + u_backup_auto: q.u_backup_auto || null, + u_backup_data: q.u_backup_data || null, + u_backup_method: q.u_backup_method || null, + u_basline_config_documents: q.u_basline_config_documents || null, + u_change_default_pw: q.u_change_default_pw || null, + u_connect_deny_default: q.u_connect_deny_default || null, + u_cyber_incident_plan: q.u_cyber_incident_plan || null, + u_departing_employee_return: q.u_departing_employee_return || null, + u_device_config: q.u_device_config || null, + u_device_type: q.u_device_type || null, + u_disable_svcs: q.u_disable_svcs || null, + u_email_tls_dkim: q.u_email_tls_dkim || null, + u_fw_av: q.u_fw_av || null, + u_governance_training: q.u_governance_training || null, + u_hw_software_firmware_approval: + q.u_hw_software_firmware_approval || null, + u_iam_security: q.u_iam_security || null, + u_inc_reporting_policy: q.u_inc_reporting_policy || null, + u_inc_response_plan: q.u_inc_response_plan || null, + u_it_org_size: q.u_it_org_size || null, + u_kev_mitagation: q.u_kev_mitagation || null, + u_log_storage: q.u_log_storage || null, + u_log_storage_detect_resp: q.u_log_storage_detect_resp || null, + u_log_unsuccessful_login: q.u_log_unsuccessful_login || null, + u_macros_disabled: q.u_macros_disabled || null, + u_mfa_enabled: q.u_mfa_enabled || null, + u_named_cyber_role: q.u_named_cyber_role || null, + u_network_diagrams: q.u_network_diagrams || null, + u_no_public_devices: q.u_no_public_devices || null, + u_ot_cyber_training: q.u_ot_cyber_training || null, + u_prohibit_unauth_devices: q.u_prohibit_unauth_devices || null, + u_public_services_disabled: q.u_public_services_disabled || null, + u_pw_different: q.u_pw_different || null, + u_pw_length: q.u_pw_length || null, + u_pw_strong: q.u_pw_strong || null, + u_regular_backups: q.u_regular_backups || null, + u_reputable_software: q.u_reputable_software || null, + u_secure_credential_storage: q.u_secure_credential_storage || null, + u_security_features: q.u_security_features || null, + u_security_inc_sla: q.u_security_inc_sla || null, + u_security_research_contact: q.u_security_research_contact || null, + u_security_vuln_sla: q.u_security_vuln_sla || null, + u_separate_admin: q.u_separate_admin || null, + u_software_updates: q.u_software_updates || null, + u_strong_mfa: q.u_strong_mfa || null, + u_supply_chain_risk: q.u_supply_chain_risk || null, + u_tls_enabled: q.u_tls_enabled || null, + u_ttp_list: q.u_ttp_list || null, + u_unique_svc_accounts: q.u_unique_svc_accounts || null, + u_vendor_eval_docs: q.u_vendor_eval_docs || null, + u_vul_management: q.u_vul_management || null + } + })); +} + +export const handler = async () => { + await fetchRecentAssessments(); +}; + const authorizationHeader = Buffer.from( `${process.env.MI_ACCOUNT_NAME}:${process.env.MI_PASSWORD}`, 'utf8' ).toString('base64'); -const questionDictionary = async (): Promise<{ +// Creates a temporary dictionary to reduce calls to the database +const generateQuestionDictionary = async (): Promise<{ [key: string]: string; }> => { + await connectToDatabase(); const questionRepository = getRepository(Question); const questions = await questionRepository.find({ select: ['id', 'name'] }); return questions.reduce((acc, question) => { @@ -88,9 +113,9 @@ const questionDictionary = async (): Promise<{ return acc; }, {}); }; +let questionDictionary: { [key: string]: string } = {}; -// Intended to run every 6 hours; retrieves assessments created in the last 48 hours -export const fetchRecentAssessments = async () => { +const fetchRecentAssessments = async () => { console.log('Fetching assessments'); // Create a date object and subtract 48 hours @@ -111,7 +136,7 @@ export const fetchRecentAssessments = async () => { } }; -// Intended to run on user registration +// TODO: Tie this to RSC user registration export const fetchAssessmentsByUser = async (email: string) => { console.log('Fetching assessments for user'); try { @@ -132,8 +157,13 @@ const getUserIdByEmail = async (email: string): Promise => { return user ? user.id : null; }; -const saveAssessmentsToDb = async (assessments: AssessmentEntry[]) => { +export const saveAssessmentsToDb = async (assessments: any[]) => { console.log('Saving assessments to database'); + await connectToDatabase(); + questionDictionary = await generateQuestionDictionary(); + assessments = castApiResponseToAssessmentInterface( + assessments + ) as AssessmentEntry[]; try { for (const assessment of assessments) { const user = await getUserIdByEmail(assessment.metadata.u_customer_email); @@ -158,19 +188,27 @@ const saveAssessmentsToDb = async (assessments: AssessmentEntry[]) => { } }; -export const saveResponsesToDb = async ( +const saveResponsesToDb = async ( assessment: AssessmentEntry, assessmentId: string ) => { const responseArray: Response[] = []; + console.log('assessmentId: ', assessmentId); for (const [question, response] of Object.entries(assessment.questions)) { - responseArray.push( - plainToClass(Response, { - selection: response, - assessmentId, - questionId: questionDictionary[question] - }) - ); + const questionId = questionDictionary[question]; + console.log(`\nquestion: `, question); + console.log('response: ', response); + console.log('questionId: ', questionId); + if (response) { + responseArray.push( + plainToClass(Response, { + selection: response, + assessment: assessmentId, + question: questionId + }) + ); + } } + console.log('responseArray: ', responseArray); await Response.save(responseArray); }; diff --git a/backend/src/worker.ts b/backend/src/worker.ts index 0e4cdebd..c29b6059 100644 --- a/backend/src/worker.ts +++ b/backend/src/worker.ts @@ -1,31 +1,32 @@ import { bootstrap } from 'global-agent'; import { CommandOptions } from './tasks/ecs-client'; +import { connectToDatabase } from './models'; +import fetchPublicSuffixList from './tasks/helpers/fetchPublicSuffixList'; import { handler as amass } from './tasks/amass'; import { handler as censys } from './tasks/censys'; -import { handler as findomain } from './tasks/findomain'; -import { handler as portscanner } from './tasks/portscanner'; -import { handler as wappalyzer } from './tasks/wappalyzer'; -import { handler as censysIpv4 } from './tasks/censysIpv4'; import { handler as censysCertificates } from './tasks/censysCertificates'; -import { handler as savedSearch } from './tasks/saved-search'; -import { handler as sslyze } from './tasks/sslyze'; -import { handler as searchSync } from './tasks/search-sync'; -import { handler as intrigueIdent } from './tasks/intrigue-ident'; +import { handler as censysIpv4 } from './tasks/censysIpv4'; import { handler as cve } from './tasks/cve'; +import { handler as cveSync } from './tasks/cve-sync'; +import { handler as dnstwist } from './tasks/dnstwist'; import { handler as dotgov } from './tasks/dotgov'; -import { handler as webscraper } from './tasks/webscraper'; -import { handler as shodan } from './tasks/shodan'; -import { handler as testProxy } from './tasks/test-proxy'; +import { handler as findomain } from './tasks/findomain'; import { handler as hibp } from './tasks/hibp'; +import { handler as intrigueIdent } from './tasks/intrigue-ident'; import { handler as lookingGlass } from './tasks/lookingGlass'; -import { handler as dnstwist } from './tasks/dnstwist'; +import { handler as portscanner } from './tasks/portscanner'; import { handler as rootDomainSync } from './tasks/rootDomainSync'; +import { handler as rscSync } from './tasks/rscSync'; +import { handler as savedSearch } from './tasks/saved-search'; +import { handler as searchSync } from './tasks/search-sync'; +import { handler as shodan } from './tasks/shodan'; +import { handler as sslyze } from './tasks/sslyze'; +import { handler as testProxy } from './tasks/test-proxy'; import { handler as trustymail } from './tasks/trustymail'; -import { handler as cveSync } from './tasks/cve-sync'; import { handler as vulnSync } from './tasks/vuln-sync'; +import { handler as wappalyzer } from './tasks/wappalyzer'; +import { handler as webscraper } from './tasks/webscraper'; import { SCAN_SCHEMA } from './api/scans'; -import { connectToDatabase } from './models'; -import fetchPublicSuffixList from './tasks/helpers/fetchPublicSuffixList'; /** * Worker entrypoint. @@ -41,31 +42,32 @@ async function main() { const scanFn: any = { amass, censys, - censysIpv4, censysCertificates, - cveSync, - sslyze, - searchSync, + censysIpv4, cve, + cveSync, + dnstwist, dotgov, findomain, - portscanner, - wappalyzer, - intrigueIdent, - webscraper, - savedSearch, - shodan, hibp, + intrigueIdent, lookingGlass, - dnstwist, - testProxy, + portscanner, rootDomainSync, - trustymail, - vulnSync, + rscSync, + savedSearch, + searchSync, + shodan, + sslyze, test: async () => { await connectToDatabase(); console.log('test'); - } + }, + testProxy, + trustymail, + vulnSync, + wappalyzer, + webscraper }[scanName]; if (!scanFn) { throw new Error('Invalid scan name ' + scanName);