From 314e6dd1ee05f43193eb0ff27f2eda9ef4baa2f6 Mon Sep 17 00:00:00 2001 From: edujosemena <58452754+edujosemena@users.noreply.github.com> Date: Thu, 6 Jun 2024 12:12:24 -0400 Subject: [PATCH 01/23] Add xpanse-sync --- backend/src/api/scans.ts | 9 ++ backend/src/tasks/xpanse-sync.ts | 225 +++++++++++++++++++++++++++++++ backend/src/worker.ts | 2 + 3 files changed, 236 insertions(+) create mode 100644 backend/src/tasks/xpanse-sync.ts diff --git a/backend/src/api/scans.ts b/backend/src/api/scans.ts index 8a34358a..ee2bdcf7 100644 --- a/backend/src/api/scans.ts +++ b/backend/src/api/scans.ts @@ -246,6 +246,15 @@ export const SCAN_SCHEMA: ScanSchema = { cpu: '1024', memory: '4096', description: 'Scrapes all webpages on a given domain, respecting robots.txt' + }, + xpanseSync: { + type: 'fargate', + isPassive: true, + global: true, + description: + 'Pull in xpanse vulnerability data from PEs Vulnerability database', + cpu: '1024', + memory: '8192' } }; diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts new file mode 100644 index 00000000..1d4a9101 --- /dev/null +++ b/backend/src/tasks/xpanse-sync.ts @@ -0,0 +1,225 @@ +import { Cpe, Cve } from '../models'; +import axios from 'axios'; +import { plainToClass } from 'class-transformer'; + +interface XpanseVulnOutput { + alert_name?: string; + description?: string; + last_modified_ts?: Date; + local_insert_ts?: Date; + event_timestamp?: Date[]; + host_name?: string; + alert_action?: string; + action_country?: string[]; + action_remote_port?: number[]; + external_id?: string; + related_external_id?: string; + alert_occurrence?: number; + severity?: string; + matching_status?: string; + alert_type?: string; + resolution_status?: string; + resolution_comment?: string; + last_observed?: string; + country_codes?: string[]; + cloud_providers?: string[]; + ipv4_addresses?: string[]; + domain_names?: string[]; + port_protocol?: string; + time_pulled_from_xpanse?: string; + action_pretty?: string; + attack_surface_rule_name?: string; + certificate?: Record; + remediation_guidance?: string; + asset_identifiers?: Record[]; + services?: XpanseServiceOutput[]; +} + +interface XpanseServiceOutput { + service_id?: string; + service_name?: string; + service_type?: string; + ip_address?: string[]; + domain?: string[]; + externally_detected_providers?: string[]; + is_active?: string; + first_observed?: string; + last_observed?: string; + port?: number; + protocol?: string; + active_classifications?: string[]; + inactive_classifications?: string[]; + discovery_type?: string; + externally_inferred_vulnerability_score?: string; + externally_inferred_cves?: string[]; + service_key?: string; + service_key_type?: string; + cves?: XpanseCveOutput[]; +} + +interface XpanseCveOutput { + cve_id?: string; + cvss_score_v2?: string; + cve_severity_v2?: string; + cvss_score_v3?: string; + cve_severity_v3?: string; + inferred_cve_match_type?: string; + product?: string; + confidence?: string; + vendor?: string; + version_number?: string; + activity_status?: string; + first_observed?: string; + last_observed?: string; +} +interface TaskResponse { + task_id: string; + status: 'Pending' | 'Completed' | 'Failure'; + result: XpanseVulnOutput[]; + error?: string; +} + +const fetchPEVulnTask = async (org_acronym: string) => { + console.log('Creating task to fetch PE vuln data'); + try { + console.log(String(process.env.CF_API_KEY)); + const response = await axios({ + url: 'https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/xpanse_vulns', + method: 'POST', + headers: { + Authorization: String(process.env.CF_API_KEY), + access_token: String(process.env.PE_API_KEY), + 'Content-Type': '' // This is needed or else it breaks because axios defaults to application/json + }, + data: { + org_acronym: org_acronym + } + }); + if (response.status >= 200 && response.status < 300) { + console.log('Task request was successful'); + } else { + console.log('Task request failed'); + } + return response.data as TaskResponse; + } catch (error) { + console.log(`Error making POST request: ${error}`); + } +}; + +const fetchPEVulnData = async (task_id: string) => { + console.log('Creating task to fetch CVE data'); + try { + const response = await axios({ + url: `https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/xpanse_vulns/task/?task_id=${task_id}`, + headers: { + Authorization: String(process.env.CF_API_KEY), + access_token: String(process.env.PE_API_KEY), + 'Content-Type': '' // This is needed or else it breaks because axios defaults to application/json + } + }); + if (response.status >= 200 && response.status < 300) { + console.log('Request was successful'); + } else { + console.log('Request failed'); + } + return response.data as TaskResponse; + } catch (error) { + console.log(`Error making GET request: ${error}`); + } +}; + +async function main() { + const taskResponse: TaskResponse | undefined = + await fetchPEVulnTask('TEST_ORG'); + let validTaskResponse: TaskResponse; + + if (taskResponse !== undefined) { + validTaskResponse = taskResponse; + const data: TaskResponse | undefined = await fetchPEVulnData( + validTaskResponse.task_id + ); + if (data == undefined) { + console.log('error'); + } + } else { + console.log('error'); + } +} +export const handler = async (CommandOptions) => { + await main(); +}; +/** + * +class XpanseCveOutput(BaseModel): + """XpanseCveOutput schema class.""" + + cve_id: Optional[str] = None + cvss_score_v2: Optional[str] = None + cve_severity_v2: Optional[str] = None + cvss_score_v3: Optional[str] = None + cve_severity_v3: Optional[str] = None + inferred_cve_match_type: Optional[str] = None + product: Optional[str] = None + confidence: Optional[str] = None + vendor: Optional[str] = None + version_number: Optional[str] = None + activity_status: Optional[str] = None + first_observed: Optional[str] = None + last_observed: Optional[str] = None + +class XpanseServiceOutput(BaseModel): + """XpanseServiceOutput schema class.""" + + service_id: Optional[str] = None + service_name: Optional[str] = None + service_type: Optional[str] = None + ip_address: Optional[List[str]] = None + domain: Optional[List[str]] = None + externally_detected_providers: Optional[List[str]] = None + is_active: Optional[str] = None + first_observed: Optional[str] = None + last_observed: Optional[str] = None + port: Optional[int] = None + protocol: Optional[str] = None + active_classifications: Optional[List[str]] = None + inactive_classifications: Optional[List[str]] = None + discovery_type: Optional[str] = None + externally_inferred_vulnerability_score: Optional[str] = None + externally_inferred_cves: Optional[List[str]] = None + service_key: Optional[str] = None + service_key_type: Optional[str] = None + cves: Optional[List[XpanseCveOutput]] = None +class XpanseVulnOutput(BaseModel): + """XpanseVulnOutput schhema class.""" + + alert_name: Optional[str] = None + description: Optional[str] = None + last_modified_ts: Optional[datetime] = None + local_insert_ts: Optional[datetime] = None + event_timestamp: Optional[List[datetime]] = None + host_name: Optional[str] = None + alert_action: Optional[str] = None + action_country: Optional[List[str]] = None + action_remote_port: Optional[List[int]] = None + external_id: Optional[str] = None + related_external_id: Optional[str] = None + alert_occurrence: Optional[int] = None + severity: Optional[str] = None + matching_status: Optional[str] = None + alert_type: Optional[str] = None + resolution_status: Optional[str] = None + resolution_comment: Optional[str] = None + last_observed: Optional[str] = None + country_codes: Optional[List[str]] = None + cloud_providers: Optional[List[str]] = None + ipv4_addresses: Optional[List[str]] = None + domain_names: Optional[List[str]] = None + port_protocol: Optional[str] = None + time_pulled_from_xpanse: Optional[str] = None + action_pretty: Optional[str] = None + attack_surface_rule_name: Optional[str] = None + certificate: Optional[Dict] = None + remediation_guidance: Optional[str] = None + asset_identifiers: Optional[List[Dict]] = None + services: Optional[List[XpanseServiceOutput]] = None + */ diff --git a/backend/src/worker.ts b/backend/src/worker.ts index c30dfae4..86bf00ae 100644 --- a/backend/src/worker.ts +++ b/backend/src/worker.ts @@ -27,6 +27,7 @@ import { handler as vulnSync } from './tasks/vuln-sync'; import { handler as vulnScanningSync } from './tasks/vs_sync'; import { handler as wappalyzer } from './tasks/wappalyzer'; import { handler as webscraper } from './tasks/webscraper'; +import { handler as xpanseSync } from './tasks/xpanse-sync'; import { SCAN_SCHEMA } from './api/scans'; /** @@ -63,6 +64,7 @@ async function main() { searchSync, shodan, sslyze, + xpanseSync, test: async () => { await connectToDatabase(); console.log('test'); From 17908604c1e68907d7548fb5bc0411d862c4da76 Mon Sep 17 00:00:00 2001 From: edujosemena <58452754+edujosemena@users.noreply.github.com> Date: Fri, 7 Jun 2024 11:05:10 -0400 Subject: [PATCH 02/23] Update xpanse-sync.ts --- backend/src/tasks/xpanse-sync.ts | 94 +++++++++++++++++++++----------- 1 file changed, 62 insertions(+), 32 deletions(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index 1d4a9101..cfa1f87a 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -1,3 +1,4 @@ +import { int } from 'aws-sdk/clients/datapipeline'; import { Cpe, Cve } from '../models'; import axios from 'axios'; import { plainToClass } from 'class-transformer'; @@ -74,76 +75,105 @@ interface XpanseCveOutput { } interface TaskResponse { task_id: string; - status: 'Pending' | 'Completed' | 'Failure'; - result: XpanseVulnOutput[]; - error?: string; + status: string; + result: null | { + total_pages: number; + current_page: number; + data: XpanseVulnOutput[]; + }; + error: string | null; } -const fetchPEVulnTask = async (org_acronym: string) => { - console.log('Creating task to fetch PE vuln data'); +const createTask = async (page: number, orgId: string) => { + console.log('Creating task to fetch CVE data'); try { - console.log(String(process.env.CF_API_KEY)); const response = await axios({ - url: 'https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/xpanse_vulns', + url: 'https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/cves_by_modified_date', method: 'POST', headers: { Authorization: String(process.env.CF_API_KEY), access_token: String(process.env.PE_API_KEY), - 'Content-Type': '' // This is needed or else it breaks because axios defaults to application/json + 'Content-Type': '' //this is needed or else it breaks because axios defaults to application/json }, data: { - org_acronym: org_acronym + page: page, + per_page: 100 // Tested with 150 and 200 but this results in 502 errors on certain pages with a lot of CPEs } }); if (response.status >= 200 && response.status < 300) { - console.log('Task request was successful'); + //console.log('Request was successful'); } else { - console.log('Task request failed'); + console.log('Request failed'); } return response.data as TaskResponse; } catch (error) { console.log(`Error making POST request: ${error}`); } }; - -const fetchPEVulnData = async (task_id: string) => { - console.log('Creating task to fetch CVE data'); +const fetchData = async (task_id: string) => { + console.log('Fetching CVE data'); try { const response = await axios({ - url: `https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/xpanse_vulns/task/?task_id=${task_id}`, + url: `https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/cves_by_modified_date/task/${task_id}`, headers: { Authorization: String(process.env.CF_API_KEY), access_token: String(process.env.PE_API_KEY), - 'Content-Type': '' // This is needed or else it breaks because axios defaults to application/json + 'Content-Type': '' } }); if (response.status >= 200 && response.status < 300) { - console.log('Request was successful'); + //console.log('Request was successful'); } else { console.log('Request failed'); } return response.data as TaskResponse; } catch (error) { - console.log(`Error making GET request: ${error}`); + console.log(`Error making POST request: ${error}`); } }; +const getVulnData = async (orgId: string) => { + let done = false; + let page = 1; + let total_pages = 2; + let fullVulnArray: XpanseVulnOutput[] = []; + while (!done) { + let taskRequest = await createTask(page, orgId); + console.log(`Fetching page ${page} of page ${total_pages}`); + await new Promise((r) => setTimeout(r, 1000)); + if (taskRequest?.status == 'Processing') { + while (taskRequest?.status == 'Processing') { + //console.log('Waiting for task to complete'); + await new Promise((r) => setTimeout(r, 1000)); + taskRequest = await fetchData(taskRequest.task_id); + //console.log(taskRequest?.status); + } + if (taskRequest?.status == 'Completed') { + console.log(`Task completed successfully for page: ${page}`); -async function main() { - const taskResponse: TaskResponse | undefined = - await fetchPEVulnTask('TEST_ORG'); - let validTaskResponse: TaskResponse; - - if (taskResponse !== undefined) { - validTaskResponse = taskResponse; - const data: TaskResponse | undefined = await fetchPEVulnData( - validTaskResponse.task_id - ); - if (data == undefined) { - console.log('error'); + const vulnArray = taskRequest?.result?.data || []; //TODO, change this to CveEntry[] + fullVulnArray = fullVulnArray.concat(vulnArray); + total_pages = taskRequest?.result?.total_pages || 1; + const current_page = taskRequest?.result?.current_page || 1; + if (current_page >= total_pages) { + done = true; + console.log(`Finished fetching CVE data`); + return fullVulnArray; + } + page = page + 1; + } + } else { + done = true; + console.log( + `Error fetching CVE data: ${taskRequest?.error} and status: ${taskRequest?.status}` + ); } - } else { - console.log('error'); } +}; +async function main() { + const org_id = 'ADF'; //testing purposes + await getVulnData(org_id); + + getVulnData; } export const handler = async (CommandOptions) => { await main(); From f394c704f3d5d22d2406500ceccaeea213b9a44c Mon Sep 17 00:00:00 2001 From: edujosemena <58452754+edujosemena@users.noreply.github.com> Date: Fri, 7 Jun 2024 11:29:08 -0400 Subject: [PATCH 03/23] remove null from xpanse interface --- backend/src/tasks/xpanse-sync.ts | 145 ++++++++----------------------- 1 file changed, 35 insertions(+), 110 deletions(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index cfa1f87a..7ce365d5 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -4,38 +4,37 @@ import axios from 'axios'; import { plainToClass } from 'class-transformer'; interface XpanseVulnOutput { - alert_name?: string; - description?: string; - last_modified_ts?: Date; - local_insert_ts?: Date; - event_timestamp?: Date[]; - host_name?: string; - alert_action?: string; - action_country?: string[]; - action_remote_port?: number[]; - external_id?: string; - related_external_id?: string; - alert_occurrence?: number; - severity?: string; - matching_status?: string; - alert_type?: string; - resolution_status?: string; - resolution_comment?: string; - last_observed?: string; - country_codes?: string[]; - cloud_providers?: string[]; - ipv4_addresses?: string[]; - domain_names?: string[]; - port_protocol?: string; - time_pulled_from_xpanse?: string; - action_pretty?: string; - attack_surface_rule_name?: string; - certificate?: Record; - remediation_guidance?: string; - asset_identifiers?: Record[]; - services?: XpanseServiceOutput[]; + alert_name: string; + description: string; + last_modified_ts: Date; + local_insert_ts: Date; + event_timestamp: Date[]; + host_name: string; + alert_action: string; + action_country: string[]; + action_remote_port: number[]; + external_id: string; + related_external_id: string; + alert_occurrence: number; + severity: string; + matching_status: string; + alert_type: string; + resolution_status: string; + resolution_comment: string; + last_observed: string; + country_codes: string[]; + cloud_providers: string[]; + ipv4_addresses: string[]; + domain_names: string[]; + port_protocol: string; + time_pulled_from_xpanse: string; + action_pretty: string; + attack_surface_rule_name: string; + certificate: Record; //Change this to a + remediation_guidance: string; + asset_identifiers: Record[]; + services: XpanseServiceOutput[]; } - interface XpanseServiceOutput { service_id?: string; service_name?: string; @@ -57,7 +56,6 @@ interface XpanseServiceOutput { service_key_type?: string; cves?: XpanseCveOutput[]; } - interface XpanseCveOutput { cve_id?: string; cvss_score_v2?: string; @@ -149,7 +147,6 @@ const getVulnData = async (orgId: string) => { } if (taskRequest?.status == 'Completed') { console.log(`Task completed successfully for page: ${page}`); - const vulnArray = taskRequest?.result?.data || []; //TODO, change this to CveEntry[] fullVulnArray = fullVulnArray.concat(vulnArray); total_pages = taskRequest?.result?.total_pages || 1; @@ -166,90 +163,18 @@ const getVulnData = async (orgId: string) => { console.log( `Error fetching CVE data: ${taskRequest?.error} and status: ${taskRequest?.status}` ); + return fullVulnArray; } } }; async function main() { const org_id = 'ADF'; //testing purposes - await getVulnData(org_id); - + const vulnArray: XpanseVulnOutput[] = (await getVulnData(org_id)) || []; + for (const vuln of vulnArray) { + console.log(vuln); + } getVulnData; } export const handler = async (CommandOptions) => { await main(); }; -/** - * -class XpanseCveOutput(BaseModel): - """XpanseCveOutput schema class.""" - - cve_id: Optional[str] = None - cvss_score_v2: Optional[str] = None - cve_severity_v2: Optional[str] = None - cvss_score_v3: Optional[str] = None - cve_severity_v3: Optional[str] = None - inferred_cve_match_type: Optional[str] = None - product: Optional[str] = None - confidence: Optional[str] = None - vendor: Optional[str] = None - version_number: Optional[str] = None - activity_status: Optional[str] = None - first_observed: Optional[str] = None - last_observed: Optional[str] = None - -class XpanseServiceOutput(BaseModel): - """XpanseServiceOutput schema class.""" - - service_id: Optional[str] = None - service_name: Optional[str] = None - service_type: Optional[str] = None - ip_address: Optional[List[str]] = None - domain: Optional[List[str]] = None - externally_detected_providers: Optional[List[str]] = None - is_active: Optional[str] = None - first_observed: Optional[str] = None - last_observed: Optional[str] = None - port: Optional[int] = None - protocol: Optional[str] = None - active_classifications: Optional[List[str]] = None - inactive_classifications: Optional[List[str]] = None - discovery_type: Optional[str] = None - externally_inferred_vulnerability_score: Optional[str] = None - externally_inferred_cves: Optional[List[str]] = None - service_key: Optional[str] = None - service_key_type: Optional[str] = None - cves: Optional[List[XpanseCveOutput]] = None -class XpanseVulnOutput(BaseModel): - """XpanseVulnOutput schhema class.""" - - alert_name: Optional[str] = None - description: Optional[str] = None - last_modified_ts: Optional[datetime] = None - local_insert_ts: Optional[datetime] = None - event_timestamp: Optional[List[datetime]] = None - host_name: Optional[str] = None - alert_action: Optional[str] = None - action_country: Optional[List[str]] = None - action_remote_port: Optional[List[int]] = None - external_id: Optional[str] = None - related_external_id: Optional[str] = None - alert_occurrence: Optional[int] = None - severity: Optional[str] = None - matching_status: Optional[str] = None - alert_type: Optional[str] = None - resolution_status: Optional[str] = None - resolution_comment: Optional[str] = None - last_observed: Optional[str] = None - country_codes: Optional[List[str]] = None - cloud_providers: Optional[List[str]] = None - ipv4_addresses: Optional[List[str]] = None - domain_names: Optional[List[str]] = None - port_protocol: Optional[str] = None - time_pulled_from_xpanse: Optional[str] = None - action_pretty: Optional[str] = None - attack_surface_rule_name: Optional[str] = None - certificate: Optional[Dict] = None - remediation_guidance: Optional[str] = None - asset_identifiers: Optional[List[Dict]] = None - services: Optional[List[XpanseServiceOutput]] = None - */ From beafdf14f5d696b0937cb766ebffaa4bab36a403 Mon Sep 17 00:00:00 2001 From: edujosemena <58452754+edujosemena@users.noreply.github.com> Date: Fri, 7 Jun 2024 14:13:55 -0400 Subject: [PATCH 04/23] Update api call --- backend/src/tasks/xpanse-sync.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index 7ce365d5..27c1eafc 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -86,7 +86,7 @@ const createTask = async (page: number, orgId: string) => { console.log('Creating task to fetch CVE data'); try { const response = await axios({ - url: 'https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/cves_by_modified_date', + url: 'https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/xpanse_vulns', method: 'POST', headers: { Authorization: String(process.env.CF_API_KEY), @@ -94,6 +94,7 @@ const createTask = async (page: number, orgId: string) => { 'Content-Type': '' //this is needed or else it breaks because axios defaults to application/json }, data: { + org_acronym: orgId, page: page, per_page: 100 // Tested with 150 and 200 but this results in 502 errors on certain pages with a lot of CPEs } @@ -112,7 +113,7 @@ const fetchData = async (task_id: string) => { console.log('Fetching CVE data'); try { const response = await axios({ - url: `https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/cves_by_modified_date/task/${task_id}`, + url: `https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/xpanse_vulns/task/${task_id}`, headers: { Authorization: String(process.env.CF_API_KEY), access_token: String(process.env.PE_API_KEY), From d1165916d9da5e60cf465b739f5b942909c30632 Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Fri, 7 Jun 2024 16:56:49 -0400 Subject: [PATCH 05/23] create crossfeed compatible objects create crossfeed compatible objects to save xpanse findings --- backend/src/tasks/xpanse-sync.ts | 130 +++++++++++++++++++++++++++---- 1 file changed, 116 insertions(+), 14 deletions(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index 27c1eafc..4b6fefc3 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -1,7 +1,18 @@ import { int } from 'aws-sdk/clients/datapipeline'; +import { CommandOptions } from './ecs-client'; import { Cpe, Cve } from '../models'; import axios from 'axios'; import { plainToClass } from 'class-transformer'; +import { + connectToDatabase, + Domain, + Service, + Vulnerability, + Organization +} from '../models'; +import { isIP } from 'net'; +import * as dns from 'dns'; +import saveDomainsReturn from './helpers/saveDomainsReturn'; interface XpanseVulnOutput { alert_name: string; @@ -81,6 +92,12 @@ interface TaskResponse { }; error: string | null; } +function isIPAddress(input: string): boolean { + // Regular expression to match an IP address + const ipRegex = /^(?:\d{1,3}\.){3}\d{1,3}$/; + + return ipRegex.test(input); +} const createTask = async (page: number, orgId: string) => { console.log('Creating task to fetch CVE data'); @@ -130,13 +147,13 @@ const fetchData = async (task_id: string) => { console.log(`Error making POST request: ${error}`); } }; -const getVulnData = async (orgId: string) => { +const getVulnData = async (org: Organization, commandOptions: CommandOptions) => { let done = false; let page = 1; let total_pages = 2; - let fullVulnArray: XpanseVulnOutput[] = []; + // let fullVulnArray: XpanseVulnOutput[] = []; while (!done) { - let taskRequest = await createTask(page, orgId); + let taskRequest = await createTask(page, org.acronym); console.log(`Fetching page ${page} of page ${total_pages}`); await new Promise((r) => setTimeout(r, 1000)); if (taskRequest?.status == 'Processing') { @@ -149,33 +166,118 @@ const getVulnData = async (orgId: string) => { if (taskRequest?.status == 'Completed') { console.log(`Task completed successfully for page: ${page}`); const vulnArray = taskRequest?.result?.data || []; //TODO, change this to CveEntry[] - fullVulnArray = fullVulnArray.concat(vulnArray); + // fullVulnArray = fullVulnArray.concat(vulnArray); + saveXpanseAlert(vulnArray, org, commandOptions.scanId) total_pages = taskRequest?.result?.total_pages || 1; const current_page = taskRequest?.result?.current_page || 1; if (current_page >= total_pages) { done = true; - console.log(`Finished fetching CVE data`); - return fullVulnArray; + console.log(`Finished fetching Xpanse Alert data`); + return 1 } page = page + 1; } } else { done = true; console.log( - `Error fetching CVE data: ${taskRequest?.error} and status: ${taskRequest?.status}` + `Error fetching Xpanse Alert data: ${taskRequest?.error} and status: ${taskRequest?.status}` ); - return fullVulnArray; + return 1 } } }; -async function main() { - const org_id = 'ADF'; //testing purposes - const vulnArray: XpanseVulnOutput[] = (await getVulnData(org_id)) || []; - for (const vuln of vulnArray) { + +const saveXpanseAlert = async(alerts:XpanseVulnOutput[], org: Organization, scan_id: string) => { + for (const vuln of alerts) { console.log(vuln); + let domainId; + let service_domain; + let service_ip; + let ipOnly = false; + try{ + if (isIPAddress(vuln.host_name)) { + service_ip = vuln.host_name; + try { + service_domain = (await dns.promises.reverse(service_ip))[0]; + } catch { + service_domain = service_ip; + ipOnly = true; + } + service_ip = vuln.host_name + } else { + service_domain = vuln.host_name; + try { + service_ip = (await dns.promises.lookup(service_domain)).address; + } catch { + service_ip = null; + } + } + [domainId] = await saveXpanseDomain([ + plainToClass(Domain, { + name: service_domain, + ip: service_ip, + organization: {id: org.id}, + fromRootDomain: !ipOnly + ? service_domain.split('.').slice(-2).join('.') + : null, + discoveredBy: { id: scan_id}, + subdomainSource: `Palo Alto Expanse`, + ipOnly: ipOnly + }) + ]) + + } catch (e) { + console.error(`Failed to save domain ${vuln.host_name}`); + console.error(e); + console.error('Continuing to next vulnerability'); + continue; + } + + try { + let resolution; + if (vuln.resolution_status.includes("RESOLVED")) { + resolution = "closed"; + } else { + resolution = "open"; + } + + const vuln_obj: Vulnerability = plainToClass(Vulnerability, { + domain: domainId, + last_seen: vuln.last_modified_ts, + title: vuln.alert_name, + cve: "Xpanse Alert", + description: vuln.description, + severity: vuln.severity, + state: resolution, + structuredData:{ + + }, + source: `Palo Alto Expanse`, + needsPopulation: true, + service: null + }) + + saveXpanseVuln(vuln_obj) + // save vulns if this vulns hasn't been seen or if last_seen >= last_seen on alert in db + } catch (e) { + console.error('Could not save vulnerability. Continuing.'); + console.error(e); + } + } - getVulnData; } + export const handler = async (CommandOptions) => { - await main(); + try { + await connectToDatabase(); + const allOrgs: Organization[] = await Organization.find(); + + for (const org of allOrgs) { + (await getVulnData(org, CommandOptions )) || []; + } + } catch (e) { + console.error('Unknown failure.'); + console.error(e); + } + }; From 1423c24896cedbd0f744d9d66606e4ad328f525f Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Fri, 7 Jun 2024 17:21:28 -0400 Subject: [PATCH 06/23] update structuredData update structuredData to insert --- backend/src/tasks/xpanse-sync.ts | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index 4b6fefc3..b7884315 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -15,6 +15,7 @@ import * as dns from 'dns'; import saveDomainsReturn from './helpers/saveDomainsReturn'; interface XpanseVulnOutput { + alert_id: string; alert_name: string; description: string; last_modified_ts: Date; @@ -250,7 +251,28 @@ const saveXpanseAlert = async(alerts:XpanseVulnOutput[], org: Organization, scan severity: vuln.severity, state: resolution, structuredData:{ - + alert_id: vuln.alert_id, + xpanse_modified_ts: vuln.last_modified_ts, + xpanse_insert_ts: vuln.local_insert_ts, + xpanse_last_observed: vuln.last_observed, + event_timestamps: vuln.event_timestamp, + host_name: vuln.host_name, + action_countries: vuln.action_country, + action_remote_port: vuln.action_remote_port, + external_id: vuln.external_id, + related_external_id: vuln.related_external_id, + alert_occurrence: vuln.alert_occurrence, + resolution_status: vuln.resolution_status, + resolution_comment: vuln.resolution_comment, + country_codes: vuln.country_codes, + cloud_providers: vuln.cloud_providers, + ipv4_addresses: vuln.ipv4_addresses, + domain_names: vuln.domain_names, + port_protocol: vuln.port_protocol, + time_pulled_from_xpanse: vuln.time_pulled_from_xpanse, + attack_surface_rule_name: vuln.attack_surface_rule_name, + certificate: vuln.certificate, + remediation_guidance: vuln.remediation_guidance }, source: `Palo Alto Expanse`, needsPopulation: true, From 8391707b82c84daf07ec6249b5a9e105b2db2328 Mon Sep 17 00:00:00 2001 From: edujosemena <58452754+edujosemena@users.noreply.github.com> Date: Mon, 10 Jun 2024 12:18:47 -0400 Subject: [PATCH 07/23] Fix xpanse sync --- backend/src/tasks/xpanse-sync.ts | 160 +++++++++++++++++++------------ 1 file changed, 97 insertions(+), 63 deletions(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index b7884315..bf52cfaa 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -148,7 +148,10 @@ const fetchData = async (task_id: string) => { console.log(`Error making POST request: ${error}`); } }; -const getVulnData = async (org: Organization, commandOptions: CommandOptions) => { +const getVulnData = async ( + org: Organization, + commandOptions: CommandOptions +) => { let done = false; let page = 1; let total_pages = 2; @@ -168,13 +171,13 @@ const getVulnData = async (org: Organization, commandOptions: CommandOptions) => console.log(`Task completed successfully for page: ${page}`); const vulnArray = taskRequest?.result?.data || []; //TODO, change this to CveEntry[] // fullVulnArray = fullVulnArray.concat(vulnArray); - saveXpanseAlert(vulnArray, org, commandOptions.scanId) + await saveXpanseAlert(vulnArray, org, commandOptions.scanId); total_pages = taskRequest?.result?.total_pages || 1; const current_page = taskRequest?.result?.current_page || 1; if (current_page >= total_pages) { done = true; console.log(`Finished fetching Xpanse Alert data`); - return 1 + return 1; } page = page + 1; } @@ -183,19 +186,55 @@ const getVulnData = async (org: Organization, commandOptions: CommandOptions) => console.log( `Error fetching Xpanse Alert data: ${taskRequest?.error} and status: ${taskRequest?.status}` ); - return 1 + return 1; } } }; +const saveXpanseDomain = async (domain) => { + console.log(`Service domain: ${domain.service_domain}`); + const existingDomain = await Domain.findOneBy({ + name: domain.name, + organization: { id: domain.organization.id } + }); + if (!existingDomain) { + const newDomain = await Domain.save(domain); + return newDomain; + } else { + return existingDomain; + } +}; -const saveXpanseAlert = async(alerts:XpanseVulnOutput[], org: Organization, scan_id: string) => { +const saveXpanseVuln = async (vuln: Vulnerability, savedDomain: Domain) => { + const vulns: Vulnerability[] = []; + const existingVuln = await Vulnerability.findOneBy({ + domain: { id: savedDomain.id }, + title: vuln.title + }); + if (!existingVuln) { + const newVuln = await Vulnerability.save(vuln); + return newVuln; + } else { + if (vuln.lastSeen) { + existingVuln.lastSeen = new Date(vuln.lastSeen); + } + existingVuln.state = vuln.state; + existingVuln.structuredData = vuln.structuredData; + const newVuln = await Vulnerability.save(existingVuln); + return newVuln; + } +}; +const saveXpanseAlert = async ( + alerts: XpanseVulnOutput[], + org: Organization, + scan_id: string +) => { for (const vuln of alerts) { console.log(vuln); - let domainId; + let savedDomain: Domain; let service_domain; let service_ip; let ipOnly = false; - try{ + try { if (isIPAddress(vuln.host_name)) { service_ip = vuln.host_name; try { @@ -204,7 +243,7 @@ const saveXpanseAlert = async(alerts:XpanseVulnOutput[], org: Organization, scan service_domain = service_ip; ipOnly = true; } - service_ip = vuln.host_name + service_ip = vuln.host_name; } else { service_domain = vuln.host_name; try { @@ -213,20 +252,19 @@ const saveXpanseAlert = async(alerts:XpanseVulnOutput[], org: Organization, scan service_ip = null; } } - [domainId] = await saveXpanseDomain([ + savedDomain = await saveXpanseDomain( plainToClass(Domain, { name: service_domain, ip: service_ip, - organization: {id: org.id}, + organization: { org }, fromRootDomain: !ipOnly - ? service_domain.split('.').slice(-2).join('.') - : null, - discoveredBy: { id: scan_id}, + ? service_domain.split('.').slice(-2).join('.') + : null, + discoveredBy: { id: scan_id }, subdomainSource: `Palo Alto Expanse`, ipOnly: ipOnly }) - ]) - + ); } catch (e) { console.error(`Failed to save domain ${vuln.host_name}`); console.error(e); @@ -236,58 +274,55 @@ const saveXpanseAlert = async(alerts:XpanseVulnOutput[], org: Organization, scan try { let resolution; - if (vuln.resolution_status.includes("RESOLVED")) { - resolution = "closed"; - } else { - resolution = "open"; - } - - const vuln_obj: Vulnerability = plainToClass(Vulnerability, { - domain: domainId, - last_seen: vuln.last_modified_ts, - title: vuln.alert_name, - cve: "Xpanse Alert", - description: vuln.description, - severity: vuln.severity, - state: resolution, - structuredData:{ - alert_id: vuln.alert_id, - xpanse_modified_ts: vuln.last_modified_ts, - xpanse_insert_ts: vuln.local_insert_ts, - xpanse_last_observed: vuln.last_observed, - event_timestamps: vuln.event_timestamp, - host_name: vuln.host_name, - action_countries: vuln.action_country, - action_remote_port: vuln.action_remote_port, - external_id: vuln.external_id, - related_external_id: vuln.related_external_id, - alert_occurrence: vuln.alert_occurrence, - resolution_status: vuln.resolution_status, - resolution_comment: vuln.resolution_comment, - country_codes: vuln.country_codes, - cloud_providers: vuln.cloud_providers, - ipv4_addresses: vuln.ipv4_addresses, - domain_names: vuln.domain_names, - port_protocol: vuln.port_protocol, - time_pulled_from_xpanse: vuln.time_pulled_from_xpanse, - attack_surface_rule_name: vuln.attack_surface_rule_name, - certificate: vuln.certificate, - remediation_guidance: vuln.remediation_guidance - }, - source: `Palo Alto Expanse`, - needsPopulation: true, - service: null - }) - - saveXpanseVuln(vuln_obj) + if (vuln.resolution_status.includes('RESOLVED')) { + resolution = 'closed'; + } else { + resolution = 'open'; + } + const vuln_obj: Vulnerability = plainToClass(Vulnerability, { + domain: { savedDomain }, + last_seen: vuln.last_modified_ts, + title: vuln.alert_name, + cve: 'Xpanse Alert', + description: vuln.description, + severity: vuln.severity, + state: resolution, + structuredData: { + alert_id: vuln.alert_id, + xpanse_modified_ts: vuln.last_modified_ts, + xpanse_insert_ts: vuln.local_insert_ts, + xpanse_last_observed: vuln.last_observed, + event_timestamps: vuln.event_timestamp, + host_name: vuln.host_name, + action_countries: vuln.action_country, + action_remote_port: vuln.action_remote_port, + external_id: vuln.external_id, + related_external_id: vuln.related_external_id, + alert_occurrence: vuln.alert_occurrence, + resolution_status: vuln.resolution_status, + resolution_comment: vuln.resolution_comment, + country_codes: vuln.country_codes, + cloud_providers: vuln.cloud_providers, + ipv4_addresses: vuln.ipv4_addresses, + domain_names: vuln.domain_names, + port_protocol: vuln.port_protocol, + time_pulled_from_xpanse: vuln.time_pulled_from_xpanse, + attack_surface_rule_name: vuln.attack_surface_rule_name, + certificate: vuln.certificate, + remediation_guidance: vuln.remediation_guidance + }, + source: `Palo Alto Expanse`, + needsPopulation: true, + service: null + }); + await saveXpanseVuln(vuln_obj, savedDomain); // save vulns if this vulns hasn't been seen or if last_seen >= last_seen on alert in db } catch (e) { console.error('Could not save vulnerability. Continuing.'); console.error(e); } - } -} +}; export const handler = async (CommandOptions) => { try { @@ -295,11 +330,10 @@ export const handler = async (CommandOptions) => { const allOrgs: Organization[] = await Organization.find(); for (const org of allOrgs) { - (await getVulnData(org, CommandOptions )) || []; + (await getVulnData(org, CommandOptions)) || []; } } catch (e) { console.error('Unknown failure.'); console.error(e); } - }; From a5b016d2a46a50efbe6e50c9d486e0cfdaa19f5f Mon Sep 17 00:00:00 2001 From: edujosemena <58452754+edujosemena@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:26:38 -0400 Subject: [PATCH 08/23] Update findOne --- backend/src/tasks/xpanse-sync.ts | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index bf52cfaa..7a41b5e7 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -192,33 +192,44 @@ const getVulnData = async ( }; const saveXpanseDomain = async (domain) => { console.log(`Service domain: ${domain.service_domain}`); - const existingDomain = await Domain.findOneBy({ - name: domain.name, - organization: { id: domain.organization.id } + const existingDomain = await Domain.findOne({ + where: { + name: domain.name, + organization: { id: domain.organization.id } + }, + relations: ['organization'] }); if (!existingDomain) { + console.log(`Saving domain ${domain}`); const newDomain = await Domain.save(domain); return newDomain; } else { return existingDomain; } -}; - +}; // save vulns if this vulns hasn't been seen or if last_seen >= last_seen on alert in db const saveXpanseVuln = async (vuln: Vulnerability, savedDomain: Domain) => { const vulns: Vulnerability[] = []; - const existingVuln = await Vulnerability.findOneBy({ - domain: { id: savedDomain.id }, - title: vuln.title + const existingVuln = await Vulnerability.findOne({ + where: { + domain: { id: savedDomain.id }, + title: vuln.title //Change this for typeorm bump + } }); if (!existingVuln) { const newVuln = await Vulnerability.save(vuln); return newVuln; } else { if (vuln.lastSeen) { + if (existingVuln.lastSeen) { + if (existingVuln.lastSeen >= vuln.lastSeen) { + return existingVuln; + } + } existingVuln.lastSeen = new Date(vuln.lastSeen); } existingVuln.state = vuln.state; existingVuln.structuredData = vuln.structuredData; + existingVuln.domain = savedDomain; const newVuln = await Vulnerability.save(existingVuln); return newVuln; } @@ -256,7 +267,7 @@ const saveXpanseAlert = async ( plainToClass(Domain, { name: service_domain, ip: service_ip, - organization: { org }, + organization: org, fromRootDomain: !ipOnly ? service_domain.split('.').slice(-2).join('.') : null, @@ -280,7 +291,7 @@ const saveXpanseAlert = async ( resolution = 'open'; } const vuln_obj: Vulnerability = plainToClass(Vulnerability, { - domain: { savedDomain }, + domain: savedDomain, last_seen: vuln.last_modified_ts, title: vuln.alert_name, cve: 'Xpanse Alert', From a90af2958d545973c1042bdd43092592ea40acaf Mon Sep 17 00:00:00 2001 From: Amelia Vance Date: Mon, 10 Jun 2024 17:43:00 -0400 Subject: [PATCH 09/23] Refactor vuln details to put cve details in its own file and conditional to having a cve --- .../src/pages/Vulnerability/CveSection.tsx | 248 +++++++ .../src/pages/Vulnerability/Vulnerability.tsx | 631 ++++++++---------- 2 files changed, 514 insertions(+), 365 deletions(-) create mode 100644 frontend/src/pages/Vulnerability/CveSection.tsx diff --git a/frontend/src/pages/Vulnerability/CveSection.tsx b/frontend/src/pages/Vulnerability/CveSection.tsx new file mode 100644 index 00000000..fe32dd29 --- /dev/null +++ b/frontend/src/pages/Vulnerability/CveSection.tsx @@ -0,0 +1,248 @@ +import React from 'react'; +import { + Box, + Grid, + Link as LinkMui, + Paper, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, + Typography +} from '@mui/material'; +import { getCVSSColor } from 'pages/Risk/utils'; +import { + Cve as CveType, + Cpe as ProductInfoType, + Vulnerability as VulnerabilityType +} from 'types'; + +interface GroupedByVendor { + [key: string]: ProductInfoType[]; +} + +interface HasCVEProps { + vulnerability: VulnerabilityType; + cve: CveType | undefined; + VulnDescription: any; + CweSection: React.FC; +} + +export const CveSection: React.FC = ({ + vulnerability, + cve, + VulnDescription, + CweSection +}) => { + const references = vulnerability.references.map((ref) => ref); + if (vulnerability.cve) + references.unshift({ + name: 'NIST National Vulnerability Database', + url: `https://nvd.nist.gov/vuln/detail/${vulnerability.cve}`, + source: '', + tags: [] + }); + const groupedByVendor: GroupedByVendor = (cve?.cpes ?? []).reduce( + (acc: GroupedByVendor, current: ProductInfoType) => { + const { vendor, ...rest } = current; + // If the vendor exists, push the current object to its array + if (acc[vendor]) { + acc[vendor].push(rest); + } else { + // Create a new array with the current object + acc[vendor] = [rest]; + } + return acc; + }, + {} as GroupedByVendor + ); + const SeverityMetricsSection = () => { + return ( + + + CVSS 3.x Severity & Metrics + + + + + + NIST:{' '} + + + {cve?.cvssV3Source != null + ? cve?.cvssV3Source.split('@')[0].toUpperCase() + : null} + + + + + Base Score:{' '} + + +   {cve?.cvssV3BaseScore}  + {getCVSSColor(Number(cve?.cvssV3BaseScore))[1]}   + + + + + Vector: + + + {cve?.cvssV3VectorString} + + + + + CVSS 2.0 Severity & Metrics + + + + + + NIST:{' '} + + + {cve?.cvssV2Source != null + ? cve?.cvssV2Source.split('@')[0].toUpperCase() + : null} + + + + + Base Score:{' '} + + +   {cve?.cvssV2BaseScore}  + {getCVSSColor(Number(cve?.cvssV2BaseScore))[1]}   + + + + + Vector:{' '} + + + {cve?.cvssV2VectorString} + + + + + ); + }; + const CVEReferences = () => { + return ( + <> + + CVE References + + + + + + Hyperlink + Resource + + + + {vulnerability.references.map((row) => ( + + + {row.url} + + {row.source} + + ))} + +
+ + + ); + }; + const ProductsAffectedSection = () => { + return ( + + Products Affected + {Object.entries(groupedByVendor).map(([vendor, values]) => ( +
+ {vendor} + +
    + {values.map((value, index) => ( +
  • {value.name}
    • + ))} +
+ +
+ ))} + + ); + }; + return ( + <> + + + + + + + + + + + ); +}; diff --git a/frontend/src/pages/Vulnerability/Vulnerability.tsx b/frontend/src/pages/Vulnerability/Vulnerability.tsx index f46522bc..da7701a1 100644 --- a/frontend/src/pages/Vulnerability/Vulnerability.tsx +++ b/frontend/src/pages/Vulnerability/Vulnerability.tsx @@ -1,11 +1,10 @@ import React, { useCallback, useEffect, useState } from 'react'; import { Link, useParams, useHistory } from 'react-router-dom'; -import { differenceInCalendarDays, format, parseISO } from 'date-fns'; +import { differenceInCalendarDays, parseISO } from 'date-fns'; import { ChevronLeft, OpenInNew } from '@mui/icons-material'; import { AppBar, Box, - Button, Grid, IconButton, Link as LinkMui, @@ -19,24 +18,11 @@ import { Toolbar, Typography } from '@mui/material'; -import { getSeverityColor, getCVSSColor } from 'pages/Risk/utils'; +import { tableCellClasses } from '@mui/material/TableCell'; +import { getSeverityColor } from 'pages/Risk/utils'; import { useAuthContext } from 'context'; -import { - Cve as CveType, - Cpe as ProductInfoType, - Vulnerability as VulnerabilityType -} from 'types'; - -const formatDate = (date: string | null) => { - if (date) { - return format(parseISO(date), 'MM/dd/yyyy'); - } - return 'Date not found'; -}; - -interface GroupedByVendor { - [key: string]: ProductInfoType[]; -} +import { Cve as CveType, Vulnerability as VulnerabilityType } from 'types'; +import { CveSection } from './CveSection'; interface Product { cpe?: string | null; @@ -102,8 +88,11 @@ export const Vulnerability: React.FC = () => { const result = await apiGet( `/vulnerabilities/${vulnerabilityId}` ); + console.log(result); setVulnerability(result); - getCve(result.cve); + if (result.cve) { + getCve(result.cve); + } } catch (e) { console.error(e); } @@ -116,21 +105,6 @@ export const Vulnerability: React.FC = () => { if (!vulnerability) return <>No Vulnerabilities; - const groupedByVendor: GroupedByVendor = (cve?.cpes ?? []).reduce( - (acc: GroupedByVendor, current: ProductInfoType) => { - const { vendor, ...rest } = current; - // If the vendor exists, push the current object to its array - if (acc[vendor]) { - acc[vendor].push(rest); - } else { - // Create a new array with the current object - acc[vendor] = [rest]; - } - return acc; - }, - {} as GroupedByVendor - ); - const generateWebInfo = (service: Service): WebInfoItem[] => { const categoriesToProducts: Record> = {}; for (const product of service.products) { @@ -209,7 +183,7 @@ export const Vulnerability: React.FC = () => { return ( - + {vulnerability.domain.name} @@ -218,7 +192,9 @@ export const Vulnerability: React.FC = () => { edge="start" color="inherit" aria-label="menu" - href={`/inventory/domain/${vulnerability.domain.id}`} + onClick={handleOpenInNewTab( + `/inventory/domain/${vulnerability.domain.id}` + )} > @@ -226,37 +202,78 @@ export const Vulnerability: React.FC = () => { - - Overview - - IP: {vulnerability.domain.ip} -
- First Seen:{' '} - {differenceInCalendarDays( - Date.now(), - parseISO(vulnerability.domain.createdAt) - )}{' '} - days ago -
- Last Seen:{' '} - {differenceInCalendarDays( - Date.now(), - parseISO(vulnerability.domain.updatedAt) - )}{' '} - days ago -
- Country:{' '} - {vulnerability.domain.country - ? vulnerability.domain.country - : 'Not found'} -
- Organization: {vulnerability.domain.organization.name} -
+ + + Overview + + + + + + IP: + + + {vulnerability.domain.ip} + + + + + First Seen: + + + {differenceInCalendarDays( + Date.now(), + parseISO(vulnerability.domain.createdAt) + )}{' '} + day(s) ago + + + + + Last Seen: + + + {differenceInCalendarDays( + Date.now(), + parseISO(vulnerability.domain.updatedAt) + )}{' '} + day(s) ago + + + + + Country: + + + {vulnerability.domain.country + ? vulnerability.domain.country + : 'Not found'} + + + + + Organization: + + + {vulnerability.domain.organization.name} + + + +
+
{generateWebInfo(vulnerability.service).length > 0 && ( <> - + Installed (Known) Products {generateWebInfo(vulnerability.service).map( @@ -269,319 +286,162 @@ export const Vulnerability: React.FC = () => { )} - + Provenance - - - Root Domain: {vulnerability.domain.fromRootDomain} -
- Subdomain: {vulnerability.domain.name} ( - {vulnerability.domain.subdomainSource})
- Service/Port:{' '} - {vulnerability.service.service - ? vulnerability.service.service - : vulnerability.service.port}{' '} - ({vulnerability.service.serviceSource})
- Product: {vulnerability.cpe} -
- Vulnerability: {vulnerability.title} ( - {vulnerability.source}) - - + + + + + + Root Domain: + + + {vulnerability.domain.fromRootDomain} + + + + + Subdomain: + + + {vulnerability.domain.name} ( + {vulnerability.domain.subdomainSource}) + + + + + Service/Port: + + + {vulnerability.service.service + ? vulnerability.service.service + : vulnerability.service.port}{' '} + ({vulnerability.service.serviceSource}) + + + {vulnerability.cpe && ( + + + Product: + + {vulnerability.cpe} + + )} + + + Vulnerability: + + + {vulnerability.title} ({vulnerability.source}) + + + +
+ ); }; - const CVEHighlightSection = () => { + const VulnHighlightSection = () => { return ( - - - - - - - - {vulnerability.cve} - - - - - {vulnerability.severity} - - - {vulnerability.isKev ? 'Yes' : 'No'} - - - {vulnerability.domain.name} - - - - {product - ? product.name + - (product.version ? ' ' + product.version : '') - : vulnerability.cpe} - - - {lastState === 'open' - ? (daysOpen += differenceInCalendarDays( - new Date(), - parseISO(lastOpenDate) - )) - : daysOpen} - - {vulnState} - - -
- - + + + + + {vulnerability.cve || vulnerability.title} + + + {vulnerability?.severity} + + + {vulnerability?.isKev ? 'Yes' : 'No'} + + + {vulnerability?.domain.name} + + + + {product + ? product.name + + (product.version ? ' ' + product.version : '') + : vulnerability?.cpe} + + + {lastState === 'open' + ? (daysOpen += differenceInCalendarDays( + new Date(), + parseISO(lastOpenDate) + )) + : daysOpen} + + {vulnState} + + +
+ ); }; - const CVEReferencesCWESection = () => { + + const VulnDescription = () => { return ( - - - {vulnerability.cve} - - - + <> + Description - {vulnerability.description} - - References - - - - - - Hyperlink - Resource - - - - {vulnerability.references.map((row) => ( - - - {row.url} - - {row.source} - - ))} - -
- - - - - - CWE-ID - Source - - - - - - - {vulnerability.cwe} - - - {vulnerability.source} - - -
- - + {vulnerability.description} + ); }; - const SeverityAffectedHistorySection = () => { + const CweSection = () => { return ( - - - - CVSS 3.x Severity & Metrics - - - - - - NIST:{' '} - - - {cve?.cvssV3Source != null - ? cve?.cvssV3Source.split('@')[0].toUpperCase() - : null} - - - - - Base Score:{' '} - - -   {cve?.cvssV3BaseScore}  - {getCVSSColor(Number(cve?.cvssV3BaseScore))[1]}   - - - - - Vector: - - - {cve?.cvssV3VectorString} - - - - - CVSS 2.0 Severity & Metrics - - - - - - NIST:{' '} - - - {cve?.cvssV2Source != null - ? cve?.cvssV2Source.split('@')[0].toUpperCase() - : null} - - - - - Base Score:{' '} - - -   {cve?.cvssV2BaseScore}  - {getCVSSColor(Number(cve?.cvssV2BaseScore))[1]}   - - - - - Vector:{' '} - - - {cve?.cvssV2VectorString} - - - - - - Products Affected - {Object.entries(groupedByVendor).map(([vendor, values]) => ( -
- {vendor} - -
    - {values.map((value, index) => ( -
  • {value.name}
    • - ))} -
- -
- ))} - - - Vulnerability Detection History - - First Detected - - - Vulnerability Opened: {formatDate(vulnerability.createdAt)} - - - Last Detected - - - State automatically changed to {vulnState.toLowerCase()}:{' '} - {formatDate(vulnerability.lastSeen)} - - - + + + + + CWE-ID + Source + + + + + + {vulnerability.cwe ? ( + + {vulnerability.cwe} + + ) : ( + 'Not found' + )} + + {vulnerability.source} + + +
+ ); }; // TODO: Discuss options to add manual notes. https://github.com/cisagov/crossfeed/issues/2519 @@ -593,6 +453,12 @@ export const Vulnerability: React.FC = () => { // // ; // }; + + const handleOpenInNewTab = (url: string) => (event: React.MouseEvent) => { + event.preventDefault(); + window.open(url, '_blank', 'noopener,noreferrer'); + }; + return ( @@ -616,15 +482,50 @@ export const Vulnerability: React.FC = () => { - - - + <> + + + + + + - - - {/* */} + + + + {vulnerability.cve || vulnerability.title} + + {vulnerability.cve && ( + + + + )} + + + {!vulnerability.cve && ( + <> + + + + + + + + )} + {vulnerability.cve && ( + + )} From dd629d7b93fe6f81ee2bb5ecc09a08be6401bbe6 Mon Sep 17 00:00:00 2001 From: Amelia Vance Date: Mon, 10 Jun 2024 17:44:51 -0400 Subject: [PATCH 10/23] Add placeholder file for xpanse section for vuln details --- .../src/pages/Vulnerability/XpanseSection.tsx | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 frontend/src/pages/Vulnerability/XpanseSection.tsx diff --git a/frontend/src/pages/Vulnerability/XpanseSection.tsx b/frontend/src/pages/Vulnerability/XpanseSection.tsx new file mode 100644 index 00000000..4ef38517 --- /dev/null +++ b/frontend/src/pages/Vulnerability/XpanseSection.tsx @@ -0,0 +1,172 @@ +import React from 'react'; +import { + Grid, + Table, + TableBody, + TableCell, + TableContainer, + TableRow, + Typography +} from '@mui/material'; +import { tableCellClasses } from '@mui/material/TableCell'; + +// TODO: Remove and update to vulnerability data and format once it is available. +const testXpanseData = { + firstObserved: 'Jun 04 2024', + lastObserved: 'Jun 05 2024', + ipv4Address: '1000.00.00.00', + portNum: '10000', + portProtocol: 'TCP', + httpPath: '/', + remediation: + // "1. Verify backported versions requires system access and cannot be accurately done from the Internet, it is recommended to confirm vulnerabilities on the system through a change log or other means.\n2. Outdated versions may no longer get security updates on a regular basis and are prime targets for attackers. It is recommended to upgrade to the latest version of Apache Web Server and apply latest security updates.This can be done via [https://httpd.apache.org/download.cgi](https://httpd.apache.org/download.cgi).\n3. Ensure all files outside the document root are secured with 'require all denied' in the server's configuration. Details on how to do so can be found at [https://httpd.apache.org/docs/2.4/misc/security_tips.html](https://httpd.apache.org/docs/2.4/misc/security_tips.html)." + 'OpenSSH versions under 9.3 allow for methods of remote code execution, privilege escalation, information disclosure and modification, as well as denial of service. When using OpenSSH ensure the following:\n\n1. Backported versions require system access and cannot be done accurately from the Internet, it is recommended to verify vulnerabilities on the system through a change log or other means.\n2. Outdated versions may no longer get security updates on a regular basis and are prime targets for attackers. It is recommended to upgrade to a supported version of OpenSSH and apply latest security updates. Latest versions can be found at [https://www.openssh.com/openbsd.html](https://www.openssh.com/openbsd.html).' +}; + +const parseTextToJSX = (text: string) => { + const lines = text.split('\n'); + return lines.map((line, index) => { + const parts = line.split(/(\[.*?\]\(.*?\))/g).map((part, i) => { + const match = part.match(/\[(.*?)\]\((.*?)\)/); + if (match) { + return ( + + {match[1]} + + ); + } + return part; + }); + return ( + + {parts} + + ); + }); +}; + +export const XpanseSection: React.FC = () => { + return ( + + + + Alert Policy Description + + + This issue flags all Eclipse Jetty web servers with versions xx, xx, + or xx. These versions of Jetty web server have multiple CVEs + associated with them with impacts ranging from denial of service to + local privilege escalation. + + + Alert Evidence + + + + + + + First Observed + + + {testXpanseData.firstObserved} + + + + + Last Observed + + {testXpanseData.lastObserved} + + + + IPv4 Address + + {testXpanseData.ipv4Address} + + + + Port Number + + {testXpanseData.portNum} + + + + Port Protocol + + {testXpanseData.portProtocol} + + + + HTTP Path + + {testXpanseData.httpPath} + + +
+ + + Certificate + + + + + + + Subject Name + + + localhost.localdomain + + + + + Issuer Name + + + localhost.localdomain + + + + + Serial Number + + + 123456789012345678901234567890 + + + + + Valid Not Before + + Apr 05 2024 + + + + Valid Not After + + Apr 03 2034 + + +
+ + + + + How to Remediate + + {parseTextToJSX(testXpanseData.remediation)} + + + ); +}; From db5a30f46c9c1ef0d87f4b0818104779903d564d Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Wed, 12 Jun 2024 09:25:03 -0400 Subject: [PATCH 11/23] add logging and minor fixes add logging and minor fixes --- backend/src/tasks/xpanse-sync.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index 7a41b5e7..d4b0aceb 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -101,7 +101,7 @@ function isIPAddress(input: string): boolean { } const createTask = async (page: number, orgId: string) => { - console.log('Creating task to fetch CVE data'); + console.log('Creating task to fetch Xpanse Alert data'); try { const response = await axios({ url: 'https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/xpanse_vulns', @@ -128,7 +128,7 @@ const createTask = async (page: number, orgId: string) => { } }; const fetchData = async (task_id: string) => { - console.log('Fetching CVE data'); + console.log('Fetching Xpanse alert data'); try { const response = await axios({ url: `https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/xpanse_vulns/task/${task_id}`, @@ -154,7 +154,7 @@ const getVulnData = async ( ) => { let done = false; let page = 1; - let total_pages = 2; + let total_pages = 1; // let fullVulnArray: XpanseVulnOutput[] = []; while (!done) { let taskRequest = await createTask(page, org.acronym); @@ -176,7 +176,7 @@ const getVulnData = async ( const current_page = taskRequest?.result?.current_page || 1; if (current_page >= total_pages) { done = true; - console.log(`Finished fetching Xpanse Alert data`); + console.log(`Finished fetching Xpanse Alert data for ${org.name}`); return 1; } page = page + 1; @@ -191,7 +191,6 @@ const getVulnData = async ( } }; const saveXpanseDomain = async (domain) => { - console.log(`Service domain: ${domain.service_domain}`); const existingDomain = await Domain.findOne({ where: { name: domain.name, @@ -240,7 +239,6 @@ const saveXpanseAlert = async ( scan_id: string ) => { for (const vuln of alerts) { - console.log(vuln); let savedDomain: Domain; let service_domain; let service_ip; @@ -265,7 +263,7 @@ const saveXpanseAlert = async ( } savedDomain = await saveXpanseDomain( plainToClass(Domain, { - name: service_domain, + name: service_domain.toLowerCase(), ip: service_ip, organization: org, fromRootDomain: !ipOnly @@ -277,7 +275,7 @@ const saveXpanseAlert = async ( }) ); } catch (e) { - console.error(`Failed to save domain ${vuln.host_name}`); + console.error(`Failed to save domain for host: ${vuln.host_name}`); console.error(e); console.error('Continuing to next vulnerability'); continue; @@ -292,7 +290,7 @@ const saveXpanseAlert = async ( } const vuln_obj: Vulnerability = plainToClass(Vulnerability, { domain: savedDomain, - last_seen: vuln.last_modified_ts, + lastSeen: vuln.last_modified_ts, title: vuln.alert_name, cve: 'Xpanse Alert', description: vuln.description, @@ -320,7 +318,8 @@ const saveXpanseAlert = async ( time_pulled_from_xpanse: vuln.time_pulled_from_xpanse, attack_surface_rule_name: vuln.attack_surface_rule_name, certificate: vuln.certificate, - remediation_guidance: vuln.remediation_guidance + remediation_guidance: vuln.remediation_guidance, + asset_identifiers: vuln.asset_identifiers }, source: `Palo Alto Expanse`, needsPopulation: true, @@ -341,6 +340,7 @@ export const handler = async (CommandOptions) => { const allOrgs: Organization[] = await Organization.find(); for (const org of allOrgs) { + console.log(`Gathering Expanse alerts for ${org.name}`); (await getVulnData(org, CommandOptions)) || []; } } catch (e) { From 3dbf5858b36f2d2b539c15a0f105ce5ccd06b0cf Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Wed, 12 Jun 2024 11:31:01 -0400 Subject: [PATCH 12/23] Fix naming of Xpanse alerts Fix naming of Xpanse alerts --- backend/src/tasks/xpanse-sync.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index d4b0aceb..706afb49 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -199,7 +199,7 @@ const saveXpanseDomain = async (domain) => { relations: ['organization'] }); if (!existingDomain) { - console.log(`Saving domain ${domain}`); + console.log(`Saving domain ${domain.name}`); const newDomain = await Domain.save(domain); return newDomain; } else { @@ -270,7 +270,7 @@ const saveXpanseAlert = async ( ? service_domain.split('.').slice(-2).join('.') : null, discoveredBy: { id: scan_id }, - subdomainSource: `Palo Alto Expanse`, + subdomainSource: `Palo Alto Xpanse`, ipOnly: ipOnly }) ); @@ -321,7 +321,7 @@ const saveXpanseAlert = async ( remediation_guidance: vuln.remediation_guidance, asset_identifiers: vuln.asset_identifiers }, - source: `Palo Alto Expanse`, + source: `Palo Alto Xpanse`, needsPopulation: true, service: null }); @@ -340,7 +340,7 @@ export const handler = async (CommandOptions) => { const allOrgs: Organization[] = await Organization.find(); for (const org of allOrgs) { - console.log(`Gathering Expanse alerts for ${org.name}`); + console.log(`Gathering Xpanse alerts for ${org.name}`); (await getVulnData(org, CommandOptions)) || []; } } catch (e) { From d8ad00a2f9c68ecaeee859d945dd7194331196c7 Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Fri, 14 Jun 2024 12:27:38 -0400 Subject: [PATCH 13/23] capitalize severity when inserting capitalize severity when inserting --- backend/src/tasks/xpanse-sync.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index 706afb49..e3165a39 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -294,7 +294,7 @@ const saveXpanseAlert = async ( title: vuln.alert_name, cve: 'Xpanse Alert', description: vuln.description, - severity: vuln.severity, + severity: vuln.severity !== null ? vuln.severity.charAt(0).toUpperCase() + vuln.severity.slice(1).toLowerCase() : null, state: resolution, structuredData: { alert_id: vuln.alert_id, From 85bbf8016f14004b67d9da645f51ab2af96e2fee Mon Sep 17 00:00:00 2001 From: Amelia Vance Date: Fri, 14 Jun 2024 12:34:57 -0400 Subject: [PATCH 14/23] Add conditions per vuln type to display in Vulnerability.tsx --- .../src/pages/Vulnerability/Vulnerability.tsx | 100 ++++++++++-------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/frontend/src/pages/Vulnerability/Vulnerability.tsx b/frontend/src/pages/Vulnerability/Vulnerability.tsx index da7701a1..93f57c84 100644 --- a/frontend/src/pages/Vulnerability/Vulnerability.tsx +++ b/frontend/src/pages/Vulnerability/Vulnerability.tsx @@ -23,6 +23,7 @@ import { getSeverityColor } from 'pages/Risk/utils'; import { useAuthContext } from 'context'; import { Cve as CveType, Vulnerability as VulnerabilityType } from 'types'; import { CveSection } from './CveSection'; +import { XpanseSection } from './XpanseSection'; interface Product { cpe?: string | null; @@ -271,20 +272,21 @@ export const Vulnerability: React.FC = () => {
- {generateWebInfo(vulnerability.service).length > 0 && ( - <> - - Installed (Known) Products - - {generateWebInfo(vulnerability.service).map( - ({ label, value }) => ( - - {label}: {value} - - ) - )} - - )} + {vulnerability.service && + generateWebInfo(vulnerability.service).length > 0 && ( + <> + + Installed (Known) Products + + {generateWebInfo(vulnerability.service).map( + ({ label, value }) => ( + + {label}: {value} + + ) + )} + + )} @@ -318,17 +320,30 @@ export const Vulnerability: React.FC = () => { {vulnerability.domain.subdomainSource}) - - - Service/Port: - - - {vulnerability.service.service - ? vulnerability.service.service - : vulnerability.service.port}{' '} - ({vulnerability.service.serviceSource}) - - + {vulnerability.service && vulnerability.service.service ? ( + + + Service/Port: + + + {vulnerability.service.service} + + + ) : vulnerability.service && + vulnerability.service.port && + vulnerability.service.serviceSource ? ( + + + Service/Port: + + + {vulnerability.service.port} ( + {vulnerability.service.serviceSource}) + + + ) : ( + <> + )} {vulnerability.cpe && ( @@ -360,7 +375,7 @@ export const Vulnerability: React.FC = () => { - {vulnerability.cve || vulnerability.title} + {vulnerability.title} { ); }; - const VulnDescription = () => { return ( <> @@ -444,15 +458,6 @@ export const Vulnerability: React.FC = () => { ); }; - // TODO: Discuss options to add manual notes. https://github.com/cisagov/crossfeed/issues/2519 - // const NotesSection = () => { - // - // - // Team Notes - // Add new note - // - // ; - // }; const handleOpenInNewTab = (url: string) => (event: React.MouseEvent) => { event.preventDefault(); @@ -495,9 +500,9 @@ export const Vulnerability: React.FC = () => { - {vulnerability.cve || vulnerability.title} + {vulnerability.title} - {vulnerability.cve && ( + {vulnerability.cve && vulnerability.cve.startsWith('CVE') && ( { )} - {!vulnerability.cve && ( + {vulnerability.source === 'Palo Alto Xpanse' ? ( + + ) : vulnerability.cve && vulnerability.cve.startsWith('CVE') ? ( + + ) : ( <> @@ -518,14 +532,6 @@ export const Vulnerability: React.FC = () => { )} - {vulnerability.cve && ( - - )} From 9bf40433feeeea1483480a312ab477b7c3372bbe Mon Sep 17 00:00:00 2001 From: Amelia Vance Date: Fri, 14 Jun 2024 12:35:48 -0400 Subject: [PATCH 15/23] Add resolution status section to XpanseSection.tsx --- frontend/src/pages/Vulnerability/XpanseSection.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frontend/src/pages/Vulnerability/XpanseSection.tsx b/frontend/src/pages/Vulnerability/XpanseSection.tsx index 4ef38517..98bbebd2 100644 --- a/frontend/src/pages/Vulnerability/XpanseSection.tsx +++ b/frontend/src/pages/Vulnerability/XpanseSection.tsx @@ -102,6 +102,12 @@ export const XpanseSection: React.FC = () => { {testXpanseData.httpPath} + + + Resolution Status + + STATUS_000_NEW +
From d7d0c21573bfb97449529cb393aab24895ee4f6d Mon Sep 17 00:00:00 2001 From: Amelia Vance Date: Fri, 14 Jun 2024 15:42:46 -0400 Subject: [PATCH 16/23] Update XpanseSection.tsx with vulnerability data --- .../src/pages/Vulnerability/XpanseSection.tsx | 217 +++++++++--------- 1 file changed, 113 insertions(+), 104 deletions(-) diff --git a/frontend/src/pages/Vulnerability/XpanseSection.tsx b/frontend/src/pages/Vulnerability/XpanseSection.tsx index 98bbebd2..ad85b1cd 100644 --- a/frontend/src/pages/Vulnerability/XpanseSection.tsx +++ b/frontend/src/pages/Vulnerability/XpanseSection.tsx @@ -9,55 +9,24 @@ import { Typography } from '@mui/material'; import { tableCellClasses } from '@mui/material/TableCell'; +import Markdown from 'react-markdown'; +import { Vulnerability as VulnerabilityType } from 'types'; +import { humanReadableDate } from 'utils/dateUtils'; -// TODO: Remove and update to vulnerability data and format once it is available. -const testXpanseData = { - firstObserved: 'Jun 04 2024', - lastObserved: 'Jun 05 2024', - ipv4Address: '1000.00.00.00', - portNum: '10000', - portProtocol: 'TCP', - httpPath: '/', - remediation: - // "1. Verify backported versions requires system access and cannot be accurately done from the Internet, it is recommended to confirm vulnerabilities on the system through a change log or other means.\n2. Outdated versions may no longer get security updates on a regular basis and are prime targets for attackers. It is recommended to upgrade to the latest version of Apache Web Server and apply latest security updates.This can be done via [https://httpd.apache.org/download.cgi](https://httpd.apache.org/download.cgi).\n3. Ensure all files outside the document root are secured with 'require all denied' in the server's configuration. Details on how to do so can be found at [https://httpd.apache.org/docs/2.4/misc/security_tips.html](https://httpd.apache.org/docs/2.4/misc/security_tips.html)." - 'OpenSSH versions under 9.3 allow for methods of remote code execution, privilege escalation, information disclosure and modification, as well as denial of service. When using OpenSSH ensure the following:\n\n1. Backported versions require system access and cannot be done accurately from the Internet, it is recommended to verify vulnerabilities on the system through a change log or other means.\n2. Outdated versions may no longer get security updates on a regular basis and are prime targets for attackers. It is recommended to upgrade to a supported version of OpenSSH and apply latest security updates. Latest versions can be found at [https://www.openssh.com/openbsd.html](https://www.openssh.com/openbsd.html).' -}; - -const parseTextToJSX = (text: string) => { - const lines = text.split('\n'); - return lines.map((line, index) => { - const parts = line.split(/(\[.*?\]\(.*?\))/g).map((part, i) => { - const match = part.match(/\[(.*?)\]\((.*?)\)/); - if (match) { - return ( - - {match[1]} - - ); - } - return part; - }); - return ( - - {parts} - - ); - }); -}; +interface XpanseSectionProps { + vulnerability: VulnerabilityType; +} -export const XpanseSection: React.FC = () => { +export const XpanseSection: React.FC = ({ + vulnerability +}) => { return ( Alert Policy Description - - This issue flags all Eclipse Jetty web servers with versions xx, xx, - or xx. These versions of Jetty web server have multiple CVEs - associated with them with impacts ranging from denial of service to - local privilege escalation. - + {vulnerability.description} Alert Evidence @@ -66,112 +35,152 @@ export const XpanseSection: React.FC = () => { - First Observed + First Observed in Xpanse - {testXpanseData.firstObserved} + {humanReadableDate( + vulnerability.structuredData?.xpanse_insert_ts + ) || 'N/A'} - Last Observed + Last Observed in Xpanse + + + {humanReadableDate( + vulnerability.structuredData?.xpanse_last_observed + ) || 'N/A'} - {testXpanseData.lastObserved} IPv4 Address - {testXpanseData.ipv4Address} + + {vulnerability.structuredData?.ipv4_addresses || 'N/A'} + Port Number - {testXpanseData.portNum} + + {vulnerability.structuredData?.action_remote_port || 'N/A'} + Port Protocol - {testXpanseData.portProtocol} - - - - HTTP Path + + {vulnerability.structuredData?.port_protocol || 'N/A'} - {testXpanseData.httpPath} - Resolution Status - - STATUS_000_NEW - - - - - - Certificate - - - - - - - Subject Name + HTTP Path - - localhost.localdomain + + {vulnerability.structuredData?.asset_identifiers[0] + .httpPath || 'N/A'} - - Issuer Name - - localhost.localdomain - - - - - Serial Number + Resolution Status - 123456789012345678901234567890 - - - - - Valid Not Before - - Apr 05 2024 - - - - Valid Not After + {vulnerability.structuredData?.resolution_status || 'N/A'} - Apr 03 2034 + {!vulnerability.structuredData?.certificate && ( + + + Certificate + + N/A + + )}
+ {vulnerability.structuredData?.certificate && ( + <> + + Certificate + + + + + + + Subject Name + + + {vulnerability.structuredData?.certificate?.subjectName || + 'N/A'} + + + + + Issuer Name + + + {vulnerability.structuredData?.certificate?.issuerName || + 'N/A'} + + + + + Serial Number + + + {vulnerability.structuredData?.certificate + ?.serialNumber || 'N/A'} + + + + + Valid Not Before + + + {vulnerability.structuredData?.certificate + ?.validNotBefore || 'N/A'} + + + + + Valid Not After + + + {vulnerability.structuredData?.certificate + ?.validNotAfter || 'N/A'} + + + +
+ + + )} How to Remediate - {parseTextToJSX(testXpanseData.remediation)} + + {vulnerability.structuredData?.remediation_guidance} + ); From 417ec3d9128a9510f2ee300c24b1544f56999fe0 Mon Sep 17 00:00:00 2001 From: Amelia Vance Date: Fri, 14 Jun 2024 15:45:05 -0400 Subject: [PATCH 17/23] Update Vulnerability.tsx to handle null values --- .../src/pages/Vulnerability/Vulnerability.tsx | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/frontend/src/pages/Vulnerability/Vulnerability.tsx b/frontend/src/pages/Vulnerability/Vulnerability.tsx index 93f57c84..53d6589c 100644 --- a/frontend/src/pages/Vulnerability/Vulnerability.tsx +++ b/frontend/src/pages/Vulnerability/Vulnerability.tsx @@ -234,8 +234,7 @@ export const Vulnerability: React.FC = () => { {differenceInCalendarDays( Date.now(), parseISO(vulnerability.domain.createdAt) - )}{' '} - day(s) ago + ) + ' day(s) ago' || 'N/A'} @@ -246,8 +245,7 @@ export const Vulnerability: React.FC = () => { {differenceInCalendarDays( Date.now(), parseISO(vulnerability.domain.updatedAt) - )}{' '} - day(s) ago + ) + ' day(s) ago' || 'N/A'} @@ -257,7 +255,7 @@ export const Vulnerability: React.FC = () => { {vulnerability.domain.country ? vulnerability.domain.country - : 'Not found'} + : 'N/A'} @@ -265,7 +263,7 @@ export const Vulnerability: React.FC = () => { Organization: - {vulnerability.domain.organization.name} + {vulnerability.domain.organization.name || 'N/A'} @@ -308,7 +306,7 @@ export const Vulnerability: React.FC = () => { Root Domain: - {vulnerability.domain.fromRootDomain} + {vulnerability.domain.fromRootDomain || 'N/A'} @@ -316,8 +314,10 @@ export const Vulnerability: React.FC = () => { Subdomain: - {vulnerability.domain.name} ( - {vulnerability.domain.subdomainSource}) + {vulnerability.domain.name + + ' (' + + vulnerability.domain.subdomainSource + + ')' || 'N/A'} {vulnerability.service && vulnerability.service.service ? ( @@ -357,7 +357,10 @@ export const Vulnerability: React.FC = () => { Vulnerability: - {vulnerability.title} ({vulnerability.source}) + {vulnerability.title + + ' (' + + vulnerability.source + + ')' || 'N/A'} @@ -375,7 +378,7 @@ export const Vulnerability: React.FC = () => { - {vulnerability.title} + {vulnerability.title || ''} { width: '80px' }} > - {vulnerability?.severity} + {vulnerability?.severity || ''} {vulnerability?.isKev ? 'Yes' : 'No'} @@ -396,10 +399,12 @@ export const Vulnerability: React.FC = () => { - {product + {product?.name && product?.version ? product.name + (product.version ? ' ' + product.version : '') - : vulnerability?.cpe} + : vulnerability?.cpe + ? vulnerability.cpe + : ''} {lastState === 'open' @@ -448,10 +453,10 @@ export const Vulnerability: React.FC = () => { {vulnerability.cwe} ) : ( - 'Not found' + 'N/A' )} - {vulnerability.source} + {vulnerability.source || 'N/A'}
@@ -500,7 +505,7 @@ export const Vulnerability: React.FC = () => { - {vulnerability.title} + {vulnerability.title || ''} {vulnerability.cve && vulnerability.cve.startsWith('CVE') && ( { {vulnerability.source === 'Palo Alto Xpanse' ? ( - + ) : vulnerability.cve && vulnerability.cve.startsWith('CVE') ? ( Date: Fri, 14 Jun 2024 15:59:43 -0400 Subject: [PATCH 18/23] Add aria labels to links in Vulnerability.tsx --- frontend/src/pages/Vulnerability/Vulnerability.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/Vulnerability/Vulnerability.tsx b/frontend/src/pages/Vulnerability/Vulnerability.tsx index 53d6589c..8ccccc19 100644 --- a/frontend/src/pages/Vulnerability/Vulnerability.tsx +++ b/frontend/src/pages/Vulnerability/Vulnerability.tsx @@ -192,7 +192,7 @@ export const Vulnerability: React.FC = () => { size="large" edge="start" color="inherit" - aria-label="menu" + aria-label="link to the domain page" onClick={handleOpenInNewTab( `/inventory/domain/${vulnerability.domain.id}` )} @@ -449,7 +449,10 @@ export const Vulnerability: React.FC = () => { {vulnerability.cwe ? ( - + {vulnerability.cwe} ) : ( @@ -512,6 +515,7 @@ export const Vulnerability: React.FC = () => { size="large" color="inherit" onClick={handleOpenInNewTab(references[0]?.url)} + aria-label="link to this CVE's site" > From 7ccb953b0c123f594fac039431f7919be923cf75 Mon Sep 17 00:00:00 2001 From: Amelia Vance Date: Mon, 17 Jun 2024 12:31:36 -0400 Subject: [PATCH 19/23] Update display grid for Vulnerability Details pages --- .../src/pages/Vulnerability/CveSection.tsx | 46 +++++++++++-------- .../src/pages/Vulnerability/Vulnerability.tsx | 3 +- .../src/pages/Vulnerability/XpanseSection.tsx | 4 +- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/frontend/src/pages/Vulnerability/CveSection.tsx b/frontend/src/pages/Vulnerability/CveSection.tsx index fe32dd29..0f434f11 100644 --- a/frontend/src/pages/Vulnerability/CveSection.tsx +++ b/frontend/src/pages/Vulnerability/CveSection.tsx @@ -60,7 +60,7 @@ export const CveSection: React.FC = ({ ); const SeverityMetricsSection = () => { return ( - + CVSS 3.x Severity & Metrics @@ -183,7 +183,7 @@ export const CveSection: React.FC = ({ const CVEReferences = () => { return ( <> - + CVE References @@ -217,18 +217,20 @@ export const CveSection: React.FC = ({ return ( Products Affected - {Object.entries(groupedByVendor).map(([vendor, values]) => ( -
- {vendor} - -
    - {values.map((value, index) => ( -
  • {value.name}
    • - ))} -
- -
- ))} + {Object.keys(groupedByVendor).length > 0 + ? Object.entries(groupedByVendor).map(([vendor, values]) => ( +
+ {vendor} + +
    + {values.map((value, index) => ( +
  • {value.name}
    • + ))} +
+ +
+ )) + : 'No products found'} ); }; @@ -237,12 +239,18 @@ export const CveSection: React.FC = ({ - - - - - + {cve && } + {cve ? ( + + + + + ) : ( + + + + )} ); }; diff --git a/frontend/src/pages/Vulnerability/Vulnerability.tsx b/frontend/src/pages/Vulnerability/Vulnerability.tsx index 8ccccc19..059aedcc 100644 --- a/frontend/src/pages/Vulnerability/Vulnerability.tsx +++ b/frontend/src/pages/Vulnerability/Vulnerability.tsx @@ -89,7 +89,6 @@ export const Vulnerability: React.FC = () => { const result = await apiGet( `/vulnerabilities/${vulnerabilityId}` ); - console.log(result); setVulnerability(result); if (result.cve) { getCve(result.cve); @@ -424,7 +423,7 @@ export const Vulnerability: React.FC = () => { const VulnDescription = () => { return ( <> - + Description {vulnerability.description} diff --git a/frontend/src/pages/Vulnerability/XpanseSection.tsx b/frontend/src/pages/Vulnerability/XpanseSection.tsx index ad85b1cd..a667404c 100644 --- a/frontend/src/pages/Vulnerability/XpanseSection.tsx +++ b/frontend/src/pages/Vulnerability/XpanseSection.tsx @@ -23,7 +23,7 @@ export const XpanseSection: React.FC = ({ return ( - + Alert Policy Description {vulnerability.description} @@ -175,7 +175,7 @@ export const XpanseSection: React.FC = ({ )} - + How to Remediate From 65234952d54d7ea2f6c000cc2d605ae09157707d Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:04:02 -0400 Subject: [PATCH 20/23] update logic to avoid unused variable update logic to avoid unused variable --- backend/src/tasks/xpanse-sync.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index e3165a39..ca7b84d8 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -177,7 +177,7 @@ const getVulnData = async ( if (current_page >= total_pages) { done = true; console.log(`Finished fetching Xpanse Alert data for ${org.name}`); - return 1; + } page = page + 1; } @@ -186,9 +186,9 @@ const getVulnData = async ( console.log( `Error fetching Xpanse Alert data: ${taskRequest?.error} and status: ${taskRequest?.status}` ); - return 1; } - } + } + return 1 }; const saveXpanseDomain = async (domain) => { const existingDomain = await Domain.findOne({ @@ -341,7 +341,7 @@ export const handler = async (CommandOptions) => { for (const org of allOrgs) { console.log(`Gathering Xpanse alerts for ${org.name}`); - (await getVulnData(org, CommandOptions)) || []; + (await getVulnData(org, CommandOptions)); } } catch (e) { console.error('Unknown failure.'); From 3b8bd24202816db15e5516040f7de3378b93bc44 Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:20:53 -0400 Subject: [PATCH 21/23] run prettier run prettier --- backend/src/tasks/xpanse-sync.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index ca7b84d8..f9b9cba7 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -177,7 +177,6 @@ const getVulnData = async ( if (current_page >= total_pages) { done = true; console.log(`Finished fetching Xpanse Alert data for ${org.name}`); - } page = page + 1; } @@ -187,8 +186,8 @@ const getVulnData = async ( `Error fetching Xpanse Alert data: ${taskRequest?.error} and status: ${taskRequest?.status}` ); } - } - return 1 + } + return 1; }; const saveXpanseDomain = async (domain) => { const existingDomain = await Domain.findOne({ @@ -294,7 +293,11 @@ const saveXpanseAlert = async ( title: vuln.alert_name, cve: 'Xpanse Alert', description: vuln.description, - severity: vuln.severity !== null ? vuln.severity.charAt(0).toUpperCase() + vuln.severity.slice(1).toLowerCase() : null, + severity: + vuln.severity !== null + ? vuln.severity.charAt(0).toUpperCase() + + vuln.severity.slice(1).toLowerCase() + : null, state: resolution, structuredData: { alert_id: vuln.alert_id, @@ -341,7 +344,7 @@ export const handler = async (CommandOptions) => { for (const org of allOrgs) { console.log(`Gathering Xpanse alerts for ${org.name}`); - (await getVulnData(org, CommandOptions)); + await getVulnData(org, CommandOptions); } } catch (e) { console.error('Unknown failure.'); From ea2524d852037e246e65942a1c48a63961fa9287 Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Tue, 18 Jun 2024 13:39:14 -0400 Subject: [PATCH 22/23] remove outdated comments Remove unnecessary comments from backend xpanse pull script --- backend/src/tasks/xpanse-sync.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index f9b9cba7..8a350857 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -42,7 +42,7 @@ interface XpanseVulnOutput { time_pulled_from_xpanse: string; action_pretty: string; attack_surface_rule_name: string; - certificate: Record; //Change this to a + certificate: Record; remediation_guidance: string; asset_identifiers: Record[]; services: XpanseServiceOutput[]; @@ -155,22 +155,18 @@ const getVulnData = async ( let done = false; let page = 1; let total_pages = 1; - // let fullVulnArray: XpanseVulnOutput[] = []; while (!done) { let taskRequest = await createTask(page, org.acronym); console.log(`Fetching page ${page} of page ${total_pages}`); await new Promise((r) => setTimeout(r, 1000)); if (taskRequest?.status == 'Processing') { while (taskRequest?.status == 'Processing') { - //console.log('Waiting for task to complete'); await new Promise((r) => setTimeout(r, 1000)); taskRequest = await fetchData(taskRequest.task_id); - //console.log(taskRequest?.status); } if (taskRequest?.status == 'Completed') { console.log(`Task completed successfully for page: ${page}`); - const vulnArray = taskRequest?.result?.data || []; //TODO, change this to CveEntry[] - // fullVulnArray = fullVulnArray.concat(vulnArray); + const vulnArray = taskRequest?.result?.data || []; await saveXpanseAlert(vulnArray, org, commandOptions.scanId); total_pages = taskRequest?.result?.total_pages || 1; const current_page = taskRequest?.result?.current_page || 1; @@ -210,7 +206,7 @@ const saveXpanseVuln = async (vuln: Vulnerability, savedDomain: Domain) => { const existingVuln = await Vulnerability.findOne({ where: { domain: { id: savedDomain.id }, - title: vuln.title //Change this for typeorm bump + title: vuln.title } }); if (!existingVuln) { From b8198cad1e01a3af04145bf9f587043996e39741 Mon Sep 17 00:00:00 2001 From: DJensen94 <79864006+DJensen94@users.noreply.github.com> Date: Tue, 18 Jun 2024 13:50:32 -0400 Subject: [PATCH 23/23] run linter run pre-commit --- backend/src/tasks/xpanse-sync.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/tasks/xpanse-sync.ts b/backend/src/tasks/xpanse-sync.ts index 8a350857..2a1f62a8 100644 --- a/backend/src/tasks/xpanse-sync.ts +++ b/backend/src/tasks/xpanse-sync.ts @@ -42,7 +42,7 @@ interface XpanseVulnOutput { time_pulled_from_xpanse: string; action_pretty: string; attack_surface_rule_name: string; - certificate: Record; + certificate: Record; remediation_guidance: string; asset_identifiers: Record[]; services: XpanseServiceOutput[]; @@ -166,7 +166,7 @@ const getVulnData = async ( } if (taskRequest?.status == 'Completed') { console.log(`Task completed successfully for page: ${page}`); - const vulnArray = taskRequest?.result?.data || []; + const vulnArray = taskRequest?.result?.data || []; await saveXpanseAlert(vulnArray, org, commandOptions.scanId); total_pages = taskRequest?.result?.total_pages || 1; const current_page = taskRequest?.result?.current_page || 1; @@ -206,7 +206,7 @@ const saveXpanseVuln = async (vuln: Vulnerability, savedDomain: Domain) => { const existingVuln = await Vulnerability.findOne({ where: { domain: { id: savedDomain.id }, - title: vuln.title + title: vuln.title } }); if (!existingVuln) {