diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 95b0a0b9..d76c7b74 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -189,6 +189,13 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Run syncmdl + run: aws lambda invoke --function-name crossfeed-staging-syncmdl --region us-east-1 /dev/stdout + working-directory: backend + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + deploy_prod: needs: [build_worker, lint, test, test_python] runs-on: ubuntu-latest @@ -238,3 +245,10 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: Run syncmdl + run: aws lambda invoke --function-name crossfeed-prod-syncmdl --region us-east-1 /dev/stdout + working-directory: backend + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/backend/package.json b/backend/package.json index 915067f1..fa42e826 100644 --- a/backend/package.json +++ b/backend/package.json @@ -112,6 +112,7 @@ "deploy-worker-prod": "./tools/deploy-worker.sh crossfeed-prod-worker", "syncdb": "docker-compose exec -T backend npx ts-node src/tools/run-syncdb.ts", "pesyncdb": "docker-compose exec -T backend npx ts-node src/tools/run-pesyncdb.ts", + "syncmdl": "docker-compose exec -T backend npx ts-node src/tools/run-syncmdl.ts", "control-queue": "docker-compose exec -T backend npx ts-node src/tools/consumeControlQueue.ts" }, "author": "", diff --git a/backend/src/models/connection.ts b/backend/src/models/connection.ts index ec8c8cd3..5fad88f6 100644 --- a/backend/src/models/connection.ts +++ b/backend/src/models/connection.ts @@ -1,5 +1,6 @@ import { createConnection, Connection } from 'typeorm'; import { + // Models for the Crossfeed database Domain, Service, Vulnerability, @@ -13,11 +14,95 @@ import { SavedSearch, OrganizationTag, Cpe, - Cve + Cve, + + // Models for the Mini Data Lake database + CertScan, + Cidr, + Contact, + DL_Cpe, + DL_Cve, + DL_Domain, + HostScan, + Host, + Ip, + Kev, + Location, + DL_Organization, + PortScan, + PrecertScan, + Report, + Request, + Sector, + Snapshot, + SslyzeScan, + Tag, + Tally, + TicketEvent, + Ticket, + TrustymailScan, + VulnScan + } from '.'; let connection: Connection | null = null; +let dl_connection: Connection | null = null; + +const connectDl = async (logging?:boolean) => { + const dl_connection = createConnection({ + type: 'postgres', + host: process.env.MDL_HOST, + port: parseInt(process.env.MDL_PORT ?? ''), + username: process.env.MDL_USERNAME, + password: process.env.MDL_PASSWORD, + database: process.env.MDL_NAME, + entities: [ + CertScan, + Cidr, + Contact, + DL_Cpe, + DL_Cve, + DL_Domain, + HostScan, + Host, + Ip, + Kev, + Location, + DL_Organization, + PortScan, + PrecertScan, + Report, + Request, + Sector, + Snapshot, + SslyzeScan, + Tag, + Tally, + TicketEvent, + Ticket, + TrustymailScan, + VulnScan + ], + synchronize: false, + name: 'mini_data_lake', + dropSchema: false, + logging: logging ?? false, + cache: true + }); + return dl_connection +} + +export const connectToDatalake = async (logging?: boolean) => { + if (!dl_connection?.isConnected) { + dl_connection = await connectDl(logging); + } + else { + console.log("didn't connect") + } + return dl_connection; +}; + const connectDb = async (logging?: boolean) => { const connection = createConnection({ type: 'postgres', diff --git a/backend/src/models/index.ts b/backend/src/models/index.ts index 310cfb76..1de7bdf5 100644 --- a/backend/src/models/index.ts +++ b/backend/src/models/index.ts @@ -13,3 +13,29 @@ export * from './webpage'; export * from './api-key'; export * from './saved-search'; export * from './organization-tag'; + +export * from './mini_data_lake/cert_scans'; +export * from './mini_data_lake/cidrs'; +export * from './mini_data_lake/contacts'; +export {Cpe as DL_Cpe} from './mini_data_lake/cpes'; +export {Cve as DL_Cve} from './mini_data_lake/cves'; +export {Domain as DL_Domain} from './mini_data_lake/domains'; +export * from './mini_data_lake/host_scans'; +export * from './mini_data_lake/hosts'; +export * from './mini_data_lake/ips'; +export * from './mini_data_lake/kevs'; +export * from './mini_data_lake/locations'; +export {Organization as DL_Organization} from './mini_data_lake/organizations'; +export * from './mini_data_lake/port_scans'; +export * from './mini_data_lake/precert_scans'; +export * from './mini_data_lake/reports'; +export * from './mini_data_lake/requests'; +export * from './mini_data_lake/sectors'; +export * from './mini_data_lake/snapshots'; +export * from './mini_data_lake/sslyze_scan'; +export * from './mini_data_lake/tag'; +export * from './mini_data_lake/tallies'; +export * from './mini_data_lake/ticket_events'; +export * from './mini_data_lake/tickets'; +export * from './mini_data_lake/trustymail_scans'; +export * from './mini_data_lake/vuln_scans'; diff --git a/backend/src/models/mini_data_lake/cert_scans.ts b/backend/src/models/mini_data_lake/cert_scans.ts new file mode 100644 index 00000000..4f81e838 --- /dev/null +++ b/backend/src/models/mini_data_lake/cert_scans.ts @@ -0,0 +1,61 @@ +// The data in this table is derived from the Vulnerability Scans Database, +// the [certs Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#certs-collection). + +import { + Entity, + Column, + PrimaryColumn, + BaseEntity, + ManyToMany, + JoinTable + } from 'typeorm'; + + import {Domain} from "./domains" + +@Entity() +export class CertScan extends BaseEntity { + @PrimaryColumn() + id: string + + @Column({ + nullable: true, + type: 'varchar' + }) + issuer: string | null; + + @Column({ nullable: true, type: 'timestamp' }) + expirationTimestamp: Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + certStartTimestamp: Date | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + pem: string | null; + + @Column() + sctExists: boolean; + + @Column({ nullable: true, type: 'timestamp' }) + sctOrNotBefore: Date | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + serial: string | null; + + @ManyToMany( + (type) => Domain, + (domain) => domain.certScans, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + domains: Domain[]; + +} diff --git a/backend/src/models/mini_data_lake/cidrs.ts b/backend/src/models/mini_data_lake/cidrs.ts new file mode 100644 index 00000000..1c5d601c --- /dev/null +++ b/backend/src/models/mini_data_lake/cidrs.ts @@ -0,0 +1,68 @@ +import { + Entity, + Index, + Column, + PrimaryColumn, + CreateDateColumn, + BaseEntity, + ManyToMany, + JoinTable + } from 'typeorm'; + +import {Request } from './requests'; +import { Organization } from './organizations'; +@Entity() +export class Cidr extends BaseEntity { + @PrimaryColumn() + id: string + + @CreateDateColumn() + createdDate: Date; + + @Index() + @Column({ + nullable: true, + type: 'cidr', + unique: true + }) + network: string | null; + + @Column({ + nullable: true, + type: 'inet' + }) + startIp: string | null; + + @Column({ + nullable: true, + type: 'inet' + }) + endIp: string | null; + + @Column({nullable: true}) + retired: boolean; + + @ManyToMany( + (type) => Request, + (request) => request.cidrs, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + requests: Request[]; + + @ManyToMany( + (type) => Organization, + (org) => org.cidrs, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + organizations: Organization[]; + + +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/contacts.ts b/backend/src/models/mini_data_lake/contacts.ts new file mode 100644 index 00000000..96ba792a --- /dev/null +++ b/backend/src/models/mini_data_lake/contacts.ts @@ -0,0 +1,58 @@ +// The data in this table is derived from the Vulnerability Scans Database, +// the [requests Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#requests-collection). + +import { + Entity, + Column, + PrimaryGeneratedColumn, + BaseEntity, + ManyToMany, + Unique, + JoinTable + } from 'typeorm'; +import {Organization} from './organizations' +@Entity() +@Unique(['name','email','type']) +export class Contact extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ + nullable: true, + type: 'varchar' + }) + name: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + email: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + phoneNumber: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + type: string | null; + + @Column() + retired: boolean; + + @ManyToMany( + (type) => Organization, + (org) => org.contacts, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + organizations: Organization[]; + +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/cpes.ts b/backend/src/models/mini_data_lake/cpes.ts new file mode 100644 index 00000000..a4e1b14c --- /dev/null +++ b/backend/src/models/mini_data_lake/cpes.ts @@ -0,0 +1,32 @@ +import { + Entity, + PrimaryGeneratedColumn, + Column, + ManyToMany, + BaseEntity, + Unique + } from 'typeorm'; + import { Cve } from './cves'; + + @Entity() + @Unique(['name', 'version', 'vendor']) + export class Cpe extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column() + name: string; + + @Column() + version: string; + + @Column() + vendor: string; + + @Column() + lastSeenAt: Date; + + @ManyToMany(() => Cve, (cve) => cve.cpes) + cves: Cve[]; + } + \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/cves.ts b/backend/src/models/mini_data_lake/cves.ts new file mode 100644 index 00000000..dbb3019d --- /dev/null +++ b/backend/src/models/mini_data_lake/cves.ts @@ -0,0 +1,131 @@ +import { + Entity, + PrimaryGeneratedColumn, + Column, + ManyToMany, + OneToMany, + BaseEntity, + JoinTable, + Unique +} from 'typeorm'; +import { Cpe } from './cpes'; +import { Ticket } from './tickets'; +import { VulnScan } from './vuln_scans'; + +@Entity() +@Unique(['name']) +export class Cve extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ nullable: true }) + name: string; + + @Column({ nullable: true }) + publishedAt: Date; + + @Column({ nullable: true }) + modifiedAt: Date; + + @Column({ nullable: true }) + status: string; + + @Column({ nullable: true }) + description: string; + + @Column({ nullable: true }) + cvssV2Source: string; + + @Column({ nullable: true }) + cvssV2Type: string; + + @Column({ nullable: true }) + cvssV2Version: string; + + @Column({ nullable: true }) + cvssV2VectorString: string; + + @Column({ nullable: true }) + cvssV2BaseScore: string; + + @Column({ nullable: true }) + cvssV2BaseSeverity: string; + + @Column({ nullable: true }) + cvssV2ExploitabilityScore: string; + + @Column({ nullable: true }) + cvssV2ImpactScore: string; + + @Column({ nullable: true }) + cvssV3Source: string; + + @Column({ nullable: true }) + cvssV3Type: string; + + @Column({ nullable: true }) + cvssV3Version: string; + + @Column({ nullable: true }) + cvssV3VectorString: string; + + @Column({ nullable: true }) + cvssV3BaseScore: string; + + @Column({ nullable: true }) + cvssV3BaseSeverity: string; + + @Column({ nullable: true }) + cvssV3ExploitabilityScore: string; + + @Column({ nullable: true }) + cvssV3ImpactScore: string; + + @Column({ nullable: true }) + cvssV4Source: string; + + @Column({ nullable: true }) + cvssV4Type: string; + + @Column({ nullable: true }) + cvssV4Version: string; + + @Column({ nullable: true }) + cvssV4VectorString: string; + + @Column({ nullable: true }) + cvssV4BaseScore: string; + + @Column({ nullable: true }) + cvssV4BaseSeverity: string; + + @Column({ nullable: true }) + cvssV4ExploitabilityScore: string; + + @Column({ nullable: true }) + cvssV4ImpactScore: string; + + @Column('simple-array', { nullable: true }) + weaknesses: string[]; + + @Column('simple-array', { nullable: true }) + references: string[]; + + @ManyToMany(() => Cpe, (cpe) => cpe.cves, { + cascade: true + }) + @JoinTable() + cpes: Cpe[]; + + @OneToMany((type) => Ticket, (ticket) => ticket.cve, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + tickets: Ticket[]; + + @OneToMany((type) => VulnScan, (vuln_scan) => vuln_scan.cve, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + vulnScans: VulnScan[]; +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/domains.ts b/backend/src/models/mini_data_lake/domains.ts new file mode 100644 index 00000000..bd556682 --- /dev/null +++ b/backend/src/models/mini_data_lake/domains.ts @@ -0,0 +1,106 @@ +// The data in this table is derived from domains collected by our +// [gatherer](https://github.com/cisagov/gatherer), which pulls in +// domains from Cyber Hygiene and the GSA. NOTE: More details may be +// available in the GitHub +// [README](https://github.com/cisagov/cyhy-ct-logs/blob/initial/README.md) +// documents for [gatherer](https://github.com/cisagov/gatherer) and +// [saver](https://github.com/cisagov/saver). + + +import { + Entity, + Column, + PrimaryGeneratedColumn, + CreateDateColumn, + BaseEntity, + OneToMany, + ManyToMany, + ManyToOne, + Unique + } from 'typeorm'; + + import {Organization } from './organizations'; + import {CertScan} from './cert_scans' + import {PrecertScan} from './precert_scans' + import {Ip } from './ips'; + import { SslyzeScan } from './sslyze_scan'; +import { TrustymailScan } from './trustymail_scans'; + +@Entity() +@Unique(['domain']) +export class Domain extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @ManyToOne(type => Organization, organization => organization.domains, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + organization: Organization; + + @ManyToOne(type => Ip, ip => ip.domains, { + onDelete: 'SET NULL', + onUpdate: 'CASCADE' + }) + ip: Ip; + + @Column({ + nullable: true, + type: 'varchar' + }) + ipString: string | null; + + @CreateDateColumn() + createdAt: Date; + + @Column({ + nullable: true, + type: 'varchar' + }) + domain: string | null; + + @Column({ nullable: true }) + retired: boolean; + + @Column({ nullable: true }) + falsePositive: boolean; + + @Column({ nullable: true }) + identifiedFromIp: boolean; + + @ManyToOne((type) => Domain, (dom) => dom.subDomains, { + onDelete: 'CASCADE', + nullable: true + }) + rootDomain: Domain; + + @OneToMany((type) => Domain, (dom) => dom.rootDomain, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + subDomains: Domain[]; + + @ManyToMany((type) => CertScan, (cert_scan) => cert_scan.domains, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + certScans: CertScan[]; + + @ManyToMany((type) => PrecertScan, (precert_scan) => precert_scan.domains, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + precertScans: PrecertScan[]; + + @OneToMany((type) => SslyzeScan, (sslyze_scan) => sslyze_scan.domain, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + sslyzeScans: SslyzeScan[]; + + @OneToMany((type) => TrustymailScan, (trustymail_scan) => trustymail_scan.domain, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + trustymailScans: TrustymailScan[]; +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/host_scans.ts b/backend/src/models/mini_data_lake/host_scans.ts new file mode 100644 index 00000000..bfd54d8f --- /dev/null +++ b/backend/src/models/mini_data_lake/host_scans.ts @@ -0,0 +1,90 @@ +// The data in this table is derived from IP addresses supplied by the +// CyHy stakeholders. + +import { DirectConnect } from 'aws-sdk'; +import { toInteger } from 'lodash'; +import {Snapshot} from './snapshots' +import { + Entity, + Column, + PrimaryColumn, + BaseEntity, + ManyToMany, + ManyToOne + } from 'typeorm'; + + import {Ip } from './ips'; + import { Organization } from './organizations'; + +@Entity() +export class HostScan extends BaseEntity { + @PrimaryColumn() + id: string; + + @Column({ + nullable: true, + type: 'varchar' + }) + ipString: string | null; + + @ManyToOne(type => Ip, ip => ip.hostScans, { + onDelete: 'SET NULL', + onUpdate: 'CASCADE' + }) + ip: Ip; + + @Column({ + nullable: true, + type: 'integer' + }) + accuracy: number | null; + + @Column({ + type: 'jsonb', + default: [] + }) + classes: Object[]; + + @Column({ + nullable: true, + type: 'varchar' + }) + hostname: string | null; + + @Column() + latest: boolean; + + @Column({ + nullable: true, + type: 'integer' + }) + line: number | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + name: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + source: string | null; + + @Column({ nullable: true, type: 'timestamp' }) + scanTimestamp : Date | null; + + + @ManyToMany((type) => Snapshot, (snapshot) => snapshot.hostScans, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + snapshots: Snapshot[]; + + @ManyToOne((type) => Organization, (org) => org.hostScans, { + onDelete: 'CASCADE', + nullable: true + }) + organization: Organization ; +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/hosts.ts b/backend/src/models/mini_data_lake/hosts.ts new file mode 100644 index 00000000..669d9792 --- /dev/null +++ b/backend/src/models/mini_data_lake/hosts.ts @@ -0,0 +1,108 @@ +// The data in this table is derived from IP addresses supplied by the +// CyHy stakeholders. + +import { + Entity, + Index, + Column, + PrimaryColumn, + BaseEntity, + ManyToOne + } from 'typeorm'; +import { Organization } from './organizations'; +import { Ip } from './ips'; +// TODO Determine if this should be merged with the ips column +@Entity() +@Index(['ip']) +export class Host extends BaseEntity { + @PrimaryColumn() + id: string; + + @Column({ + nullable: true, + type: 'varchar' + }) + ipString: string | null; + + @ManyToOne(type => Ip, ip => ip.hosts, { + onDelete: 'SET NULL', + onUpdate: 'CASCADE' + }) + ip: Ip; + + @Column({ nullable: true, type: 'timestamp' }) + updatedTimestamp : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + latestNetscan1Timestamp : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + latestNetscan2Timestamp : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + latestVulnscanTimestamp : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + latestPortscanTimestamp : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + latestScanCompletionTimestamp : Date | null; + + @Column({ + nullable: true, + type: 'decimal' + }) + locationLongitude: number | null; + + @Column({ + nullable: true, + type: 'decimal' + }) + locationLatitude: number | null; + + @Column({ + nullable: true, + type: 'integer' + }) + priority: number | null; + + @Column({ nullable: true, type: 'timestamp' }) + nextScanTimestamp : Date | null; + + @Column({ + nullable: true, + type: 'decimal' + }) + rand: number | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + currStage: string | null; + + @Column() + hostLive: boolean; + + + @Column({ + nullable: true, + type: 'varchar' + }) + hostLiveReason: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + status: string | null; + + @ManyToOne((type) => Organization, (org) => org.hosts, { + onDelete: 'CASCADE', + nullable: true + }) + organization: Organization ; + + + +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/ips.ts b/backend/src/models/mini_data_lake/ips.ts new file mode 100644 index 00000000..12a86488 --- /dev/null +++ b/backend/src/models/mini_data_lake/ips.ts @@ -0,0 +1,85 @@ +// The data in this table is derived from IP addresses supplied by the +// CyHy stakeholders. + +import { + Entity, + Column, + PrimaryGeneratedColumn, + BaseEntity, + OneToMany, + ManyToOne + } from 'typeorm'; + import {Domain } from './domains'; + import {HostScan } from './host_scans'; + import {Host} from './hosts' + import {Organization} from './organizations' + import { Ticket } from './tickets'; + import { VulnScan } from './vuln_scans'; +import { PortScan } from './port_scans'; +@Entity() +export class Ip extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + // TODO Determine if this should be many to many or FK + @ManyToOne(type => Organization, organization => organization.ips, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + organization: Organization; + + @Column({ nullable: true, type: 'timestamp' }) + createdTimestamp : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + lastSeenTimestamp : Date | null; + + @Column({ + nullable: true, + type: 'inet' + }) + ip: string | null; + + @Column({ nullable: true }) + live: boolean; + + @Column({ nullable: true }) + falsePositive: boolean; + + @Column({ nullable: true }) + fromCidr: boolean; + + @Column({ nullable: true }) + retired: boolean; + + @OneToMany((type) => Domain, (domain) => domain.ip, {onDelete: 'CASCADE', + onUpdate: 'CASCADE'}) + domains: Domain[]; + + @OneToMany((type) => HostScan, (host_scan) => host_scan.ip, {onDelete: 'CASCADE', + onUpdate: 'CASCADE'}) + hostScans: HostScan[]; + + @OneToMany((type) => Host, (host) => host.ip, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE'}) + hosts: Host[]; + + @OneToMany((type) => Ticket, (ticket) => ticket.ip, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + tickets: Ticket[]; + + @OneToMany((type) => VulnScan, (vuln_scan) => vuln_scan.ip, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + vulnScans: VulnScan[]; + + @OneToMany((type) => PortScan, (port_scan) => port_scan.ip, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + portScans: PortScan[]; +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/kevs.ts b/backend/src/models/mini_data_lake/kevs.ts new file mode 100644 index 00000000..7ec55216 --- /dev/null +++ b/backend/src/models/mini_data_lake/kevs.ts @@ -0,0 +1,35 @@ +// The data in this table is derived from the +// [JSON feed](https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json) +// of the [CISA Known Exploited Vulnerabilities +// Catalog](https://www.cisa.gov/known-exploited-vulnerabilities-catalog). + + +import { + Entity, + Column, + PrimaryGeneratedColumn, + BaseEntity, + OneToMany + } from 'typeorm'; +import { Ticket } from './tickets'; +@Entity() +export class Kev extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ + nullable: true, + type: 'varchar', + unique: true + }) + cve: string | null; + + @Column() + knownRansomware: boolean; + + @OneToMany((type) => Ticket, (ticket) => ticket.kev, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + tickets: Ticket[]; +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/locations.ts b/backend/src/models/mini_data_lake/locations.ts new file mode 100644 index 00000000..e09738fc --- /dev/null +++ b/backend/src/models/mini_data_lake/locations.ts @@ -0,0 +1,86 @@ +// The data in this table is derived from the "Government Units" and +// "Populated Places" Topical Gazetteers files from +// [USGS](https://geonames.usgs.gov/domestic/download_data.htm). + + +import { + Entity, + Column, + PrimaryGeneratedColumn, + BaseEntity, + ManyToMany, + JoinTable + } from 'typeorm'; + +import {Organization} from './organizations' +@Entity() +export class Location extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ + nullable: true, + type: 'varchar' + }) + name: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + countryAbvr: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + country: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + county: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + countyFips: string | null; + + @Column({ + nullable: true, + type: 'varchar', + unique: true + }) + gnisId: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + stateAbrv: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + stateFips: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + state: string | null; + + @ManyToMany( + (type) => Organization, + (org) => org.locations, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + organizations: Organization[]; +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/organizations.ts b/backend/src/models/mini_data_lake/organizations.ts new file mode 100644 index 00000000..995646d3 --- /dev/null +++ b/backend/src/models/mini_data_lake/organizations.ts @@ -0,0 +1,208 @@ +// The data in this table is derived from the Vulnerability Scans Database, +// the [requests Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#requests-collection). + + +import { + Entity, + Index, + Column, + PrimaryGeneratedColumn, + BaseEntity, + OneToMany, + ManyToMany, + ManyToOne + } from 'typeorm'; + import {Domain } from './domains'; + import {Ip} from './ips'; + import {Location} from './locations'; + import {Contact} from './contacts'; + import {Tag} from './tag'; + import {Sector} from './sectors'; + import {Report} from './reports'; + import {Request} from './requests'; + import { SslyzeScan } from './sslyze_scan'; + import {Snapshot} from './snapshots'; + import { Tally } from './tallies'; + import { TrustymailScan } from './trustymail_scans'; + import { VulnScan } from './vuln_scans'; + import { Cidr } from './cidrs'; + import { HostScan } from './host_scans'; + import {Host} from './hosts' + import { Ticket } from './tickets'; +@Entity() +export class Organization extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ + nullable: true, + type: 'varchar' + }) + name: string | null; + + @Column({ + nullable: true, + type: 'varchar', + unique: true + }) + @Index() + acronym: string | null; + + @Column({ nullable: true, type: 'timestamp' }) + firstEngageDate : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + createdDate : Date | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + createdEmplyeeId: string | null; + + @Column({ nullable: true, type: 'timestamp' }) + updatedDate : Date | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + updatedEmployeeId: string | null; + + @Column({ nullable: true }) + retired: boolean; + + @Column({ nullable: true }) + peReportOn: boolean; + + @Column({ nullable: true }) + pePremium: boolean; + + @Column({ nullable: true }) + peDemo: boolean; + + @Column({ nullable: true }) + peRunScans: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + type: string | null; + + @Column({ nullable: true }) + stakeholder: boolean; + + @OneToMany((type) => Domain, (domain) => domain.organization, {onDelete: 'CASCADE', + onUpdate: 'CASCADE'}) + domains: Domain[]; + + @OneToMany((type) => Ip, (ip) => ip.organization, {onDelete: 'CASCADE', + onUpdate: 'CASCADE'}) + ips: Ip[]; + + @OneToMany((type) => Ticket, (ticket) => ticket.organization, {onDelete: 'CASCADE', + onUpdate: 'CASCADE'}) + tickets: Ticket[]; + + @ManyToMany((type) => Location, (location) => location.organizations, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + locations: Location[]; + + @ManyToMany((type) => Contact, (contact) => contact.organizations, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + contacts: Contact[]; + + @ManyToMany((type) => Tag, (tag) => tag.organizations, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + tags: Tag[]; + + @ManyToMany((type) => Sector, (sector) => sector.organizations, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + sectors: Sector[]; + + @ManyToMany((type) => Cidr, (cidr) => cidr.organizations, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + cidrs: Cidr[]; + + @ManyToOne((type) => Organization, (org) => org.children, { + onDelete: 'CASCADE', + nullable: true + }) + parent: Organization; + + @OneToMany((type) => Organization, (org) => org.parent, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + children: Organization[]; + + @OneToMany((type) => Report, (report) => report.organization, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + reports: Report[]; + + @OneToMany((type) => Request, (request) => request.organization, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + requests: Request[]; + + @OneToMany((type) => SslyzeScan, (sslyze) => sslyze.organization, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + sslyzeScans: SslyzeScan[]; + + + @OneToMany((type) => Snapshot, (snapshot) => snapshot.organization, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + snapshots: Snapshot[]; + + @OneToMany((type) => Tally, (tally) => tally.organization, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + tallies: Tally[]; + + @OneToMany((type) => TrustymailScan, (trustymail_scan) => trustymail_scan.organization, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + })@Column({ + nullable: true, + type: 'varchar' + }) + reportPeriod: string | null; + trustymailScans: TrustymailScan[]; + + @OneToMany((type) => VulnScan, (vuln_scan) => vuln_scan.organization, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + vulnScans: VulnScan[]; + + @OneToMany((type) => HostScan, (host_scan) => host_scan.organization, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + hostScans: HostScan[]; + + @OneToMany((type) => Host, (host) => host.organization, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + hosts: Host[]; +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/port_scans.ts b/backend/src/models/mini_data_lake/port_scans.ts new file mode 100644 index 00000000..58141124 --- /dev/null +++ b/backend/src/models/mini_data_lake/port_scans.ts @@ -0,0 +1,81 @@ +// The data in this table is derived from the Vulnerability Scans Database, +// the [port_scans Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#port_scans-collection). + +import { + Entity, + Column, + PrimaryColumn, + BaseEntity, + ManyToMany, + ManyToOne + } from 'typeorm'; + +import {Snapshot} from './snapshots' +import { Ip } from './ips'; +@Entity() +export class PortScan extends BaseEntity { + @PrimaryColumn() + id: string; + + @Column({ + nullable: true, + type: 'varchar' + }) + ipString: string | null; + + @ManyToOne((type) => Ip, (ip) => ip.portScans, { + onDelete: 'CASCADE', + nullable: true + }) + ip: Ip ; + + @Column() + latest: boolean; + + @Column({ + nullable: true, + type: 'integer' + }) + port: number | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + protocol: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + reason: string | null; + + @Column({ + type: 'jsonb', + default: {} + }) + service: object; + + @Column({ + nullable: true, + type: 'varchar' + }) + source: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + state: string | null; + + @Column({ nullable: true, type: 'timestamp' }) + timeScanned : Date | null; + + @ManyToMany((type) => Snapshot, (snapshot) => snapshot.portScans, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + snapshots: Snapshot[]; + + +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/precert_scans.ts b/backend/src/models/mini_data_lake/precert_scans.ts new file mode 100644 index 00000000..0ea16dbb --- /dev/null +++ b/backend/src/models/mini_data_lake/precert_scans.ts @@ -0,0 +1,62 @@ +// The data in this table is derived from the Vulnerability Scans Database, +// the [precerts Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#precerts-collection). + +import { + Entity, + Column, + PrimaryColumn, + BaseEntity, + ManyToMany, + JoinTable + } from 'typeorm'; + + import {Domain} from "./domains" + +@Entity() +export class PrecertScan extends BaseEntity { + @PrimaryColumn() + id: string; + + @Column({ + nullable: true, + type: 'varchar' + }) + issuer: string | null; + + @Column({ nullable: true, type: 'timestamp' }) + expirationTimestamp : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + certStartTimestamp : Date | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + pem: string | null; + + @Column() + sctExists: boolean; + + @Column({ nullable: true, type: 'timestamp' }) + sctOrNotBefore : Date | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + serial: string | null; + + @ManyToMany( + (type) => Domain, + (domain) => domain.precertScans, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + domains: Domain[]; + + +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/reports.ts b/backend/src/models/mini_data_lake/reports.ts new file mode 100644 index 00000000..90e95e41 --- /dev/null +++ b/backend/src/models/mini_data_lake/reports.ts @@ -0,0 +1,39 @@ +// The data in this collection is derived from the Vulnerability Scans Database, +// the [reports Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#reports-collection). + +import { + Entity, + Column, + PrimaryColumn, + BaseEntity, + ManyToOne + } from 'typeorm'; + +import {Organization} from './organizations' +import {Snapshot} from './snapshots' + +@Entity() +export class Report extends BaseEntity { + @PrimaryColumn() + id: string; + + @ManyToOne((type) => Organization, (org) => org.reports, { + onDelete: 'CASCADE', + nullable: true + }) + organization: Organization; + + @Column({ nullable: true, type: 'timestamp' }) + createdTimestamp : Date | null; + + @Column("simple-array") + reportTypes: string[] + + + @ManyToOne((type) => Snapshot, (snapshot) => snapshot.reports, { + onDelete: 'CASCADE', + nullable: true + }) + snapshot: Snapshot; + +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/requests.ts b/backend/src/models/mini_data_lake/requests.ts new file mode 100644 index 00000000..79d91c72 --- /dev/null +++ b/backend/src/models/mini_data_lake/requests.ts @@ -0,0 +1,83 @@ +// The data in this collection is derived from the Vulnerability Scans Database, +// the [requests Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#requests-collection). + + +import { + Entity, + Index, + Column, + PrimaryGeneratedColumn, + BaseEntity, + ManyToMany, + ManyToOne + } from 'typeorm'; + +import { Organization } from './organizations'; +import {Cidr} from './cidrs' +@Entity() +export class Request extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @ManyToOne((type) => Organization, (org) => org.requests, { + onDelete: 'CASCADE', + nullable: true + }) + organization: Organization; + + @Column({ + nullable: true, + type: 'varchar', + unique: true + }) + @Index() + acronym: string | null; + + @Column({ nullable: true, type: 'timestamp' }) + enrolledTimestamp : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + periodStartTimestamp : Date | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + reportPeriod: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + initStage: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + scheduler: string | null; + + @Column({ nullable: true }) + retired: boolean; + + @Column("simple-array") + reportTypes: string[] + + @Column({ + type: 'jsonb', + default: [] + }) + scanWindows: Object[]; + + @Column({ + type: 'jsonb', + default: [] + }) + scanLimits: Object[]; + + @ManyToMany((type) => Cidr, (cidr) => cidr.requests, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + cidrs: Cidr[]; +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/sectors.ts b/backend/src/models/mini_data_lake/sectors.ts new file mode 100644 index 00000000..4c7082ad --- /dev/null +++ b/backend/src/models/mini_data_lake/sectors.ts @@ -0,0 +1,43 @@ +// The data in this collection is derived from the requests collection. + +import { + Entity, + Index, + Column, + PrimaryGeneratedColumn, + BaseEntity, + ManyToMany, + JoinTable + } from 'typeorm'; +import { Organization } from './organizations'; +@Entity() +export class Sector extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ + nullable: true, + type: 'varchar', + unique: true + }) + name: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + @Index() + acronym: string | null; + + @ManyToMany( + (type) => Organization, + (org) => org.sectors, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + organizations: Organization[]; + +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/snapshots.ts b/backend/src/models/mini_data_lake/snapshots.ts new file mode 100644 index 00000000..f3494c40 --- /dev/null +++ b/backend/src/models/mini_data_lake/snapshots.ts @@ -0,0 +1,171 @@ +// The data in this collection is derived from the Vulnerability Scans Database, +// the [snapshots Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#snapshots-collection). + +import { + Entity, + Column, + PrimaryColumn, + BaseEntity, + OneToMany, + ManyToMany, + ManyToOne, + JoinTable + } from 'typeorm'; + + import {HostScan} from './host_scans'; + import {PortScan} from './port_scans' + import {Report} from './reports' + import { Organization } from './organizations'; + import { Ticket } from './tickets'; + import { VulnScan } from './vuln_scans'; +@Entity() +export class Snapshot extends BaseEntity { + @PrimaryColumn() + id: string; + + @Column({ + nullable: true, + type: 'decimal' + }) + cvssAverageAll: number | null; + + @Column({ + nullable: true, + type: 'decimal' + }) + cvssAverageVulnerable: number | null; + + @Column({ nullable: true, type: 'timestamp' }) + startTimestamp : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + endTimestamp : Date | null; + + @Column({ + nullable: true, + type: 'integer' + }) + hostCount: number | null; + + @Column({ nullable: true, type: 'timestamp' }) + updatedTimestamp : Date | null; + + @Column() + latest: boolean; + + @Column("simple-array") + networks: string[] + + @ManyToOne((type) => Organization, (org) => org.snapshots, { + onDelete: 'CASCADE', + nullable: true + }) + organization: Organization ; + + @Column({ + nullable: true, + type: 'integer' + }) + portCount: number | null; + + @Column({ + type: 'jsonb', + default: {} + }) + services: object; + + + + @Column({ + nullable: true, + type: 'integer' + }) + uniqueOsCount: number | null; + + @Column({ + nullable: true, + type: 'integer' + }) + uniquePortCount: number | null; + + @Column({ + type: 'jsonb', + default: {} + }) + uniqueVulnerabilities: object; + + @Column({ + nullable: true, + type: 'integer' + }) + vulnHostCount: number | null; + + + @Column({ + type: 'jsonb', + default: {} + }) + vulnerabilities: object; + + @ManyToMany( + (type) => HostScan, + (hostscan) => hostscan.snapshots, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + hostScans: HostScan[]; + + @ManyToMany( + (type) => PortScan, + (portscan) => portscan.snapshots, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + portScans: PortScan[]; + + @OneToMany((type) => Report, (report) => report.snapshot, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + reports: Report[]; + + @ManyToMany((type) => Ticket, (ticket) => ticket.snapshots, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + tickets: Ticket[]; + + @ManyToMany((type) => VulnScan, (vuln_scan) => vuln_scan.snapshots, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + vulnScans: VulnScan[]; + + @ManyToMany((type) => Snapshot, (snapshot) => snapshot.parents, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + children: Snapshot[]; + + @ManyToMany( + (type) => Snapshot, + (snapshot) => snapshot.children, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + parents: Snapshot[]; + + + + + +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/sslyze_scan.ts b/backend/src/models/mini_data_lake/sslyze_scan.ts new file mode 100644 index 00000000..b0787159 --- /dev/null +++ b/backend/src/models/mini_data_lake/sslyze_scan.ts @@ -0,0 +1,141 @@ +// The data in this collection is derived from domain names collected by +// our [gatherer](https://github.com/cisagov/gatherer), which pulls in +// domains from Cyber Hygiene and the GSA. NOTE: More details may be +// available in the GitHub +// [README](https://github.com/nabla-c0d3/sslyze/blob/master/README.md) +// document for [SSLyze](https://github.com/nabla-c0d3/sslyze). + + +import { + Entity, + Column, + PrimaryColumn, + BaseEntity, + ManyToOne + } from 'typeorm'; + +import {Domain} from './domains'; +import { Organization } from './organizations'; + +@Entity() +export class SslyzeScan extends BaseEntity { + @PrimaryColumn() + id: string; + + @ManyToOne((type) => Organization, (org) => org.sslyzeScans, { + onDelete: 'CASCADE', + nullable: true + }) + organization: Organization; + + @ManyToOne((type) => Domain, (domain) => domain.sslyzeScans, { + onDelete: 'CASCADE', + nullable: true + }) + domain: Domain; + + @Column({nullable: true}) + all_forward_secrecy: boolean; + + @Column({nullable: true}) + allRc4: boolean; + + @Column({nullable: true}) + any_3des: boolean; + + @Column({nullable: true}) + anyForwardSecrecy: boolean; + + @Column({nullable: true}) + anyRc4: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + errors: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + highestConstructedIssuer: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + highestServedIssuer: string | null; + + @Column({nullable: true}) + isSymantecCert: boolean; + + + @Column({ + nullable: true, + type: 'integer' + }) + keyLength: number | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + keyType: string | null; + + @Column({nullable: true}) + latest: boolean; + + @Column({ nullable: true, type: 'timestamp' }) + scanTimestamp : Date | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + scannedHostname: string | null; + + @Column({ + nullable: true, + type: 'integer' + }) + scannedPort: number | null; + + + @Column({nullable: true}) + sha1InConstrustedChain: boolean; + + @Column({nullable: true}) + sha1InServedChain: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + signatureAlgorithm: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + symantecDistrustDate: string | null; + + @Column({nullable: true}) + sslv2: boolean; + + + @Column({nullable: true}) + sslv3: boolean; + + @Column({nullable: true}) + starttlsSmtp: boolean; + + @Column({nullable: true}) + tlsv1_0: boolean; + + @Column({nullable: true}) + tlsv1_1: boolean; + + @Column({nullable: true}) + tlsv1_2: boolean; +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/tag.ts b/backend/src/models/mini_data_lake/tag.ts new file mode 100644 index 00000000..2944ce90 --- /dev/null +++ b/backend/src/models/mini_data_lake/tag.ts @@ -0,0 +1,35 @@ +// The data in this collection is derived from + +import { + Entity, + Column, + PrimaryColumn, + BaseEntity, + ManyToMany, + JoinTable + } from 'typeorm'; + +import { Organization } from './organizations'; +@Entity() +export class Tag extends BaseEntity { + @PrimaryColumn() + id: string; + + @Column({ + nullable: true, + type: 'varchar', + unique: true + }) + name: string | null; + + @ManyToMany( + (type) => Organization, + (org) => org.tags, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + organizations: Organization[]; +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/tallies.ts b/backend/src/models/mini_data_lake/tallies.ts new file mode 100644 index 00000000..4c233775 --- /dev/null +++ b/backend/src/models/mini_data_lake/tallies.ts @@ -0,0 +1,59 @@ +// The data in this table is derived from the Vulnerability Scans Database, +// the [tallies Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#tallies-collection). + +import { + Entity, + Index, + Column, + PrimaryGeneratedColumn, + BaseEntity, + ManyToOne + } from 'typeorm'; +import { Organization } from './organizations'; +@Entity() +export class Tally extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ + nullable: true, + type: 'varchar', + unique: true + }) + @Index() + acronym: string | null; + + @ManyToOne((type) => Organization, (org) => org.tallies, { + onDelete: 'CASCADE', + nullable: true + }) + organization: Organization; + + @Column({ + type: 'jsonb', + default: {} + }) + netscan1Counts: object; + + @Column({ + type: 'jsonb', + default: {} + }) + netscan2Counts: object; + + @Column({ + type: 'jsonb', + default: {} + }) + portscanCounts: object; + + @Column({ + type: 'jsonb', + default: {} + }) + vulnscanCounts: object; + + @Column({ nullable: true, type: 'timestamp' }) + updatedTimestamp : Date | null; + +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/ticket_events.ts b/backend/src/models/mini_data_lake/ticket_events.ts new file mode 100644 index 00000000..0768ad71 --- /dev/null +++ b/backend/src/models/mini_data_lake/ticket_events.ts @@ -0,0 +1,54 @@ +// The data in this table is derived from the Vulnerability Scans Database, +// the [tickets Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#tickets-collection). + + +import { + Entity, + Column, + PrimaryGeneratedColumn, + BaseEntity, + ManyToOne + } from 'typeorm'; + + import {Ticket} from './tickets' + +@Entity() +export class TicketEvent extends BaseEntity { + @PrimaryGeneratedColumn() + id: string; + + // TODO verify this is unique + @Column({ + nullable: true, + type: 'varchar', + unique: true + }) + reference: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + action: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + reason: string | null; + + @Column({ nullable: true, type: 'timestamp' }) + eventTimestamp : Date | null; + + @Column({ + type: 'jsonb', + default: [] + }) + delta: Object[]; + + @ManyToOne((type) => Ticket, (ticket) => ticket.ticketEvents, { + onDelete: 'CASCADE', + nullable: true + }) + ticket: Ticket ; +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/tickets.ts b/backend/src/models/mini_data_lake/tickets.ts new file mode 100644 index 00000000..0c10b98d --- /dev/null +++ b/backend/src/models/mini_data_lake/tickets.ts @@ -0,0 +1,169 @@ +// The data in this table is derived from the Vulnerability Scans Database, +// the [tickets Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#tickets-collection). + +import { + Entity, + Column, + PrimaryColumn, + BaseEntity, + OneToMany, + ManyToMany, + ManyToOne, + JoinTable + } from 'typeorm'; + +import { Organization } from './organizations'; +import { Kev } from './kevs'; +import { Ip } from './ips'; +import { Snapshot } from './snapshots'; +import { TicketEvent } from './ticket_events'; +import { Cve } from './cves'; +@Entity() +export class Ticket extends BaseEntity { + @PrimaryColumn() + id: string; + + @Column({ + nullable: true, + type: 'varchar' + }) + cveString: string | null; + + @ManyToOne((type) => Cve, (cve) => cve.tickets, { + onDelete: 'CASCADE', + nullable: true + }) + cve: Cve ; + + @Column({ + nullable: true, + type: 'decimal' + }) + cvss_base_score: number | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + cvss_version: string | null; + + @ManyToOne((type) => Kev, (kev) => kev.tickets, { + onDelete: 'CASCADE', + nullable: true + }) + kev: Kev ; + + @Column({ + nullable: true, + type: 'varchar' + }) + vulnName: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + cvssScoreSource: string | null; + + @Column({ + nullable: true, + type: 'decimal' + }) + cvssSeverity: number | null; + + @Column({ + nullable: true, + type: 'decimal' + }) + vprScore: number | null; + + @Column() + falsePositive: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + ipString: string | null; + + @ManyToOne((type) => Ip, (ip) => ip.tickets, { + onDelete: 'CASCADE', + nullable: true + }) + ip: Ip ; + + @Column({ nullable: true, type: 'timestamp' }) + updatedTimestamp : Date | null; + + @Column({ + nullable: true, + type: 'decimal' + }) + locationLongitude: number | null; + + @Column({ + nullable: true, + type: 'decimal' + }) + locationLatitude: number | null; + + @Column({nullable: true}) + foundInLatestHostScan: boolean; + + @ManyToOne((type) => Organization, (org) => org.tickets, { + onDelete: 'CASCADE', + nullable: true + }) + organization: Organization; + + @Column({ + nullable: true, + type: 'integer' + }) + vulnPort: number | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + portProtocol: string | null; + + @Column({nullable: true}) + snapshotsBool: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + vulnSource: string | null; + + @Column({ + nullable: true, + type: 'integer' + }) + vulnSourceId: number | null; + + @Column({ nullable: true, type: 'timestamp' }) + closedTimestamp : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + openedTimestamp : Date | null; + + @ManyToMany( + (type) => Snapshot, + (snapshot) => snapshot.tickets, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + snapshots: Snapshot[]; + + @OneToMany((type) => TicketEvent, (ticket_event) => ticket_event.ticket, { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + }) + ticketEvents: TicketEvent[]; + +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/trustymail_scans.ts b/backend/src/models/mini_data_lake/trustymail_scans.ts new file mode 100644 index 00000000..29096ec7 --- /dev/null +++ b/backend/src/models/mini_data_lake/trustymail_scans.ts @@ -0,0 +1,158 @@ +// The data in this collection is derived from domain names collected by +// our [gatherer](https://github.com/cisagov/gatherer), which pulls in +// domains from Cyber Hygiene and the GSA. NOTE: More details may be +// available in the GitHub +// [README](https://github.com/cisagov/trustymail/blob/develop/README.md) +// document for [trustymail](https://github.com/cisagov/trustymail). + +import { + Entity, + Column, + PrimaryColumn, + BaseEntity, + ManyToOne + } from 'typeorm'; +import { Organization } from './organizations'; +import { Domain } from './domains'; + +@Entity() +export class TrustymailScan extends BaseEntity { + @PrimaryColumn() + id: string; + + @ManyToOne((type) => Organization, (org) => org.trustymailScans, { + onDelete: 'CASCADE', + nullable: true + }) + organization: Organization ; + + @ManyToOne((type) => Domain, (domain) => domain.trustymailScans, { + onDelete: 'CASCADE', + nullable: true + }) + domain: Domain ; + + @Column({ + type: 'jsonb', + default: [] + }) + aggregateReportUris: Object[]; + + @Column({ + nullable: true, + type: 'varchar' + }) + debugInfo: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + dmarcPolicy: string | null; + + @Column({ + nullable: true, + type: 'integer' + }) + dmarcPolicyPercentage: number | null; + + @Column({nullable:true}) + dmarcRecord: boolean; + + @Column({nullable:true}) + dmarcRecordBaseDomain: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + dmarcResults: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + dmarcResultsBaseDomain: string | null; + + @Column() + domainSupportsSmtp: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + domainSupportsSmtpResults: string | null; + + @Column() + domainSupportsStarttls: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + domainSupportsStarttlsResults: string | null; + + @Column({ + type: 'jsonb', + default: [] + }) + forensicReportUris: Object[]; + + @Column({nullable:true}) + hasAggregateReportUri: boolean; + + @Column({nullable:true}) + hasForensicReportUri: boolean; + + @Column({nullable:true}) + isBaseDomain: boolean; + + @Column({nullable:true}) + latest: boolean; + + @Column({nullable:true}) + live: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + mailServerPortsTested: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + mailServers: string | null; + + @Column({nullable:true}) + mxRecord: boolean; + + + @Column({ nullable: true, type: 'timestamp' }) + scanTimestamp : Date | null; + + + @Column({ + nullable: true, + type: 'varchar' + }) + spfResults: string | null; + + + @Column({ + nullable: true, + type: 'varchar' + }) + syntaxErrors: string | null; + + @Column({nullable:true}) + validDmarc: boolean; + + @Column({nullable:true}) + validDmarcBaseDomain: boolean; + + @Column({nullable:true}) + validSpf: boolean; + +} \ No newline at end of file diff --git a/backend/src/models/mini_data_lake/vuln_scans.ts b/backend/src/models/mini_data_lake/vuln_scans.ts new file mode 100644 index 00000000..d0d7ae88 --- /dev/null +++ b/backend/src/models/mini_data_lake/vuln_scans.ts @@ -0,0 +1,482 @@ +// The data in this table is derived from the Vulnerability Scans Database, +// the [vuln_scans Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#vuln_scans-collection). + +import { + Entity, + Column, + PrimaryColumn, + BaseEntity, + ManyToMany, + ManyToOne, + JoinTable + } from 'typeorm'; +import { Snapshot } from './snapshots'; +import { Ip } from './ips'; +import { Organization } from './organizations'; +import { Cve } from './cves'; + +@Entity() +export class VulnScan extends BaseEntity { + @PrimaryColumn() + id: string; + + @Column({ + nullable: true, + type: 'varchar' + }) + bugtraqId: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + certId: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + cpe: string | null; + + //TODO link cve + @Column({ + nullable: true, + type: 'varchar' + }) + cveString: string | null; + + @ManyToOne((type) => Cve, (cve) => cve.vulnScans, { + onDelete: 'CASCADE', + nullable: true + }) + cve: Cve; + + @Column({ + nullable: true, + type: 'varchar' + }) + cvssBaseScore: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + cvssTemporalScore: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + cvssTemporalVector: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + cvssVector: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + description: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + exploitAvailable: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + exploitabilityEase: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + pluginFilename: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + ipString: string | null; + + @ManyToOne((type) => Ip, (ip) => ip.vulnScans, { + onDelete: 'CASCADE', + nullable: true + }) + ip: Ip ; + + @Column() + latest: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + owner: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + osvdbId: string | null; + + @ManyToOne((type) => Organization, (org) => org.vulnScans, { + onDelete: 'CASCADE', + nullable: true + }) + organization: Organization ; + + @Column({ nullable: true, type: 'timestamp' }) + patchPublicationTimestamp : Date | null; + + @Column({ + nullable: true, + type: 'integer' + }) + port: number | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + portProtocol: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + riskFactor: string | null; + + + @Column({ + nullable: true, + type: 'varchar' + }) + scriptVersion: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + seeAlso: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + service: string | null; + + @Column({ + nullable: true, + type: 'integer' + }) + severity: number | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + solution: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + source: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + synopsis: string | null; + + @Column({ nullable: true, type: 'timestamp' }) + vulnDetectionTimestamp : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + vulnPublicationTimestamp : Date | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + xref: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + cwe: string | null; + + @Column({ nullable: true }) + exploitFrameworkMetasploit: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + metasploitName: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + edbId: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + bid: string | null; + + @Column({ nullable: true }) + exploitedByMalware: boolean; + + @Column({ nullable: true }) + exploitFrameworkCore: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + stigSeverity: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + iava: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + iavb: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + tra: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + msft: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + canvasPackage: string | null; + + @Column({ nullable: true }) + exploitFrameworkCanvas: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + secunia: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + agent: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + rhsa: string | null; + + @Column({ nullable: true }) + inTheNews: boolean; + + @Column({ nullable: true }) + thoroughTests: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + cvssScoreRationale: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + attachment: string | null; + + @Column({ + nullable: true, + type: 'decimal' + }) + vprScore: number | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + threatSourcesLast28: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + exploitCodeMaturity: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + threatIntensityLast28: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + ageOfVuln: string | null; + + @Column({ + nullable: true, + type: 'decimal' + }) + cvssV3ImpactScore: number | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + threatRecency: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + productCoverage: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + cvssScoreSource: string | null; + + @Column({ + nullable: true, + type: 'decimal' + }) + cvss3BaseScore: number | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + cvss3Vector: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + cvss3TemporalVector: string | null; + + @Column({ + nullable: true, + type: 'decimal' + }) + cvss3TemporalScore: number | null; + + @Column({ nullable: true }) + exploitedByNessus: boolean; + + @Column({ nullable: true }) + unsupportedByVendor: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + requiredKey: string | null; + + @Column({ + nullable: true, + type: 'integer' + }) + requiredPort: number | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + scriptCopyright: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + mskb: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + dependency: string | null; + + @Column({ nullable: true }) + assetInventory: boolean; + + @Column({ + nullable: true, + type: 'varchar' + }) + pluginId: string | null; + + @Column({ nullable: true, type: 'timestamp' }) + pluginModificationDate : Date | null; + + @Column({ nullable: true, type: 'timestamp' }) + pluginPublicationDate : Date | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + pluginName: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + pluginType: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + pluginFamily: string | null; + + @Column({ + nullable: true, + type: 'varchar' + }) + pluginOutput: string | null; + + @ManyToMany( + (type) => Snapshot, + (snapshot) => snapshot.vulnScans, + { + onDelete: 'CASCADE', + onUpdate: 'CASCADE' + } + ) + @JoinTable() + snapshots: Snapshot[]; +} \ No newline at end of file diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index c821a00c..3109c32f 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -23,6 +23,10 @@ pesyncdb: timeout: 900 handler: src/tasks/pesyncdb.handler +syncmdl: + timeout: 900 + handler: src/tasks/syncmdl.handler + bastion: timeout: 900 handler: src/tasks/bastion.handler diff --git a/backend/src/tasks/syncmdl.ts b/backend/src/tasks/syncmdl.ts new file mode 100644 index 00000000..29c6904b --- /dev/null +++ b/backend/src/tasks/syncmdl.ts @@ -0,0 +1,59 @@ +import { Handler } from 'aws-lambda'; +import { + connectToDatalake, + connectToDatabase +} from '../models'; + +export const handler: Handler = async (event) => { + + const connection = await connectToDatabase(); + try { + await connection.query( + `CREATE USER ${process.env.MDL_USERNAME} WITH PASSWORD '${process.env.MDL_PASSWORD}';` + ); + } catch (e) { + console.log( + "Create user failed. This usually means that the user already exists, so you're OK if that was the case. Here's the exact error:", + e + ); + } + try { + await connection.query( + `GRANT ${process.env.MDL_USERNAME} to ${process.env.DB_USERNAME};` + ); + } catch (e) { + console.log('Grant role failed. Error:', e); + } + try { + await connection.query( + `CREATE DATABASE ${process.env.MDL_NAME} owner ${process.env.MDL_USERNAME};` + ); + } catch (e) { + console.log( + "Create database failed. This usually means that the database already exists, so you're OK if that was the case. Here's the exact error:", + e + ); + } + + try { + await connection.query( + `GRANT ALL PRIVILEGES ON DATABASE ${process.env.MDL_NAME} TO ${process.env.MDL_USERNAME};` + ); + } catch (e) { + console.log( + "Failed to give user all role. This usually means the user already has this role, so you're OK if that was the case. Here's the exact error:", + e + ); + } + + const mdl_connection = await connectToDatalake(true); + const type = event?.type || event; + const dangerouslyforce = type === 'dangerouslyforce'; + + if (mdl_connection) { + await mdl_connection.synchronize(dangerouslyforce); + } else { + console.error('Error: could not sync'); + } + +}; diff --git a/backend/src/tools/run-syncmdl.ts b/backend/src/tools/run-syncmdl.ts new file mode 100644 index 00000000..ace4776e --- /dev/null +++ b/backend/src/tools/run-syncmdl.ts @@ -0,0 +1,15 @@ +import { handler as syncmdl } from '../tasks/syncmdl'; + +/** + * Runs syncmdl locally. This script is called from running "npm run syncmdl". + * Call "npm run syncmdl -- -d dangerouslyforce" to force dropping and + * recreating the SQL database. + * */ + +process.env.MDL_HOST = 'db'; +process.env.MDL_USERNAME = 'mdl'; +process.env.MDL_PASSWORD = 'password'; +process.env.MDL_DATABASE = 'crossfeed_mini_datalake' + + +syncmdl(process.argv[2] === '-d' ? process.argv[3] : '', {} as any, () => null);