Extract cyhy orgs & vuln scans data

backend/package.json

@@ -25,6 +25,7 @@
     "helmet": "^4.1.1",
     "http-proxy-middleware": "^2.0.6",
     "ip": "^1.1.9",
+    "ip-address":"^9.0.5",
     "jsdom": "^22.1",
     "jsonwebtoken": "^9.0.2",
     "jwks-rsa": "^3.0",
@@ -39,6 +40,7 @@
     "reflect-metadata": "^0.1.13",
     "serverless-http": "^3.2.0",
     "ssl-checker": "^2.0.7",
+    "tar":"^7.0.0",
     "tough-cookie": "^4.1.3",
     "typeorm": "^0.2.45",
     "utf-8-validate": "^6.0.3",

backend/src/api/scans.ts

@@ -88,6 +88,14 @@ export const SCAN_SCHEMA: ScanSchema = {
     description:
       "Matches detected software versions to CVEs from NIST NVD and CISA's Known Exploited Vulnerabilities Catalog."
   },
+  vulnScanningSync: {
+    type: 'fargate',
+    isPassive: true,
+    global: true,
+    description: 'Pull in vulnerability data from VSs Vulnerability database',
+    cpu: '1024',
+    memory: '8192'
+  },
   cveSync: {
     type: 'fargate',
     isPassive: true,

backend/src/models/connection.ts

@@ -37,7 +37,6 @@ import {
   PortScan,
   PrecertScan,
   Report,
-  Request,
   Sector,
   Snapshot,
   SslyzeScan,
@@ -54,6 +53,14 @@ let connection: Connection | null = null;
 let dl_connection: Connection | null = null;
 
 const connectDl = async (logging?: boolean) => {
+  // process.env.DB_HOST = 'db';
+  // process.env.MDL_USERNAME = 'mdl';
+  // process.env.MDL_PASSWORD = 'password';
+  // process.env.MDL_NAME = 'crossfeed_mini_datalake';
+
+  // console.log(process.env.MDL_USERNAME)
+  // console.log(process.env.MDL_PASSWORD)
+  // console.log(process.env.MDL_NAME)
   const dl_connection = createConnection({
     type: 'postgres',
     host: process.env.DB_HOST,
@@ -77,7 +84,6 @@ const connectDl = async (logging?: boolean) => {
       PortScan,
       PrecertScan,
       Report,
-      Request,
       Sector,
       Snapshot,
       SslyzeScan,
@@ -89,7 +95,7 @@ const connectDl = async (logging?: boolean) => {
       VulnScan
     ],
     synchronize: false,
-    name: 'mini_data_lake',
+    name: 'default',
     dropSchema: false,
     logging: logging ?? false,
     cache: true
@@ -99,6 +105,7 @@ const connectDl = async (logging?: boolean) => {
 
 export const connectToDatalake = async (logging?: boolean) => {
   if (!dl_connection?.isConnected) {
+    console.log("Connected to datalake")
     dl_connection = await connectDl(logging);
   } else {
     console.log("didn't connect");

backend/src/models/index.ts

@@ -34,7 +34,6 @@ export * from './mini_data_lake/locations';
 export * from './mini_data_lake/port_scans';
 export * from './mini_data_lake/precert_scans';
 export * from './mini_data_lake/reports';
-export * from './mini_data_lake/requests';
 export * from './mini_data_lake/sectors';
 export * from './mini_data_lake/snapshots';
 export * from './mini_data_lake/sslyze_scan';

backend/src/models/mini_data_lake/cidrs.ts

@@ -6,10 +6,10 @@ import {
   CreateDateColumn,
   BaseEntity,
   ManyToMany,
-  JoinTable
+  JoinTable,
+  UpdateDateColumn
 } from 'typeorm';
 
-import { Request } from './requests';
 import { Organization } from './organizations';
 @Entity()
 export class Cidr extends BaseEntity {
@@ -22,7 +22,7 @@ export class Cidr extends BaseEntity {
   @Index()
   @Column({
     nullable: true,
-    type: 'cidr',
+    type: 'inet',
     unique: true
   })
   network: string | null;
@@ -42,12 +42,8 @@ export class Cidr extends BaseEntity {
   @Column({ nullable: true })
   retired: boolean;
 
-  @ManyToMany((type) => Request, (request) => request.cidrs, {
-    onDelete: 'CASCADE',
-    onUpdate: 'CASCADE'
-  })
-  @JoinTable()
-  requests: Request[];
+  @UpdateDateColumn()
+  updatedAt: Date | null;
 
   @ManyToMany((type) => Organization, (org) => org.cidrs, {
     onDelete: 'CASCADE',
@@ -56,3 +52,5 @@ export class Cidr extends BaseEntity {
   @JoinTable()
   organizations: Organization[];
 }
+
+

backend/src/models/mini_data_lake/ips.ts

@@ -3,11 +3,14 @@
 
 import {
   Entity,
+  Unique,
   Column,
   PrimaryGeneratedColumn,
   BaseEntity,
   OneToMany,
-  ManyToOne
+  ManyToOne,
+  UpdateDateColumn,
+  CreateDateColumn
 } from 'typeorm';
 import { Domain } from './domains';
 import { HostScan } from './host_scans';
@@ -17,6 +20,7 @@ import { Ticket } from './tickets';
 import { VulnScan } from './vuln_scans';
 import { PortScan } from './port_scans';
 @Entity()
+@Unique(['ip','organization'])
 export class Ip extends BaseEntity {
   @PrimaryGeneratedColumn('uuid')
   id: string;
@@ -28,8 +32,11 @@ export class Ip extends BaseEntity {
   })
   organization: Organization;
 
-  @Column({ nullable: true, type: 'timestamp' })
-  createdTimestamp: Date | null;
+  @CreateDateColumn()
+  createdTimestamp: Date;
+
+  @UpdateDateColumn()
+  updatedTimestamp: Date | null;
 
   @Column({ nullable: true, type: 'timestamp' })
   lastSeenTimestamp: Date | null;

backend/src/models/mini_data_lake/kevs.ts

@@ -26,6 +26,50 @@ export class Kev extends BaseEntity {
   @Column({ nullable: true })
   knownRansomware: boolean;
 
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  vendorProject: string | null;
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  product: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  vulnerabilityName: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'timestamp'
+  })
+  dateAdded: Date | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  shortDescription: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  requiredAction: string | null;
+
+  @Column({ nullable: true, type: 'timestamp' })
+  dueDate: Date | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  notes: string | null;
+
   @OneToMany((type) => Ticket, (ticket) => ticket.kev, {
     onDelete: 'CASCADE',
     onUpdate: 'CASCADE'

backend/src/models/mini_data_lake/locations.ts

@@ -7,8 +7,7 @@ import {
   Column,
   PrimaryGeneratedColumn,
   BaseEntity,
-  ManyToMany,
-  JoinTable
+  OneToMany
 } from 'typeorm';
 
 import { Organization } from './organizations';
@@ -27,7 +26,7 @@ export class Location extends BaseEntity {
     nullable: true,
     type: 'varchar'
   })
-  countryAbvr: string | null;
+  countryAbrv: string | null;
 
   @Column({
     nullable: true,
@@ -72,10 +71,9 @@ export class Location extends BaseEntity {
   })
   state: string | null;
 
-  @ManyToMany((type) => Organization, (org) => org.locations, {
+  @OneToMany((type) => Organization, (org) => org.location, {
     onDelete: 'CASCADE',
     onUpdate: 'CASCADE'
   })
-  @JoinTable()
-  organizations: Organization[];
+  organizations?: Organization[];
 }

backend/src/models/mini_data_lake/organizations.ts

@@ -9,7 +9,9 @@ import {
   BaseEntity,
   OneToMany,
   ManyToMany,
-  ManyToOne
+  ManyToOne,
+  CreateDateColumn,
+  UpdateDateColumn
 } from 'typeorm';
 import { Domain } from './domains';
 import { Ip } from './ips';
@@ -18,7 +20,6 @@ import { Contact } from './contacts';
 import { Tag } from './tag';
 import { Sector } from './sectors';
 import { Report } from './reports';
-import { Request } from './requests';
 import { SslyzeScan } from './sslyze_scan';
 import { Snapshot } from './snapshots';
 import { Tally } from './tallies';
@@ -48,27 +49,18 @@ export class Organization extends BaseEntity {
   acronym: string | null;
 
   @Column({ nullable: true, type: 'timestamp' })
-  firstEngageDate: Date | null;
+  enrolledInVsTimestamp: Date | null;
 
   @Column({ nullable: true, type: 'timestamp' })
-  createdDate: Date | null;
+  periodStartVsTimestamp: Date | null;
 
-  @Column({
-    nullable: true,
-    type: 'varchar'
-  })
-  createdEmplyeeId: string | null;
+  @CreateDateColumn()
+  createdDate: Date;
 
-  @Column({ nullable: true, type: 'timestamp' })
+  @UpdateDateColumn()
   updatedDate: Date | null;
 
-  @Column({
-    nullable: true,
-    type: 'varchar'
-  })
-  updatedEmployeeId: string | null;
-
-  @Column({ nullable: true })
+  @Column({ nullable: true})
   retired: boolean;
 
   @Column({ nullable: true })
@@ -92,6 +84,44 @@ export class Organization extends BaseEntity {
   @Column({ nullable: true })
   stakeholder: boolean;
 
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  reportPeriod: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  initStage: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  scheduler: string | null;
+
+  @Column('varchar', { array: true, default: [], nullable: true })
+  reportTypes: string[] | null;
+
+  @Column('varchar', { array: true, default: [], nullable: true })
+  scanTypes: string[] | null;
+
+  @Column({
+    nullable: true,
+    type: 'jsonb',
+    default: []
+  })
+  scanWindows: Object[] | null;
+
+  @Column({
+    nullable: true,
+    type: 'jsonb',
+    default: []
+  })
+  scanLimits: Object[] | null;
+
   @OneToMany((type) => Domain, (domain) => domain.organization, {
     onDelete: 'CASCADE',
     onUpdate: 'CASCADE'
@@ -110,11 +140,11 @@ export class Organization extends BaseEntity {
   })
   tickets: Ticket[];
 
-  @ManyToMany((type) => Location, (location) => location.organizations, {
+  @ManyToOne((type) => Location, (location) => location.organizations, {
     onDelete: 'CASCADE',
-    onUpdate: 'CASCADE'
+    nullable: true
   })
-  locations: Location[];
+  location: Location;
 
   @ManyToMany((type) => Contact, (contact) => contact.organizations, {
     onDelete: 'CASCADE',
@@ -158,12 +188,6 @@ export class Organization extends BaseEntity {
   })
   reports: Report[];
 
-  @OneToMany((type) => Request, (request) => request.organization, {
-    onDelete: 'CASCADE',
-    onUpdate: 'CASCADE'
-  })
-  requests: Request[];
-
   @OneToMany((type) => SslyzeScan, (sslyze) => sslyze.organization, {
     onDelete: 'CASCADE',
     onUpdate: 'CASCADE'
@@ -183,18 +207,12 @@ export class Organization extends BaseEntity {
   tallies: Tally[];
 
   @OneToMany(
-    (type) => TrustymailScan,
-    (trustymail_scan) => trustymail_scan.organization,
+    (type) => TrustymailScan, (trustymail_scan) => trustymail_scan.organization,
     {
       onDelete: 'CASCADE',
       onUpdate: 'CASCADE'
     }
   )
-  @Column({
-    nullable: true,
-    type: 'varchar'
-  })
-  reportPeriod: string | null;
   trustymailScans: TrustymailScan[];
 
   @OneToMany((type) => VulnScan, (vuln_scan) => vuln_scan.organization, {