Create mini datalake generation

Create models and handler to create mini datalake to store org and scan data

.github/workflows/backend.yml

@@ -189,6 +189,13 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
+      - name: Run syncmdl
+        run: aws lambda invoke --function-name crossfeed-staging-syncmdl --region us-east-1 /dev/stdout
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
   deploy_prod:
     needs: [build_worker, lint, test, test_python]
     runs-on: ubuntu-latest
@@ -238,3 +245,10 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Run syncmdl
+        run: aws lambda invoke --function-name crossfeed-prod-syncmdl --region us-east-1 /dev/stdout
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

backend/package.json

@@ -112,6 +112,7 @@
     "deploy-worker-prod": "./tools/deploy-worker.sh crossfeed-prod-worker",
     "syncdb": "docker-compose exec -T backend npx ts-node src/tools/run-syncdb.ts",
     "pesyncdb": "docker-compose exec -T backend npx ts-node src/tools/run-pesyncdb.ts",
+    "syncmdl": "docker-compose exec -T backend npx ts-node src/tools/run-syncmdl.ts",
     "control-queue": "docker-compose exec -T backend npx ts-node src/tools/consumeControlQueue.ts"
   },
   "author": "",

backend/src/models/connection.ts

@@ -1,5 +1,6 @@
 import { createConnection, Connection } from 'typeorm';
 import {
+  // Models for the Crossfeed database
   Domain,
   Service,
   Vulnerability,
@@ -13,11 +14,95 @@ import {
   SavedSearch,
   OrganizationTag,
   Cpe,
-  Cve
+  Cve,
+
+  //  Models for the Mini Data Lake database
+  CertScan,
+  Cidr,
+  Contact,
+  DL_Cpe,
+  DL_Cve,
+  DL_Domain,
+  HostScan,
+  Host,
+  Ip,
+  Kev,
+  Location,
+  DL_Organization,
+  PortScan,
+  PrecertScan,
+  Report,
+  Request,
+  Sector,
+  Snapshot,
+  SslyzeScan,
+  Tag,
+  Tally,
+  TicketEvent,
+  Ticket,
+  TrustymailScan,
+  VulnScan
+
 } from '.';
 
 let connection: Connection | null = null;
 
+let dl_connection: Connection | null = null;
+
+const connectDl = async (logging?:boolean) => {
+  const dl_connection = createConnection({
+    type: 'postgres',
+    host: process.env.MDL_HOST,
+    port: parseInt(process.env.MDL_PORT ?? ''),
+    username: process.env.MDL_USERNAME,
+    password: process.env.MDL_PASSWORD,
+    database: process.env.MDL_NAME,
+    entities: [
+      CertScan,
+      Cidr,
+      Contact,
+      DL_Cpe,
+      DL_Cve,
+      DL_Domain,
+      HostScan,
+      Host,
+      Ip,
+      Kev,
+      Location,
+      DL_Organization,
+      PortScan,
+      PrecertScan,
+      Report,
+      Request,
+      Sector,
+      Snapshot,
+      SslyzeScan,
+      Tag,
+      Tally,
+      TicketEvent,
+      Ticket,
+      TrustymailScan,
+      VulnScan
+    ],
+    synchronize: false,
+    name: 'mini_data_lake',
+    dropSchema: false,
+    logging: logging ?? false,
+    cache: true
+  });
+  return dl_connection
+}
+
+export const connectToDatalake = async (logging?: boolean) => {
+  if (!dl_connection?.isConnected) {
+    dl_connection = await connectDl(logging);
+  }
+  else {
+    console.log("didn't connect")
+  }
+  return dl_connection;
+};
+
 const connectDb = async (logging?: boolean) => {
   const connection = createConnection({
     type: 'postgres',

backend/src/models/index.ts

@@ -13,3 +13,29 @@ export * from './webpage';
 export * from './api-key';
 export * from './saved-search';
 export * from './organization-tag';
+
+export * from './mini_data_lake/cert_scans';
+export * from './mini_data_lake/cidrs';
+export * from './mini_data_lake/contacts';
+export {Cpe as DL_Cpe} from './mini_data_lake/cpes';
+export {Cve as DL_Cve} from './mini_data_lake/cves';
+export {Domain as DL_Domain} from './mini_data_lake/domains';
+export * from './mini_data_lake/host_scans';
+export * from './mini_data_lake/hosts';
+export * from './mini_data_lake/ips';
+export * from './mini_data_lake/kevs';
+export * from './mini_data_lake/locations';
+export {Organization as DL_Organization} from './mini_data_lake/organizations';
+export * from './mini_data_lake/port_scans';
+export * from './mini_data_lake/precert_scans';
+export * from './mini_data_lake/reports';
+export * from './mini_data_lake/requests';
+export * from './mini_data_lake/sectors';
+export * from './mini_data_lake/snapshots';
+export * from './mini_data_lake/sslyze_scan';
+export * from './mini_data_lake/tag';
+export * from './mini_data_lake/tallies';
+export * from './mini_data_lake/ticket_events';
+export * from './mini_data_lake/tickets';
+export * from './mini_data_lake/trustymail_scans';
+export * from './mini_data_lake/vuln_scans';

backend/src/models/mini_data_lake/cert_scans.ts

@@ -0,0 +1,61 @@
+// The data in this table is derived from the Vulnerability Scans Database, 
+// the [certs Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#certs-collection).
+
+import {
+    Entity,
+    Column,
+    PrimaryColumn,
+    BaseEntity,
+    ManyToMany,
+    JoinTable
+  } from 'typeorm';
+
+  import {Domain} from "./domains"
+
+@Entity()
+export class CertScan extends BaseEntity {
+  @PrimaryColumn()
+  id: string
+
+  @Column({
+      nullable: true,
+      type: 'varchar'
+    })
+    issuer: string | null;
+
+  @Column({ nullable: true, type: 'timestamp' })
+  expirationTimestamp: Date | null;
+
+  @Column({ nullable: true, type: 'timestamp' })
+  certStartTimestamp: Date | null;
+
+  @Column({
+      nullable: true,
+      type: 'varchar'
+    })
+    pem: string | null;
+
+  @Column()
+    sctExists: boolean;
+
+  @Column({ nullable: true, type: 'timestamp' })
+    sctOrNotBefore: Date | null;
+
+  @Column({
+      nullable: true,
+      type: 'varchar'
+    })
+    serial: string | null;
+
+  @ManyToMany(
+      (type) => Domain,
+      (domain) => domain.certScans,
+      {
+        onDelete: 'CASCADE',
+        onUpdate: 'CASCADE'
+      }
+    )
+    @JoinTable()
+    domains: Domain[];
+
+}

backend/src/models/mini_data_lake/cidrs.ts

@@ -0,0 +1,68 @@
+import {
+    Entity,
+    Index,
+    Column,
+    PrimaryColumn,
+    CreateDateColumn,
+    BaseEntity,
+    ManyToMany,
+    JoinTable
+  } from 'typeorm';
+
+import {Request } from './requests';
+import { Organization } from './organizations';
+@Entity()
+export class Cidr extends BaseEntity {
+  @PrimaryColumn()
+  id: string
+
+  @CreateDateColumn()
+    createdDate: Date;
+
+  @Index()
+  @Column({
+      nullable: true,
+      type: 'cidr',
+      unique: true
+    })
+    network: string | null;
+
+  @Column({
+      nullable: true,
+      type: 'inet'
+    })
+    startIp: string | null;
+
+  @Column({
+      nullable: true,
+      type: 'inet'
+    })
+    endIp: string | null;
+
+  @Column({nullable: true})
+    retired: boolean;
+
+  @ManyToMany(
+    (type) => Request,
+    (request) => request.cidrs,
+    {
+      onDelete: 'CASCADE',
+      onUpdate: 'CASCADE'
+    }
+  )
+  @JoinTable()
+  requests: Request[];
+
+  @ManyToMany(
+    (type) => Organization,
+    (org) => org.cidrs,
+    {
+      onDelete: 'CASCADE',
+      onUpdate: 'CASCADE'
+    }
+  )
+  @JoinTable()
+  organizations: Organization[];
+
+
+}

backend/src/models/mini_data_lake/contacts.ts

@@ -0,0 +1,58 @@
+// The data in this table is derived from the Vulnerability Scans Database,
+// the [requests Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#requests-collection).
+
+import {
+    Entity,
+    Column,
+    PrimaryGeneratedColumn,
+    BaseEntity,
+    ManyToMany,
+    Unique,
+    JoinTable
+  } from 'typeorm';
+import {Organization} from './organizations'
+@Entity()
+@Unique(['name','email','type'])
+export class Contact extends BaseEntity {
+    @PrimaryGeneratedColumn('uuid')
+    id: string;
+
+    @Column({
+        nullable: true,
+        type: 'varchar'
+      })
+      name: string | null;
+
+    @Column({
+        nullable: true,
+        type: 'varchar'
+      })
+      email: string | null;
+
+    @Column({
+        nullable: true,
+        type: 'varchar'
+      })
+      phoneNumber: string | null;
+
+    @Column({
+        nullable: true,
+        type: 'varchar'
+      })
+      type: string | null;
+
+    @Column()
+      retired: boolean;
+
+    @ManyToMany(
+        (type) => Organization,
+        (org) => org.contacts,
+        {
+          onDelete: 'CASCADE',
+          onUpdate: 'CASCADE'
+        }
+      )
+      @JoinTable()
+      organizations: Organization[];
+
+}

backend/src/models/mini_data_lake/cpes.ts

@@ -0,0 +1,32 @@
+import {
+    Entity,
+    PrimaryGeneratedColumn,
+    Column,
+    ManyToMany,
+    BaseEntity,
+    Unique
+  } from 'typeorm';
+  import { Cve } from './cves';
+
+  @Entity()
+  @Unique(['name', 'version', 'vendor'])
+  export class Cpe extends BaseEntity {
+    @PrimaryGeneratedColumn('uuid')
+    id: string;
+
+    @Column()
+    name: string;
+
+    @Column()
+    version: string;
+
+    @Column()
+    vendor: string;
+
+    @Column()
+    lastSeenAt: Date;
+
+    @ManyToMany(() => Cve, (cve) => cve.cpes)
+    cves: Cve[];
+  }
+