Create mini datalake database

.github/workflows/backend.yml

@@ -189,6 +189,15 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
+      - name: Run syncmdl
+        run: |
+          aws lambda invoke --function-name crossfeed-staging-syncmdl \
+          --region us-east-1 /dev/stdout
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
   deploy_prod:
     needs: [build_worker, lint, test, test_python]
     runs-on: ubuntu-latest
@@ -238,3 +247,12 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Run syncmdl
+        run: |
+          aws lambda invoke --function-name crossfeed-prod-syncmdl --region us-east-1 \
+          /dev/stdout
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

backend/env.yml

@@ -9,6 +9,9 @@ staging:
   DB_NAME: ${ssm:/crossfeed/staging/DATABASE_NAME}
   DB_USERNAME: ${ssm:/crossfeed/staging/DATABASE_USER}
   DB_PASSWORD: ${ssm:/crossfeed/staging/DATABASE_PASSWORD}
+  MDL_USERNAME: ${ssm:/crossfeed/staging/MDL_USERNAME}
+  MDL_PASSWORD: ${ssm:/crossfeed/staging/MDL_PASSWORD}
+  MDL_NAME: ${ssm:/crossfeed/staging/MDL_NAME}
   PE_DB_NAME: ${ssm:/crossfeed/staging/PE_DB_NAME}
   PE_DB_USERNAME: ${ssm:/crossfeed/staging/PE_DB_USERNAME}
   PE_DB_PASSWORD: ${ssm:/crossfeed/staging/PE_DB_PASSWORD}
@@ -67,6 +70,9 @@ prod:
   DB_NAME: ${ssm:/crossfeed/prod/DATABASE_NAME}
   DB_USERNAME: ${ssm:/crossfeed/prod/DATABASE_USER}
   DB_PASSWORD: ${ssm:/crossfeed/prod/DATABASE_PASSWORD}
+  MDL_USERNAME: ${ssm:/crossfeed/prod/MDL_USERNAME}
+  MDL_PASSWORD: ${ssm:/crossfeed/prod/MDL_PASSWORD}
+  MDL_NAME: ${ssm:/crossfeed/prod/MDL_NAME}
   JWT_SECRET: ${ssm:/crossfeed/prod/APP_JWT_SECRET}
   LOGIN_GOV_REDIRECT_URI: ${ssm:/crossfeed/prod/LOGIN_GOV_REDIRECT_URI}
   LOGIN_GOV_BASE_URL: ${ssm:/crossfeed/prod/LOGIN_GOV_BASE_URL}

backend/package.json

@@ -112,6 +112,7 @@
     "lint:fix": "eslint '**/*.{ts,tsx,js,jsx}' --fix",
     "pesyncdb": "docker-compose exec -T backend npx ts-node src/tools/run-pesyncdb.ts",
     "syncdb": "docker-compose exec -T backend npx ts-node src/tools/run-syncdb.ts",
+    "syncmdl": "docker-compose exec -T backend npx ts-node src/tools/run-syncmdl.ts",
     "test": "jest --detectOpenHandles",
     "test-python": "pytest"
   },

backend/src/models/connection.ts

@@ -1,5 +1,6 @@
 import { createConnection, Connection } from 'typeorm';
 import {
+  // Models for the Crossfeed database
   Domain,
   Service,
   Vulnerability,
@@ -13,11 +14,93 @@ import {
   SavedSearch,
   OrganizationTag,
   Cpe,
-  Cve
+  Cve,
+
+  //  Models for the Mini Data Lake database
+  CertScan,
+  Cidr,
+  Contact,
+  DL_Cpe,
+  DL_Cve,
+  DL_Domain,
+  HostScan,
+  Host,
+  Ip,
+  Kev,
+  Location,
+  DL_Organization,
+  PortScan,
+  PrecertScan,
+  Report,
+  Request,
+  Sector,
+  Snapshot,
+  SslyzeScan,
+  Tag,
+  Tally,
+  TicketEvent,
+  Ticket,
+  TrustymailScan,
+  VulnScan
 } from '.';
 
 let connection: Connection | null = null;
 
+let dl_connection: Connection | null = null;
+
+const connectDl = async (logging?: boolean) => {
+  const dl_connection = createConnection({
+    type: 'postgres',
+    host: process.env.DB_HOST,
+    port: parseInt(process.env.DB_PORT ?? ''),
+    username: process.env.MDL_USERNAME,
+    password: process.env.MDL_PASSWORD,
+    database: process.env.MDL_NAME,
+    entities: [
+      CertScan,
+      Cidr,
+      Contact,
+      DL_Cpe,
+      DL_Cve,
+      DL_Domain,
+      HostScan,
+      Host,
+      Ip,
+      Kev,
+      Location,
+      DL_Organization,
+      PortScan,
+      PrecertScan,
+      Report,
+      Request,
+      Sector,
+      Snapshot,
+      SslyzeScan,
+      Tag,
+      Tally,
+      TicketEvent,
+      Ticket,
+      TrustymailScan,
+      VulnScan
+    ],
+    synchronize: false,
+    name: 'mini_data_lake',
+    dropSchema: false,
+    logging: logging ?? false,
+    cache: true
+  });
+  return dl_connection;
+};
+
+export const connectToDatalake = async (logging?: boolean) => {
+  if (!dl_connection?.isConnected) {
+    dl_connection = await connectDl(logging);
+  } else {
+    console.log("didn't connect");
+  }
+  return dl_connection;
+};
+
 const connectDb = async (logging?: boolean) => {
   const connection = createConnection({
     type: 'postgres',

backend/src/models/index.ts

@@ -13,3 +13,29 @@ export * from './webpage';
 export * from './api-key';
 export * from './saved-search';
 export * from './organization-tag';
+
+export * from './mini_data_lake/cert_scans';
+export * from './mini_data_lake/cidrs';
+export * from './mini_data_lake/contacts';
+export { Cpe as DL_Cpe } from './mini_data_lake/cpes';
+export { Cve as DL_Cve } from './mini_data_lake/cves';
+export { Domain as DL_Domain } from './mini_data_lake/domains';
+export * from './mini_data_lake/host_scans';
+export * from './mini_data_lake/hosts';
+export * from './mini_data_lake/ips';
+export * from './mini_data_lake/kevs';
+export * from './mini_data_lake/locations';
+export { Organization as DL_Organization } from './mini_data_lake/organizations';
+export * from './mini_data_lake/port_scans';
+export * from './mini_data_lake/precert_scans';
+export * from './mini_data_lake/reports';
+export * from './mini_data_lake/requests';
+export * from './mini_data_lake/sectors';
+export * from './mini_data_lake/snapshots';
+export * from './mini_data_lake/sslyze_scan';
+export * from './mini_data_lake/tag';
+export * from './mini_data_lake/tallies';
+export * from './mini_data_lake/ticket_events';
+export * from './mini_data_lake/tickets';
+export * from './mini_data_lake/trustymail_scans';
+export * from './mini_data_lake/vuln_scans';

backend/src/models/mini_data_lake/cert_scans.ts

@@ -0,0 +1,56 @@
+// The data in this table is derived from the Vulnerability Scans Database,
+// the [certs Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#certs-collection).
+
+import {
+  Entity,
+  Column,
+  PrimaryColumn,
+  BaseEntity,
+  ManyToMany,
+  JoinTable
+} from 'typeorm';
+
+import { Domain } from './domains';
+
+@Entity()
+export class CertScan extends BaseEntity {
+  @PrimaryColumn()
+  id: string;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  issuer: string | null;
+
+  @Column({ nullable: true, type: 'timestamp' })
+  expirationTimestamp: Date | null;
+
+  @Column({ nullable: true, type: 'timestamp' })
+  certStartTimestamp: Date | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  pem: string | null;
+
+  @Column({ nullable: true })
+  sctExists: boolean;
+
+  @Column({ nullable: true, type: 'timestamp' })
+  sctOrNotBefore: Date | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  serial: string | null;
+
+  @ManyToMany((type) => Domain, (domain) => domain.certScans, {
+    onDelete: 'CASCADE',
+    onUpdate: 'CASCADE'
+  })
+  @JoinTable()
+  domains: Domain[];
+}

backend/src/models/mini_data_lake/cidrs.ts

@@ -0,0 +1,58 @@
+import {
+  Entity,
+  Index,
+  Column,
+  PrimaryGeneratedColumn,
+  CreateDateColumn,
+  BaseEntity,
+  ManyToMany,
+  JoinTable
+} from 'typeorm';
+
+import { Request } from './requests';
+import { Organization } from './organizations';
+@Entity()
+export class Cidr extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  id: string;
+
+  @CreateDateColumn()
+  createdDate: Date;
+
+  @Index()
+  @Column({
+    nullable: true,
+    type: 'cidr',
+    unique: true
+  })
+  network: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'inet'
+  })
+  startIp: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'inet'
+  })
+  endIp: string | null;
+
+  @Column({ nullable: true })
+  retired: boolean;
+
+  @ManyToMany((type) => Request, (request) => request.cidrs, {
+    onDelete: 'CASCADE',
+    onUpdate: 'CASCADE'
+  })
+  @JoinTable()
+  requests: Request[];
+
+  @ManyToMany((type) => Organization, (org) => org.cidrs, {
+    onDelete: 'CASCADE',
+    onUpdate: 'CASCADE'
+  })
+  @JoinTable()
+  organizations: Organization[];
+}

backend/src/models/mini_data_lake/contacts.ts

@@ -0,0 +1,53 @@
+// The data in this table is derived from the Vulnerability Scans Database,
+// the [requests Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#requests-collection).
+
+import {
+  Entity,
+  Column,
+  PrimaryGeneratedColumn,
+  BaseEntity,
+  ManyToMany,
+  Unique,
+  JoinTable
+} from 'typeorm';
+import { Organization } from './organizations';
+@Entity()
+@Unique(['name', 'email', 'type'])
+export class Contact extends BaseEntity {
+  @PrimaryGeneratedColumn('uuid')
+  id: string;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  name: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  email: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  phoneNumber: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  type: string | null;
+
+  @Column({ nullable: true })
+  retired: boolean;
+
+  @ManyToMany((type) => Organization, (org) => org.contacts, {
+    onDelete: 'CASCADE',
+    onUpdate: 'CASCADE'
+  })
+  @JoinTable()
+  organizations: Organization[];
+}

backend/src/models/mini_data_lake/cpes.ts

@@ -0,0 +1,31 @@
+import {
+  Entity,
+  PrimaryGeneratedColumn,
+  Column,
+  ManyToMany,
+  BaseEntity,
+  Unique
+} from 'typeorm';
+import { Cve } from './cves';
+
+@Entity()
+@Unique(['name', 'version', 'vendor'])
+export class Cpe extends BaseEntity {
+  @PrimaryGeneratedColumn('uuid')
+  id: string;
+
+  @Column()
+  name: string;
+
+  @Column()
+  version: string;
+
+  @Column()
+  vendor: string;
+
+  @Column()
+  lastSeenAt: Date;
+
+  @ManyToMany(() => Cve, (cve) => cve.cpes)
+  cves: Cve[];
+}