Merge branch 'develop' into ui-header-issue16

.github/workflows/backend.yml

@@ -189,6 +189,15 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
+      - name: Run syncmdl
+        run: |
+          aws lambda invoke --function-name crossfeed-staging-syncmdl \
+          --region us-east-1 /dev/stdout
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
   deploy_prod:
     needs: [build_worker, lint, test, test_python]
     runs-on: ubuntu-latest
@@ -238,3 +247,12 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Run syncmdl
+        run: |
+          aws lambda invoke --function-name crossfeed-prod-syncmdl --region us-east-1 \
+          /dev/stdout
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/build.yml

@@ -101,6 +101,8 @@ jobs:
           pip install --upgrade --requirement requirements-test.txt
       - name: Set up pre-commit hook environments
         run: pre-commit install-hooks
+      - name: Create .env file needed by docker-compose-check pre-commit hook
+        run: cp dev.env.example .env
       - name: Run pre-commit on all files
         run: pre-commit run --all-files
       - name: Setup tmate debug session

backend/env.yml

@@ -9,6 +9,9 @@ staging:
   DB_NAME: ${ssm:/crossfeed/staging/DATABASE_NAME}
   DB_USERNAME: ${ssm:/crossfeed/staging/DATABASE_USER}
   DB_PASSWORD: ${ssm:/crossfeed/staging/DATABASE_PASSWORD}
+  MDL_USERNAME: ${ssm:/crossfeed/staging/MDL_USERNAME}
+  MDL_PASSWORD: ${ssm:/crossfeed/staging/MDL_PASSWORD}
+  MDL_NAME: ${ssm:/crossfeed/staging/MDL_NAME}
   PE_DB_NAME: ${ssm:/crossfeed/staging/PE_DB_NAME}
   PE_DB_USERNAME: ${ssm:/crossfeed/staging/PE_DB_USERNAME}
   PE_DB_PASSWORD: ${ssm:/crossfeed/staging/PE_DB_PASSWORD}
@@ -67,6 +70,9 @@ prod:
   DB_NAME: ${ssm:/crossfeed/prod/DATABASE_NAME}
   DB_USERNAME: ${ssm:/crossfeed/prod/DATABASE_USER}
   DB_PASSWORD: ${ssm:/crossfeed/prod/DATABASE_PASSWORD}
+  MDL_USERNAME: ${ssm:/crossfeed/prod/MDL_USERNAME}
+  MDL_PASSWORD: ${ssm:/crossfeed/prod/MDL_PASSWORD}
+  MDL_NAME: ${ssm:/crossfeed/prod/MDL_NAME}
   JWT_SECRET: ${ssm:/crossfeed/prod/APP_JWT_SECRET}
   LOGIN_GOV_REDIRECT_URI: ${ssm:/crossfeed/prod/LOGIN_GOV_REDIRECT_URI}
   LOGIN_GOV_BASE_URL: ${ssm:/crossfeed/prod/LOGIN_GOV_BASE_URL}

backend/package.json

@@ -112,6 +112,7 @@
     "lint:fix": "eslint '**/*.{ts,tsx,js,jsx}' --fix",
     "pesyncdb": "docker-compose exec -T backend npx ts-node src/tools/run-pesyncdb.ts",
     "syncdb": "docker-compose exec -T backend npx ts-node src/tools/run-syncdb.ts",
+    "syncmdl": "docker-compose exec -T backend npx ts-node src/tools/run-syncmdl.ts",
     "test": "jest --detectOpenHandles",
     "test-python": "pytest"
   },

backend/src/api/helpers.ts

@@ -12,6 +12,67 @@ import { SES } from 'aws-sdk';
 import * as nodemailer from 'nodemailer';
 import * as handlebars from 'handlebars';
 
+export const REGION_STATE_MAP = {
+  Alabama: '4',
+  Alaska: '10',
+  'American Samoa': '9',
+  Arkansas: '6',
+  Arizona: '9',
+  California: '9',
+  Colorado: '8',
+  'Commonwealth Northern Mariana Islands': '9',
+  Connecticut: '1',
+  Delaware: '3',
+  'District of Columbia': '3',
+  'Federal States of Micronesia': '9',
+  Florida: '4',
+  Georgia: '4',
+  Guam: '9',
+  Hawaii: '9',
+  Idaho: '10',
+  Illinois: '5',
+  Indiana: '5',
+  Iowa: '7',
+  Kansas: '7',
+  Kentucky: '4',
+  Louisiana: '6',
+  Maine: '1',
+  Maryland: '3',
+  Massachusetts: '1',
+  Michigan: '5',
+  Minnesota: '5',
+  Mississippi: '4',
+  Missouri: '7',
+  Montana: '8',
+  Nebraska: '7',
+  Nevada: '9',
+  'New Hampshire': '1',
+  'New Jersey': '2',
+  'New Mexico': '6',
+  'New York': '2',
+  'North Carolina': '4',
+  'North Dakota': '8',
+  Ohio: '5',
+  Oklahoma: '6',
+  Oregon: '10',
+  Pennsylvania: '3',
+  'Puerto Rico': '2',
+  'Republic of Marshall Islands': '9',
+  'Rhode Island': '1',
+  'South Carolina': '4',
+  'South Dakota': '8',
+  Tennessee: '4',
+  Texas: '6',
+  Utah: '8',
+  Vermont: '1',
+  'Virgin Islands': '2',
+  Virginia: '3',
+  Washington: '10',
+  'West Virginia': '3',
+  Wisconsin: '5',
+  Wyoming: '8'
+};
+
 export const validateBody = async <T>(
   obj: ClassType<T>,
   body: string | null,

backend/src/api/organizations.ts

@@ -19,7 +19,13 @@ import {
   OrganizationTag,
   PendingDomain
 } from '../models';
-import { validateBody, wrapHandler, NotFound, Unauthorized } from './helpers';
+import {
+  validateBody,
+  wrapHandler,
+  NotFound,
+  REGION_STATE_MAP,
+  Unauthorized
+} from './helpers';
 import {
   isOrgAdmin,
   isGlobalWriteAdmin,
@@ -986,67 +992,6 @@ export const addUserV2 = wrapHandler(async (event) => {
   return NotFound;
 });
 
-export const REGION_STATE_MAP = {
-  Alabama: '4',
-  Alaska: '10',
-  'American Samoa': '9',
-  Arizona: '9',
-  Arkansas: '6',
-  California: '9',
-  Colorado: '8',
-  'Commonwealth Northern Mariana Islands': '9',
-  Connecticut: '1',
-  Delaware: '3',
-  'District of Columbia': '3',
-  'Federal States of Micronesia': '9',
-  Florida: '4',
-  Georgia: '4',
-  Guam: '9',
-  Hawaii: '9',
-  Idaho: '10',
-  Illinois: '5',
-  Indiana: '5',
-  Iowa: '7',
-  Kansas: '7',
-  Kentucky: '4',
-  Louisiana: '6',
-  Maine: '1',
-  Maryland: '3',
-  Massachusetts: '1',
-  Michigan: '5',
-  Minnesota: '5',
-  Mississippi: '4',
-  Missouri: '7',
-  Montana: '8',
-  Nebraska: '7',
-  Nevada: '9',
-  'New Hampshire': '1',
-  'New Jersey': '2',
-  'New Mexico': '6',
-  'New York': '2',
-  'North Carolina': '4',
-  'North Dakota': '8',
-  Ohio: '5',
-  Oklahoma: '6',
-  Oregon: '10',
-  Pennsylvania: '3',
-  'Puerto Rico': '2',
-  'Republic of Marshall Islands': '9',
-  'Rhode Island': '1',
-  'South Carolina': '4',
-  'South Dakota': '8',
-  Tennessee: '4',
-  Texas: '6',
-  Utah: '8',
-  Vermont: '1',
-  Virginia: '3',
-  'Virgin Islands': '2',
-  Washington: '10',
-  'West Virginia': '3',
-  Wisconsin: '5',
-  Wyoming: '8'
-};
-
 /**
  * @swagger
  *

backend/src/api/users.ts

@@ -19,6 +19,7 @@ import {
   wrapHandler,
   NotFound,
   Unauthorized,
+  REGION_STATE_MAP,
   sendEmail,
   sendUserRegistrationEmail,
   sendRegistrationApprovedEmail,
@@ -146,67 +147,6 @@ class UpdateUser {
   role: string;
 }
 
-const REGION_STATE_MAP = {
-  Connecticut: '1',
-  Maine: '1',
-  Massachusetts: '1',
-  'New Hampshire': '1',
-  'Rhode Island': '1',
-  Vermont: '1',
-  'New Jersey': '2',
-  'New York': '2',
-  'Puerto Rico': '2',
-  'Virgin Islands': '2',
-  Delaware: '3',
-  Maryland: '3',
-  Pennsylvania: '3',
-  Virginia: '3',
-  'District of Columbia': '3',
-  'West Virginia': '3',
-  Alabama: '4',
-  Florida: '4',
-  Georgia: '4',
-  Kentucky: '4',
-  Mississippi: '4',
-  'North Carolina': '4',
-  'South Carolina': '4',
-  Tennessee: '4',
-  Illinois: '5',
-  Indiana: '5',
-  Michigan: '5',
-  Minnesota: '5',
-  Ohio: '5',
-  Wisconsin: '5',
-  Arkansas: '6',
-  Louisiana: '6',
-  'New Mexico': '6',
-  Oklahoma: '6',
-  Texas: '6',
-  Iowa: '7',
-  Kansas: '7',
-  Missouri: '7',
-  Nebraska: '7',
-  Colorado: '8',
-  Montana: '8',
-  'North Dakota': '8',
-  'South Dakota': '8',
-  Utah: '8',
-  Wyoming: '8',
-  Arizona: '9',
-  California: '9',
-  Hawaii: '9',
-  Nevada: '9',
-  Guam: '9',
-  'American Samoa': '9',
-  'Commonwealth Northern Mariana Islands': '9',
-  'Republic of Marshall Islands': '9',
-  'Federal States of Micronesia': '9',
-  Alaska: '10',
-  Idaho: '10',
-  Oregon: '10',
-  Washington: '10'
-};
-
 /**
  * @swagger
  *

backend/src/models/connection.ts

@@ -1,5 +1,6 @@
 import { createConnection, Connection } from 'typeorm';
 import {
+  // Models for the Crossfeed database
   Domain,
   Service,
   Vulnerability,
@@ -13,11 +14,93 @@ import {
   SavedSearch,
   OrganizationTag,
   Cpe,
-  Cve
+  Cve,
+
+  //  Models for the Mini Data Lake database
+  CertScan,
+  Cidr,
+  Contact,
+  DL_Cpe,
+  DL_Cve,
+  DL_Domain,
+  DL_Organization,
+  HostScan,
+  Host,
+  Ip,
+  Kev,
+  Location,
+  PortScan,
+  PrecertScan,
+  Report,
+  Request,
+  Sector,
+  Snapshot,
+  SslyzeScan,
+  Tag,
+  Tally,
+  TicketEvent,
+  Ticket,
+  TrustymailScan,
+  VulnScan
 } from '.';
 
 let connection: Connection | null = null;
 
+let dl_connection: Connection | null = null;
+
+const connectDl = async (logging?: boolean) => {
+  const dl_connection = createConnection({
+    type: 'postgres',
+    host: process.env.DB_HOST,
+    port: parseInt(process.env.DB_PORT ?? ''),
+    username: process.env.MDL_USERNAME,
+    password: process.env.MDL_PASSWORD,
+    database: process.env.MDL_NAME,
+    entities: [
+      CertScan,
+      Cidr,
+      Contact,
+      DL_Cpe,
+      DL_Cve,
+      DL_Domain,
+      HostScan,
+      Host,
+      Ip,
+      Kev,
+      Location,
+      DL_Organization,
+      PortScan,
+      PrecertScan,
+      Report,
+      Request,
+      Sector,
+      Snapshot,
+      SslyzeScan,
+      Tag,
+      Tally,
+      TicketEvent,
+      Ticket,
+      TrustymailScan,
+      VulnScan
+    ],
+    synchronize: false,
+    name: 'mini_data_lake',
+    dropSchema: false,
+    logging: logging ?? false,
+    cache: true
+  });
+  return dl_connection;
+};
+
+export const connectToDatalake = async (logging?: boolean) => {
+  if (!dl_connection?.isConnected) {
+    dl_connection = await connectDl(logging);
+  } else {
+    console.log("didn't connect");
+  }
+  return dl_connection;
+};
+
 const connectDb = async (logging?: boolean) => {
   const connection = createConnection({
     type: 'postgres',