LZ to DMZ Sync

.gitignore

@@ -25,6 +25,7 @@ frontend/.pnp.js
 backend/coverage
 docs/coverage
 frontend/coverage
+backend/dmzLzSyncTestData.ts
 
 # production
 frontend/build

backend/.eslintrc.yml

@@ -3,7 +3,7 @@ env:
   es6: true
   node: true
 parser: '@typescript-eslint/parser'
-ignorePatterns: [dist/**]
+ignorePatterns: [dist/**, dmzLzSyncTestDAta.ts]
 extends:
   - plugin:prettier/recommended
   - plugin:@typescript-eslint/eslint-recommended

backend/db-init/initmdl.sql

@@ -0,0 +1,13 @@
+-- Create the user (role)
+CREATE USER mdl_local WITH PASSWORD 'mini_data_lake';
+
+SELECT CURRENT_USER;
+
+-- Optionally grant privileges on a specific database (if required)
+CREATE DATABASE mini_data_lake_local;
+GRANT ALL PRIVILEGES ON DATABASE mini_data_lake_local TO mdl_local;
+ALTER SCHEMA public OWNER TO mdl_local;
+ALTER DATABASE mini_data_lake_local OWNER TO mdl_local;
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA public;
+
+SET ROLE mdl_local;

backend/package.json

@@ -111,10 +111,12 @@
     "deploy-worker-staging": "./tools/deploy-worker.sh",
     "lint": "eslint '**/*.{ts,tsx,js,jsx}'",
     "lint:fix": "eslint '**/*.{ts,tsx,js,jsx}' --fix",
+    "mdl-datagen": "ts-node src/tools/generators.ts",
     "pesyncdb": "docker compose exec -T backend npx ts-node src/tools/run-pesyncdb.ts",
     "scan-exec": "docker compose exec -T backend npx ts-node src/tools/run-scanExecution.ts",
     "syncdb": "docker compose exec -T backend npx ts-node src/tools/run-syncdb.ts",
     "syncmdl": "docker compose exec -T backend npx ts-node src/tools/run-syncmdl.ts",
+    "syncnewmdl": "docker compose exec -T backend npx ts-node src/tools/run-syncnewmdl.ts",
     "test": "jest --detectOpenHandles",
     "test-python": "pytest"
   },

backend/src/api/app.ts

@@ -10,6 +10,7 @@ import * as cves from './cves';
 import * as domains from './domains';
 import * as notifications from './notifications';
 import * as search from './search';
+import * as sync from './sync';
 import * as vulnerabilities from './vulnerabilities';
 import * as organizations from './organizations';
 import * as scans from './scans';
@@ -30,6 +31,7 @@ import { Request, Response, NextFunction } from 'express';
 import fetch from 'node-fetch';
 import * as searchOrganizations from './organizationSearch';
 import { Logger, RecordMessage } from '../tools/logger';
+import { parse } from 'papaparse';
 
 const sanitizer = require('sanitizer');
 
@@ -52,7 +54,10 @@ const handlerToExpress =
         pathParameters: req.params,
         query: req.query,
         requestContext: req.requestContext,
-        body: JSON.stringify(req.body || '{}'),
+        body:
+          typeof req.body !== 'string'
+            ? JSON.stringify(req.body || '{}')
+            : req.body,
         headers: req.headers,
         path: req.originalUrl
       },
@@ -89,6 +94,39 @@ app.use(
   })
 ); // limit 1000 requests per 15 minutes
 
+app.use((req, res, next) => {
+  // Middleware to parse CSV data
+  if (req.headers['content-type'] === 'text/csv') {
+    let data = '';
+
+    req.on('data', (chunk) => {
+      data += chunk;
+    });
+    req.on('end', () => {
+      const parsedData = parse(data, {
+        header: true,
+        skipEmptyLines: true
+      });
+
+      if (parsedData.errors.length > 0) {
+        return res
+          .status(400)
+          .json({ message: 'CSV Parsing Error', errors: parsedData.errors });
+      }
+      // We don't need to parse the data, just validate it
+      // CSV Parsing will happen in the handler
+      req.body = data;
+      next();
+    });
+
+    req.on('error', () => {
+      res.status(500).json({ message: 'Error reading CSV data' });
+    });
+  } else {
+    next();
+  }
+});
+
 app.use(express.json({ strict: false }));
 
 // These CORS origins work in all Crossfeed environments
@@ -762,6 +800,15 @@ authenticatedRoute.post(
   handlerToExpress(reports.list_reports)
 );
 
+if (process.env.IS_LOCAL) {
+  console.log('Unauthenticated local route');
+  app.post('/sync', handlerToExpress(sync.ingest));
+} else {
+  authenticatedRoute.post('/sync', handlerToExpress(sync.ingest));
+}
+
+// authenticatedRoute.post('/sync', handlerToExpress(sync.ingest));
+
 //Authenticated Registration Routes
 authenticatedRoute.put(
   '/users/:userId/register/approve',

backend/src/api/domains.ts

@@ -233,7 +233,7 @@ export const export_ = wrapHandler(async (event) => {
     res.products = Object.values(products).join(', ');
     return res;
   });
-  const url = await client.saveCSV(
+  const res = await client.saveCSV(
     Papa.unparse({
       fields: [
         'name',
@@ -247,13 +247,14 @@ export const export_ = wrapHandler(async (event) => {
       ],
       data: result
     }),
-    'domains'
+    'domains',
+    process.env.EXPORT_BUCKET_NAME
   );
 
   return {
     statusCode: 200,
     body: JSON.stringify({
-      url
+      url: res.url
     })
   };
 });

backend/src/api/search.ts

@@ -152,7 +152,7 @@ export const export_ = wrapHandler(async (event) => {
     return res;
   });
   const client = new S3Client();
-  const url = await client.saveCSV(
+  const { url } = await client.saveCSV(
     Papa.unparse({
       fields: [
         'name',