Remove log changes in backend/src/tasks

cisagov · May 2, 2024 · 3f7f25a · 3f7f25a
1 parent e3f1f1c
commit 3f7f25a
Show file tree

Hide file tree

Showing 31 changed files with 214 additions and 247 deletions.
diff --git a/backend/src/tasks/amass.ts b/backend/src/tasks/amass.ts
@@ -6,14 +6,13 @@ import { CommandOptions } from './ecs-client';
 import getRootDomains from './helpers/getRootDomains';
 import saveDomainsToDb from './helpers/saveDomainsToDb';
 import * as path from 'path';
-import logger from '../tools/lambda-logger';
 
 const OUT_PATH = path.join(__dirname, 'out-' + Math.random() + '.txt');
 
 export const handler = async (commandOptions: CommandOptions) => {
   const { organizationId, organizationName, scanId } = commandOptions;
 
-  logger.info(`Running amass on organization ${organizationName}`);
+  console.log('Running amass on organization', organizationName);
 
   const rootDomains = await getRootDomains(organizationId!);
 
@@ -28,7 +27,7 @@ export const handler = async (commandOptions: CommandOptions) => {
         '-json',
         OUT_PATH
       ];
-      logger.info(`Running amass with args ${JSON.stringify(args)}`);
+      console.log('Running amass with args', args);
       spawnSync('amass', args, { stdio: 'pipe' });
       const output = String(readFileSync(OUT_PATH));
       const lines = output.split('\n');
@@ -49,9 +48,9 @@ export const handler = async (commandOptions: CommandOptions) => {
         );
       }
       await saveDomainsToDb(domains);
-      logger.info(`amass created/updated ${domains.length} new domains`);
+      console.log(`amass created/updated ${domains.length} new domains`);
     } catch (e) {
-      logger.error(JSON.stringify(e));
+      console.error(e);
       continue;
     }
   }

diff --git a/backend/src/tasks/bastion.ts b/backend/src/tasks/bastion.ts
@@ -1,22 +1,21 @@
 import { Handler } from 'aws-lambda';
 import { connectToDatabase, User } from '../models';
 import ESClient from '../tasks/es-client';
-import logger from '../tools/lambda-logger';
 
-export const handler: Handler = async (event, context) => {
+export const handler: Handler = async (event) => {
   if (event.mode === 'db') {
-    const connection = await connectToDatabase();
+    const connection = await connectToDatabase(true);
     const res = await connection.query(event.query);
-    logger.info(JSON.stringify(res), { context });
+    console.log(res);
   } else if (event.mode === 'es') {
     if (event.query === 'delete') {
       const client = new ESClient();
       await client.deleteAll();
-      logger.info('Index successfully deleted', { context });
+      console.log('Index successfully deleted');
     } else {
-      logger.info(`Query not found: ${event.query}`, { context });
+      console.log('Query not found: ' + event.query);
     }
   } else {
-    logger.info(`Mode not found: ${event.mode}`, { context });
+    console.log('Mode not found: ' + event.mode);
   }
 };
diff --git a/backend/src/tasks/censys.ts b/backend/src/tasks/censys.ts
@@ -5,7 +5,6 @@ import * as dns from 'dns';
 import saveDomainsToDb from './helpers/saveDomainsToDb';
 import { CommandOptions } from './ecs-client';
 import getRootDomains from './helpers/getRootDomains';
-import logger from '../tools/lambda-logger';
 
 interface CensysAPIResponse {
   result: {
@@ -26,9 +25,9 @@ const sleep = (milliseconds: number) => {
 };
 
 const fetchCensysData = async (rootDomain: string) => {
-  logger.info(`Fetching certificates for ${rootDomain}`);
+  console.log(`Fetching certificates for ${rootDomain}`);
   const data = await fetchPage(rootDomain);
-  logger.info(
+  console.log(
     `Censys found ${data.result.total} certificates for ${rootDomain}
     Fetching ${Math.min(data.result.total, resultLimit)} of them...`
   );
@@ -67,7 +66,7 @@ const fetchPage = async (rootDomain: string, nextToken?: string) => {
 export const handler = async (commandOptions: CommandOptions) => {
   const { organizationId, organizationName, scanId } = commandOptions;
 
-  logger.info(`Running Censys on: ${organizationName}`);
+  console.log(`Running Censys on: ${organizationName}`);
 
   const rootDomains = await getRootDomains(organizationId!);
   const uniqueNames = new Set<string>(); //used to dedupe domain names
@@ -108,7 +107,7 @@ export const handler = async (commandOptions: CommandOptions) => {
   // Project Sonar has both forward & reverse DNS for finding subdomains
 
   // Save domains to database
-  logger.info(`Saving ${organizationName} subdomains to database...`);
+  console.log(`Saving ${organizationName} subdomains to database...`);
   const domains: Domain[] = [];
   for (const domain of foundDomains) {
     let ip: string | null;
@@ -130,7 +129,7 @@ export const handler = async (commandOptions: CommandOptions) => {
     );
   }
   await saveDomainsToDb(domains);
-  logger.info(
+  console.log(
     `Censys saved or updated ${domains.length} subdomains for ${organizationName}`
   );
 };
diff --git a/backend/src/tasks/censysCertificates.ts b/backend/src/tasks/censysCertificates.ts
@@ -1,24 +1,17 @@
-import {
-  connectToDatabase,
-  Domain,
-  Organization,
-  Scan,
-  Service
-} from '../models';
+import { connectToDatabase, Domain, Scan } from '../models';
 import { plainToClass } from 'class-transformer';
 import saveDomainsToDb from './helpers/saveDomainsToDb';
 import { CommandOptions } from './ecs-client';
 import { CensysCertificatesData } from '../models/generated/censysCertificates';
-import saveServicesToDb from './helpers/saveServicesToDb';
 import getAllDomains from './helpers/getAllDomains';
+import sanitizeChunkValues from './helpers/sanitizeChunkValues';
 import * as zlib from 'zlib';
 import * as readline from 'readline';
 import got from 'got';
 import PQueue from 'p-queue';
 import pRetry from 'p-retry';
 import axios from 'axios';
 import getScanOrganizations from './helpers/getScanOrganizations';
-import logger from '../tools/lambda-logger';
 
 interface CommonNameToDomainsMap {
   [commonName: string]: Domain[];
@@ -42,10 +35,12 @@ const downloadPath = async (
   path: string,
   commonNameToDomainsMap: CommonNameToDomainsMap,
   i: number,
-  numFiles: number,
-  commandOptions: CommandOptions
+  numFiles: number
 ): Promise<void> => {
-  logger.info(`i: ${i} of ${numFiles}: starting download of url ${path}`);
+  if (i >= 100) {
+    throw new Error('Invalid chunk number.');
+  }
+  console.log(`i: ${i} of ${numFiles}: starting download of url ${path}`);
 
   const domains: Domain[] = [];
   const gunzip = zlib.createGunzip();
@@ -105,11 +100,11 @@ const downloadPath = async (
     readInterface.on('SIGTSTP', reject);
   });
   if (!domains.length) {
-    logger.info(
+    console.log(
       `censysCertificates - processed file ${i} of ${numFiles}: got no results`
     );
   } else {
-    logger.info(
+    console.log(
       `censysCertificates - processed file ${i} of ${numFiles}: got some results: ${domains.length} domains`
     );
   }
@@ -118,24 +113,22 @@ const downloadPath = async (
 };
 
 export const handler = async (commandOptions: CommandOptions) => {
-  const { chunkNumber, numChunks, organizationId } = commandOptions;
+  const { organizationId } = commandOptions;
 
-  if (chunkNumber === undefined || numChunks === undefined) {
-    throw new Error('Chunks not specified.');
-  }
+  const { chunkNumber, numChunks } = await sanitizeChunkValues(commandOptions);
 
   const {
     data: { results }
   } = await pRetry(() => axios.get(CENSYS_CERTIFICATES_ENDPOINT, { auth }), {
-    // Perform less retries on jest to make tests faster
+    // Perform fewer retries on jest to make tests faster
     retries: typeof jest === 'undefined' ? 5 : 2,
     randomize: true
   });
 
   const {
     data: { files }
   } = await pRetry(() => axios.get(results.latest.details_url, { auth }), {
-    // Perform less retries on jest to make tests faster
+    // Perform fewer retries on jest to make tests faster
     retries: typeof jest === 'undefined' ? 5 : 2,
     randomize: true
   });
@@ -162,9 +155,9 @@ export const handler = async (commandOptions: CommandOptions) => {
   const fileNames = Object.keys(files).sort();
   const jobs: Promise<void>[] = [];
 
-  let startIndex = Math.floor(((1.0 * chunkNumber) / numChunks) * numFiles);
+  let startIndex = Math.floor(((1.0 * chunkNumber!) / numChunks!) * numFiles);
   let endIndex =
-    Math.floor(((1.0 * (chunkNumber + 1)) / numChunks) * numFiles) - 1;
+    Math.floor(((1.0 * (chunkNumber! + 1)) / numChunks!) * numFiles) - 1;
 
   if (process.env.IS_LOCAL && typeof jest === 'undefined') {
     // For local testing.
@@ -200,20 +193,19 @@ export const handler = async (commandOptions: CommandOptions) => {
               files[fileName].download_path,
               commonNameToDomainsMap,
               idx,
-              numFiles,
-              commandOptions
+              numFiles
             ),
           {
-            // Perform less retries on jest to make tests faster
+            // Perform fewer retries on jest to make tests faster
             retries: typeof jest === 'undefined' ? 5 : 2,
             randomize: true
           }
         )
       )
     );
   }
-  logger.info(`censysCertificates: scheduled all tasks`);
+  console.log(`censysCertificates: scheduled all tasks`);
   await Promise.all(jobs);
 
-  logger.info(`censysCertificates done`);
+  console.log(`censysCertificates done`);
 };
diff --git a/backend/src/tasks/censysIpv4.ts b/backend/src/tasks/censysIpv4.ts
@@ -13,7 +13,7 @@ import PQueue from 'p-queue';
 import pRetry from 'p-retry';
 import axios from 'axios';
 import getScanOrganizations from './helpers/getScanOrganizations';
-import logger from '../tools/lambda-logger';
+import sanitizeChunkValues from './helpers/sanitizeChunkValues';
 
 export interface IpToDomainsMap {
   [ip: string]: Domain[];
@@ -39,7 +39,10 @@ const downloadPath = async (
   numFiles: number,
   commandOptions: CommandOptions
 ): Promise<void> => {
-  logger.info(`i: ${i} of ${numFiles}: starting download of url ${path}`);
+  if (i >= 100) {
+    throw new Error('Invalid chunk number.');
+  }
+  console.log(`i: ${i} of ${numFiles}: starting download of url ${path}`);
 
   const domains: Domain[] = [];
   const services: Service[] = [];
@@ -106,11 +109,11 @@ const downloadPath = async (
     readInterface.on('SIGTSTP', reject);
   });
   if (!domains.length) {
-    logger.info(
+    console.log(
       `censysipv4 - processed file ${i} of ${numFiles}: got no results`
     );
   } else {
-    logger.info(
+    console.log(
       `censysipv4 - processed file ${i} of ${numFiles}: got some results: ${domains.length} domains and ${services.length} services`
     );
   }
@@ -120,11 +123,9 @@ const downloadPath = async (
 };
 
 export const handler = async (commandOptions: CommandOptions) => {
-  const { chunkNumber, numChunks, organizationId } = commandOptions;
+  const { organizationId } = commandOptions;
 
-  if (chunkNumber === undefined || numChunks === undefined) {
-    throw new Error('Chunks not specified.');
-  }
+  const { chunkNumber, numChunks } = await sanitizeChunkValues(commandOptions);
 
   const {
     data: { results }
@@ -156,9 +157,9 @@ export const handler = async (commandOptions: CommandOptions) => {
   const fileNames = Object.keys(files).sort();
   const jobs: Promise<void>[] = [];
 
-  let startIndex = Math.floor(((1.0 * chunkNumber) / numChunks) * numFiles);
+  let startIndex = Math.floor(((1.0 * chunkNumber!) / numChunks!) * numFiles);
   let endIndex =
-    Math.floor(((1.0 * (chunkNumber + 1)) / numChunks) * numFiles) - 1;
+    Math.floor(((1.0 * (chunkNumber! + 1)) / numChunks!) * numFiles) - 1;
 
   if (process.env.IS_LOCAL && typeof jest === 'undefined') {
     // For local testing.
@@ -192,16 +193,16 @@ export const handler = async (commandOptions: CommandOptions) => {
               commandOptions
             ),
           {
-            // Perform less retries on jest to make tests faster
+            // Perform fewer retries on jest to make tests faster
             retries: typeof jest === 'undefined' ? 5 : 2,
             randomize: true
           }
         )
       )
     );
   }
-  logger.info(`censysipv4: scheduled all tasks`);
+  console.log(`censysipv4: scheduled all tasks`);
   await Promise.all(jobs);
 
-  logger.info(`censysipv4 done`);
+  console.log(`censysipv4 done`);
 };