From e3f1f1c4d1ed5670b238a7476182de7114b229e6 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 2 May 2024 11:46:53 -0400 Subject: [PATCH] Fix backend/src/api files --- backend/env.yml | 16 ++++++- backend/src/api/app.ts | 6 +-- backend/src/api/auth.ts | 2 +- backend/src/api/helpers.ts | 68 +++--------------------------- backend/src/api/saved-searches.ts | 2 - backend/src/api/scan-tasks.ts | 29 +++++++++++-- backend/src/api/scans.ts | 30 ------------- backend/src/api/users.ts | 5 --- backend/src/api/vulnerabilities.ts | 9 ++-- dev.env.example | 4 ++ 10 files changed, 58 insertions(+), 113 deletions(-) diff --git a/backend/env.yml b/backend/env.yml index 47d3d8bc..0e048755 100644 --- a/backend/env.yml +++ b/backend/env.yml @@ -4,12 +4,15 @@ dev: staging: REGION: us-east-1 + ENDPOINT_TYPE: REGIONAL RESOURCE_POLICY: - Effect: Allow Principal: '*' Action: execute-api:Invoke Resource: execute-api:/${self:provider.stage}/*/* - ENDPOINT_TYPE: REGIONAL + COGNITO_URL: https://cognito-idp.us-east-1.amazonaws.com + BACKEND_DOMAIN: https://api.staging-cd.crossfeed.cyber.dhs.gov + EMAIL_REGION: us-east-1 DB_DIALECT: postgres DB_PORT: 5432 DB_HOST: ${ssm:/crossfeed/staging/DATABASE_HOST} @@ -72,6 +75,9 @@ prod: Principal: '*' Action: execute-api:Invoke Resource: execute-api:/${self:provider.stage}/*/* + COGNITO_URL: https://cognito-idp.us-east-1.amazonaws.com + BACKEND_DOMAIN: https://api.crossfeed.cyber.dhs.gov + EMAIL_REGION: us-east-1 DB_DIALECT: postgres DB_PORT: 5432 DB_HOST: ${ssm:/crossfeed/prod/DATABASE_HOST} @@ -120,6 +126,7 @@ prod: staging-lz: REGION: us-gov-east-1 + ENDPOINT_TYPE: PRIVATE RESOURCE_POLICY: - Effect: Deny Principal: '*' @@ -132,7 +139,9 @@ staging-lz: Principal: '*' Action: execute-api:Invoke Resource: execute-api:/${self:provider.stage}/*/* - ENDPOINT_TYPE: PRIVATE + COGNITO_URL: https://cognito-idp.us-gov-west-1.amazonaws.com + BACKEND_DOMAIN: https://api.staging.crossfeed.cyber.dhs.gov + EMAIL_REGION: us-gov-west-1 DB_DIALECT: postgres DB_PORT: 5432 DB_HOST: ${ssm:/crossfeed/staging/DATABASE_HOST} @@ -201,6 +210,9 @@ prod-lz: Principal: '*' Action: execute-api:Invoke Resource: execute-api:/${self:provider.stage}/*/* + COGNITO_URL: https://cognito-idp.us-gov-west-1.amazonaws.com + BACKEND_DOMAIN: https://api.crossfeed.cyber.dhs.gov + EMAIL_REGION: us-gov-west-1 DB_DIALECT: postgres DB_PORT: 5432 DB_HOST: ${ssm:/crossfeed/prod/DATABASE_HOST} diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index f9e227bd..54bd3b7e 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -110,13 +110,13 @@ app.use( directives: { defaultSrc: [ "'self'", - 'https://cognito-idp.us-gov-west-1.amazonaws.com', - 'https://api.crossfeed.cyber.dhs.gov' + `'${process.env.COGNITO_URL}'`, + `'${process.env.BACKEND_DOMAIN}'` ], objectSrc: ["'none'"], scriptSrc: [ "'self'", - 'https://api.crossfeed.cyber.dhs.gov' + `'${process.env.BACKEND_DOMAIN}'` // Add any other allowed script sources here ], frameAncestors: ["'none'"] diff --git a/backend/src/api/auth.ts b/backend/src/api/auth.ts index eedcffc1..bc0a032f 100644 --- a/backend/src/api/auth.ts +++ b/backend/src/api/auth.ts @@ -46,7 +46,7 @@ interface UserInfo { } const client = jwksClient({ - jwksUri: `https://cognito-idp.us-gov-west-1.amazonaws.com/${process.env.REACT_APP_USER_POOL_ID}/.well-known/jwks.json`, + jwksUri: `${process.env.COGNITO_URL}/${process.env.REACT_APP_USER_POOL_ID}/.well-known/jwks.json`, getKeysInterceptor: () => { const jwksJson = JSON.parse(process.env.REACT_APP_USER_POOL_KEY!); return jwksJson.keys; diff --git a/backend/src/api/helpers.ts b/backend/src/api/helpers.ts index e66886cc..56e09724 100644 --- a/backend/src/api/helpers.ts +++ b/backend/src/api/helpers.ts @@ -13,8 +13,6 @@ import * as nodemailer from 'nodemailer'; import logger from '../tools/lambda-logger'; import * as handlebars from 'handlebars'; -const AWS = require('aws-sdk'); -const httpProxy = require('https-proxy-agent'); export const REGION_STATE_MAP = { Alabama: '4', Alaska: '10', @@ -151,18 +149,9 @@ export const sendEmail = async ( body: string ) => { try { - process.env.HTTPS_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - process.env.HTTP_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - const proxyAgent = process.env.HTTPS_PROXY || process.env.HTTP_PROXY; - AWS.config.update({ - httpOptions: { - agent: proxyAgent ? httpProxy(proxyAgent) : undefined - } - }); const transporter = nodemailer.createTransport({ SES: new SES({ - region: 'us-gov-west-1', - endpoint: 'https://email.us-gov-west-1.amazonaws.com' + region: process.env.EMAIL_REGION! }) }); @@ -185,18 +174,9 @@ export const sendEmail = async ( }; export const sendRegistrationTextEmail = async (recipient: string) => { - process.env.HTTPS_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - process.env.HTTP_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - const proxyAgent = process.env.HTTPS_PROXY || process.env.HTTP_PROXY; - AWS.config.update({ - httpOptions: { - agent: proxyAgent ? httpProxy(proxyAgent) : undefined - } - }); const transporter = nodemailer.createTransport({ SES: new SES({ - region: 'us-gov-west-1', - endpoint: 'https://email.us-gov-west-1.amazonaws.com' + region: process.env.EMAIL_REGION! }) }); @@ -217,18 +197,9 @@ export const sendRegistrationTextEmail = async (recipient: string) => { }; export const sendRegistrationHtmlEmail = async (recipient: string) => { - process.env.HTTPS_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - process.env.HTTP_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - const proxyAgent = process.env.HTTPS_PROXY || process.env.HTTP_PROXY; - AWS.config.update({ - httpOptions: { - agent: proxyAgent ? httpProxy(proxyAgent) : undefined - } - }); const transporter = nodemailer.createTransport({ SES: new SES({ - region: 'us-gov-west-1', - endpoint: 'https://email.us-gov-west-1.amazonaws.com' + region: process.env.EMAIL_REGION! }) }); @@ -273,18 +244,9 @@ export const sendUserRegistrationEmail = async ( replyTo: process.env.CROSSFEED_SUPPORT_EMAIL_REPLYTO! }; - process.env.HTTPS_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - process.env.HTTP_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - const proxyAgent = process.env.HTTPS_PROXY || process.env.HTTP_PROXY; - AWS.config.update({ - httpOptions: { - agent: proxyAgent ? httpProxy(proxyAgent) : undefined - } - }); const transporter = nodemailer.createTransport({ SES: new SES({ - region: 'us-gov-west-1', - endpoint: 'https://email.us-gov-west-1.amazonaws.com' + region: process.env.EMAIL_REGION! }) }); await transporter.sendMail(mailOptions); @@ -318,18 +280,9 @@ export const sendRegistrationDeniedEmail = async ( replyTo: process.env.CROSSFEED_SUPPORT_EMAIL_REPLYTO! }; - process.env.HTTPS_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - process.env.HTTP_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - const proxyAgent = process.env.HTTPS_PROXY || process.env.HTTP_PROXY; - AWS.config.update({ - httpOptions: { - agent: proxyAgent ? httpProxy(proxyAgent) : undefined - } - }); const transporter = nodemailer.createTransport({ SES: new SES({ - region: 'us-gov-west-1', - endpoint: 'https://email.us-gov-west-1.amazonaws.com' + region: process.env.EMAIL_REGION! }) }); await transporter.sendMail(mailOptions); @@ -363,18 +316,9 @@ export const sendRegistrationApprovedEmail = async ( replyTo: process.env.CROSSFEED_SUPPORT_EMAIL_REPLYTO! }; - process.env.HTTPS_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - process.env.HTTP_PROXY = 'http://proxy.lz.us-cert.gov:8080'; - const proxyAgent = process.env.HTTPS_PROXY || process.env.HTTP_PROXY; - AWS.config.update({ - httpOptions: { - agent: proxyAgent ? httpProxy(proxyAgent) : undefined - } - }); const transporter = nodemailer.createTransport({ SES: new SES({ - region: 'us-gov-west-1', - endpoint: 'https://email.us-gov-west-1.amazonaws.com' + region: process.env.EMAIL_REGION! }) }); await transporter.sendMail(mailOptions); diff --git a/backend/src/api/saved-searches.ts b/backend/src/api/saved-searches.ts index 4cce3e53..195caf65 100644 --- a/backend/src/api/saved-searches.ts +++ b/backend/src/api/saved-searches.ts @@ -10,7 +10,6 @@ import { import { connectToDatabase, SavedSearch, Vulnerability } from '../models'; import { validateBody, wrapHandler, NotFound, Unauthorized } from './helpers'; import { FindManyOptions } from 'typeorm'; -import logger from '../tools/lambda-logger'; export const del = wrapHandler(async (event) => { const id = event.pathParameters?.searchId; @@ -125,7 +124,6 @@ export const list = wrapHandler(async (event) => { : PAGE_SIZE; const page = event.query?.page ? parseInt(event.query?.page) : 1; - logger.info(event.query); const result = await SavedSearch.findAndCount({ where, take: pageSize, diff --git a/backend/src/api/scan-tasks.ts b/backend/src/api/scan-tasks.ts index b8ba18bd..1fc97848 100644 --- a/backend/src/api/scan-tasks.ts +++ b/backend/src/api/scan-tasks.ts @@ -5,6 +5,7 @@ import { IsIn, ValidateNested, isUUID, + IsUUID, IsOptional, IsObject } from 'class-validator'; @@ -12,7 +13,11 @@ import { Type } from 'class-transformer'; import { ScanTask, connectToDatabase } from '../models'; import { validateBody, wrapHandler, NotFound, Unauthorized } from './helpers'; import { SelectQueryBuilder } from 'typeorm'; -import { isGlobalViewAdmin, isGlobalWriteAdmin } from './auth'; +import { + getTagOrganizations, + isGlobalViewAdmin, + isGlobalWriteAdmin +} from './auth'; import ECSClient from '../tasks/ecs-client'; const PAGE_SIZE = parseInt(process.env.PAGE_SIZE ?? '') || 25; @@ -25,6 +30,14 @@ class ScanTaskFilters { @IsString() @IsOptional() status?: string; + + @IsUUID() + @IsOptional() + organization?: string; + + @IsUUID() + @IsOptional() + tag?: string; } class ScanTaskSearch { @@ -46,7 +59,7 @@ class ScanTaskSearch { @IsOptional() filters?: ScanTaskFilters; - filterResultQueryset(qs: SelectQueryBuilder) { + async filterResultQueryset(qs: SelectQueryBuilder, event) { if (this.filters?.name) { qs.andWhere('scan.name ILIKE :name', { name: `${this.filters?.name}` @@ -57,6 +70,16 @@ class ScanTaskSearch { status: `${this.filters?.status}` }); } + if (this.filters?.organization) { + qs.andWhere('organization.id = :org', { + org: this.filters.organization + }); + } + if (this.filters?.tag) { + qs.andWhere('organization.id IN (:...orgs)', { + orgs: await getTagOrganizations(event, this.filters.tag) + }); + } return qs; } @@ -67,7 +90,7 @@ class ScanTaskSearch { .skip(PAGE_SIZE * (this.page - 1)) .take(PAGE_SIZE); - this.filterResultQueryset(qs); + await this.filterResultQueryset(qs, event); return qs.getManyAndCount(); } } diff --git a/backend/src/api/scans.ts b/backend/src/api/scans.ts index 750ea8d6..91875c33 100644 --- a/backend/src/api/scans.ts +++ b/backend/src/api/scans.ts @@ -48,19 +48,6 @@ interface ScanSchema { } export const SCAN_SCHEMA: ScanSchema = { - testProxy: { - type: 'fargate', - isPassive: false, - global: true, - description: 'Not a real scan, used to test proxy' - }, - test: { - type: 'fargate', - isPassive: false, - global: true, - description: 'Not a real scan, used to test' - }, - censys: { amass: { type: 'fargate', isPassive: false, @@ -234,23 +221,6 @@ export const SCAN_SCHEMA: ScanSchema = { cpu: '1024', memory: '8192' }, - dotgov: { - type: 'fargate', - isPassive: true, - global: true, - description: - 'Create organizations based on root domains from the dotgov registrar dataset. All organizations are created with the "dotgov" tag and have a " (dotgov)" suffix added to their name.' - }, - searchSync: { - type: 'fargate', - isPassive: true, - global: true, - cpu: '2048', - memory: '16384', - description: - 'Syncs records with Elasticsearch so that they appear in search results.' - }, - intrigueIdent: { wappalyzer: { type: 'fargate', isPassive: true, diff --git a/backend/src/api/users.ts b/backend/src/api/users.ts index 44daf459..1ddae23e 100644 --- a/backend/src/api/users.ts +++ b/backend/src/api/users.ts @@ -35,10 +35,6 @@ import { isOrgAdmin, isGlobalWriteAdmin } from './auth'; -import { Type, plainToClass } from 'class-transformer'; -import { IsNull } from 'typeorm'; -import { create } from './organizations'; -import logger from '../tools/lambda-logger'; import { fetchAssessmentsByUser } from '../tasks/rscSync'; class UserSearch { @@ -327,7 +323,6 @@ export const invite = wrapHandler(async (event) => { let user = await User.findOne({ email: body.email }); - logger.info(user); let organization: Organization | undefined; if (body.organization) { diff --git a/backend/src/api/vulnerabilities.ts b/backend/src/api/vulnerabilities.ts index 86416504..218df49b 100644 --- a/backend/src/api/vulnerabilities.ts +++ b/backend/src/api/vulnerabilities.ts @@ -170,7 +170,8 @@ class VulnerabilitySearch { : `vulnerability.${this.sort}`; let qs = Vulnerability.createQueryBuilder('vulnerability') .leftJoinAndSelect('vulnerability.domain', 'domain') - .leftJoinAndSelect('domain.organization', 'organization'); + .leftJoinAndSelect('domain.organization', 'organization') + .leftJoinAndSelect('vulnerability.service', 'service'); if (groupBy) { qs = qs @@ -185,9 +186,7 @@ class VulnerabilitySearch { ]) .orderBy('cnt', 'DESC'); } else { - qs = qs - .leftJoinAndSelect('vulnerability.service', 'service') - .orderBy(sort, this.order); + qs = qs.orderBy(sort, this.order); } if (pageSize !== -1) { @@ -283,7 +282,7 @@ export const update = wrapHandler(async (event) => { * - Vulnerabilities */ export const list = wrapHandler(async (event) => { - await connectToDatabase(); + await connectToDatabase(true); const search = await validateBody(VulnerabilitySearch, event.body); const [result, count] = await search.getResults(event); return { diff --git a/dev.env.example b/dev.env.example index 56dc76ad..8aeed1c2 100644 --- a/dev.env.example +++ b/dev.env.example @@ -6,6 +6,10 @@ DB_PASSWORD=password DB_NAME=crossfeed JWT_SECRET=CHANGE_ME +BACKEND_DOMAIN=http://localhost:3000 +COGNITO_URL=https://cognito-idp.us-east-1.amazonaws.com +EMAIL_REGION= + MDL_USERNAME=mdl MDL_PASSWORD=password MDL_NAME=crossfeed_mini_datalake