Merge branch 'develop' into 20-merge-lz-infra

backend/.snyk

@@ -5,9 +5,5 @@ version: v1.22.1
 # ignores vulnerabilities until expiry date; change duration by modifying expiry date
 
 ignore:
-    # ignore scrapy 2.x.x for 6 months.
-    SNYK-PYTHON-SCRAPY-40690:
-        - '*':
-            reason: No fix available up to version 2.11.0
-            expires: 2024-06-01T00:00:00.000Z
+
 patch: {}

backend/Dockerfile.pe

@@ -26,9 +26,9 @@ RUN ./aws/install
 
 # Install pe-source module
 # Sync the latest from cf-staging branch
-RUN git clone -b AL-staging-SQS https://github.com/cisagov/pe-reports.git && \
-    cd pe-reports && \
-    git checkout 6405a2041656152b176b5fc9b3becb5dc11a5f3e && \
+RUN git clone -b crossfeed-SQS https://github.com/cisagov/ATC-Framework.git && \
+    cd ATC-Framework && \
+    git checkout a4036e94ad54876cb592fc966f57e7fd84750e9a && \
     pip install .
 
 RUN python -m spacy download en_core_web_lg

backend/scripts/populateCountiesCities/requirements.txt

@@ -1,4 +1,4 @@
 beautifulsoup4==4.11.2
 pandas==1.5.1
-requests==2.28.2
+requests==2.32.3
 typer==0.7.0

backend/src/api/app.ts

@@ -209,9 +209,6 @@ app.post('/auth/okta-callback', async (req, res) => {
   const clientId = process.env.REACT_APP_COGNITO_CLIENT_ID;
   const callbackUrl = process.env.REACT_APP_COGNITO_CALLBACK_URL;
   const domain = process.env.REACT_APP_COGNITO_DOMAIN;
-  console.log('Okta ClientID: ', clientId);
-  console.log('Okta CallbackURL: ', callbackUrl);
-  console.log('Okta Domain: ', domain);
 
   if (!code) {
     return res.status(400).json({ message: 'Missing authorization code' });
@@ -235,7 +232,6 @@ app.post('/auth/okta-callback', async (req, res) => {
       body: tokenData,
       agent: new HttpsProxyAgent('http://proxy.lz.us-cert.gov:8080')
     });
-    console.log('Okta token response: ', response);
     const { id_token, access_token, refresh_token } = await response.json();
 
     if (!id_token) {
@@ -249,12 +245,6 @@ app.post('/auth/okta-callback', async (req, res) => {
 
     const cognitoUsername = decodedToken['cognito:username'];
     const oktaId = decodedToken['custom:OKTA_ID'];
-    console.log('Cognito Username:', cognitoUsername);
-    console.log('Cognito OKTA_ID:', oktaId);
-
-    console.log('ID Token:', id_token);
-    console.log('Decoded Token:', decodedToken);
-
     jwt.verify(
       id_token,
       auth.getOktaKey,

backend/src/api/domains.ts

@@ -21,7 +21,7 @@ import {
 import S3Client from '../tasks/s3-client';
 import * as Papa from 'papaparse';
 
-const PAGE_SIZE = parseInt(process.env.PAGE_SIZE ?? '') || 25;
+const PAGE_SIZE = 15;
 
 class DomainFilters {
   @IsString()

backend/src/api/helpers.ts

@@ -391,7 +391,6 @@ export const sendRegistrationApprovedEmail = async (
  */
 async function isMajorActiveMaintenance(): Promise<boolean> {
   const now = new Date();
-  console.log(now);
   try {
     // DB connection
     await connectToDatabase();

backend/src/api/notifications.ts

@@ -4,6 +4,18 @@ import { validateBody, wrapHandler, NotFound, Unauthorized } from './helpers';
 import { isGlobalWriteAdmin } from './auth';
 import S3Client from '../tasks/s3-client';
 
+// 508 Warning Banner S3 filename
+const bannerFileName = '508warningtext.txt';
+
+// Default 508 Warning Banner for local dev
+const default508Banner =
+  'CISA is committed to providing access for all individuals with disabilities, \
+  including members of the public and all employees. While this website is not \
+  yet fully accessible to users with disabilities as required by Section \
+  [508](https://cisa.gov/accessibility) federal law, CISA is working diligently \
+  to resolve those issues. If you experience accessibility issues, please email \
+  [[email protected]](mailto:[email protected]) for assistance.';
+
 class NewNotification {
   @IsDateString()
   startDatetime?: string;
@@ -99,8 +111,6 @@ export const list = wrapHandler(async (event) => {
     }
   });
 
-  console.log('Notification.find result: ', result);
-
   return {
     statusCode: 200,
     body: JSON.stringify(result)
@@ -161,16 +171,28 @@ export const update = wrapHandler(async (event) => {
  *    - Notifications
  */
 export const get508Banner = wrapHandler(async () => {
-  const bannerFileName = '508warningtext.txt';
+  // Return hardcoded banner for local builds
+  if (process.env.IS_LOCAL) {
+    console.log('Using default banner for 508 warning: ', default508Banner);
+    // API Response
+    return {
+      statusCode: 200,
+      body: JSON.stringify(default508Banner)
+    };
+  }
+
+  // Handle normal S3 logic
   try {
     const client = new S3Client();
     const bannerResult = await client.getEmailAsset(bannerFileName);
+    // API Response
     return {
       statusCode: 200,
       body: JSON.stringify(bannerResult)
     };
   } catch (error) {
     console.log('S3 Banner Error: ', error);
+    // API Error Response
     return {
       statusCode: 500,
       body: 'Error retrieving file from S3. See details in logs.'

backend/src/api/scan-tasks.ts

@@ -20,7 +20,7 @@ import {
 } from './auth';
 import ECSClient from '../tasks/ecs-client';
 
-const PAGE_SIZE = parseInt(process.env.PAGE_SIZE ?? '') || 25;
+const PAGE_SIZE = 15;
 
 class ScanTaskFilters {
   @IsString()
@@ -59,6 +59,11 @@ class ScanTaskSearch {
   @IsOptional()
   filters?: ScanTaskFilters;
 
+  @IsInt()
+  @IsOptional()
+  // If set to -1, returns all results.
+  pageSize?: number;
+
   async filterResultQueryset(qs: SelectQueryBuilder<ScanTask>, event) {
     if (this.filters?.name) {
       qs.andWhere('scan.name ILIKE :name', {
@@ -84,11 +89,16 @@ class ScanTaskSearch {
   }
 
   async getResults(event) {
+    const pageSize = this.pageSize || PAGE_SIZE;
+
     const qs = ScanTask.createQueryBuilder('scan_task')
       .leftJoinAndSelect('scan_task.scan', 'scan')
-      .orderBy(`scan_task.${this.sort}`, this.order)
-      .skip(PAGE_SIZE * (this.page - 1))
-      .take(PAGE_SIZE);
+      .leftJoinAndSelect('scan_task.organizations', 'organization')
+      .orderBy(`scan_task.${this.sort}`, this.order);
+
+    if (pageSize !== -1) {
+      qs.skip(pageSize * (this.page - 1)).take(pageSize);
+    }
 
     await this.filterResultQueryset(qs, event);
     return qs.getManyAndCount();

backend/src/api/scans.ts

@@ -246,6 +246,15 @@ export const SCAN_SCHEMA: ScanSchema = {
     cpu: '1024',
     memory: '4096',
     description: 'Scrapes all webpages on a given domain, respecting robots.txt'
+  },
+  xpanseSync: {
+    type: 'fargate',
+    isPassive: true,
+    global: true,
+    description:
+      'Pull in xpanse vulnerability data from PEs Vulnerability database',
+    cpu: '1024',
+    memory: '8192'
   }
 };
 

backend/src/api/users.ts

@@ -176,7 +176,9 @@ class UpdateUser {
  *    - Users
  */
 export const del = wrapHandler(async (event) => {
-  if (!isGlobalWriteAdmin(event)) return Unauthorized;
+  if (!isGlobalWriteAdmin(event) && !isRegionalAdmin(event))
+    return Unauthorized;
+
   await connectToDatabase();
   const id = event.pathParameters?.userId;
   if (!id || !isUUID(id)) {
@@ -916,6 +918,10 @@ export const updateV2 = wrapHandler(async (event) => {
     return NotFound;
   }
 
+  if (body.state) {
+    body.regionId = REGION_STATE_MAP[body.state];
+  }
+
   // Update the user
   const updatedResp = await User.update(userId, body);
 

backend/src/tasks/s3-client.ts

@@ -11,16 +11,14 @@ class S3Client {
   isLocal: boolean;
 
   constructor(isLocal?: boolean) {
-    // TODO: Remove once minio service is fixed
-    // this.isLocal =
-    //   isLocal ??
-    //   (process.env.IS_OFFLINE || process.env.IS_LOCAL ? true : false);
+    this.isLocal =
+      isLocal ??
+      (process.env.IS_OFFLINE || process.env.IS_LOCAL ? true : false);
     if (this.isLocal) {
-      // TODO: Remove once minio service is fixed
-      // this.s3 = new S3({
-      //   endpoint: 'http://minio:9000',
-      //   s3ForcePathStyle: true
-      // });
+      this.s3 = new S3({
+        endpoint: 'http://minio:9000',
+        s3ForcePathStyle: true
+      });
     } else {
       const agent = new https.Agent({
         keepAlive: false