Merge pull request #351 from cisagov/AL-add-xpanse-SQS

Add Xpanse to the SQS process
cisagov · Jul 17, 2024 · 581442c · 581442c
2 parents 4b4ed6f + 0d7740f
commit 581442c
Show file tree

Hide file tree

Showing 11 changed files with 79 additions and 11 deletions.
diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe
@@ -28,7 +28,7 @@ RUN ./aws/install
 # Sync the latest from cf-staging branch
 RUN git clone -b crossfeed-SQS https://github.com/cisagov/ATC-Framework.git && \
     cd ATC-Framework && \
-    git checkout a4036e94ad54876cb592fc966f57e7fd84750e9a && \
+    git checkout 995ec3c28de41c3bb9ac8381441d8214b7e87a1d && \
     pip install .
 
 RUN python -m spacy download en_core_web_lg

diff --git a/backend/serverless.yml b/backend/serverless.yml
@@ -143,6 +143,13 @@ resources:
         VisibilityTimeout: 18000  # 5 hours
         MaximumMessageSize: 262144  # 256 KB
         MessageRetentionPeriod: 604800  # 7 days
+    XpanseQueue:
+      Type: AWS::SQS::Queue
+      Properties:
+        QueueName: ${self:provider.stage}-xpanse-queue
+        VisibilityTimeout: 18000  # 5 hours
+        MaximumMessageSize: 262144  # 256 KB
+        MessageRetentionPeriod: 604800  # 7 days
 
 functions:
   - ${file(./src/tasks/functions.yml)}

diff --git a/backend/src/tasks/scanExecution.ts b/backend/src/tasks/scanExecution.ts
@@ -5,7 +5,14 @@ import { integer } from 'aws-sdk/clients/cloudfront';
 const ecs = new AWS.ECS();
 let docker: any;
 const QUEUE_URL = process.env.QUEUE_URL!;
-const SCAN_LIST = ['dnstwist', 'hibp', 'intelx', 'cybersixgill', 'shodan'];
+const SCAN_LIST = [
+  'dnstwist',
+  'hibp',
+  'intelx',
+  'cybersixgill',
+  'shodan',
+  'xpanse'
+];
 
 if (process.env.IS_LOCAL) {
   const Docker = require('dockerode');
@@ -138,6 +145,8 @@ async function startLocalContainers(
           `SIXGILL_CLIENT_ID=${process.env.SIXGILL_CLIENT_ID}`,
           `SIXGILL_CLIENT_SECRET=${process.env.SIXGILL_CLIENT_SECRET}`,
           `INTELX_API_KEY=${process.env.INTELX_API_KEY}`,
+          `XPANSE_API_KEY=${process.env.XPANSE_API_KEY}`,
+          `XPANSE_AUTH_ID=${process.env.XPANSE_AUTH_ID}`,
           `PE_SHODAN_API_KEYS=${shodan_api_key}`,
           `WORKER_SIGNATURE_PUBLIC_KEY=${process.env.WORKER_SIGNATURE_PUBLIC_KEY}`,
           `WORKER_SIGNATURE_PRIVATE_KEY=${process.env.WORKER_SIGNATURE_PRIVATE_KEY}`,
@@ -148,7 +157,10 @@ async function startLocalContainers(
           `LG_API_KEY=${process.env.LG_API_KEY}`,
           `LG_WORKSPACE_NAME=${process.env.LG_WORKSPACE_NAME}`,
           `SERVICE_QUEUE_URL=${queueUrl}`,
-          `SERVICE_TYPE=${scanType}`
+          `SERVICE_TYPE=${scanType}`,
+          `PE_API_URL=${process.env.PE_API_URL}`,
+          `PE_API_KEY=${process.env.PE_API_KEY}`,
+          `CF_API_KEY=${process.env.CF_API_KEY}`
         ]
       } as any);
       await container.start();
@@ -211,7 +223,7 @@ export const handler: Handler = async (event) => {
       await startDesiredTasks(scanType, desiredCount);
     } else {
       console.log(
-        'Shodan, DNSTwist, HIBP, IntelX, and Cybersixgill are the only script types available right now. Must be all lowercase.'
+        'Shodan, DNSTwist, HIBP, IntelX, Xpanse, and Cybersixgill are the only script types available right now. Must be all lowercase.'
       );
     }
   } catch (error) {

diff --git a/backend/src/tools/run-scanExecution.ts b/backend/src/tools/run-scanExecution.ts
@@ -31,9 +31,9 @@ async function sendMessageToQueue(message, queue) {
 }
 
 // Simulate sending a message
-const SCAN_TYPE = 'dnstwist';
+const SCAN_TYPE = 'xpanse';
 const DESIRED_COUNT = 1;
-const ORG_LIST = ['DHS', 'DOI'];
+const ORG_LIST = ['National Science Foundation (NSF) - CISA'];
 const QUEUE = `staging-${SCAN_TYPE}-queue`;
 const API_KEY_LIST = '';
 

diff --git a/backend/worker/generate_config.sh b/backend/worker/generate_config.sh
@@ -16,8 +16,9 @@ key1=${PE_SHODAN_API_KEYS}
 key=${HIBP_API_KEY}
 
 [pe_api]
-pe_api_key=
-pe_api_url=
+pe_api_key=${PE_API_KEY}
+pe_api_url=https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/
+cf_api_key=${CF_API_KEY}
 
 [staging]
 [cyhy_mongo]
@@ -42,6 +43,10 @@ api_key=${INTELX_API_KEY}
 [API_Client_secret]
 [API_WHOIS]
 
+[xpanse]
+api_key=${XPANSE_API_KEY}
+auth_id=${XPANSE_AUTH_ID}
+
 
 EOF
 

diff --git a/backend/worker/pe-worker-entry.sh b/backend/worker/pe-worker-entry.sh
@@ -62,6 +62,8 @@ while true; do
     COMMAND="pe-source intelx --org=$ORG --soc_med_included"
   elif [[ "$SERVICE_TYPE" = *"cybersixgill"* ]]; then
     COMMAND="pe-source cybersixgill --org=$ORG --soc_med_included"
+  elif [[ "$SERVICE_TYPE" = *"xpanse"* ]]; then
+    COMMAND="pe-source xpanse --org='$ORG'"
   else
     echo "Unsupported SERVICE_TYPE: $SERVICE_TYPE"
     break

diff --git a/docs/src/documentation-pages/dev/quickstart.md b/docs/src/documentation-pages/dev/quickstart.md
@@ -55,21 +55,27 @@ This quickstart describes the initial setup required to run an instance of Cross
       cp dev.env.example .env
    ```
 
-3. Generate the P&E DB Schema
+3. Start application:
+
+   ```bash
+      npm start
+   ```
+
+4. Generate the P&E DB Schema:
 
    ```bash
    cd backend
    npm run pesyncdb
    ```
 
-4. Invoke scans by running below. You can edit the backend/src/tools/run-scanExecution.ts to run the desired scan type."
+5. Invoke scans by running below. You can edit the backend/src/tools/run-scanExecution.ts to run the desired scan type."
 
    ```bash
    cd backend
    npm run scan-exec
    ```
 
-5. Observe logs in docker containers.
+6. Observe logs in docker containers.
 
 ### Running tests
 

diff --git a/infrastructure/pe_worker.tf b/infrastructure/pe_worker.tf
@@ -129,6 +129,22 @@ resource "aws_ecs_task_definition" "pe_worker" {
         "name": "INTELX_API_KEY",
         "valueFrom": "${data.aws_ssm_parameter.intelx_api_key.arn}"
       },
+      {
+        "name": "XPANSE_API_KEY",
+        "valueFrom": "${data.aws_ssm_parameter.xpanse_api_key.arn}"
+      },
+      {
+        "name": "XPANSE_AUTH_ID",
+        "valueFrom": "${data.aws_ssm_parameter.xpanse_auth_id.arn}"
+      },
+      {
+        "name": "PE_API_KEY",
+        "valueFrom": "${data.aws_ssm_parameter.pe_api_key.arn}"
+      },
+      {
+        "name": "CF_API_KEY",
+        "valueFrom": "${data.aws_ssm_parameter.cf_api_key.arn}"
+      },
       {
         "name": "LG_API_KEY",
         "valueFrom": "${data.aws_ssm_parameter.lg_api_key.arn}"

diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars
@@ -53,6 +53,8 @@ ssm_pe_shodan_api_keys               = "/crossfeed/staging/PE_SHODAN_API_KEYS"
 ssm_sixgill_client_id                = "/crossfeed/staging/SIXGILL_CLIENT_ID"
 ssm_sixgill_client_secret            = "/crossfeed/staging/SIXGILL_CLIENT_SECRET"
 ssm_intelx_api_key                   = "/crossfeed/staging/INTELX_API_KEY"
+ssm_xpanse_api_key                   = "/crossfeed/staging/XPANSE_API_KEY"
+ssm_xpanse_auth_id                   = "/crossfeed/staging/XPANSE_AUTH_ID"
 ssm_lg_api_key                       = "/crossfeed/staging/LG_API_KEY"
 ssm_lg_workspace_name                = "/crossfeed/staging/LG_WORKSPACE_NAME"
 ssm_pe_api_key                       = "/crossfeed/staging/PE_API_KEY"

diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf
@@ -327,6 +327,18 @@ variable "ssm_intelx_api_key" {
   default     = "/crossfeed/staging/INTELX_API_KEY"
 }
 
+variable "ssm_xpanse_api_key" {
+  description = "ssm_xpanse_api_key"
+  type        = string
+  default     = "/crossfeed/staging/XPANSE_API_KEY"
+}
+
+variable "ssm_xpanse_auth_id" {
+  description = "ssm_xpanse_auth_id"
+  type        = string
+  default     = "/crossfeed/staging/XPANSE_AUTH_ID"
+}
+
 
 variable "ssm_lg_api_key" {
   description = "ssm_lg_api_key"

diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf
@@ -86,6 +86,8 @@ resource "aws_iam_role_policy" "worker_task_execution_role_policy" {
           "${data.aws_ssm_parameter.pe_shodan_api_keys.arn}",
           "${data.aws_ssm_parameter.sixgill_client_id.arn}",
           "${data.aws_ssm_parameter.intelx_api_key.arn}",
+          "${data.aws_ssm_parameter.xpanse_api_key.arn}",
+          "${data.aws_ssm_parameter.xpanse_auth_id.arn}",
           "${data.aws_ssm_parameter.sixgill_client_secret.arn}",
           "${data.aws_ssm_parameter.lg_api_key.arn}",
           "${data.aws_ssm_parameter.lg_workspace_name.arn}",
@@ -389,6 +391,10 @@ data "aws_ssm_parameter" "sixgill_client_id" { name = var.ssm_sixgill_client_id
 
 data "aws_ssm_parameter" "intelx_api_key" { name = var.ssm_intelx_api_key }
 
+data "aws_ssm_parameter" "xpanse_api_key" { name = var.ssm_xpanse_api_key }
+
+data "aws_ssm_parameter" "xpanse_auth_id" { name = var.ssm_xpanse_auth_id }
+
 data "aws_ssm_parameter" "sixgill_client_secret" { name = var.ssm_sixgill_client_secret }
 
 data "aws_ssm_parameter" "pe_db_name" { name = var.ssm_pe_db_name }