Add Whoisxml, Qualys and MDL functionality from ATC to ASM SQS queue (#…

…716) * Add whoisxml api key variable * Run terraform linter * Fix typo in pe_worker * fix typo * Add latest ASM Sync code from crossfeed-SQS commit * Add Qualys and latest ATC-Framework * Add was-report-pull to qualys command * Fix terraform checks * Fix pre-commit checks
cisagov · Dec 13, 2024 · 6cc39b6 · 6cc39b6
1 parent 3e772fd
commit 6cc39b6
Show file tree

Hide file tree

Showing 10 changed files with 71 additions and 7 deletions.
diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe
@@ -28,7 +28,7 @@ RUN ./aws/install
 # Sync the latest from cf-staging branch
 RUN git clone -b crossfeed-SQS https://github.com/cisagov/ATC-Framework.git && \
     cd ATC-Framework && \
-    git checkout 22c3de519337e577775f2580ba09c7ba51c08904 && \
+    git checkout 074fcaf4c4e3dd1cd2d71aeeab71319db3701c48 && \
     pip install .
 
 RUN python -m spacy download en_core_web_lg

diff --git a/backend/serverless.yml b/backend/serverless.yml
@@ -147,10 +147,17 @@ resources:
         VisibilityTimeout: 18000  # 5 hours
         MaximumMessageSize: 262144  # 256 KB
         MessageRetentionPeriod: 604800  # 7 days
-    ASMQueue:
+    ASMSyncQueue:
       Type: AWS::SQS::Queue
       Properties:
-        QueueName: ${self:provider.stage}-asm-queue
+        QueueName: ${self:provider.stage}-asmSync-queue
+        VisibilityTimeout: 18000  # 5 hours
+        MaximumMessageSize: 262144  # 256 KB
+        MessageRetentionPeriod: 604800  # 7 days
+    QualysQueue:
+      Type: AWS::SQS::Queue
+      Properties:
+        QueueName: ${self:provider.stage}-qualys-queue
         VisibilityTimeout: 18000  # 5 hours
         MaximumMessageSize: 262144  # 256 KB
         MessageRetentionPeriod: 604800  # 7 days

diff --git a/backend/src/tasks/scanExecution.ts b/backend/src/tasks/scanExecution.ts
@@ -11,7 +11,8 @@ const SCAN_LIST = [
   'cybersixgill',
   'shodan',
   'xpanse',
-  'asmSync'
+  'asmSync',
+  'qualys'
 ];
 
 if (process.env.IS_LOCAL) {
@@ -159,7 +160,10 @@ async function startLocalContainers(
           `SERVICE_TYPE=${scanType}`,
           `PE_API_URL=${process.env.PE_API_URL}`,
           `PE_API_KEY=${process.env.PE_API_KEY}`,
-          `CF_API_KEY=${process.env.CF_API_KEY}`
+          `CF_API_KEY=${process.env.CF_API_KEY}`,
+          `WHOIS_XML_KEY=${process.env.WHOIS_XML_KEY}`,
+          `QUALYS_USERNAME=${process.env.QUALYS_USERNAME}`,
+          `QUALYS_PASSWORD=${process.env.QUALYS_PASSWORD}`
         ]
       } as any);
       await container.start();
@@ -222,7 +226,7 @@ export const handler: Handler = async (event) => {
       await startDesiredTasks(scanType, desiredCount);
     } else {
       console.log(
-        'Shodan, ASMSync, DNSTwist, IntelX, Xpanse, and Cybersixgill are the only script types available right now. Must be all lowercase.'
+        'Shodan, Qualys, ASMSync, DNSTwist, IntelX, Xpanse, and Cybersixgill are the only script types available right now. Must be all lowercase.'
       );
     }
   } catch (error) {

diff --git a/backend/worker/generate_config.sh b/backend/worker/generate_config.sh
@@ -25,7 +25,7 @@ client_id=${SIXGILL_CLIENT_ID}
 client_secret=${SIXGILL_CLIENT_SECRET}
 
 [whoisxml]
-key=
+key=${WHOIS_XML_KEY}
 
 [intelx]
 api_key=${INTELX_API_KEY}
@@ -44,6 +44,10 @@ api_key=${INTELX_API_KEY}
 api_key=${XPANSE_API_KEY}
 auth_id=${XPANSE_AUTH_ID}
 
+[was]
+username=${QUALYS_USERNAME}
+password=${QUALYS_PASSWORD}
+
 
 EOF
 

diff --git a/backend/worker/pe-worker-entry.sh b/backend/worker/pe-worker-entry.sh
@@ -64,6 +64,9 @@ while true; do
     COMMAND="pe-source xpanse --org='$ORG'"
   elif [[ "$SERVICE_TYPE" = *"asmSync"* ]]; then
     COMMAND="pe-asm-sync asm-sqs --org='$ORG'"
+  elif [[ "$SERVICE_TYPE" = *"qualys"* ]]; then
+    COMMAND="pe-source was-report-pull --org='$ORG' && pe-source was-findings-sync --org='$ORG'"
+
   else
     echo "Unsupported SERVICE_TYPE: $SERVICE_TYPE"
     break

diff --git a/dev.env.example b/dev.env.example
@@ -105,3 +105,7 @@ PE_SHODAN_API_KEYS=
 
 PE_FARGATE_CLUSTER_NAME=pe-staging-worker
 PE_FARGATE_TASK_DEFINITION_NAME=pe-staging-worker
+
+WHOIS_XML_KEY=change_me
+QUALYS_USERNAME=change_me
+QUALYS_PASSWORD=change_me
diff --git a/infrastructure/pe_worker.tf b/infrastructure/pe_worker.tf
@@ -160,6 +160,18 @@ resource "aws_ecs_task_definition" "pe_worker" {
       {
         "name": "ELASTICSEARCH_ENDPOINT",
         "valueFrom": "${aws_ssm_parameter.es_endpoint.arn}"
+      },
+      {
+        "name": "WHOIS_XML_KEY",
+        "valueFrom": "${data.aws_ssm_parameter.whoisxml_api_key.arn}"
+      },
+      {
+        "name": "QUALYS_USERNAME",
+        "valueFrom": "${data.aws_ssm_parameter.qualys_username.arn}"
+      },
+      {
+        "name": "QUALYS_PASSWORD",
+        "valueFrom": "${data.aws_ssm_parameter.qualys_password.arn}"
       }
     ]
   }

diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars
@@ -58,6 +58,9 @@ ssm_lg_api_key                       = "/crossfeed/staging/LG_API_KEY"
 ssm_lg_workspace_name                = "/crossfeed/staging/LG_WORKSPACE_NAME"
 ssm_pe_api_key                       = "/crossfeed/staging/PE_API_KEY"
 ssm_cf_api_key                       = "/crossfeed/staging/CF_API_KEY"
+ssm_whoisxml_api_key                 = "/crossfeed/staging/WHOIS_XML_KEY"
+ssm_qualys_username                  = "/crossfeed/staging/QUALYS_USERNAME"
+ssm_qualys_password                  = "/crossfeed/staging/QUALYS_PASSWORD"
 db_group_name                        = "crossfeed-staging-db-group"
 worker_ecs_repository_name           = "crossfeed-staging-worker"
 worker_ecs_cluster_name              = "crossfeed-staging-worker"

diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf
@@ -327,6 +327,24 @@ variable "ssm_xpanse_api_key" {
   default     = "/crossfeed/staging/XPANSE_API_KEY"
 }
 
+variable "ssm_whoisxml_api_key" {
+  description = "ssm_whoisxml_api_key"
+  type        = string
+  default     = "/crossfeed/staging/WHOIS_XML_KEY"
+}
+
+variable "ssm_qualys_username" {
+  description = "ssm_qualys_username"
+  type        = string
+  default     = "/crossfeed/staging/QUALYS_USERNAME"
+}
+
+variable "ssm_qualys_password" {
+  description = "ssm_qualys_password"
+  type        = string
+  default     = "/crossfeed/staging/QUALYS_PASSWORD"
+}
+
 variable "ssm_xpanse_auth_id" {
   description = "ssm_xpanse_auth_id"
   type        = string

diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf
@@ -87,6 +87,9 @@ resource "aws_iam_role_policy" "worker_task_execution_role_policy" {
           "${data.aws_ssm_parameter.intelx_api_key.arn}",
           "${data.aws_ssm_parameter.xpanse_api_key.arn}",
           "${data.aws_ssm_parameter.xpanse_auth_id.arn}",
+          "${data.aws_ssm_parameter.whoisxml_api_key.arn}",
+          "${data.aws_ssm_parameter.qualys_username.arn}",
+          "${data.aws_ssm_parameter.qualys_password.arn}",
           "${data.aws_ssm_parameter.sixgill_client_secret.arn}",
           "${data.aws_ssm_parameter.lg_api_key.arn}",
           "${data.aws_ssm_parameter.lg_workspace_name.arn}",
@@ -386,6 +389,12 @@ data "aws_ssm_parameter" "intelx_api_key" { name = var.ssm_intelx_api_key }
 
 data "aws_ssm_parameter" "xpanse_api_key" { name = var.ssm_xpanse_api_key }
 
+data "aws_ssm_parameter" "whoisxml_api_key" { name = var.ssm_whoisxml_api_key }
+
+data "aws_ssm_parameter" "qualys_username" { name = var.ssm_qualys_username }
+
+data "aws_ssm_parameter" "qualys_password" { name = var.ssm_qualys_password }
+
 data "aws_ssm_parameter" "xpanse_auth_id" { name = var.ssm_xpanse_auth_id }
 
 data "aws_ssm_parameter" "sixgill_client_secret" { name = var.ssm_sixgill_client_secret }