From 6cc39b67e8c709eaad1d853a9eb0281baa72b2a5 Mon Sep 17 00:00:00 2001 From: aloftus23 <79927030+aloftus23@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:00:22 -0500 Subject: [PATCH] Add Whoisxml, Qualys and MDL functionality from ATC to ASM SQS queue (#716) * Add whoisxml api key variable * Run terraform linter * Fix typo in pe_worker * fix typo * Add latest ASM Sync code from crossfeed-SQS commit * Add Qualys and latest ATC-Framework * Add was-report-pull to qualys command * Fix terraform checks * Fix pre-commit checks --- backend/Dockerfile.pe | 2 +- backend/serverless.yml | 11 +++++++++-- backend/src/tasks/scanExecution.ts | 10 +++++++--- backend/worker/generate_config.sh | 6 +++++- backend/worker/pe-worker-entry.sh | 3 +++ dev.env.example | 4 ++++ infrastructure/pe_worker.tf | 12 ++++++++++++ infrastructure/stage.tfvars | 3 +++ infrastructure/vars.tf | 18 ++++++++++++++++++ infrastructure/worker.tf | 9 +++++++++ 10 files changed, 71 insertions(+), 7 deletions(-) diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe index 3d90f73a..0b5aab71 100644 --- a/backend/Dockerfile.pe +++ b/backend/Dockerfile.pe @@ -28,7 +28,7 @@ RUN ./aws/install # Sync the latest from cf-staging branch RUN git clone -b crossfeed-SQS https://github.com/cisagov/ATC-Framework.git && \ cd ATC-Framework && \ - git checkout 22c3de519337e577775f2580ba09c7ba51c08904 && \ + git checkout 074fcaf4c4e3dd1cd2d71aeeab71319db3701c48 && \ pip install . RUN python -m spacy download en_core_web_lg diff --git a/backend/serverless.yml b/backend/serverless.yml index dd7e1f41..7ff67f77 100644 --- a/backend/serverless.yml +++ b/backend/serverless.yml @@ -147,10 +147,17 @@ resources: VisibilityTimeout: 18000 # 5 hours MaximumMessageSize: 262144 # 256 KB MessageRetentionPeriod: 604800 # 7 days - ASMQueue: + ASMSyncQueue: Type: AWS::SQS::Queue Properties: - QueueName: ${self:provider.stage}-asm-queue + QueueName: ${self:provider.stage}-asmSync-queue + VisibilityTimeout: 18000 # 5 hours + MaximumMessageSize: 262144 # 256 KB + MessageRetentionPeriod: 604800 # 7 days + QualysQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: ${self:provider.stage}-qualys-queue VisibilityTimeout: 18000 # 5 hours MaximumMessageSize: 262144 # 256 KB MessageRetentionPeriod: 604800 # 7 days diff --git a/backend/src/tasks/scanExecution.ts b/backend/src/tasks/scanExecution.ts index d1dff72d..7dd6347d 100644 --- a/backend/src/tasks/scanExecution.ts +++ b/backend/src/tasks/scanExecution.ts @@ -11,7 +11,8 @@ const SCAN_LIST = [ 'cybersixgill', 'shodan', 'xpanse', - 'asmSync' + 'asmSync', + 'qualys' ]; if (process.env.IS_LOCAL) { @@ -159,7 +160,10 @@ async function startLocalContainers( `SERVICE_TYPE=${scanType}`, `PE_API_URL=${process.env.PE_API_URL}`, `PE_API_KEY=${process.env.PE_API_KEY}`, - `CF_API_KEY=${process.env.CF_API_KEY}` + `CF_API_KEY=${process.env.CF_API_KEY}`, + `WHOIS_XML_KEY=${process.env.WHOIS_XML_KEY}`, + `QUALYS_USERNAME=${process.env.QUALYS_USERNAME}`, + `QUALYS_PASSWORD=${process.env.QUALYS_PASSWORD}` ] } as any); await container.start(); @@ -222,7 +226,7 @@ export const handler: Handler = async (event) => { await startDesiredTasks(scanType, desiredCount); } else { console.log( - 'Shodan, ASMSync, DNSTwist, IntelX, Xpanse, and Cybersixgill are the only script types available right now. Must be all lowercase.' + 'Shodan, Qualys, ASMSync, DNSTwist, IntelX, Xpanse, and Cybersixgill are the only script types available right now. Must be all lowercase.' ); } } catch (error) { diff --git a/backend/worker/generate_config.sh b/backend/worker/generate_config.sh index 2057aa68..dbec4454 100755 --- a/backend/worker/generate_config.sh +++ b/backend/worker/generate_config.sh @@ -25,7 +25,7 @@ client_id=${SIXGILL_CLIENT_ID} client_secret=${SIXGILL_CLIENT_SECRET} [whoisxml] -key= +key=${WHOIS_XML_KEY} [intelx] api_key=${INTELX_API_KEY} @@ -44,6 +44,10 @@ api_key=${INTELX_API_KEY} api_key=${XPANSE_API_KEY} auth_id=${XPANSE_AUTH_ID} +[was] +username=${QUALYS_USERNAME} +password=${QUALYS_PASSWORD} + EOF diff --git a/backend/worker/pe-worker-entry.sh b/backend/worker/pe-worker-entry.sh index e2a8377e..10c60b5e 100755 --- a/backend/worker/pe-worker-entry.sh +++ b/backend/worker/pe-worker-entry.sh @@ -64,6 +64,9 @@ while true; do COMMAND="pe-source xpanse --org='$ORG'" elif [[ "$SERVICE_TYPE" = *"asmSync"* ]]; then COMMAND="pe-asm-sync asm-sqs --org='$ORG'" + elif [[ "$SERVICE_TYPE" = *"qualys"* ]]; then + COMMAND="pe-source was-report-pull --org='$ORG' && pe-source was-findings-sync --org='$ORG'" + else echo "Unsupported SERVICE_TYPE: $SERVICE_TYPE" break diff --git a/dev.env.example b/dev.env.example index 87fe6b15..a516ed39 100644 --- a/dev.env.example +++ b/dev.env.example @@ -105,3 +105,7 @@ PE_SHODAN_API_KEYS= PE_FARGATE_CLUSTER_NAME=pe-staging-worker PE_FARGATE_TASK_DEFINITION_NAME=pe-staging-worker + +WHOIS_XML_KEY=change_me +QUALYS_USERNAME=change_me +QUALYS_PASSWORD=change_me diff --git a/infrastructure/pe_worker.tf b/infrastructure/pe_worker.tf index 3d34ca3f..f650a924 100644 --- a/infrastructure/pe_worker.tf +++ b/infrastructure/pe_worker.tf @@ -160,6 +160,18 @@ resource "aws_ecs_task_definition" "pe_worker" { { "name": "ELASTICSEARCH_ENDPOINT", "valueFrom": "${aws_ssm_parameter.es_endpoint.arn}" + }, + { + "name": "WHOIS_XML_KEY", + "valueFrom": "${data.aws_ssm_parameter.whoisxml_api_key.arn}" + }, + { + "name": "QUALYS_USERNAME", + "valueFrom": "${data.aws_ssm_parameter.qualys_username.arn}" + }, + { + "name": "QUALYS_PASSWORD", + "valueFrom": "${data.aws_ssm_parameter.qualys_password.arn}" } ] } diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars index 10c71f1c..54e94c0f 100644 --- a/infrastructure/stage.tfvars +++ b/infrastructure/stage.tfvars @@ -58,6 +58,9 @@ ssm_lg_api_key = "/crossfeed/staging/LG_API_KEY" ssm_lg_workspace_name = "/crossfeed/staging/LG_WORKSPACE_NAME" ssm_pe_api_key = "/crossfeed/staging/PE_API_KEY" ssm_cf_api_key = "/crossfeed/staging/CF_API_KEY" +ssm_whoisxml_api_key = "/crossfeed/staging/WHOIS_XML_KEY" +ssm_qualys_username = "/crossfeed/staging/QUALYS_USERNAME" +ssm_qualys_password = "/crossfeed/staging/QUALYS_PASSWORD" db_group_name = "crossfeed-staging-db-group" worker_ecs_repository_name = "crossfeed-staging-worker" worker_ecs_cluster_name = "crossfeed-staging-worker" diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index 53232f6d..36497c82 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -327,6 +327,24 @@ variable "ssm_xpanse_api_key" { default = "/crossfeed/staging/XPANSE_API_KEY" } +variable "ssm_whoisxml_api_key" { + description = "ssm_whoisxml_api_key" + type = string + default = "/crossfeed/staging/WHOIS_XML_KEY" +} + +variable "ssm_qualys_username" { + description = "ssm_qualys_username" + type = string + default = "/crossfeed/staging/QUALYS_USERNAME" +} + +variable "ssm_qualys_password" { + description = "ssm_qualys_password" + type = string + default = "/crossfeed/staging/QUALYS_PASSWORD" +} + variable "ssm_xpanse_auth_id" { description = "ssm_xpanse_auth_id" type = string diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf index 6108cba2..104fcb3c 100644 --- a/infrastructure/worker.tf +++ b/infrastructure/worker.tf @@ -87,6 +87,9 @@ resource "aws_iam_role_policy" "worker_task_execution_role_policy" { "${data.aws_ssm_parameter.intelx_api_key.arn}", "${data.aws_ssm_parameter.xpanse_api_key.arn}", "${data.aws_ssm_parameter.xpanse_auth_id.arn}", + "${data.aws_ssm_parameter.whoisxml_api_key.arn}", + "${data.aws_ssm_parameter.qualys_username.arn}", + "${data.aws_ssm_parameter.qualys_password.arn}", "${data.aws_ssm_parameter.sixgill_client_secret.arn}", "${data.aws_ssm_parameter.lg_api_key.arn}", "${data.aws_ssm_parameter.lg_workspace_name.arn}", @@ -386,6 +389,12 @@ data "aws_ssm_parameter" "intelx_api_key" { name = var.ssm_intelx_api_key } data "aws_ssm_parameter" "xpanse_api_key" { name = var.ssm_xpanse_api_key } +data "aws_ssm_parameter" "whoisxml_api_key" { name = var.ssm_whoisxml_api_key } + +data "aws_ssm_parameter" "qualys_username" { name = var.ssm_qualys_username } + +data "aws_ssm_parameter" "qualys_password" { name = var.ssm_qualys_password } + data "aws_ssm_parameter" "xpanse_auth_id" { name = var.ssm_xpanse_auth_id } data "aws_ssm_parameter" "sixgill_client_secret" { name = var.ssm_sixgill_client_secret }