From d9e6a04ee8f34c06b0d7752469e0f89a639eb5b1 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 18 Jun 2024 09:31:12 -0400 Subject: [PATCH 1/5] Add xpanse to the SQS process --- backend/Dockerfile.pe | 6 +++--- backend/serverless.yml | 7 +++++++ backend/src/tasks/scanExecution.ts | 11 +++++++++-- backend/worker/generate_config.sh | 4 ++-- backend/worker/pe-worker-entry.sh | 2 ++ docs/src/documentation-pages/dev/quickstart.md | 12 +++++++++--- 6 files changed, 32 insertions(+), 10 deletions(-) diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe index eef90641..c51b91bd 100644 --- a/backend/Dockerfile.pe +++ b/backend/Dockerfile.pe @@ -26,9 +26,9 @@ RUN ./aws/install # Install pe-source module # Sync the latest from cf-staging branch -RUN git clone -b AL-staging-SQS https://github.com/cisagov/pe-reports.git && \ - cd pe-reports && \ - git checkout 6405a2041656152b176b5fc9b3becb5dc11a5f3e && \ +RUN git clone -b crossfeed-SQS https://github.com/cisagov/ATC-Framework.git && \ + cd ATC-Framework && \ + git checkout b077f19d6c2f7a72b474264e857465dcaeb56d1a && \ pip install . RUN python -m spacy download en_core_web_lg diff --git a/backend/serverless.yml b/backend/serverless.yml index 9c6fa74d..fccb2cd3 100644 --- a/backend/serverless.yml +++ b/backend/serverless.yml @@ -143,6 +143,13 @@ resources: VisibilityTimeout: 18000 # 5 hours MaximumMessageSize: 262144 # 256 KB MessageRetentionPeriod: 604800 # 7 days + XpanseQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: ${self:provider.stage}-xpanse-queue + VisibilityTimeout: 18000 # 5 hours + MaximumMessageSize: 262144 # 256 KB + MessageRetentionPeriod: 604800 # 7 days functions: - ${file(./src/tasks/functions.yml)} diff --git a/backend/src/tasks/scanExecution.ts b/backend/src/tasks/scanExecution.ts index c0ed64ff..94788dcf 100644 --- a/backend/src/tasks/scanExecution.ts +++ b/backend/src/tasks/scanExecution.ts @@ -5,7 +5,14 @@ import { integer } from 'aws-sdk/clients/cloudfront'; const ecs = new AWS.ECS(); let docker: any; const QUEUE_URL = process.env.QUEUE_URL!; -const SCAN_LIST = ['dnstwist', 'hibp', 'intelx', 'cybersixgill', 'shodan']; +const SCAN_LIST = [ + 'dnstwist', + 'hibp', + 'intelx', + 'cybersixgill', + 'shodan', + 'xpanse' +]; if (process.env.IS_LOCAL) { const Docker = require('dockerode'); @@ -211,7 +218,7 @@ export const handler: Handler = async (event) => { await startDesiredTasks(scanType, desiredCount); } else { console.log( - 'Shodan, DNSTwist, HIBP, IntelX, and Cybersixgill are the only script types available right now. Must be all lowercase.' + 'Shodan, DNSTwist, HIBP, IntelX, Xpanse, and Cybersixgill are the only script types available right now. Must be all lowercase.' ); } } catch (error) { diff --git a/backend/worker/generate_config.sh b/backend/worker/generate_config.sh index 6a461dbe..85acda82 100755 --- a/backend/worker/generate_config.sh +++ b/backend/worker/generate_config.sh @@ -1,7 +1,7 @@ #!/bin/bash # Generate database.ini -cat << EOF > pe-reports/src/pe_reports/data/database.ini +cat << EOF > ATC-Framework/src/pe_reports/data/database.ini [postgres] host=${DB_HOST} database=${PE_DB_NAME} @@ -52,6 +52,6 @@ pe_reports_path=$(pip show pe-reports | grep -E '^Location:' | awk '{print $2}') pe_reports_path="${pe_reports_path%/pe-reports}/pe_reports" # Copy database.ini to the module's installation directory -cp /app/pe-reports/src/pe_reports/data/database.ini "${pe_reports_path}/data/" +cp /app/ATC-Framework/src/pe_reports/data/database.ini "${pe_reports_path}/data/" exec "$@" diff --git a/backend/worker/pe-worker-entry.sh b/backend/worker/pe-worker-entry.sh index 0a4c2162..b3a9413e 100755 --- a/backend/worker/pe-worker-entry.sh +++ b/backend/worker/pe-worker-entry.sh @@ -62,6 +62,8 @@ while true; do COMMAND="pe-source intelx --org=$ORG --soc_med_included" elif [[ "$SERVICE_TYPE" = *"cybersixgill"* ]]; then COMMAND="pe-source cybersixgill --org=$ORG --soc_med_included" + elif [[ "$SERVICE_TYPE" = *"xpanse"* ]]; then + COMMAND="pe-source xpanse --org=$ORG" else echo "Unsupported SERVICE_TYPE: $SERVICE_TYPE" break diff --git a/docs/src/documentation-pages/dev/quickstart.md b/docs/src/documentation-pages/dev/quickstart.md index ebd06f22..2b420405 100644 --- a/docs/src/documentation-pages/dev/quickstart.md +++ b/docs/src/documentation-pages/dev/quickstart.md @@ -55,21 +55,27 @@ This quickstart describes the initial setup required to run an instance of Cross cp dev.env.example .env ``` -3. Generate the P&E DB Schema +3. Start application: + + ```bash + npm start + ``` + +4. Generate the P&E DB Schema: ```bash cd backend npm run pesyncdb ``` -4. Invoke scans by running below. You can edit the backend/src/tools/run-scanExecution.ts to run the desired scan type." +5. Invoke scans by running below. You can edit the backend/src/tools/run-scanExecution.ts to run the desired scan type." ```bash cd backend npm run scan-exec ``` -5. Observe logs in docker containers. +6. Observe logs in docker containers. ### Running tests From 2512ed758356ae1cc7eaf59f1c07fabdaf2a6e82 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 15 Jul 2024 16:07:29 -0400 Subject: [PATCH 2/5] Add api keys needed and make changes based on xpanse updates --- backend/Dockerfile.pe | 2 +- backend/src/tasks/scanExecution.ts | 5 ++++- backend/src/tools/run-scanExecution.ts | 4 ++-- backend/worker/generate_config.sh | 5 +++-- backend/worker/pe-worker-entry.sh | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe index e0767353..c410c43a 100644 --- a/backend/Dockerfile.pe +++ b/backend/Dockerfile.pe @@ -28,7 +28,7 @@ RUN ./aws/install # Sync the latest from cf-staging branch RUN git clone -b crossfeed-SQS https://github.com/cisagov/ATC-Framework.git && \ cd ATC-Framework && \ - git checkout a4036e94ad54876cb592fc966f57e7fd84750e9a && \ + git checkout 8a42ebdb378e187f8ae4b8e4dd91f8aa051b30da && \ pip install . RUN python -m spacy download en_core_web_lg diff --git a/backend/src/tasks/scanExecution.ts b/backend/src/tasks/scanExecution.ts index 94788dcf..082b39f3 100644 --- a/backend/src/tasks/scanExecution.ts +++ b/backend/src/tasks/scanExecution.ts @@ -155,7 +155,10 @@ async function startLocalContainers( `LG_API_KEY=${process.env.LG_API_KEY}`, `LG_WORKSPACE_NAME=${process.env.LG_WORKSPACE_NAME}`, `SERVICE_QUEUE_URL=${queueUrl}`, - `SERVICE_TYPE=${scanType}` + `SERVICE_TYPE=${scanType}`, + `PE_API_URL=${process.env.PE_API_URL}`, + `PE_API_KEY=${process.env.PE_API_KEY}`, + `CF_API_KEY=${process.env.CF_API_KEY}` ] } as any); await container.start(); diff --git a/backend/src/tools/run-scanExecution.ts b/backend/src/tools/run-scanExecution.ts index 86664858..64bbdf91 100644 --- a/backend/src/tools/run-scanExecution.ts +++ b/backend/src/tools/run-scanExecution.ts @@ -31,9 +31,9 @@ async function sendMessageToQueue(message, queue) { } // Simulate sending a message -const SCAN_TYPE = 'dnstwist'; +const SCAN_TYPE = 'xpanse'; const DESIRED_COUNT = 1; -const ORG_LIST = ['DHS', 'DOI']; +const ORG_LIST = ['National Science Foundation (NSF) - CISA']; const QUEUE = `staging-${SCAN_TYPE}-queue`; const API_KEY_LIST = ''; diff --git a/backend/worker/generate_config.sh b/backend/worker/generate_config.sh index 85acda82..68bbd2f2 100755 --- a/backend/worker/generate_config.sh +++ b/backend/worker/generate_config.sh @@ -16,8 +16,9 @@ key1=${PE_SHODAN_API_KEYS} key=${HIBP_API_KEY} [pe_api] -pe_api_key= -pe_api_url= +pe_api_key=${PE_API_KEY} +pe_api_url=https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/ +cf_api_key=${CF_API_KEY} [staging] [cyhy_mongo] diff --git a/backend/worker/pe-worker-entry.sh b/backend/worker/pe-worker-entry.sh index b3a9413e..9a270b44 100755 --- a/backend/worker/pe-worker-entry.sh +++ b/backend/worker/pe-worker-entry.sh @@ -63,7 +63,7 @@ while true; do elif [[ "$SERVICE_TYPE" = *"cybersixgill"* ]]; then COMMAND="pe-source cybersixgill --org=$ORG --soc_med_included" elif [[ "$SERVICE_TYPE" = *"xpanse"* ]]; then - COMMAND="pe-source xpanse --org=$ORG" + COMMAND="python3 ATC-Framework/src/pe_source/xpanse_alert_pull.py --org='$ORG'" else echo "Unsupported SERVICE_TYPE: $SERVICE_TYPE" break From 2ef6a80a45fd7551accbdb08253cba6da30dafee Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 15 Jul 2024 16:44:22 -0400 Subject: [PATCH 3/5] Use xpanse in the pe-source module instead of calling directly --- backend/Dockerfile.pe | 2 +- backend/worker/pe-worker-entry.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe index c410c43a..fa894f85 100644 --- a/backend/Dockerfile.pe +++ b/backend/Dockerfile.pe @@ -28,7 +28,7 @@ RUN ./aws/install # Sync the latest from cf-staging branch RUN git clone -b crossfeed-SQS https://github.com/cisagov/ATC-Framework.git && \ cd ATC-Framework && \ - git checkout 8a42ebdb378e187f8ae4b8e4dd91f8aa051b30da && \ + git checkout 995ec3c28de41c3bb9ac8381441d8214b7e87a1d && \ pip install . RUN python -m spacy download en_core_web_lg diff --git a/backend/worker/pe-worker-entry.sh b/backend/worker/pe-worker-entry.sh index 9a270b44..d2b1142b 100755 --- a/backend/worker/pe-worker-entry.sh +++ b/backend/worker/pe-worker-entry.sh @@ -63,7 +63,7 @@ while true; do elif [[ "$SERVICE_TYPE" = *"cybersixgill"* ]]; then COMMAND="pe-source cybersixgill --org=$ORG --soc_med_included" elif [[ "$SERVICE_TYPE" = *"xpanse"* ]]; then - COMMAND="python3 ATC-Framework/src/pe_source/xpanse_alert_pull.py --org='$ORG'" + COMMAND="pe-source xpanse --org='$ORG'" else echo "Unsupported SERVICE_TYPE: $SERVICE_TYPE" break From 66c781b09b774a74c716db8c533648a8d863d81d Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 15 Jul 2024 17:25:03 -0400 Subject: [PATCH 4/5] Add xpanse env variables --- backend/src/tasks/scanExecution.ts | 2 ++ backend/worker/generate_config.sh | 4 ++++ infrastructure/pe_worker.tf | 16 ++++++++++++++++ infrastructure/stage.tfvars | 2 ++ infrastructure/vars.tf | 12 ++++++++++++ infrastructure/worker.tf | 6 ++++++ 6 files changed, 42 insertions(+) diff --git a/backend/src/tasks/scanExecution.ts b/backend/src/tasks/scanExecution.ts index 082b39f3..2a53b14b 100644 --- a/backend/src/tasks/scanExecution.ts +++ b/backend/src/tasks/scanExecution.ts @@ -145,6 +145,8 @@ async function startLocalContainers( `SIXGILL_CLIENT_ID=${process.env.SIXGILL_CLIENT_ID}`, `SIXGILL_CLIENT_SECRET=${process.env.SIXGILL_CLIENT_SECRET}`, `INTELX_API_KEY=${process.env.INTELX_API_KEY}`, + `XPANSE_API_KEY=${process.env.XPANSE_API_KEY}`, + `XPANSE_AUTH_ID=${process.env.XPANSE_AUTH_ID}`, `PE_SHODAN_API_KEYS=${shodan_api_key}`, `WORKER_SIGNATURE_PUBLIC_KEY=${process.env.WORKER_SIGNATURE_PUBLIC_KEY}`, `WORKER_SIGNATURE_PRIVATE_KEY=${process.env.WORKER_SIGNATURE_PRIVATE_KEY}`, diff --git a/backend/worker/generate_config.sh b/backend/worker/generate_config.sh index 68bbd2f2..291f3c3a 100755 --- a/backend/worker/generate_config.sh +++ b/backend/worker/generate_config.sh @@ -43,6 +43,10 @@ api_key=${INTELX_API_KEY} [API_Client_secret] [API_WHOIS] +[xpanse] +api_key=${XPANSE_API_KEY} +auth_id=${XPANSE_AUTH_ID} + EOF diff --git a/infrastructure/pe_worker.tf b/infrastructure/pe_worker.tf index 0afae126..35eb4a82 100644 --- a/infrastructure/pe_worker.tf +++ b/infrastructure/pe_worker.tf @@ -129,6 +129,22 @@ resource "aws_ecs_task_definition" "pe_worker" { "name": "INTELX_API_KEY", "valueFrom": "${data.aws_ssm_parameter.intelx_api_key.arn}" }, + { + "name": "XPANSE_API_KEY", + "valueFrom": "${data.aws_ssm_parameter.xpanse_api_key.arn}" + }, + { + "name": "XPANSE_AUTH_ID", + "valueFrom": "${data.aws_ssm_parameter.xpanse_auth_id.arn}" + }, + { + "name": "PE_API_KEY", + "valueFrom": "${data.aws_ssm_parameter.pe_api_key.arn}" + }, + { + "name": "CF_API_KEY", + "valueFrom": "${data.aws_ssm_parameter.cf_api_key.arn}" + }, { "name": "LG_API_KEY", "valueFrom": "${data.aws_ssm_parameter.lg_api_key.arn}" diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars index e40e34f0..0fb68f0d 100644 --- a/infrastructure/stage.tfvars +++ b/infrastructure/stage.tfvars @@ -53,6 +53,8 @@ ssm_pe_shodan_api_keys = "/crossfeed/staging/PE_SHODAN_API_KEYS" ssm_sixgill_client_id = "/crossfeed/staging/SIXGILL_CLIENT_ID" ssm_sixgill_client_secret = "/crossfeed/staging/SIXGILL_CLIENT_SECRET" ssm_intelx_api_key = "/crossfeed/staging/INTELX_API_KEY" +ssm_xpanse_api_key = "/crossfeed/staging/XPANSE_API_KEY" +ssm_xpanse_auth_id = "/crossfeed/staging/XPANSE_AUTH_ID" ssm_lg_api_key = "/crossfeed/staging/LG_API_KEY" ssm_lg_workspace_name = "/crossfeed/staging/LG_WORKSPACE_NAME" ssm_pe_api_key = "/crossfeed/staging/PE_API_KEY" diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index f920dc56..187b50d3 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -327,6 +327,18 @@ variable "ssm_intelx_api_key" { default = "/crossfeed/staging/INTELX_API_KEY" } +variable "ssm_xpanse_api_key" { + description = "ssm_xpanse_api_key" + type = string + default = "/crossfeed/staging/XPANSE_API_KEY" +} + +variable "ssm_xpanse_auth_id" { + description = "ssm_xpanse_auth_id" + type = string + default = "/crossfeed/staging/XPANSE_AUTH_ID" +} + variable "ssm_lg_api_key" { description = "ssm_lg_api_key" diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf index 4cbdb6b4..d37161cb 100644 --- a/infrastructure/worker.tf +++ b/infrastructure/worker.tf @@ -86,6 +86,8 @@ resource "aws_iam_role_policy" "worker_task_execution_role_policy" { "${data.aws_ssm_parameter.pe_shodan_api_keys.arn}", "${data.aws_ssm_parameter.sixgill_client_id.arn}", "${data.aws_ssm_parameter.intelx_api_key.arn}", + "${data.aws_ssm_parameter.xpanse_api_key.arn}", + "${data.aws_ssm_parameter.xpanse_auth_id.arn}", "${data.aws_ssm_parameter.sixgill_client_secret.arn}", "${data.aws_ssm_parameter.lg_api_key.arn}", "${data.aws_ssm_parameter.lg_workspace_name.arn}", @@ -389,6 +391,10 @@ data "aws_ssm_parameter" "sixgill_client_id" { name = var.ssm_sixgill_client_id data "aws_ssm_parameter" "intelx_api_key" { name = var.ssm_intelx_api_key } +data "aws_ssm_parameter" "spanse_api_key" { name = var.ssm_xpanse_api_key } + +data "aws_ssm_parameter" "xpanse_auth_id" { name = var.ssm_xpanse_auth_id } + data "aws_ssm_parameter" "sixgill_client_secret" { name = var.ssm_sixgill_client_secret } data "aws_ssm_parameter" "pe_db_name" { name = var.ssm_pe_db_name } From 468e337934af1c0213b883def4f0f356d1d1b641 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 15 Jul 2024 17:29:27 -0400 Subject: [PATCH 5/5] fix typo --- infrastructure/worker.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf index d37161cb..390f1f5b 100644 --- a/infrastructure/worker.tf +++ b/infrastructure/worker.tf @@ -391,7 +391,7 @@ data "aws_ssm_parameter" "sixgill_client_id" { name = var.ssm_sixgill_client_id data "aws_ssm_parameter" "intelx_api_key" { name = var.ssm_intelx_api_key } -data "aws_ssm_parameter" "spanse_api_key" { name = var.ssm_xpanse_api_key } +data "aws_ssm_parameter" "xpanse_api_key" { name = var.ssm_xpanse_api_key } data "aws_ssm_parameter" "xpanse_auth_id" { name = var.ssm_xpanse_auth_id }