diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe index e0767353..fa894f85 100644 --- a/backend/Dockerfile.pe +++ b/backend/Dockerfile.pe @@ -28,7 +28,7 @@ RUN ./aws/install # Sync the latest from cf-staging branch RUN git clone -b crossfeed-SQS https://github.com/cisagov/ATC-Framework.git && \ cd ATC-Framework && \ - git checkout a4036e94ad54876cb592fc966f57e7fd84750e9a && \ + git checkout 995ec3c28de41c3bb9ac8381441d8214b7e87a1d && \ pip install . RUN python -m spacy download en_core_web_lg diff --git a/backend/serverless.yml b/backend/serverless.yml index 9c6fa74d..fccb2cd3 100644 --- a/backend/serverless.yml +++ b/backend/serverless.yml @@ -143,6 +143,13 @@ resources: VisibilityTimeout: 18000 # 5 hours MaximumMessageSize: 262144 # 256 KB MessageRetentionPeriod: 604800 # 7 days + XpanseQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: ${self:provider.stage}-xpanse-queue + VisibilityTimeout: 18000 # 5 hours + MaximumMessageSize: 262144 # 256 KB + MessageRetentionPeriod: 604800 # 7 days functions: - ${file(./src/tasks/functions.yml)} diff --git a/backend/src/tasks/scanExecution.ts b/backend/src/tasks/scanExecution.ts index c0ed64ff..2a53b14b 100644 --- a/backend/src/tasks/scanExecution.ts +++ b/backend/src/tasks/scanExecution.ts @@ -5,7 +5,14 @@ import { integer } from 'aws-sdk/clients/cloudfront'; const ecs = new AWS.ECS(); let docker: any; const QUEUE_URL = process.env.QUEUE_URL!; -const SCAN_LIST = ['dnstwist', 'hibp', 'intelx', 'cybersixgill', 'shodan']; +const SCAN_LIST = [ + 'dnstwist', + 'hibp', + 'intelx', + 'cybersixgill', + 'shodan', + 'xpanse' +]; if (process.env.IS_LOCAL) { const Docker = require('dockerode'); @@ -138,6 +145,8 @@ async function startLocalContainers( `SIXGILL_CLIENT_ID=${process.env.SIXGILL_CLIENT_ID}`, `SIXGILL_CLIENT_SECRET=${process.env.SIXGILL_CLIENT_SECRET}`, `INTELX_API_KEY=${process.env.INTELX_API_KEY}`, + `XPANSE_API_KEY=${process.env.XPANSE_API_KEY}`, + `XPANSE_AUTH_ID=${process.env.XPANSE_AUTH_ID}`, `PE_SHODAN_API_KEYS=${shodan_api_key}`, `WORKER_SIGNATURE_PUBLIC_KEY=${process.env.WORKER_SIGNATURE_PUBLIC_KEY}`, `WORKER_SIGNATURE_PRIVATE_KEY=${process.env.WORKER_SIGNATURE_PRIVATE_KEY}`, @@ -148,7 +157,10 @@ async function startLocalContainers( `LG_API_KEY=${process.env.LG_API_KEY}`, `LG_WORKSPACE_NAME=${process.env.LG_WORKSPACE_NAME}`, `SERVICE_QUEUE_URL=${queueUrl}`, - `SERVICE_TYPE=${scanType}` + `SERVICE_TYPE=${scanType}`, + `PE_API_URL=${process.env.PE_API_URL}`, + `PE_API_KEY=${process.env.PE_API_KEY}`, + `CF_API_KEY=${process.env.CF_API_KEY}` ] } as any); await container.start(); @@ -211,7 +223,7 @@ export const handler: Handler = async (event) => { await startDesiredTasks(scanType, desiredCount); } else { console.log( - 'Shodan, DNSTwist, HIBP, IntelX, and Cybersixgill are the only script types available right now. Must be all lowercase.' + 'Shodan, DNSTwist, HIBP, IntelX, Xpanse, and Cybersixgill are the only script types available right now. Must be all lowercase.' ); } } catch (error) { diff --git a/backend/src/tools/run-scanExecution.ts b/backend/src/tools/run-scanExecution.ts index 86664858..64bbdf91 100644 --- a/backend/src/tools/run-scanExecution.ts +++ b/backend/src/tools/run-scanExecution.ts @@ -31,9 +31,9 @@ async function sendMessageToQueue(message, queue) { } // Simulate sending a message -const SCAN_TYPE = 'dnstwist'; +const SCAN_TYPE = 'xpanse'; const DESIRED_COUNT = 1; -const ORG_LIST = ['DHS', 'DOI']; +const ORG_LIST = ['National Science Foundation (NSF) - CISA']; const QUEUE = `staging-${SCAN_TYPE}-queue`; const API_KEY_LIST = ''; diff --git a/backend/worker/generate_config.sh b/backend/worker/generate_config.sh index 85acda82..291f3c3a 100755 --- a/backend/worker/generate_config.sh +++ b/backend/worker/generate_config.sh @@ -16,8 +16,9 @@ key1=${PE_SHODAN_API_KEYS} key=${HIBP_API_KEY} [pe_api] -pe_api_key= -pe_api_url= +pe_api_key=${PE_API_KEY} +pe_api_url=https://api.staging-cd.crossfeed.cyber.dhs.gov/pe/apiv1/ +cf_api_key=${CF_API_KEY} [staging] [cyhy_mongo] @@ -42,6 +43,10 @@ api_key=${INTELX_API_KEY} [API_Client_secret] [API_WHOIS] +[xpanse] +api_key=${XPANSE_API_KEY} +auth_id=${XPANSE_AUTH_ID} + EOF diff --git a/backend/worker/pe-worker-entry.sh b/backend/worker/pe-worker-entry.sh index 0a4c2162..d2b1142b 100755 --- a/backend/worker/pe-worker-entry.sh +++ b/backend/worker/pe-worker-entry.sh @@ -62,6 +62,8 @@ while true; do COMMAND="pe-source intelx --org=$ORG --soc_med_included" elif [[ "$SERVICE_TYPE" = *"cybersixgill"* ]]; then COMMAND="pe-source cybersixgill --org=$ORG --soc_med_included" + elif [[ "$SERVICE_TYPE" = *"xpanse"* ]]; then + COMMAND="pe-source xpanse --org='$ORG'" else echo "Unsupported SERVICE_TYPE: $SERVICE_TYPE" break diff --git a/docs/src/documentation-pages/dev/quickstart.md b/docs/src/documentation-pages/dev/quickstart.md index ebd06f22..2b420405 100644 --- a/docs/src/documentation-pages/dev/quickstart.md +++ b/docs/src/documentation-pages/dev/quickstart.md @@ -55,21 +55,27 @@ This quickstart describes the initial setup required to run an instance of Cross cp dev.env.example .env ``` -3. Generate the P&E DB Schema +3. Start application: + + ```bash + npm start + ``` + +4. Generate the P&E DB Schema: ```bash cd backend npm run pesyncdb ``` -4. Invoke scans by running below. You can edit the backend/src/tools/run-scanExecution.ts to run the desired scan type." +5. Invoke scans by running below. You can edit the backend/src/tools/run-scanExecution.ts to run the desired scan type." ```bash cd backend npm run scan-exec ``` -5. Observe logs in docker containers. +6. Observe logs in docker containers. ### Running tests diff --git a/infrastructure/pe_worker.tf b/infrastructure/pe_worker.tf index 0afae126..35eb4a82 100644 --- a/infrastructure/pe_worker.tf +++ b/infrastructure/pe_worker.tf @@ -129,6 +129,22 @@ resource "aws_ecs_task_definition" "pe_worker" { "name": "INTELX_API_KEY", "valueFrom": "${data.aws_ssm_parameter.intelx_api_key.arn}" }, + { + "name": "XPANSE_API_KEY", + "valueFrom": "${data.aws_ssm_parameter.xpanse_api_key.arn}" + }, + { + "name": "XPANSE_AUTH_ID", + "valueFrom": "${data.aws_ssm_parameter.xpanse_auth_id.arn}" + }, + { + "name": "PE_API_KEY", + "valueFrom": "${data.aws_ssm_parameter.pe_api_key.arn}" + }, + { + "name": "CF_API_KEY", + "valueFrom": "${data.aws_ssm_parameter.cf_api_key.arn}" + }, { "name": "LG_API_KEY", "valueFrom": "${data.aws_ssm_parameter.lg_api_key.arn}" diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars index e40e34f0..0fb68f0d 100644 --- a/infrastructure/stage.tfvars +++ b/infrastructure/stage.tfvars @@ -53,6 +53,8 @@ ssm_pe_shodan_api_keys = "/crossfeed/staging/PE_SHODAN_API_KEYS" ssm_sixgill_client_id = "/crossfeed/staging/SIXGILL_CLIENT_ID" ssm_sixgill_client_secret = "/crossfeed/staging/SIXGILL_CLIENT_SECRET" ssm_intelx_api_key = "/crossfeed/staging/INTELX_API_KEY" +ssm_xpanse_api_key = "/crossfeed/staging/XPANSE_API_KEY" +ssm_xpanse_auth_id = "/crossfeed/staging/XPANSE_AUTH_ID" ssm_lg_api_key = "/crossfeed/staging/LG_API_KEY" ssm_lg_workspace_name = "/crossfeed/staging/LG_WORKSPACE_NAME" ssm_pe_api_key = "/crossfeed/staging/PE_API_KEY" diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index f920dc56..187b50d3 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -327,6 +327,18 @@ variable "ssm_intelx_api_key" { default = "/crossfeed/staging/INTELX_API_KEY" } +variable "ssm_xpanse_api_key" { + description = "ssm_xpanse_api_key" + type = string + default = "/crossfeed/staging/XPANSE_API_KEY" +} + +variable "ssm_xpanse_auth_id" { + description = "ssm_xpanse_auth_id" + type = string + default = "/crossfeed/staging/XPANSE_AUTH_ID" +} + variable "ssm_lg_api_key" { description = "ssm_lg_api_key" diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf index 4cbdb6b4..390f1f5b 100644 --- a/infrastructure/worker.tf +++ b/infrastructure/worker.tf @@ -86,6 +86,8 @@ resource "aws_iam_role_policy" "worker_task_execution_role_policy" { "${data.aws_ssm_parameter.pe_shodan_api_keys.arn}", "${data.aws_ssm_parameter.sixgill_client_id.arn}", "${data.aws_ssm_parameter.intelx_api_key.arn}", + "${data.aws_ssm_parameter.xpanse_api_key.arn}", + "${data.aws_ssm_parameter.xpanse_auth_id.arn}", "${data.aws_ssm_parameter.sixgill_client_secret.arn}", "${data.aws_ssm_parameter.lg_api_key.arn}", "${data.aws_ssm_parameter.lg_workspace_name.arn}", @@ -389,6 +391,10 @@ data "aws_ssm_parameter" "sixgill_client_id" { name = var.ssm_sixgill_client_id data "aws_ssm_parameter" "intelx_api_key" { name = var.ssm_intelx_api_key } +data "aws_ssm_parameter" "xpanse_api_key" { name = var.ssm_xpanse_api_key } + +data "aws_ssm_parameter" "xpanse_auth_id" { name = var.ssm_xpanse_auth_id } + data "aws_ssm_parameter" "sixgill_client_secret" { name = var.ssm_sixgill_client_secret } data "aws_ssm_parameter" "pe_db_name" { name = var.ssm_pe_db_name }