Move over the lz-infra branch and capture updates

cisagov · Mar 13, 2024 · cd25a90 · cd25a90
1 parent 2197a69
commit cd25a90
Show file tree

Hide file tree

Showing 79 changed files with 2,917 additions and 5,853 deletions.
diff --git a/backend/Dockerfile.worker b/backend/Dockerfile.worker
@@ -60,7 +60,7 @@ RUN apt remove dav1d && apt autoclean && apt autoremove
 
 # Install pe-source module
 # Sync the latest from cf-staging branch
-RUN git clone -b cf-source-staging https://github.com/cisagov/pe-reports.git && cd pe-reports && git checkout c9cbbd73b22ef38cabe1da6ba50aeb2dc0be4f99 && sed -i 's/"pandas == 1.1.5"/"pandas == 1.5.1"/g' setup.py && sed -i 's/psycopg2-binary == 2.9.3/psycopg2-binary == 2.9.5/g' setup.py && sed -i 's/psycopg2-binary == 2.9.3/psycopg2-binary == 2.9.5/g' setup_reports.py  && pip install .
+# RUN git clone -b cf-source-staging https://github.com/cisagov/pe-reports.git && cd pe-reports && git checkout c9cbbd73b22ef38cabe1da6ba50aeb2dc0be4f99 && sed -i 's/"pandas == 1.1.5"/"pandas == 1.5.1"/g' setup.py && sed -i 's/psycopg2-binary == 2.9.3/psycopg2-binary == 2.9.5/g' setup.py && sed -i 's/psycopg2-binary == 2.9.3/psycopg2-binary == 2.9.5/g' setup_reports.py  && pip install .
 # Python dependencies
 
 COPY worker/requirements.txt worker/requirements.txt

diff --git a/backend/env.yml b/backend/env.yml
@@ -1,4 +1,6 @@
----
+## added dev option for environment to remove the warning
+## from 'npx sls package' during backend 'test' github action
+## Warning: Invalid configuration encountered at 'provider.environment': must be object
 dev:
   DUMMY:
 
@@ -14,8 +16,6 @@ staging:
   PE_DB_PASSWORD: ${ssm:/crossfeed/staging/PE_DB_PASSWORD}
   SIXGILL_CLIENT_ID: ${ssm:/crossfeed/staging/SIXGILL_CLIENT_ID}
   SIXGILL_CLIENT_SECRET: ${ssm:/crossfeed/staging/SIXGILL_CLIENT_SECRET}
-  INTELX_API_KEY: ${ssm:/crossfeed/staging/INTELX_API_KEY}
-  HIBP_API_KEY: ${ssm:/crossfeed/staging/HIBP_API_KEY}
   PE_SHODAN_API_KEYS: ${ssm:/crossfeed/staging/PE_SHODAN_API_KEYS}
   JWT_SECRET: ${ssm:/crossfeed/staging/APP_JWT_SECRET}
   LOGIN_GOV_REDIRECT_URI: ${ssm:/crossfeed/staging/LOGIN_GOV_REDIRECT_URI}
@@ -32,33 +32,20 @@ staging:
   FARGATE_LOG_GROUP_NAME: 'crossfeed-staging-worker'
   CROSSFEED_SUPPORT_EMAIL_SENDER: '[email protected]'
   CROSSFEED_SUPPORT_EMAIL_REPLYTO: '[email protected]'
-  FRONTEND_DOMAIN: 'https://staging-cd.crossfeed.cyber.dhs.gov'
+  FRONTEND_DOMAIN: 'https://staging.crossfeed.cyber.dhs.gov'
   SLS_LAMBDA_PREFIX: '${self:service}-${self:provider.stage}'
   USE_COGNITO: 1
-  REACT_APP_USER_POOL_ID: us-east-1_uxiY8DOum
+  REACT_APP_USER_POOL_ID: ${ssm:/crossfeed/staging/USER_POOL_ID}
+  REACT_APP_USER_POOL_KEY: ${ssm(raw):/crossfeed/staging/USER_POOL_KEY}
   WORKER_USER_AGENT: ${ssm:/crossfeed/staging/WORKER_USER_AGENT}
   WORKER_SIGNATURE_PUBLIC_KEY: ${ssm:/crossfeed/staging/WORKER_SIGNATURE_PUBLIC_KEY}
   ELASTICSEARCH_ENDPOINT: ${ssm:/crossfeed/staging/ELASTICSEARCH_ENDPOINT}
   REACT_APP_TERMS_VERSION: ${ssm:/crossfeed/staging/REACT_APP_TERMS_VERSION}
-  REACT_APP_RANDOM_PASSWORD: ${ssm:/crossfeed/staging/REACT_APP_RANDOM_PASSWORD}
-  MATOMO_URL: http://matomo.crossfeed.local
+  MATOMO_URL: http://matomo.cfs.lz.us-cert.gov
   EXPORT_BUCKET_NAME: cisa-crossfeed-staging-exports
   PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL}
   REPORTS_BUCKET_NAME: cisa-crossfeed-staging-reports
-  CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-staging-cloudwatch
-  STAGE: staging
-  PE_CLUSTER_NAME: pe-staging-worker
-  SHODAN_QUEUE_URL: ${ssm:/crossfeed/staging/SHODAN_QUEUE_URL}
-  SHODAN_SERVICE_NAME: pe-staging-shodan
-  DNSTWIST_QUEUE_URL: ${ssm:/crossfeed/staging/DNSTWIST_QUEUE_URL}
-  DNSTWIST_SERVICE_NAME: pe-staging-dnstwist
-  HIBP_QUEUE_URL: ${ssm:/crossfeed/staging/HIBP_QUEUE_URL}
-  HIBP_SERVICE_NAME: pe-staging-hibp
-  INTELX_QUEUE_URL: ${ssm:/crossfeed/staging/INTELX_QUEUE_URL}
-  INTELX_SERVICE_NAME: pe-staging-intelx
-  CYBERSIXGILL_QUEUE_URL: ${ssm:/crossfeed/staging/CYBERSIXGILL_QUEUE_URL}
-  CYBERSIXGILL_SERVICE_NAME: pe-staging-cybersixgill
-  EMAIL_BUCKET_NAME: cisa-crossfeed-staging-html-email
+  VPC_ENDPOINT: ${ssm:/crossfeed/staging/BACKEND_VPC_ENDPOINT}
 
 prod:
   DB_DIALECT: 'postgres'
@@ -85,23 +72,21 @@ prod:
   FRONTEND_DOMAIN: 'https://crossfeed.cyber.dhs.gov'
   SLS_LAMBDA_PREFIX: '${self:service}-${self:provider.stage}'
   USE_COGNITO: 1
-  REACT_APP_USER_POOL_ID: us-east-1_MZgKoBmkN
+  REACT_APP_USER_POOL_ID: ${ssm:/crossfeed/prod/USER_POOL_ID}
+  REACT_APP_USER_POOL_KEY: ${ssm(raw):/crossfeed/prod/USER_POOL_KEY}
   WORKER_USER_AGENT: ${ssm:/crossfeed/prod/WORKER_USER_AGENT}
   WORKER_SIGNATURE_PUBLIC_KEY: ${ssm:/crossfeed/prod/WORKER_SIGNATURE_PUBLIC_KEY}
   ELASTICSEARCH_ENDPOINT: ${ssm:/crossfeed/prod/ELASTICSEARCH_ENDPOINT}
   REACT_APP_TERMS_VERSION: ${ssm:/crossfeed/prod/REACT_APP_TERMS_VERSION}
-  REACT_APP_RANDOM_PASSWORD: ${ssm:/crossfeed/prod/REACT_APP_RANDOM_PASSWORD}
-  MATOMO_URL: http://matomo.crossfeed.local
+  MATOMO_URL: http://matomo.cfs.lz.us-cert.gov
   EXPORT_BUCKET_NAME: cisa-crossfeed-prod-exports
   PE_API_URL: ${ssm:/crossfeed/prod/PE_API_URL}
   REPORTS_BUCKET_NAME: cisa-crossfeed-prod-reports
-  CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-prod-cloudwatch
-  STAGE: prod
-  PE_CLUSTER_NAME: pe-prod-worker
-  SHODAN_QUEUE_URL: ${ssm:/crossfeed/prod/SHODAN_QUEUE_URL}
-  SHODAN_SERVICE_NAME: pe-prod-shodan
-  EMAIL_BUCKET_NAME: cisa-crossfeed-staging-html-email
+  VPC_ENDPOINT: ${ssm:/crossfeed/prod/BACKEND_VPC_ENDPOINT}
 
+## added dev option for vpc to remove the warning
+## from 'npx sls package' during backend 'test' github action
+## Warning: Invalid configuration encountered at 'provider.vpc': must be object
 dev-vpc:
   securityGroupIds:
     - dummy

diff --git a/backend/serverless.yml b/backend/serverless.yml
@@ -1,4 +1,3 @@
----
 service: crossfeed
 
 frameworkVersion: '3'
@@ -17,18 +16,27 @@ custom:
 
 provider:
   name: aws
-  region: us-east-1
-  endpointType: REGIONAL
+  region: us-gov-east-1
+  endpointType: PRIVATE
   runtime: nodejs16.x
   timeout: 30
   stage: ${opt:stage, 'dev'}
   environment: ${file(env.yml):${self:provider.stage}, ''}
   vpc: ${file(env.yml):${self:provider.stage}-vpc, ''}
+  vpcEndpointIds:
+    - ${file(env.yml):${self:provider.stage}.VPC_ENDPOINT, ''}
   apiGateway:
     binaryMediaTypes:
       - 'image/*'
       - 'font/*'
     resourcePolicy:
+      - Effect: Deny
+        Principal: '*'
+        Action: 'execute-api:Invoke'
+        Resource: 'execute-api:/${self:provider.stage}/*/*'
+        Condition:
+          StringNotEquals:
+            'aws:sourceVpce': ${file(env.yml):${self:provider.stage}.VPC_ENDPOINT, ''}
       - Effect: Allow
         Principal: '*'
         Action: 'execute-api:Invoke'
@@ -40,122 +48,37 @@ provider:
   iam:
     role:
       statements:
-        # TODO: make the resources more specific. See Resource: '*' was
+        # TODO: make the resources more specific.
         - Effect: Allow
           Action:
             - lambda:InvokeAsync
             - lambda:InvokeFunction
-            - cognito-idp:AdminDisableUser
-            - cognito-idp:ListUsers
-            - cognito-idp:AdminSetUserPassword
-          Resource: "*"
+          Resource: '*'
         - Effect: Allow
           Action:
             - ecs:RunTask
             - ecs:ListTasks
-            - ecs:DescribeTasks
-            - ecs:DescribeServices
-            - ecs:UpdateService
             - iam:PassRole
+            - logs:GetLogEvents
           Resource: '*'
         - Effect: Allow
           Action:
             - ses:SendRawEmail
+            - ses:SendEmail
           Resource: '*'
         - Effect: Allow
           Action:
             - s3:GetObject
             - s3:GetObjectAcl
             - s3:PutObject
             - s3:PutObjectAcl
-            - s3:PutBucketAcl
-            - s3:GetBucketAcl
           Resource: '*'
-        - Effect: Allow
-          Action:
-            - sts:AssumeRole
-          Resource: '*'
-        - Effect: Allow
-          Action:
-            - sqs:ReceiveMessage
-            - sqs:DeleteMessage
-            - sqs:SendMessage
-            - sqs:GetQueueAttributes
-          Resource: '*'
-        - Effect: Allow
-          Action:
-            - logs:CreateExportTask
-            - logs:CreateLogStream
-            - logs:Describe*
-            - logs:Get*
-            - logs:List*
-            - logs:PutLogEvents
-            - logs:StartQuery
-            - logs:StopQuery
-            - logs:TestMetricFilter
-            - logs:FilterLogEvents
-            - logs:StartLiveTail
-            - logs:StopLiveTail
-          Resource: '*'
-        - Effect: Allow
-          Action:
-            - ssm:DescribeParameters
-            - ssm:GetParameter
-            - ssm:GetParameters
-            - ssm:GetParametersByPath
-            - ssm:PutParameter
-          Resource: '*'
-
-resources:
-  Resources:
-    WorkerControlQueue:
-      Type: AWS::SQS::Queue
-      Properties:
-        QueueName: ${self:provider.stage}-worker-control-queue
-        VisibilityTimeout: 300  # Should match or exceed function timeout
-        MaximumMessageSize: 262144  # 256 KB
-        MessageRetentionPeriod: 604800  # 7 days
-    ShodanQueue:
-      Type: AWS::SQS::Queue
-      Properties:
-        QueueName: ${self:provider.stage}-shodan-queue
-        VisibilityTimeout: 300
-        MaximumMessageSize: 262144  # 256 KB
-        MessageRetentionPeriod: 604800  # 7 days
-    DnstwistQueue:
-      Type: AWS::SQS::Queue
-      Properties:
-        QueueName: ${self:provider.stage}-dnstwist-queue
-        VisibilityTimeout: 300
-        MaximumMessageSize: 262144  # 256 KB
-        MessageRetentionPeriod: 604800  # 7 days
-    HibpQueue:
-      Type: AWS::SQS::Queue
-      Properties:
-        QueueName: ${self:provider.stage}-hibp-queue
-        VisibilityTimeout: 300
-        MaximumMessageSize: 262144  # 256 KB
-        MessageRetentionPeriod: 604800  # 7 days
-    IntelxQueue:
-      Type: AWS::SQS::Queue
-      Properties:
-        QueueName: ${self:provider.stage}-intelx-queue
-        VisibilityTimeout: 300
-        MaximumMessageSize: 262144  # 256 KB
-        MessageRetentionPeriod: 604800  # 7 days
-    CybersixgillQueue:
-      Type: AWS::SQS::Queue
-      Properties:
-        QueueName: ${self:provider.stage}-cybersixgill-queue
-        VisibilityTimeout: 300
-        MaximumMessageSize: 262144  # 256 KB
-        MessageRetentionPeriod: 604800  # 7 days
 
 functions:
   - ${file(./src/tasks/functions.yml)}
   - ${file(./src/api/functions.yml)}
 
 plugins:
+  - serverless-better-credentials
   - serverless-domain-manager
   - serverless-webpack
-  - serverless-dotenv-plugin