Merge branch 'integration' into JD-add-request-field-to-response

.github/workflows/backend.yml

@@ -117,6 +117,26 @@ jobs:
           DB_USERNAME: crossfeed
           DB_PASSWORD: password
 
+  build_worker:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+      - name: Restore npm cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
+          restore-keys: ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: npm ci
+      - name: Build worker container
+        run: npm run build-worker
+        working-directory: ./backend
+
   deploy_staging:
     needs: [lint, test, test_worker]
     runs-on: ubuntu-latest
@@ -208,12 +228,19 @@ jobs:
           SLS_DEBUG: '*'
 
       - name: Deploy backend
-        run: npx sls deploy --stage=integration --force
+        run: npx sls deploy --stage=integration
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           SLS_DEBUG: '*'
 
+      - name: Deploy worker
+        run: npm run deploy-worker-integration
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
       - name: Run syncdb
         run: |
           aws lambda invoke --function-name crossfeed-integration-syncdb \

backend/Makefile

@@ -1,9 +1,24 @@
 .PHONY: syncdb
+.PHONY: populate-cache
 
 # Synchronize and populate the database
 syncdb:
 	docker compose exec backend python manage.py syncdb
 
+syncdb-populate:
+	docker compose exec backend python manage.py syncdb --populate
+
+# Pytest
 pytest:
 	docker compose exec backend python manage.py makemigrations
 	docker compose exec backend pytest --no-migrations --reuse-db $(FILE)
+
+# Redis lambdas
+populate-cache:
+	docker compose exec backend python manage.py populate_services_cache
+	docker compose exec backend python manage.py populate_ports_cache
+	docker compose exec backend python manage.py populate_vulns_cache
+	docker compose exec backend python manage.py populate_most_common_vulns_cache
+	docker compose exec backend python manage.py populate_latest_vulns_cache
+	docker compose exec backend python manage.py populate_severity_count_cache
+	docker compose exec backend python manage.py populate_by_orgs_cache

backend/env.yml

@@ -137,6 +137,7 @@ integration:
   REACT_APP_COGNITO_CLIENT_ID: 6pohrlhoa20675c0rds6tcsk62
   REACT_APP_COGNITO_USER_POOL_ID: us-east-1_AuTPyIYDK
   REACT_APP_COGNITO_CALLBACK_URL: https://integration.crossfeed.cyber.dhs.gov/okta-callback
+  ELASTICACHE_ENDPOINT: ${ssm:/crossfeed/integration/ELASTICACHE_ENDPOINT}
 
 staging:
   REGION: us-gov-east-1
@@ -198,6 +199,7 @@ staging:
   MDL_USERNAME: ${ssm:/crossfeed/staging/MDL_USERNAME}
   MDL_PASSWORD: ${ssm:/crossfeed/staging/MDL_PASSWORD}
   MDL_NAME: ${ssm:/crossfeed/staging/MDL_NAME}
+  ELASTICACHE_ENDPOINT: ${ssm:/crossfeed/staging/ELASTICACHE_ENDPOINT}
 
 prod:
   REGION: us-gov-east-1

backend/package.json

@@ -13,7 +13,7 @@
     "build-worker": "sh ./tools/build-worker.sh",
     "codegen": "ts-node src/tools/generate-types.ts",
     "control-queue": "docker compose exec -T backend npx ts-node src/tools/consumeControlQueue.ts",
-    "deploy-worker-prod": "./tools/deploy-worker.sh crossfeed-prod-worker",
+    "deploy-worker-integration": "./tools/deploy-worker.sh crossfeed-integration-worker",
     "deploy-worker-staging": "./tools/deploy-worker.sh",
     "lint": "eslint '**/*.{ts,tsx,js,jsx}'",
     "lint:fix": "eslint '**/*.{ts,tsx,js,jsx}' --fix",

backend/requirements.txt

@@ -87,4 +87,4 @@ uvicorn==0.30.1
 uvloop==0.21.0
 watchfiles==1.0.0
 websockets==14.1
-yarl==1.18.3
+yarl==1.18.3

backend/src/tasks/functions.yml

@@ -48,124 +48,86 @@ updateScanTaskStatus:
             clusterArn:
               - ${file(env.yml):${self:provider.stage}-ecs-cluster, ''}
 
-# populateServiceStatsElasticache:
-#   handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_ServicesStatscache
-#   runtime: python3.11
-#   environment:
-#     DJANGO_SETTINGS_MODULE: xfd_django.settings
-#     REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
-#   events:
-#     - schedule:
-#         rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
-#         enabled: true
-#   package:
-#     include:
-#       - src/**
-#       - requirements.txt
-#     exclude:
-#       - node_modules/**
-
-# populatePortStatsElasticache:
-#   handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_PortsStatscache
-#   runtime: python3.11
-#   environment:
-#     DJANGO_SETTINGS_MODULE: xfd_django.settings
-#     REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
-#   events:
-#     - schedule:
-#         rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
-#         enabled: true
-#   package:
-#     include:
-#       - src/**
-#       - requirements.txt
-#     exclude:
-#       - node_modules/**
-
-# populateNumVulnsStatsElasticache:
-#   handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_NumVulnerabilitiesStatscache
-#   runtime: python3.11
-#   environment:
-#     DJANGO_SETTINGS_MODULE: xfd_django.settings
-#     REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
-#   events:
-#     - schedule:
-#         rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
-#         enabled: true
-#   package:
-#     include:
-#       - src/**
-#       - requirements.txt
-#     exclude:
-#       - node_modules/**
-
-# populateLatestVulnsStatsElasticache:
-#   handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_LatestVulnerabilitiesCache
-#   runtime: python3.11
-#   environment:
-#     DJANGO_SETTINGS_MODULE: xfd_django.settings
-#     REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
-#   events:
-#     - schedule:
-#         rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
-#         enabled: true
-#   package:
-#     include:
-#       - src/**
-#       - requirements.txt
-#     exclude:
-#       - node_modules/**
-
-# populateMostCommonVulnerabilityElasticache:
-#   handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_MostCommonVulnerabilitiesCache
-
-#   runtime: python3.11
-#   environment:
-#     DJANGO_SETTINGS_MODULE: xfd_django.settings
-#     REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
-#   events:
-#     - schedule:
-#         rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
-#         enabled: true
-#   package:
-#     include:
-#       - src/**
-#       - requirements.txt
-#     exclude:
-#       - node_modules/**
-
-# populateSeverityCountsCache:
-#   handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_SeverityCountsCache
-
-#   runtime: python3.11
-#   environment:
-#     DJANGO_SETTINGS_MODULE: xfd_django.settings
-#     REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
-#   events:
-#     - schedule:
-#         rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
-#         enabled: true
-#   package:
-#     include:
-#       - src/**
-#       - requirements.txt
-#     exclude:
-#       - node_modules/**
-
-# populateByOrgCache:
-#   handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_VulnerabilitiesByOrgCache
-
-#   runtime: python3.11
-#   environment:
-#     DJANGO_SETTINGS_MODULE: xfd_django.settings
-#     REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
-#   events:
-#     - schedule:
-#         rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
-#         enabled: true
-#   package:
-#     include:
-#       - src/**
-#       - requirements.txt
-#     exclude:
-#       - node_modules/**
+serviceStatsCache:
+  handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_services_cache
+  events:
+    - schedule:
+        rate: cron(0 0 * * ? *)  # This triggers the function every day it midnight
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**
+
+portStatsCache:
+  handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_ports_cache
+  events:
+    - schedule:
+        rate: cron(0 0 * * ? *)  # This triggers the function every day it midnight
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**
+
+numVulnsCache:
+  handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_num_vulns_cache
+  events:
+    - schedule:
+        rate: cron(0 0 * * ? *)  # This triggers the function every day it midnight
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**
+
+latestVulnsCache:
+  handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_latest_vulns_cache
+  events:
+    - schedule:
+        rate: cron(0 0 * * ? *)  # This triggers the function every day it midnight
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**
+
+mostCommonVulnsCache:
+  handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_most_common_vulns_cache
+  events:
+    - schedule:
+        rate: cron(0 0 * * ? *)  # This triggers the function every day it midnight
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**
+
+severityCache:
+  handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_severity_cache
+  events:
+    - schedule:
+        rate: cron(0 0 * * ? *)  # This triggers the function every day it midnight
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**
+
+byOrgCache:
+  handler: src/xfd_django/xfd_api/tasks/elasticache_tasks.populate_by_org_cache
+  events:
+    - schedule:
+        rate: cron(0 0 * * ? *)  # This triggers the function every day it midnight
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**