Merge branch 'develop' into SQS-fixes

.bandit.yml

@@ -6,8 +6,6 @@
 # If `tests` is empty, all tests are considered included.
 
 tests:
-# - B101
-# - B102
 
 skips:
-# - B101 # skip "assert used" check since assertions are required in pytests
+  - B101  # skip "assert used" check since assertions are required in pytests

.dockerignore

@@ -11,4 +11,4 @@ nvd-dump
 minio-data
 **/node_modules
 **/.cache
-./docs/node_modules
+./docs/node_modules

.github/workflows/backend.yml

@@ -111,6 +111,8 @@ jobs:
         uses: actions/[email protected]
         with:
           python-version: '3.10'
+      - name: Copy .env file
+        run: cp ../dev.env.example .env
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
@@ -187,6 +189,15 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
+      - name: Run syncmdl
+        run: |
+          aws lambda invoke --function-name crossfeed-staging-syncmdl \
+          --region us-east-1 /dev/stdout
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
   deploy_prod:
     needs: [build_worker, lint, test, test_python]
     runs-on: ubuntu-latest
@@ -236,3 +247,12 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Run syncmdl
+        run: |
+          aws lambda invoke --function-name crossfeed-prod-syncmdl --region us-east-1 \
+          /dev/stdout
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/build.yml

@@ -101,6 +101,8 @@ jobs:
           pip install --upgrade --requirement requirements-test.txt
       - name: Set up pre-commit hook environments
         run: pre-commit install-hooks
+      - name: Create .env file needed by docker-compose-check pre-commit hook
+        run: cp dev.env.example .env
       - name: Run pre-commit on all files
         run: pre-commit run --all-files
       - name: Setup tmate debug session

.gitignore

@@ -2,6 +2,10 @@
 # Files already tracked by Git are not affected.
 # See: https://git-scm.com/docs/gitignore
 
+# python
+__pycache__
+.mypy_cache
+.python-version
 
 # terraform
 .terraform
@@ -50,4 +54,3 @@ minio-data
 infrastructure/lambdas/security_headers.zip
 *.hcl
 .iac-data
-

.pre-commit-config.yaml

@@ -16,6 +16,7 @@ repos:
         args:
           - --allow-missing-credentials
       - id: detect-private-key
+        exclude: dev.env.example
       - id: end-of-file-fixer
         exclude: files/(issue|motd)
       - id: mixed-line-ending
@@ -26,6 +27,7 @@ repos:
           - --autofix
       - id: requirements-txt-fixer
       - id: trailing-whitespace
+        exclude: .*.snap
 
   # Text file hooks
   - repo: https://github.com/igorshubovych/markdownlint-cli
@@ -103,6 +105,8 @@ repos:
     rev: v1.5.1
     hooks:
       - id: mypy
+        additional_dependencies:
+          - types-requests
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.10.1
     hooks:

Dockerfile.docs

@@ -21,4 +21,4 @@ COPY ./docs .
 # Configure port used by Gatsby
 ENV INTERNAL_STATUS_PORT=44475
 
-CMD npm run codegen; npm run develop -- -H 0.0.0.0 --port 4000
+CMD npm run codegen; npm run develop -- -H 0.0.0.0 --port 4000

LICENSE

@@ -113,4 +113,4 @@ party to this document and has no duty or obligation with respect to this
 CC0 or use of the Work.
 
 For more information, please see
-<http://creativecommons.org/publicdomain/zero/1.0/>
+<http://creativecommons.org/publicdomain/zero/1.0/>

backend/.dockerignore

@@ -4,4 +4,4 @@ node_modules
 dist
 Dockerfile*
 nvd-dump
-coverage
+coverage

backend/.npmrc

@@ -1 +1 @@
-engine-strict=true
+engine-strict=true

backend/.snyk

@@ -4,9 +4,9 @@ version: v1.22.1
 
 # ignores vulnerabilities until expiry date; change duration by modifying expiry date
 
-ignore: 
+ignore:
     # ignore scrapy 2.x.x for 6 months.
-    SNYK-PYTHON-SCRAPY-40690: 
+    SNYK-PYTHON-SCRAPY-40690:
         - '*':
             reason: No fix available up to version 2.11.0
             expires: 2024-06-01T00:00:00.000Z

backend/Dockerfile.pe

@@ -33,9 +33,9 @@ RUN git clone -b AL-staging-SQS https://github.com/cisagov/pe-reports.git && \
 
 RUN python -m spacy download en_core_web_lg
 
-# Create database.ini 
-RUN echo "[database]" > database.ini \ 
-    && echo "user=$(cat db_user.txt)" >> database.ini \ 
+# Create database.ini
+RUN echo "[database]" > database.ini \
+    && echo "user=$(cat db_user.txt)" >> database.ini \
     && echo "password=$(cat db_password.txt)" >> database.ini
 
 COPY worker worker

backend/Dockerfile.worker

@@ -26,7 +26,7 @@ WORKDIR /app
 
 RUN apk add --no-cache curl unzip musl-dev
 
-RUN curl -4LO http://github.com/Findomain/Findomain/releases/latest/download/findomain-linux.zip 
+RUN curl -4LO http://github.com/Findomain/Findomain/releases/latest/download/findomain-linux.zip
 RUN unzip findomain-linux.zip && chmod +x findomain && cp findomain /usr/bin/findomain
 
 RUN go mod init crossfeed-worker

backend/env.yml

@@ -9,6 +9,9 @@ staging:
   DB_NAME: ${ssm:/crossfeed/staging/DATABASE_NAME}
   DB_USERNAME: ${ssm:/crossfeed/staging/DATABASE_USER}
   DB_PASSWORD: ${ssm:/crossfeed/staging/DATABASE_PASSWORD}
+  MDL_USERNAME: ${ssm:/crossfeed/staging/MDL_USERNAME}
+  MDL_PASSWORD: ${ssm:/crossfeed/staging/MDL_PASSWORD}
+  MDL_NAME: ${ssm:/crossfeed/staging/MDL_NAME}
   PE_DB_NAME: ${ssm:/crossfeed/staging/PE_DB_NAME}
   PE_DB_USERNAME: ${ssm:/crossfeed/staging/PE_DB_USERNAME}
   PE_DB_PASSWORD: ${ssm:/crossfeed/staging/PE_DB_PASSWORD}
@@ -63,6 +66,9 @@ prod:
   DB_NAME: ${ssm:/crossfeed/prod/DATABASE_NAME}
   DB_USERNAME: ${ssm:/crossfeed/prod/DATABASE_USER}
   DB_PASSWORD: ${ssm:/crossfeed/prod/DATABASE_PASSWORD}
+  MDL_USERNAME: ${ssm:/crossfeed/prod/MDL_USERNAME}
+  MDL_PASSWORD: ${ssm:/crossfeed/prod/MDL_PASSWORD}
+  MDL_NAME: ${ssm:/crossfeed/prod/MDL_NAME}
   JWT_SECRET: ${ssm:/crossfeed/prod/APP_JWT_SECRET}
   LOGIN_GOV_REDIRECT_URI: ${ssm:/crossfeed/prod/LOGIN_GOV_REDIRECT_URI}
   LOGIN_GOV_BASE_URL: ${ssm:/crossfeed/prod/LOGIN_GOV_BASE_URL}