Merge branch 'main' into experiment/migrate-to-ruff

.devcontainer/docker-compose.yml

@@ -20,7 +20,7 @@ services:
       - db
 
   db:
-    image: postgres:11.20-bullseye@sha256:98fac4e8dc6fb58a75f2be563e876842f53db5baadb0d98abdd3205a20f6e6eb
+    image: postgres:11.22-bullseye@sha256:c886a3236b3d11abc302e64309186c90a69b49e53ccff23fd8c8b057b5b4bce9
     volumes:
       - ./initdb:/docker-entrypoint-initdb.d
     restart: always
@@ -38,7 +38,7 @@ services:
       - "5432:5432"
 
   redis:
-    image: redis:6.2@sha256:9e75c88539241ad7f61bc9c39ea4913b354064b8a75ca5fc40e1cef41b645bc0
+    image: redis:6.2@sha256:d4948d011cc38e94f0aafb8f9a60309bd93034e07d10e0767af534512cf012a9
     restart: always
     command: redis-server --port 6380
     ports:

.devcontainer/scripts/notify-dev-entrypoint.sh

@@ -25,6 +25,7 @@ echo -e "complete -F __start_kubectl k" >> ~/.zshrc
 
 # Smoke test
 # requires adding files .env_staging and .env_prod to the root of the project
+echo -e "alias smoke-local='cd /workspace && cp .env_smoke_local tests_smoke/.env && poetry run make smoke-test-local'" >> ~/.zshrc
 echo -e "alias smoke-staging='cd /workspace && cp .env_smoke_staging tests_smoke/.env && poetry run make smoke-test'" >> ~/.zshrc
 echo -e "alias smoke-prod='cd /workspace && cp .env_smoke_prod tests_smoke/.env && poetry run make smoke-test'" >> ~/.zshrc
 

.env.example

@@ -19,3 +19,7 @@ AWS_PINPOINT_REGION=us-west-2
 AWS_EMF_ENVIRONMENT=local
 
 CONTACT_FORM_EMAIL_ADDRESS = ""
+
+AWS_PINPOINT_SC_POOL_ID=
+AWS_PINPOINT_SC_TEMPLATE_IDS=
+AWS_PINPOINT_DEFAULT_POOL_ID=

.github/workflows/codeql.yml

@@ -27,15 +27,15 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@2f93e4319b2f04a2efc38fa7f78bd681bc3f7b2f # v2.23.2
+        uses: github/codeql-action/init@ffd3158cb9024ebd018dbf20756f28befbd168c7 # v2.24.10
         with:
           languages: ${{ matrix.language }}
           queries: +security-and-quality
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@2f93e4319b2f04a2efc38fa7f78bd681bc3f7b2f # v2.23.2
+        uses: github/codeql-action/autobuild@ffd3158cb9024ebd018dbf20756f28befbd168c7 # v2.24.10
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@2f93e4319b2f04a2efc38fa7f78bd681bc3f7b2f # v2.23.2
+        uses: github/codeql-action/analyze@ffd3158cb9024ebd018dbf20756f28befbd168c7 # v2.24.10
         with:
           category: "/language:${{ matrix.language }}"

.github/workflows/docker.yaml

@@ -9,6 +9,7 @@ env:
   DOCKER_ORG: public.ecr.aws/v6b8u5o6
   DOCKER_SLUG: public.ecr.aws/v6b8u5o6/notify-api
   KUBECTL_VERSION: '1.23.6'
+  WORKFLOW_PAT: ${{ secrets.WORKFLOW_GITHUB_PAT }}
 
 permissions:
   id-token: write   # This is required for requesting the OIDC JWT
@@ -26,21 +27,14 @@ jobs:
         unzip -q awscliv2.zip
         sudo ./aws/install --update
         aws --version
-    - name: Install kubectl
-      run: |
-        curl -LO https://storage.googleapis.com/kubernetes-release/release/v$KUBECTL_VERSION/bin/linux/amd64/kubectl
-        chmod +x ./kubectl
-        sudo mv ./kubectl /usr/local/bin/kubectl
-        kubectl version --client
-        mkdir -p $HOME/.kube
 
     - name: Configure credentials to CDS public ECR using OIDC
       uses: aws-actions/configure-aws-credentials@master
       with:
         role-to-assume: arn:aws:iam::283582579564:role/notification-api-apply
         role-session-name: NotifyApiGitHubActions
         aws-region: "us-east-1"
-
+  
     - name: Login to ECR
       id: login-ecr
       uses: aws-actions/amazon-ecr-login@5a88a04c91d5c6f97aae0d9be790e64d9b1d47b7 # v1.7.1
@@ -56,43 +50,14 @@ jobs:
         -t $DOCKER_SLUG:${GITHUB_SHA::7} \
         -t $DOCKER_SLUG:latest \
         -f ci/Dockerfile .
+
     - name: Publish
       run: |
         docker push $DOCKER_SLUG:latest && docker push $DOCKER_SLUG:${GITHUB_SHA::7}
 
-    - name: Configure credentials to Notify account using OIDC
-      uses: aws-actions/configure-aws-credentials@master
-      with:
-        role-to-assume: arn:aws:iam::239043911459:role/notification-api-apply
-        role-session-name: NotifyApiGitHubActions
-        aws-region: "ca-central-1"
-
-    - name: Get Kubernetes configuration
-      run: |
-        aws eks --region $AWS_REGION update-kubeconfig --name notification-canada-ca-staging-eks-cluster --kubeconfig $HOME/.kube/config
-    - name: Update images in staging
+    - name: Rollout in Kubernetes
       run: |
-        kubectl set image deployment.apps/api api=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-beat celery-beat=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-sms celery-sms=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-primary celery-primary=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-scalable celery-scalable=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-sms-send-primary celery-sms-send-primary=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-sms-send-scalable celery-sms-send-scalable=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-email-send-primary celery-email-send-primary=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-        kubectl set image deployment.apps/celery-email-send-scalable celery-email-send-scalable=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
-
-    - name: Restart deployments in staging
-      run: |
-        kubectl rollout restart deployment/api -n notification-canada-ca
-        kubectl rollout restart deployment/celery-beat -n notification-canada-ca
-        kubectl rollout restart deployment/celery-sms -n notification-canada-ca
-        kubectl rollout restart deployment/celery-primary -n notification-canada-ca
-        kubectl rollout restart deployment/celery-scalable -n notification-canada-ca
-        kubectl rollout restart deployment/celery-sms-send-primary -n notification-canada-ca
-        kubectl rollout restart deployment/celery-sms-send-scalable -n notification-canada-ca
-        kubectl rollout restart deployment/celery-email-send-primary -n notification-canada-ca
-        kubectl rollout restart deployment/celery-email-send-scalable -n notification-canada-ca
+        ./scripts/callManifestsRollout.sh ${GITHUB_SHA::7}
 
     - name: my-app-install token
       id: notify-pr-bot
@@ -118,3 +83,4 @@ jobs:
       run: |
         json="{'text':'<!here> CI is failing in <https://github.com/cds-snc/notification-api/actions/runs/${GITHUB_RUN_ID}|notification-api> !'}"
         curl -X POST -H 'Content-type: application/json' --data "$json"  ${{ secrets.SLACK_WEBHOOK }}
+

.github/workflows/ossf-scorecard.yml

@@ -25,7 +25,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@0ae0fb3a2ca18a43d6dea9c07cfb9bd01d17eae1
+        uses: ossf/scorecard-action@50aaf84fb1a9f22255cb8bfb1729f4dd085c838c
         with:
           results_file: ossf-results.json
           results_format: json

.github/workflows/s3-backup.yml

@@ -15,7 +15,7 @@ jobs:
         fetch-depth: 0 # retrieve all history
 
     - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-access-key-id: ${{ secrets.AWS_S3_BACKUP_ACCESS_KEY_ID }}
         aws-secret-access-key: ${{ secrets.AWS_S3_BACKUP_SECRET_ACCESS_KEY }}

.github/workflows/test.yaml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: postgres:11.20-bullseye@sha256:4e4b23580ada59c9ec5a712bdff9f91b0e6a7898d9ea954306b953c426727cef
+        image: postgres:11.22-bullseye@sha256:c886a3236b3d11abc302e64309186c90a69b49e53ccff23fd8c8b057b5b4bce9
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -67,7 +67,7 @@ jobs:
       run: |
         cp -f .env.example .env
     - name: Checks for new endpoints against AWS WAF rules
-      uses: cds-snc/notification-utils/.github/actions/waffles@06a40db6286f525fe3551e029418458d33342592 # 52.1.0
+      uses: cds-snc/notification-utils/.github/actions/waffles@52.2.2
       with:
         app-loc: '/github/workspace'
         app-libs: '/github/workspace/env/site-packages'

.vscode/launch.json

@@ -43,7 +43,7 @@
                 "-l",
                 "DEBUG",
                 "-Q",
-                "database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,send-email-tasks,service-callbacks,delivery-receipts",
+                "database-tasks,-priority-database-tasks.fifo,-normal-database-tasks,-bulk-database-tasks,job-tasks,notify-internal-tasks,periodic-tasks,priority-tasks,normal-tasks,bulk-tasks,reporting-tasks,research-mode-tasks,retry-tasks,service-callbacks-retry,send-sms-tasks,send-sms-high,send-sms-medium,send-sms-low,send-throttled-sms-tasks,send-email-high,send-email-medium,send-email-low,send-email-tasks,service-callbacks,delivery-receipts",
             ]
         },
         {

Makefile

@@ -45,21 +45,25 @@ format:
 smoke-test:
 	cd tests_smoke && poetry run python smoke_test.py
 
+.PHONY: smoke-test-local
+smoke-test-local:
+	cd tests_smoke && poetry run python smoke_test.py --local --nofiles
+
 .PHONY: run
 run: ## Run the web app
-	flask run -p 6011 --host=0.0.0.0
+	poetry run flask run -p 6011 --host=0.0.0.0
 
 .PHONY: run-celery-local
 run-celery-local: ## Run the celery workers with all the queues
-	./scripts/run_celery_local.sh
+	poetry run ./scripts/run_celery_local.sh
 
 .PHONY: run-celery-local-filtered
 run-celery-local-filtered: ## Run the celery workers with all queues but filter out common scheduled tasks
-	./scripts/run_celery_local.sh 2>&1 >/dev/null | grep -iEv 'beat|in-flight-to-inbox|run-scheduled-jobs|check-job-status'
+	poetry run ./scripts/run_celery_local.sh 2>&1 >/dev/null | grep -iEv 'beat|in-flight-to-inbox|run-scheduled-jobs|check-job-status'
 
 .PHONY: run-celery-purge
 run-celery-purge: ## Purge the celery queues
-	./scripts/run_celery_purge.sh
+	poetry run ./scripts/run_celery_purge.sh
 
 .PHONY: run-db
 run-db: ## psql to access dev database

README.md

@@ -17,78 +17,7 @@ Contains:
 For any issues during the following instructions, make sure to review the
 **Frequent problems** section toward the end of the document.
 
-### Local installation instruction
-
-#### On OS X:
-
-1. Install PyEnv with Homebrew. This will preserve your sanity.
-
-`brew install pyenv`
-
-2. Install Python 3.10.8 or whatever is the latest
-
-`pyenv install 3.10.8`
-
-3. If you expect no conflicts, set `3.10.8` as you default
-
-`pyenv global 3.10.8`
-
-4. Ensure it installed by running
-
-`python --version`
-
-if it did not, take a look here: https://github.com/pyenv/pyenv/issues/660
-
-5. Install `poetry`:
-
-`pip install poetry==1.3.2`
-
-6. Restart your terminal and make your virtual environtment:
-
-`poetry env use $(which python)`
-
-8. Verify that the environment was created and activated by poetry
-
-`poetry env list`
-
-9. Install [Postgres.app](http://postgresapp.com/).
-
-10. Create the database for the application
-
-`createdb --user=postgres notification_api`
-
-11. Install the required environment variables via our LastPast Vault
-
-Within the team's *LastPass Vault*, you should find corresponding folders for this
-project containing the `.env` content that you should copy in your project root folder. This
-will grant the application necessary access to our internal infrastructure.
-
-If you don't have access to our *LastPass Vault* (as you evaluate our notification
-platform for example), you will find a sane set of defaults exists in the `.env.example`
-file. Copy that file to `.env` and customize it to your needs.
-
-12. Install all dependencies
-
-`poetry install`
-
-1.  Generate the version file ?!?
-
-`make generate-version-file`
-
-14. Run all DB migrations
-
-`flask db upgrade`
-
-15. Run the service
-
-`make run`
-
-15a. To test
-
-`poetry install --with test`
-
-`make test`
-
+### Local installation instruction (Use Dev Containers)
 #### In a [VS Code devcontainer](https://code.visualstudio.com/docs/remote/containers-tutorial)
 
 1. Install VS Code

app/__init__.py

@@ -28,6 +28,7 @@
     PerformancePlatformClient,
 )
 from app.clients.salesforce.salesforce_client import SalesforceClient
+from app.clients.sms.aws_pinpoint import AwsPinpointClient
 from app.clients.sms.aws_sns import AwsSnsClient
 from app.dbsetup import RoutingSQLAlchemy
 from app.encryption import CryptoSigner
@@ -45,6 +46,7 @@
 notify_celery = NotifyCelery()
 aws_ses_client = AwsSesClient()
 aws_sns_client = AwsSnsClient()
+aws_pinpoint_client = AwsPinpointClient()
 signer_notification = CryptoSigner()
 signer_personalisation = CryptoSigner()
 signer_complaint = CryptoSigner()
@@ -107,6 +109,7 @@ def create_app(application, config=None):
     statsd_client.init_app(application)
     logging.init_app(application, statsd_client)
     aws_sns_client.init_app(application, statsd_client=statsd_client)
+    aws_pinpoint_client.init_app(application, statsd_client=statsd_client)
     aws_ses_client.init_app(application.config["AWS_REGION"], statsd_client=statsd_client)
     notify_celery.init_app(application)
 
@@ -120,7 +123,7 @@ def create_app(application, config=None):
 
     performance_platform_client.init_app(application)
     document_download_client.init_app(application)
-    clients.init_app(sms_clients=[aws_sns_client], email_clients=[aws_ses_client])
+    clients.init_app(sms_clients=[aws_sns_client, aws_pinpoint_client], email_clients=[aws_ses_client])
 
     if application.config["FF_SALESFORCE_CONTACT"]:
         salesforce_client.init_app(application)

app/api_key/rest.py

@@ -121,8 +121,7 @@ def revoke_api_keys():
 
     # Step 1
     try:
-        # take last 36 chars of string so that it works even if the full key is provided.
-        api_key_token = api_key_data["token"][-36:]
+        api_key_token = api_key_data["token"]
         api_key = get_api_key_by_secret(api_key_token)
     except Exception:
         current_app.logger.error(

app/authentication/auth.py

@@ -63,7 +63,7 @@ def get_auth_token(req):
     for el in AUTH_TYPES:
         scheme, auth_type, _ = el
         if auth_header.lower().startswith(scheme.lower()):
-            token = auth_header[len(scheme) + 1 :]
+            token = auth_header[len(scheme) + 1 :].strip()
             return auth_type, token
 
     raise AuthError(
@@ -152,21 +152,12 @@ def requires_auth():
 
 
 def _auth_by_api_key(auth_token):
-    # TODO: uncomment this when the grace period for the token prefix is over
-    # orig_token = auth_token
-
     try:
-        # take last 36 chars of string so that it works even if the full key is provided.
-        auth_token = auth_token[-36:]
         api_key = get_api_key_by_secret(auth_token)
-
-        # TODO: uncomment this when the grace period for the token prefix is over
-        # check for token prefix
-        # if current_app.config["API_KEY_PREFIX"] not in orig_token:
-        #     raise AuthError("Invalid token: you must re-generate your API key to continue using GC Notify", 403, service_id=api_key.service.id, api_key_id=api_key.id)
-
     except NoResultFound:
         raise AuthError("Invalid token: API key not found", 403)
+    except ValueError:
+        raise AuthError("Invalid token: Enter your full API key", 403)
     _auth_with_api_key(api_key, api_key.service)