Merge branch 'main' into feat/golive-form-updates

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/vscode/devcontainers/python:1-3.10@sha256:c5b379b09a94ac1ccb437e000dd54c96164a8322d0c53d2bcb25f225e27924e6
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.12
 
 ENV POETRY_VERSION="1.7.1"
 

.devcontainer/devcontainer.json

@@ -33,12 +33,11 @@
 				"vsliveshare.vsliveshare"
 			],
 			"settings": {
+				"ruff.lint.ignore": ["F401"],
 				"ruff.lint.run": "onSave",
-				"ruff.configurationPreference": "filesystemFirst",
-				"editor.defaultFormatter": "charliermarsh.ruff",
-				"editor.formatOnSave": true,
-				"editor.codeActionsOnSave": {
-					"source.fixAll": "explicit"
+				"ruff.organizeImports": false,
+				"[python]": {
+					"editor.defaultFormatter": "charliermarsh.ruff"
 				}
 			}
 		}

.devcontainer/scripts/installations.sh

@@ -38,6 +38,9 @@ poetry completions zsh > ~/.zfunc/_poetry
 cd /workspace
 poetry install
 
+# Install pre-commit hooks
+poetry run pre-commit install
+
 # Poe the Poet plugin tab completions
 touch ~/.zfunc/_poe
 poetry run poe _zsh_completion > ~/.zfunc/_poe

.github/workflows/cypress-staging.yaml

@@ -3,6 +3,7 @@ name: Cypress staging a11y tests
 on:
   schedule:
     - cron: 0 */3 * * *
+  workflow_dispatch:
 defaults:
   run:
     shell: bash

.github/workflows/docker.yaml

@@ -8,8 +8,7 @@ env:
   AWS_REGION: ca-central-1
   DOCKER_ORG: public.ecr.aws/v6b8u5o6
   DOCKER_SLUG: public.ecr.aws/v6b8u5o6/notify-admin
-  WORKFLOW_PAT: ${{ secrets.WORKFLOW_GITHUB_PAT }}
-  OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+  OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN_STAGING }}
 
 permissions:
   id-token: write   # This is required for requesting the OIDC JWT
@@ -57,8 +56,8 @@ jobs:
     - name: Configure AWS credentials
       uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
       with:
-        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
         aws-region: ca-central-1
 
     - name: Install OpenVPN
@@ -87,7 +86,6 @@ jobs:
       uses: "kota65535/github-openvpn-connect-action@cd2ed8a90cc7b060dc4e001143e811b5f7ea0af5"
       with:
         config_file: /var/tmp/staging.ovpn
-        client_key: ${{ secrets.STAGING_OVPN_CLIENT_KEY }}
         echo_config: false       
 
     - name: Configure kubeconfig
@@ -97,7 +95,7 @@ jobs:
     - name: Update images in staging
       run: |
         DOCKER_TAG=${GITHUB_SHA::7}
-        kubectl set image deployment.apps/admin admin=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/notify-admin notify-admin=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
 
     - name: my-app-install token
       id: notify-pr-bot

.github/workflows/test-admin-delete-unused.yaml

@@ -19,8 +19,8 @@ jobs:
         id: aws-creds
         uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
           aws-region: ca-central-1
 
       - name: Delete old PR review environments

.github/workflows/test-admin-deploy.yaml

@@ -32,8 +32,8 @@ jobs:
         id: aws-creds
         uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
           aws-region: ca-central-1
 
       - name: Login to ECR
@@ -79,8 +79,8 @@ jobs:
         id: aws-creds
         uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
           aws-region: ca-central-1
 
       - name: Create/Update lambda function

.github/workflows/test-admin-remove.yaml

@@ -26,8 +26,8 @@ jobs:
         id: aws-creds
         uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
           aws-region: ca-central-1
 
       - name: Delete lambda function resources

.github/workflows/test.yaml

@@ -11,7 +11,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     - uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0
       with:
-        python-version: '3.10'
+        python-version: '3.12'
     - uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v3.8.2
       with:
         node-version: '16.x'

.github/workflows/test_endpoints.yaml

@@ -8,10 +8,10 @@ jobs:
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
-    - name: Set up Python 3.10
+    - name: Set up Python 3.12
       uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0
       with:
-        python-version: '3.10'
+        python-version: '3.12'
 
     - name: Upgrade pip
       run: python -m pip install --upgrade pip
@@ -45,7 +45,7 @@ jobs:
       working-directory: ${{ github.workspace }}
       shell: bash
       run: |
-        mkdir -p "${{ github.workspace }}/env/" && cp -fR $(poetry env list | poetry env info -p)/lib/python3.10/site-packages "${{ github.workspace }}/env/"
+        mkdir -p "${{ github.workspace }}/env/" && cp -fR $(poetry env list | poetry env info -p)/lib/python3.12/site-packages "${{ github.workspace }}/env/"
 
     - name: Install development .env file
       working-directory: ${{ github.workspace }}
@@ -58,7 +58,7 @@ jobs:
         echo "PYTHONPATH=/github/workspace/env/site-packages:${{ env.PYTHONPATH}}" >> $GITHUB_ENV
 
     - name: Checks for new endpoints against AWS WAF rules
-      uses: cds-snc/notification-utils/.github/actions/waffles@52.2.2
+      uses: cds-snc/notification-utils/.github/actions/waffles@53.0.1
       with:
         app-loc: '/github/workspace'
         app-libs: '/github/workspace/env/site-packages'

.github/workflows/test_prod_config.yaml

@@ -11,7 +11,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     - uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0
       with:
-        python-version: '3.10'
+        python-version: '3.12'
     - uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v3.8.2
       with:
         node-version: '16.x'

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.5.5
+  hooks:
+    # Run the linter
+    - id: ruff
+      args: [ --fix ]
+    # Run the formatter
+    - id: ruff-format

Makefile

@@ -55,9 +55,13 @@ coverage: venv ## Create coverage report
 run-dev:
 	poetry run flask run -p 6012 --host=localhost
 
+.PHONY: run-gunicorn
+run-gunicorn:
+	PORT=6012 poetry run gunicorn -c gunicorn_config.py application
+
 .PHONY: format
 format:
-	ruff check --select I --fix .
+	ruff check --fix .
 	ruff check
 	ruff format .
 	mypy ./

README.md

@@ -31,7 +31,7 @@ https://github.com/alphagov/notifications-admin
 ```
 
 Languages needed
-- Python 3.10
+- Python 3.12
 - [Node](https://nodejs.org/) 10.15.3 or greater
 - [npm](https://www.npmjs.com/) 6.4.1 or greater
 
@@ -58,15 +58,15 @@ On OS X:
 
 `brew install pyenv`
 
-2. Install Python 3.10.8 or whatever is the latest
+2. Install Python 3.12.7 or whatever is the latest
 
-`pyenv install 3.10.8`
+`pyenv install 3.12.7`
 
-3. If you expect no conflicts, set `3.10.8` as you default
+3. If you expect no conflicts, set `3.12.7` as you default
 
-`pyenv global 3.10.8`
+`pyenv global 3.12.7`
 
-4. Ensure that version `3.10.8` is now the default by running
+4. Ensure that version `3.12.7` is now the default by running
 
 `python --version`
 
@@ -77,15 +77,15 @@ eval "$(pyenv init -)"
 ```
 and open a new terminal.
 
-If you are still not running Python 3.10.8 take a look here: https://github.com/pyenv/pyenv/issues/660
+If you are still not running Python 3.12.7 take a look here: https://github.com/pyenv/pyenv/issues/660
 
 5. Install `poetry`:
 
 `pip install poetry==1.3.2`
 
 6. Restart your terminal and make your virtual environtment:
 
-`mkvirtualenv -p ~/.pyenv/versions/3.10.8/bin/python notifications-admin`
+`mkvirtualenv -p ~/.pyenv/versions/3.12.7/bin/python notifications-admin`
 
 7. You can now return to your environment any time by entering
 
@@ -278,7 +278,7 @@ https://github.com/alphagov/notifications-admin
 ```
 
 Langages nécessaires
-- Python 3.10
+- Python 3.12
 - [Node](https://nodejs.org/) 10.15.3 ou supérieur
 - [npm](https://www.npmjs.com/) 6.4.1 ou plus
 ```shell
@@ -302,15 +302,15 @@ Sur macOS :
 
 `brew install pyenv`
 
-2. Installez Python 3.10.8 ou la dernière version
+1. Installez Python 3.12.7 ou la dernière version
 
-`pyenv install 3.10.8`
+`pyenv install 3.12.7`
 
-3. Si vous n'attendez aucun conflit, mettez `3.10.8` comme valeur par défaut
+3. Si vous n'attendez aucun conflit, mettez `3.12.7` comme valeur par défaut
 
-`pyenv global 3.10.8`
+`pyenv global 3.12.7`
 
-4. Assurez-vous que la version 3.10.8 est maintenant la version par défaut en exécutant
+4. Assurez-vous que la version 3.12.7 est maintenant la version par défaut en exécutant
 
 `python --version`
 
@@ -320,7 +320,7 @@ eval "$(pyenv init --path)"
 eval "$(pyenv init -)"
 ```
 et ouvrez un nouveau terminal.
-Si vous n’utilisez toujours pas Python 3.10.8, jetez un coup d’œil ici : https://github.com/pyenv/pyenv/issues/660
+Si vous n’utilisez toujours pas Python 3.12.7, jetez un coup d’œil ici : https://github.com/pyenv/pyenv/issues/660
 
 5. Installez `virtualenv` :
 
@@ -331,12 +331,12 @@ Si vous n’utilisez toujours pas Python 3.10.8, jetez un coup d’œil ici : ht
 ```
 export WORKON_HOME=$HOME/.virtualenvs
 export PROJECT_HOME=$HOME/Devel
-source ~/.pyenv/versions/3.10.8/bin/virtualenvwrapper.sh
+source ~/.pyenv/versions/3.12.7/bin/virtualenvwrapper.sh
 ```
 
 7. Redémarrez votre terminal et créez votre environnement virtuel :
 
-`mkvirtualenv -p ~/.pyenv/versions/3.10.8/bin/python notifications-admin`
+`mkvirtualenv -p ~/.pyenv/versions/3.12.7/bin/python notifications-admin`
 
 8. Vous pouvez maintenant retourner dans votre environnement à tout moment en entrant
 

app/config.py

@@ -188,7 +188,6 @@ class Development(Config):
     SESSION_PROTECTION = None
     SYSTEM_STATUS_URL = "https://localhost:3000"
     NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef"
-    FF_ANNUAL_LIMIT = env.bool("FF_ANNUAL_LIMIT", True)
 
 
 class Test(Development):
@@ -252,15 +251,13 @@ class Production(Config):
     NOTIFY_LOG_LEVEL = "INFO"
     SYSTEM_STATUS_URL = "https://status.notification.canada.ca"
     NO_BRANDING_ID = "760c802a-7762-4f71-b19e-f93c66c92f1a"
-    FF_ANNUAL_LIMIT = False
 
 
 class Staging(Production):
     NOTIFY_ENVIRONMENT = "staging"
     NOTIFY_LOG_LEVEL = "INFO"
     SYSTEM_STATUS_URL = "https://status.staging.notification.cdssandbox.xyz"
     NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef"
-    FF_ANNUAL_LIMIT = True
 
 
 class Scratch(Production):

app/main/views/dashboard.py

@@ -273,27 +273,34 @@ def aggregate_by_type(notification_data):
         return counts
 
     year, current_financial_year = requested_and_current_financial_year(request)
-    monthly_data = service_api_client.get_monthly_notification_stats(service_id, year)
-    annual_data = aggregate_by_type(monthly_data)
 
-    todays_data = annual_limit_client.get_all_notification_counts(current_service.id)
+    # if FF_ANNUAL is on
+    if current_app.config["FF_ANNUAL_LIMIT"]:
+        monthly_data = service_api_client.get_monthly_notification_stats(service_id, year)
+        annual_data = aggregate_by_type(monthly_data)
 
-    # if redis is empty, query the db
-    if todays_data is None:
-        todays_data = service_api_client.get_service_statistics(service_id, limit_days=1, today_only=False)
-        annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "db")
+        todays_data = annual_limit_client.get_all_notification_counts(current_service.id)
 
-        months = (format_monthly_stats_to_list(monthly_data["data"]),)
-        monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "db")
-    else:
-        # aggregate daily + annual
-        current_app.logger.info("todays data" + str(todays_data))
-        annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "redis")
+        # if redis is empty, query the db
+        if all(value == 0 for value in todays_data.values()):
+            todays_data = service_api_client.get_service_statistics(service_id, limit_days=1, today_only=False)
+            annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "db")
 
-        months = (format_monthly_stats_to_list(monthly_data["data"]),)
-        monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "redis")
+            months = (format_monthly_stats_to_list(monthly_data["data"]),)
+            monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "db")
+        else:
+            # aggregate daily + annual
+            current_app.logger.info("todays data" + str(todays_data))
+            annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "redis")
 
-    # add today's data to monthly data
+            months = (format_monthly_stats_to_list(monthly_data["data"]),)
+            monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "redis")
+    else:
+        monthly_data_aggregate = (
+            format_monthly_stats_to_list(service_api_client.get_monthly_notification_stats(service_id, year)["data"]),
+        )
+        monthly_data_aggregate = monthly_data_aggregate[0]
+        annual_data_aggregate = None
 
     return render_template(
         "views/dashboard/monthly.html",