Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into monkeytype-auto-annotate
Browse files Browse the repository at this point in the history
  • Loading branch information
whabanks committed Apr 12, 2024
2 parents ed35a40 + 7c19d9f commit ec8a6b6
Showing 66 changed files with 912 additions and 372 deletions.
2 changes: 1 addition & 1 deletion .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@ FROM mcr.microsoft.com/vscode/devcontainers/python:0-3.10@sha256:ef9cc483a593c95

ARG KUBENS_VERSION="0.9.4"
ARG OCTANT_VERSION="0.25.1"
ENV POETRY_VERSION="1.3.2"
ENV POETRY_VERSION="1.7.1"

# Install packages
RUN apt-get update \
6 changes: 4 additions & 2 deletions .devcontainer/scripts/notify-dev-entrypoint.sh
Original file line number Diff line number Diff line change
@@ -34,8 +34,10 @@ cd /workspace
echo -e "fpath+=/.zfunc" >> ~/.zshrc
echo -e "autoload -Uz compinit && compinit"

pip install poetry==${POETRY_VERSION} \
&& poetry --version
pip install poetry==${POETRY_VERSION}
export PATH=$PATH:/home/vscode/.local/bin/
which poetry
poetry --version

# Initialize poetry autocompletions
mkdir ~/.zfunc
46 changes: 6 additions & 40 deletions .github/workflows/docker.yaml
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ env:
DOCKER_ORG: public.ecr.aws/v6b8u5o6
DOCKER_SLUG: public.ecr.aws/v6b8u5o6/notify-api
KUBECTL_VERSION: '1.23.6'
WORKFLOW_PAT: ${{ secrets.WORKFLOW_GITHUB_PAT }}

permissions:
id-token: write # This is required for requesting the OIDC JWT
@@ -26,21 +27,14 @@ jobs:
unzip -q awscliv2.zip
sudo ./aws/install --update
aws --version
- name: Install kubectl
run: |
curl -LO https://storage.googleapis.com/kubernetes-release/release/v$KUBECTL_VERSION/bin/linux/amd64/kubectl
chmod +x ./kubectl
sudo mv ./kubectl /usr/local/bin/kubectl
kubectl version --client
mkdir -p $HOME/.kube
- name: Configure credentials to CDS public ECR using OIDC
uses: aws-actions/configure-aws-credentials@master
with:
role-to-assume: arn:aws:iam::283582579564:role/notification-api-apply
role-session-name: NotifyApiGitHubActions
aws-region: "us-east-1"

- name: Login to ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@5a88a04c91d5c6f97aae0d9be790e64d9b1d47b7 # v1.7.1
@@ -56,43 +50,14 @@ jobs:
-t $DOCKER_SLUG:${GITHUB_SHA::7} \
-t $DOCKER_SLUG:latest \
-f ci/Dockerfile .
- name: Publish
run: |
docker push $DOCKER_SLUG:latest && docker push $DOCKER_SLUG:${GITHUB_SHA::7}
- name: Configure credentials to Notify account using OIDC
uses: aws-actions/configure-aws-credentials@master
with:
role-to-assume: arn:aws:iam::239043911459:role/notification-api-apply
role-session-name: NotifyApiGitHubActions
aws-region: "ca-central-1"

- name: Get Kubernetes configuration
run: |
aws eks --region $AWS_REGION update-kubeconfig --name notification-canada-ca-staging-eks-cluster --kubeconfig $HOME/.kube/config
- name: Update images in staging
- name: Rollout in Kubernetes
run: |
kubectl set image deployment.apps/api api=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
kubectl set image deployment.apps/celery-beat celery-beat=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
kubectl set image deployment.apps/celery-sms celery-sms=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
kubectl set image deployment.apps/celery-primary celery-primary=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
kubectl set image deployment.apps/celery-scalable celery-scalable=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
kubectl set image deployment.apps/celery-sms-send-primary celery-sms-send-primary=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
kubectl set image deployment.apps/celery-sms-send-scalable celery-sms-send-scalable=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
kubectl set image deployment.apps/celery-email-send-primary celery-email-send-primary=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
kubectl set image deployment.apps/celery-email-send-scalable celery-email-send-scalable=$DOCKER_SLUG:${GITHUB_SHA::7} -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
- name: Restart deployments in staging
run: |
kubectl rollout restart deployment/api -n notification-canada-ca
kubectl rollout restart deployment/celery-beat -n notification-canada-ca
kubectl rollout restart deployment/celery-sms -n notification-canada-ca
kubectl rollout restart deployment/celery-primary -n notification-canada-ca
kubectl rollout restart deployment/celery-scalable -n notification-canada-ca
kubectl rollout restart deployment/celery-sms-send-primary -n notification-canada-ca
kubectl rollout restart deployment/celery-sms-send-scalable -n notification-canada-ca
kubectl rollout restart deployment/celery-email-send-primary -n notification-canada-ca
kubectl rollout restart deployment/celery-email-send-scalable -n notification-canada-ca
./scripts/callManifestsRollout.sh ${GITHUB_SHA::7}
- name: my-app-install token
id: notify-pr-bot
@@ -118,3 +83,4 @@ jobs:
run: |
json="{'text':'<!here> CI is failing in <https://github.com/cds-snc/notification-api/actions/runs/${GITHUB_RUN_ID}|notification-api> !'}"
curl -X POST -H 'Content-type: application/json' --data "$json" ${{ secrets.SLACK_WEBHOOK }}
2 changes: 1 addition & 1 deletion .github/workflows/ossf-scorecard.yml
Original file line number Diff line number Diff line change
@@ -25,7 +25,7 @@ jobs:
persist-credentials: false

- name: "Run analysis"
uses: ossf/scorecard-action@155cf0ea68b491a7c47af606d2741b54963ecb04
uses: ossf/scorecard-action@50aaf84fb1a9f22255cb8bfb1729f4dd085c838c
with:
results_file: ossf-results.json
results_format: json
2 changes: 1 addition & 1 deletion .github/workflows/s3-backup.yml
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ jobs:
fetch-depth: 0 # retrieve all history

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_S3_BACKUP_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_S3_BACKUP_SECRET_ACCESS_KEY }}
6 changes: 3 additions & 3 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
@@ -33,10 +33,10 @@ jobs:
${{ runner.os }}-pip-
- name: Install poetry
env:
POETRY_VERSION: "1.3.2"
POETRY_VERSION: "1.7.1"
run: pip install poetry==${POETRY_VERSION} && poetry --version
- name: Check poetry.lock aligns with pyproject.toml
run: poetry lock --check
run: poetry check --lock
- name: Install requirements
run: poetry install --with test
- name: Run tests
@@ -67,7 +67,7 @@ jobs:
run: |
cp -f .env.example .env
- name: Checks for new endpoints against AWS WAF rules
uses: cds-snc/notification-utils/.github/actions/waffles@2da74685e0ffb220f0403e1f2584e783be99bbad # 52.1.0
uses: cds-snc/notification-utils/.github/actions/waffles@06a40db6286f525fe3551e029418458d33342592 # 52.1.0
with:
app-loc: '/github/workspace'
app-libs: '/github/workspace/env/site-packages'
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -25,7 +25,7 @@ freeze-requirements:

.PHONY: test-requirements
test-requirements:
poetry lock --check
poetry check --lock

.PHONY: coverage
coverage: venv ## Create coverage report
3 changes: 1 addition & 2 deletions app/api_key/rest.py
Original file line number Diff line number Diff line change
@@ -121,8 +121,7 @@ def revoke_api_keys():

# Step 1
try:
# take last 36 chars of string so that it works even if the full key is provided.
api_key_token = api_key_data["token"][-36:]
api_key_token = api_key_data["token"]
api_key = get_api_key_by_secret(api_key_token)
except Exception:
current_app.logger.error(
15 changes: 3 additions & 12 deletions app/authentication/auth.py
Original file line number Diff line number Diff line change
@@ -63,7 +63,7 @@ def get_auth_token(req):
for el in AUTH_TYPES:
scheme, auth_type, _ = el
if auth_header.lower().startswith(scheme.lower()):
token = auth_header[len(scheme) + 1 :]
token = auth_header[len(scheme) + 1 :].strip()
return auth_type, token

raise AuthError(
@@ -152,21 +152,12 @@ def requires_auth():


def _auth_by_api_key(auth_token):
# TODO: uncomment this when the grace period for the token prefix is over
# orig_token = auth_token

try:
# take last 36 chars of string so that it works even if the full key is provided.
auth_token = auth_token[-36:]
api_key = get_api_key_by_secret(auth_token)

# TODO: uncomment this when the grace period for the token prefix is over
# check for token prefix
# if current_app.config["API_KEY_PREFIX"] not in orig_token:
# raise AuthError("Invalid token: you must re-generate your API key to continue using GC Notify", 403, service_id=api_key.service.id, api_key_id=api_key.id)

except NoResultFound:
raise AuthError("Invalid token: API key not found", 403)
except ValueError:
raise AuthError("Invalid token: Enter your full API key", 403)
_auth_with_api_key(api_key, api_key.service)


19 changes: 17 additions & 2 deletions app/aws/s3.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
import uuid
from datetime import datetime, timedelta
from typing import List

import botocore
import pytz
from boto3 import client, resource
from flask import current_app
from notifications_utils.s3 import s3upload as utils_s3upload

from app.models import Job

FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv"


@@ -60,8 +63,20 @@ def get_job_metadata_from_s3(service_id, job_id):
return obj.get()["Metadata"]


def remove_job_from_s3(service_id, job_id):
return remove_s3_object(*get_job_location(service_id, job_id))
def remove_jobs_from_s3(jobs: List[Job], batch_size=1000):
"""
Remove the files from S3 for the given jobs.
Args:
jobs (List[Job]): The jobs whose files need to be removed from S3.
batch_size (int, optional): The number of jobs to process in each boto call. Defaults to the AWS maximum of 1000.
"""

bucket = resource("s3").Bucket(current_app.config["CSV_UPLOAD_BUCKET_NAME"])

for start in range(0, len(jobs), batch_size):
object_keys = [FILE_LOCATION_STRUCTURE.format(job.service_id, job.id) for job in jobs[start : start + batch_size]]
bucket.delete_objects(Delete={"Objects": [{"Key": key} for key in object_keys]})


def get_s3_bucket_objects(bucket_name, subfolder="", older_than=7, limit_days=2):
41 changes: 28 additions & 13 deletions app/celery/nightly_tasks.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from datetime import datetime, timedelta
from typing import List

import pytz
from flask import current_app
@@ -12,7 +13,7 @@
from app.config import QueueNames
from app.cronitor import cronitor
from app.dao.inbound_sms_dao import delete_inbound_sms_older_than_retention
from app.dao.jobs_dao import dao_archive_job, dao_get_jobs_older_than_data_retention
from app.dao.jobs_dao import dao_archive_jobs, dao_get_jobs_older_than_data_retention
from app.dao.notifications_dao import (
dao_timeout_notifications,
delete_notifications_older_than_retention_by_type,
@@ -37,23 +38,37 @@
@notify_celery.task(name="remove_sms_email_jobs")
@cronitor("remove_sms_email_jobs")
@statsd(namespace="tasks")
def remove_sms_email_csv_files():
_remove_csv_files([EMAIL_TYPE, SMS_TYPE])
def remove_sms_email_jobs():
"""
Remove csv files from s3 and archive email and sms jobs older than data retention period.
"""

_archive_jobs([EMAIL_TYPE, SMS_TYPE])


@notify_celery.task(name="remove_letter_jobs")
@cronitor("remove_letter_jobs")
@statsd(namespace="tasks")
def remove_letter_csv_files():
_remove_csv_files([LETTER_TYPE])


def _remove_csv_files(job_types):
jobs = dao_get_jobs_older_than_data_retention(notification_types=job_types)
for job in jobs:
s3.remove_job_from_s3(job.service_id, job.id)
dao_archive_job(job)
current_app.logger.info("Job ID {} has been removed from s3.".format(job.id))
def remove_letter_jobs():
_archive_jobs([LETTER_TYPE])


def _archive_jobs(job_types: List[str]):
"""
Remove csv files from s3 and archive jobs older than data retention period.
Args:
job_types (List[str]): list of job types to remove csv files and archive jobs for
"""

while True:
jobs = dao_get_jobs_older_than_data_retention(notification_types=job_types, limit=100)
if len(jobs) == 0:
break
current_app.logger.info("Archiving {} jobs.".format(len(jobs)))
s3.remove_jobs_from_s3(jobs)
dao_archive_jobs(jobs)
current_app.logger.info(f"Jobs archived: {[job.id for job in jobs]}")


@notify_celery.task(name="delete-sms-notifications")
8 changes: 4 additions & 4 deletions app/celery/provider_tasks.py
Original file line number Diff line number Diff line change
@@ -42,10 +42,10 @@ def deliver_throttled_sms(self, notification_id):

# Celery rate limits are per worker instance and not a global rate limit.
# https://docs.celeryproject.org/en/stable/userguide/tasks.html#Task.rate_limit
# This task is dispatched through the `send-sms-tasks` queue.
# This queue is consumed by 6 Celery instances with 4 workers in production.
# The maximum throughput is therefore 6 instances * 4 workers = 24 tasks per second
# if we set rate_limit="1/s" on the Celery task
# We currently set rate_limit="1/s" on the Celery task and 4 workers per pod, and so a limit of 4 tasks per second per pod.
# The number of pods is controlled by the Kubernetes HPA and scales up and down with demand.
# Currently in production we have 3 celery-sms-send-primary pods, and up to 20 celery-sms-send-scalable pods
# This means we can send up to 92 messages per second.
@notify_celery.task(
bind=True,
name="deliver_sms",
4 changes: 4 additions & 0 deletions app/clients/freshdesk.py
Original file line number Diff line number Diff line change
@@ -59,11 +59,15 @@ def _generate_description(self):
f"A new logo has been uploaded by {self.contact.name} ({self.contact.email_address}) for the following service:",
f"- Service id: {self.contact.service_id}",
f"- Service name: {self.contact.service_name}",
f"- Organisation id: {self.contact.organisation_id}",
f"- Organisation name: {self.contact.department_org_name}",
f"- Logo filename: {self.contact.branding_url}",
"<hr>",
f"Un nouveau logo a été téléchargé par {self.contact.name} ({self.contact.email_address}) pour le service suivant :",
f"- Identifiant du service : {self.contact.service_id}",
f"- Nom du service : {self.contact.service_name}",
f"- Identifiant de l'organisation: {self.contact.organisation_id}",
f"- Nom de l'organisation: {self.contact.department_org_name}",
f"- Nom du fichier du logo : {self.contact.branding_url}",
]
)
12 changes: 0 additions & 12 deletions app/config.py
Original file line number Diff line number Diff line change
@@ -78,18 +78,11 @@ class QueueNames(object):
# A queue for the tasks associated with the batch saving
NOTIFY_CACHE = "notifiy-cache-tasks"

# For normal send of notifications. This is relatively normal volume and flushed
# pretty quickly.
SEND_NORMAL_QUEUE = "send-{}-tasks" # notification type to be filled in the queue name

# Queues for sending all SMS, except long dedicated numbers.
SEND_SMS_HIGH = "send-sms-high"
SEND_SMS_MEDIUM = "send-sms-medium"
SEND_SMS_LOW = "send-sms-low"

# TODO: Delete this queue once we verify that it is not used anymore.
SEND_SMS = "send-sms-tasks"

# Primarily used for long dedicated numbers sent from us-west-2 upon which
# we have a limit to send per second and hence, needs to be throttled.
SEND_THROTTLED_SMS = "send-throttled-sms-tasks"
@@ -99,9 +92,6 @@ class QueueNames(object):
SEND_EMAIL_MEDIUM = "send-email-medium"
SEND_EMAIL_LOW = "send-email-low"

# TODO: Delete this queue once we verify that it is not used anymore.
SEND_EMAIL = "send-email-tasks"

# The research mode queue for notifications that are tested by users trying
# out Notify.
RESEARCH_MODE = "research-mode-tasks"
@@ -158,12 +148,10 @@ def all_queues():
QueueNames.SEND_SMS_HIGH,
QueueNames.SEND_SMS_MEDIUM,
QueueNames.SEND_SMS_LOW,
QueueNames.SEND_SMS,
QueueNames.SEND_THROTTLED_SMS,
QueueNames.SEND_EMAIL_HIGH,
QueueNames.SEND_EMAIL_MEDIUM,
QueueNames.SEND_EMAIL_LOW,
QueueNames.SEND_EMAIL,
QueueNames.RESEARCH_MODE,
QueueNames.REPORTING,
QueueNames.JOBS,
Loading

0 comments on commit ec8a6b6

Please sign in to comment.