Merge branch 'master' into feat/uuid-query-endpoint

.github/workflows/build_and_push.yml → .github/workflows/build_and_push_performance_test.yml

@@ -1,10 +1,12 @@
 
-name: Build and Push Container to ECR, deploy to lambda
+name: Build and Push Performance Testing Container to ECR, deploy to lambda
 
 on:
   workflow_dispatch:
   push:
-    branches: [master]
+    paths:
+      - 'tests-perf/**'
+      - 'tests_smoke/**'
 
 env:
   GITHUB_SHA: ${{ github.sha }}
@@ -23,7 +25,9 @@ jobs:
         id: filter
         with:
           filters: |
-            api: 'app/**'
+            performance-test: 
+              - 'tests-perf/**'
+              - 'tests_smoke/**'
 
   build-push-and-deploy:
     if: ${{ needs.changes.outputs.images != '[]' }}
@@ -44,7 +48,7 @@ jobs:
           --build-arg git_sha=$GITHUB_SHA \
           -t $REGISTRY/${{ matrix.image }}:$GITHUB_SHA \
           -t $REGISTRY/${{ matrix.image }}:latest . \
-          -f ci/Dockerfile.lambda
+          -f tests-perf/ops/Dockerfile
 
       - name: Configure AWS credentials
         id: aws-creds
@@ -65,9 +69,3 @@ jobs:
 
       - name: Logout of Amazon ECR
         run: docker logout ${{ steps.login-ecr.outputs.registry }}
-
-      - name: Deploy lambda
-        run: |
-          aws lambda update-function-code \
-            --function-name ${{ matrix.image }} \
-            --image-uri $REGISTRY/${{ matrix.image }}:latest

.github/workflows/lambda_production.yml

@@ -0,0 +1,49 @@
+
+name: Build and push lambda image to production
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [master]
+
+env:
+  REGISTRY: ${{ secrets.PRODUCTION_API_LAMBDA_ECR_ACCOUNT }}.dkr.ecr.ca-central-1.amazonaws.com/notify
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        image: ["api-lambda"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Build container
+        run: |
+          docker build \
+          --build-arg GIT_SHA=${GITHUB_SHA::7} \
+          -t $REGISTRY/${{ matrix.image }}:${GITHUB_SHA::7} \
+          . \
+          -f ci/Dockerfile.lambda
+
+      - name: Configure AWS credentials
+        id: aws-creds
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.PRODUCTION_ECR_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.PRODUCTION_ECR_SECRET_ACCESS_KEY }}
+          aws-region: ca-central-1
+
+      - name: Login to ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Push containers to ECR
+        run: |
+          docker push $REGISTRY/${{ matrix.image }}:${GITHUB_SHA::7}
+
+      - name: Logout of Amazon ECR
+        run: docker logout ${{ steps.login-ecr.outputs.registry }}

.github/workflows/lambda_staging.yml

@@ -0,0 +1,55 @@
+
+name: Build, push, and deploy lambda image to staging
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [master]
+
+env:
+  REGISTRY: ${{ secrets.STAGING_API_LAMBDA_ECR_ACCOUNT }}.dkr.ecr.ca-central-1.amazonaws.com/notify
+
+jobs:
+  build-push-and-deploy:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        image: ["api-lambda"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Build container
+        run: |
+          docker build \
+          --build-arg GIT_SHA=${GITHUB_SHA::7} \
+          -t $REGISTRY/${{ matrix.image }}:${GITHUB_SHA::7} \
+          . \
+          -f ci/Dockerfile.lambda
+
+      - name: Configure AWS credentials
+        id: aws-creds
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.STAGING_ECR_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.STAGING_ECR_SECRET_ACCESS_KEY }}
+          aws-region: ca-central-1
+
+      - name: Login to ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Push containers to ECR
+        run: |
+          docker push $REGISTRY/${{ matrix.image }}:${GITHUB_SHA::7}
+
+      - name: Logout of Amazon ECR
+        run: docker logout ${{ steps.login-ecr.outputs.registry }}
+
+      - name: Deploy lambda
+        run: |
+          aws lambda update-function-code \
+            --function-name ${{ matrix.image }} \
+            --image-uri $REGISTRY/${{ matrix.image }}:${GITHUB_SHA::7}

.github/workflows/performance.yml

@@ -0,0 +1,31 @@
+name: Notify Performance / Load Tests
+
+on:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install libcurl
+        run: sudo apt-get update && sudo apt-get install libssl-dev libcurl4-openssl-dev
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+      - uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Run performance tests
+        run: /bin/bash -c "pip install -r requirements_for_test.txt && locust --headless --config tests-perf/locust/locust.conf -f tests-perf/locust/locust-notifications.py"
+      - name: Notify Slack channel if this performance test job fails
+        if: ${{ failure() && github.ref == 'refs/heads/master' }}
+        run: |
+          json="{'text':'Scheduled CI Performance testing failed: <https://github.com/cds-snc/notification-api/actions|GitHub actions>'}"
+          curl -X POST -H 'Content-type: application/json' --data "$json"  ${{ secrets.SLACK_WEBHOOK }}

.github/workflows/test.yaml

@@ -1,15 +1,12 @@
 on:
   push:
-  schedule:
-    - cron: '2 0-12,20-23 * * 1-5'
-    - cron: '2 * * * 0,6'
-name: CI testing
+name: Python tests
 jobs:
   test:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: postgres:11.8
+        image: postgres:13.4
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres

SECURITY.md

@@ -1,9 +1,9 @@
 # Security
 
-**Do not post any security issues on the public repository!** Security vulnerabilities must be reported by email to [[email protected]](mailto:[email protected]) and [[email protected]](mailto:[email protected].ca).
+**Do not post any security issues on the public repository!** Security vulnerabilities must be reported by email to [[email protected] ](mailto:security@cds-snc.ca).
 
 ______________________
 
 ## Sécurité
 
-**Ne publiez aucun problème de sécurité sur le dépôt publique!** Les vulnérabilités de sécurité doivent être signalées par courriel à [[email protected]](mailto:[email protected]) et [[email protected]](mailto:[email protected].ca).
+**Ne publiez aucun problème de sécurité sur le dépôt publique!** Les vulnérabilités de sécurité doivent être signalées par courriel à [[email protected] ](mailto:security@cds-snc.ca).

app/celery/provider_tasks.py

@@ -9,6 +9,7 @@
 from app.dao.notifications_dao import update_notification_status_by_id
 from app.delivery import send_to_providers
 from app.exceptions import (
+    InvalidUrlException,
     MalwarePendingException,
     NotificationTechnicalFailureException,
 )
@@ -65,6 +66,10 @@ def deliver_email(self, notification_id):
         current_app.logger.info(f"Cannot send notification {notification_id}, got an invalid email address: {str(e)}.")
         update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE)
         _check_and_queue_callback_task(notification)
+    except InvalidUrlException:
+        current_app.logger.error(f"Cannot send notification {notification_id}, got an invalid direct file url.")
+        update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE)
+        _check_and_queue_callback_task(notification)
     except MalwarePendingException:
         current_app.logger.info("RETRY: Email notification {} is pending malware scans".format(notification_id))
         self.retry(queue=QueueNames.RETRY, countdown=60)
@@ -90,6 +95,10 @@ def _deliver_sms(self, notification_id):
         if not notification:
             raise NoResultFound()
         send_to_providers.send_sms_to_provider(notification)
+    except InvalidUrlException:
+        current_app.logger.error(f"Cannot send notification {notification_id}, got an invalid direct file url.")
+        update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE)
+        _check_and_queue_callback_task(notification)
     except Exception:
         try:
             current_app.logger.exception("SMS notification delivery for id: {} failed".format(notification_id))

app/celery/scheduled_tasks.py

@@ -122,20 +122,20 @@ def check_job_status():
     from jobs
     where job_status == 'in progress'
     and template_type in ('sms', 'email')
-    and scheduled_at or created_at is older that 60 minutes.
+    and scheduled_at or created_at is older that 120 minutes.
     if any results then
         raise error
         process the rows in the csv that are missing (in another task) just do the check here.
     """
-    sixty_minutes_ago = datetime.utcnow() - timedelta(minutes=60)
-    sixty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=65)
+    minutes_ago_120 = datetime.utcnow() - timedelta(minutes=120)
+    minutes_ago_125 = datetime.utcnow() - timedelta(minutes=125)
 
-    jobs_not_complete_after_60_minutes = (
+    jobs_not_complete_after_120_minutes = (
         Job.query.filter(
             Job.job_status == JOB_STATUS_IN_PROGRESS,
             and_(
-                sixty_five_minutes_ago < Job.processing_started,
-                Job.processing_started < sixty_minutes_ago,
+                minutes_ago_125 < Job.processing_started,
+                Job.processing_started < minutes_ago_120,
             ),
         )
         .order_by(Job.processing_started)
@@ -145,7 +145,7 @@ def check_job_status():
     # temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
     # if they haven't been re-processed in time.
     job_ids = []
-    for job in jobs_not_complete_after_60_minutes:
+    for job in jobs_not_complete_after_120_minutes:
         job.job_status = JOB_STATUS_ERROR
         dao_update_job(job)
         job_ids.append(str(job.id))

app/clients/freshdesk.py

@@ -85,10 +85,5 @@ def send_ticket(self) -> int:
             return response.status_code
         except requests.RequestException as e:
             content = json.loads(response.content)
-            current_app.logger.warning(f"Failed to create Freshdesk ticket: {content['errors']}")
+            current_app.logger.error(f"Failed to create Freshdesk ticket: {content['errors']}")
             raise e
-        except NotImplementedError:
-            # There are cases in development when we do not want to send to freshdesk
-            # because configuration is not defined, lets return a 200 OK
-            current_app.logger.warning("Did not send ticket to Freshdesk")
-            return 200

app/clients/zendesk.py

@@ -0,0 +1,82 @@
+from typing import Dict, List, Union
+from urllib.parse import urljoin
+
+import requests
+from flask import current_app
+from requests.auth import HTTPBasicAuth
+
+from app.user.contact_request import ContactRequest
+
+__all__ = ["Zendesk"]
+
+
+class Zendesk(object):
+    def __init__(self, contact: ContactRequest):
+        self.api_url = current_app.config["ZENDESK_API_URL"]
+        self.token = current_app.config["ZENDESK_API_KEY"]
+        self.contact = contact
+
+    def _generate_description(self):
+        message = self.contact.message
+        if self.contact.is_go_live_request():
+            message = "<br>".join(
+                [
+                    f"{self.contact.service_name} just requested to go live.",
+                    "",
+                    f"- Department/org: {self.contact.department_org_name}",
+                    f"- Intended recipients: {self.contact.intended_recipients}",
+                    f"- Purpose: {self.contact.main_use_case}",
+                    f"- Notification types: {self.contact.notification_types}",
+                    f"- Expected monthly volume: {self.contact.expected_volume}",
+                    "---",
+                    self.contact.service_url,
+                ]
+            )
+        elif self.contact.is_branding_request():
+            message = "<br>".join(
+                [
+                    f"A new logo has been uploaded by {self.contact.name} ({self.contact.email_address}) for the following service:",
+                    f"- Service id: {self.contact.service_id}",
+                    f"- Service name: {self.contact.service_name}",
+                    f"- Logo filename: {self.contact.branding_url}",
+                    "<hr>",
+                    f"Un nouveau logo a été téléchargé par {self.contact.name} ({self.contact.email_address}) pour le service suivant :",
+                    f"- Identifiant du service : {self.contact.service_id}",
+                    f"- Nom du service : {self.contact.service_name}",
+                    f"- Nom du fichier du logo : {self.contact.branding_url}",
+                ]
+            )
+
+        if len(self.contact.user_profile):
+            message += f"<br><br>---<br><br> {self.contact.user_profile}"
+
+        return message
+
+    # Update for Zendesk API Ticket format
+    # read docs: https://developer.zendesk.com/rest_api/docs/core/tickets#create-ticket
+    def _generate_ticket(self) -> Dict[str, Dict[str, Union[str, int, List[str]]]]:
+        return {
+            "ticket": {
+                "subject": self.contact.friendly_support_type,
+                "description": self._generate_description(),
+                "email": self.contact.email_address,
+                "tags": self.contact.tags
+                + ["notification_api"],  # Custom tag used to auto-assign ticket to the notification support group
+            }
+        }
+
+    def send_ticket(self):
+        if not self.api_url or not self.token:
+            raise NotImplementedError
+
+        # The API and field definitions are defined here:
+        # https://developer.zendesk.com/rest_api/docs/support/tickets
+        response = requests.post(
+            urljoin(self.api_url, "/api/v2/tickets"),
+            json=self._generate_ticket(),
+            auth=HTTPBasicAuth(f"{self.contact.email_address}/token", self.token),
+            timeout=5,
+        )
+
+        if response.status_code != 201:
+            raise requests.HTTPError(response.status_code, "Failed to create zendesk ticket")

app/config.py

@@ -143,6 +143,7 @@ class Config(object):
     PERFORMANCE_PLATFORM_URL = "https://www.performance.service.gov.uk/data/govuk-notify/"
 
     # Zendesk
+    ZENDESK_API_URL = os.getenv("ZENDESK_API_URL")
     ZENDESK_API_KEY = os.getenv("ZENDESK_API_KEY")
     ZENDESK_SELL_API_URL = os.getenv("ZENDESK_SELL_API_URL")
     ZENDESK_SELL_API_KEY = os.getenv("ZENDESK_SELL_API_KEY")
@@ -491,6 +492,9 @@ class Test(Development):
 
     TEMPLATE_PREVIEW_API_HOST = "http://localhost:9999"
 
+    # FEATURE FLAGS
+    FF_BATCH_INSERTION = os.getenv("FF_BATCH_INSERTION", False)
+
 
 class Production(Config):
     NOTIFY_EMAIL_DOMAIN = os.getenv("NOTIFY_EMAIL_DOMAIN", "notification.canada.ca")