Merge branch 'main' into feat/notify-user-failed-callbacks

.github/workflows/docker.yaml

@@ -10,6 +10,7 @@ env:
   DOCKER_SLUG: public.ecr.aws/v6b8u5o6/notify-api
   KUBECTL_VERSION: '1.23.6'
   WORKFLOW_PAT: ${{ secrets.WORKFLOW_GITHUB_PAT }}
+  OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
 
 permissions:
   id-token: write   # This is required for requesting the OIDC JWT
@@ -55,9 +56,51 @@ jobs:
       run: |
         docker push $DOCKER_SLUG:latest && docker push $DOCKER_SLUG:${GITHUB_SHA::7}
 
-    - name: Rollout in Kubernetes
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ca-central-1
+
+    - name: Install OpenVPN
+      run: |
+        sudo apt update
+        sudo apt install -y openvpn openvpn-systemd-resolved
+
+    - name: Install 1Pass CLI
+      run: |
+        curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
+        sudo dpkg -i 1pass.deb
+
+    - name: One Password Fetch
       run: |
-        ./scripts/callManifestsRollout.sh ${GITHUB_SHA::7}
+        op read op://4eyyuwddp6w4vxlabrr2i2duxm/"Staging Github Actions VPN"/notesPlain > /var/tmp/staging.ovpn
+
+    - name: Connect to VPN
+      uses: "kota65535/github-openvpn-connect-action@cd2ed8a90cc7b060dc4e001143e811b5f7ea0af5"
+      with:
+        config_file: /var/tmp/staging.ovpn
+        client_key: ${{ secrets.STAGING_OVPN_CLIENT_KEY }}
+        echo_config: false       
+
+    - name: Configure kubeconfig
+      run: |
+        aws eks update-kubeconfig --name notification-canada-ca-staging-eks-cluster     
+
+    - name: Update images in staging
+      run: |
+        DOCKER_TAG=${GITHUB_SHA::7}
+        kubectl set image deployment.apps/api api=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/celery-beat celery-beat=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/celery-sms celery-sms=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/celery-primary celery-primary=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/celery-scalable celery-scalable=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/celery-sms-send-primary celery-sms-send-primary=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/celery-sms-send-scalable celery-sms-send-scalable=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/celery-email-send-primary celery-email-send-primary=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config
+        kubectl set image deployment.apps/celery-email-send-scalable celery-email-send-scalable=$DOCKER_SLUG:$DOCKER_TAG -n=notification-canada-ca --kubeconfig=$HOME/.kube/config        
+
 
     - name: my-app-install token
       id: notify-pr-bot

.github/workflows/export_github_data.yml

@@ -14,7 +14,7 @@ jobs:
           DNS_PROXY_FORWARDTOSENTINEL: "true"
           DNS_PROXY_LOGANALYTICSWORKSPACEID: ${{ secrets.LOG_ANALYTICS_WORKSPACE_ID }}
           DNS_PROXY_LOGANALYTICSSHAREDKEY: ${{ secrets.LOG_ANALYTICS_WORKSPACE_KEY }}
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Export Data
         uses: cds-snc/github-repository-metadata-exporter@main
         with:

.github/workflows/ossf-scorecard.yml

@@ -20,12 +20,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@7699f539c2b9ff754039f0e173fdf1a4e4a1e143
+        uses: ossf/scorecard-action@8c9e2c1222f54716a1df7d7bbb245e2a045b4423
         with:
           results_file: ossf-results.json
           results_format: json

.github/workflows/s3-backup.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
 
     - name: Checkout
-      uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         fetch-depth: 0 # retrieve all history
 

app/__init__.py

@@ -153,10 +153,16 @@ def create_app(application, config=None):
     # Log the application configuration
     application.logger.info(f"Notify config: {config.get_safe_config()}")
 
-    # avoid circular imports by importing this file later
-    from app.commands import setup_commands
+    # avoid circular imports by importing these files later
+    from app.commands.bulk_db import setup_bulk_db_commands
+    from app.commands.deprecated import setup_deprecated_commands
+    from app.commands.support import setup_support_commands
+    from app.commands.test_data import setup_test_data_commands
 
-    setup_commands(application)
+    setup_support_commands(application)
+    setup_bulk_db_commands(application)
+    setup_test_data_commands(application)
+    setup_deprecated_commands(application)
 
     return application
 
@@ -201,6 +207,7 @@ def register_blueprint(application):
     from app.service.rest import service_blueprint
     from app.status.healthcheck import status as status_blueprint
     from app.template.rest import template_blueprint
+    from app.template.template_category_rest import template_category_blueprint
     from app.template_folder.rest import template_folder_blueprint
     from app.template_statistics.rest import (
         template_statistics as template_statistics_blueprint,
@@ -259,6 +266,8 @@ def register_blueprint(application):
 
     register_notify_blueprint(application, letter_branding_blueprint, requires_admin_auth)
 
+    register_notify_blueprint(application, template_category_blueprint, requires_admin_auth)
+
 
 def register_v2_blueprints(application):
     from app.authentication.auth import requires_auth

app/clients/freshdesk.py

@@ -77,7 +77,20 @@ def _generate_description(self):
                     f"- Texte alternatif français : {self.contact.alt_text_fr}",
                 ]
             )
-
+        elif self.contact.is_new_template_category_request():
+            message = "<br>".join(
+                [
+                    f"New template category request from {self.contact.name} ({self.contact.email_address}):",
+                    f"- Service id: {self.contact.service_id}",
+                    f"- New Template Category Request name: {self.contact.template_category_name_en}",
+                    f"- Template id request: {self.contact.template_id_link}",
+                    "<hr>",
+                    f"Demande de nouvelle catégorie de modèle de {self.contact.name} ({self.contact.email_address}):",
+                    f"- Identifiant du service: {self.contact.service_id}",
+                    f"- Nom de la nouvelle catégorie de modèle demandée: {self.contact.template_category_name_fr}",
+                    f"- Demande d'identifiant de modèle: {self.contact.template_id_link}",
+                ]
+            )
         if len(self.contact.user_profile):
             message += f"<br><br>---<br><br> {self.contact.user_profile}"
 

app/clients/sms/__init__.py

@@ -1,6 +1,13 @@
+from enum import Enum
+
 from app.clients import Client, ClientException
 
 
+class SmsSendingVehicles(Enum):
+    SHORT_CODE = "short_code"
+    LONG_CODE = "long_code"
+
+
 class SmsClientResponseException(ClientException):
     """
     Base Exception for SmsClientsResponses

app/clients/sms/aws_pinpoint.py

@@ -3,7 +3,7 @@
 import boto3
 import phonenumbers
 
-from app.clients.sms import SmsClient
+from app.clients.sms import SmsClient, SmsSendingVehicles
 
 
 class AwsPinpointClient(SmsClient):
@@ -21,38 +21,66 @@ def init_app(self, current_app, statsd_client, *args, **kwargs):
     def get_name(self):
         return self.name
 
-    def send_sms(self, to, content, reference, multi=True, sender=None, template_id=None):
+    def send_sms(self, to, content, reference, multi=True, sender=None, template_id=None, service_id=None, sending_vehicle=None):
         messageType = "TRANSACTIONAL"
         matched = False
+        opted_out = False
+        response = {}
 
-        if template_id is not None and str(template_id) in self.current_app.config["AWS_PINPOINT_SC_TEMPLATE_IDS"]:
+        if self.current_app.config["FF_TEMPLATE_CATEGORY"]:
+            use_shortcode_pool = (
+                sending_vehicle == SmsSendingVehicles.SHORT_CODE
+                or str(service_id) == self.current_app.config["NOTIFY_SERVICE_ID"]
+            )
+        else:
+            use_shortcode_pool = (
+                str(template_id) in self.current_app.config["AWS_PINPOINT_SC_TEMPLATE_IDS"]
+                or str(service_id) == self.current_app.config["NOTIFY_SERVICE_ID"]
+            )
+        if use_shortcode_pool:
             pool_id = self.current_app.config["AWS_PINPOINT_SC_POOL_ID"]
         else:
             pool_id = self.current_app.config["AWS_PINPOINT_DEFAULT_POOL_ID"]
 
         for match in phonenumbers.PhoneNumberMatcher(to, "US"):
             matched = True
+            opted_out = False
             to = phonenumbers.format_number(match.number, phonenumbers.PhoneNumberFormat.E164)
             destinationNumber = to
-
             try:
                 start_time = monotonic()
-                response = self._client.send_text_message(
-                    DestinationPhoneNumber=destinationNumber,
-                    OriginationIdentity=pool_id,
-                    MessageBody=content,
-                    MessageType=messageType,
-                    ConfigurationSetName=self.current_app.config["AWS_PINPOINT_CONFIGURATION_SET_NAME"],
-                )
+                # For international numbers we send with an AWS number for the corresponding country, using our default sender id.
+                # Note that Canada does not currently support sender ids.
+                if phonenumbers.region_code_for_number(match.number) != "CA":
+                    response = self._client.send_text_message(
+                        DestinationPhoneNumber=destinationNumber,
+                        MessageBody=content,
+                        MessageType=messageType,
+                        ConfigurationSetName=self.current_app.config["AWS_PINPOINT_CONFIGURATION_SET_NAME"],
+                    )
+                else:
+                    response = self._client.send_text_message(
+                        DestinationPhoneNumber=destinationNumber,
+                        OriginationIdentity=pool_id,
+                        MessageBody=content,
+                        MessageType=messageType,
+                        ConfigurationSetName=self.current_app.config["AWS_PINPOINT_CONFIGURATION_SET_NAME"],
+                    )
+            except self._client.exceptions.ConflictException as e:
+                if e.response.get("Reason") == "DESTINATION_PHONE_NUMBER_OPTED_OUT":
+                    opted_out = True
+                else:
+                    raise e
+
             except Exception as e:
                 self.statsd_client.incr("clients.pinpoint.error")
-                raise Exception(e)
+                raise e
             finally:
                 elapsed_time = monotonic() - start_time
                 self.current_app.logger.info("AWS Pinpoint request finished in {}".format(elapsed_time))
                 self.statsd_client.timing("clients.pinpoint.request-time", elapsed_time)
                 self.statsd_client.incr("clients.pinpoint.success")
-            return response["MessageId"]
+            return "opted_out" if opted_out else response.get("MessageId")
 
         if not matched:
             self.statsd_client.incr("clients.pinpoint.error")

app/clients/sms/aws_sns.py

@@ -26,7 +26,7 @@ def get_name(self):
         return self.name
 
     @statsd(namespace="clients.sns")
-    def send_sms(self, to, content, reference, multi=True, sender=None, template_id=None):
+    def send_sms(self, to, content, reference, multi=True, sender=None, template_id=None, service_id=None, sending_vehicle=None):
         matched = False
 
         for match in phonenumbers.PhoneNumberMatcher(to, "US"):

app/commands/README.md

@@ -0,0 +1,15 @@
+# Flask commands
+
+Changes to the database outside Notify admin/api should be done via flask commands versus `psql` or a database client.
+
+Flask commands must be run on a server that has the api repository installed and has access to the database, for example an api or celery pod in the kubernetes cluster.
+
+Commands are run by specifying the group and command. You must also have the environment variable `FLASK_APP` set correctly. For example
+```
+FLASK_APP=application flask support list-routes
+```
+
+We currently have 4 groups of commands available: `support`, `bulk-db`, `test-data`, and `deprecated`. To see what commands are available for a group run a command such as
+```
+FLASK_APP=application flask support
+```