Merge branch 'main' into task/pythonupgrade

.github/workflows/export_github_data.yml

@@ -14,7 +14,7 @@ jobs:
           DNS_PROXY_FORWARDTOSENTINEL: "true"
           DNS_PROXY_LOGANALYTICSWORKSPACEID: ${{ secrets.LOG_ANALYTICS_WORKSPACE_ID }}
           DNS_PROXY_LOGANALYTICSSHAREDKEY: ${{ secrets.LOG_ANALYTICS_WORKSPACE_KEY }}
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - name: Export Data
         uses: cds-snc/github-repository-metadata-exporter@main
         with:

.github/workflows/ossf-scorecard.yml

@@ -20,7 +20,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           persist-credentials: false
 

.github/workflows/s3-backup.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
 
     - name: Checkout
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       with:
         fetch-depth: 0 # retrieve all history
 

notifications_utils/clients/redis/annual_limit.py

@@ -0,0 +1,188 @@
+"""
+This module stores daily notification counts and annual limit statuses for a service in Redis using a hash structure:
+
+
+annual-limit: {
+    {service_id}: {
+        notifications: {
+            sms_delivered: int,
+            email_delivered: int,
+            sms_failed: int,
+            email_failed: int
+        },
+        status: {
+            near_sms_limit: Datetime,
+            near_email_limit: Datetime,
+            over_sms_limit: Datetime,
+            over_email_limit: Datetime
+            seeded_at: Datetime
+        }
+    }
+}
+
+
+"""
+
+from datetime import datetime
+
+from notifications_utils.clients.redis.redis_client import RedisClient
+
+SMS_DELIVERED = "sms_delivered"
+EMAIL_DELIVERED = "email_delivered"
+SMS_FAILED = "sms_failed"
+EMAIL_FAILED = "email_failed"
+
+NOTIFICATIONS = [SMS_DELIVERED, EMAIL_DELIVERED, SMS_FAILED, EMAIL_FAILED]
+
+NEAR_SMS_LIMIT = "near_sms_limit"
+NEAR_EMAIL_LIMIT = "near_email_limit"
+OVER_SMS_LIMIT = "over_sms_limit"
+OVER_EMAIL_LIMIT = "over_email_limit"
+SEEDED_AT = "seeded_at"
+
+STATUSES = [NEAR_SMS_LIMIT, NEAR_EMAIL_LIMIT, OVER_SMS_LIMIT, OVER_EMAIL_LIMIT]
+
+
+def annual_limit_notifications_key(service_id):
+    """
+    Generates the Redis hash key for storing daily metrics of a service.
+    """
+    return f"annual-limit:{service_id}:notifications"
+
+
+def annual_limit_status_key(service_id):
+    """
+    Generates the Redis hash key for storing annual limit information of a service.
+    """
+    return f"annual-limit:{service_id}:status"
+
+
+def decode_byte_dict(dict: dict, value_type=str):
+    """
+    Redis-py returns byte strings for keys and values. This function decodes them to UTF-8 strings.
+    """
+    # Check if expected_value_type is one of the allowed types
+    if value_type not in {int, float, str}:
+        raise ValueError("expected_value_type must be int, float, or str")
+    return {key.decode("utf-8"): value_type(value.decode("utf-8")) for key, value in dict.items() if dict.items()}
+
+
+class RedisAnnualLimit:
+    def __init__(self, redis: RedisClient):
+        self._redis_client = redis
+
+    def init_app(self, app, *args, **kwargs):
+        pass
+
+    def increment_notification_count(self, service_id: str, field: str):
+        self._redis_client.increment_hash_value(annual_limit_notifications_key(service_id), field)
+
+    def get_notification_count(self, service_id: str, field: str):
+        """
+        Retrieves the specified daily notification count for a service. (e.g. SMS_DELIVERED, EMAIL_FAILED, etc.)
+        """
+        return int(self._redis_client.get_hash_field(annual_limit_notifications_key(service_id), field))
+
+    def get_all_notification_counts(self, service_id: str):
+        """
+        Retrieves all daily notification metrics for a service.
+        """
+        return decode_byte_dict(self._redis_client.get_all_from_hash(annual_limit_notifications_key(service_id)), int)
+
+    def reset_all_notification_counts(self, service_ids=None):
+        """
+        Resets all daily notification metrics.
+        :param: service_ids: list of service_ids to reset, if None, resets all services
+        """
+        hashes = (
+            annual_limit_notifications_key("*")
+            if not service_ids
+            else [annual_limit_notifications_key(service_id) for service_id in service_ids]
+        )
+
+        self._redis_client.delete_hash_fields(hashes=hashes)
+
+    def seed_annual_limit_notifications(self, service_id: str, mapping: dict):
+        self._redis_client.bulk_set_hash_fields(key=annual_limit_notifications_key(service_id), mapping=mapping)
+
+    def was_seeded_today(self, service_id):
+        last_seeded_time = self.get_seeded_at(service_id)
+        return last_seeded_time == datetime.utcnow().strftime("%Y-%m-%d") if last_seeded_time else False
+
+    def get_seeded_at(self, service_id: str):
+        seeded_at = self._redis_client.get_hash_field(annual_limit_status_key(service_id), SEEDED_AT)
+        return seeded_at and seeded_at.decode("utf-8")
+
+    def set_seeded_at(self, service_id):
+        self._redis_client.set_hash_value(annual_limit_status_key(service_id), SEEDED_AT, datetime.utcnow().strftime("%Y-%m-%d"))
+
+    def clear_notification_counts(self, service_id: str):
+        """
+        Clears all daily notification metrics for a service.
+        """
+        self._redis_client.expire(annual_limit_notifications_key(service_id), -1)
+
+    def set_annual_limit_status(self, service_id: str, field: str, value: datetime):
+        """
+        Sets the status (e.g., 'nearing_limit', 'over_limit') in the annual limits Redis hash.
+        """
+        self._redis_client.set_hash_value(annual_limit_status_key(service_id), field, value.strftime("%Y-%m-%d"))
+
+    def get_annual_limit_status(self, service_id: str, field: str):
+        """
+        Retrieves the value of a specific annual limit status from the Redis hash.
+        """
+        response = self._redis_client.get_hash_field(annual_limit_status_key(service_id), field)
+        return response.decode("utf-8") if response is not None else None
+
+    def get_all_annual_limit_statuses(self, service_id: str):
+        return decode_byte_dict(self._redis_client.get_all_from_hash(annual_limit_status_key(service_id)))
+
+    def clear_annual_limit_statuses(self, service_id: str):
+        self._redis_client.expire(f"{annual_limit_status_key(service_id)}", -1)
+
+    # Helper methods for daily metrics
+    def increment_sms_delivered(self, service_id: str):
+        self.increment_notification_count(service_id, SMS_DELIVERED)
+
+    def increment_sms_failed(self, service_id: str):
+        self.increment_notification_count(service_id, SMS_FAILED)
+
+    def increment_email_delivered(self, service_id: str):
+        self.increment_notification_count(service_id, EMAIL_DELIVERED)
+
+    def increment_email_failed(self, service_id: str):
+        self.increment_notification_count(service_id, EMAIL_FAILED)
+
+    # Helper methods for annual limits
+    def set_nearing_sms_limit(self, service_id: str):
+        self.set_annual_limit_status(service_id, NEAR_SMS_LIMIT, datetime.utcnow())
+
+    def set_nearing_email_limit(self, service_id: str):
+        self.set_annual_limit_status(service_id, NEAR_EMAIL_LIMIT, datetime.utcnow())
+
+    def set_over_sms_limit(self, service_id: str):
+        self.set_annual_limit_status(service_id, OVER_SMS_LIMIT, datetime.utcnow())
+
+    def set_over_email_limit(self, service_id: str):
+        self.set_annual_limit_status(service_id, OVER_EMAIL_LIMIT, datetime.utcnow())
+
+    def check_has_warning_been_sent(self, service_id: str, message_type: str):
+        """
+        Check if an annual limit warning email has been sent to the service.
+        Returns None if no warning has been sent, otherwise returns the date the
+        last warning was issued.
+        When a service's annual limit is increased this value is reset.
+        """
+        field_to_fetch = NEAR_SMS_LIMIT if message_type == "sms" else NEAR_EMAIL_LIMIT
+        return self.get_annual_limit_status(service_id, field_to_fetch)
+
+    def check_has_over_limit_been_sent(self, service_id: str, message_type: str):
+        """
+        Check if an annual limit exceeded email has been sent to the service.
+        Returns None if no exceeded email has been sent, otherwise returns the date the
+        last exceeded email was issued.
+        When a service's annual limit is increased this value is reset.
+        """
+        field_to_fetch = OVER_SMS_LIMIT if message_type == "sms" else OVER_EMAIL_LIMIT
+        return self.get_annual_limit_status(service_id, field_to_fetch)

notifications_utils/clients/redis/redis_client.py

@@ -1,7 +1,7 @@
 import numbers
 import uuid
 from time import time
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 from flask import current_app
 from flask_redis import FlaskRedis
@@ -81,6 +81,64 @@ def delete_cache_keys_by_pattern(self, pattern):
             return self.scripts["delete-keys-by-pattern"](args=[pattern])
         return 0
 
+    # TODO: Refactor and simplify this to use HEXPIRE when we upgrade Redis to 7.4.0
+    def delete_hash_fields(self, hashes: (str | list), fields: Optional[list] = None, raise_exception=False):
+        """Deletes fields from the specified hashes. if fields is `None`, then all fields from the hashes are deleted, deleting the hash entirely.
+
+        Args:
+            hashes (str|list): The hash pattern or list of hash keys to delete fields from.
+            fields (list): A list of fields to delete from the hashes. If `None`, then all fields are deleted.
+
+        Returns:
+            _type_: _description_
+        """
+        if self.active:
+            try:
+                hashes = [prepare_value(h) for h in hashes] if isinstance(hashes, list) else prepare_value(hashes)
+                # When fields are passed in, use the list as is
+                # When hashes is a list, and no fields are passed in, fetch the fields from the first hash in the list
+                # otherwise we know we're going scan iterate over a pattern so we'll fetch the fields on the first pass in the loop below
+                fields = (
+                    [prepare_value(f) for f in fields]
+                    if fields is not None
+                    else self.redis_store.hkeys(hashes[0])
+                    if isinstance(hashes, list)
+                    else None
+                )
+                # Use a pipeline to atomically delete fields from each hash.
+                pipe = self.redis_store.pipeline()
+                # if hashes is not a list, we're scan iterating over keys matching a pattern
+                for key in hashes if isinstance(hashes, list) else self.redis_store.scan_iter(hashes):
+                    if not fields:
+                        fields = self.redis_store.hkeys(key)
+                    key = prepare_value(key)
+                    pipe.hdel(key, *fields)
+                result = pipe.execute()
+                # TODO: May need to double check that the pipeline result count matches the number of hashes deleted
+                # and retry any failures
+                return result
+            except Exception as e:
+                self.__handle_exception(e, raise_exception, "expire_hash_fields", hashes)
+        return False
+
+    def bulk_set_hash_fields(self, mapping, pattern=None, key=None, raise_exception=False):
+        """
+        Bulk set hash fields.
+        :param pattern: the pattern to match keys
+        :param mappting: the mapping of fields to set
+        :param raise_exception: True if we should allow the exception to bubble up
+        """
+        if self.active:
+            try:
+                if pattern:
+                    for key in self.redis_store.scan_iter(pattern):
+                        self.redis_store.hmset(key, mapping)
+                if key:
+                    return self.redis_store.hmset(key, mapping)
+            except Exception as e:
+                self.__handle_exception(e, raise_exception, "bulk_set_hash_fields", pattern)
+        return False
+
     def exceeded_rate_limit(self, cache_key, limit, interval, raise_exception=False):
         """
         Rate limiting.
@@ -228,17 +286,44 @@ def get(self, key, raise_exception=False):
 
         return None
 
+    def set_hash_value(self, key, field, value, raise_exception=False):
+        key = prepare_value(key)
+        field = prepare_value(field)
+        value = prepare_value(value)
+
+        if self.active:
+            try:
+                return self.redis_store.hset(key, field, value)
+            except Exception as e:
+                self.__handle_exception(e, raise_exception, "set_hash_value", key)
+
+        return None
+
     def decrement_hash_value(self, key, value, raise_exception=False):
         return self.increment_hash_value(key, value, raise_exception, incr_by=-1)
 
     def increment_hash_value(self, key, value, raise_exception=False, incr_by=1):
         key = prepare_value(key)
         value = prepare_value(value)
+
         if self.active:
             try:
                 return self.redis_store.hincrby(key, value, incr_by)
             except Exception as e:
                 self.__handle_exception(e, raise_exception, "increment_hash_value", key)
+        return None
+
+    def get_hash_field(self, key, field, raise_exception=False):
+        key = prepare_value(key)
+        field = prepare_value(field)
+
+        if self.active:
+            try:
+                return self.redis_store.hget(key, field)
+            except Exception as e:
+                self.__handle_exception(e, raise_exception, "get_hash_field", key)
+
+        return None
 
     def get_all_from_hash(self, key, raise_exception=False):
         key = prepare_value(key)
@@ -248,6 +333,8 @@ def get_all_from_hash(self, key, raise_exception=False):
             except Exception as e:
                 self.__handle_exception(e, raise_exception, "get_all_from_hash", key)
 
+        return None
+
     def set_hash_and_expire(self, key, values, expire_in_seconds, raise_exception=False):
         key = prepare_value(key)
         values = {prepare_value(k): prepare_value(v) for k, v in values.items()}
@@ -258,6 +345,8 @@ def set_hash_and_expire(self, key, values, expire_in_seconds, raise_exception=Fa
             except Exception as e:
                 self.__handle_exception(e, raise_exception, "set_hash_and_expire", key)
 
+        return None
+
     def expire(self, key, expire_in_seconds, raise_exception=False):
         key = prepare_value(key)
         if self.active:

notifications_utils/formatters.py

@@ -417,7 +417,7 @@ def list(self, body, ordered=True):
                 '<table role="presentation" style="padding: 0 0 20px 0;">'
                 "<tr>"
                 '<td style="font-family: Helvetica, Arial, sans-serif;">'
-                '<ol style="Margin: 0 0 0 20px; padding: 0; list-style-type: decimal;">'
+                '<ol style="margin: 0; padding: 0; list-style-type: decimal; margin-inline-start: 20px;">'
                 "{}"
                 "</ol>"
                 "</td>"
@@ -429,7 +429,7 @@ def list(self, body, ordered=True):
                 '<table role="presentation" style="padding: 0 0 20px 0;">'
                 "<tr>"
                 '<td style="font-family: Helvetica, Arial, sans-serif;">'
-                '<ul style="Margin: 0 0 0 20px; padding: 0; list-style-type: disc;">'
+                '<ul style="margin: 0; padding: 0; list-style-type: disc; margin-inline-start: 20px;">'
                 "{}"
                 "</ul>"
                 "</td>"
@@ -440,8 +440,8 @@ def list(self, body, ordered=True):
 
     def list_item(self, text):
         return (
-            '<li style="Margin: 5px 0 5px; padding: 0 0 0 5px; font-size: 19px;'
-            'line-height: 25px; color: #0B0C0C;">'
+            '<li style="Margin: 5px 0 5px; padding: 0 0 0 5px; font-size: 19px; '
+            'line-height: 25px; color: #0B0C0C; text-align:start;">'
             "{}"
             "</li>"
         ).format(text.strip())

notifications_utils/jinja_templates/email/email_preview_template.jinja2

@@ -75,7 +75,7 @@
         </p>
       {% endif %}
 
-      <div style="padding: 0 10px">
+      <div style="padding: 0 10px" dir="{{ 'rtl' if text_direction_rtl else 'ltr' }}">
         {{ body }}
       </div>