Merge branch 'main' into fix/formatter-check-in-ci

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/vscode/devcontainers/python:0-3.10@sha256:ef9cc483a593c95e1e83f2cf00b6a0e1ec7df43344416a41ccb3a88aef27beac
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.12
 
 ENV POETRY_VERSION="1.7.1"
 

.github/actions/waffles/Dockerfile

@@ -1,5 +1,5 @@
 # Container image that runs your code
-FROM ubuntu:22.04@sha256:2b7412e6465c3c7fc5bb21d3e6f1917c167358449fecac8176c6e496e5c1f05f
+FROM python:3.12
 
 RUN apt update
 RUN apt install -y python3-pip python3 git libmagic-dev

.github/actions/waffles/requirements.txt

@@ -1,4 +1,5 @@
 docopt==0.6.2
 Flask==2.3.3
 markupsafe==2.1.5
-git+https://github.com/cds-snc/[email protected]#egg=notifications-utils
+setuptools==75.6.0 # required for distutils in Python 3.12
+git+https://github.com/cds-snc/[email protected]#egg=notifications-utils

.github/workflows/ci.yaml

@@ -7,22 +7,28 @@ jobs:
 
     steps:
     - uses: actions/checkout@main
-    - name: docker://python:3.10
-      uses: docker://python:3.10  # utils is currently 3.10 so we need to use 3.10 in ci
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.12
 
     - name: Install poetry
       env:
         POETRY_VERSION: "1.7.1"
       run: pip install poetry==${POETRY_VERSION} poetry-plugin-sort && poetry --version
 
+    - name: Use Python 3.12 for Poetry
+      run: poetry env use python3.12
+
     - name: Install requirements
       run: poetry install
 
     - name: Copy site-packages in workspace
       working-directory: ${{ github.workspace }}
       shell: bash
       run: |
-        mkdir -p "${{ github.workspace }}/env/" && cp -fR $(poetry env list | poetry env info -p)/lib/python3.10/site-packages "${{ github.workspace }}/env/"
+        mkdir -p "${{ github.workspace }}/env/" && cp -fR $(poetry env list | poetry env info -p)/lib/python3.12/site-packages "${{ github.workspace }}/env/"
 
     - name: Bootstrap and run tests
       working-directory: ${{ github.workspace }}

Makefile

@@ -21,6 +21,10 @@ build: dependencies ## Build project
 test: ## Run tests
 	poetry run ./scripts/run_tests.sh
 
+.PHONY: freeze-requirements
+freeze-requirements:
+	poetry lock --no-update
+
 .PHONY: prepare-docker-build-image
 prepare-docker-build-image: ## Prepare the Docker builder image
 	make -C docker build

mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.10
+python_version = 3.12
 check_untyped_defs = True
 exclude = venv|build|site-packages
 

notifications_utils/formatters.py

@@ -31,10 +31,14 @@
 FR_CLOSE = r"\[\[/fr\]\]"  # matches [[/fr]]
 EN_OPEN = r"\[\[en\]\]"  # matches [[en]]
 EN_CLOSE = r"\[\[/en\]\]"  # matches [[/en]]
+RTL_OPEN = r"\[\[rtl\]\]"  # matches [[rtl]]
+RTL_CLOSE = r"\[\[/rtl\]\]"  # matches [[/rtl]]
 FR_OPEN_LITERAL = "[[fr]]"
 FR_CLOSE_LITERAL = "[[/fr]]"
 EN_OPEN_LITERAL = "[[en]]"
 EN_CLOSE_LITERAL = "[[/en]]"
+RTL_OPEN_LITERAL = "[[rtl]]"
+RTL_CLOSE_LITERAL = "[[/rtl]]"
 BR_TAG = r"<br\s?/>"
 
 
@@ -621,6 +625,20 @@ def escape_lang_tags(_content: str) -> str:
     return _content
 
 
+def escape_rtl_tags(_content: str) -> str:
+    """
+    Escape RTL tags into code tags in the content so mistune doesn't put them inside p tags.  This makes it simple
+    to replace them afterwards, and avoids creating invalid HTML in the process
+    """
+
+    # check to ensure we have the same number of opening and closing tags before escaping tags
+    if _content.count(RTL_OPEN_LITERAL) == _content.count(RTL_CLOSE_LITERAL):
+        _content = _content.replace(RTL_OPEN_LITERAL, f"\n```\n{RTL_OPEN_LITERAL}\n```\n")
+        _content = _content.replace(RTL_CLOSE_LITERAL, f"\n```\n{RTL_CLOSE_LITERAL}\n```\n")
+
+    return _content
+
+
 def add_language_divs(_content: str) -> str:
     """
     Custom parser to add the language divs.
@@ -646,6 +664,27 @@ def remove_language_divs(_content: str) -> str:
     return remove_tags(_content, FR_OPEN, FR_CLOSE, EN_OPEN, EN_CLOSE)
 
 
+def add_rtl_divs(_content: str) -> str:
+    """
+    Custom parser to add the language divs.
+
+    String replace language tags in-place
+    """
+
+    # check to ensure we have the same number of opening and closing tags before escaping tags
+    if _content.count(RTL_OPEN_LITERAL) == _content.count(RTL_CLOSE_LITERAL):
+        _content = _content.replace(RTL_OPEN_LITERAL, '<div dir="rtl">')
+        _content = _content.replace(RTL_CLOSE_LITERAL, "</div>")
+
+    return _content
+
+
+def remove_rtl_divs(_content: str) -> str:
+    """Remove the tags from content. This fn is for use in the email
+    preheader, since this is plain text not html"""
+    return remove_tags(_content, RTL_OPEN, RTL_CLOSE)
+
+
 def remove_tags(_content: str, *tags) -> str:
     """Remove the tags in parameters from content.
 

notifications_utils/template.py

@@ -13,10 +13,12 @@
 from notifications_utils.formatters import (
     add_language_divs,
     add_prefix,
+    add_rtl_divs,
     add_trailing_newline,
     autolink_sms,
     escape_html,
     escape_lang_tags,
+    escape_rtl_tags,
     make_quotes_smart,
     nl2br,
     nl2li,
@@ -28,6 +30,7 @@
     notify_plain_text_email_markdown,
     remove_empty_lines,
     remove_language_divs,
+    remove_rtl_divs,
     remove_smart_quotes_from_email_addresses,
     remove_whitespace_before_punctuation,
     replace_hyphens_with_en_dashes,
@@ -398,6 +401,7 @@ def preheader(self):
             .then(add_trailing_newline)
             .then(notify_email_preheader_markdown)
             .then(remove_language_divs)
+            .then(remove_rtl_divs)
             .then(do_nice_typography)
             .split()
         )[: self.PREHEADER_LENGTH_IN_CHARACTERS].strip()
@@ -805,8 +809,10 @@ def get_html_email_body(template_content, template_values, redact_missing_person
         .then(strip_unsupported_characters)
         .then(add_trailing_newline)
         .then(escape_lang_tags)
+        .then(escape_rtl_tags)
         .then(notify_email_markdown)
         .then(add_language_divs)
+        .then(add_rtl_divs)
         .then(do_nice_typography)
     )