From 14881f7287e0d6f454c26d6223c9d4f4a4c61a2b Mon Sep 17 00:00:00 2001 From: David Kalbfleisch <1.21e9W@protonmail.com> Date: Wed, 18 Dec 2024 14:40:39 -0500 Subject: [PATCH] #169 Upgrade Mistune to v3 (#187) --- notifications_utils/field.py | 11 +- notifications_utils/formatters.py | 641 ++++------- notifications_utils/take.py | 4 - notifications_utils/template.py | 459 ++------ notifications_utils/template_change.py | 4 +- notifications_utils/version.py | 2 +- setup.py | 2 +- tests/test_base_template.py | 4 +- tests/test_formatted_list.py | 0 tests/test_formatters.py | 1212 +++++++++----------- tests/test_take.py | 25 - tests/test_template_types.py | 1407 ++++++------------------ 12 files changed, 1251 insertions(+), 2520 deletions(-) delete mode 100644 notifications_utils/take.py delete mode 100644 tests/test_formatted_list.py delete mode 100644 tests/test_take.py diff --git a/notifications_utils/field.py b/notifications_utils/field.py index 06bf0e86..4e5805a0 100644 --- a/notifications_utils/field.py +++ b/notifications_utils/field.py @@ -86,13 +86,11 @@ def __init__( markdown_lists=False, redact_missing_personalisation=False, preview_mode=False, - is_letter_template=False ): self.content = content self.values = values self.markdown_lists = markdown_lists self.preview_mode = preview_mode - self.is_letter_template = is_letter_template if not with_brackets: self.placeholder_tag = self.placeholder_tag_no_brackets if preview_mode: @@ -105,10 +103,10 @@ def __init__( }[html] self.redact_missing_personalisation = redact_missing_personalisation - def __str__(self): + def __str__(self) -> str: if self.values: return self.replaced - return self.formatted + return str(self.formatted) def __repr__(self): return "{}(\"{}\", {})".format(self.__class__.__name__, self.content, self.values) # TODO: more real @@ -155,11 +153,12 @@ def replace_match(self, match): elif replaced_value is not None: return self.get_replacement(placeholder) -# TODO: investigate why this fallback is necessary and potentially remove to enable truly conditional placeholders + # TODO - Investigate why this fallback is necessary, and potentially remove + # it to enable truly conditional placeholders. return self.format_match(match) def is_okay_to_have_null_values(self, placeholder) -> bool: - return self.redact_missing_personalisation or placeholder.is_conditional() or self.is_letter_template + return self.redact_missing_personalisation or placeholder.is_conditional() def get_replacement(self, placeholder): replacement = self.values.get(placeholder.name) diff --git a/notifications_utils/formatters.py b/notifications_utils/formatters.py index 6bcb65ed..d922fd02 100644 --- a/notifications_utils/formatters.py +++ b/notifications_utils/formatters.py @@ -1,18 +1,30 @@ import os -import string import re -import urllib +import string -import mistune import bleach -from itertools import count +import mistune +import smartypants from markupsafe import Markup +from mistune.renderers.html import HTMLRenderer +from mistune.renderers.markdown import MarkdownRenderer from . import email_with_smart_quotes_regex from notifications_utils.sanitise_text import SanitiseSMS -import smartypants -PARAGRAPH_STYLE = 'Margin: 0 0 20px 0; font-size: 16px; line-height: 25px; color: #323A45;' +ACTION_LINK_IMAGE_STYLE = 'vertical-align: middle;' +BLOCK_QUOTE_STYLE = 'background: #F1F1F1; ' \ + 'padding: 24px 24px 0.1px 24px; ' \ + 'font-family: Helvetica, Arial, sans-serif; ' \ + 'font-size: 16px; line-height: 25px;' +COLUMN_WIDTH = 65 LINK_STYLE = 'word-wrap: break-word; color: #004795;' +ORDERED_LIST_STYLE = 'Margin: 0 0 0 20px; padding: 0 0 20px 0; list-style-type: decimal; ' \ + 'font-family: Helvetica, Arial, sans-serif;' +LIST_ITEM_STYLE = 'Margin: 5px 0 5px; padding: 0 0 0 5px; font-size: 16px; line-height: 25px; color: #323A45;' +PARAGRAPH_STYLE = 'Margin: 0 0 20px 0; font-size: 16px; line-height: 25px; color: #323A45;' +THEMATIC_BREAK_STYLE = 'border: 0; height: 1px; background: #BFC1C3; Margin: 30px 0 30px 0;' +UNORDERED_LIST_STYLE = 'Margin: 0 0 0 20px; padding: 0 0 20px 0; list-style-type: disc; ' \ + 'font-family: Helvetica, Arial, sans-serif;' OBSCURE_WHITESPACE = ( '\u180E' # Mongolian vowel separator @@ -24,36 +36,6 @@ ) -mistune._block_quote_leading_pattern = re.compile(r'^ *\^ ?', flags=re.M) -mistune.BlockGrammar.block_quote = re.compile(r'^( *\^[^\n]+(\n[^\n]+)*\n*)+') -mistune.BlockGrammar.list_block = re.compile( - r'^( *)([•*-]|\d+\.)[\s\S]+?' - r'(?:' - r'\n+(?=\1?(?:[-*_] *){3,}(?:\n+|$))' # hrule - r'|\n+(?=%s)' # def links - r'|\n+(?=%s)' # def footnotes - r'|\n{2,}' - r'(?! )' - r'(?!\1(?:[•*-]|\d+\.) )\n*' - r'|' - r'\s*$)' % ( - mistune._pure_pattern(mistune.BlockGrammar.def_links), - mistune._pure_pattern(mistune.BlockGrammar.def_footnotes), - ) -) -mistune.BlockGrammar.list_item = re.compile( - r'^(( *)(?:[•*-]|\d+\.)[^\n]*' - r'(?:\n(?!\2(?:[•*-]|\d+\.))[^\n]*)*)', - flags=re.M -) -mistune.BlockGrammar.list_bullet = re.compile(r'^ *(?:[•*-]|\d+\.)') -mistune.InlineGrammar.url = re.compile(r'''^(https?:\/\/[^\s<]+[^<.,:"')\]\s])''') - -govuk_not_a_link = re.compile( - r'(?'.format(tag) for tag in { 'cr', 'h1', 'h2', 'p', 'normal', 'op', 'np', 'bul', 'tab' @@ -69,22 +51,6 @@ multiple_newlines = re.compile(r'((\n)\2{2,})') -MAGIC_SEQUENCE = "🇬🇧🐦✉️" - -magic_sequence_regex = re.compile(MAGIC_SEQUENCE) - -# The Mistune URL regex only matches URLs at the start of a string, -# using `^`, so we slice that off and recompile -url = re.compile(mistune.InlineGrammar.url.pattern[1:]) - - -def unlink_govuk_escaped(message): - return re.sub( - govuk_not_a_link, - r'\1' + '.\u200B' + r'\2', # Unicode zero-width space - message - ) - def nl2br(value): return re.sub(r'\n|\r', '
', value.strip()) @@ -96,15 +62,21 @@ def nl2li(value): )) -def add_prefix(body, prefix=None): +def add_prefix(body, prefix=None) -> str: if prefix: - return "{}: {}".format(prefix.strip(), body) + return f'{prefix.strip()}: {body}' return body +# The Mistune URL regex only matches URLs at the start of a string, +# using `^`, so we slice that off and recompile +# 17 DEC 2024: The above comment might be stale. +url = re.compile(r'''(https?:\/\/[^\s<]+[^<.,:"')\]\s])''') + + def autolink_sms(body): return url.sub( - lambda match: '{}'.format( + lambda match: '{}'.format( LINK_STYLE, match.group(1), match.group(1), ), @@ -121,14 +93,22 @@ def remove_empty_lines(lines): def sms_encode(content): - return SanitiseSMS.encode(content) + return SanitiseSMS.encode(str(content)) def strip_html(value): + """ + Calls to bleach.clean escapes HTML. This function strips and escapes the input. + """ + return bleach.clean(value, tags=[], strip=True) def escape_html(value): + """ + Calls to bleach.clean escapes HTML. This function escapes, but does not strip, the input. + """ + if not value: return value value = str(value).replace('<', '<') @@ -203,16 +183,20 @@ def strip_pipes(value): def remove_whitespace_before_punctuation(value): + """ + Remove spaces and tabs before various punctuation marks. + """ + return re.sub( whitespace_before_punctuation, lambda match: match.group(1), - value + str(value) ) def make_quotes_smart(value): return smartypants.smartypants( - value, + str(value), smartypants.Attr.q | smartypants.Attr.u ) @@ -245,7 +229,7 @@ def strip_leading_whitespace(value): def add_trailing_newline(value): - return '{}\n'.format(value) + return f'{value}\n' def tweak_dvla_list_markup(value): @@ -283,86 +267,96 @@ def strip_unsupported_characters(value): return value.replace('\u2028', '') -def normalise_whitespace(value): - # leading and trailing whitespace removed, all inner whitespace becomes a single space - return ' '.join(strip_and_remove_obscure_whitespace(value).split()) +def strip_unsupported_characters_in_preheader(value): + """ + Preheaders should not contain headers or links, and unordered lists should use the literal bullet. + """ + # No headers + value = re.sub(r'''^(\s+)#''', '', value, flags=re.M) -def get_action_links(html: str) -> list[str]: - """Get the action links from the html email body and return them as a list. (insert_action_link helper)""" - # set regex to find action link in html, should look like this: - # >>link_text - action_link_regex = re.compile( - r'(>|(>)){2}()(.*?)' - ) + # No links (regular or action) + value = re.sub(r'''((>|>){2})?\[([\w -]+)\]\(\S+\)''', r'\3', value) + + # Bullets for unordered lists + value = re.sub(r'''^(\s*)[-+*]''', r'\1•', value, flags=re.M) + + return value + + +def normalise_whitespace(value): + """ + Remove leading and trailing whitespace. All inner whitespace becomes a single space. + """ - return re.findall(action_link_regex, html) + return ' '.join(strip_and_remove_obscure_whitespace(value).split()) def get_action_link_image_url() -> str: - """Get action link image url for the current environment. (insert_action_link helper)""" + """Get the action link image url for the current environment.""" + env_map = { 'production': 'prod', 'staging': 'staging', 'performance': 'staging', } - # default to dev if NOTIFY_ENVIRONMENT isn't provided + img_env = env_map.get(os.environ.get('NOTIFY_ENVIRONMENT'), 'dev') return f'https://{img_env}-va-gov-assets.s3-us-gov-west-1.amazonaws.com/img/vanotify-action-link.png' def insert_action_link(html: str) -> str: """ - Finds an action link and replaces it with the desired format. The action link is placed on it's own line, the link - image is inserted into the link, and the styling is updated appropriately. + Finds an "action link," and replaces it with the desired format. This preprocessing should take place before + any manipulation by Mistune. + + Given: + >>[text](url) + + Output: + \n\ncall to action img text\n\n + """ + + img_src = get_action_link_image_url() + substitution = r'\n\n' \ + fr'call to action img ' \ + r'\2\n\n' + + # text url + return re.sub(r'''(>|>){2}\[([\w -]+)\]\((\S+)\)''', substitution, html) + + +def insert_block_quotes(md: str) -> str: """ - # common html used - p_start = f'

' - p_end = '

' - - action_link_list = get_action_links(html) - - img_link = get_action_link_image_url() - - for item in action_link_list: - # Puts the action link in a new

tag with appropriate styling. - # item[0] and item[1] values will be '>' symbols - # item[2] is the html link tag info - # item[-1] is the link text and end of the link tag - action_link = ( - f'{item[2]}call to action img {item[-1][:-4]}' - ) + Template markup uses ^ to denote a block quote, but Github markdown, which Mistune reflects, specifies a block + quote with the > character. Rather than write a custom parser, templates should preprocess their text to replace + the former with the latter. This preprocessing should take place before any manipulation by Mistune. - action_link_p_tags = f'{p_start}{action_link}{p_end}' - - # get the text around the action link if there is any - # ensure there are only two items in list with maxsplit - before_link, after_link = html.split("".join(item), maxsplit=1) - - # value is the converted action link if there's nothing around it, otherwise

tags will need to be - # closed / open around the action link - if before_link == p_start and after_link == p_end: - # action link exists on its own, unlikely to happen - html = action_link_p_tags - elif before_link.endswith(p_start) and after_link.startswith(p_end): - # an action link on it's own line, as it should be - html = f'{before_link}{action_link}{after_link}' - elif before_link.endswith(p_start): - # action link is on a newline, but has something after it on that line - html = f'{before_link}{action_link}{p_end}{p_start}{after_link}' - elif after_link == p_end: - # paragraph ends with action link - html = f'{before_link}{"

" if "" if " This is a block quote. + """ + + return re.sub(r'''^(\s*)\^(\s*)''', r'''\1>\2''', md, flags=re.M) + + +def insert_list_spaces(md: str) -> str: + """ + Proper markdown for lists has a space after the number or bullet. This is a preprocessing step to insert + any missing spaces in lists. This preprocessing should take place before any manipulation by Mistune. + + The regular expression for unordered lists replaces the bullet with the minus, which Mistune handles. + This is necessary because Utils allows the non-standard literal • in markdown to denote an unordered list. + Performing this substitution avoids having to write custom parsing logic for Mistune. + """ + + # Ordered lists + md = re.sub(r'''^(\s*)(\d+\.)(?=\S)''', r'''\1\2 ''', md, flags=re.M) + + # Unordered lists + return re.sub(r'''^(\s*)(\*|-|\+|•)(?!\2)(\s*)''', r'''\1- ''', md, flags=re.M) def strip_parentheses_in_link_placeholders(value: str) -> str: @@ -420,348 +414,157 @@ def replace_symbols_with_placeholder_parens(value: str) -> str: return value -class NotifyLetterMarkdownPreviewRenderer(mistune.Renderer): - - def block_code(self, code, language=None): - return code +class NotifyHTMLRenderer(HTMLRenderer): + def action_link(self, text, url): + raise NotImplementedError('MADE IT HERE') def block_quote(self, text): - return text - - def header(self, text, level, raw=None): - if level == 1: - return super().header(text, 2) - return self.paragraph(text) - - def hrule(self): - return '
 
' - - def paragraph(self, text): - if text.strip(): - return '

{}

'.format(text) - return '' - - def table(self, header, body): - return "" - - def autolink(self, link, is_email=False): - return '{}'.format( - link.replace('http://', '').replace('https://', '') - ) - - def codespan(self, text): - return text - - def double_emphasis(self, text): - return text - - def emphasis(self, text): - return text - - def image(self, src, title, alt_text): - return "" - - def linebreak(self): - return "
" - - def newline(self): - return self.linebreak() - - def list_item(self, text): - return '
  • {}
  • \n'.format(text.strip()) - - def link(self, link, title, content): - return '{}: {}'.format(content, self.autolink(link)) - - def strikethrough(self, text): - return text - - def footnote_ref(self, key, index): - return "" - - def footnote_item(self, key, text): - return text - - def footnotes(self, text): - return text - - -class NotifyEmailMarkdownRenderer(NotifyLetterMarkdownPreviewRenderer): + value = super().block_quote(text) + return value[:11] + f' style="{BLOCK_QUOTE_STYLE}"' + value[11:] - def header(self, text, level, raw=None): + def heading(self, text, level, **attrs): if level == 1: - return ( - '

    ' - '{}' - '

    ' - ).format( - text - ) + style = 'Margin: 0 0 20px 0; padding: 0; font-size: 32px; ' \ + 'line-height: 35px; font-weight: bold; color: #323A45;' elif level == 2: - return ( - '

    ' - '{}' - '

    ' - ).format( - text - ) + style = 'Margin: 0 0 15px 0; padding: 0; line-height: 26px; color: #323A45; ' \ + 'font-size: 24px; font-weight: bold; font-family: Helvetica, Arial, sans-serif;' elif level == 3: - return ( - '

    ' - '{}' - '

    ' - ).format( - text - ) - return self.paragraph(text) - - def hrule(self): - return ( - '
    ' - ) + style = 'Margin: 0 0 15px 0; padding: 0; line-height: 26px; color: #323A45; ' \ + 'font-size: 20.8px; font-weight: bold; font-family: Helvetica, Arial, sans-serif;' + else: + return self.paragraph(text) - def linebreak(self): - return "
    " + value = super().heading(text, level, **attrs) + return value[:3] + f' style="{style}"' + value[3:] - def list(self, body, ordered=True): - return ( - '' - '' - '' - '' - '
    ' - '
      ' - '{}' - '
    ' - '
    ' - ).format( - body - ) if ordered else ( - '' - '' - '' - '' - '
    ' - '
      ' - '{}' - '
    ' - '
    ' - ).format( - body - ) + def image(self, alt, url, title=None): + """ + VA e-mail messages generally contain only 1 header image that is not managed by clients. + There is also an image associated with "action links", but action links are handled + in preprocessing. (See insert_action_link above.) + """ - def list_item(self, text): - return ( - '
  • ' - '{}' - '
  • ' - ).format( - text.strip() - ) + return '' - def paragraph(self, text, is_inside_list=False): - margin = 'Margin: 5px 0 5px 0' if is_inside_list else 'Margin: 0 0 20px 0' - if text.strip(): - return f'

    {text}

    ' - return "" + def link(self, text, url, title=None): + """ + Add CSS to links. + """ - def block_quote(self, text): - return ( - '' - '' - '
    ' - '{}' - '
    ' - ).format( - text - ) + value = super().link(text, url, title) + return value[:2] + f' style="{LINK_STYLE}" target="_blank"' + value[2:] - def link(self, link, title, content): - return ( - '{}' - ).format( - LINK_STYLE, - ' href="{}"'.format(link), - ' title="{}"'.format(title) if title else "", - content, - ) + def list(self, text, ordered, **attrs): + value = super().list(text, ordered, **attrs) + style = ORDERED_LIST_STYLE if ordered else UNORDERED_LIST_STYLE + return value[:3] + f' role="presentation" style="{style}"' + value[3:] - def autolink(self, link, is_email=False): - if is_email: - return link - return '{}'.format( - LINK_STYLE, - urllib.parse.quote( - urllib.parse.unquote(link), - safe=':/?#=&;' - ), - link - ) + def list_item(self, text, **attrs): + value = super().list_item(text, **attrs) + return value[:3] + f' style="{LIST_ITEM_STYLE}"' + value[3:] - def double_emphasis(self, text): - return '{}'.format(text) - - def emphasis(self, text): - return '{}'.format(text) + def paragraph(self, text): + """ + Add CSS to paragraphs. + """ + value = super().paragraph(text) -class NotifyPlainTextEmailMarkdownRenderer(NotifyEmailMarkdownRenderer): + if value == '

    \n': + # This is the case when all child elements, such as tables and images, are deleted. + return '' - COLUMN_WIDTH = 65 + return value[:2] + f' style="{PARAGRAPH_STYLE}"' + value[2:] - def header(self, text, level, raw=None): - if level == 1: - return ''.join(( - self.linebreak() * 3, - text, - self.linebreak(), - '-' * self.COLUMN_WIDTH, - )) - elif level in (2, 3): - return ''.join(( - self.linebreak() * 2, - text, - self.linebreak(), - '-' * self.COLUMN_WIDTH - )) - return self.paragraph(text) - - def hrule(self): - return self.paragraph( - '=' * self.COLUMN_WIDTH - ) + def table(self, text): + """ + Delete tables. + """ - def linebreak(self): - return '\n' - - def list(self, body, ordered=True): - - def _get_list_marker(): - decimal = count(1) - return lambda _: '{}.'.format(next(decimal)) if ordered else '•' - - return ''.join(( - self.linebreak(), - re.sub( - magic_sequence_regex, - _get_list_marker(), - body, - ), - )) - - def list_item(self, text): - return ''.join(( - self.linebreak(), - MAGIC_SEQUENCE, - ' ', - text.strip(), - )) - - def paragraph(self, text, is_inside_list=False): - if text.strip(): - return ''.join(( - self.linebreak() * 2, - text, - )) - return "" + return '' - def block_quote(self, text): - return text + def thematic_break(self): + """ + Thematic breaks were known as horizontal rules (hrule) in earlier versions of Mistune. + """ - def link(self, link, title, content): - return ''.join(( - content, - ' ({})'.format(title) if title else '', - ': ', - link, - )) + value = super().thematic_break() + return value[:3] + f' style="{THEMATIC_BREAK_STYLE}"' + value[3:] - def autolink(self, link, is_email=False): - return link - def double_emphasis(self, text): - return text +class NotifyMarkdownRenderer(MarkdownRenderer): + def block_quote(self, token, state): + return '\n\n' + super().block_quote(token, state)[2:] - def emphasis(self, text): - return text + def heading(self, token, state): + level = token['attrs']['level'] + if level > 3: + token['type'] = 'paragraph' + return self.paragraph(token, state) -class NotifyEmailPreheaderMarkdownRenderer(NotifyPlainTextEmailMarkdownRenderer): + value = super().heading(token, state) + indentation = 3 if level == 1 else 2 + return ('\n' * indentation) + value.strip('#\n ') + '\n' + ('-' * COLUMN_WIDTH) + '\n' - def header(self, text, level, raw=None): - return self.paragraph(text) + def image(self, token, state): + """ + Delete images. VA e-mail messages contain only 1 image that is not managed by clients. + """ - def hrule(self): return '' - def link(self, link, title, content): - return ''.join(( - content, - ' ({})'.format(title) if title else '', - )) + def link(self, token, state): + """ + Input: + [text](url) + Output: + text: url + """ + return self.render_children(token, state) + ': ' + token['attrs']['url'] -class NotifyEmailBlockLexer(mistune.BlockLexer): + def list(self, token, state): + """ + Use the bullet character as the actual bullet output for all input (asterisks, pluses, and minues) + when the list is unordered. + """ - def __init__(self, rules=None, **kwargs): - super().__init__(rules, **kwargs) + if not token['attrs']['ordered']: + token['bullet'] = '•' - def parse_newline(self, m): - if self._list_depth == 0: - super().parse_newline(m) + return super().list(token, state) + def strikethrough(self, token, state): + """ + https://mistune.lepture.com/en/latest/renderers.html#with-plugins + """ -class NotifyEmailMarkdown(mistune.Markdown): + return '\n\n' + self.render_children(token, state) - def __init__(self, renderer=None, inline=None, block=None, **kwargs): - super().__init__(renderer, inline, block, **kwargs) - self._is_inside_list = False + def table(self, token, state): + """ + Delete tables. + """ - def output_loose_item(self): - body = self.renderer.placeholder() - self._is_inside_list = True - while self.pop()['type'] != 'list_item_end': - body += self.tok() + return '' - self._is_inside_list = False - return self.renderer.list_item(body) + def thematic_break(self, token, state): + """ + Thematic breaks were known as horizontal rules (hrule) in earlier versions of Mistune. + """ - def tok_text(self): - if self._is_inside_list: - return self.inline(self.token['text']) - else: - return super().tok_text() + return '=' * COLUMN_WIDTH + '\n' - def output_text(self): - return self.renderer.paragraph(self.tok_text(), self._is_inside_list) - -notify_email_markdown = NotifyEmailMarkdown( - renderer=NotifyEmailMarkdownRenderer(), - block=NotifyEmailBlockLexer, - hard_wrap=True, - use_xhtml=False, -) -notify_plain_text_email_markdown = mistune.Markdown( - renderer=NotifyPlainTextEmailMarkdownRenderer(), +notify_html_markdown = mistune.create_markdown( hard_wrap=True, + renderer=NotifyHTMLRenderer(escape=False), + plugins=['strikethrough', 'table', 'url'], ) -notify_email_preheader_markdown = mistune.Markdown( - renderer=NotifyEmailPreheaderMarkdownRenderer(), - hard_wrap=True, -) -notify_letter_preview_markdown = mistune.Markdown( - renderer=NotifyLetterMarkdownPreviewRenderer(), - hard_wrap=True, - use_xhtml=False, + +notify_markdown = mistune.create_markdown( + renderer=NotifyMarkdownRenderer(), + plugins=['strikethrough', 'table'], ) diff --git a/notifications_utils/take.py b/notifications_utils/take.py deleted file mode 100644 index 732f0c1d..00000000 --- a/notifications_utils/take.py +++ /dev/null @@ -1,4 +0,0 @@ -class Take(str): - - def then(self, func, *args, **kwargs): - return self.__class__(func(self, *args, **kwargs)) diff --git a/notifications_utils/template.py b/notifications_utils/template.py index a207110f..ab82342a 100644 --- a/notifications_utils/template.py +++ b/notifications_utils/template.py @@ -1,6 +1,5 @@ import math import sys -from datetime import datetime from html import unescape from os import path @@ -15,30 +14,25 @@ add_trailing_newline, autolink_sms, escape_html, insert_action_link, + insert_block_quotes, + insert_list_spaces, make_quotes_smart, nl2br, - nl2li, normalise_newlines, normalise_whitespace, - notify_email_markdown, - notify_email_preheader_markdown, - notify_letter_preview_markdown, - notify_plain_text_email_markdown, - remove_empty_lines, + notify_html_markdown, + notify_markdown, remove_smart_quotes_from_email_addresses, remove_whitespace_before_punctuation, replace_hyphens_with_en_dashes, - replace_hyphens_with_non_breaking_hyphens, replace_symbols_with_placeholder_parens, - sms_encode, strip_dvla_markup, + sms_encode, strip_leading_whitespace, - strip_pipes, strip_parentheses_in_link_placeholders, strip_unsupported_characters, - tweak_dvla_list_markup, - unlink_govuk_escaped) + strip_unsupported_characters_in_preheader, +) from notifications_utils.sanitise_text import SanitiseSMS -from notifications_utils.take import Take from notifications_utils.template_change import TemplateChange template_env = Environment(loader=FileSystemLoader( @@ -49,6 +43,17 @@ )) +def compose1(value, *fs): + """ + Return the composition of functions applied to a single value. + """ + + return_value = value + for f in fs: + return_value = f(return_value) + return return_value + + class Template(): encoding = "utf-8" @@ -108,16 +113,16 @@ def values(self, value): if not value: self._values = {} else: - placeholders = Columns.from_keys(self.placeholders) + placeholders = Columns.from_keys(self.placeholder_names) self._values = Columns(value).as_dict_with_keys( - self.placeholders | set( + self.placeholder_names | set( key for key in value.keys() if Columns.make_key(key) not in placeholders.keys() ) ) @property - def placeholders(self): # TODO: rename to placeholder_names + def placeholder_names(self): return Field(self.content).placeholder_names @property @@ -129,7 +134,7 @@ def missing_data(self): @property def additional_data(self): - return self.values.keys() - self.placeholders + return self.values.keys() - self.placeholder_names def get_raw(self, key, default=None): return self._template.get(key, default) @@ -158,18 +163,14 @@ def __init__( super().__init__(template, values, jinja_path=jinja_path) def __str__(self): - return Take(Field( - self.content, self.values, html='passthrough' - )).then( - add_prefix, self.prefix - ).then( - sms_encode - ).then( - remove_whitespace_before_punctuation - ).then( - normalise_newlines - ).then( - str.strip + field = str(Field(self.content, self.values, html='passthrough')) + field = add_prefix(field, self.prefix) + return compose1( + field, + sms_encode, + remove_whitespace_before_punctuation, + normalise_newlines, + str.strip, ) @property @@ -182,12 +183,12 @@ def prefix(self, value): @property def content_count(self): - return len(( - # we always want to call SMSMessageTemplate.__str__ regardless of subclass, to avoid any html formatting - SMSMessageTemplate.__str__(self) - if self._values - else sms_encode(add_prefix(self.content.strip(), self.prefix)) - ).encode(self.encoding)) + if self._values: + # Always call SMSMessageTemplate.__str__, regardless of subclass, to avoid any html formatting. + content = SMSMessageTemplate.__str__(self) + else: + content = sms_encode(add_prefix(self.content.strip(), self.prefix)) + return len(content) @property def fragment_count(self): @@ -220,30 +221,28 @@ def __init__( self.redact_missing_personalisation = redact_missing_personalisation self.jinja_template = self.template_env.get_template('sms_preview_template.jinja2') - def __str__(self): + def __str__(self) -> str: + field = str(Field( + self.content, + self.values, + html='escape', + redact_missing_personalisation=self.redact_missing_personalisation, + )) + field = add_prefix(field, escape_html(self.prefix) if self.show_prefix else None) - return Markup(self.jinja_template.render({ + return str(Markup(self.jinja_template.render({ 'sender': self.sender, 'show_sender': self.show_sender, 'recipient': Field('((phone number))', self.values, with_brackets=False, html='escape'), 'show_recipient': self.show_recipient, - 'body': Take(Field( - self.content, - self.values, - html='escape', - redact_missing_personalisation=self.redact_missing_personalisation, - )).then( - add_prefix, (escape_html(self.prefix) or None) if self.show_prefix else None - ).then( - sms_encode if self.downgrade_non_sms_characters else str - ).then( - remove_whitespace_before_punctuation - ).then( - nl2br - ).then( - autolink_sms + 'body': compose1( + field, + sms_encode if self.downgrade_non_sms_characters else str, + remove_whitespace_before_punctuation, + nl2br, + autolink_sms, ) - })) + }))) class WithSubjectTemplate(Template): @@ -272,57 +271,46 @@ def __str__(self): @property def subject(self): - return Markup(Take(Field( + field = Field( self._subject, self.values, html='escape', redact_missing_personalisation=self.redact_missing_personalisation, - )).then( - do_nice_typography - ).then( - normalise_whitespace - )) + ) + return Markup(compose1(field, do_nice_typography, normalise_whitespace)) @property - def placeholders(self): + def placeholder_names(self): return Field(self._subject).placeholder_names | Field(self.content).placeholder_names class PlainTextEmailTemplate(WithSubjectTemplate): def __str__(self): - return Take(Field( - self.content, self.values, html='passthrough', markdown_lists=True - )).then( - unlink_govuk_escaped - ).then( - strip_unsupported_characters - ).then( - add_trailing_newline - ).then( - notify_plain_text_email_markdown - ).then( - do_nice_typography - ).then( - unescape - ).then( - strip_leading_whitespace - ).then( - add_trailing_newline + field = str(Field(self.content, self.values, html='passthrough', markdown_lists=True)) + return compose1( + field, + strip_unsupported_characters, + add_trailing_newline, + insert_block_quotes, + insert_list_spaces, + notify_markdown, + do_nice_typography, + unescape, + strip_leading_whitespace, + add_trailing_newline, ) @property def subject(self): - return Markup(Take(Field( + field = Field( self._subject, self.values, html='passthrough', redact_missing_personalisation=self.redact_missing_personalisation - )).then( - do_nice_typography - ).then( - normalise_whitespace - )) + ) + field = compose1(field, do_nice_typography, normalise_whitespace) + return Markup(field) class HTMLEmailTemplate(WithSubjectTemplate): @@ -359,36 +347,32 @@ def __init__( self.ga_pixel_url = ga_pixel_url self.ga4_open_email_event_url = ga4_open_email_event_url self.preview_mode = preview_mode - # set this again to make sure the correct either utils / downstream local jinja is used - # however, don't set if we are in a test environment (to preserve the above mock) + if "pytest" not in sys.modules: + # Ensure use of the correct utils or downstream local jinja template. self.jinja_template = self.template_env.get_template('email_template.jinja2') + # Else, this is a test environment. The above mock prevails. @property def preheader(self): - return " ".join(Take(Field( + field = str(Field( self.content, self.values, html='escape', markdown_lists=True - )).then( - unlink_govuk_escaped - ).then( - strip_unsupported_characters - ).then( - add_trailing_newline - ).then( - notify_email_preheader_markdown - ).then( - do_nice_typography - ).split())[:self.PREHEADER_LENGTH_IN_CHARACTERS].strip() + )) + field = compose1( + field, + strip_unsupported_characters, + add_trailing_newline, + strip_unsupported_characters_in_preheader, + do_nice_typography, + ).split() + return ' '.join(field)[:self.PREHEADER_LENGTH_IN_CHARACTERS].strip() def __str__(self): - return self.jinja_template.render({ - 'body': get_html_email_body( - self.content, self.values, preview_mode=self.preview_mode - ), + 'body': get_html_email_body(self.content, self.values, preview_mode=self.preview_mode), 'preheader': self.preheader if not self.preview_mode else '', 'default_banner': self.default_banner, 'complete_html': self.complete_html, @@ -444,224 +428,13 @@ def __str__(self): @property def subject(self): - return Take(Field( + field = Field( self._subject, self.values, html='escape', redact_missing_personalisation=self.redact_missing_personalisation - )).then( - do_nice_typography - ).then( - normalise_whitespace - ) - - -class LetterPreviewTemplate(WithSubjectTemplate): - - jinja_template = template_env.get_template('letter_pdf/preview.jinja2') - - address_block = '\n'.join([ - '((address line 1))', - '((address line 2))', - '((address line 3))', - '((address line 4))', - '((address line 5))', - '((address line 6))', - '((postcode))', - ]) - - def __init__( - self, - template, - values=None, - contact_block=None, - admin_base_url='http://localhost:6012', - logo_file_name=None, - redact_missing_personalisation=False, - date=None, - ): - self.contact_block = (contact_block or '').strip() - super().__init__(template, values, redact_missing_personalisation=redact_missing_personalisation) - self.admin_base_url = admin_base_url - self.logo_file_name = logo_file_name - self.date = date or datetime.utcnow() - - def __str__(self): - return Markup(self.jinja_template.render({ - 'admin_base_url': self.admin_base_url, - 'logo_file_name': self.logo_file_name, - # logo_class should only ever be None, svg or png - 'logo_class': self.logo_file_name.lower()[-3:] if self.logo_file_name else None, - 'subject': self.subject, - 'message': self._message, - 'address': self._address_block, - 'contact_block': self._contact_block, - 'date': self._date, - })) - - @property - def subject(self): - return Take(Field( - self._subject, - self.values, - redact_missing_personalisation=self.redact_missing_personalisation, - html='escape', - is_letter_template=True - )).then( - do_nice_typography - ).then( - strip_pipes - ).then( - strip_dvla_markup - ).then( - normalise_whitespace - ) - - @property - def placeholders(self): - return super().placeholders | Field(self.contact_block, is_letter_template=True).placeholder_names - - @property - def values_with_default_optional_address_lines(self): - keys = Columns.from_keys( - set(self.values.keys()) | { - 'address line 3', - 'address line 4', - 'address line 5', - 'address line 6', - } - ).keys() - - return { - key: Columns(self.values).get(key) or '' - for key in keys - } - - @property - def _address_block(self): - return Take(Field( - self.address_block, - ( - self.values_with_default_optional_address_lines - if all(Columns(self.values).get(key) for key in { - 'address line 1', - 'address line 2', - 'postcode', - }) else self.values - ), - html='escape', - with_brackets=False, - is_letter_template=True - )).then( - strip_pipes - ).then( - remove_empty_lines - ).then( - remove_whitespace_before_punctuation - ).then( - nl2li - ) - - @property - def _contact_block(self): - return Take(Field( - '\n'.join( - line.strip() - for line in self.contact_block.split('\n') - ), - self.values, - redact_missing_personalisation=self.redact_missing_personalisation, - html='escape', - )).then( - remove_whitespace_before_punctuation - ).then( - nl2br - ).then( - strip_pipes - ) - - @property - def _date(self): - return self.date.strftime('%-d %B %Y') - - @property - def _message(self): - return Take(Field( - strip_dvla_markup(self.content), - self.values, - html='escape', - markdown_lists=True, - redact_missing_personalisation=self.redact_missing_personalisation, - )).then( - strip_pipes - ).then( - add_trailing_newline - ).then( - notify_letter_preview_markdown - ).then( - do_nice_typography - ).then( - replace_hyphens_with_non_breaking_hyphens - ).then( - tweak_dvla_list_markup ) - - -class LetterPrintTemplate(LetterPreviewTemplate): - - jinja_template = template_env.get_template('letter_pdf/print.jinja2') - - -class LetterImageTemplate(LetterPreviewTemplate): - - jinja_template = template_env.get_template('letter_image_template.jinja2') - first_page_number = 1 - max_page_count = 10 - - def __init__( - self, - template, - values=None, - image_url=None, - page_count=None, - contact_block=None, - postage='second', - ): - super().__init__(template, values, contact_block=contact_block) - if not image_url: - raise TypeError('image_url is required') - if not page_count: - raise TypeError('page_count is required') - if postage not in {'first', 'second'}: - raise TypeError('postage must be first or second') - self.image_url = image_url - self.page_count = int(page_count) - self.postage = postage - - @property - def last_page_number(self): - return min(self.page_count, self.max_page_count) + self.first_page_number - - @property - def page_numbers(self): - return list(range(self.first_page_number, self.last_page_number)) - - @property - def too_many_pages(self): - return self.page_count > self.max_page_count - - def __str__(self): - return Markup(self.jinja_template.render({ - 'image_url': self.image_url, - 'page_numbers': self.page_numbers, - 'too_many_pages': self.too_many_pages, - 'address': self._address_block, - 'contact_block': self._contact_block, - 'date': self._date, - 'subject': self.subject, - 'message': self._message, - 'postage': self.postage, - })) + return compose1(field, do_nice_typography, normalise_whitespace) class NeededByTemplateError(Exception): @@ -686,46 +459,38 @@ def is_unicode(content): def get_html_email_body( - template_content, template_values, redact_missing_personalisation=False, preview_mode=False -): - - return Take(Field( + template_content, template_values, redact_missing_personalisation=False, preview_mode=False +) -> str: + field = str(Field( template_content, template_values, - html='escape', + html='passthrough', # notify_html_markdown will escape the input markdown_lists=True, redact_missing_personalisation=redact_missing_personalisation, preview_mode=preview_mode - )).then( - unlink_govuk_escaped - ).then( - strip_unsupported_characters - ).then( - add_trailing_newline - ).then( - # before converting to markdown, strip out the "(())" for placeholders (preview mode or test emails) - strip_parentheses_in_link_placeholders - ).then( - notify_email_markdown - ).then( + )) + + return compose1( + field, + strip_unsupported_characters, + add_trailing_newline, + # before converting from markdown, strip out the "(())" for placeholders (preview mode or test emails) + strip_parentheses_in_link_placeholders, + insert_action_link, + insert_block_quotes, + insert_list_spaces, + notify_html_markdown, # after converting to html link, replace !!foo## with ((foo)) - replace_symbols_with_placeholder_parens - ).then( - do_nice_typography - ).then( - insert_action_link + replace_symbols_with_placeholder_parens, + do_nice_typography, ) def do_nice_typography(value): - return Take( - value - ).then( - remove_whitespace_before_punctuation - ).then( - make_quotes_smart - ).then( - remove_smart_quotes_from_email_addresses - ).then( - replace_hyphens_with_en_dashes + return compose1( + value, + remove_whitespace_before_punctuation, + make_quotes_smart, + remove_smart_quotes_from_email_addresses, + replace_hyphens_with_en_dashes, ) diff --git a/notifications_utils/template_change.py b/notifications_utils/template_change.py index f9d28b28..bf06cd33 100644 --- a/notifications_utils/template_change.py +++ b/notifications_utils/template_change.py @@ -4,8 +4,8 @@ class TemplateChange(): def __init__(self, old_template, new_template): - self.old_placeholders = Columns.from_keys(old_template.placeholders) - self.new_placeholders = Columns.from_keys(new_template.placeholders) + self.old_placeholders = Columns.from_keys(old_template.placeholder_names) + self.new_placeholders = Columns.from_keys(new_template.placeholder_names) @property def has_different_placeholders(self): diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 5d724e8d..82190396 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -1 +1 @@ -__version__ = '2.2.6' +__version__ = '2.3.0' diff --git a/setup.py b/setup.py index a5a2b222..435b28dc 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ 'Flask-Redis>=0.4.0', 'Jinja2>=2.11.3', 'MarkupSafe>=2.1.3', - 'mistune==0.8.4', + 'mistune==3.0.2', 'monotonic>=1.6', 'phonenumbers~=8.12.12', 'pypdf >= 3.15.0', diff --git a/tests/test_base_template.py b/tests/test_base_template.py index b8a506f1..ddf7abac 100644 --- a/tests/test_base_template.py +++ b/tests/test_base_template.py @@ -135,12 +135,12 @@ def test_include_placeholder_in_missing_data_if_placeholder_is_conditional(perso ] ) def test_extracting_placeholders(template_content, template_subject, expected): - assert WithSubjectTemplate({"content": template_content, 'subject': template_subject}).placeholders == expected + assert WithSubjectTemplate({"content": template_content, 'subject': template_subject}).placeholder_names == expected @pytest.mark.parametrize('template_cls', [SMSMessageTemplate, SMSPreviewTemplate]) @pytest.mark.parametrize( - "content,prefix, expected_length, expected_replaced_length", + "content, prefix, expected_length, expected_replaced_length", [ ("The quick brown fox jumped over the lazy dog", None, 44, 44), # should be replaced with a ? diff --git a/tests/test_formatted_list.py b/tests/test_formatted_list.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/test_formatters.py b/tests/test_formatters.py index dae66c73..562cf4a4 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -2,34 +2,32 @@ from markupsafe import Markup from notifications_utils.formatters import ( - unlink_govuk_escaped, - notify_email_markdown, - notify_letter_preview_markdown, - notify_plain_text_email_markdown, - sms_encode, - formatted_list, - strip_dvla_markup, - strip_pipes, escape_html, - remove_whitespace_before_punctuation, + formatted_list, + insert_list_spaces, make_quotes_smart, - replace_hyphens_with_en_dashes, - tweak_dvla_list_markup, nl2li, - strip_whitespace, - strip_and_remove_obscure_whitespace, + normalise_whitespace, + notify_html_markdown, + notify_markdown, remove_smart_quotes_from_email_addresses, + remove_whitespace_before_punctuation, + replace_hyphens_with_en_dashes, + sms_encode, + strip_and_remove_obscure_whitespace, + strip_dvla_markup, + strip_pipes, strip_unsupported_characters, - normalise_whitespace, + strip_whitespace, + tweak_dvla_list_markup, ) from notifications_utils.template import ( HTMLEmailTemplate, - PlainTextEmailTemplate, SMSMessageTemplate, SMSPreviewTemplate ) -PARAGRAPH_TEXT = '

    {}

    ' +PARAGRAPH_TEXT = '

    {}

    \n' @pytest.mark.parametrize( @@ -38,25 +36,22 @@ "http://www.gov.uk/", "https://www.gov.uk/", "http://service.gov.uk", - "http://service.gov.uk/blah.ext?q=a%20b%20c&order=desc#fragment", - pytest.param("http://service.gov.uk/blah.ext?q=one two three", marks=pytest.mark.xfail), ] ) def test_makes_links_out_of_urls(url): - link = '{}'.format(url, url) - assert (notify_email_markdown(url) == ( - PARAGRAPH_TEXT - ).format(link)) + link = f'{url}' + assert notify_html_markdown(url) == PARAGRAPH_TEXT.format(link) -@pytest.mark.parametrize('input, output', [ +@pytest.mark.parametrize('input_text, output', [ ( ( 'this is some text with a link http://example.com in the middle' ), ( 'this is some text with a link ' - 'http://example.com' + 'http://example.com' ' in the middle' ), ), @@ -66,14 +61,13 @@ def test_makes_links_out_of_urls(url): ), ( 'this link is in brackets ' - '(http://example.com)' + '(http://example.com)' ), ) ]) -def test_makes_links_out_of_urls_in_context(input, output): - assert notify_email_markdown(input) == ( - PARAGRAPH_TEXT - ).format(output) +def test_makes_links_out_of_urls_in_context(input_text, output): + assert notify_html_markdown(input_text) == PARAGRAPH_TEXT.format(output) @pytest.mark.parametrize( @@ -83,53 +77,50 @@ def test_makes_links_out_of_urls_in_context(input, output): "ftp://example.com", "test@example.com", "mailto:test@example.com", - "Example", ] ) -def test_doesnt_make_links_out_of_invalid_urls(url): - assert notify_email_markdown(url) == ( - PARAGRAPH_TEXT - ).format(url) +def test_makes_paragraphs_out_of_invalid_urls(url): + assert notify_html_markdown(url) == PARAGRAPH_TEXT.format(url) def test_handles_placeholders_in_urls(): - assert notify_email_markdown( + assert notify_html_markdown( "http://example.com/?token=((token))&key=1" ) == ( '

    ' - '' + '' 'http://example.com/?token=' '' '((token))&key=1' - '

    ' + '

    \n' ) @pytest.mark.parametrize( - "url, expected_html, expected_html_in_template", [ + "url, expected_html, expected_html_in_template", + [ ( """https://example.com"onclick="alert('hi')""", - """https://example.com"onclick="alert('hi')""", # noqa - """https://example.com"onclick="alert('hi‘)""", # noqa + """https://example.com"onclick="alert('hi')""", # noqa + """https://example.com"onclick="alert('hi‘)""", # noqa ), ( """https://example.com"style='text-decoration:blink'""", - """https://example.com"style='text-decoration:blink'""", # noqa - """https://example.com"style='text-decoration:blink’""", # noqa + """https://example.com"style='text-decoration:blink'""", # noqa + """https://example.com"style='text-decoration:blink’""", # noqa ), - ] + ], + ids=['js', 'style'] ) def test_urls_get_escaped(url, expected_html, expected_html_in_template): - assert notify_email_markdown(url) == ( - PARAGRAPH_TEXT - ).format(expected_html) + assert notify_html_markdown(url) == (PARAGRAPH_TEXT).format(expected_html) assert expected_html_in_template in str(HTMLEmailTemplate({'content': url, 'subject': ''})) def test_html_template_has_urls_replaced_with_links(): assert ( - '' - 'https://service.example.com/accept_invite/a1b2c3d4' + 'https://service.example.com/accept_invite/a1b2c3d4' '' ) in str(HTMLEmailTemplate({'content': ( 'You’ve been invited to a service. Click this link:\n' @@ -139,24 +130,26 @@ def test_html_template_has_urls_replaced_with_links(): ), 'subject': ''})) -@pytest.mark.parametrize('markdown_function, expected_output', [ - (notify_email_markdown, ( - '

    ' - '' - 'https://example.com' - '' - '

    ' - '

    ' - 'Next paragraph' - '

    ' - )), - (notify_plain_text_email_markdown, ( - '\n' - '\nhttps://example.com' - '\n' - '\nNext paragraph' - )), -]) +@pytest.mark.parametrize( + 'markdown_function, expected_output', + [ + (notify_html_markdown, ( + '

    ' + '' + 'https://example.com' + '' + '

    \n' + '

    ' + 'Next paragraph' + '

    \n' + )), + (notify_markdown, ( + 'https://example.com\n\n' + 'Next paragraph\n' + )), + ], + ids=('notify_html_markdown', 'notify_markdown'), +) def test_preserves_whitespace_when_making_links( markdown_function, expected_output ): @@ -167,24 +160,6 @@ def test_preserves_whitespace_when_making_links( ) == expected_output -@pytest.mark.parametrize( - "template_content,expected", [ - ("gov.uk", u"gov.\u200Buk"), - ("GOV.UK", u"GOV.\u200BUK"), - ("Gov.uk", u"Gov.\u200Buk"), - ("https://gov.uk", "https://gov.uk"), - ("https://www.gov.uk", "https://www.gov.uk"), - ("www.gov.uk", "www.gov.uk"), - ("gov.uk/register-to-vote", "gov.uk/register-to-vote"), - ("gov.uk?q=", "gov.uk?q=") - ] -) -def test_escaping_govuk_in_email_templates(template_content, expected): - assert unlink_govuk_escaped(template_content) == expected - assert expected in str(PlainTextEmailTemplate({'content': template_content, 'subject': ''})) - assert expected in str(HTMLEmailTemplate({'content': template_content, 'subject': ''})) - - @pytest.mark.parametrize( "prefix, body, expected", [ ("a", "b", "a: b"), @@ -214,459 +189,343 @@ def test_sms_preview_adds_newlines(): 'markdown_function, expected', ( [ - notify_letter_preview_markdown, - 'print("hello")' - ], - [ - notify_email_markdown, - 'print("hello")' + notify_html_markdown, + '
    print("hello")\n
    \n' ], [ - notify_plain_text_email_markdown, - 'print("hello")' + notify_markdown, + '```\nprint("hello")\n```\n' ], - ) + ), + ids=['notify_html_markdown', 'notify_markdown'] ) def test_block_code(markdown_function, expected): assert markdown_function('```\nprint("hello")\n```') == expected -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_letter_preview_markdown, - ( - '

    inset text

    ' - ) - ], - [ - notify_email_markdown, - ( - '' - '' - '
    ' - '

    inset text

    ' - '
    ' - ) - ], - [ - notify_plain_text_email_markdown, - ( - '\n' - '\ninset text' - ), - ], -)) +@pytest.mark.parametrize( + 'markdown_function, expected', + ( + [ + notify_html_markdown, + ( + '
    \n' + '

    inset text

    \n
    \n' + ) + ], + [ + notify_markdown, + '\n\ninset text\n' + ], + ), + ids=['notify_html_markdown', 'notify_markdown'] +) def test_block_quote(markdown_function, expected): - assert markdown_function('^ inset text') == expected + assert markdown_function('> inset text') == expected -@pytest.mark.parametrize('heading', ( - '# heading', - '#heading', -)) @pytest.mark.parametrize( 'markdown_function, expected', ( [ - notify_letter_preview_markdown, - '

    heading

    \n' - ], - [ - notify_email_markdown, + notify_html_markdown, ( '

    ' 'heading' - '

    ' + '\n' ) ], [ - notify_plain_text_email_markdown, + notify_markdown, ( '\n' '\n' '\nheading' - '\n-----------------------------------------------------------------' + '\n-----------------------------------------------------------------\n' ), ], - ) + ), + ids=['notify_html_markdown', 'notify_markdown'] ) -def test_level_1_header(markdown_function, heading, expected): - assert markdown_function(heading) == expected +def test_level_1_header(markdown_function, expected): + assert markdown_function('# heading') == expected -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_letter_preview_markdown, - '

    inset text

    ' - ], - [ - notify_email_markdown, - '

    inset text

    ' - ], - [ - notify_plain_text_email_markdown, - ( - '\n' - '\ninset text' - '\n-----------------------------------------------------------------' - ), - ], -)) +@pytest.mark.parametrize( + 'markdown_function, expected', + ( + [ + notify_html_markdown, + '

    inset text

    \n' + ], + [ + notify_markdown, + ( + '\n' + '\ninset text' + '\n-----------------------------------------------------------------\n' + ), + ], + ), + ids=['notify_html_markdown', 'notify_markdown'] +) def test_level_2_header(markdown_function, expected): assert markdown_function('## inset text') == expected -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_email_markdown, - '

    inset text

    ' - ], - [ - notify_plain_text_email_markdown, - ( - '\n' - '\ninset text' - '\n-----------------------------------------------------------------' - ), - ], -)) +@pytest.mark.parametrize( + 'markdown_function, expected', + ( + [ + notify_html_markdown, + '

    inset text

    \n' + ], + [ + notify_markdown, + ( + '\n' + '\ninset text' + '\n-----------------------------------------------------------------\n' + ), + ], + ), + ids=['notify_html_markdown', 'notify_markdown'] +) def test_level_3_header(markdown_function, expected): assert markdown_function('### inset text') == expected -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_letter_preview_markdown, - ( - '

    a

    ' - '
     
    ' - '

    b

    ' - ) - ], - [ - notify_email_markdown, - ( - '

    a

    ' - '
    ' - '

    b

    ' - ) - ], - [ - notify_plain_text_email_markdown, - ( - '\n' - '\na' - '\n' - '\n=================================================================' - '\n' - '\nb' - ), - ], -)) -def test_hrule(markdown_function, expected): +@pytest.mark.parametrize( + 'markdown_function, expected', + ( + [ + notify_html_markdown, + ( + '

    a

    \n' + '
    \n' + '

    b

    \n' + ) + ], + [ + notify_markdown, + ( + 'a\n\n' + '=================================================================\n' + 'b\n' + ), + ], + ), + ids=['notify_html_markdown', 'notify_markdown'] +) +def test_thematic_break(markdown_function, expected): + """ + Thematic breaks were known as horizontal rules (hrule) in earlier versions of Mistune. + """ + assert markdown_function('a\n\n***\n\nb') == expected assert markdown_function('a\n\n---\n\nb') == expected -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_email_markdown, - ( - '' - '' - '' - '' - '
    ' - '
      ' - '
    1. one
    2. ' - '
    3. two
    4. ' - '
    5. three
    6. ' - '
    ' - '
    ' - ) - ], -)) -def test_ordered_list(markdown_function, expected): - assert markdown_function( +def test_ordered_list(): + assert notify_html_markdown( '1. one\n' '2. two\n' '3. three\n' - ) == expected - assert markdown_function( - '1.one\n' - '2.two\n' - '3.three\n' - ) == expected + ) == ( + '
      \n' + '
    1. ' + 'one
    2. \n' + '
    3. ' + 'two
    4. \n' + '
    5. ' + 'three
    6. \n' + '
    \n' + ) -@pytest.mark.parametrize('markdown_function, test_text, expected', ( # noqa: E126 - [ - notify_email_markdown, - ( - '1. List item 1\n\n' - '\tShould be paragraph in the list item without extra br above' +@pytest.mark.parametrize( + 'test_text, expected', + ( + [ + ( + '1. List item 1\n\n' + '\tShould be paragraph in the list item without extra br above' + ), + ( + '
      \n' + '
    1. ' + '

      List item 1

      \n' + '

      ' + 'Should be paragraph in the list item without extra br above

      \n' + '
    2. \n' + '
    \n' + ) + ], + [ + ( + '1. List item 1\n\n' + ' Should not be paragraph in the list item without extra br above' + ), + ( + '
      \n' + '
    1. ' + 'List item 1' + '
    2. \n' + '
    \n' + '

    ' + 'Should not be paragraph in the list item without extra br above

    \n' + ) + ], + [ + ( + '1. one' + '\n\n nested 1' + '\n\n nested 2' + '\n1. two' + '\n1. three' + ), + ( + '
      \n' + '
    1. ' + 'one' + '
    2. \n' + '
    \n' + '

    nested 1

    \n' + '

    nested 2

    \n' + '
      \n' + '
    1. two
    2. \n' + '
    3. three
    4. \n' + '
    \n' + ) + ], + [ + ( + '* one' + '\n\n nested 1' + '\n\n nested 2' + '\n* two' + '\n* three' + ), + ( + '\n' + '

    nested 1

    \n' + '

    nested 2

    \n' + '\n' + ) + ], + ), + ids=['paragraph_in_list', 'paragraph_not_in_list', 'ordered_nested_list', 'unordered_nested_list'] +) +def test_paragraph_in_list_has_no_linebreak(test_text, expected): + assert notify_html_markdown(test_text) == expected + + +@pytest.mark.parametrize( + 'markdown', + ( + ( # two spaces + '* one\n' + '* two\n' + '* three\n' ), - ( - '' - '' - '' - '' - '
    ' - '
      ' - '
    1. ' - '

      List item 1

      ' - '

      ' - ' Should be paragraph in the list item without extra br above

      ' - '
    2. ' - '
    ' - '
    ' - ) - ], - [ - notify_email_markdown, - ( - '1. List item 1\n\n' - ' Should be paragraph in the list item without extra br above' + ( # tab + '* one\n' + '* two\n' + '* three\n' ), - ( - '' - '' - '' - '' - '
    ' - '
      ' - '
    1. ' - '

      List item 1

      ' - '

      ' - 'Should be paragraph in the list item without extra br above

      ' - '
    2. ' - '
    ' - '
    ' - ) - ], - [ - notify_email_markdown, - ( - '1. one' - '\n\n nested 1' - '\n\n nested 2' - '\n1. two' - '\n1. three' + ( # dash as bullet + '- one\n' + '- two\n' + '- three\n' ), - ( - '' - '' - '' - '' - '
    ' - '
      ' - '
    1. ' - '

      one

      ' - '

      nested 1

      ' - '

      nested 2

      ' - '
    2. ' - '
    3. two
    4. ' - '
    5. three
    6. ' - '
    ' - '
    ' - ) - ], - [ - notify_email_markdown, - ( - '* one' - '\n\n nested 1' - '\n\n nested 2' - '\n* two' - '\n* three' + ( # plus as bullet + '+ one\n' + '+ two\n' + '+ three\n' ), - ( - '' - '' - '' - '' - '
    ' - '
      ' - '
    • ' - '

      one

      ' - '

      nested 1

      ' - '

      nested 2

      ' - '
    • ' - '
    • two
    • ' - '
    • three
    • ' - '
    ' - '
    ' - ) - ], -)) -def test_paragraph_in_list_has_no_linebreak(markdown_function, test_text, expected): - assert markdown_function(test_text) == expected - - -@pytest.mark.parametrize('markdown', ( - ( # no space - '*one\n' - '*two\n' - '*three\n' - ), - ( # single space - '* one\n' - '* two\n' - '* three\n' - ), - ( # two spaces - '* one\n' - '* two\n' - '* three\n' - ), - ( # tab - '* one\n' - '* two\n' - '* three\n' - ), - ( # dash as bullet - '- one\n' - '- two\n' - '- three\n' ), - pytest.param(( # plus as bullet - '+ one\n' - '+ two\n' - '+ three\n' - ), marks=pytest.mark.xfail(raises=AssertionError)), - ( # bullet as bullet - '• one\n' - '• two\n' - '• three\n' + ids=['two_spaces', 'tab', 'dash_as_bullet', 'plus_as_bullet'] +) +@pytest.mark.parametrize( + 'markdown_function, expected', + ( + [ + notify_html_markdown, + ( + '\n' + ) + ], + [ + notify_markdown, + '• one\n• two\n• three\n' + ], ), -)) -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_letter_preview_markdown, - ( - '\n' - ) - ], - [ - notify_email_markdown, - ( - '' - '' - '' - '' - '
    ' - '
      ' - '
    • one
    • ' - '
    • two
    • ' - '
    • three
    • ' - '
    ' - '
    ' - ) - ], - [ - notify_plain_text_email_markdown, - ( - '\n' - '\n• one' - '\n• two' - '\n• three' - ), - ], -)) + ids=['notify_html_markdown', 'notify_markdown'] +) def test_unordered_list(markdown, markdown_function, expected): assert markdown_function(markdown) == expected -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_letter_preview_markdown, - '

    + one

    + two

    + three

    ', - ], - [ - notify_email_markdown, - ( - '

    + one

    ' - '

    + two

    ' - '

    + three

    ' - ), - ], - [ - notify_plain_text_email_markdown, - ( - '\n\n+ one' - '\n\n+ two' - '\n\n+ three' - ), - ], -)) -def test_pluses_dont_render_as_lists(markdown_function, expected): - assert markdown_function( - '+ one\n' - '+ two\n' - '+ three\n' - ) == expected - - -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_letter_preview_markdown, - ( - '

    ' - 'line one
    ' - 'line two' - '

    ' - '

    ' - 'new paragraph' - '

    ' - ) - ], - [ - notify_email_markdown, - ( - '

    line one
    ' - 'line two

    ' - '

    new paragraph

    ' - ) - ], - [ - notify_plain_text_email_markdown, - ( - '\n' - '\nline one' - '\nline two' - '\n' - '\nnew paragraph' - ), - ], -), - ids=['notify_letter_preview_markdown', 'notify_email_markdown', 'notify_plain_text_email_markdown'] +@pytest.mark.parametrize( + 'markdown_function, expected', + ( + [ + notify_html_markdown, + ( + '

    line one
    \n' + 'line two

    \n' + '

    new paragraph

    \n' + ) + ], + [ + notify_markdown, + ( + 'line one\n' + 'line two\n' + '\nnew paragraph\n' + ), + ], + ), + ids=['notify_html_markdown', 'notify_markdown'] ) def test_paragraphs(markdown_function, expected): assert markdown_function( @@ -677,31 +536,23 @@ def test_paragraphs(markdown_function, expected): ) == expected -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_letter_preview_markdown, - ( - '

    before

    ' - '

    after

    ' - ) - ], - [ - notify_email_markdown, - ( - '

    before

    ' - '

    after

    ' - ) - ], - [ - notify_plain_text_email_markdown, - ( - '\n' - '\nbefore' - '\n' - '\nafter' - ), - ], -)) +@pytest.mark.parametrize( + 'markdown_function, expected', + ( + [ + notify_html_markdown, + ( + '

    before

    \n' + '

    after

    \n' + ) + ], + [ + notify_markdown, + 'before\n\nafter\n', + ], + ), + ids=['notify_html_markdown', 'notify_markdown'] +) def test_multiple_newlines_get_truncated(markdown_function, expected): assert markdown_function( 'before\n\n\n\n\n\nafter' @@ -709,224 +560,191 @@ def test_multiple_newlines_get_truncated(markdown_function, expected): @pytest.mark.parametrize('markdown_function', ( - notify_letter_preview_markdown, notify_email_markdown, notify_plain_text_email_markdown + notify_html_markdown, notify_markdown )) def test_table(markdown_function): + """ + Delete tables. Note that supporting them would be very easy. Both renderers use Mistune's "table" + plugin. To support tables, delete the overridden "table" method from the renderer. + """ + assert markdown_function( 'col | col\n' '----|----\n' 'val | val\n' - ) == ( + ).rstrip() == ( '' ) -@pytest.mark.parametrize('markdown_function, link, expected', ( - [ - notify_letter_preview_markdown, - 'http://example.com', - '

    example.com

    ' - ], - [ - notify_email_markdown, - 'http://example.com', - ( - '

    ' - 'http://example.com' - '

    ' - ) - ], - [ - notify_email_markdown, - """https://example.com"onclick="alert('hi')""", - ( - '

    ' - '' - 'https://example.com"onclick="alert(\'hi' - '\')' - '

    ' - ) - ], - [ - notify_plain_text_email_markdown, - 'http://example.com', - ( - '\n' - '\nhttp://example.com' - ), - ], -)) +@pytest.mark.parametrize( + 'markdown_function, link, expected', + ( + [ + notify_html_markdown, + 'http://example.com', + ( + '

    ' + 'http://example.com' + '

    \n' + ) + ], + [ + notify_html_markdown, + """https://example.com"onclick="alert('hi')""", + ( + '

    ' + '' + 'https://example.com"onclick="alert(\'hi' + '\')' + '

    \n' + ) + ], + [ + notify_markdown, + 'http://example.com', + 'http://example.com\n' + ], + ), + ids=['html_link', 'html_link_js', 'markdown'] +) def test_autolink(markdown_function, link, expected): assert markdown_function(link) == expected -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_letter_preview_markdown, - '

    variable called thing

    ' - ], - [ - notify_email_markdown, - '

    variable called thing

    ' - ], - [ - notify_plain_text_email_markdown, - '\n\nvariable called thing', - ], -)) +@pytest.mark.parametrize( + 'markdown_function, expected', + ( + [ + notify_html_markdown, + '

    ' + 'variable called thing

    \n' + ], + [ + notify_markdown, + 'variable called `thing`\n', + ], + ), + ids=['notify_html_markdown', 'notify_markdown'] +) def test_codespan(markdown_function, expected): - assert markdown_function( - 'variable called `thing`' - ) == expected + assert markdown_function('variable called `thing`') == expected @pytest.mark.parametrize('markdown_function, expected', ( [ - notify_letter_preview_markdown, - '

    something important

    ' - ], - [ - notify_email_markdown, + notify_html_markdown, '

    something important

    ' + 'color: #323A45;">something important

    \n' ], [ - notify_plain_text_email_markdown, - '\n\nsomething important', + notify_markdown, + 'something **important**\n', ], )) def test_double_emphasis(markdown_function, expected): - assert markdown_function( - 'something __important__' - ) == expected + assert markdown_function('something __important__') == expected @pytest.mark.parametrize('markdown_function, expected', ( [ - notify_letter_preview_markdown, - '

    something important

    ' - ], - [ - notify_email_markdown, + notify_html_markdown, '

    something important

    ' + 'color: #323A45;">something important

    \n' ], [ - notify_plain_text_email_markdown, - '\n\nsomething important', + notify_markdown, + 'something *important*\n', ], )) def test_emphasis(markdown_function, expected): - assert markdown_function( - 'something _important_' - ) == expected + assert markdown_function('something _important_') == expected @pytest.mark.parametrize('markdown_function, expected', ( [ - notify_email_markdown, + notify_html_markdown, '

    foo bar

    ' + 'color: #323A45;">foo bar

    \n' ], [ - notify_plain_text_email_markdown, - '\n\nfoo bar', + notify_markdown, + 'foo ***bar***\n', ], )) def test_nested_emphasis(markdown_function, expected): - assert markdown_function( - 'foo ___bar___' - ) == expected + """ + Note that this behavior has no correstpondence with Github markdown. The expected + output is simply what the renderer actually does. + """ - -@pytest.mark.parametrize('markdown_function', ( - notify_letter_preview_markdown, notify_email_markdown, notify_plain_text_email_markdown -)) -def test_image(markdown_function): - assert markdown_function( - '![alt text](http://example.com/image.png)' - ) == ( - '' - ) + assert markdown_function('foo ___bar___') == expected -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_letter_preview_markdown, - ( - '

    Example: example.com

    ' - ) - ], - [ - notify_email_markdown, - ( - '

    ' - 'Example' - '

    ' - ) - ], +@pytest.mark.parametrize( + 'markdown_function, expected', [ - notify_plain_text_email_markdown, - ( - '\n' - '\nExample: http://example.com' - ), + (notify_html_markdown, ''), + (notify_markdown, '\n'), ], -)) + ids=['notify_html_markdown', 'notify_markdown'] +) +def test_image(markdown_function, expected): + assert markdown_function('![alt text](http://example.com/image.png)') == expected + + +@pytest.mark.parametrize( + 'markdown_function, expected', + ( + [ + notify_html_markdown, + ( + '

    ' + 'Example' + '

    \n' + ) + ], + [ + notify_markdown, + 'Example: http://example.com\n', + ], + ), + ids=['notify_html_markdown', 'notify_markdown'] +) def test_link(markdown_function, expected): - assert markdown_function( - '[Example](http://example.com)' - ) == expected + assert markdown_function('[Example](http://example.com)') == expected -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_letter_preview_markdown, - ( - '

    Example: example.com

    ' - ) - ], - [ - notify_email_markdown, - ( - '

    ' - '' - 'Example' - '' - '

    ' - ) - ], - [ - notify_plain_text_email_markdown, - ( - '\n' - '\nExample (An example URL): http://example.com' - ), - ], -)) -def test_link_with_title(markdown_function, expected): - assert markdown_function( - '[Example](http://example.com "An example URL")' - ) == expected +def test_link_with_title(): + assert notify_html_markdown('[Example](http://example.com "An example URL")') == ( + '

    ' + '' + 'Example' + '' + '

    \n' + ) -@pytest.mark.parametrize('markdown_function, expected', ( - [ - notify_letter_preview_markdown, - '

    Strike

    ' - ], - [ - notify_email_markdown, - '

    Strike

    ' - ], - [ - notify_plain_text_email_markdown, - '\n\nStrike' - ], -)) +@pytest.mark.parametrize( + 'markdown_function, expected', + ( + [ + notify_html_markdown, + '

    Strike

    \n' + ], + [ + notify_markdown, + '\n\nStrike\n' + ], + ), + ids=['notify_html_markdown', 'notify_markdown'] +) def test_strikethrough(markdown_function, expected): assert markdown_function('~~Strike~~') == expected @@ -1171,3 +989,63 @@ def test_strip_unsupported_characters(): def test_normalise_whitespace(): assert normalise_whitespace('\u200C Your tax is\ndue\n\n') == 'Your tax is due' + + +@pytest.mark.parametrize( + 'actual, expected', + [ + ( + '1.one\n2.two\n3.three', + '1. one\n2. two\n3. three', + ), + ( + '-one\n -two\n-three', + '- one\n - two\n- three', + ), + ( + '+one\n +two\n+three', + '- one\n - two\n- three', + ), + ( + '*one\n *two\n*three', + '- one\n - two\n- three', + ), + ( + '•one\n •two\n•three', + '- one\n - two\n- three', + ), + # Next 2 tests: Shouldn't misinterpret a thematic break as a list item + ( + '***one\n *two\n*three', + '***one\n - two\n- three', + ), + ( + '-one\n ---two\n-three', + '- one\n ---two\n- three', + ), + ( + '# This is Heading 1\n' + '## This is heading 2\n' + '### This is heading 3\n' + '\n' + '__This has been emboldened__\n' + '\n' + '- this is a bullet list\n' + '- list list list\n' + '\n' + 'Testing personalisation, ((name)).\n', + '# This is Heading 1\n' + '## This is heading 2\n' + '### This is heading 3\n' + '\n' + '__This has been emboldened__\n' + '\n' + '- this is a bullet list\n' + '- list list list\n' + '\n' + 'Testing personalisation, ((name)).\n', + ), + ] +) +def test_insert_list_spaces(actual, expected): + assert insert_list_spaces(actual) == expected diff --git a/tests/test_take.py b/tests/test_take.py deleted file mode 100644 index afe6685a..00000000 --- a/tests/test_take.py +++ /dev/null @@ -1,25 +0,0 @@ -from notifications_utils.take import Take - - -def _uppercase(value): - return value.upper() - - -def _append(value, to_append): - return value + to_append - - -def _prepend_with_service_name(value, service_name=None): - return '{}: {}'.format(service_name, value) - - -def test_take(): - assert 'Service name: HELLO WORLD!' == Take( - 'hello world' - ).then( - _uppercase - ).then( - _append, '!' - ).then( - _prepend_with_service_name, service_name='Service name' - ) diff --git a/tests/test_template_types.py b/tests/test_template_types.py index c14369c2..eea2a643 100644 --- a/tests/test_template_types.py +++ b/tests/test_template_types.py @@ -1,24 +1,25 @@ -import datetime from time import process_time -import os -import pytest -from functools import partial from unittest import mock + +import pytest from markupsafe import Markup -from freezegun import freeze_time -from notifications_utils.formatters import LINK_STYLE, PARAGRAPH_STYLE, unlink_govuk_escaped +from notifications_utils.formatters import ( + BLOCK_QUOTE_STYLE, + LINK_STYLE, + LIST_ITEM_STYLE, + ORDERED_LIST_STYLE, + PARAGRAPH_STYLE, + UNORDERED_LIST_STYLE +) from notifications_utils.template import ( Template, HTMLEmailTemplate, - LetterPreviewTemplate, - LetterImageTemplate, PlainTextEmailTemplate, SMSMessageTemplate, SMSPreviewTemplate, WithSubjectTemplate, EmailPreviewTemplate, - LetterPrintTemplate, get_html_email_body, ) @@ -39,18 +40,18 @@ def test_pass_through_renderer(): 'line one\nline two with ((name))\n\nnew paragraph', {'name': 'bob'}, ( - f'

    line one
    line two with bob

    ' - f'

    new paragraph

    ' + f'

    line one
    \nline two with bob

    \n' + f'

    new paragraph

    \n' ), ), ( '>>[action](https://example.com/foo?a=b)', {}, ( - f'

    ' - ' action

    ' + f'

    ' + 'call to action img action

    \n' ) ), ( @@ -63,81 +64,90 @@ def test_pass_through_renderer(): {'color': 'brown'}, ( '

    foo

    Bar

    ' - f'

    The quick brown fox

    ' - f'

    ' - ' the action_link-of doom

    ' + 'color: #323A45;">foo\n' + '

    Bar

    \n' + f'

    The quick brown fox

    \n' + f'

    ' + 'call to action img the action_link-of doom

    \n' ), ), ( 'text before link\n\n>>[great link](http://example.com)\n\ntext after link', {}, ( - f'

    text before link

    ' - f'

    ' - ' great link

    ' - f'

    text after link

    ' + f'

    text before link

    \n' + f'

    ' + 'call to action img great link

    \n' + f'

    text after link

    \n' ) ), ( 'action link: >>[Example](http://example.com)\nanother link: [test](https://example2.com)', {}, ( - f'

    action link:

    ' - f'

    ' - ' Example

    ' - f'


    ' - f'another link: test

    ' + f'

    action link:

    \n' + f'

    ' + 'call to action img Example

    \n' + f'

    ' + f'another link: test

    \n' ) ), ( 'action link: >>[grin](http://example.com) another action link: >>[test](https://example2.com)', {}, ( - f'

    action link:

    ' - f'

    ' - ' grin

    ' - f'

    another action link:

    ' - f'

    ' - ' test

    ' + f'

    action link:

    \n' + f'

    ' + 'call to action img grin

    \n' + f'

    another action link:

    \n' + f'

    ' + 'call to action img test

    \n' ) ), ( 'text before && link >>[Example](http://example.com) text after & link', {}, ( - f'

    text before && link

    ' - f'

    ' - ' Example

    ' - f'

    text after & link

    ' + f'

    text before && link

    \n' + f'

    ' + 'call to action img Example

    \n' + f'

    text after & link

    \n' ) ), ( 'text before >> link >>[great action](http://example.com) text after >>link', {}, ( - f'

    text before >> link

    ' - f'

    ' - ' great action

    ' - f'

    text after >>link

    ' + f'

    text before >> link

    \n' + f'

    ' + 'call to action img great action

    \n' + f'

    text after >>link

    \n' ) ), ( 'text >> then [item] and (things) then >>[action](link)', {}, ( - f'

    text >> then [item] and (things) then

    ' - f'

    ' - ' action

    ' + f'

    text >> then [item] and (things) then

    \n' + f'

    ' + 'call to action img action

    \n' ) ), ( @@ -148,19 +158,22 @@ def test_pass_through_renderer(): ), {}, ( - f'

    ' - ' action link

    ' - f'

    testing the new

    ' - f'

    ' - ' action link

    ' - f'

    thingy...

    ' - f'

    ' - ' click me

    ' + f'

    ' + 'call to action img action link

    \n' + f'

    testing the new

    \n' + f'

    ' + 'call to action img action link

    \n' + f'

    thingy...

    \n' + f'

    ' + 'call to action img click me

    \n' f'

    ! Text with a ' - 'regular link

    ' + 'regular link

    \n' ) ) ], @@ -188,50 +201,51 @@ def test_get_html_email_body_with_action_links(content, values, expected): 'normal placeholder formatting: ((foo))', ( f'

    normal placeholder formatting: ((foo))' - '

    ' + '

    \n' ), ), ( 'regular markdown link: [link text](#)', ( f'

    regular markdown link: ' - f'link text

    ' + f'link text

    \n' ), ), ( 'placeholder in link text, without placeholder in link: [link ((foo))](https://test.com/)', ( f'

    placeholder in link text, without placeholder in link: ' - f'link ' - '((foo))

    ' + f'link ' + '((foo))

    \n' ), ), ( 'no format within link, placeholder at end: [link text](https://test.com/((foo)))', ( f'

    no format within link, placeholder at end: ' - f'link text

    ' + f'link text

    \n' ) ), ( 'no format within link, placeholder in middle: [link text](https://test.com/((foo))?xyz=123)', ( f'

    no format within link, placeholder in middle: ' - f'link text

    ' + f'link text

    \n' ) ), ( 'no format in link, with only placeholder: [link text](((foo)))', ( f'

    no format in link, with only placeholder: ' - f'link text

    ' + f'link text

    \n' ) ), ( 'no format within link, multiple placeholders: [link text](https://test.com/((foo))?xyz=((bar)))', ( f'

    no format within link, multiple placeholders: ' - f'link text

    ' + f'' + 'link text

    \n' ) ), ], @@ -250,9 +264,8 @@ def test_get_html_email_body_preview_with_placeholder_in_markdown_link(content, def test_html_email_inserts_body(): - assert 'the <em>quick</em> brown fox' in str(HTMLEmailTemplate( - {'content': 'the quick brown fox', 'subject': ''} - )) + content = 'the quick brown fox' + assert content in str(HTMLEmailTemplate({'content': content, 'subject': ''})) @pytest.mark.parametrize( @@ -454,133 +467,108 @@ def test_preheader_is_at_start_of_html_emails(): 'font-size: 16px;Margin: 0;color:#323A45;">\n' '\n' 'content…' - ) in str( - HTMLEmailTemplate({'content': 'content', 'subject': 'subject'}) - ) + ) in str(HTMLEmailTemplate({'content': 'content', 'subject': 'subject'})) -@pytest.mark.parametrize('content, values, expected_preheader', [ - ( +@pytest.mark.parametrize( + 'content, values, expected_preheader', + [ ( - 'Hello (( name ))\n' - '\n' - '# This - is a "heading"\n' - '\n' - 'My favourite websites\' URLs are:\n' - '- GOV.UK\n' - '- https://www.example.com\n' + ( + 'Hello (( name ))\n' + '\n' + '# This - is a "heading"\n' + '\n' + 'My favourite websites\' URLs are:\n' + '- va.gov\n' + '- https://www.example.com\n' + ), + {'name': 'Jo'}, + 'Hello Jo This – is a “heading” My favourite websites’ URLs are: • va.gov • https://www.example.com', ), - {'name': 'Jo'}, - 'Hello Jo This – is a “heading” My favourite websites’ URLs are: • GOV.​UK • https://www.example.com', - ), - ( ( - '[Markdown link](https://www.example.com)\n' + ( + '[Markdown link](https://www.example.com)\n' + ), + {}, + 'Markdown link', ), - {}, - 'Markdown link', - ), - ( - """ - Lorem Ipsum is simply dummy text of the printing and - typesetting industry. + ( + ( + '>>[action link](https://www.example.com)\n' + ), + {}, + 'action link', + ), + ( + """ + Lorem Ipsum is simply dummy text of the printing and + typesetting industry. - Lorem Ipsum has been the industry’s standard dummy text - ever since the 1500s, when an unknown printer took a galley - of type and scrambled it to make a type specimen book. + Lorem Ipsum has been the industry’s standard dummy text + ever since the 1500s, when an unknown printer took a galley + of type and scrambled it to make a type specimen book. - Lorem Ipsum is simply dummy text of the printing and - typesetting industry. + Lorem Ipsum is simply dummy text of the printing and + typesetting industry. - Lorem Ipsum has been the industry’s standard dummy text - ever since the 1500s, when an unknown printer took a galley - of type and scrambled it to make a type specimen book. - """, - {}, + Lorem Ipsum has been the industry’s standard dummy text + ever since the 1500s, when an unknown printer took a galley + of type and scrambled it to make a type specimen book. + """, + {}, + ( + 'Lorem Ipsum is simply dummy text of the printing and ' + 'typesetting industry. Lorem Ipsum has been the industry’s ' + 'standard dummy text ever since the 1500s, when an unknown ' + 'printer took a galley of type and scrambled it to make a ' + 'type specimen book. Lorem Ipsu' + ), + ), ( - 'Lorem Ipsum is simply dummy text of the printing and ' - 'typesetting industry. Lorem Ipsum has been the industry’s ' - 'standard dummy text ever since the 1500s, when an unknown ' - 'printer took a galley of type and scrambled it to make a ' - 'type specimen book. Lorem Ipsu' + 'short email', + {}, + 'short email', ), - ), - ( - 'short email', - {}, - 'short email', - ), -]) -@mock.patch( - 'notifications_utils.template.HTMLEmailTemplate.jinja_template.render', - return_value='mocked' + ], + ids=['1', '2', '3', '4', '5'] ) def test_content_of_preheader_in_html_emails( - mock_jinja_template, content, values, expected_preheader, ): - assert str(HTMLEmailTemplate( + assert HTMLEmailTemplate( {'content': content, 'subject': 'subject'}, values - )) == 'mocked' - assert mock_jinja_template.call_args[0][0]['preheader'] == expected_preheader + ).preheader == expected_preheader -@pytest.mark.parametrize('template_class, extra_args, result, markdown_renderer', [ - [ - HTMLEmailTemplate, - {}, - ( - 'the quick brown fox\n' - '\n' - 'jumped over the lazy dog\n' - ), - 'notifications_utils.template.notify_email_markdown', - ], - [ - LetterPreviewTemplate, - {}, - ( - 'the quick brown fox\n' - '\n' - 'jumped over the lazy dog\n' - ), - 'notifications_utils.template.notify_letter_preview_markdown' - ], -]) -def test_markdown_in_templates( - template_class, - extra_args, - result, - markdown_renderer, -): - with mock.patch(markdown_renderer, return_value='') as mock_markdown_renderer: - str(template_class( - { - "content": ( - 'the quick ((colour)) ((animal))\n' - '\n' - 'jumped over the lazy dog' - ), - 'subject': 'animal story' - }, - {'animal': 'fox', 'colour': 'brown'}, - **extra_args - )) - mock_markdown_renderer.assert_called_once_with(result) +def test_markdown_in_templates(): + str(HTMLEmailTemplate( + { + "content": ( + 'the quick ((colour)) ((animal))\n' + '\n' + 'jumped over the lazy dog' + ), + 'subject': 'animal story' + }, + {'animal': 'fox', 'colour': 'brown'}, + )) == 'the quick brown fox\n\njumped over the lazy dog\n' @pytest.mark.parametrize( - 'template_class', [ + 'template_class', + [ HTMLEmailTemplate, EmailPreviewTemplate, SMSPreviewTemplate, ] ) @pytest.mark.parametrize( - "url, url_with_entities_replaced", [ + "url, url_with_entities_replaced", + [ ("http://example.com", "http://example.com"), ("http://www.gov.uk/", "http://www.gov.uk/"), ("https://www.gov.uk/", "https://www.gov.uk/"), @@ -589,72 +577,49 @@ def test_markdown_in_templates( "http://service.gov.uk/blah.ext?q=a%20b%20c&order=desc#fragment", "http://service.gov.uk/blah.ext?q=a%20b%20c&order=desc#fragment", ), - pytest.param("example.com", "example.com", marks=pytest.mark.xfail), - pytest.param("www.example.com", "www.example.com", marks=pytest.mark.xfail), - pytest.param( - "http://service.gov.uk/blah.ext?q=one two three", - "http://service.gov.uk/blah.ext?q=one two three", - marks=pytest.mark.xfail, - ), - pytest.param("ftp://example.com", "ftp://example.com", marks=pytest.mark.xfail), - pytest.param("mailto:test@example.com", "mailto:test@example.com", marks=pytest.mark.xfail), ] ) def test_makes_links_out_of_URLs(template_class, url, url_with_entities_replaced): - assert '{}'.format( - url_with_entities_replaced, url_with_entities_replaced + assert '{}'.format( + LINK_STYLE, url_with_entities_replaced, url_with_entities_replaced ) in str(template_class({'content': url, 'subject': ''})) -@pytest.mark.parametrize('content, html_snippet', ( - ( - ( - 'You’ve been invited to a service. Click this link:\n' - 'https://service.example.com/accept_invite/a1b2c3d4\n' - '\n' - 'Thanks\n' - ), - ( - '' - 'https://service.example.com/accept_invite/a1b2c3d4' - '' - ), - ), +@pytest.mark.parametrize( + 'content, html_snippet', ( ( - 'https://service.example.com/accept_invite/?a=b&c=d&' + ( + 'You’ve been invited to a service. Click this link:\n' + 'https://service.example.com/accept_invite/a1b2c3d4\n' + '\n' + 'Thanks\n' + ), + ( + '' + 'https://service.example.com/accept_invite/a1b2c3d4' + '' + ), ), ( - '' - 'https://service.example.com/accept_invite/?a=b&c=d&' - '' + ( + 'https://service.example.com/accept_invite/?a=b&c=d&' + ), + ( + '' + 'https://service.example.com/accept_invite/?a=b&c=d&' + '' + ), ), ), -)) + ids=['no_url_params', 'with_url_params'] +) def test_HTML_template_has_URLs_replaced_with_links(content, html_snippet): assert html_snippet in str(HTMLEmailTemplate({'content': content, 'subject': ''})) -@pytest.mark.parametrize( - "template_content,expected", [ - ("gov.uk", u"gov.\u200Buk"), - ("GOV.UK", u"GOV.\u200BUK"), - ("Gov.uk", u"Gov.\u200Buk"), - ("https://gov.uk", "https://gov.uk"), - ("https://www.gov.uk", "https://www.gov.uk"), - ("www.gov.uk", "www.gov.uk"), - ("gov.uk/register-to-vote", "gov.uk/register-to-vote"), - ("gov.uk?q=", "gov.uk?q=") - ] -) -def test_escaping_govuk_in_email_templates(template_content, expected): - assert unlink_govuk_escaped(template_content) == expected - assert expected in str(PlainTextEmailTemplate({'content': template_content, 'subject': ''})) - assert expected in str(HTMLEmailTemplate({'content': template_content, 'subject': ''})) - - def test_stripping_of_unsupported_characters_in_email_templates(): template_content = "line one\u2028line two" expected = "line oneline two" @@ -662,44 +627,48 @@ def test_stripping_of_unsupported_characters_in_email_templates(): assert expected in str(HTMLEmailTemplate({'content': template_content, 'subject': ''})) -@mock.patch('notifications_utils.template.add_prefix', return_value='') @pytest.mark.parametrize( - "template_class, prefix, body, expected_call", [ - (SMSMessageTemplate, "a", "b", (Markup("b"), "a")), - (SMSPreviewTemplate, "a", "b", (Markup("b"), "a")), - (SMSMessageTemplate, None, "b", (Markup("b"), None)), - (SMSPreviewTemplate, None, "b", (Markup("b"), None)), - (SMSMessageTemplate, 'ht&ml', "b", (Markup("b"), 'ht&ml')), - (SMSPreviewTemplate, 'ht&ml', "b", (Markup("b"), '<em>ht&ml</em>')), - ] + "template_class, prefix, body, expected", + [ + (SMSMessageTemplate, 'a', 'b', 'a: b'), + (SMSMessageTemplate, None, 'b', 'b'), + (SMSMessageTemplate, 'ht&ml', 'b', 'ht&ml: b'), + (SMSPreviewTemplate, 'a', 'b', '\n\n
    \n a: b\n
    '), + (SMSPreviewTemplate, None, 'b', '\n\n
    \n b\n
    '), + ( + SMSPreviewTemplate, + 'ht&ml', + 'b', + '\n\n
    \n <em>ht&ml</em>: b\n
    ', + ), + ], + ids=['message_a', 'message_none', 'message_html', 'preview_a', 'preview_none', 'preview_html'] ) -def test_sms_message_adds_prefix(add_prefix, template_class, prefix, body, expected_call): +def test_sms_templates_add_prefix(template_class, prefix, body, expected): template = template_class({'content': body}) template.prefix = prefix template.sender = None - str(template) - add_prefix.assert_called_once_with(*expected_call) + assert str(template) == expected -@mock.patch('notifications_utils.template.add_prefix', return_value='') -@pytest.mark.parametrize( - 'template_class', [SMSMessageTemplate, SMSPreviewTemplate] -) @pytest.mark.parametrize( - "show_prefix, prefix, body, sender, expected_call", [ - (False, "a", "b", "c", (Markup("b"), None)), - (True, "a", "b", None, (Markup("b"), "a")), - (True, "a", "b", False, (Markup("b"), "a")), + "template_class, show_prefix, prefix, body, sender, expected", + [ + (SMSMessageTemplate, False, "a", "b", "c", 'b'), + (SMSMessageTemplate, True, "a", "b", None, 'a: b'), + (SMSMessageTemplate, True, "a", "b", False, 'a: b'), + (SMSPreviewTemplate, False, "a", "b", "c", '\n\n
    \n b\n
    '), + (SMSPreviewTemplate, True, "a", "b", None, '\n\n
    \n a: b\n
    '), + (SMSPreviewTemplate, True, "a", "b", False, '\n\n
    \n a: b\n
    '), ] ) def test_sms_message_adds_prefix_only_if_asked_to( - add_prefix, + template_class, show_prefix, prefix, body, sender, - expected_call, - template_class, + expected, ): template = template_class( {'content': body}, @@ -707,8 +676,7 @@ def test_sms_message_adds_prefix_only_if_asked_to( show_prefix=show_prefix, sender=sender, ) - str(template) - add_prefix.assert_called_once_with(*expected_call) + assert str(template) == expected @pytest.mark.parametrize('content_to_look_for', [ @@ -754,7 +722,7 @@ def test_sms_messages_dont_downgrade_non_sms_if_setting_is_false(mock_sms_encode assert mock_sms_encode.called is False -@mock.patch('notifications_utils.template.nl2br') +@mock.patch('notifications_utils.template.nl2br', return_value='') def test_sms_preview_adds_newlines(nl2br): content = "the\nquick\n\nbrown fox" str(SMSPreviewTemplate({'content': content})) @@ -796,267 +764,6 @@ def test_sms_message_normalises_newlines(content): ) -@pytest.mark.skip(reason="not in use") -@freeze_time("2012-12-12 12:12:12") -@mock.patch('notifications_utils.template.LetterPreviewTemplate.jinja_template.render') -@mock.patch('notifications_utils.template.remove_empty_lines', return_value='123 Street') -@mock.patch('notifications_utils.template.unlink_govuk_escaped') -@mock.patch('notifications_utils.template.notify_letter_preview_markdown', return_value='Bar') -@mock.patch('notifications_utils.template.strip_pipes', side_effect=lambda x: x) -@pytest.mark.parametrize('values, expected_address', [ - ({}, Markup( - "address line 1\n" - "address line 2\n" - "address line 3\n" - "address line 4\n" - "address line 5\n" - "address line 6\n" - "postcode" - )), - ({ - 'address line 1': '123 Fake Street', - 'address line 6': 'United Kingdom', - }, Markup( - "123 Fake Street\n" - "address line 2\n" - "address line 3\n" - "address line 4\n" - "address line 5\n" - "United Kingdom\n" - "postcode" - )), - ({ - 'address line 1': '123 Fake Street', - 'address line 2': 'City of Town', - 'postcode': 'SW1A 1AA', - }, Markup( - "123 Fake Street\n" - "City of Town\n" - "\n" - "\n" - "\n" - "\n" - "SW1A 1AA" - )) -]) -@pytest.mark.parametrize('contact_block, expected_rendered_contact_block', [ - ( - None, - '' - ), - ( - '', - '' - ), - ( - """ - The Pension Service - Mail Handling Site A - Wolverhampton WV9 1LU - - Telephone: 0845 300 0168 - Email: fpc.customercare@dwp.gsi.gov.uk - Monday - Friday 8am - 6pm - www.gov.uk - """, - ( - 'The Pension Service
    ' - 'Mail Handling Site A
    ' - 'Wolverhampton WV9 1LU
    ' - '
    ' - 'Telephone: 0845 300 0168
    ' - 'Email: fpc.customercare@dwp.gsi.gov.uk
    ' - 'Monday - Friday 8am - 6pm
    ' - 'www.gov.uk' - ) - ) -]) -@pytest.mark.parametrize('extra_args, expected_logo_file_name, expected_logo_class', [ - ({}, None, None), - ({'logo_file_name': 'example.foo'}, 'example.foo', 'foo'), -]) -@pytest.mark.parametrize('additional_extra_args, expected_date', [ - ({}, '12 December 2012'), - ({'date': None}, '12 December 2012'), - ({'date': datetime.date.fromtimestamp(0)}, '1 January 1970'), -]) -def test_letter_preview_renderer( - strip_pipes, - letter_markdown, - unlink_govuk, - remove_empty_lines, - jinja_template, - values, - expected_address, - contact_block, - expected_rendered_contact_block, - extra_args, - expected_logo_file_name, - expected_logo_class, - additional_extra_args, - expected_date, -): - extra_args.update(additional_extra_args) - str(LetterPreviewTemplate( - {'content': 'Foo', 'subject': 'Subject'}, - values, - contact_block=contact_block, - **extra_args - )) - remove_empty_lines.assert_called_once_with(expected_address) - jinja_template.assert_called_once_with({ - 'address': '', - 'subject': 'Subject', - 'message': 'Bar', - 'date': expected_date, - 'contact_block': expected_rendered_contact_block, - 'admin_base_url': 'http://localhost:6012', - 'logo_file_name': expected_logo_file_name, - 'logo_class': expected_logo_class, - }) - letter_markdown.assert_called_once_with(Markup('Foo\n')) - unlink_govuk.assert_not_called() - assert strip_pipes.call_args_list == [ - mock.call('Subject'), - mock.call('Foo'), - mock.call(expected_address), - mock.call(expected_rendered_contact_block), - ] - - -@freeze_time("2001-01-01 12:00:00.000000") -@mock.patch('notifications_utils.template.LetterPreviewTemplate.jinja_template.render') -def test_letter_preview_renderer_without_mocks(jinja_template): - - str(LetterPreviewTemplate( - {'content': 'Foo', 'subject': 'Subject'}, - {'addressline1': 'name', 'addressline2': 'street', 'postcode': 'SW1 1AA'}, - contact_block='', - )) - - jinja_template_locals = jinja_template.call_args_list[0][0][0] - - assert jinja_template_locals['address'] == ( - '' - ) - assert jinja_template_locals['subject'] == 'Subject' - assert jinja_template_locals['message'] == "

    Foo

    " - assert jinja_template_locals['date'] == '1 January 2001' - assert jinja_template_locals['contact_block'] == '' - assert jinja_template_locals['admin_base_url'] == 'http://localhost:6012' - assert jinja_template_locals['logo_file_name'] is None - - -@freeze_time("2012-12-12 12:12:12") -@mock.patch('notifications_utils.template.LetterImageTemplate.jinja_template.render') -@pytest.mark.parametrize('page_count, expected_oversized, expected_page_numbers', [ - ( - 1, False, - [1], - ), - ( - 5, False, - [1, 2, 3, 4, 5], - ), - ( - 10, False, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - ), - ( - 11, True, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - ), - ( - 99, True, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - ), -]) -@pytest.mark.parametrize('postage_args, expected_postage', ( - pytest.param({}, 'second'), - pytest.param({'postage': 'first'}, 'first'), - pytest.param({'postage': 'second'}, 'second'), - pytest.param({'postage': 'third'}, 'third', marks=pytest.mark.xfail(raises=TypeError)), -)) -def test_letter_image_renderer( - jinja_template, - page_count, - expected_page_numbers, - expected_oversized, - postage_args, - expected_postage, -): - str(LetterImageTemplate( - {'content': 'Content', 'subject': 'Subject'}, - image_url='http://example.com/endpoint.png', - page_count=page_count, - contact_block='10 Downing Street', - **postage_args - )) - jinja_template.assert_called_once_with({ - 'image_url': 'http://example.com/endpoint.png', - 'page_numbers': expected_page_numbers, - 'too_many_pages': expected_oversized, - 'address': ( - "" - ), - 'contact_block': '10 Downing Street', - 'date': '12 December 2012', - 'subject': 'Subject', - 'message': '

    Content

    ', - 'postage': expected_postage, - }) - - -@pytest.mark.parametrize('page_image_url', [ - pytest.param('http://example.com/endpoint.png?page=0', marks=pytest.mark.xfail), - 'http://example.com/endpoint.png?page=1', - 'http://example.com/endpoint.png?page=2', - 'http://example.com/endpoint.png?page=3', - pytest.param('http://example.com/endpoint.png?page=4', marks=pytest.mark.xfail), -]) -def test_letter_image_renderer_pagination(page_image_url): - assert page_image_url in str(LetterImageTemplate( - {'content': '', 'subject': ''}, - image_url='http://example.com/endpoint.png', - page_count=3, - )) - - -@pytest.mark.parametrize('partial_call, expected_exception', [ - ( - partial(LetterImageTemplate), - TypeError - ), - ( - partial(LetterImageTemplate, page_count=1), - TypeError - ), - ( - partial(LetterImageTemplate, image_url='foo'), - TypeError - ), - ( - partial(LetterImageTemplate, image_url='foo', page_count='foo'), - ValueError - ), -]) -def test_letter_image_renderer_requires_arguments(partial_call, expected_exception): - with pytest.raises(expected_exception): - partial_call({'content': '', 'subject': ''}) - - def test_sets_subject(): assert WithSubjectTemplate({"content": '', 'subject': 'Your tax is due'}).subject == 'Your tax is due' @@ -1066,8 +773,6 @@ def test_subject_line_gets_applied_to_correct_template_types(): EmailPreviewTemplate, HTMLEmailTemplate, PlainTextEmailTemplate, - LetterPreviewTemplate, - LetterImageTemplate, ]: assert issubclass(cls, WithSubjectTemplate) for cls in [ @@ -1095,12 +800,12 @@ def test_subject_line_gets_replaced(): mock.call('content', {}, html='passthrough', markdown_lists=True) ]), (HTMLEmailTemplate, {}, [ - mock.call('content', {}, preview_mode=False, html='escape', markdown_lists=True, - redact_missing_personalisation=False), + mock.call('content', {}, html='passthrough', markdown_lists=True, + redact_missing_personalisation=False, preview_mode=False), mock.call('content', {}, html='escape', markdown_lists=True), ]), (EmailPreviewTemplate, {}, [ - mock.call('content', {}, preview_mode=False, html='escape', markdown_lists=True, + mock.call('content', {}, preview_mode=False, html='passthrough', markdown_lists=True, redact_missing_personalisation=False), mock.call('subject', {}, html='escape', redact_missing_personalisation=False), mock.call('((email address))', {}, with_brackets=False), @@ -1109,38 +814,8 @@ def test_subject_line_gets_replaced(): mock.call('content', {}, html='passthrough'), ]), (SMSPreviewTemplate, {}, [ - mock.call('((phone number))', {}, with_brackets=False, html='escape'), mock.call('content', {}, html='escape', redact_missing_personalisation=False), - ]), - (LetterPreviewTemplate, {'contact_block': 'www.gov.uk'}, [ - mock.call('subject', {}, html='escape', is_letter_template=True, redact_missing_personalisation=False), - mock.call('content', {}, html='escape', markdown_lists=True, redact_missing_personalisation=False), - mock.call(( - '((address line 1))\n' - '((address line 2))\n' - '((address line 3))\n' - '((address line 4))\n' - '((address line 5))\n' - '((address line 6))\n' - '((postcode))' - ), {}, with_brackets=False, html='escape', is_letter_template=True), - mock.call('www.gov.uk', {}, html='escape', redact_missing_personalisation=False), - ]), - (LetterImageTemplate, { - 'image_url': 'http://example.com', 'page_count': 1, 'contact_block': 'www.gov.uk' - }, [ - mock.call(( - '((address line 1))\n' - '((address line 2))\n' - '((address line 3))\n' - '((address line 4))\n' - '((address line 5))\n' - '((address line 6))\n' - '((postcode))' - ), {}, with_brackets=False, html='escape', is_letter_template=True), - mock.call('www.gov.uk', {}, html='escape', redact_missing_personalisation=False), - mock.call('subject', {}, html='escape', redact_missing_personalisation=False, is_letter_template=True), - mock.call('content', {}, html='escape', markdown_lists=True, redact_missing_personalisation=False), + mock.call('((phone number))', {}, with_brackets=False, html='escape'), ]), (Template, {'redact_missing_personalisation': True}, [ mock.call('content', {}, html='escape', redact_missing_personalisation=True), @@ -1149,28 +824,14 @@ def test_subject_line_gets_replaced(): mock.call('content', {}, html='passthrough', redact_missing_personalisation=True, markdown_lists=True), ]), (EmailPreviewTemplate, {'redact_missing_personalisation': True}, [ - mock.call('content', {}, preview_mode=False, html='escape', markdown_lists=True, + mock.call('content', {}, preview_mode=False, html='passthrough', markdown_lists=True, redact_missing_personalisation=True), mock.call('subject', {}, html='escape', redact_missing_personalisation=True), mock.call('((email address))', {}, with_brackets=False), ]), (SMSPreviewTemplate, {'redact_missing_personalisation': True}, [ - mock.call('((phone number))', {}, with_brackets=False, html='escape'), mock.call('content', {}, html='escape', redact_missing_personalisation=True), - ]), - (LetterPreviewTemplate, {'contact_block': 'www.gov.uk', 'redact_missing_personalisation': True}, [ - mock.call('subject', {}, html='escape', redact_missing_personalisation=True, is_letter_template=True), - mock.call('content', {}, html='escape', markdown_lists=True, redact_missing_personalisation=True), - mock.call(( - '((address line 1))\n' - '((address line 2))\n' - '((address line 3))\n' - '((address line 4))\n' - '((address line 5))\n' - '((address line 6))\n' - '((postcode))' - ), {}, with_brackets=False, html='escape', is_letter_template=True), - mock.call('www.gov.uk', {}, html='escape', redact_missing_personalisation=True), + mock.call('((phone number))', {}, with_brackets=False, html='escape'), ]), ]) @mock.patch('notifications_utils.template.Field.__init__', return_value=None) @@ -1184,123 +845,26 @@ def test_templates_handle_html_and_redacting( ): assert str(template_class({'content': 'content', 'subject': 'subject'}, **extra_args)) assert mock_field_init.call_args_list == expected_field_calls + mock_field_str.assert_called() -@pytest.mark.parametrize('template_class, extra_args, expected_remove_whitespace_calls', [ - (PlainTextEmailTemplate, {}, [ - mock.call('\n\ncontent'), - mock.call(Markup('subject')), - mock.call(Markup('subject')), - ]), - (HTMLEmailTemplate, {}, [ - mock.call( - '

    ' - 'content' - '

    ' - ), - mock.call('\n\ncontent'), - mock.call(Markup('subject')), - mock.call(Markup('subject')), - ]), - (EmailPreviewTemplate, {}, [ - mock.call( - '

    ' - 'content' - '

    ' - ), - mock.call(Markup('subject')), - mock.call(Markup('subject')), - mock.call(Markup('subject')), - ]), - (SMSMessageTemplate, {}, [ - mock.call('content'), - ]), - (SMSPreviewTemplate, {}, [ - mock.call('content'), - ]), - (LetterPreviewTemplate, {'contact_block': 'www.gov.uk'}, [ - mock.call(Markup('subject')), - mock.call(Markup('

    content

    ')), - mock.call(( - "address line 1\n" - "address line 2\n" - "address line 3\n" - "address line 4\n" - "address line 5\n" - "address line 6\n" - "postcode" - )), - mock.call(Markup('www.gov.uk')), - mock.call(Markup('subject')), - mock.call(Markup('subject')), - ]), -]) -@mock.patch('notifications_utils.template.remove_whitespace_before_punctuation', side_effect=lambda x: x) -def test_templates_remove_whitespace_before_punctuation( - mock_remove_whitespace, - template_class, - extra_args, - expected_remove_whitespace_calls, -): - template = template_class({'content': 'content', 'subject': 'subject'}, **extra_args) - - assert str(template) - - if hasattr(template, 'subject'): - assert template.subject - - assert mock_remove_whitespace.call_args_list == expected_remove_whitespace_calls - - -@pytest.mark.parametrize('template_class, extra_args, expected_calls', [ - (PlainTextEmailTemplate, {}, [ - mock.call('\n\ncontent'), - mock.call(Markup('subject')), - ]), - (HTMLEmailTemplate, {}, [ - mock.call( - '

    ' - 'content' - '

    ' - ), - mock.call('\n\ncontent'), - mock.call(Markup('subject')), - ]), - (EmailPreviewTemplate, {}, [ - mock.call( - '

    ' - 'content' - '

    ' - ), - mock.call(Markup('subject')), - ]), - (SMSMessageTemplate, {}, [ - ]), - (SMSPreviewTemplate, {}, [ - ]), - (LetterPreviewTemplate, {'contact_block': 'www.gov.uk'}, [ - mock.call(Markup('subject')), - mock.call(Markup('

    content

    ')), - ]), -]) -@mock.patch('notifications_utils.template.make_quotes_smart', side_effect=lambda x: x) -@mock.patch('notifications_utils.template.replace_hyphens_with_en_dashes', side_effect=lambda x: x) -def test_templates_make_quotes_smart_and_dashes_en( - mock_en_dash_replacement, - mock_smart_quotes, - template_class, - extra_args, - expected_calls, -): - template = template_class({'content': 'content', 'subject': 'subject'}, **extra_args) +@pytest.mark.parametrize( + 'template_class', + [ + PlainTextEmailTemplate, + HTMLEmailTemplate, + EmailPreviewTemplate, + SMSMessageTemplate, + SMSPreviewTemplate, + ], +) +def test_templates_remove_whitespace_before_punctuation(template_class): + template = template_class({'content': 'content \t\t .', 'subject': 'subject\t \t,'}) - assert str(template) + assert 'content.' in str(template) if hasattr(template, 'subject'): - assert template.subject - - mock_smart_quotes.assert_has_calls(expected_calls) - mock_en_dash_replacement.assert_has_calls(expected_calls) + assert template.subject == 'subject,' @pytest.mark.parametrize('content', ( @@ -1332,18 +896,6 @@ def test_smart_quotes_removed_from_long_template_in_under_a_second(): assert process_time() - start_time < 1 -def test_basic_templates_return_markup(): - - template_dict = {'content': 'content', 'subject': 'subject'} - - for output in [ - str(Template(template_dict)), - str(WithSubjectTemplate(template_dict)), - WithSubjectTemplate(template_dict).subject, - ]: - assert isinstance(output, Markup) - - @pytest.mark.parametrize('template_instance, expected_placeholders', [ ( SMSMessageTemplate( @@ -1375,28 +927,12 @@ def test_basic_templates_return_markup(): ), ['content', 'subject'], ), - ( - LetterPreviewTemplate( - {"content": "((content))", "subject": "((subject))"}, - contact_block='((contact_block))', - ), - ['content', 'subject', 'contact_block'], - ), - ( - LetterImageTemplate( - {"content": "((content))", "subject": "((subject))"}, - contact_block='((contact_block))', - image_url='http://example.com', - page_count=99, - ), - ['content', 'subject', 'contact_block'], - ), ]) def test_templates_extract_placeholders( template_instance, expected_placeholders, ): - assert template_instance.placeholders == set(expected_placeholders) + assert template_instance.placeholder_names == set(expected_placeholders) @pytest.mark.parametrize('extra_args', [ @@ -1465,20 +1001,6 @@ def test_email_preview_shows_recipient_address( assert expected_content in str(template) -@mock.patch('notifications_utils.template.strip_dvla_markup', return_value='FOOBARBAZ') -def test_letter_preview_strips_dvla_markup(mock_strip_dvla_markup): - assert 'FOOBARBAZ' in str(LetterPreviewTemplate( - { - "content": 'content', - 'subject': 'subject', - }, - )) - assert mock_strip_dvla_markup.call_args_list == [ - mock.call(Markup('subject')), - mock.call('content'), - ] - - dvla_file_spec = [ { 'Field number': '1', @@ -1856,171 +1378,6 @@ def test_letter_preview_strips_dvla_markup(mock_strip_dvla_markup): ] -@pytest.mark.parametrize("address, expected", [ - ( - { - "address line 1": "line 1", - "address line 2": "line 2", - "address line 3": "line 3", - "address line 4": "line 4", - "address line 5": "line 5", - "address line 6": "line 6", - "postcode": "N1 4W2", - }, - { - "addressline1": "line 1", - "addressline2": "line 2", - "addressline3": "line 3", - "addressline4": "line 4", - "addressline5": "line 5", - "addressline6": "line 6", - "postcode": "N1 4W2", - }, - ), ( - { - "addressline1": "line 1", - "addressline2": "line 2", - "addressline3": "line 3", - "addressline4": "line 4", - "addressline5": "line 5", - "addressLine6": "line 6", - "postcode": "N1 4W2", - }, - { - "addressline1": "line 1", - "addressline2": "line 2", - "addressline3": "line 3", - "addressline4": "line 4", - "addressline5": "line 5", - "addressline6": "line 6", - "postcode": "N1 4W2", - }, - ), - ( - { - "addressline1": "line 1", - "addressline3": "line 3", - "addressline5": "line 5", - "addressline6": "line 6", - "postcode": "N1 4W2", - }, - { - "addressline1": "line 1", - # addressline2 is required, but not given - "addressline3": "line 3", - "addressline4": "", - "addressline5": "line 5", - "addressline6": "line 6", - "postcode": "N1 4W2", - }, - ), - ( - { - "addressline1": "line 1", - "addressline2": "line 2", - "addressline3": None, - "addressline6": None, - "postcode": "N1 4W2", - }, - { - "addressline1": "line 1", - "addressline2": "line 2", - "addressline3": "", - "addressline4": "", - "addressline5": "", - "addressline6": "", - "postcode": "N1 4W2", - }, - ), - ( - { - "addressline1": "line 1", - "addressline2": "\t ,", - "postcode": "N1 4W2", - }, - { - "addressline1": "line 1", - "addressline2": "\t ,", - "addressline3": "", - "addressline4": "", - "addressline5": "", - "addressline6": "", - "postcode": "N1 4W2", - }, - ), -]) -def test_letter_address_format(address, expected): - template = LetterPreviewTemplate( - {'content': '', 'subject': ''}, - address, - ) - assert template.values_with_default_optional_address_lines == expected - # check that we can actually build a valid letter from this data - assert str(template) - - -@freeze_time("2001-01-01 12:00:00.000000") -@pytest.mark.parametrize('markdown, expected', [ - ( - ( - 'Here is a list of bullets:\n' - '\n' - '* one\n' - '* two\n' - '* three\n' - '\n' - 'New paragraph' - ), - ( - '\n' - '

    New paragraph

    \n' - ) - ), - ( - ( - '# List title:\n' - '\n' - '* one\n' - '* two\n' - '* three\n' - ), - ( - '

    List title:

    \n' - '\n' - ) - ), - ( - ( - 'Here’s an ordered list:\n' - '\n' - '1. one\n' - '2. two\n' - '3. three\n' - ), - ( - '

    Here’s an ordered list:

      \n' - '
    1. one
    2. \n' - '
    3. two
    4. \n' - '
    5. three
    6. \n' - '
    ' - ) - ), -]) -def test_lists_in_combination_with_other_elements_in_letters(markdown, expected): - assert expected in str(LetterPreviewTemplate( - {'content': markdown, 'subject': 'Hello'}, - {}, - )) - - @pytest.mark.parametrize('template_class', [ SMSMessageTemplate, SMSPreviewTemplate, @@ -2035,8 +1392,6 @@ def test_message_too_long(template_class): (EmailPreviewTemplate, {}), (HTMLEmailTemplate, {}), (PlainTextEmailTemplate, {}), - (LetterPreviewTemplate, {}), - (LetterImageTemplate, {'image_url': 'foo', 'page_count': 1}), ]) def test_non_sms_ignores_message_too_long(template_class, kwargs): body = 'a' * 1000 @@ -2044,52 +1399,6 @@ def test_non_sms_ignores_message_too_long(template_class, kwargs): assert template.is_message_too_long() is False -@pytest.mark.parametrize( - ( - 'content,' - 'expected_preview_markup,' - ), [ - ( - 'a\n\n\nb', - ( - '

    a

    ' - '

    b

    ' - ), - ), - ( - ( - 'a\n' - '\n' - '* one\n' - '* two\n' - '* three\n' - 'and a half\n' - '\n' - '\n' - '\n' - '\n' - 'foo' - ), - ( - '

    a

    \n' - '

    foo

    ' - ), - ), - ] -) -def test_multiple_newlines_in_letters( - content, - expected_preview_markup, -): - assert expected_preview_markup in str(LetterPreviewTemplate( - {'content': content, 'subject': 'foo'} - )) - - @pytest.mark.parametrize('subject', [ ' no break ', ' no\tbreak ', @@ -2103,7 +1412,6 @@ def test_multiple_newlines_in_letters( (PlainTextEmailTemplate, {}), (HTMLEmailTemplate, {}), (EmailPreviewTemplate, {}), - (LetterPreviewTemplate, {}), ]) def test_whitespace_in_subjects(template_class, subject, extra_args): @@ -2127,20 +1435,24 @@ def test_whitespace_in_subject_placeholders(template_class): ).subject == 'Your tax is due' -@pytest.mark.parametrize('template_class, expected_output', [ - ( - PlainTextEmailTemplate, - 'paragraph one\n\n\xa0\n\nparagraph two', - ), - ( - HTMLEmailTemplate, +@pytest.mark.parametrize( + 'template_class, expected_output', + [ ( - '

    paragraph one

    ' - '

     

    ' - '

    paragraph two

    ' + PlainTextEmailTemplate, + 'paragraph one\n\n\xa0\n\nparagraph two', ), - ), -]) + ( + HTMLEmailTemplate, + ( + '

    paragraph one

    \n' + '

    \xa0

    \n' + '

    paragraph two

    \n' + ), + ), + ], + ids=['plain', 'html'] +) def test_govuk_email_whitespace_hack(template_class, expected_output): template_instance = template_class({ @@ -2150,61 +1462,6 @@ def test_govuk_email_whitespace_hack(template_class, expected_output): assert expected_output in str(template_instance) -def test_letter_preview_uses_non_breaking_hyphens(): - assert 'non\u2011breaking' in str(LetterPreviewTemplate( - {'content': 'non-breaking', 'subject': 'foo'} - )) - assert '–' in str(LetterPreviewTemplate( - {'content': 'en dash - not hyphen - when set with spaces', 'subject': 'foo'} - )) - - -@freeze_time("2001-01-01 12:00:00.000000") -def test_nested_lists_in_lettr_markup(): - - template_content = str(LetterPreviewTemplate({ - 'content': ( - 'nested list:\n' - '\n' - '1. one\n' - '2. two\n' - '3. three\n' - ' - three one\n' - ' - three two\n' - ' - three three\n' - ), - 'subject': 'foo', - })) - - assert ( - '

    \n' - ' 1 January 2001\n' - '

    \n' - '

    \n' - ' foo\n' - '

    \n' - '

    nested list:

      \n' - '
    1. one
    2. \n' - '
    3. two
    4. \n' - '
    5. three
        \n' - '
      • three one
      • \n' - '
      • three two
      • \n' - '
      • three three
      • \n' - '
    6. \n' - '
    \n' - '\n' - ' \n' - ' \n' - '' - ) in template_content - - -def test_that_print_template_is_the_same_as_preview(): - assert dir(LetterPreviewTemplate) == dir(LetterPrintTemplate) - assert os.path.basename(LetterPreviewTemplate.jinja_template.filename) == 'preview.jinja2' - assert os.path.basename(LetterPrintTemplate.jinja_template.filename) == 'print.jinja2' - - def test_plain_text_email_whitespace(): email = PlainTextEmailTemplate({'subject': 'foo', 'content': ( '# Heading\n' @@ -2229,71 +1486,129 @@ def test_plain_text_email_whitespace(): assert str(email) == ( 'Heading\n' '-----------------------------------------------------------------\n' - '\n' '1. one\n' '2. two\n' '3. three\n' '\n' '=================================================================\n' '\n' - '\n' - 'Heading\n' + '\n\nHeading\n' '-----------------------------------------------------------------\n' - '\n' 'Paragraph\n' '\n' 'Paragraph\n' '\n' - 'callout\n' - '\n' + '\n\ncallout\n\n\n\n' '1. one not four\n' '2. two not five\n' + '\n' ) -@pytest.mark.parametrize('renderer, expected_content', ( - (PlainTextEmailTemplate, ( - 'Heading link: https://example.com\n' - '-----------------------------------------------------------------\n' - )), - (HTMLEmailTemplate, ( - '

    ' - 'Heading link' - '

    ' - )), - (LetterPreviewTemplate, ( - '

    Heading link: example.com

    ' - )), - (LetterPrintTemplate, ( - '

    Heading link: example.com

    ' - )), -)) +@pytest.mark.parametrize( + 'renderer, expected_content', + ( + (PlainTextEmailTemplate, ( + 'Heading link: https://example.com\n' + '-----------------------------------------------------------------\n' + )), + (HTMLEmailTemplate, ( + '

    ' + 'Heading link' + '

    \n' + )), + ), + ids=['PlainTextEmailTemplate', 'HTMLEmailTemplate'] +) def test_heading_only_template_renders(renderer, expected_content): - assert expected_content in str(renderer({'subject': 'foo', 'content': ( - '# Heading [link](https://example.com)' - )})) + assert expected_content in str(renderer( + { + 'subject': 'foo', + 'content': '# Heading [link](https://example.com)', + } + )) -@pytest.mark.parametrize("template_class", [ - LetterPreviewTemplate, - LetterPrintTemplate, -]) -@pytest.mark.parametrize("filename, expected_html_class", [ - ('example.png', 'class="png"'), - ('example.svg', 'class="svg"'), -]) -def test_image_class_applied_to_logo(template_class, filename, expected_html_class): - assert expected_html_class in str(template_class( - {'content': 'Foo', 'subject': 'Subject'}, - logo_file_name=filename, - )) +@pytest.mark.parametrize( + 'template_type, expected_content', + ( + (PlainTextEmailTemplate, 'Hi\n\n\n\nThis is a block quote.\n\n\n\nhello\n\n'), + (HTMLEmailTemplate, ( + f'

    Hi

    \n' + f'
    \n' + f'

    This is a block quote.

    \n' + '
    \n' + f'

    hello

    \n' + )), + ), + ids=['plain', 'html'] +) +def test_block_quotes(template_type, expected_content): + """ + Template markup uses ^ to denote a block quote, but Github markdown, which Mistune reflects, specifies a block + quote with the > character. Rather than write a custom parser, templates should preprocess their text to replace + the former with the latter. + """ + + assert expected_content in str( + template_type({'content': '\nHi\n\n^ This is a block quote.\n\nhello', 'subject': ''}) + ) -@pytest.mark.parametrize("template_class", [ - LetterPreviewTemplate, - LetterPrintTemplate, -]) -def test_image_not_present_if_no_logo(template_class): - # can't test that the html doesn't move in utils - tested in template preview instead - assert '\n' + f'
  • one
  • \n' + f'
  • two
  • \n' + f'
  • three
  • \n' + '\n' + ), + ] +) +def test_ordered_list_without_spaces(template_type, expected): + """ + Proper markdown for ordered lists has a space after the number and period. + """ + + content = '1.one\n2.two\n3.three\n' + + if isinstance(template_type, PlainTextEmailTemplate): + assert str(template_type({'content': content, 'subject': ''})) == expected + else: + assert expected in str(template_type({'content': content, 'subject': ''})) + + +@pytest.mark.parametrize('with_spaces', [True, False]) +@pytest.mark.parametrize( + 'template_type, expected', + [ + (PlainTextEmailTemplate, '• one\n• two\n• three\n\n'), + ( + HTMLEmailTemplate, + f'
      \n' + f'
    • one
    • \n' + f'
    • two
    • \n' + f'
    • three
    • \n' + '
    \n' + ), + ] +) +@pytest.mark.parametrize('bullet', ['*', '-', '+', '•']) +def test_unordered_list(bullet, template_type, expected, with_spaces): + """ + Proper markdown for unordered lists has a space after the bullet. + """ + + space = ' ' if with_spaces else '' + content = f'{bullet}{space}one\n{bullet}{space}two\n{bullet}{space}three\n' + + if isinstance(template_type, PlainTextEmailTemplate): + assert str(template_type({'content': content, 'subject': ''})) == expected + else: + assert expected in str(template_type({'content': content, 'subject': ''}))