From b3657117dd02c21103cf8c32ffc40216acc96577 Mon Sep 17 00:00:00 2001
From: Christine Cunningham <5705329+ccunningham101@users.noreply.github.com>
Date: Sun, 11 Feb 2024 20:15:23 +0000
Subject: [PATCH 1/2] Clarify email text

We are getting feedback from authors saying there was a mistake, they
cited the paper before the retraction. Clarify that we are interested
in whether they know about the retraction now (for the future rate of
citations).
---
 .../management/commands/send_retraction_emails.py        | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/retractions/management/commands/send_retraction_emails.py b/retractions/management/commands/send_retraction_emails.py
index 29aa9ea..cd8061f 100644
--- a/retractions/management/commands/send_retraction_emails.py
+++ b/retractions/management/commands/send_retraction_emails.py
@@ -223,7 +223,10 @@ def _get_body_and_subject(self, pairs, recentest_citing_paper, author):
         body = "<p>Dear %s,</p>" % aliases[0].full_name()
         body += "<p>"
         body += """We're writing to let you know that the following paper(s) cited a
-                paper which has been retracted."""
+                paper which has been retracted.<br>Please note, we are
+                interested in reducing future citations of retracted papers, so
+                your citation may have happened before or after the paper was
+                retracted."""
         body += "</p>"
         body += "<table border='1' cellpadding='0' cellspacing='0' width='80%' style='border-collapse: collapse;'>"
         body += "<tr>"
@@ -262,8 +265,8 @@ def _get_body_and_subject(self, pairs, recentest_citing_paper, author):
             papers.</p>
 
             <p><strong>Was this information useful?</strong><br/>
-            Please click below to let us know whether you knew about the
-            retraction.
+            Please click below to let us know <strong>whether you know about
+            the retraction now, not based on when you cited it</strong>.
 
             <br>Your voluntary click, below, is taken as consent for your
             response to be included in our aggregated analysis. If you have any

From 97c01e103c9249b13e097be6dd1458df8fa27077 Mon Sep 17 00:00:00 2001
From: Christine Cunningham <5705329+ccunningham101@users.noreply.github.com>
Date: Sun, 11 Feb 2024 20:24:03 +0000
Subject: [PATCH 2/2] Ignore undeliverable emails

Our bounce rate was too high. We used external validations to check a
percentage of the scheduled emails. Rather than changing the datamodel
to store this information, provide the full underliverable email list
csv as an argument.
---
 .../commands/send_retraction_emails.py        | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/retractions/management/commands/send_retraction_emails.py b/retractions/management/commands/send_retraction_emails.py
index cd8061f..600f81f 100644
--- a/retractions/management/commands/send_retraction_emails.py
+++ b/retractions/management/commands/send_retraction_emails.py
@@ -1,12 +1,14 @@
 import email.utils
 import logging
 import mailbox
+import pathlib
 
 import anymail.exceptions
 import anymail.utils
 import django.core.exceptions
 import django.db
 import html2text
+import pandas
 from django.core.mail import EmailMultiAlternatives
 from django.core.management.base import BaseCommand
 
@@ -59,6 +61,11 @@ def add_arguments(self, parser):
             help="Maximum number of authors to send to",
             default=None,
         )
+        parser.add_argument(
+            "--undeliverable-file",
+            type=pathlib.Path,
+            help="File with validation result",
+        )
 
     def handle(self, *args, **options):
         setup.setup_logger(options["verbosity"])
@@ -78,6 +85,11 @@ def handle(self, *args, **options):
 
         self.test_email = options["test-email"]
 
+        if options["undeliverable_file"]:
+            self.undeliverable = self._get_undeliverable(options["undeliverable_file"])
+        else:
+            self.undeliverable = None
+
         # Filter for the authors who we would send mail to
         authors = (
             Author.objects.filter(pairs__retractedpaper__rct_group="i")
@@ -97,6 +109,15 @@ def handle(self, *args, **options):
 
             self._send_for_author(author)
 
+    def _get_undeliverable(self, results_path):
+        results_df = pandas.read_csv(results_path)
+        assert "result" in results_df.columns
+        undeliverable = results_df[
+            (results_df.result == "undeliverable")
+            | (results_df.result == "do_not_send")
+        ]
+        return undeliverable.address
+
     def _send_for_author(self, author):
         """
         Create and send mails for the author.
@@ -148,6 +169,16 @@ def _get_mail_to_send(self, author, intervention_pairs):
                     )
                     valid = False
 
+                if (
+                    self.undeliverable is not None
+                    and author_alias.email_address in self.undeliverable.values
+                ):
+                    logging.warning(
+                        "  Ignoring undeliverable email %s",
+                        author_alias.email_address,
+                    )
+                    valid = False
+
                 if valid:
                     if self.test_email:
                         logging.warning(
@@ -376,6 +407,8 @@ def _actually_send_mails(self, author, pairs, mail_to_send):
                 )
         except anymail.exceptions.AnymailError as e:
             logging.exception("  Error trying to send email: %s", str(e))
+        except UnicodeError as e:
+            logging.exception("  Unicode error trying to send email: %s", str(e))
         except django.db.utils.Error as e:
             logging.exception("  Database error while mailing: %s", str(e))
             raise