Unsubscribe links and domain-only "From" email addresses (#3721)

* Generate unsubscribe links and add unsubscribe headers to emails * Add separate view for oneclick POSTs; add unsubscribe confirmation button * Constrain "from" email address for comm panel * lint fixes * Fix regex * Make director email match domain/subdomain pattern * Add migration; fix regex * Fix tests * fix more tests * require that the from_email for send_mail matches our DMARC; always include a plaintext version of emails * lint fix * Add info@ PlainRedirect if it doesn't exist with migration * Misc fixes * lint fix * Include email in error message to help with debugging * Support named email addresses * Fix commpanel test * Add username
learning-unlimited · Mar 9, 2024 · d08d039 · d08d039
1 parent 324e0a6
commit d08d039
Show file tree

Hide file tree

Showing 19 changed files with 291 additions and 38 deletions.
diff --git a/esp/esp/dbmail/admin.py b/esp/esp/dbmail/admin.py
@@ -61,8 +61,8 @@ class MessageRequestAdmin(admin.ModelAdmin):
 admin_site.register(MessageRequest, MessageRequestAdmin)
 
 class TextOfEmailAdmin(admin.ModelAdmin):
-    list_display = ('id', 'send_from', 'send_to', 'subject', 'sent')
-    search_fields = ('=id', 'send_from', 'send_to', 'subject')
+    list_display = ('id', 'send_from', 'send_to', 'subject', 'sent', 'user')
+    search_fields = ('=id', 'send_from', 'send_to', 'subject', 'user')
     date_hierarchy = 'sent'
     list_filter = ('send_from',)
 admin_site.register(TextOfEmail, TextOfEmailAdmin)
diff --git a/esp/esp/dbmail/migrations/0006_textofemail_user.py b/esp/esp/dbmail/migrations/0006_textofemail_user.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2024-02-08 20:52
+from __future__ import unicode_literals
+
+from django.db import migrations
+import django.db.models.deletion
+import esp.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0034_auto_20230525_2024'),
+        ('dbmail', '0005_auto_20220719_2056'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='textofemail',
+            name='user',
+            field=esp.db.fields.AjaxForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='users.ESPUser'),
+        ),
+    ]
diff --git a/esp/esp/dbmail/models.py b/esp/esp/dbmail/models.py
@@ -52,22 +52,37 @@
 
 from django.conf import settings
 from django.contrib.sites.models import Site
+from django.utils.html import strip_tags
 
 from django.core.mail import get_connection
 from django.core.mail.backends.smtp import EmailBackend as SMTPEmailBackend
 from django.core.mail.message import sanitize_address
 from django.core.exceptions import ImproperlyConfigured
 
+# `user` is required for marketing and subscribed messages to add unsubscribe headers
+# this includes all comm panel emails
+# https://support.google.com/a/answer/81126?visit_id=638428689824104778-3542874255&rd=1#subscriptions
 def send_mail(subject, message, from_email, recipient_list, fail_silently=False, bcc=None,
-              return_path=settings.DEFAULT_EMAIL_ADDRESSES['bounces'], extra_headers={},
+              return_path=settings.DEFAULT_EMAIL_ADDRESSES['bounces'], extra_headers={}, user=None,
               *args, **kwargs):
     from_email = from_email.strip()
+    # the from_email must match one of our DMARC domains/subdomains
+    # or the email may be rejected by email clients
+    if not re.match(r"(^.+@%s>?$)|(^.+@(\w+\.)?learningu\.org>?$)" % settings.SITE_INFO[1].replace(".", "\."), from_email):
+        raise ESPError("Invalid 'From' email address (" + from_email + "). The 'From' email address must " +
+                       "end in @" + settings.SITE_INFO[1] + " (your website), " +
+                       "@learningu.org, or a valid subdomain of learningu.org " +
+                       "(i.e., @subdomain.learningu.org).")
+
     if 'Reply-To' in extra_headers:
         extra_headers['Reply-To'] = extra_headers['Reply-To'].strip()
     if isinstance(recipient_list, basestring):
         new_list = [ recipient_list ]
     else:
         new_list = [ x for x in recipient_list ]
+    if user is not None:
+        extra_headers['List-Unsubscribe-Post'] = "List-Unsubscribe=One-Click"
+        extra_headers['List-Unsubscribe'] = '<%s>' % (user.unsubscribe_oneclick())
 
     # remove duplicate email addresses (sendgrid doesn't like them)
     recipients = []
@@ -83,17 +98,24 @@ def send_mail(subject, message, from_email, recipient_list, fail_silently=False,
                 recipients.append(x)
                 emails.remove(tmp)
 
-    from django.core.mail import EmailMessage #send_mail as django_send_mail
+    from django.core.mail import EmailMessage, EmailMultiAlternatives #send_mail as django_send_mail
     logger.info("Sent mail to %s", recipients)
 
     #   Get whatever type of email connection Django provides.
     #   Normally this will be SMTP, but it also has an in-memory backend for testing.
     connection = get_connection(fail_silently=fail_silently, return_path=return_path)
-    msg = EmailMessage(subject, message, from_email, recipients, bcc=bcc, connection=connection, headers=extra_headers)
 
     #   Detect HTML tags in message and change content-type if they are found
     if '<html>' in message:
-        msg.content_subtype = 'html'
+        # Generate a plaintext version of the email
+        # Remove html tags and continuous whitespaces
+        text_only = re.sub('[ \t]+', ' ', strip_tags(message))
+        # Strip single spaces in the beginning of each line
+        message_text = text_only.replace('\n ', '\n').strip()
+        msg = EmailMultiAlternatives(subject, message_text, from_email, recipients, bcc=bcc, connection=connection, headers=extra_headers)
+        msg.attach_alternative(message, "text/html")
+    else:
+        msg = EmailMessage(subject, message, from_email, recipients, bcc=bcc, connection=connection, headers=extra_headers)
 
     msg.send()
 
@@ -350,6 +372,7 @@ def process(self):
                 newemailrequest = {'target': user, 'msgreq': self}
                 send_to = ESPUser.email_sendto_address(*address_pair)
                 newtxt = {
+                    'user': user,
                     'send_to': send_to,
                     'send_from': send_from,
                     'subject': subject,
@@ -383,6 +406,7 @@ def process(self):
 
 class TextOfEmail(models.Model):
     """ Contains the processed form of an EmailRequest, ready to be sent.  SmartText becomes plain text. """
+    user = AjaxForeignKey(ESPUser, blank=True, null=True) # blank=True because there isn't an easy way to backfill this
     send_to = models.CharField(max_length=1024)  # Valid email address, "Name" <[email protected]>
     send_from = models.CharField(max_length=1024) # Valid email address
     subject = models.TextField() # Email subject; plain text
@@ -430,7 +454,8 @@ def send(self):
                       self.send_from,
                       self.send_to,
                       False,
-                      extra_headers=extra_headers)
+                      extra_headers=extra_headers,
+                      user = self.user)
         except Exception as e:
             self.tries += 1
             self.save()

diff --git a/esp/esp/local_settings.py.travis b/esp/esp/local_settings.py.travis
@@ -48,6 +48,10 @@ SITE_INFO = (1, 'testserver.learningu.org', 'ESP Test Server')
 CACHE_PREFIX = "Test"
 SECRET_KEY = 'b36cfdfb78aba27ddbde330b23352bc6d40973b4'
 
+SERVER_EMAIL = '[email protected]'
+EMAIL_HOST = 'testserver.learningu.org'
+EMAIL_HOST_SENDER = 'testserver.learningu.org'
+
 ###################
 # Plugin settings #
 ###################

diff --git a/esp/esp/program/forms.py b/esp/esp/program/forms.py
@@ -36,6 +36,7 @@
 import re
 import unicodedata
 
+from django.conf import settings
 from esp.users.models import StudentInfo, K12School, RecordType
 from esp.program.models import Program, ProgramModule, ClassFlag, ClassFlagType, ClassCategories
 from esp.dbmail.models import PlainRedirect
@@ -150,9 +151,10 @@ class Meta:
             'program_modules': forms.SelectMultiple(attrs={'class': 'hidden-field'}),
         }
         model = Program
-ProgramCreationForm.base_fields['director_email'].widget = forms.TextInput(attrs={'size': 40})
-ProgramCreationForm.base_fields['director_cc_email'].widget = forms.TextInput(attrs={'size': 40})
-ProgramCreationForm.base_fields['director_confidential_email'].widget = forms.TextInput(attrs={'size': 40})
+ProgramCreationForm.base_fields['director_email'].widget = forms.EmailInput(attrs={'size': 40,
+                                                                                   'pattern': r'(^.+@%s$)|(^.+@(\w+\.)+learningu\.org$)' % settings.SITE_INFO[1].replace('.', '\.')})
+ProgramCreationForm.base_fields['director_cc_email'].widget = forms.EmailInput(attrs={'size': 40})
+ProgramCreationForm.base_fields['director_confidential_email'].widget = forms.EmailInput(attrs={'size': 40})
 '''
 ProgramCreationForm.base_fields['term'].line_group = -4
 ProgramCreationForm.base_fields['term_friendly'].line_group = -4

diff --git a/esp/esp/program/migrations/0027_auto_20240220_2124.py b/esp/esp/program/migrations/0027_auto_20240220_2124.py
@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2024-02-20 21:24
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.core import validators
+from django.db import migrations, models
+import django.db.models.fields
+import re
+
+def replace_director_emails(apps, schema_editor):
+    Program = apps.get_model('program', 'Program')
+    for prog in Program.objects.all():
+        if not re.match(r"(^.+@%s$)|(^.+@(\w+\.)*learningu\.org$)" % settings.SITE_INFO[1].replace(".", "\."), prog.director_email):
+            prog.director_email = 'info@' + settings.SITE_INFO[1]
+            prog.save()
+
+def create_info_redirect(apps, schema_editor):
+    PlainRedirect = apps.get_model('dbmail', 'PlainRedirect')
+    prs = PlainRedirect.objects.filter(original = "info")
+    if not prs.exists():
+        redirect = PlainRedirect.objects.create(original = "info", destination = settings.DEFAULT_EMAIL_ADDRESSES['default'])
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('program', '0026_auto_20221122_2100'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='program',
+            name='director_email',
+            field=models.EmailField(default=b'info@' + settings.SITE_INFO[1], max_length=75,
+                                    help_text=b'The director email address must end in @' + settings.SITE_INFO[1] +
+                                              ' (your website), @learningu.org, or a valid subdomain of learningu.org (i.e., @subdomain.learningu.org). The default is <b>info@' + settings.SITE_INFO[1] +
+                                              '</b>, which redirects to the "default" email address from your site\'s settings by default. You can create and manage your email redirects <a href="/manage/redirects/">here</a>.',
+                                    validators=[validators.RegexValidator(r'(^.+@%s$)|(^.+@(\w+\.)?learningu\.org$)' % settings.SITE_INFO[1].replace('.', '\.'))]),
+        ),
+        # This will run backwards, but won't do anything
+        migrations.RunPython(replace_director_emails, lambda a, s: None),
+        migrations.RunPython(create_info_redirect, lambda a, s: None),
+    ]
diff --git a/esp/esp/program/models/__init__.py b/esp/esp/program/models/__init__.py
@@ -249,7 +249,14 @@ class Program(models.Model, CustomFormsLinkModel):
     name = models.CharField(max_length=80)
     grade_min = models.IntegerField()
     grade_max = models.IntegerField()
-    director_email = models.EmailField(max_length=75) # director contact email address used for from field and display
+    # director contact email address used for from field and display
+    director_email = models.EmailField(default='info@' + settings.SITE_INFO[1], max_length=75,
+                                       validators=[validators.RegexValidator(r'(^.+@%s$)|(^.+@(\w+\.)?learningu\.org$)' % settings.SITE_INFO[1].replace('.', '\.'))],
+                                       help_text=mark_safe('The director email address must end in @' + settings.SITE_INFO[1] + ' (your website), ' +
+                                                           '@learningu.org, or a valid subdomain of learningu.org (i.e., @subdomain.learningu.org). ' +
+                                                           'The default is <b>info@' + settings.SITE_INFO[1] + '</b>, which redirects to the "default" ' +
+                                                           'email address from your site\'s settings by default. ' +
+                                                           'You can create and manage your email redirects <a href="/manage/redirects/">here</a>.'))
     director_cc_email = models.EmailField(blank=True, default='', max_length=75, help_text=mark_safe('If set, automated outgoing mail (except class cancellations) will be sent to this address <i>instead of</i> the director email. Use this if you do not want to spam the director email with teacher class registration emails. Otherwise, leave this field blank.')) # "carbon-copy" address for most automated outgoing mail to or CC'd to directors (except class cancellations)
     director_confidential_email = models.EmailField(blank=True, default='', max_length=75, help_text='If set, confidential emails such as financial aid applications will be sent to this address <i>instead of</i> the director email.')
     program_size_max = models.IntegerField(null=True, help_text='Set to 0 for no cap. Student registration performance is best when no cap is set.')

diff --git a/esp/esp/program/modules/forms/admincore.py b/esp/esp/program/modules/forms/admincore.py
@@ -1,5 +1,6 @@
 from decimal import Decimal
 from django import forms
+from django.conf import settings
 from django.contrib import admin
 from django.template import Template, Context
 from django.template.loader import select_template
@@ -93,6 +94,7 @@ class Meta:
             'program_modules': forms.SelectMultiple(attrs={'class': 'hidden-field'}),
         }
         model = Program
+ProgramSettingsForm.base_fields['director_email'].widget = forms.EmailInput(attrs={'pattern': r'(^.+@%s$)|(^.+@(\w+\.)?learningu\.org$)' % settings.SITE_INFO[1].replace('.', '\.')})
 
 class TeacherRegSettingsForm(BetterModelForm):
     """ Form for changing teacher class registration settings. """

diff --git a/esp/esp/program/modules/handlers/commmodule.py b/esp/esp/program/modules/handlers/commmodule.py
@@ -35,7 +35,7 @@
 from esp.program.modules.base import ProgramModuleObj, needs_admin, main_call, aux_call
 from esp.program.modules.handlers.listgenmodule import ListGenModule
 from esp.utils.web import render_to_response
-from esp.dbmail.models import MessageRequest
+from esp.dbmail.models import MessageRequest, PlainRedirect
 from esp.users.models   import ESPUser, PersistentQueryFilter
 from esp.users.controllers.usersearch import UserSearchController
 from esp.users.views.usersearch import get_user_checklist
@@ -45,6 +45,8 @@
 from esp.middleware import ESPError
 from django.template import loader
 
+import re
+
 class CommModule(ProgramModuleObj):
     doc = """Email users that match specific search criteria."""
     """ Want to email all ESP students within a 60 mile radius of NYC?
@@ -79,9 +81,17 @@ def commprev(self, request, tl, one, two, module, extra, prog):
         # Set From address
         if request.POST.get('from', '').strip():
             fromemail = request.POST['from']
+            if not re.match(r"(^.+@%s$)|(^.+@(\w+\.)?learningu\.org$)" % settings.SITE_INFO[1].replace(".", "\."), fromemail):
+                raise ESPError("Invalid 'From' email address. The 'From' email address must " +
+                               "end in @" + settings.SITE_INFO[1] + " (your website), " +
+                               "@learningu.org, or a valid subdomain of learningu.org " +
+                               "(i.e., @subdomain.learningu.org).")
         else:
-            # String together an address like [email protected]
-            fromemail = '%s@%s' % (request.user.username, settings.SITE_INFO[1])
+            # Use the info redirect (make one for the default email address if it doesn't exist)
+            prs = PlainRedirect.objects.filter(original = "info")
+            if not prs.exists():
+                redirect = PlainRedirect.objects.create(original = "info", destination = settings.DEFAULT_EMAIL_ADDRESSES['default'])
+            fromemail = '%s@%s' % ("info", settings.SITE_INFO[1])
 
         # Set Reply-To address
         if request.POST.get('replyto', '').strip():
@@ -241,6 +251,7 @@ def commpanel_old(self, request, tl, one, two, module, extra, prog):
     @main_call
     @needs_admin
     def commpanel(self, request, tl, one, two, module, extra, prog):
+        from django.conf import settings
 
         usc = UserSearchController()
 
@@ -267,6 +278,11 @@ def commpanel(self, request, tl, one, two, module, extra, prog):
                 context['sendto_fn_name'] = sendto_fn_name
                 context['listcount'] = self.approx_num_of_recipients(filterObj, sendto_fn)
                 context['selected'] = selected
+                # Use the info redirect (make one for the default email address if it doesn't exist)
+                prs = PlainRedirect.objects.filter(original = "info")
+                if not prs.exists():
+                    redirect = PlainRedirect.objects.create(original = "info", destination = settings.DEFAULT_EMAIL_ADDRESSES['default'])
+                context['from'] = '%s@%s' % ("info", settings.SITE_INFO[1])
                 return render_to_response(self.baseDir()+'step2.html', request, context)
 
             ##  Prepare a message starting from an earlier request

diff --git a/esp/esp/program/modules/handlers/teachereventsmodule.py b/esp/esp/program/modules/handlers/teachereventsmodule.py
@@ -41,6 +41,7 @@
 from esp.users.models import ESPUser, UserAvailability
 from esp.middleware.threadlocalrequest import get_current_request
 from django.contrib.auth.models import Group
+from django.conf import settings
 
 class TeacherEventsModule(ProgramModuleObj):
     doc = """Allows teachers to sign up for one or more teacher events (e.g. interviews, training)."""
@@ -132,8 +133,8 @@ def event_signup(self, request, tl, one, two, module, extra, prog):
                         send_mail('['+self.program.niceName()+'] Teacher Interview for ' + request.user.first_name + ' ' + request.user.last_name + ': ' + event_name, \
                               """Teacher Interview Registration Notification\n--------------------------------- \n\nTeacher: %s %s\n\nTime: %s\n\n""" % \
                               (request.user.first_name, request.user.last_name, event_name) , \
-                              (request.user.get_email_sendto_address()), \
-                              [self.program.getDirectorCCEmail()], True)
+                              '%s Registration System <server@%s>' % (self.program.program_type, settings.EMAIL_HOST_SENDER), \
+                              [self.program.getDirectorCCEmail()], True, extra_headers = {'Reply-To': request.user.get_email_sendto_address()})
 
                 # Register for training
                 if data['training']:

diff --git a/esp/esp/program/modules/tests/commpanel.py b/esp/esp/program/modules/tests/commpanel.py
@@ -90,8 +90,8 @@ def runTest(self):
         post_data = {
             'subject': 'Test Subject 123',
             'body':    'Test Body 123',
-            'from':    'from@email-server',
-            'replyto': 'replyto@email-server',
+            'from':    '[email protected]',
+            'replyto': 'replyto@testserver.learningu.org',
             'filterid': filterid,
         }
         response = self.client.post('/manage/%s/%s' % (self.program.getUrlBase(), 'commfinal'), post_data)
@@ -113,8 +113,8 @@ def runTest(self):
         msg = mail.outbox[0]
         self.assertEqual(msg.subject, 'Test Subject 123')
         self.assertEqual(msg.body, 'Test Body 123')
-        self.assertEqual(msg.from_email, 'from@email-server')
-        self.assertEqual(msg.extra_headers.get('Reply-To', ''), 'replyto@email-server')
+        self.assertEqual(msg.from_email, '[email protected]')
+        self.assertEqual(msg.extra_headers.get('Reply-To', ''), 'replyto@testserver.learningu.org')
 
         #   Check that the MessageRequest was marked as processed
         m = MessageRequest.objects.filter(recipients__id=filterid, subject='Test Subject 123')