Merge pull request mailgun#189 from mailgun/maxim/py3

One step closer to Python 3 support

build.sh

@@ -10,6 +10,8 @@ else
     nosetests --with-coverage --cover-package=flanker \
         tests/addresslib \
         tests/mime/bounce_tests.py \
+        tests/mime/message/fallback \
+        tests/mime/message/headers \
         tests/mime/message/threading_test.py \
         tests/mime/message/tokenizer_test.py \
         tests/mime/message/headers/encodedword_test.py \

flanker/mime/message/fallback/create.py

@@ -1,8 +1,14 @@
 import email
+
+import six
+
 from flanker.mime.message.fallback.part import FallbackMimePart
 
 
 def from_string(string):
+    if six.PY3 and isinstance(string, six.binary_type):
+        string = string.decode('utf-8')
+
     return FallbackMimePart(email.message_from_string(string))
 
 

flanker/mime/message/fallback/part.py

@@ -141,16 +141,16 @@ def add(self, key, value):
         MimeHeaders.add(self, key, value)
         self._m[key] = headers.to_mime(normalize(key), remove_newlines(value))
 
-    def transform(self, func):
+    def transform(self, fn, decode=False):
         changed = [False]
 
-        def wrapped_func(key, value):
-            new_key, new_value = func(key, value)
-            if new_value != value or new_key != key:
+        def wrapper(key, val):
+            new_key, new_value = fn(key, val)
+            if new_value != val or new_key != key:
                 changed[0] = True
             return new_key, new_value
 
-        transformed_headers = [wrapped_func(k, v) for k, v in self._m.items()]
+        transformed_headers = [wrapper(k, v) for k, v in self._m.items()]
         if changed[0]:
             self._m._headers = transformed_headers
             self._v = MultiDict([(normalize(k), remove_newlines(v))
@@ -161,15 +161,22 @@ def wrapped_func(key, value):
 def _try_decode(key, value):
     if isinstance(value, (tuple, list)):
         return value
-    elif isinstance(value, six.binary_type):
+
+    if six.PY3:
+        assert (isinstance(key, six.text_type) and
+                isinstance(value, six.text_type))
+        try:
+            return headers.parse_header_value(key, value)
+        except Exception:
+            return value
+
+    if isinstance(value, six.binary_type):
         try:
             return headers.parse_header_value(key, value)
         except Exception:
             return value.decode('utf-8', 'ignore')
 
-    elif isinstance(value, six.text_type):
+    if isinstance(value, six.text_type):
         return value
-    else:
-        return ""
-
 
+    raise TypeError('%s is not allowed type of header %s' % (type(value), key))

flanker/mime/message/headers/encoding.py

@@ -11,15 +11,20 @@
 
 _log = logging.getLogger(__name__)
 
+# The value of email.header.MAXLINELEN constant changed from 76 to 78 in
+# Python 3. To make sure that the library behaviour is consistent across all
+# Python versions we introduced our own constant.
+_MAX_LINE_LEN = 76
+
 _ADDRESS_HEADERS = ('From', 'To', 'Delivered-To', 'Cc', 'Bcc', 'Reply-To')
 
 
 def to_mime(key, value):
     if not value:
-        return ""
+        return ''
 
     if type(value) == list:
-        return "; ".join(encode(key, v) for v in value)
+        return '; '.join(encode(key, v) for v in value)
 
     return encode(key, value)
 
@@ -32,38 +37,40 @@ def encode(name, value):
 
         return _encode_unstructured(name, value)
     except Exception:
-        _log.exception("Failed to encode %s %s" % (name, value))
+        _log.exception('Failed to encode %s %s' % (name, value))
         raise
 
 
 def _encode_unstructured(name, value):
     try:
-        return Header(
-            value.encode("ascii"), "ascii",
-            header_name=name).encode(splitchars=' ;,')
+        header = Header(value.encode('ascii'), 'ascii', header_name=name)
+        return header.encode(splitchars=' ;,')
     except (UnicodeEncodeError, UnicodeDecodeError):
         if _is_address_header(name, value):
             return _encode_address_header(name, value)
 
-        return Header(
-            to_utf8(value), "utf-8",
-            header_name=name).encode(splitchars=' ;,')
+        header = Header(to_utf8(value), 'utf-8', header_name=name)
+        return header.encode(splitchars=' ;,')
 
 
 def _encode_address_header(name, value):
     out = deque()
     for addr in flanker.addresslib.address.parse_list(value):
         if addr.requires_non_ascii():
-            out.append(addr.to_unicode().encode('utf-8'))
+            encoded_addr = addr.to_unicode()
+            if six.PY2:
+                encoded_addr = encoded_addr.encode('utf-8')
         else:
-            out.append(addr.full_spec().encode('utf-8'))
+            encoded_addr = addr.full_spec()
+
+        out.append(encoded_addr)
     return '; '.join(out)
 
 
 def _encode_parametrized(key, value, params):
     if params:
         params = [_encode_param(key, n, v) for n, v in six.iteritems(params)]
-        return value + "; " + ("; ".join(params))
+        return value + '; ' + ('; '.join(params))
 
     return value
 
@@ -75,16 +82,16 @@ def _encode_param(key, name, value):
 
         return email.message._formatparam(name, value)
     except Exception:
-        value = Header(value.encode("utf-8"), "utf-8",  header_name=key).encode(splitchars=' ;,')
+        header = Header(value.encode('utf-8'), 'utf-8', header_name=key)
+        value = header.encode(splitchars=' ;,')
         return email.message._formatparam(name, value)
 
 
-def encode_string(name, value, maxlinelen=None):
+def encode_string(name, value, maxlinelen=_MAX_LINE_LEN):
     try:
-        header = Header(value.encode("ascii"), "ascii", maxlinelen,
-                        header_name=name)
+        header = Header(value.encode('ascii'), 'ascii', maxlinelen, name)
     except UnicodeEncodeError:
-        header = Header(value.encode("utf-8"), "utf-8", header_name=name)
+        header = Header(value.encode('utf-8'), 'utf-8', maxlinelen, name)
 
     return header.encode(splitchars=' ;,')
 

flanker/mime/message/headers/headers.py

@@ -71,16 +71,15 @@ def transform(self, fn, decode=False):
         a new pair of key, val and applies the function to all
         header, value pairs in the message.
         """
-
         changed = [False]
 
-        def tracking_fn(key, val):
+        def wrapper(key, val):
             new_key, new_val = fn(key, val)
             if new_val != val or new_key != key:
                 changed[0] = True
             return new_key, new_val
 
-        v = MultiDict(tracking_fn(key, val) for key, val in self.iteritems(raw=not decode))
+        v = MultiDict(wrapper(k, v) for k, v in self.iteritems(raw=not decode))
         if changed[0]:
             self._v = v
             self.changed = True

tests/__init__.py

@@ -1,8 +1,6 @@
 # coding:utf-8
 
-from os.path import join, abspath, dirname, exists
-from nose.tools import *
-import codecs
+from os.path import join, abspath, dirname
 
 
 def fixtures_path():
@@ -20,92 +18,88 @@ def skip_if_asked():
         raise SkipTest()
 
 
-def read_fixture_bytes(path):
+def read_fixture(path, binary=False):
     absolute_path = join(abspath(dirname(__file__)), 'fixtures', path)
-    with open(absolute_path, 'rb') as f:
+    mode = 'rb' if binary else 'r'
+    with open(absolute_path, mode) as f:
         return f.read()
 
 
 # mime fixture files
-BOUNCE = read_fixture_bytes('messages/bounce/zed.eml')
-BOUNCE_OFFICE365 = read_fixture_bytes('messages/bounce/office365.eml')
-MAILBOX_FULL = read_fixture_bytes('messages/bounce/mailbox-full.eml')
-NDN = read_fixture_bytes('messages/bounce/delayed.eml')
-NDN_BROKEN = read_fixture_bytes('messages/bounce/delayed-broken.eml')
-
-SIGNED = read_fixture_bytes('messages/signed.eml')
-LONG_LINKS = read_fixture_bytes('messages/long-links.eml')
-MULTI_RECEIVED_HEADERS = read_fixture_bytes(
+BOUNCE = read_fixture('messages/bounce/zed.eml')
+BOUNCE_OFFICE365 = read_fixture('messages/bounce/office365.eml')
+MAILBOX_FULL = read_fixture('messages/bounce/mailbox-full.eml')
+NDN = read_fixture('messages/bounce/delayed.eml')
+NDN_BROKEN = read_fixture('messages/bounce/delayed-broken.eml')
+
+SIGNED = read_fixture('messages/signed.eml')
+LONG_LINKS = read_fixture('messages/long-links.eml')
+MULTI_RECEIVED_HEADERS = read_fixture(
     'messages/multi-received-headers.eml')
-MAILGUN_PNG = read_fixture_bytes('messages/attachments/mailgun.png')
-MAILGUN_WAV = read_fixture_bytes('messages/attachments/mailgun-rocks.wav')
-
-TORTURE = read_fixture_bytes('messages/torture.eml')
-TORTURE_PART = read_fixture_bytes('messages/torture-part.eml')
-BILINGUAL = read_fixture_bytes('messages/bilingual-simple.eml')
-RELATIVE = read_fixture_bytes('messages/relative.eml')
-IPHONE = read_fixture_bytes('messages/iphone.eml')
-
-MULTIPART = read_fixture_bytes('messages/multipart.eml')
-FROM_ENCODING = read_fixture_bytes('messages/from-encoding.eml')
-NO_CTYPE = read_fixture_bytes('messages/no-ctype.eml')
-APACHE_MIME_MESSAGE_NEWS = read_fixture_bytes(
+MAILGUN_PNG = read_fixture('messages/attachments/mailgun.png', binary=True)
+MAILGUN_WAV = read_fixture('messages/attachments/mailgun-rocks.wav',
+                           binary=True)
+
+TORTURE = read_fixture('messages/torture.eml')
+TORTURE_PART = read_fixture('messages/torture-part.eml')
+BILINGUAL = read_fixture('messages/bilingual-simple.eml')
+RELATIVE = read_fixture('messages/relative.eml')
+IPHONE = read_fixture('messages/iphone.eml')
+
+MULTIPART = read_fixture('messages/multipart.eml')
+FROM_ENCODING = read_fixture('messages/from-encoding.eml', binary=True)
+NO_CTYPE = read_fixture('messages/no-ctype.eml')
+APACHE_MIME_MESSAGE_NEWS = read_fixture(
     'messages/apache-message-news-mime.eml')
-ENCLOSED = read_fixture_bytes('messages/enclosed.eml')
-ENCLOSED_BROKEN_BOUNDARY = read_fixture_bytes('messages/enclosed-broken.eml')
-ENCLOSED_ENDLESS = read_fixture_bytes('messages/enclosed-endless.eml')
-ENCLOSED_BROKEN_BODY = read_fixture_bytes('messages/enclosed-broken-body.eml')
-ENCLOSED_BROKEN_ENCODING = read_fixture_bytes(
-    'messages/enclosed-bad-encoding.eml')
-FALSE_MULTIPART = read_fixture_bytes('messages/false-multipart.eml')
-ENCODED_HEADER = read_fixture_bytes('messages/encoded-header.eml')
-MESSAGE_EXTERNAL_BODY= read_fixture_bytes(
+ENCLOSED = read_fixture('messages/enclosed.eml')
+ENCLOSED_BROKEN_BOUNDARY = read_fixture('messages/enclosed-broken.eml')
+ENCLOSED_ENDLESS = read_fixture('messages/enclosed-endless.eml')
+ENCLOSED_BROKEN_BODY = read_fixture('messages/enclosed-broken-body.eml')
+ENCLOSED_BROKEN_ENCODING = read_fixture(
+    'messages/enclosed-bad-encoding.eml', binary=True)
+FALSE_MULTIPART = read_fixture('messages/false-multipart.eml')
+ENCODED_HEADER = read_fixture('messages/encoded-header.eml')
+MESSAGE_EXTERNAL_BODY= read_fixture(
     'messages/message-external-body.eml')
-EIGHT_BIT = read_fixture_bytes('messages/8bitmime.eml')
-BIG = read_fixture_bytes('messages/big.eml')
-RUSSIAN_ATTACH_YAHOO = read_fixture_bytes(
-    'messages/russian-attachment-yahoo.eml')
-QUOTED_PRINTABLE = read_fixture_bytes('messages/quoted-printable.eml')
-TEXT_ONLY = read_fixture_bytes('messages/text-only.eml')
-MAILGUN_PIC = read_fixture_bytes('messages/mailgun-pic.eml')
-BZ2_ATTACHMENT  = read_fixture_bytes('messages/bz2-attachment.eml')
-OUTLOOK_EXPRESS = read_fixture_bytes('messages/outlook-express.eml')
-
-AOL_FBL = read_fixture_bytes('messages/complaints/aol.eml')
-YAHOO_FBL = read_fixture_bytes('messages/complaints/yahoo.eml')
-NOTIFICATION = read_fixture_bytes('messages/bounce/no-mx.eml')
-DASHED_BOUNDARIES = read_fixture_bytes('messages/dashed-boundaries.eml')
-WEIRD_BOUNCE = read_fixture_bytes('messages/bounce/gmail-no-dns.eml')
-WEIRD_BOUNCE_2 = read_fixture_bytes(
+EIGHT_BIT = read_fixture('messages/8bitmime.eml')
+BIG = read_fixture('messages/big.eml')
+RUSSIAN_ATTACH_YAHOO = read_fixture(
+    'messages/russian-attachment-yahoo.eml', binary=True)
+QUOTED_PRINTABLE = read_fixture('messages/quoted-printable.eml')
+TEXT_ONLY = read_fixture('messages/text-only.eml')
+MAILGUN_PIC = read_fixture('messages/mailgun-pic.eml')
+BZ2_ATTACHMENT  = read_fixture('messages/bz2-attachment.eml')
+OUTLOOK_EXPRESS = read_fixture('messages/outlook-express.eml')
+
+AOL_FBL = read_fixture('messages/complaints/aol.eml')
+YAHOO_FBL = read_fixture('messages/complaints/yahoo.eml')
+NOTIFICATION = read_fixture('messages/bounce/no-mx.eml')
+DASHED_BOUNDARIES = read_fixture('messages/dashed-boundaries.eml')
+WEIRD_BOUNCE = read_fixture('messages/bounce/gmail-no-dns.eml')
+WEIRD_BOUNCE_2 = read_fixture(
     'messages/bounce/gmail-invalid-address.eml')
 
-WEIRD_BOUNCE_3 = read_fixture_bytes('messages/bounce/broken-mime.eml')
-MISSING_BOUNDARIES = read_fixture_bytes('messages/missing-boundaries.eml')
-MISSING_FINAL_BOUNDARY = read_fixture_bytes(
+WEIRD_BOUNCE_3 = read_fixture('messages/bounce/broken-mime.eml')
+MISSING_BOUNDARIES = read_fixture('messages/missing-boundaries.eml')
+MISSING_FINAL_BOUNDARY = read_fixture(
     'messages/missing-final-boundary.eml')
-DISPOSITION_NOTIFICATION = read_fixture_bytes(
+DISPOSITION_NOTIFICATION = read_fixture(
     'messages/disposition-notification.eml')
-MAILFORMED_HEADERS = read_fixture_bytes('messages/mailformed-headers.eml')
-
-SPAM_BROKEN_HEADERS = read_fixture_bytes('messages/spam/broken-headers.eml')
-SPAM_BROKEN_CTYPE = read_fixture_bytes('messages/spam/broken-ctype.eml')
-LONG_HEADER = read_fixture_bytes('messages/long-header.eml')
-ATTACHED_PDF = read_fixture_bytes('messages/attached-pdf.eml')
+MAILFORMED_HEADERS = read_fixture(
+    'messages/mailformed-headers.eml', binary=True)
+SPAM_BROKEN_HEADERS = read_fixture(
+    'messages/spam/broken-headers.eml', binary=True)
+SPAM_BROKEN_CTYPE = read_fixture('messages/spam/broken-ctype.eml')
+LONG_HEADER = read_fixture('messages/long-header.eml')
+ATTACHED_PDF = read_fixture('messages/attached-pdf.eml')
 
 # addresslib fixture files
-MAILBOX_VALID_TESTS = read_fixture_bytes(
-    'mailbox_valid.txt').decode('utf-8')
-MAILBOX_INVALID_TESTS = read_fixture_bytes(
-    'mailbox_invalid.txt').decode('utf-8')
-ABRIDGED_LOCALPART_VALID_TESTS = read_fixture_bytes(
-    'abridged_localpart_valid.txt').decode('utf-8')
-ABRIDGED_LOCALPART_INVALID_TESTS = read_fixture_bytes(
-    'abridged_localpart_invalid.txt').decode('utf-8')
-URL_VALID_TESTS = read_fixture_bytes(
-    'url_valid.txt').decode('utf-8')
-URL_INVALID_TESTS = read_fixture_bytes(
-    'url_invalid.txt').decode('utf-8')
-DOMAIN_TYPO_VALID_TESTS = read_fixture_bytes(
-    'domain_typos_valid.txt').decode('utf-8')
-DOMAIN_TYPO_INVALID_TESTS = read_fixture_bytes(
-    'domain_typos_invalid.txt').decode('utf-8')
+MAILBOX_VALID_TESTS = read_fixture('mailbox_valid.txt')
+MAILBOX_INVALID_TESTS = read_fixture('mailbox_invalid.txt')
+ABRIDGED_LOCALPART_VALID_TESTS = read_fixture('abridged_localpart_valid.txt')
+ABRIDGED_LOCALPART_INVALID_TESTS = read_fixture(
+    'abridged_localpart_invalid.txt')
+URL_VALID_TESTS = read_fixture('url_valid.txt')
+URL_INVALID_TESTS = read_fixture('url_invalid.txt')
+DOMAIN_TYPO_VALID_TESTS = read_fixture('domain_typos_valid.txt')
+DOMAIN_TYPO_INVALID_TESTS = read_fixture('domain_typos_invalid.txt')