Merge pull request mailgun#192 from mailgun/maxim/py3

Create adaptor module _email for the standard email package

flanker/_email.py

@@ -0,0 +1,98 @@
+import email
+from contextlib import closing
+from email.generator import Generator
+from email.header import Header
+from email.mime import audio
+from email.utils import make_msgid
+
+import six
+from six.moves import StringIO
+
+_CRLF = '\r\n'
+_SPLIT_CHARS = ' ;,'
+
+# The value of email.header.MAXLINELEN constant changed from 76 to 78 in
+# Python 3. To make sure that the library behaviour is consistent across all
+# Python versions we introduced our own constant.
+_MAX_LINE_LEN = 76
+
+
+if six.PY3:
+    from email.policy import Compat32
+
+    class _Compat32CRLF(Compat32):
+        linesep = _CRLF
+
+    _compat32_crlf = _Compat32CRLF()
+
+else:
+    from email import generator, header, quoprimime, feedparser
+    generator.NL = _CRLF
+    header.NL = _CRLF
+    quoprimime.NL = _CRLF
+    feedparser.NL = _CRLF
+
+
+def message_from_string(string):
+    if six.PY3:
+        if isinstance(string, six.binary_type):
+            return email.message_from_bytes(string, policy=_compat32_crlf)
+
+        return email.message_from_string(string, policy=_compat32_crlf)
+
+    return email.message_from_string(string)
+
+
+def message_to_string(msg):
+    """
+    Converts python message to string in a proper way.
+    """
+    with closing(StringIO()) as fp:
+        if six.PY3:
+            g = Generator(fp, mangle_from_=False, policy=_compat32_crlf)
+            g.flatten(msg, unixfrom=False)
+            return fp.getvalue()
+
+        g = Generator(fp, mangle_from_=False)
+        g.flatten(msg, unixfrom=False)
+
+        # In Python 2 Generator.flatten uses `print >> ` to write to fp, that
+        # adds `\n` regardless of generator.NL value. So we resort to a hackish
+        # way of replacing LF with RFC complaint CRLF.
+        for i, v in enumerate(fp.buflist):
+            if v == '\n':
+                fp.buflist[i] = _CRLF
+
+        return fp.getvalue()
+
+
+def format_param(name, val):
+    return email.message._formatparam(name, val)
+
+
+def decode_base64(val):
+    return email.base64mime.decode(val)
+
+
+def encode_base64(val):
+    return email.encoders._bencode(val)
+
+
+def decode_quoted_printable(val):
+    return email.quoprimime.header_decode(val)
+
+
+def detect_audio_type(val):
+    return audio._whatsnd(val)
+
+
+def make_message_id():
+    return make_msgid()
+
+
+def encode_header(name, val, encoding='ascii', max_line_len=_MAX_LINE_LEN):
+    header = Header(val, encoding, max_line_len, name)
+    if six.PY3:
+        return header.encode(_SPLIT_CHARS, linesep=_CRLF)
+
+    return header.encode(_SPLIT_CHARS)

flanker/addresslib/address.py

@@ -34,16 +34,17 @@
 See the parser.py module for implementation details of the parser.
 """
 from logging import getLogger
-from time import time
 
 import idna
 import six
 from idna import IDNAError
 from ply.lex import LexError
 from ply.yacc import YaccError
 from six.moves.urllib_parse import urlparse
+from time import time
 from tld import get_tld
 
+from flanker import _email
 from flanker.addresslib._parser.lexer import lexer
 from flanker.addresslib._parser.parser import (Mailbox, Url, mailbox_parser,
                                                mailbox_or_url_parser,
@@ -53,7 +54,6 @@
 from flanker.addresslib.validate import (mail_exchanger_lookup,
                                          plugin_for_esp)
 from flanker.mime.message.headers.encodedword import mime_to_unicode
-from flanker.mime.message.headers.encoding import encode_string
 from flanker.utils import is_pure_ascii, metrics_wrapper
 
 _log = getLogger(__name__)
@@ -503,8 +503,10 @@ def display_name(self):
 
     @property
     def ace_display_name(self):
-        return _to_str(encode_string(None, smart_quote(self._display_name),
-                                     maxlinelen=MAX_ADDRESS_LENGTH))
+        quoted_display_name = smart_quote(self._display_name)
+        encoded_display_name = _email.encode_header(None, quoted_display_name,
+                                                    'ascii', MAX_ADDRESS_LENGTH)
+        return _to_str(encoded_display_name)
 
     @property
     def mailbox(self):

flanker/mime/__init__.py

@@ -63,5 +63,4 @@
 from flanker.mime import create
 from flanker.mime.create import from_string
 from flanker.mime.message.fallback.create import from_string as recover
-from flanker.mime.message.utils import python_message_to_string
 from flanker.mime.message.headers.parametrized import fix_content_type

flanker/mime/create.py

@@ -1,12 +1,15 @@
-""" This package is a set of utilities and methods for building mime messages """
+"""
+This package is a set of utilities and methods for building mime messages.
+"""
 
 import uuid
+
+from flanker import _email
 from flanker.mime import DecodingError
-from flanker.mime.message import ContentType, utils
-from flanker.mime.message.part import MimePart, Body, Part, adjust_content_type
-from flanker.mime.message import scanner
-from flanker.mime.message.headers.parametrized import fix_content_type
+from flanker.mime.message import ContentType, scanner
 from flanker.mime.message.headers import WithParams
+from flanker.mime.message.headers.parametrized import fix_content_type
+from flanker.mime.message.part import MimePart, Body, Part, adjust_content_type
 
 
 def multipart(subtype):
@@ -84,8 +87,7 @@ def from_string(string):
 
 
 def from_python(message):
-    return from_string(
-        utils.python_message_to_string(message))
+    return from_string(_email.message_to_string(message))
 
 
 def from_message(message):

flanker/mime/message/fallback/create.py

@@ -1,15 +1,9 @@
-import email
-
-import six
-
+from flanker import _email
 from flanker.mime.message.fallback.part import FallbackMimePart
 
 
 def from_string(string):
-    if six.PY3 and isinstance(string, six.binary_type):
-        string = string.decode('utf-8')
-
-    return FallbackMimePart(email.message_from_string(string))
+    return FallbackMimePart(_email.message_from_string(string))
 
 
 def from_python(message):

flanker/mime/message/fallback/part.py

@@ -1,14 +1,15 @@
 import logging
-import email
 
 import six
 from webob.multidict import MultiDict
+
+from flanker import _email
+from flanker.mime.message import headers
 from flanker.mime.message.charsets import convert_to_unicode
+from flanker.mime.message.headers import parametrized, normalize
 from flanker.mime.message.headers.headers import remove_newlines, MimeHeaders
 from flanker.mime.message.part import RichPartMixin
 from flanker.mime.message.scanner import ContentType
-from flanker.mime.message import utils, headers
-from flanker.mime.message.headers import parametrized, normalize
 
 log = logging.getLogger(__name__)
 
@@ -85,7 +86,7 @@ def charset(self, value):
         pass  # FIXME Not implement
 
     def to_string(self):
-        return utils.python_message_to_string(self._m)
+        return _email.message_to_string(self._m)
 
     def to_stream(self, out):
         out.write(self.to_string())
@@ -98,7 +99,7 @@ def to_python_message(self):
 
     def append(self, *messages):
         for m in messages:
-            part = FallbackMimePart(email.message_from_string(m.to_string()))
+            part = FallbackMimePart(_email.message_from_string(m.to_string()))
             self._m.attach(part)
 
     @property

flanker/mime/message/headers/encodedword.py

@@ -1,12 +1,11 @@
 # coding:utf-8
-import email.base64mime
-import email.quoprimime
 import logging
 from base64 import b64encode
 
 import regex as re
 import six
 
+from flanker import _email
 from flanker.mime.message import charsets, errors
 
 _log = logging.getLogger(__name__)
@@ -113,7 +112,7 @@ def _decode_part(charset, encoding, value):
         if paderr:
             value += '==='[:4 - paderr]
 
-        return charset, email.base64mime.decode(value)
+        return charset, _email.decode_base64(value)
 
     if not encoding:
         return charset, value
@@ -123,7 +122,7 @@ def _decode_part(charset, encoding, value):
 
 def _decode_quoted_printable(qp):
     if six.PY2:
-        return email.quoprimime.header_decode(str(qp))
+        return _email.decode_quoted_printable(str(qp))
 
     buf = bytearray()
     size = len(qp)

flanker/mime/message/headers/encoding.py

@@ -1,21 +1,15 @@
-import email.message
 import logging
 from collections import deque
-from email.header import Header
 
 import six
 
 import flanker.addresslib.address
+from flanker import _email
 from flanker.mime.message.headers import parametrized
 from flanker.mime.message.utils import to_utf8
 
 _log = logging.getLogger(__name__)
 
-# The value of email.header.MAXLINELEN constant changed from 76 to 78 in
-# Python 3. To make sure that the library behaviour is consistent across all
-# Python versions we introduced our own constant.
-_MAX_LINE_LEN = 76
-
 _ADDRESS_HEADERS = ('From', 'To', 'Delivered-To', 'Cc', 'Bcc', 'Reply-To')
 
 
@@ -43,14 +37,12 @@ def encode(name, value):
 
 def _encode_unstructured(name, value):
     try:
-        header = Header(value.encode('ascii'), 'ascii', header_name=name)
-        return header.encode(splitchars=' ;,')
+        return _email.encode_header(name, value.encode('ascii'), 'ascii')
     except (UnicodeEncodeError, UnicodeDecodeError):
         if _is_address_header(name, value):
             return _encode_address_header(name, value)
 
-        header = Header(to_utf8(value), 'utf-8', header_name=name)
-        return header.encode(splitchars=' ;,')
+        return _email.encode_header(name, to_utf8(value), 'utf-8')
 
 
 def _encode_address_header(name, value):
@@ -80,20 +72,11 @@ def _encode_param(key, name, value):
         if six.PY2:
             value = value.encode('ascii')
 
-        return email.message._formatparam(name, value)
+        return _email.format_param(name, value)
     except Exception:
-        header = Header(value.encode('utf-8'), 'utf-8', header_name=key)
-        value = header.encode(splitchars=' ;,')
-        return email.message._formatparam(name, value)
-
-
-def encode_string(name, value, maxlinelen=_MAX_LINE_LEN):
-    try:
-        header = Header(value.encode('ascii'), 'ascii', maxlinelen, name)
-    except UnicodeEncodeError:
-        header = Header(value.encode('utf-8'), 'utf-8', maxlinelen, name)
-
-    return header.encode(splitchars=' ;,')
+        value = value.encode('utf-8')
+        encoded_param = _email.encode_header(key, value, 'utf-8')
+        return _email.format_param(name, encoded_param)
 
 
 def _is_address_header(key, val):

flanker/mime/message/headers/wrappers.py

@@ -1,13 +1,11 @@
 """ Useful wrappers for headers with parameters,
 provide some convenience access methods
 """
-
-from email.utils import make_msgid
-
 import regex as re
 import six
 
 import flanker.addresslib.address
+from flanker import _email
 
 
 class WithParams(tuple):
@@ -164,7 +162,7 @@ def from_string(cls, string):
 
     @classmethod
     def generate(cls, domain=None):
-        message_id = make_msgid().strip("<>")
+        message_id = _email.make_message_id().strip("<>")
         if domain:
             local = message_id.split('@')[0]
             message_id = "{0}@{1}".format(local, domain)

flanker/mime/message/part.py

@@ -1,16 +1,14 @@
 import base64
-import email.encoders
 import imghdr
 import logging
 import mimetypes
 import quopri
 from contextlib import closing
-from email.mime import audio
 from os import path
 
 import six
 
-from flanker import metrics
+from flanker import metrics, _email
 from flanker.mime import bounce
 from flanker.mime.message import headers, charsets
 from flanker.mime.message.errors import EncodingError, DecodingError
@@ -114,7 +112,7 @@ def adjust_content_type(content_type, body=None, filename=None):
             content_type = ContentType('image', sub)
 
     elif content_type.main == 'audio' and body:
-        sub = audio._whatsnd(body)
+        sub = _email.detect_audio_type(body)
         if sub:
             content_type = ContentType('audio', sub)
 
@@ -509,7 +507,7 @@ def was_changed(self, ignore_prepends=False):
             return self.enclosed.was_changed()
 
     def to_python_message(self):
-        return email.message_from_string(self.to_string())
+        return _email.message_from_string(self.to_string())
 
     def append(self, *messages):
         for m in messages:
@@ -628,7 +626,7 @@ def encode_transfer_encoding(encoding, body):
     if encoding == 'quoted-printable':
         return quopri.encodestring(body, quotetabs=False)
     elif encoding == 'base64':
-        return email.encoders._bencode(body)
+        return _email.encode_base64(body)
     else:
         return body