Merge pull request #106 from mailgun/sergey/html5lib

use html5lib to parse html

MANIFEST.in

@@ -1,9 +1,7 @@
-recursive-include tests *
-recursive-include talon *
 recursive-exclude tests *.pyc *~
 recursive-exclude talon *.pyc *~
 include train.data
 include classifier
 include LICENSE
 include MANIFEST.in
-include README.rst
+include README.rst

setup.py

@@ -1,8 +1,35 @@
+from __future__ import absolute_import
 from setuptools import setup, find_packages
+from setuptools.command.install import install
+
+
+class InstallCommand(install):
+    user_options = install.user_options + [
+        ('no-ml', None, "Don't install without Machine Learning modules."),
+    ]
+
+    boolean_options = install.boolean_options + ['no-ml']
+
+    def initialize_options(self):
+        install.initialize_options(self)
+        self.no_ml = None
+
+    def finalize_options(self):
+        install.finalize_options(self)
+        if self.no_ml:
+            dist = self.distribution
+            dist.packages=find_packages(exclude=[
+                'tests',
+                'tests.*',
+                'talon.signature',
+                'talon.signature.*',
+            ])
+            for not_required in ['numpy', 'scipy', 'scikit-learn==0.16.1']:
+                dist.install_requires.remove(not_required)
 
 
 setup(name='talon',
-      version='1.2.10',
+      version='1.3.0',
       description=("Mailgun library "
                    "to extract message quotations and signatures."),
       long_description=open("README.rst").read(),
@@ -10,7 +37,10 @@ setup(name='talon',
       author_email='admin@mailgunhq.com',
       url='https://github.com/mailgun/talon',
       license='APACHE2',
-      packages=find_packages(exclude=['tests']),
+      cmdclass={
+          'install': InstallCommand,
+      },
+      packages=find_packages(exclude=['tests', 'tests.*']),
       include_package_data=True,
       zip_safe=True,
       install_requires=[
@@ -21,7 +51,9 @@ setup(name='talon',
           "scikit-learn==0.16.1", # pickled versions of classifier, else rebuild
           'chardet>=1.0.1',
           'cchardet>=0.3.5',
-          'cssselect'
+          'cssselect',
+          'six>=1.10.0',
+          'html5lib'
           ],
       tests_require=[
           "mock",

talon/__init__.py

@@ -1,7 +1,13 @@
+from __future__ import absolute_import
 from talon.quotations import register_xpath_extensions
-from talon import signature
+try:
+    from talon import signature
+    ML_ENABLED = True
+except ImportError:
+    ML_ENABLED = False
 
 
 def init():
     register_xpath_extensions()
-    signature.initialize()
+    if ML_ENABLED:
+        signature.initialize()

talon/constants.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import regex as re
 
 

talon/html_quotations.py

@@ -3,8 +3,10 @@ The module's functions operate on message bodies trying to extract original
 messages (without quoted messages) from html
 """
 
+from __future__ import absolute_import
 import regex as re
 
+from talon.utils import cssselect 
 
 CHECKPOINT_PREFIX = '#!%!'
 CHECKPOINT_SUFFIX = '!%!#'
@@ -77,7 +79,7 @@ def delete_quotation_tags(html_note, counter, quotation_checkpoints):
 
 def cut_gmail_quote(html_message):
     ''' Cuts the outermost block element with class gmail_quote. '''
-    gmail_quote = html_message.cssselect('div.gmail_quote')
+    gmail_quote = cssselect('div.gmail_quote', html_message)
     if gmail_quote and (gmail_quote[0].text is None or not RE_FWD.match(gmail_quote[0].text)):
         gmail_quote[0].getparent().remove(gmail_quote[0])
         return True
@@ -134,7 +136,7 @@ def cut_microsoft_quote(html_message):
 def cut_by_id(html_message):
     found = False
     for quote_id in QUOTE_IDS:
-        quote = html_message.cssselect('#{}'.format(quote_id))
+        quote = cssselect('#{}'.format(quote_id), html_message)
         if quote:
             found = True
             quote[0].getparent().remove(quote[0])

talon/quotations.py

@@ -5,14 +5,18 @@ The module's functions operate on message bodies trying to extract
 original messages (without quoted messages)
 """
 
+from __future__ import absolute_import
 import regex as re
 import logging
 from copy import deepcopy
 
 from lxml import html, etree
 
-from talon.utils import get_delimiter, html_to_text
+from talon.utils import (get_delimiter, html_tree_to_text,
+                         html_document_fromstring)
 from talon import html_quotations
+from six.moves import range
+import six
 
 
 log = logging.getLogger(__name__)
@@ -161,6 +165,9 @@ RE_PARENTHESIS_LINK = re.compile("\(https?://")
 
 SPLITTER_MAX_LINES = 4
 MAX_LINES_COUNT = 1000
+# an extensive research shows that exceeding this limit
+# leads to excessive processing time
+MAX_HTML_LEN = 2794202
 
 QUOT_PATTERN = re.compile('^>+ ?')
 NO_QUOT_LINE = re.compile('^[^>].*[\S].*')
@@ -191,7 +198,7 @@ def mark_message_lines(lines):
     >>> mark_message_lines(['answer', 'From: foo@bar.com', '', '> question'])
     'tsem'
     """
-    markers = bytearray(len(lines))
+    markers = ['e' for _ in lines]
     i = 0
     while i < len(lines):
         if not lines[i].strip():
@@ -207,7 +214,7 @@ def mark_message_lines(lines):
             if splitter:
                 # append as many splitter markers as lines in splitter
                 splitter_lines = splitter.group().splitlines()
-                for j in xrange(len(splitter_lines)):
+                for j in range(len(splitter_lines)):
                     markers[i + j] = 's'
 
                 # skip splitter lines
@@ -217,7 +224,7 @@ def mark_message_lines(lines):
                 markers[i] = 't'
         i += 1
 
-    return markers
+    return ''.join(markers)
 
 
 def process_marked_lines(lines, markers, return_flags=[False, -1, -1]):
@@ -231,6 +238,7 @@ def process_marked_lines(lines, markers, return_flags=[False, -1, -1]):
     return_flags = [were_lines_deleted, first_deleted_line,
                     last_deleted_line]
     """
+    markers = ''.join(markers)
     # if there are no splitter there should be no markers
     if 's' not in markers and not re.search('(me*){3}', markers):
         markers = markers.replace('m', 't')
@@ -276,10 +284,15 @@ def preprocess(msg_body, delimiter, content_type='text/plain'):
     Replaces link brackets so that they couldn't be taken for quotation marker.
     Splits line in two if splitter pattern preceded by some text on the same
     line (done only for 'On <date> <person> wrote:' pattern).
+
+    Converts msg_body into a unicode.
     """
     # normalize links i.e. replace '<', '>' wrapping the link with some symbols
     # so that '>' closing the link couldn't be mistakenly taken for quotation
     # marker.
+    if isinstance(msg_body, bytes):
+        msg_body = msg_body.decode('utf8')
+
     def link_wrapper(link):
         newline_index = msg_body[:link.start()].rfind("\n")
         if msg_body[newline_index + 1] == ">":
@@ -342,15 +355,49 @@ def extract_from_html(msg_body):
     then extracting quotations from text,
     then checking deleted checkpoints,
     then deleting necessary tags.
+
+    Returns a unicode string.
     """
-    if msg_body.strip() == '':
+    if isinstance(msg_body, six.text_type):
+        msg_body = msg_body.encode('utf8')
+    elif not isinstance(msg_body, bytes):
+        msg_body = msg_body.encode('ascii')
+
+    result = _extract_from_html(msg_body)
+    if isinstance(result, bytes):
+        result = result.decode('utf8')
+
+    return result
+
+
+def _extract_from_html(msg_body):
+    """
+    Extract not quoted message from provided html message body
+    using tags and plain text algorithm.
+
+    Cut out the 'blockquote', 'gmail_quote' tags.
+    Cut Microsoft quotations.
+
+    Then use plain text algorithm to cut out splitter or
+    leftover quotation.
+    This works by adding checkpoint text to all html tags,
+    then converting html to text,
+    then extracting quotations from text,
+    then checking deleted checkpoints,
+    then deleting necessary tags.
+    """
+    if len(msg_body) > MAX_HTML_LEN:
+        return msg_body
+
+    if msg_body.strip() == b'':
+        return msg_body
+
+    msg_body = msg_body.replace(b'\r\n', b'\n')
+    html_tree = html_document_fromstring(msg_body)
+
+    if html_tree is None:
         return msg_body
 
-    msg_body = msg_body.replace('\r\n', '').replace('\n', '')
-    html_tree = html.document_fromstring(
-        msg_body,
-        parser=html.HTMLParser(encoding="utf-8")
-    )
     cut_quotations = (html_quotations.cut_gmail_quote(html_tree) or
                       html_quotations.cut_zimbra_quote(html_tree) or
                       html_quotations.cut_blockquote(html_tree) or
@@ -362,8 +409,7 @@ def extract_from_html(msg_body):
 
     number_of_checkpoints = html_quotations.add_checkpoint(html_tree, 0)
     quotation_checkpoints = [False] * number_of_checkpoints
-    msg_with_checkpoints = html.tostring(html_tree)
-    plain_text = html_to_text(msg_with_checkpoints)
+    plain_text = html_tree_to_text(html_tree)
     plain_text = preprocess(plain_text, '\n', content_type='text/html')
     lines = plain_text.splitlines()
 
@@ -386,25 +432,31 @@ def extract_from_html(msg_body):
     return_flags = []
     process_marked_lines(lines, markers, return_flags)
     lines_were_deleted, first_deleted, last_deleted = return_flags
+
+    if not lines_were_deleted and not cut_quotations:
+        return msg_body
+
     if lines_were_deleted:
         #collect checkpoints from deleted lines
-        for i in xrange(first_deleted, last_deleted):
+        for i in range(first_deleted, last_deleted):
             for checkpoint in line_checkpoints[i]:
                 quotation_checkpoints[checkpoint] = True
-    else:
-        if cut_quotations:
-            return html.tostring(html_tree_copy)
-        else:
-            return msg_body
 
-    # Remove tags with quotation checkpoints
-    html_quotations.delete_quotation_tags(
-        html_tree_copy, 0, quotation_checkpoints
-    )
+        # Remove tags with quotation checkpoints
+        html_quotations.delete_quotation_tags(
+            html_tree_copy, 0, quotation_checkpoints
+        )
+
+    if _readable_text_empty(html_tree_copy):
+        return msg_body
 
     return html.tostring(html_tree_copy)
 
 
+def _readable_text_empty(html_tree):
+    return not bool(html_tree_to_text(html_tree).strip())
+
+
 def is_splitter(line):
     '''
     Returns Matcher object if provided string is a splitter and
@@ -418,7 +470,7 @@ def is_splitter(line):
 
 def text_content(context):
     '''XPath Extension function to return a node text content.'''
-    return context.context_node.text_content().strip()
+    return context.context_node.xpath("string()").strip()
 
 
 def tail(context):

talon/signature/__init__.py

@@ -20,6 +20,7 @@ trained against, don't forget to regenerate:
 * signature/data/classifier
 """
 
+from __future__ import absolute_import
 import os
 
 from . import extraction

talon/signature/bruteforce.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import logging
 
 import regex as re
@@ -111,7 +112,7 @@ def extract_signature(msg_body):
 
             return (stripped_body.strip(),
                     signature.strip())
-    except Exception, e:
+    except Exception as e:
         log.exception('ERROR extracting signature')
         return (msg_body, None)
 

talon/signature/extraction.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
 import logging
 
 import regex as re

talon/signature/learning/classifier.py

@@ -5,6 +5,7 @@ The classifier could be used to detect if a certain line of the message
 body belongs to the signature.
 """
 
+from __future__ import absolute_import
 from numpy import genfromtxt
 from sklearn.svm import LinearSVC
 from sklearn.externals import joblib

talon/signature/learning/dataset.py

@@ -16,11 +16,13 @@ suffix and the corresponding sender file has the same name except for the
 suffix which should be `_sender`.
 """
 
+from __future__ import absolute_import
 import os
 import regex as re
 
 from talon.signature.constants import SIGNATURE_MAX_LINES
 from talon.signature.learning.featurespace import build_pattern, features
+from six.moves import range
 
 
 SENDER_SUFFIX = '_sender'
@@ -144,7 +146,7 @@ def build_extraction_dataset(folder, dataset_filename,
             if not sender or not msg:
                 continue
             lines = msg.splitlines()
-            for i in xrange(1, min(SIGNATURE_MAX_LINES,
+            for i in range(1, min(SIGNATURE_MAX_LINES,
                                    len(lines)) + 1):
                 line = lines[-i]
                 label = -1

talon/signature/learning/featurespace.py

@@ -7,9 +7,12 @@ The body and the message sender string are converted into unicode before
 applying features to them.
 """
 
+from __future__ import absolute_import
 from talon.signature.constants import (SIGNATURE_MAX_LINES,
                                        TOO_LONG_SIGNATURE_LINE)
 from talon.signature.learning.helpers import *
+from six.moves import zip
+from functools import reduce
 
 
 def features(sender=''):

talon/signature/learning/helpers.py

@@ -6,6 +6,7 @@
 
 """
 
+from __future__ import absolute_import
 import unicodedata
 import regex as re
 
@@ -184,12 +185,13 @@ def capitalized_words_percent(s):
     s = to_unicode(s, precise=True)
     words = re.split('\s', s)
     words = [w for w in words if w.strip()]
+    words = [w for w in words if len(w) > 2]    
     capitalized_words_counter = 0
     valid_words_counter = 0
     for word in words:
         if not INVALID_WORD_START.match(word):
             valid_words_counter += 1
-            if word[0].isupper():
+            if word[0].isupper() and not word[1].isupper():
                 capitalized_words_counter += 1
     if valid_words_counter > 0 and len(words) > 1:
         return 100 * float(capitalized_words_counter) / valid_words_counter

talon/utils.py

@@ -1,15 +1,19 @@
 # coding:utf-8
 
+from __future__ import absolute_import
 import logging
 from random import shuffle
 import chardet
 import cchardet
 import regex as re
 
-from lxml import html
+from lxml.html import html5parser
 from lxml.cssselect import CSSSelector
 
+import html5lib
+
 from talon.constants import RE_DELIMITER
+import six
 
 
 def safe_format(format_string, *args, **kwargs):
@@ -28,7 +32,7 @@ def safe_format(format_string, *args, **kwargs):
     except (UnicodeEncodeError, UnicodeDecodeError):
         format_string = to_utf8(format_string)
         args = [to_utf8(p) for p in args]
-        kwargs = {k: to_utf8(v) for k, v in kwargs.iteritems()}
+        kwargs = {k: to_utf8(v) for k, v in six.iteritems(kwargs)}
         return format_string.format(*args, **kwargs)
 
     # ignore other errors
@@ -45,9 +49,9 @@ def to_unicode(str_or_unicode, precise=False):
         u'привет'
     If `precise` flag is True, tries to guess the correct encoding first.
     """
-    encoding = quick_detect_encoding(str_or_unicode) if precise else 'utf-8'
-    if isinstance(str_or_unicode, str):
-        return unicode(str_or_unicode, encoding, 'replace')
+    if not isinstance(str_or_unicode, six.text_type):
+        encoding = quick_detect_encoding(str_or_unicode) if precise else 'utf-8'
+        return six.text_type(str_or_unicode, encoding, 'replace')
     return str_or_unicode
 
 
@@ -57,11 +61,12 @@ def detect_encoding(string):
 
     Defaults to UTF-8.
     """
+    assert isinstance(string, bytes)
     try:
         detected = chardet.detect(string)
         if detected:
             return detected.get('encoding') or 'utf-8'
-    except Exception, e:
+    except Exception as e:
         pass
     return 'utf-8'
 
@@ -72,11 +77,12 @@ def quick_detect_encoding(string):
 
     Uses cchardet. Fallbacks to detect_encoding.
     """
+    assert isinstance(string, bytes)
     try:
         detected = cchardet.detect(string)
         if detected:
             return detected.get('encoding') or detect_encoding(string)
-    except Exception, e:
+    except Exception as e:
         pass
     return detect_encoding(string)
 
@@ -87,7 +93,7 @@ def to_utf8(str_or_unicode):
     >>> utils.to_utf8(u'hi')
         'hi'
     """
-    if isinstance(str_or_unicode, unicode):
+    if not isinstance(str_or_unicode, six.text_type):
         return str_or_unicode.encode("utf-8", "ignore")
     return str(str_or_unicode)
 
@@ -109,26 +115,18 @@ def get_delimiter(msg_body):
     return delimiter
 
 
-def html_to_text(string):
-    """
-    Dead-simple HTML-to-text converter:
-        >>> html_to_text("one<br>two<br>three")
-        >>> "one\ntwo\nthree"
-
-    NOTES:
-        1. the string is expected to contain UTF-8 encoded HTML!
-        2. returns utf-8 encoded str (not unicode)
-    """
-    s = _prepend_utf8_declaration(string)
-    s = s.replace("\n", "")
-
-    tree = html.fromstring(s)
-
+def html_tree_to_text(tree):
     for style in CSSSelector('style')(tree):
         style.getparent().remove(style)
 
     for c in tree.xpath('//comment()'):
-        c.getparent().remove(c)
+        parent = c.getparent()
+
+        # comment with no parent does not impact produced text
+        if parent is None:
+            continue
+
+        parent.remove(c)
 
     text   = ""
     for el in tree.iter():
@@ -152,10 +150,56 @@ def html_to_text(string):
     return _encode_utf8(retval)
 
 
+def html_to_text(string):
+    """
+    Dead-simple HTML-to-text converter:
+        >>> html_to_text("one<br>two<br>three")
+        >>> "one\ntwo\nthree"
+
+    NOTES:
+        1. the string is expected to contain UTF-8 encoded HTML!
+        2. returns utf-8 encoded str (not unicode)
+        3. if html can't be parsed returns None
+    """
+    if isinstance(string, six.text_type):
+        string = string.encode('utf8')
+
+    s = _prepend_utf8_declaration(string)
+    s = s.replace(b"\n", b"")
+    tree = html_fromstring(s)
+
+    if tree is None:
+        return None
+
+    return html_tree_to_text(tree)
+
+
+def html_fromstring(s):
+    """Parse html tree from string. Return None if the string can't be parsed.
+    """
+    try:
+        return html5parser.fromstring(s, parser=_HTML5LIB_PARSER)
+    except Exception:
+        pass
+
+
+def html_document_fromstring(s):
+    """Parse html tree from string. Return None if the string can't be parsed.
+    """
+    try:
+        return html5parser.document_fromstring(s, parser=_HTML5LIB_PARSER)
+    except Exception:
+        pass
+
+
+def cssselect(expr, tree):
+    return CSSSelector(expr)(tree)
+
+
 def _contains_charset_spec(s):
     """Return True if the first 4KB contain charset spec
     """
-    return s.lower().find('html; charset=', 0, 4096) != -1
+    return s.lower().find(b'html; charset=', 0, 4096) != -1
 
 
 def _prepend_utf8_declaration(s):
@@ -173,15 +217,25 @@ def _rm_excessive_newlines(s):
 def _encode_utf8(s):
     """Encode in 'utf-8' if unicode
     """
-    return s.encode('utf-8') if isinstance(s, unicode) else s
+    return s.encode('utf-8') if isinstance(s, six.text_type) else s
 
 
-_UTF8_DECLARATION = ('<meta http-equiv="Content-Type" content="text/html;'
-                     'charset=utf-8">')
+_UTF8_DECLARATION = (b'<meta http-equiv="Content-Type" content="text/html;'
+                     b'charset=utf-8">')
 
 
 _BLOCKTAGS  = ['div', 'p', 'ul', 'li', 'h1', 'h2', 'h3']
 _HARDBREAKS = ['br', 'hr', 'tr']
 
-
 _RE_EXCESSIVE_NEWLINES = re.compile("\n{2,10}")
+
+# html5lib is a pure-python library that conforms to the WHATWG HTML spec
+# and is not vulnarable to certain attacks common for XML libraries
+_HTML5LIB_PARSER = html5lib.HTMLParser(
+    # build lxml tree
+    html5lib.treebuilders.getTreeBuilder("lxml"),
+    # remove namespace value from inside lxml.html.html5paser element tag
+    # otherwise it yields something like "{http://www.w3.org/1999/xhtml}div"
+    # instead of "div", throwing the algo off
+    namespaceHTMLElements=False
+)

tests/__init__.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from nose.tools import *
 from mock import *
 

tests/html_quotations_test.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
 from . import *
 from . fixtures import *
 
@@ -26,7 +27,7 @@ def test_quotation_splitter_inside_blockquote():
 
 </blockquote>"""
 
-    eq_("<html><body><p>Reply</p></body></html>",
+    eq_("<html><head></head><body>Reply</body></html>",
         RE_WHITESPACE.sub('', quotations.extract_from_html(msg_body)))
 
 
@@ -43,7 +44,7 @@ def test_quotation_splitter_outside_blockquote():
   </div>
 </blockquote>
 """
-    eq_("<html><body><p>Reply</p></body></html>",
+    eq_("<html><head></head><body>Reply</body></html>",
         RE_WHITESPACE.sub('', quotations.extract_from_html(msg_body)))
 
 
@@ -61,7 +62,7 @@ def test_regular_blockquote():
   </div>
 </blockquote>
 """
-    eq_("<html><body><p>Reply</p><blockquote>Regular</blockquote></body></html>",
+    eq_("<html><head></head><body>Reply<blockquote>Regular</blockquote></body></html>",
         RE_WHITESPACE.sub('', quotations.extract_from_html(msg_body)))
 
 
@@ -84,6 +85,7 @@ Reply
 
     reply = """
 <html>
+<head></head>
 <body>
 Reply
 
@@ -127,7 +129,7 @@ def test_gmail_quote():
     </div>
   </div>
 </div>"""
-    eq_("<html><body><p>Reply</p></body></html>",
+    eq_("<html><head></head><body>Reply</body></html>",
         RE_WHITESPACE.sub('', quotations.extract_from_html(msg_body)))
 
 
@@ -138,7 +140,7 @@ def test_gmail_quote_compact():
                '<div>Test</div>' \
                '</div>' \
                '</div>'
-    eq_("<html><body><p>Reply</p></body></html>",
+    eq_("<html><head></head><body>Reply</body></html>",
         RE_WHITESPACE.sub('', quotations.extract_from_html(msg_body)))
 
 
@@ -165,7 +167,7 @@ def test_unicode_in_reply():
   Quote
 </blockquote>""".encode("utf-8")
 
-    eq_("<html><body><p>Reply&#160;&#160;Text<br></p><div><br></div>"
+    eq_("<html><head></head><body>Reply&#160;&#160;Text<br><div><br></div>"
         "</body></html>",
         RE_WHITESPACE.sub('', quotations.extract_from_html(msg_body)))
 
@@ -191,6 +193,7 @@ def test_blockquote_disclaimer():
 
     stripped_html = """
 <html>
+  <head></head>
   <body>
   <div>
     <div>
@@ -222,7 +225,7 @@ def test_date_block():
   </div>
 </div>
 """
-    eq_('<html><body><div>message<br></div></body></html>',
+    eq_('<html><head></head><body><div>message<br></div></body></html>',
         RE_WHITESPACE.sub('', quotations.extract_from_html(msg_body)))
 
 
@@ -239,7 +242,7 @@ Subject: You Have New Mail From Mary!<br><br>
 text
 </div></div>
 """
-    eq_('<html><body><div>message<br></div></body></html>',
+    eq_('<html><head></head><body><div>message<br></div></body></html>',
         RE_WHITESPACE.sub('', quotations.extract_from_html(msg_body)))
 
 
@@ -257,7 +260,7 @@ def test_reply_shares_div_with_from_block():
 
   </div>
 </body>'''
-    eq_('<html><body><div>Blah<br><br></div></body></html>',
+    eq_('<html><head></head><body><div>Blah<br><br></div></body></html>',
         RE_WHITESPACE.sub('', quotations.extract_from_html(msg_body)))
 
 
@@ -268,13 +271,13 @@ def test_reply_quotations_share_block():
 
 
 def test_OLK_SRC_BODY_SECTION_stripped():
-    eq_('<html><body><div>Reply</div></body></html>',
+    eq_('<html><head></head><body><div>Reply</div></body></html>',
         RE_WHITESPACE.sub(
             '', quotations.extract_from_html(OLK_SRC_BODY_SECTION)))
 
 
 def test_reply_separated_by_hr():
-    eq_('<html><body><div>Hi<div>there</div></div></body></html>',
+    eq_('<html><head></head><body><div>Hi<div>there</div></div></body></html>',
         RE_WHITESPACE.sub(
             '', quotations.extract_from_html(REPLY_SEPARATED_BY_HR)))
 
@@ -295,7 +298,7 @@ Reply
   </div>
 </div>
 '''
-    eq_('<html><body><p>Reply</p><div><hr></div></body></html>',
+    eq_('<html><head></head><body>Reply<div><hr></div></body></html>',
         RE_WHITESPACE.sub('', quotations.extract_from_html(msg_body)))
 
 
@@ -305,6 +308,7 @@ def extract_reply_and_check(filename):
     msg_body = f.read()
     reply = quotations.extract_from_html(msg_body)
     plain_reply = u.html_to_text(reply)
+    plain_reply = plain_reply.decode('utf8')
 
     eq_(RE_WHITESPACE.sub('', "Hi. I am fine.\n\nThanks,\nAlex"),
         RE_WHITESPACE.sub('', plain_reply))
@@ -354,7 +358,8 @@ def test_CRLF():
     assert_false(symbol in extracted)
     eq_('<html></html>', RE_WHITESPACE.sub('', extracted))
 
-    msg_body = """Reply
+    msg_body = """My
+reply
 <blockquote>
 
   <div>
@@ -369,8 +374,8 @@ def test_CRLF():
     msg_body = msg_body.replace('\n', '\r\n')
     extracted = quotations.extract_from_html(msg_body)
     assert_false(symbol in extracted)    
-    eq_("<html><body><p>Reply</p></body></html>",
-        RE_WHITESPACE.sub('', extracted))
+    # Keep new lines otherwise "My reply" becomes one word - "Myreply" 
+    eq_("<html><head></head><body>My\nreply\n</body></html>", extracted)
 
 
 def test_gmail_forwarded_msg():
@@ -378,3 +383,39 @@ def test_gmail_forwarded_msg():
 </div><br></div>"""
     extracted = quotations.extract_from_html(msg_body)
     eq_(RE_WHITESPACE.sub('', msg_body), RE_WHITESPACE.sub('', extracted))
+
+
+@patch.object(quotations, 'MAX_HTML_LEN', 1)
+def test_too_large_html():
+    msg_body = 'Reply' \
+               '<div class="gmail_quote">' \
+               '<div class="gmail_quote">On 11-Apr-2011, at 6:54 PM, Bob &lt;bob@example.com&gt; wrote:' \
+               '<div>Test</div>' \
+               '</div>' \
+               '</div>'
+    eq_(RE_WHITESPACE.sub('', msg_body),
+        RE_WHITESPACE.sub('', quotations.extract_from_html(msg_body)))
+
+
+def test_readable_html_empty():
+    msg_body = """
+<blockquote>
+  Reply
+  <div>
+    On 11-Apr-2011, at 6:54 PM, Bob &lt;bob@example.com&gt; wrote:
+  </div>
+
+  <div>
+    Test
+  </div>
+
+</blockquote>"""
+
+    eq_(RE_WHITESPACE.sub('', msg_body),
+        RE_WHITESPACE.sub('', quotations.extract_from_html(msg_body)))
+
+
+@patch.object(quotations, 'html_document_fromstring', Mock(return_value=None))
+def test_bad_html():
+    bad_html = "<html></html>"
+    eq_(bad_html, quotations.extract_from_html(bad_html))

tests/quotations_test.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
 from . import *
 from . fixtures import *
 

tests/signature/bruteforce_test.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
 from .. import *
 
 from talon.signature import bruteforce

tests/signature/extraction_test.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
 from .. import *
 
 import os
@@ -8,6 +9,7 @@ from talon.signature.learning import dataset
 from talon import signature
 from talon.signature import extraction as e
 from talon.signature import bruteforce
+from six.moves import range
 
 
 def test_message_shorter_SIGNATURE_MAX_LINES():
@@ -75,6 +77,31 @@ def test_basic():
         signature.extract(msg_body, 'Sergey'))
 
 
+def test_capitalized():
+    msg_body = """Hi Mary,
+
+Do you still need a DJ for your wedding? I've included a video demo of one of our DJs available for your wedding date.
+
+DJ Doe 
+http://example.com
+Password: SUPERPASSWORD
+
+Would you like to check out more?
+
+
+At your service,
+
+John Smith
+Doe Inc
+555-531-7967"""
+
+    sig = """John Smith
+Doe Inc
+555-531-7967"""
+
+    eq_(sig, signature.extract(msg_body, 'Doe')[1])
+
+
 def test_over_2_text_lines_after_signature():
     body = """Blah
 
@@ -127,20 +154,20 @@ def test_mark_lines():
 
 def test_process_marked_lines():
     # no signature found
-    eq_((range(5), None), e._process_marked_lines(range(5), 'telt'))
+    eq_((list(range(5)), None), e._process_marked_lines(list(range(5)), 'telt'))
 
     # signature in the middle of the text
-    eq_((range(9), None), e._process_marked_lines(range(9), 'tesestelt'))
+    eq_((list(range(9)), None), e._process_marked_lines(list(range(9)), 'tesestelt'))
 
     # long line splits signature
-    eq_((range(7), [7, 8]),
-        e._process_marked_lines(range(9), 'tsslsless'))
+    eq_((list(range(7)), [7, 8]),
+        e._process_marked_lines(list(range(9)), 'tsslsless'))
 
-    eq_((range(20), [20]),
-        e._process_marked_lines(range(21), 'ttttttstttesllelelets'))
+    eq_((list(range(20)), [20]),
+        e._process_marked_lines(list(range(21)), 'ttttttstttesllelelets'))
 
     # some signature lines could be identified as text
-    eq_(([0], range(1, 9)), e._process_marked_lines(range(9), 'tsetetest'))
+    eq_(([0], list(range(1, 9))), e._process_marked_lines(list(range(9)), 'tsetetest'))
 
-    eq_(([], range(5)),
-        e._process_marked_lines(range(5), "ststt"))
+    eq_(([], list(range(5))),
+        e._process_marked_lines(list(range(5)), "ststt"))

tests/signature/learning/dataset_test.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
 from ... import *
 import os
 

tests/signature/learning/featurespace_test.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
 from ... import *
 
 from talon.signature.learning import featurespace as fs

tests/signature/learning/helpers_test.py

@@ -1,11 +1,13 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
 from ... import *
 
 import regex as re
 
 from talon.signature.learning import helpers as h
 from talon.signature.learning.helpers import *
+from six.moves import range
 
 # First testing regex constants.
 VALID = '''
@@ -154,7 +156,7 @@ def test_extract_names():
         # check that extracted names could be compiled
         try:
             re.compile("|".join(extracted_names))
-        except Exception, e:
+        except Exception as e:
             ok_(False, ("Failed to compile extracted names {}"
                         "\n\nReason: {}").format(extracted_names, e))
         if expected_names:
@@ -190,10 +192,11 @@ def test_punctuation_percent(categories_percent):
 def test_capitalized_words_percent():
     eq_(0.0, h.capitalized_words_percent(''))
     eq_(100.0, h.capitalized_words_percent('Example Corp'))
-    eq_(50.0, h.capitalized_words_percent('Qqq qqq QQQ 123 sss'))
+    eq_(50.0, h.capitalized_words_percent('Qqq qqq Aqs 123 sss'))
     eq_(100.0, h.capitalized_words_percent('Cell 713-444-7368'))
     eq_(100.0, h.capitalized_words_percent('8th Floor'))
     eq_(0.0, h.capitalized_words_percent('(212) 230-9276'))
+    eq_(50.0, h.capitalized_words_percent('Password: REMARKABLE'))
 
 
 def test_has_signature():
@@ -204,7 +207,7 @@ def test_has_signature():
                         'sender@example.com'))
     assert_false(h.has_signature('http://www.example.com/555-555-5555',
                                  'sender@example.com'))
-    long_line = ''.join(['q' for e in xrange(28)])
+    long_line = ''.join(['q' for e in range(28)])
     assert_false(h.has_signature(long_line + ' sender', 'sender@example.com'))
     # wont crash on an empty string
     assert_false(h.has_signature('', ''))

tests/text_quotations_test.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
 from . import *
 from . fixtures import *
 
@@ -7,6 +8,9 @@ import os
 
 import email.iterators
 from talon import quotations
+import six
+from six.moves import range
+from six import StringIO
 
 
 @patch.object(quotations, 'MAX_LINES_COUNT', 1)
@@ -138,7 +142,7 @@ def _check_pattern_original_message(original_message_indicator):
 -----{}-----
 
 Test"""
-    eq_('Test reply', quotations.extract_from_plain(msg_body.format(unicode(original_message_indicator))))
+    eq_('Test reply', quotations.extract_from_plain(msg_body.format(six.text_type(original_message_indicator))))
 
 def test_english_original_message():
     _check_pattern_original_message('Original Message')
@@ -662,6 +666,15 @@ def test_preprocess_postprocess_2_links():
     eq_(msg_body, quotations.extract_from_plain(msg_body))
 
 
+def body_iterator(msg, decode=False):
+    for subpart in msg.walk():
+        payload = subpart.get_payload(decode=decode)
+        if isinstance(payload, six.text_type):
+            yield payload
+        else:
+            yield payload.decode('utf8')
+
+
 def test_standard_replies():
     for filename in os.listdir(STANDARD_REPLIES):
         filename = os.path.join(STANDARD_REPLIES, filename)
@@ -669,8 +682,8 @@ def test_standard_replies():
             continue
         with open(filename) as f:
             message = email.message_from_file(f)
-            body = email.iterators.typed_subpart_iterator(message, subtype='plain').next()
-            text = ''.join(email.iterators.body_line_iterator(body, True))
+            body = next(email.iterators.typed_subpart_iterator(message, subtype='plain'))
+            text = ''.join(body_iterator(body, True))
 
             stripped_text = quotations.extract_from_plain(text)
             reply_text_fn = filename[:-4] + '_reply_text'

tests/utils_test.py

@@ -1,9 +1,12 @@
 # coding:utf-8
 
+from __future__ import absolute_import
 from . import *
 
 from talon import utils as u
 import cchardet
+import six
+from lxml import html
 
 
 def test_get_delimiter():
@@ -14,49 +17,49 @@ def test_get_delimiter():
 
 def test_unicode():
     eq_ (u'hi', u.to_unicode('hi'))
-    eq_ (type(u.to_unicode('hi')), unicode )
-    eq_ (type(u.to_unicode(u'hi')), unicode )
-    eq_ (type(u.to_unicode('привет')), unicode )
-    eq_ (type(u.to_unicode(u'привет')), unicode )
+    eq_ (type(u.to_unicode('hi')), six.text_type )
+    eq_ (type(u.to_unicode(u'hi')), six.text_type )
+    eq_ (type(u.to_unicode('привет')), six.text_type )
+    eq_ (type(u.to_unicode(u'привет')), six.text_type )
     eq_ (u"привет", u.to_unicode('привет'))
     eq_ (u"привет", u.to_unicode(u'привет'))
     # some latin1 stuff
-    eq_ (u"Versión", u.to_unicode('Versi\xf3n', precise=True))
+    eq_ (u"Versión", u.to_unicode(u'Versi\xf3n'.encode('iso-8859-2'), precise=True))
 
 
 def test_detect_encoding():
-    eq_ ('ascii', u.detect_encoding('qwe').lower())
-    eq_ ('iso-8859-2', u.detect_encoding('Versi\xf3n').lower())
-    eq_ ('utf-8', u.detect_encoding('привет').lower())
+    eq_ ('ascii', u.detect_encoding(b'qwe').lower())
+    eq_ ('iso-8859-2', u.detect_encoding(u'Versi\xf3n'.encode('iso-8859-2')).lower())
+    eq_ ('utf-8', u.detect_encoding(u'привет'.encode('utf8')).lower())
     # fallback to utf-8
     with patch.object(u.chardet, 'detect') as detect:
         detect.side_effect = Exception
-        eq_ ('utf-8', u.detect_encoding('qwe').lower())
+        eq_ ('utf-8', u.detect_encoding('qwe'.encode('utf8')).lower())
 
 
 def test_quick_detect_encoding():
-    eq_ ('ascii', u.quick_detect_encoding('qwe').lower())
-    eq_ ('windows-1252', u.quick_detect_encoding('Versi\xf3n').lower())
-    eq_ ('utf-8', u.quick_detect_encoding('привет').lower())
+    eq_ ('ascii', u.quick_detect_encoding(b'qwe').lower())
+    eq_ ('windows-1252', u.quick_detect_encoding(u'Versi\xf3n'.encode('windows-1252')).lower())
+    eq_ ('utf-8', u.quick_detect_encoding(u'привет'.encode('utf8')).lower())
 
 
 @patch.object(cchardet, 'detect')
 @patch.object(u, 'detect_encoding')
 def test_quick_detect_encoding_edge_cases(detect_encoding, cchardet_detect):
     cchardet_detect.return_value = {'encoding': 'ascii'}
-    eq_('ascii', u.quick_detect_encoding("qwe"))
-    cchardet_detect.assert_called_once_with("qwe")
+    eq_('ascii', u.quick_detect_encoding(b"qwe"))
+    cchardet_detect.assert_called_once_with(b"qwe")
 
     # fallback to detect_encoding
     cchardet_detect.return_value = {}
     detect_encoding.return_value = 'utf-8'
-    eq_('utf-8', u.quick_detect_encoding("qwe"))
+    eq_('utf-8', u.quick_detect_encoding(b"qwe"))
 
     # exception
     detect_encoding.reset_mock()
     cchardet_detect.side_effect = Exception()
     detect_encoding.return_value = 'utf-8'
-    eq_('utf-8', u.quick_detect_encoding("qwe"))
+    eq_('utf-8', u.quick_detect_encoding(b"qwe"))
     ok_(detect_encoding.called)
 
 
@@ -73,11 +76,11 @@ Haha
 </p>
 </body>"""
     text = u.html_to_text(html)
-    eq_("Hello world! \n\n  * One! \n  * Two \nHaha", text)
-    eq_("привет!", u.html_to_text("<b>привет!</b>"))
+    eq_(b"Hello world! \n\n  * One! \n  * Two \nHaha", text)
+    eq_(u"привет!", u.html_to_text("<b>привет!</b>").decode('utf8'))
 
     html = '<body><br/><br/>Hi</body>'
-    eq_ ('Hi', u.html_to_text(html))
+    eq_ (b'Hi', u.html_to_text(html))
 
     html = """Hi
 <style type="text/css">
@@ -97,11 +100,34 @@ font: 13px 'Lucida Grande', Arial, sans-serif;
 
 }
 </style>"""
-    eq_ ('Hi', u.html_to_text(html))
+    eq_ (b'Hi', u.html_to_text(html))
 
     html = """<div>
 <!-- COMMENT 1 -->
 <span>TEXT 1</span>
 <p>TEXT 2 <!-- COMMENT 2 --></p>
 </div>"""
-    eq_('TEXT 1 \nTEXT 2', u.html_to_text(html))
+    eq_(b'TEXT 1 \nTEXT 2', u.html_to_text(html))
+
+
+def test_comment_no_parent():
+    s = "<!-- COMMENT 1 --> no comment"
+    d = u.html_document_fromstring(s)
+    eq_("no comment", u.html_tree_to_text(d))
+
+
+@patch.object(u.html5parser, 'fromstring', Mock(side_effect=Exception()))
+def test_html_fromstring_exception():
+    eq_(None, u.html_fromstring("<html></html>"))
+
+
+@patch.object(u.html5parser, 'document_fromstring')
+def test_html_document_fromstring_exception(document_fromstring):
+    document_fromstring.side_effect = Exception()
+    eq_(None, u.html_document_fromstring("<html></html>"))
+
+
+@patch.object(u, 'html_fromstring', Mock(return_value=None))
+def test_bad_html_to_text():
+    bad_html = "one<br>two<br>three"
+    eq_(None, u.html_to_text(bad_html))

train.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from talon.signature import EXTRACTOR_FILENAME, EXTRACTOR_DATA
 from talon.signature.learning.classifier import train, init