Continue with quotation cut even if html cut throws an exception

.build/Dockerfile

@@ -1,20 +0,0 @@
-FROM python:3.9-slim-buster AS deps
-
-RUN apt-get update && \
-    apt-get install -y build-essential git curl python3-dev libatlas3-base libatlas-base-dev liblapack-dev libxml2 libxml2-dev libffi6 libffi-dev musl-dev libxslt-dev
-
-FROM deps AS testable
-ARG REPORT_PATH
-
-VOLUME ["/var/mailgun", "/etc/mailgun/ssl", ${REPORT_PATH}]
-
-ADD . /app
-WORKDIR /app
-COPY wheel/* /wheel/
-
-RUN mkdir -p ${REPORT_PATH}
-
-RUN python ./setup.py build bdist_wheel -d /wheel && \
-    pip install --no-deps /wheel/*
-
-ENTRYPOINT ["/bin/sh", "/app/run_tests.sh"]

.gitignore

@@ -54,6 +54,3 @@ _trial_temp
 
 # OSX
 .DS_Store
-
-# vim-backup
-*.bak

requirements.txt

@@ -1,11 +0,0 @@
-chardet>=1.0.1
-cchardet>=0.3.5
-cssselect
-html5lib
-joblib
-lxml>=2.3.3
-numpy
-regex>=1
-scikit-learn>=1.0.0
-scipy
-six>=1.10.0

run_tests.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-set -ex
-REPORT_PATH="${REPORT_PATH:-./}"
-nosetests --with-xunit --with-coverage --cover-xml --cover-xml-file $REPORT_PATH/coverage.xml --xunit-file=$REPORT_PATH/nosetests.xml --cover-package=talon .

setup.py

@@ -19,17 +19,17 @@ class InstallCommand(install):
         if self.no_ml:
             dist = self.distribution
             dist.packages=find_packages(exclude=[
-                "tests",
+                'tests',
-                "tests.*",
+                'tests.*',
-                "talon.signature",
+                'talon.signature',
-                "talon.signature.*",
+                'talon.signature.*',
             ])
-            for not_required in ["numpy", "scipy", "scikit-learn==0.24.1"]:
+            for not_required in ['numpy', 'scipy', 'scikit-learn==0.16.1']:
                 dist.install_requires.remove(not_required)
 
 
 setup(name='talon',
-      version='1.4.10',
+      version='1.4.8',
       description=("Mailgun library "
                    "to extract message quotations and signatures."),
       long_description=open("README.rst").read(),
@@ -44,21 +44,20 @@ setup(name='talon',
       include_package_data=True,
       zip_safe=True,
       install_requires=[
-          "lxml",
+          "lxml>=2.3.3",
-          "regex",
+          "regex>=1",
           "numpy",
           "scipy",
-          "scikit-learn>=1.0.0",
+          "scikit-learn==0.16.1", # pickled versions of classifier, else rebuild
-          "chardet",
+          'chardet>=1.0.1',
-          "cchardet",
+          'cchardet>=0.3.5',
-          "cssselect",
+          'cssselect',
-          "six",
+          'six>=1.10.0',
-          "html5lib",
+          'html5lib'
-          "joblib",
           ],
       tests_require=[
           "mock",
-          "nose",
+          "nose>=1.2.1",
           "coverage"
           ]
       )

talon/quotations.py

@@ -457,13 +457,15 @@ def _extract_from_html(msg_body):
 
     msg_body = msg_body.replace(b'\r\n', b'\n')
 
-    msg_body = re.sub(br"\<\?xml.+\?\>|\<\!DOCTYPE.+]\>", "", msg_body)
+    msg_body = re.sub(r"\<\?xml.+\?\>|\<\!DOCTYPE.+]\>", "", msg_body)
 
     html_tree = html_document_fromstring(msg_body)
 
     if html_tree is None:
         return msg_body
 
+    cut_quotations = False
+    try:
         cut_quotations = (html_quotations.cut_gmail_quote(html_tree) or
                           html_quotations.cut_zimbra_quote(html_tree) or
                           html_quotations.cut_blockquote(html_tree) or
@@ -471,6 +473,10 @@ def _extract_from_html(msg_body):
                           html_quotations.cut_by_id(html_tree) or
                           html_quotations.cut_from_block(html_tree)
                           )
+    except Exception as e:
+        log.exception('during html quotations cut')
+        pass
+
     html_tree_copy = deepcopy(html_tree)
 
     number_of_checkpoints = html_quotations.add_checkpoint(html_tree, 0)

talon/signature/__init__.py

@@ -23,14 +23,17 @@ trained against, don't forget to regenerate:
 from __future__ import absolute_import
 import os
 
-from talon.signature import extraction
+from . import extraction
-from talon.signature.extraction import extract
+from . extraction import extract  #noqa
-from talon.signature.learning import classifier
+from . learning import classifier
+
+
+DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
+
+EXTRACTOR_FILENAME = os.path.join(DATA_DIR, 'classifier')
+EXTRACTOR_DATA = os.path.join(DATA_DIR, 'train.data')
 
 
 def initialize():
-    data_dir = os.path.join(os.path.dirname(__file__), 'data')
+    extraction.EXTRACTOR = classifier.load(EXTRACTOR_FILENAME,
-    extractor_filename = os.path.join(data_dir, 'classifier')
+                                           EXTRACTOR_DATA)
-    extractor_data_filename = os.path.join(data_dir, 'train.data')
-    extraction.EXTRACTOR = classifier.load(extractor_filename,
-                                           extractor_data_filename)

talon/signature/bruteforce.py

@@ -62,7 +62,7 @@ RE_SIGNATURE_CANDIDATE = re.compile(r'''
 
 
 def extract_signature(msg_body):
-    '''
+    """
     Analyzes message for a presence of signature block (by common patterns)
     and returns tuple with two elements: message text without signature block
     and the signature itself.
@@ -72,7 +72,7 @@ def extract_signature(msg_body):
 
     >>> extract_signature('Hey man!')
     ('Hey man!', None)
-    '''
+    """
     try:
         # identify line delimiter first
         delimiter = get_delimiter(msg_body)

talon/signature/learning/classifier.py

@@ -8,7 +8,7 @@ body belongs to the signature.
 from __future__ import absolute_import
 
 from numpy import genfromtxt
-import joblib
+from sklearn.externals import joblib
 from sklearn.svm import LinearSVC
 
 

talon/signature/learning/helpers.py

@@ -102,7 +102,7 @@ def flatten_list(list_to_flatten):
 
 
 def contains_sender_names(sender):
-    """Returns a functions to search sender\'s name or it\'s part.
+    '''Returns a functions to search sender\'s name or it\'s part.
 
     >>> feature = contains_sender_names("Sergey N.  Obukhov <xxx@example.com>")
     >>> feature("Sergey Obukhov")
@@ -115,7 +115,7 @@ def contains_sender_names(sender):
     1
     >>> contains_sender_names("<serobnic@mail.ru>")("serobnic")
     1
-    """
+    '''
     names = '( |$)|'.join(flatten_list([[e, e.capitalize()]
                                         for e in extract_names(sender)]))
     names = names or sender
@@ -140,16 +140,10 @@ def extract_names(sender):
     sender = "".join([char if char.isalpha() else ' ' for char in sender])
     # Remove too short words and words from "black" list i.e.
     # words like `ru`, `gmail`, `com`, `org`, etc.
-    names = list()
+    sender = [word for word in sender.split() if len(word) > 1 and
-    for word in sender.split():
+              not word in BAD_SENDER_NAMES]
-        if len(word) < 2:
+    # Remove duplicates
-            continue
+    names = list(set(sender))
-        if word in BAD_SENDER_NAMES:
-            continue
-        if word in names:
-            continue
-        names.append(word)
-
     return names
 
 
@@ -214,26 +208,20 @@ def many_capitalized_words(s):
 
 
 def has_signature(body, sender):
-    """Checks if the body has signature. Returns True or False."""
+    '''Checks if the body has signature. Returns True or False.'''
     non_empty = [line for line in body.splitlines() if line.strip()]
     candidate = non_empty[-SIGNATURE_MAX_LINES:]
     upvotes = 0
-    sender_check = contains_sender_names(sender)
     for line in candidate:
         # we check lines for sender's name, phone, email and url,
         # those signature lines don't take more then 27 lines
         if len(line.strip()) > 27:
             continue
-
+        elif contains_sender_names(sender)(line):
-        if sender_check(line):
             return True
-
+        elif (binary_regex_search(RE_RELAX_PHONE)(line) +
-        if (binary_regex_search(RE_RELAX_PHONE)(line) +
               binary_regex_search(RE_EMAIL)(line) +
               binary_regex_search(RE_URL)(line) == 1):
             upvotes += 1
-
     if upvotes > 1:
         return True
-
-    return False

test-requirements.txt

@@ -1,3 +0,0 @@
-coverage
-mock
-nose>=1.2.1

tests/__init__.py

@@ -1,6 +1,4 @@
 from __future__ import absolute_import
-from nose.tools import *
-from mock import *
 
 import talon
 

tests/html_quotations_test.py

@@ -2,13 +2,11 @@
 
 from __future__ import absolute_import
 
-# noinspection PyUnresolvedReferences
+from tests.fixtures import REPLY_QUOTATIONS_SHARE_BLOCK, OLK_SRC_BODY_SECTION, REPLY_SEPARATED_BY_HR
-import re
+from nose.tools import eq_, ok_, assert_false, assert_true
-
 from talon import quotations, utils as u
-from . import *
+from mock import Mock, patch
-from .fixtures import *
+import re
-from lxml import html
 
 RE_WHITESPACE = re.compile("\s")
 RE_DOUBLE_WHITESPACE = re.compile("\s")

tests/quotations_test.py

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from . import *
-from . fixtures import *
 
+from mock import Mock, patch
 from talon import quotations
+from nose.tools import eq_
 
 
 @patch.object(quotations, 'extract_from_html')

tests/signature/bruteforce_test.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from .. import *
+from nose.tools import eq_
 
 from talon.signature import bruteforce
+from mock import patch, Mock
 
 
 def test_empty_body():

tests/signature/extraction_test.py

@@ -2,14 +2,14 @@
 
 from __future__ import absolute_import
 
-import os
-
-from six.moves import range
-
 from talon.signature import bruteforce, extraction, extract
 from talon.signature import extraction as e
 from talon.signature.learning import dataset
-from .. import *
+from nose.tools import eq_
+from .. import STRIPPED, UNICODE_MSG
+from six.moves import range
+from mock import patch
+import os
 
 
 def test_message_shorter_SIGNATURE_MAX_LINES():

tests/signature/learning/dataset_test.py

@@ -1,14 +1,13 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from ... import *
-import os
-
-from numpy import genfromtxt
-
-from talon.signature.learning import dataset as d
 
+from ... import EML_MSG_FILENAME, MSG_FILENAME_WITH_BODY_SUFFIX, TMP_DIR, EMAILS_DIR
 from talon.signature.learning.featurespace import features
+from talon.signature.learning import dataset as d
+from nose.tools import eq_, assert_false, ok_
+from numpy import genfromtxt
+import os
 
 
 def test_is_sender_filename():

tests/signature/learning/featurespace_test.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from ... import *
 
 from talon.signature.learning import featurespace as fs
+from nose.tools import eq_, assert_false, ok_
+from mock import patch
 
 
 def test_apply_features():

tests/signature/learning/helpers_test.py

@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from ... import *
-
-import regex as re
 
 from talon.signature.learning import helpers as h
-from talon.signature.learning.helpers import *
+from talon.signature.learning.helpers import RE_RELAX_PHONE, RE_NAME
+from nose.tools import eq_, ok_, assert_false, assert_in
+from mock import patch, Mock
 from six.moves import range
+import re
 
 # First testing regex constants.
 VALID = '''

tests/text_quotations_test.py

@@ -1,16 +1,15 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from . import *
-from . fixtures import *
 
-import os
+from tests.fixtures import STANDARD_REPLIES
-
-import email.iterators
 from talon import quotations
-import six
 from six.moves import range
-from six import StringIO
+from nose.tools import eq_
+from mock import patch
+import email.iterators
+import six
+import os
 
 
 @patch.object(quotations, 'MAX_LINES_COUNT', 1)
@@ -35,6 +34,7 @@ On 11-Apr-2011, at 6:54 PM, Roman Tkachenko <romant@example.com> wrote:
 
     eq_("Test reply", quotations.extract_from_plain(msg_body))
 
+
 def test_pattern_on_date_polymail():
     msg_body = """Test reply
 
@@ -190,14 +190,17 @@ Test"""
     eq_('Test reply', quotations.extract_from_plain(
         msg_body.format(six.text_type(original_message_indicator))))
 
+
 def test_english_original_message():
     _check_pattern_original_message('Original Message')
     _check_pattern_original_message('Reply Message')
 
+
 def test_german_original_message():
     _check_pattern_original_message(u'Ursprüngliche Nachricht')
     _check_pattern_original_message('Antwort Nachricht')
 
+
 def test_danish_original_message():
     _check_pattern_original_message('Oprindelig meddelelse')
 
@@ -296,6 +299,7 @@ On 04/19/2011 07:10 AM, Roman Tkachenko wrote:
 > Hello"""
     eq_("Hi", quotations.extract_from_plain(msg_body))
 
+
 def test_with_indent():
     msg_body = """YOLO salvia cillum kogi typewriter mumblecore cardigan skateboard Austin.
 
@@ -303,7 +307,8 @@ def test_with_indent():
 
 Brunch mumblecore pug Marfa tofu, irure taxidermy hoodie readymade pariatur.
     """
-    eq_("YOLO salvia cillum kogi typewriter mumblecore cardigan skateboard Austin.", quotations.extract_from_plain(msg_body))
+    eq_("YOLO salvia cillum kogi typewriter mumblecore cardigan skateboard Austin.",
+        quotations.extract_from_plain(msg_body))
 
 
 def test_short_quotation_with_newline():
@@ -343,6 +348,7 @@ Subject: The manager has commented on your Loop
 Blah-blah-blah
 """))
 
+
 def test_german_from_block():
     eq_('Allo! Follow up MIME!', quotations.extract_from_plain(
     """Allo! Follow up MIME!
@@ -355,6 +361,7 @@ Betreff: The manager has commented on your Loop
 Blah-blah-blah
 """))
 
+
 def test_french_multiline_from_block():
     eq_('Lorem ipsum', quotations.extract_from_plain(
     u"""Lorem ipsum
@@ -367,6 +374,7 @@ Objet : Follow Up
 Blah-blah-blah
 """))
 
+
 def test_french_from_block():
     eq_('Lorem ipsum', quotations.extract_from_plain(
     u"""Lorem ipsum
@@ -375,6 +383,7 @@ Le 23 janv. 2015 à 22:03, Brendan xxx <brendan.xxx@xxx.com<mailto:brendan.xxx@x
 
 Bonjour!"""))
 
+
 def test_polish_from_block():
     eq_('Lorem ipsum', quotations.extract_from_plain(
     u"""Lorem ipsum
@@ -385,6 +394,7 @@ napisał:
 Blah!
 """))
 
+
 def test_danish_from_block():
     eq_('Allo! Follow up MIME!', quotations.extract_from_plain(
     """Allo! Follow up MIME!
@@ -397,6 +407,7 @@ Emne: The manager has commented on your Loop
 Blah-blah-blah
 """))
 
+
 def test_swedish_from_block():
     eq_('Allo! Follow up MIME!', quotations.extract_from_plain(
     u"""Allo! Follow up MIME!
@@ -408,6 +419,7 @@ Till: Isacson Leiff
 Blah-blah-blah
 """))
 
+
 def test_swedish_from_line():
     eq_('Lorem', quotations.extract_from_plain(
     """Lorem
@@ -416,6 +428,7 @@ Den 14 september, 2015 02:23:18, Valentino Rudy (valentino@rudy.be) skrev:
 Veniam laborum mlkshk kale chips authentic. Normcore mumblecore laboris, fanny pack readymade eu blog chia pop-up freegan enim master cleanse.
 """))
 
+
 def test_norwegian_from_line():
     eq_('Lorem', quotations.extract_from_plain(
     u"""Lorem
@@ -424,6 +437,7 @@ På 14 september 2015 på 02:23:18, Valentino Rudy (valentino@rudy.be) skrev:
 Veniam laborum mlkshk kale chips authentic. Normcore mumblecore laboris, fanny pack readymade eu blog chia pop-up freegan enim master cleanse.
 """))
 
+
 def test_dutch_from_block():
     eq_('Gluten-free culpa lo-fi et nesciunt nostrud.', quotations.extract_from_plain(
     """Gluten-free culpa lo-fi et nesciunt nostrud.
@@ -433,6 +447,7 @@ Op 17-feb.-2015, om 13:18 heeft Julius Caesar <pantheon@rome.com> het volgende g
 Small batch beard laboris tempor, non listicle hella Tumblr heirloom.
 """))
 
+
 def test_vietnamese_from_block():
     eq_('Hello', quotations.extract_from_plain(
     u"""Hello
@@ -442,6 +457,7 @@ Vào 14:24 8 tháng 6, 2017, Hùng Nguyễn <hungnguyen@xxx.com> đã viết:
 > Xin chào
 """))
 
+
 def test_quotation_marker_false_positive():
     msg_body = """Visit us now for assistance...
 >>> >>>  http://www.domain.com <<<
@@ -826,10 +842,10 @@ The user experience was unparallelled. Please continue production. I'm sending p
 that this line is intact."""
 
     parsed = quotations.extract_from_plain(msg_body)
-    eq_(msg_body, parsed)
+    eq_(msg_body, parsed.decode('utf8'))
 
 
-def test_appointment_2():
+def test_appointment():
     msg_body = """Invitation for an interview:
 
 Date: Wednesday 3, October 2011 
@@ -838,4 +854,4 @@ Address: 130 Fox St
 
 Please bring in your ID."""
     parsed = quotations.extract_from_plain(msg_body)
-    eq_(msg_body, parsed)
+    eq_(msg_body, parsed.decode('utf8'))

tests/utils_test.py

@@ -2,12 +2,13 @@
 
 from __future__ import absolute_import
 
+
+from nose.tools import eq_, ok_, assert_false
+from talon import utils as u
+from mock import patch, Mock
 import cchardet
 import six
 
-from talon import utils as u
-from . import *
-
 
 def test_get_delimiter():
     eq_('\r\n', u.get_delimiter('abc\r\n123'))