Continue with quotation cut even if html cut throws an exception

Add __init__.py into data folder, add data files into MANIFEST.in

MANIFEST.in

@@ -5,3 +5,10 @@ include classifier
 include LICENSE
 include MANIFEST.in
 include README.rst
+include talon/signature/data/train.data
+include talon/signature/data/classifier
+include talon/signature/data/classifier_01.npy
+include talon/signature/data/classifier_02.npy
+include talon/signature/data/classifier_03.npy
+include talon/signature/data/classifier_04.npy
+include talon/signature/data/classifier_05.npy

talon/quotations.py

@@ -464,13 +464,19 @@ def _extract_from_html(msg_body):
     if html_tree is None:
         return msg_body
 
-    cut_quotations = (html_quotations.cut_gmail_quote(html_tree) or
-                      html_quotations.cut_zimbra_quote(html_tree) or
-                      html_quotations.cut_blockquote(html_tree) or
-                      html_quotations.cut_microsoft_quote(html_tree) or
-                      html_quotations.cut_by_id(html_tree) or
-                      html_quotations.cut_from_block(html_tree)
-                      )
+    cut_quotations = False
+    try:
+        cut_quotations = (html_quotations.cut_gmail_quote(html_tree) or
+                          html_quotations.cut_zimbra_quote(html_tree) or
+                          html_quotations.cut_blockquote(html_tree) or
+                          html_quotations.cut_microsoft_quote(html_tree) or
+                          html_quotations.cut_by_id(html_tree) or
+                          html_quotations.cut_from_block(html_tree)
+                          )
+    except Exception as e:
+        log.exception('during html quotations cut')
+        pass
+
     html_tree_copy = deepcopy(html_tree)
 
     number_of_checkpoints = html_quotations.add_checkpoint(html_tree, 0)

talon/signature/bruteforce.py

@@ -62,7 +62,7 @@ RE_SIGNATURE_CANDIDATE = re.compile(r'''
 
 
 def extract_signature(msg_body):
-    '''
+    """
     Analyzes message for a presence of signature block (by common patterns)
     and returns tuple with two elements: message text without signature block
     and the signature itself.
@@ -72,7 +72,7 @@ def extract_signature(msg_body):
 
     >>> extract_signature('Hey man!')
     ('Hey man!', None)
-    '''
+    """
     try:
         # identify line delimiter first
         delimiter = get_delimiter(msg_body)

talon/signature/data/__init__.py

@@ -0,0 +1 @@
+

tests/__init__.py

@@ -1,6 +1,4 @@
 from __future__ import absolute_import
-from nose.tools import *
-from mock import *
 
 import talon
 

tests/html_quotations_test.py

@@ -2,13 +2,11 @@
 
 from __future__ import absolute_import
 
-# noinspection PyUnresolvedReferences
-import re
-
+from tests.fixtures import REPLY_QUOTATIONS_SHARE_BLOCK, OLK_SRC_BODY_SECTION, REPLY_SEPARATED_BY_HR
+from nose.tools import eq_, ok_, assert_false, assert_true
 from talon import quotations, utils as u
-from . import *
-from .fixtures import *
-from lxml import html
+from mock import Mock, patch
+import re
 
 RE_WHITESPACE = re.compile("\s")
 RE_DOUBLE_WHITESPACE = re.compile("\s")

tests/quotations_test.py

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from . import *
-from . fixtures import *
 
+from mock import Mock, patch
 from talon import quotations
+from nose.tools import eq_
 
 
 @patch.object(quotations, 'extract_from_html')

tests/signature/bruteforce_test.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from .. import *
+from nose.tools import eq_
 
 from talon.signature import bruteforce
+from mock import patch, Mock
 
 
 def test_empty_body():

tests/signature/extraction_test.py

@@ -2,14 +2,14 @@
 
 from __future__ import absolute_import
 
-import os
-
-from six.moves import range
-
 from talon.signature import bruteforce, extraction, extract
 from talon.signature import extraction as e
 from talon.signature.learning import dataset
-from .. import *
+from nose.tools import eq_
+from .. import STRIPPED, UNICODE_MSG
+from six.moves import range
+from mock import patch
+import os
 
 
 def test_message_shorter_SIGNATURE_MAX_LINES():

tests/signature/learning/dataset_test.py

@@ -1,14 +1,13 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from ... import *
-import os
-
-from numpy import genfromtxt
-
-from talon.signature.learning import dataset as d
 
+from ... import EML_MSG_FILENAME, MSG_FILENAME_WITH_BODY_SUFFIX, TMP_DIR, EMAILS_DIR
 from talon.signature.learning.featurespace import features
+from talon.signature.learning import dataset as d
+from nose.tools import eq_, assert_false, ok_
+from numpy import genfromtxt
+import os
 
 
 def test_is_sender_filename():

tests/signature/learning/featurespace_test.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from ... import *
 
 from talon.signature.learning import featurespace as fs
+from nose.tools import eq_, assert_false, ok_
+from mock import patch
 
 
 def test_apply_features():

tests/signature/learning/helpers_test.py

@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from ... import *
-
-import regex as re
 
 from talon.signature.learning import helpers as h
-from talon.signature.learning.helpers import *
+from talon.signature.learning.helpers import RE_RELAX_PHONE, RE_NAME
+from nose.tools import eq_, ok_, assert_false, assert_in
+from mock import patch, Mock
 from six.moves import range
+import re
 
 # First testing regex constants.
 VALID = '''

tests/text_quotations_test.py

@@ -1,16 +1,15 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from . import *
-from . fixtures import *
 
-import os
-
-import email.iterators
+from tests.fixtures import STANDARD_REPLIES
 from talon import quotations
-import six
 from six.moves import range
-from six import StringIO
+from nose.tools import eq_
+from mock import patch
+import email.iterators
+import six
+import os
 
 
 @patch.object(quotations, 'MAX_LINES_COUNT', 1)
@@ -35,6 +34,7 @@ On 11-Apr-2011, at 6:54 PM, Roman Tkachenko <romant@example.com> wrote:
 
     eq_("Test reply", quotations.extract_from_plain(msg_body))
 
+
 def test_pattern_on_date_polymail():
     msg_body = """Test reply
 
@@ -190,14 +190,17 @@ Test"""
     eq_('Test reply', quotations.extract_from_plain(
         msg_body.format(six.text_type(original_message_indicator))))
 
+
 def test_english_original_message():
     _check_pattern_original_message('Original Message')
     _check_pattern_original_message('Reply Message')
 
+
 def test_german_original_message():
     _check_pattern_original_message(u'Ursprüngliche Nachricht')
     _check_pattern_original_message('Antwort Nachricht')
 
+
 def test_danish_original_message():
     _check_pattern_original_message('Oprindelig meddelelse')
 
@@ -296,6 +299,7 @@ On 04/19/2011 07:10 AM, Roman Tkachenko wrote:
 > Hello"""
     eq_("Hi", quotations.extract_from_plain(msg_body))
 
+
 def test_with_indent():
     msg_body = """YOLO salvia cillum kogi typewriter mumblecore cardigan skateboard Austin.
 
@@ -303,7 +307,8 @@ def test_with_indent():
 
 Brunch mumblecore pug Marfa tofu, irure taxidermy hoodie readymade pariatur.
     """
-    eq_("YOLO salvia cillum kogi typewriter mumblecore cardigan skateboard Austin.", quotations.extract_from_plain(msg_body))
+    eq_("YOLO salvia cillum kogi typewriter mumblecore cardigan skateboard Austin.",
+        quotations.extract_from_plain(msg_body))
 
 
 def test_short_quotation_with_newline():
@@ -343,6 +348,7 @@ Subject: The manager has commented on your Loop
 Blah-blah-blah
 """))
 
+
 def test_german_from_block():
     eq_('Allo! Follow up MIME!', quotations.extract_from_plain(
     """Allo! Follow up MIME!
@@ -355,6 +361,7 @@ Betreff: The manager has commented on your Loop
 Blah-blah-blah
 """))
 
+
 def test_french_multiline_from_block():
     eq_('Lorem ipsum', quotations.extract_from_plain(
     u"""Lorem ipsum
@@ -367,6 +374,7 @@ Objet : Follow Up
 Blah-blah-blah
 """))
 
+
 def test_french_from_block():
     eq_('Lorem ipsum', quotations.extract_from_plain(
     u"""Lorem ipsum
@@ -375,6 +383,7 @@ Le 23 janv. 2015 à 22:03, Brendan xxx <brendan.xxx@xxx.com<mailto:brendan.xxx@x
 
 Bonjour!"""))
 
+
 def test_polish_from_block():
     eq_('Lorem ipsum', quotations.extract_from_plain(
     u"""Lorem ipsum
@@ -385,6 +394,7 @@ napisał:
 Blah!
 """))
 
+
 def test_danish_from_block():
     eq_('Allo! Follow up MIME!', quotations.extract_from_plain(
     """Allo! Follow up MIME!
@@ -397,6 +407,7 @@ Emne: The manager has commented on your Loop
 Blah-blah-blah
 """))
 
+
 def test_swedish_from_block():
     eq_('Allo! Follow up MIME!', quotations.extract_from_plain(
     u"""Allo! Follow up MIME!
@@ -408,6 +419,7 @@ Till: Isacson Leiff
 Blah-blah-blah
 """))
 
+
 def test_swedish_from_line():
     eq_('Lorem', quotations.extract_from_plain(
     """Lorem
@@ -416,6 +428,7 @@ Den 14 september, 2015 02:23:18, Valentino Rudy (valentino@rudy.be) skrev:
 Veniam laborum mlkshk kale chips authentic. Normcore mumblecore laboris, fanny pack readymade eu blog chia pop-up freegan enim master cleanse.
 """))
 
+
 def test_norwegian_from_line():
     eq_('Lorem', quotations.extract_from_plain(
     u"""Lorem
@@ -424,6 +437,7 @@ På 14 september 2015 på 02:23:18, Valentino Rudy (valentino@rudy.be) skrev:
 Veniam laborum mlkshk kale chips authentic. Normcore mumblecore laboris, fanny pack readymade eu blog chia pop-up freegan enim master cleanse.
 """))
 
+
 def test_dutch_from_block():
     eq_('Gluten-free culpa lo-fi et nesciunt nostrud.', quotations.extract_from_plain(
     """Gluten-free culpa lo-fi et nesciunt nostrud.
@@ -433,6 +447,7 @@ Op 17-feb.-2015, om 13:18 heeft Julius Caesar <pantheon@rome.com> het volgende g
 Small batch beard laboris tempor, non listicle hella Tumblr heirloom.
 """))
 
+
 def test_vietnamese_from_block():
     eq_('Hello', quotations.extract_from_plain(
     u"""Hello
@@ -442,6 +457,7 @@ Vào 14:24 8 tháng 6, 2017, Hùng Nguyễn <hungnguyen@xxx.com> đã viết:
 > Xin chào
 """))
 
+
 def test_quotation_marker_false_positive():
     msg_body = """Visit us now for assistance...
 >>> >>>  http://www.domain.com <<<

tests/utils_test.py

@@ -2,12 +2,13 @@
 
 from __future__ import absolute_import
 
+
+from nose.tools import eq_, ok_, assert_false
+from talon import utils as u
+from mock import patch, Mock
 import cchardet
 import six
 
-from talon import utils as u
-from . import *
-
 
 def test_get_delimiter():
     eq_('\r\n', u.get_delimiter('abc\r\n123'))