From 086f5ba43b1d657ae548b3607d43e2ed02bf89c2 Mon Sep 17 00:00:00 2001
From: Yacine Filali <yfilali@hotmail.com>
Date: Tue, 23 May 2017 15:39:50 -0700
Subject: [PATCH] Updated talon for Python 3

---
 .gitignore                             |   4 ++-
 talon/signature/bruteforce.py          |  10 +++---
 talon/signature/data/classifier        | Bin 608 -> 505 bytes
 talon/signature/extraction.py          |  16 ++++-----
 talon/signature/learning/classifier.py |  11 +++++-
 talon/signature/learning/dataset.py    |   9 ++---
 talon/utils.py                         |  28 ++++++++-------
 tests/html_quotations_test.py          |   9 +++--
 tests/signature/extraction_test.py     |   2 +-
 tests/utils_test.py                    |  47 +++++++++++++------------
 10 files changed, 75 insertions(+), 61 deletions(-)

diff --git a/.gitignore b/.gitignore
index 002f03e..d1a3778 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,6 +39,8 @@ nosetests.xml
 /.emacs.desktop
 /.emacs.desktop.lock
 .elc
+.idea
+.cache
 auto-save-list
 tramp
 .\#*
@@ -51,4 +53,4 @@ tramp
 _trial_temp
 
 # OSX
-.DS_Store
\ No newline at end of file
+.DS_Store
diff --git a/talon/signature/bruteforce.py b/talon/signature/bruteforce.py
index 7f666bd..85e25ce 100644
--- a/talon/signature/bruteforce.py
+++ b/talon/signature/bruteforce.py
@@ -1,15 +1,15 @@
 from __future__ import absolute_import
+
 import logging
 
 import regex as re
 
-from talon.utils import get_delimiter
 from talon.signature.constants import (SIGNATURE_MAX_LINES,
                                        TOO_LONG_SIGNATURE_LINE)
+from talon.utils import get_delimiter
 
 log = logging.getLogger(__name__)
 
-
 # regex to fetch signature based on common signature words
 RE_SIGNATURE = re.compile(r'''
                (
@@ -28,7 +28,6 @@ RE_SIGNATURE = re.compile(r'''
                )
                ''', re.I | re.X | re.M | re.S)
 
-
 # signatures appended by phone email clients
 RE_PHONE_SIGNATURE = re.compile(r'''
                (
@@ -45,7 +44,6 @@ RE_PHONE_SIGNATURE = re.compile(r'''
                )
                ''', re.I | re.X | re.M | re.S)
 
-
 # see _mark_candidate_indexes() for details
 # c - could be signature line
 # d - line starts with dashes (could be signature or list item)
@@ -163,7 +161,7 @@ def _mark_candidate_indexes(lines, candidate):
     'cdc'
     """
     # at first consider everything to be potential signature lines
-    markers = bytearray('c'*len(candidate))
+    markers = list('c' * len(candidate))
 
     # mark lines starting from bottom up
     for i, line_idx in reversed(list(enumerate(candidate))):
@@ -174,7 +172,7 @@ def _mark_candidate_indexes(lines, candidate):
             if line.startswith('-') and line.strip("-"):
                 markers[i] = 'd'
 
-    return markers
+    return "".join(markers)
 
 
 def _process_marked_candidate_indexes(candidate, markers):
diff --git a/talon/signature/data/classifier b/talon/signature/data/classifier
index 1c3a4b0865f3e951b1b3b17fb31bacc48d8d005b..5caaf86e63a0e614ce42c48aa82942589a51890d 100644
GIT binary patch
literal 505
zcmV<V0S5kfUUgDUZ`3dl&33mRWLs$Yrj(CDfh|z0TM*)a1Za^ek&;7^sBe~&^)?2_
z$=J@)J*)&5=&$0(a6;loz{I;$LI@wM$;`Z&_nz~qnc5FZaHEMG<|I|zT4BRIsWJEV
zT`z=S^D{KYFw>!}AP7`pEi~y8h88MrM4x3+9|~yE2IhJ&yZd4T13YvAujxPrb0>Sm
zkrPI9Wy$*@Q8FRA%F7WeWr{>tTI3~nVf^Z)F?@7jcv%V)!aTBA7{jTKpK7bb4~8)H
z;86E@-5x^Q*9TGw&2s^Zbmb2TIgrAz?Jm*f2$rgZ&SPaLOj1|@r=!4kN{!r7Q95Q*
zu22}9i5gybzv2p(kG=ANbnK)PR*wD7oF6fXXgK@kN7E12{+q$M2}zO`Mv%NxPEN?y
zD>!O9KNZn;qf&%$fzH%S9>c{bh#Ftv(mq`7L{0DSGq@7R@#|lB{1pWbR+C=qTQdmY
z>K9nshiiU!iTafr!F4}O?Z6G%u1VJ0*~Gh>6hT2TG_Xz=F*02Di=0`<oq$`huew6P
z?cL|APrm<L>uu55+RQ=>7;I$qi`*H*-A)!}?d)Hpcf8T}(CF|lW1-`rn`5~DKj<S5
veSpxKmuJg93VE10co@@V%(3ZD%kTlMEHrG<IZSVp?o-kQQYGX!lONuT3}pJS

literal 608
zcmZut%Z}496iu5B4QUIMH^cBApu;1A0kJ{?RHTszlf|G5OUp^!iNSGlZKq5Ym0$z&
zReTI9Bt8OMchm*qE-d-#yzcqZ&h#h2sFIPM6;URr)=Y<QxwN}?@5do}TVJs?$My(1
zqSP9F7-+1iViS^cxzDfztue0cyxjEMKyV~N@YqV59T=u06*~G@rh=!Dtc$8470;}B
zn7n?a6m32zT2)Mi*tM3y9FLD6&~@tMFw7h!loPtZhS-B($_10OVAzMDL-8qRiadH8
z$*RFYokDlUR9b3=Ct?qVT-p>Fs|?W@6*fGcj9bu|)KuW9Ll{iCAu9|H4<RV%0pXT}
zr+0t((3S2(@XRtyl$DA_MJ)_phFZu{d%)SFuoKf%EW~rr=1=i_?8U9`cwrAOj$$8L
z&+$@{ByaxMaZl_STu-6%!}mPA{0%qu@CtM*d!aB5UWK;U#%s`Xc4S$~b@0oX!W+<6
zR8Go*XhRLdn+f=)6nJar#rm_Kzc<DYVJ$1!l;CE*==Sy;?~L*=@8$m#x9y6%Yl{n4
hJrEo@($*aB{g?E_k?vbk-=!j-8Q;$hK1k{``U^Y((T)HB

diff --git a/talon/signature/extraction.py b/talon/signature/extraction.py
index 3259171..fb88b32 100644
--- a/talon/signature/extraction.py
+++ b/talon/signature/extraction.py
@@ -1,16 +1,15 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
+
 import logging
 
-import regex as re
 import numpy
-
-from talon.signature.learning.featurespace import features, build_pattern
-from talon.utils import get_delimiter
+import regex as re
 from talon.signature.bruteforce import get_signature_candidate
+from talon.signature.learning.featurespace import features, build_pattern
 from talon.signature.learning.helpers import has_signature
-
+from talon.utils import get_delimiter
 
 log = logging.getLogger(__name__)
 
@@ -58,7 +57,7 @@ def extract(body, sender):
                 text = delimiter.join(text)
                 if text.strip():
                     return (text, delimiter.join(signature))
-    except Exception:
+    except Exception as e:
         log.exception('ERROR when extracting signature with classifiers')
 
     return (body, None)
@@ -81,7 +80,7 @@ def _mark_lines(lines, sender):
     candidate = get_signature_candidate(lines)
 
     # at first consider everything to be text no signature
-    markers = bytearray('t'*len(lines))
+    markers = list('t' * len(lines))
 
     # mark lines starting from bottom up
     # mark only lines that belong to candidate
@@ -96,7 +95,7 @@ def _mark_lines(lines, sender):
         elif is_signature_line(line, sender, EXTRACTOR):
             markers[j] = 's'
 
-    return markers
+    return "".join(markers)
 
 
 def _process_marked_lines(lines, markers):
@@ -111,3 +110,4 @@ def _process_marked_lines(lines, markers):
         return (lines[:-signature.end()], lines[-signature.end():])
 
     return (lines, None)
+
diff --git a/talon/signature/learning/classifier.py b/talon/signature/learning/classifier.py
index 8ec3228..ce062bc 100644
--- a/talon/signature/learning/classifier.py
+++ b/talon/signature/learning/classifier.py
@@ -6,6 +6,9 @@ body belongs to the signature.
 """
 
 from __future__ import absolute_import
+
+import pickle
+
 from numpy import genfromtxt
 from sklearn.svm import LinearSVC
 from sklearn.externals import joblib
@@ -29,4 +32,10 @@ def train(classifier, train_data_filename, save_classifier_filename=None):
 
 def load(saved_classifier_filename, train_data_filename):
     """Loads saved classifier. """
-    return joblib.load(saved_classifier_filename)
+    try:
+        return joblib.load(saved_classifier_filename)
+    except ValueError:
+        loaded = pickle.load(open(saved_classifier_filename, 'rb'), encoding='latin1')
+        joblib.dump(loaded, saved_classifier_filename, compress=True)
+        return loaded
+
diff --git a/talon/signature/learning/dataset.py b/talon/signature/learning/dataset.py
index 308995b..fbb37db 100644
--- a/talon/signature/learning/dataset.py
+++ b/talon/signature/learning/dataset.py
@@ -17,13 +17,14 @@ suffix which should be `_sender`.
 """
 
 from __future__ import absolute_import
+
 import os
+
 import regex as re
+from six.moves import range
 
 from talon.signature.constants import SIGNATURE_MAX_LINES
 from talon.signature.learning.featurespace import build_pattern, features
-from six.moves import range
-
 
 SENDER_SUFFIX = '_sender'
 BODY_SUFFIX = '_body'
@@ -59,7 +60,7 @@ def parse_msg_sender(filename, sender_known=True):
     """
     sender, msg = None, None
     if os.path.isfile(filename) and not is_sender_filename(filename):
-        with open(filename) as f:
+        with open(filename, encoding='utf-8') as f:
             msg = f.read()
             sender = u''
             if sender_known:
@@ -147,7 +148,7 @@ def build_extraction_dataset(folder, dataset_filename,
                 continue
             lines = msg.splitlines()
             for i in range(1, min(SIGNATURE_MAX_LINES,
-                                   len(lines)) + 1):
+                                  len(lines)) + 1):
                 line = lines[-i]
                 label = -1
                 if line[:len(SIGNATURE_ANNOTATION)] == \
diff --git a/talon/utils.py b/talon/utils.py
index e6c884b..34a21c6 100644
--- a/talon/utils.py
+++ b/talon/utils.py
@@ -1,19 +1,18 @@
 # coding:utf-8
 
 from __future__ import absolute_import
-import logging
+
 from random import shuffle
-import chardet
+
 import cchardet
-import regex as re
-
-from lxml.html import html5parser
-from lxml.cssselect import CSSSelector
-
+import chardet
 import html5lib
+import regex as re
+import six
+from lxml.cssselect import CSSSelector
+from lxml.html import html5parser
 
 from talon.constants import RE_DELIMITER
-import six
 
 
 def safe_format(format_string, *args, **kwargs):
@@ -128,7 +127,7 @@ def html_tree_to_text(tree):
 
         parent.remove(c)
 
-    text   = ""
+    text = ""
     for el in tree.iter():
         el_text = (el.text or '') + (el.tail or '')
         if len(el_text) > 1:
@@ -177,6 +176,8 @@ def html_to_text(string):
 def html_fromstring(s):
     """Parse html tree from string. Return None if the string can't be parsed.
     """
+    if isinstance(s, six.text_type):
+        s = s.encode('utf8')
     try:
         if html_too_big(s):
             return None
@@ -189,6 +190,8 @@ def html_fromstring(s):
 def html_document_fromstring(s):
     """Parse html tree from string. Return None if the string can't be parsed.
     """
+    if isinstance(s, six.text_type):
+        s = s.encode('utf8')
     try:
         if html_too_big(s):
             return None
@@ -203,7 +206,9 @@ def cssselect(expr, tree):
 
 
 def html_too_big(s):
-    return s.count('<') > _MAX_TAGS_COUNT
+    if isinstance(s, six.text_type):
+        s = s.encode('utf8')
+    return s.count(b'<') > _MAX_TAGS_COUNT
 
 
 def _contains_charset_spec(s):
@@ -248,8 +253,7 @@ def _html5lib_parser():
 _UTF8_DECLARATION = (b'<meta http-equiv="Content-Type" content="text/html;'
                      b'charset=utf-8">')
 
-
-_BLOCKTAGS  = ['div', 'p', 'ul', 'li', 'h1', 'h2', 'h3']
+_BLOCKTAGS = ['div', 'p', 'ul', 'li', 'h1', 'h2', 'h3']
 _HARDBREAKS = ['br', 'hr', 'tr']
 
 _RE_EXCESSIVE_NEWLINES = re.compile("\n{2,10}")
diff --git a/tests/html_quotations_test.py b/tests/html_quotations_test.py
index b78409b..7e3591f 100644
--- a/tests/html_quotations_test.py
+++ b/tests/html_quotations_test.py
@@ -1,13 +1,12 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
-from . import *
-from . fixtures import *
 
 import regex as re
 
 from talon import quotations, utils as u
-
+from . import *
+from .fixtures import *
 
 RE_WHITESPACE = re.compile("\s")
 RE_DOUBLE_WHITESPACE = re.compile("\s")
@@ -303,7 +302,7 @@ Reply
 
 
 def extract_reply_and_check(filename):
-    f = open(filename)
+    f = open(filename, encoding='utf8')
 
     msg_body = f.read()
     reply = quotations.extract_from_html(msg_body)
@@ -373,7 +372,7 @@ reply
 </blockquote>"""
     msg_body = msg_body.replace('\n', '\r\n')
     extracted = quotations.extract_from_html(msg_body)
-    assert_false(symbol in extracted)    
+    assert_false(symbol in extracted)
     # Keep new lines otherwise "My reply" becomes one word - "Myreply" 
     eq_("<html><head></head><body>My\nreply\n</body></html>", extracted)
 
diff --git a/tests/signature/extraction_test.py b/tests/signature/extraction_test.py
index b570303..0bce56d 100644
--- a/tests/signature/extraction_test.py
+++ b/tests/signature/extraction_test.py
@@ -31,7 +31,7 @@ def test_messages_longer_SIGNATURE_MAX_LINES():
         sender, body = dataset.parse_msg_sender(filename)
         text, extracted_signature = signature.extract(body, sender)
         extracted_signature = extracted_signature or ''
-        with open(filename[:-len('body')] + 'signature') as ms:
+        with open(filename[:-len('body')] + 'signature', encoding='utf8') as ms:
             msg_signature = ms.read()
             eq_(msg_signature.strip(), extracted_signature.strip())
             stripped_msg = body.strip()[:len(body.strip())-len(msg_signature)]
diff --git a/tests/utils_test.py b/tests/utils_test.py
index 8ddebdc..7ba4b52 100644
--- a/tests/utils_test.py
+++ b/tests/utils_test.py
@@ -1,12 +1,12 @@
 # coding:utf-8
 
 from __future__ import absolute_import
-from . import *
 
-from talon import utils as u
 import cchardet
 import six
-from lxml import html
+
+from talon import utils as u
+from . import *
 
 
 def test_get_delimiter():
@@ -16,35 +16,35 @@ def test_get_delimiter():
 
 
 def test_unicode():
-    eq_ (u'hi', u.to_unicode('hi'))
-    eq_ (type(u.to_unicode('hi')), six.text_type )
-    eq_ (type(u.to_unicode(u'hi')), six.text_type )
-    eq_ (type(u.to_unicode('привет')), six.text_type )
-    eq_ (type(u.to_unicode(u'привет')), six.text_type )
-    eq_ (u"привет", u.to_unicode('привет'))
-    eq_ (u"привет", u.to_unicode(u'привет'))
+    eq_(u'hi', u.to_unicode('hi'))
+    eq_(type(u.to_unicode('hi')), six.text_type)
+    eq_(type(u.to_unicode(u'hi')), six.text_type)
+    eq_(type(u.to_unicode('привет')), six.text_type)
+    eq_(type(u.to_unicode(u'привет')), six.text_type)
+    eq_(u"привет", u.to_unicode('привет'))
+    eq_(u"привет", u.to_unicode(u'привет'))
     # some latin1 stuff
-    eq_ (u"Versión", u.to_unicode(u'Versi\xf3n'.encode('iso-8859-2'), precise=True))
+    eq_(u"Versión", u.to_unicode(u'Versi\xf3n'.encode('iso-8859-2'), precise=True))
 
 
 def test_detect_encoding():
-    eq_ ('ascii', u.detect_encoding(b'qwe').lower())
-    ok_ (u.detect_encoding(
+    eq_('ascii', u.detect_encoding(b'qwe').lower())
+    ok_(u.detect_encoding(
         u'Versi\xf3n'.encode('iso-8859-2')).lower() in [
             'iso-8859-1', 'iso-8859-2'])
-    eq_ ('utf-8', u.detect_encoding(u'привет'.encode('utf8')).lower())
+    eq_('utf-8', u.detect_encoding(u'привет'.encode('utf8')).lower())
     # fallback to utf-8
     with patch.object(u.chardet, 'detect') as detect:
         detect.side_effect = Exception
-        eq_ ('utf-8', u.detect_encoding('qwe'.encode('utf8')).lower())
+        eq_('utf-8', u.detect_encoding('qwe'.encode('utf8')).lower())
 
 
 def test_quick_detect_encoding():
-    eq_ ('ascii', u.quick_detect_encoding(b'qwe').lower())
-    ok_ (u.quick_detect_encoding(
+    eq_('ascii', u.quick_detect_encoding(b'qwe').lower())
+    ok_(u.quick_detect_encoding(
         u'Versi\xf3n'.encode('windows-1252')).lower() in [
             'windows-1252', 'windows-1250'])
-    eq_ ('utf-8', u.quick_detect_encoding(u'привет'.encode('utf8')).lower())
+    eq_('utf-8', u.quick_detect_encoding(u'привет'.encode('utf8')).lower())
 
 
 @patch.object(cchardet, 'detect')
@@ -84,7 +84,7 @@ Haha
     eq_(u"привет!", u.html_to_text("<b>привет!</b>").decode('utf8'))
 
     html = '<body><br/><br/>Hi</body>'
-    eq_ (b'Hi', u.html_to_text(html))
+    eq_(b'Hi', u.html_to_text(html))
 
     html = """Hi
 <style type="text/css">
@@ -104,7 +104,7 @@ font: 13px 'Lucida Grande', Arial, sans-serif;
 
 }
 </style>"""
-    eq_ (b'Hi', u.html_to_text(html))
+    eq_(b'Hi', u.html_to_text(html))
 
     html = """<div>
 <!-- COMMENT 1 -->
@@ -115,15 +115,16 @@ font: 13px 'Lucida Grande', Arial, sans-serif;
 
 
 def test_comment_no_parent():
-    s = "<!-- COMMENT 1 --> no comment"
+    s = b'<!-- COMMENT 1 --> no comment'
     d = u.html_document_fromstring(s)
-    eq_("no comment", u.html_tree_to_text(d))
+    eq_(b"no comment", u.html_tree_to_text(d))
 
 
 @patch.object(u.html5parser, 'fromstring', Mock(side_effect=Exception()))
 def test_html_fromstring_exception():
     eq_(None, u.html_fromstring("<html></html>"))
 
+
 @patch.object(u, 'html_too_big', Mock())
 @patch.object(u.html5parser, 'fromstring')
 def test_html_fromstring_too_big(fromstring):
@@ -158,5 +159,5 @@ def test_html_too_big():
 
 @patch.object(u, '_MAX_TAGS_COUNT', 3)
 def test_html_to_text():
-    eq_("Hello", u.html_to_text("<div>Hello</div>"))
+    eq_(b"Hello", u.html_to_text("<div>Hello</div>"))
     eq_(None, u.html_to_text("<div><span>Hi</span></div>"))