Merge pull request #79 from mailgun/sergey/version

bump version
2016-02-29 15:13:51 -08:00 · 2016-02-29 14:50:52 -08:00 · 2016-02-29 14:14:09 -08:00 · 2016-02-21 16:56:52 +01:00 · 2016-02-19 19:08:37 -08:00 · 2016-02-19 19:07:10 -08:00
5 changed files with 41 additions and 4 deletions
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages


 setup(name='talon',
-      version='1.2.1',
+      version='1.2.4',
      description=("Mailgun library "
                   "to extract message quotations and signatures."),
      long_description=open("README.rst").read(),
--- a/talon/html_quotations.py
+++ b/talon/html_quotations.py
@@ -12,6 +12,7 @@ CHECKPOINT_PATTERN = re.compile(CHECKPOINT_PREFIX + '\d+' + CHECKPOINT_SUFFIX)

 # HTML quote indicators (tag ids)
 QUOTE_IDS = ['OLK_SRC_BODY_SECTION']
+RE_FWD = re.compile("^[-]+[ ]*Forwarded message[ ]*[-]+$", re.I | re.M)


 def add_checkpoint(html_note, counter):
@@ -77,7 +78,7 @@ def delete_quotation_tags(html_note, counter, quotation_checkpoints):
 def cut_gmail_quote(html_message):
    ''' Cuts the outermost block element with class gmail_quote. '''
    gmail_quote = html_message.cssselect('div.gmail_quote')
-    if gmail_quote:
+    if gmail_quote and not RE_FWD.match(gmail_quote[0].text):
        gmail_quote[0].getparent().remove(gmail_quote[0])
        return True

@@ -172,6 +173,7 @@ def cut_from_block(html_message):
            parent_div_is_all_content = (
                maybe_body is not None and maybe_body.tag == 'body' and
                len(maybe_body.getchildren()) == 1)
+
            if not parent_div_is_all_content:
                block.getparent().remove(block)
                return True
@@ -185,7 +187,17 @@ def cut_from_block(html_message):
         "//*[starts-with(mg:tail(), 'Date:')]"))
    if block:
        block = block[0]
+
+        if RE_FWD.match(block.getparent().text or ''):
+            return False
+        
        while(block.getnext() is not None):
            block.getparent().remove(block.getnext())
        block.getparent().remove(block)
        return True
+
+def cut_zimbra_quote(html_message):
+    zDivider = html_message.xpath('//hr[@data-marker="__DIVIDER__"]')
+    if zDivider:
+        zDivider[0].getparent().remove(zDivider[0])
+        return True
--- a/talon/quotations.py
+++ b/talon/quotations.py
@@ -137,11 +137,16 @@ RE_FROM_COLON_OR_DATE_COLON = re.compile(u'(_+\r?\n)?[\s]*(:?[*]?{})[\s]?:[*]? .

 SPLITTER_PATTERNS = [
    RE_ORIGINAL_MESSAGE,
-    # <date> <person>
-    re.compile("(\d+/\d+/\d+|\d+\.\d+\.\d+).*@", re.S),
    RE_ON_DATE_SMB_WROTE,
    RE_ON_DATE_WROTE_SMB,
    RE_FROM_COLON_OR_DATE_COLON,
+    # 02.04.2012 14:20 пользователь "bob@example.com" <
+    # bob@xxx.mailgun.org> написал:
+    re.compile("(\d+/\d+/\d+|\d+\.\d+\.\d+).*@", re.S),
+    # 2014-10-17 11:28 GMT+03:00 Bob <
+    # bob@example.com>:
+    re.compile("\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+GMT.*@", re.S),
+    # Thu, 26 Jun 2014 14:00:51 +0400 Bob <bob@example.com>:
    re.compile('\S{3,10}, \d\d? \S{3,10} 20\d\d,? \d\d?:\d\d(:\d\d)?'
               '( \S+){3,6}@\S+:')
    ]
@@ -345,6 +350,7 @@ def extract_from_html(msg_body):
        parser=html.HTMLParser(encoding="utf-8")
    )
    cut_quotations = (html_quotations.cut_gmail_quote(html_tree) or
+                      html_quotations.cut_zimbra_quote(html_tree) or
                      html_quotations.cut_blockquote(html_tree) or
                      html_quotations.cut_microsoft_quote(html_tree) or
                      html_quotations.cut_by_id(html_tree) or
--- a/tests/html_quotations_test.py
+++ b/tests/html_quotations_test.py
@@ -340,3 +340,10 @@ def test_CRLF():
    assert_false(symbol in extracted)    
    eq_("<html><body><p>Reply</p></body></html>",
        RE_WHITESPACE.sub('', extracted))
+
+
+def test_gmail_forwarded_msg():
+    msg_body = """<div dir="ltr"><br><div class="gmail_quote">---------- Forwarded message ----------<br>From: <b class="gmail_sendername">Bob</b> <span dir="ltr">&lt;<a href="mailto:bob@example.com">bob@example.com</a>&gt;</span><br>Date: Fri, Feb 11, 2010 at 5:59 PM<br>Subject: Bob WFH today<br>To: Mary &lt;<a href="mailto:mary@example.com">mary@example.com</a>&gt;<br><br><br><div dir="ltr">eom</div>
+</div><br></div>"""
+    extracted = quotations.extract_from_html(msg_body)
+    eq_(RE_WHITESPACE.sub('', msg_body), RE_WHITESPACE.sub('', extracted))
--- a/tests/text_quotations_test.py
+++ b/tests/text_quotations_test.py
@@ -54,6 +54,18 @@ On 04/19/2011 07:10 AM, Roman Tkachenko wrote:
    eq_("Test reply", quotations.extract_from_plain(msg_body))


+def test_date_time_email_splitter():
+    msg_body = """Test reply
+
+2014-10-17 11:28 GMT+03:00 Postmaster <
+postmaster@sandboxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.mailgun.org>:
+
+> First from site
+>
+    """
+    eq_("Test reply", quotations.extract_from_plain(msg_body))
+
+
 def test_pattern_on_date_somebody_wrote_allows_space_in_front():
    msg_body = """Thanks Thanmai
 On Mar 8, 2012 9:59 AM, "Example.com" <
Author	SHA1	Message	Date
Sergey Obukhov	3497b5cab4	Merge pull request #79 from mailgun/sergey/version bump version	2016-02-29 15:13:51 -08:00
Sergey Obukhov	9c17dca17c	bump version	2016-02-29 14:50:52 -08:00
Sergey Obukhov	de342d3177	Merge pull request #78 from defkev/master Added Zimbra HTML quotation extraction	2016-02-29 14:14:09 -08:00
defkev	743b452daf	Added Zimbra HTML quotation extraction	2016-02-21 16:56:52 +01:00
Sergey Obukhov	c762f3c337	Merge pull request #77 from mailgun/sergey/fix-gmail-fwd fixes mailgun/talon#18	2016-02-19 19:08:37 -08:00
Sergey Obukhov	31803d41bc	fixes mailgun/talon#18	2016-02-19 19:07:10 -08:00
Sergey Obukhov	2ecd9779fc	bump up version	2016-02-19 18:32:07 -08:00
Sergey Obukhov	5a7047233e	Merge pull request #76 from mailgun/sergey/fix-date-splitter fixes mailgun/talon#19	2016-02-19 18:28:23 -08:00
Sergey Obukhov	999e9c3725	fixes mailgun/talon#19	2016-02-19 17:53:52 -08:00