Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
2ecd9779fc | ||
|
|
5a7047233e | ||
|
|
999e9c3725 | ||
|
|
f6940fe878 | ||
|
|
ce65ff8fc8 | ||
|
f688d074b5 |
2
setup.py
2
setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
|
|||||||
|
|
||||||
|
|
||||||
setup(name='talon',
|
setup(name='talon',
|
||||||
version='1.2.0',
|
version='1.2.2',
|
||||||
description=("Mailgun library "
|
description=("Mailgun library "
|
||||||
"to extract message quotations and signatures."),
|
"to extract message quotations and signatures."),
|
||||||
long_description=open("README.rst").read(),
|
long_description=open("README.rst").read(),
|
||||||
|
|||||||
@@ -159,21 +159,33 @@ def cut_from_block(html_message):
|
|||||||
|
|
||||||
if block:
|
if block:
|
||||||
block = block[-1]
|
block = block[-1]
|
||||||
|
parent_div = None
|
||||||
while block.getparent() is not None:
|
while block.getparent() is not None:
|
||||||
if block.tag == 'div':
|
if block.tag == 'div':
|
||||||
|
parent_div = block
|
||||||
|
break
|
||||||
|
block = block.getparent()
|
||||||
|
if parent_div is not None:
|
||||||
|
maybe_body = parent_div.getparent()
|
||||||
|
# In cases where removing this enclosing div will remove all
|
||||||
|
# content, we should assume the quote is not enclosed in a tag.
|
||||||
|
parent_div_is_all_content = (
|
||||||
|
maybe_body is not None and maybe_body.tag == 'body' and
|
||||||
|
len(maybe_body.getchildren()) == 1)
|
||||||
|
if not parent_div_is_all_content:
|
||||||
block.getparent().remove(block)
|
block.getparent().remove(block)
|
||||||
return True
|
return True
|
||||||
else:
|
else:
|
||||||
block = block.getparent()
|
return False
|
||||||
else:
|
|
||||||
# handle the case when From: block goes right after e.g. <hr>
|
# handle the case when From: block goes right after e.g. <hr>
|
||||||
# and not enclosed in some tag
|
# and not enclosed in some tag
|
||||||
block = html_message.xpath(
|
block = html_message.xpath(
|
||||||
("//*[starts-with(mg:tail(), 'From:')]|"
|
("//*[starts-with(mg:tail(), 'From:')]|"
|
||||||
"//*[starts-with(mg:tail(), 'Date:')]"))
|
"//*[starts-with(mg:tail(), 'Date:')]"))
|
||||||
if block:
|
if block:
|
||||||
block = block[0]
|
block = block[0]
|
||||||
while(block.getnext() is not None):
|
while(block.getnext() is not None):
|
||||||
block.getparent().remove(block.getnext())
|
block.getparent().remove(block.getnext())
|
||||||
block.getparent().remove(block)
|
block.getparent().remove(block)
|
||||||
return True
|
return True
|
||||||
|
|||||||
@@ -137,11 +137,16 @@ RE_FROM_COLON_OR_DATE_COLON = re.compile(u'(_+\r?\n)?[\s]*(:?[*]?{})[\s]?:[*]? .
|
|||||||
|
|
||||||
SPLITTER_PATTERNS = [
|
SPLITTER_PATTERNS = [
|
||||||
RE_ORIGINAL_MESSAGE,
|
RE_ORIGINAL_MESSAGE,
|
||||||
# <date> <person>
|
|
||||||
re.compile("(\d+/\d+/\d+|\d+\.\d+\.\d+).*@", re.S),
|
|
||||||
RE_ON_DATE_SMB_WROTE,
|
RE_ON_DATE_SMB_WROTE,
|
||||||
RE_ON_DATE_WROTE_SMB,
|
RE_ON_DATE_WROTE_SMB,
|
||||||
RE_FROM_COLON_OR_DATE_COLON,
|
RE_FROM_COLON_OR_DATE_COLON,
|
||||||
|
# 02.04.2012 14:20 пользователь "bob@example.com" <
|
||||||
|
# bob@xxx.mailgun.org> написал:
|
||||||
|
re.compile("(\d+/\d+/\d+|\d+\.\d+\.\d+).*@", re.S),
|
||||||
|
# 2014-10-17 11:28 GMT+03:00 Bob <
|
||||||
|
# bob@example.com>:
|
||||||
|
re.compile("\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+GMT.*@", re.S),
|
||||||
|
# Thu, 26 Jun 2014 14:00:51 +0400 Bob <bob@example.com>:
|
||||||
re.compile('\S{3,10}, \d\d? \S{3,10} 20\d\d,? \d\d?:\d\d(:\d\d)?'
|
re.compile('\S{3,10}, \d\d? \S{3,10} 20\d\d,? \d\d?:\d\d(:\d\d)?'
|
||||||
'( \S+){3,6}@\S+:')
|
'( \S+){3,6}@\S+:')
|
||||||
]
|
]
|
||||||
|
|||||||
87
tests/fixtures/html_replies/ms_outlook_2010.html
vendored
Normal file
87
tests/fixtures/html_replies/ms_outlook_2010.html
vendored
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta http-equiv="Content-Type" content="text/html; charset=iso-2022-jp">
|
||||||
|
<meta name="Generator" content="Microsoft Word 14 (filtered medium)">
|
||||||
|
<style><!--
|
||||||
|
/* Font Definitions */
|
||||||
|
@font-face
|
||||||
|
{font-family:Calibri;
|
||||||
|
panose-1:2 15 5 2 2 2 4 3 2 4;}
|
||||||
|
@font-face
|
||||||
|
{font-family:Tahoma;
|
||||||
|
panose-1:2 11 6 4 3 5 4 4 2 4;}
|
||||||
|
/* Style Definitions */
|
||||||
|
p.MsoNormal, li.MsoNormal, div.MsoNormal
|
||||||
|
{margin:0in;
|
||||||
|
margin-bottom:.0001pt;
|
||||||
|
font-size:12.0pt;
|
||||||
|
font-family:"Times New Roman","serif";}
|
||||||
|
h3
|
||||||
|
{mso-style-priority:9;
|
||||||
|
mso-style-link:"Heading 3 Char";
|
||||||
|
mso-margin-top-alt:auto;
|
||||||
|
margin-right:0in;
|
||||||
|
mso-margin-bottom-alt:auto;
|
||||||
|
margin-left:0in;
|
||||||
|
font-size:13.5pt;
|
||||||
|
font-family:"Times New Roman","serif";
|
||||||
|
font-weight:bold;}
|
||||||
|
a:link, span.MsoHyperlink
|
||||||
|
{mso-style-priority:99;
|
||||||
|
color:blue;
|
||||||
|
text-decoration:underline;}
|
||||||
|
a:visited, span.MsoHyperlinkFollowed
|
||||||
|
{mso-style-priority:99;
|
||||||
|
color:purple;
|
||||||
|
text-decoration:underline;}
|
||||||
|
p
|
||||||
|
{mso-style-priority:99;
|
||||||
|
mso-margin-top-alt:auto;
|
||||||
|
margin-right:0in;
|
||||||
|
mso-margin-bottom-alt:auto;
|
||||||
|
margin-left:0in;
|
||||||
|
font-size:12.0pt;
|
||||||
|
font-family:"Times New Roman","serif";}
|
||||||
|
span.Heading3Char
|
||||||
|
{mso-style-name:"Heading 3 Char";
|
||||||
|
mso-style-priority:9;
|
||||||
|
mso-style-link:"Heading 3";
|
||||||
|
font-family:"Cambria","serif";
|
||||||
|
color:#4F81BD;
|
||||||
|
font-weight:bold;}
|
||||||
|
span.EmailStyle19
|
||||||
|
{mso-style-type:personal-reply;
|
||||||
|
font-family:"Calibri","sans-serif";
|
||||||
|
color:#1F497D;}
|
||||||
|
.MsoChpDefault
|
||||||
|
{mso-style-type:export-only;
|
||||||
|
font-family:"Calibri","sans-serif";}
|
||||||
|
@page WordSection1
|
||||||
|
{size:8.5in 11.0in;
|
||||||
|
margin:1.0in 1.0in 1.0in 1.0in;}
|
||||||
|
div.WordSection1
|
||||||
|
{page:WordSection1;}
|
||||||
|
--></style><!--[if gte mso 9]><xml>
|
||||||
|
<o:shapedefaults v:ext="edit" spidmax="1026" />
|
||||||
|
</xml><![endif]--><!--[if gte mso 9]><xml>
|
||||||
|
<o:shapelayout v:ext="edit">
|
||||||
|
<o:idmap v:ext="edit" data="1" />
|
||||||
|
</o:shapelayout></xml><![endif]-->
|
||||||
|
</head>
|
||||||
|
<body lang="EN-US" link="blue" vlink="purple">
|
||||||
|
<div class="WordSection1">
|
||||||
|
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";color:#1F497D">Hi. I am fine.<o:p></o:p></span></p>
|
||||||
|
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";color:#1F497D">Thanks,<o:p></o:p></span></p>
|
||||||
|
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri","sans-serif";color:#1F497D">Alex<o:p></o:p></span></p>
|
||||||
|
<p class="MsoNormal"><b><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif"">From:</span></b><span style="font-size:10.0pt;font-family:"Tahoma","sans-serif""> Foo [mailto:foo@bar.com]
|
||||||
|
<b>On Behalf Of </b>baz@bar.com<br>
|
||||||
|
<b>Sent:</b> Monday, January 01, 2000 12:00 AM<br>
|
||||||
|
<b>To:</b> john@bar.com<br>
|
||||||
|
<b>Cc:</b> jane@bar.io<br>
|
||||||
|
<b>Subject:</b> Conversation<o:p></o:p></span></p>
|
||||||
|
<p class="MsoNormal"><o:p> </o:p></p>
|
||||||
|
<p>Hello! How are you?<o:p></o:p></p>
|
||||||
|
<p class="MsoNormal"><o:p> </o:p></p>
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -299,6 +299,10 @@ def test_ms_outlook_2007_reply():
|
|||||||
extract_reply_and_check("tests/fixtures/html_replies/ms_outlook_2007.html")
|
extract_reply_and_check("tests/fixtures/html_replies/ms_outlook_2007.html")
|
||||||
|
|
||||||
|
|
||||||
|
def test_ms_outlook_2010_reply():
|
||||||
|
extract_reply_and_check("tests/fixtures/html_replies/ms_outlook_2010.html")
|
||||||
|
|
||||||
|
|
||||||
def test_thunderbird_reply():
|
def test_thunderbird_reply():
|
||||||
extract_reply_and_check("tests/fixtures/html_replies/thunderbird.html")
|
extract_reply_and_check("tests/fixtures/html_replies/thunderbird.html")
|
||||||
|
|
||||||
|
|||||||
@@ -54,6 +54,18 @@ On 04/19/2011 07:10 AM, Roman Tkachenko wrote:
|
|||||||
eq_("Test reply", quotations.extract_from_plain(msg_body))
|
eq_("Test reply", quotations.extract_from_plain(msg_body))
|
||||||
|
|
||||||
|
|
||||||
|
def test_date_time_email_splitter():
|
||||||
|
msg_body = """Test reply
|
||||||
|
|
||||||
|
2014-10-17 11:28 GMT+03:00 Postmaster <
|
||||||
|
postmaster@sandboxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.mailgun.org>:
|
||||||
|
|
||||||
|
> First from site
|
||||||
|
>
|
||||||
|
"""
|
||||||
|
eq_("Test reply", quotations.extract_from_plain(msg_body))
|
||||||
|
|
||||||
|
|
||||||
def test_pattern_on_date_somebody_wrote_allows_space_in_front():
|
def test_pattern_on_date_somebody_wrote_allows_space_in_front():
|
||||||
msg_body = """Thanks Thanmai
|
msg_body = """Thanks Thanmai
|
||||||
On Mar 8, 2012 9:59 AM, "Example.com" <
|
On Mar 8, 2012 9:59 AM, "Example.com" <
|
||||||
|
|||||||
Reference in New Issue
Block a user