initial commit

tests/signature/learning/dataset_test.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+
+from ... import *
+import os
+
+from PyML import SparseDataSet
+
+from talon.utils import to_unicode
+from talon.signature.learning import dataset as d
+
+from talon.signature.learning.featurespace import features
+
+
+def test_is_sender_filename():
+    assert_false(d.is_sender_filename("foo/bar"))
+    assert_false(d.is_sender_filename("foo/bar_body"))
+    ok_(d.is_sender_filename("foo/bar_sender"))
+
+
+def test_build_sender_filename():
+    eq_("foo/bar_sender", d.build_sender_filename("foo/bar_body"))
+
+
+def test_parse_msg_sender():
+    sender, msg = d.parse_msg_sender(EML_MSG_FILENAME)
+    # if the message in eml format
+    with open(EML_MSG_FILENAME) as f:
+        eq_(sender,
+            " Alex Q <xxx@yahoo.com>")
+        eq_(msg, f.read())
+
+    # if the message sender is stored in a separate file
+    sender, msg = d.parse_msg_sender(MSG_FILENAME_WITH_BODY_SUFFIX)
+    with open(MSG_FILENAME_WITH_BODY_SUFFIX) as f:
+        eq_(sender, u"john@example.com")
+        eq_(msg, f.read())
+
+
+def test_build_extraction_dataset():
+    if os.path.exists(os.path.join(TMP_DIR, 'extraction.data')):
+        os.remove(os.path.join(TMP_DIR, 'extraction.data'))
+    d.build_extraction_dataset(os.path.join(EMAILS_DIR, 'P'),
+                               os.path.join(TMP_DIR,
+                                            'extraction.data'), 1)
+    test_data = SparseDataSet(os.path.join(TMP_DIR, 'extraction.data'),
+                              labelsColumn=-1)
+    # the result is a loadable signature extraction dataset
+    # 32 comes from 3 emails in emails/P folder, 11 lines checked to be
+    # a signature, one email has only 10 lines
+    eq_(test_data.size(), 32)
+    eq_(len(features('')), test_data.numFeatures)

tests/signature/learning/featurespace_test.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+from ... import *
+
+from talon.signature.learning import featurespace as fs
+
+
+def test_apply_features():
+    s = '''John Doe
+
+VP Research and Development, Xxxx Xxxx Xxxxx
+
+555-226-2345
+
+john@example.com'''
+    sender = 'John <john@example.com>'
+    features = fs.features(sender)
+    result = fs.apply_features(s, features)
+    # note that we don't consider the first line because signatures don't
+    # usually take all the text, empty lines are not considered
+    eq_(result, [[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
+                 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+                 [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0],
+                 [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]])
+
+    with patch.object(fs, 'SIGNATURE_MAX_LINES', 4):
+        features = fs.features(sender)
+        new_result = fs.apply_features(s, features)
+        # result remains the same because we don't consider empty lines
+        eq_(result, new_result)
+
+
+def test_build_pattern():
+    s = '''John Doe
+
+VP Research and Development, Xxxx Xxxx Xxxxx
+
+555-226-2345
+
+john@example.com'''
+    sender = 'John <john@example.com>'
+    features = fs.features(sender)
+    result = fs.build_pattern(s, features)
+    eq_(result, [2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1])

tests/signature/learning/helpers_test.py

@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+
+from ... import *
+
+import regex as re
+
+from talon.signature.learning import helpers as h
+from talon.signature.learning.helpers import *
+
+# First testing regex constants.
+VALID = '''
+15615552323
+1-561-555-1212
+5613333
+
+18008793262
+800-879-3262
+0-800.879.3262
+
+04 3452488
+04 -3452488
+04 - 3452499
+
+(610) 310-5555 x5555
+533-1123
+
+(021)1234567
+(021)123456
+(000)000000
+
++7 920 34 57 23
++7(920) 34 57 23
++7(920)345723
++7920345723
+8920345723
+21143
+2-11-43
+2 - 11 - 43
+'''
+
+VALID_PHONE_NUMBERS = [e.strip() for e in VALID.splitlines() if e.strip()]
+
+
+def test_match_phone_numbers():
+    for phone in VALID_PHONE_NUMBERS:
+        ok_(RE_RELAX_PHONE.match(phone), "{} should be matched".format(phone))
+
+
+def test_match_names():
+    names = ['John R. Doe']
+    for name in names:
+        ok_(RE_NAME.match(name), "{} should be matched".format(name))
+
+
+def test_sender_with_name():
+    ok_lines = ['Sergey Obukhov <serobnic@example.com>',
+                '\tSergey  <serobnic@example.com>',
+                ('"Doe, John (TX)"'
+                 '<DowJ@example.com>@EXAMPLE'
+                 '<IMCEANOTES-+22Doe+2C+20John+20'
+                 '+28TX+29+22+20+3CDoeJ+40example+2Ecom+3E'
+                 '+40EXAMPLE@EXAMPLE.com>'),
+                ('Company Sleuth <csleuth@email.xxx.com>'
+                 '@EXAMPLE <XXX-Company+20Sleuth+20+3Ccsleuth'
+                 '+40email+2Exxx+2Ecom+3E+40EXAMPLE@EXAMPLE.com>'),
+                ('Doe III, John '
+                 '</O=EXAMPLE/OU=NA/CN=RECIPIENTS/CN=jDOE5>')]
+    for line in ok_lines:
+        ok_(RE_SENDER_WITH_NAME.match(line),
+            '{} should be matched'.format(line))
+
+    nok_lines = ['', '<serobnic@xxx.ru>', 'Sergey serobnic@xxx.ru']
+    for line in nok_lines:
+        assert_false(RE_SENDER_WITH_NAME.match(line),
+                     '{} should not be matched'.format(line))
+
+
+# Now test helpers functions
+def test_binary_regex_search():
+    eq_(1, h.binary_regex_search(re.compile("12"))("12"))
+    eq_(0, h.binary_regex_search(re.compile("12"))("34"))
+
+
+def binary_regex_match(prog):
+    eq_(1, h.binary_regex_match(re.compile("12"))("12 3"))
+    eq_(0, h.binary_regex_match(re.compile("12"))("3 12"))
+
+
+def test_flatten_list():
+    eq_([1, 2, 3, 4, 5], h.flatten_list([[1, 2], [3, 4, 5]]))
+
+
+@patch.object(h.re, 'compile')
+def test_contains_sender_names(re_compile):
+    with patch.object(h, 'extract_names',
+                      Mock(return_value=['bob', 'smith'])) as extract_names:
+        has_sender_names = h.contains_sender_names("bob.smith@example.com")
+        extract_names.assert_called_with("bob.smith@example.com")
+        for name in ["bob", "Bob", "smith", "Smith"]:
+            ok_(has_sender_names(name))
+
+        extract_names.return_value = ''
+        has_sender_names = h.contains_sender_names("bob.smith@example.com")
+        # if no names could be extracted fallback to the email address
+        ok_(has_sender_names('bob.smith@example.com'))
+
+        # don't crash if there are no sender
+        extract_names.return_value = ''
+        has_sender_names = h.contains_sender_names("")
+        assert_false(has_sender_names(''))
+
+
+def test_extract_names():
+    senders_names = {
+        # from example dataset
+        ('Jay Rickerts <eCenter@example.com>@EXAMPLE <XXX-Jay+20Rickerts'
+         '+20+3CeCenter+40example+2Ecom+3E+40EXAMPLE@EXAMPLE.com>'):
+        ['Jay', 'Rickerts'],
+        # if `,` is used in sender's name
+        'Williams III, Bill </O=EXAMPLE/OU=NA/CN=RECIPIENTS/CN=BWILLIA5>':
+        ['Williams', 'III', 'Bill'],
+        # if somehow `'` or `"` are used in sender's name
+        'Laura" "Goldberg <laura.goldberg@example.com>':
+        ['Laura', 'Goldberg'],
+        # extract from senders email address
+        '<sergey@xxx.ru>': ['sergey'],
+        # extract from sender's email address
+        # if dots are used in the email address
+        '<sergey.obukhov@xxx.ru>': ['sergey', 'obukhov'],
+        # extract from sender's email address
+        # if dashes are used in the email address
+        '<sergey-obukhov@xxx.ru>': ['sergey', 'obukhov'],
+        # extract from sender's email address
+        # if `_` are used in the email address
+        '<sergey_obukhov@xxx.ru>': ['sergey', 'obukhov'],
+        # old style From field, found in jangada dataset
+        'wcl@example.com (Wayne Long)': ['Wayne', 'Long'],
+        # if only sender's name provided
+        'Wayne Long': ['Wayne', 'Long'],
+        # if middle name is shortened with dot
+        'Sergey N.  Obukhov <serobnic@xxx.ru>': ['Sergey', 'Obukhov'],
+        # not only spaces could be used as name splitters
+        '  Sergey  Obukhov  <serobnic@xxx.ru>': ['Sergey', 'Obukhov'],
+        # finally normal example
+        'Sergey <serobnic@xxx.ru>': ['Sergey'],
+        # if middle name is shortened with `,`
+        'Sergey N, Obukhov': ['Sergey', 'Obukhov'],
+        # if mailto used with email address and sender's name is specified
+        'Sergey N, Obukhov [mailto: serobnic@xxx.ru]': ['Sergey', 'Obukhov'],
+        # when only email address is given
+        'serobnic@xxx.ru': ['serobnic'],
+        # when nothing is given
+        '': [],
+        # if phone is specified in the `From:` header
+        'wcl@example.com (Wayne Long +7 920 -256 - 35-09)': ['Wayne', 'Long'],
+        # from crash reports `nothing to repeat`
+        '* * * * <the_pod1@example.com>': ['the', 'pod'],
+        '"**Bobby B**" <copymycashsystem@example.com>':
+        ['Bobby', 'copymycashsystem'],
+        # from crash reports `bad escape`
+        '"M Ali B Azlan \(GHSE/PETH\)" <aliazlan@example.com>':
+        ['Ali', 'Azlan'],
+        ('"Ridthauddin B A Rahim \(DD/PCSB\)"'
+         ' <ridthauddin_arahim@example.com>'): ['Ridthauddin', 'Rahim'],
+        ('"Boland, Patrick \(Global Xxx Group, Ireland \)"'
+         ' <Patrick.Boland@example.com>'): ['Boland', 'Patrick'],
+        '"Mates Rate \(Wine\)" <amen@example.com.com>':
+        ['Mates', 'Rate', 'Wine'],
+        ('"Morgan, Paul \(Business Xxx RI, Xxx Xxx Group\)"'
+         ' <paul.morgan@example.com>'): ['Morgan', 'Paul'],
+        '"David DECOSTER \(Domicile\)" <decosterdavid@xxx.be>':
+        ['David', 'DECOSTER', 'Domicile']
+        }
+
+    for sender, expected_names in senders_names.items():
+        extracted_names = h.extract_names(sender)
+        # check that extracted names could be compiled
+        try:
+            re.compile("|".join(extracted_names))
+        except Exception, e:
+            ok_(False, ("Failed to compile extracted names {}"
+                        "\n\nReason: {}").format(extracted_names, e))
+        if expected_names:
+            for name in expected_names:
+                assert_in(name, extracted_names)
+        else:
+            eq_(expected_names, extracted_names)
+
+    # words like `ru`, `gmail`, `com`, `org`, etc. are not considered
+    # sender's names
+    for word in h.BAD_SENDER_NAMES:
+        eq_(h.extract_names(word), [])
+
+    # duplicates are not allowed
+    eq_(h.extract_names("sergey <sergey@example.com"), ["sergey"])
+
+
+def test_categories_percent():
+    eq_(0.0, h.categories_percent("qqq ggg hhh", ["Po"]))
+    eq_(50.0, h.categories_percent("q,w.", ["Po"]))
+    eq_(0.0, h.categories_percent("qqq ggg hhh", ["Nd"]))
+    eq_(50.0, h.categories_percent("q5", ["Nd"]))
+    eq_(50.0, h.categories_percent("s.s,5s", ["Po", "Nd"]))
+    eq_(0.0, h.categories_percent("", ["Po", "Nd"]))
+
+
+@patch.object(h, 'categories_percent')
+def test_punctuation_percent(categories_percent):
+    h.punctuation_percent("qqq")
+    categories_percent.assert_called_with("qqq", ['Po'])
+
+
+def test_capitalized_words_percent():
+    eq_(0.0, h.capitalized_words_percent(''))
+    eq_(100.0, h.capitalized_words_percent('Example Corp'))
+    eq_(50.0, h.capitalized_words_percent('Qqq qqq QQQ 123 sss'))
+    eq_(100.0, h.capitalized_words_percent('Cell 713-444-7368'))
+    eq_(100.0, h.capitalized_words_percent('8th Floor'))
+    eq_(0.0, h.capitalized_words_percent('(212) 230-9276'))
+
+
+def test_has_signature():
+    ok_(h.has_signature('sender', 'sender@example.com'))
+    ok_(h.has_signature('http://www.example.com\n555 555 5555',
+                        'sender@example.com'))
+    ok_(h.has_signature('http://www.example.com\naddress@example.com',
+                        'sender@example.com'))
+    assert_false(h.has_signature('http://www.example.com/555-555-5555',
+                                 'sender@example.com'))
+    long_line = ''.join(['q' for e in xrange(28)])
+    assert_false(h.has_signature(long_line + ' sender', 'sender@example.com'))
+    # wont crash on an empty string
+    assert_false(h.has_signature('', ''))
+    # dont consider empty strings when analysing signature
+    with patch.object(h, 'SIGNATURE_MAX_LINES', 1):
+        ok_('sender\n\n', 'sender@example.com')