Term Frequency Calculation Speed

1_test.py

import re
import collections
import math

import numpy as np
import lxml.etree
from lxml.html.clean import Cleaner


ITERATIONS = 2500


def cleanup_html(html):
    cleaner = Cleaner(style=True, javascript=True, comments=True, meta=True, frames=True,
                      page_structure=True, embedded=True, annoying_tags=True)

    # Strip encoding declaration, lxml does not like those in unicode strings.
    html = re.sub(r'<\?xml ([^>]*)encoding="[^"]+"', r'<?xml \1', html)

    # Apparently lxml does not like empty HTML documents either...
    if not html:
        return ''

    # Clean HTML
    try:
        cleaned_html = cleaner.clean_html(lxml.html.fromstring(html))
    except lxml.etree.ParserError:
        return ''
    text_content = cleaned_html.text_content()
    return re.sub(r'\s+', ' ', text_content).strip()


with open('doc1.txt', 'r') as f:
    doc1 = cleanup_html(f.read())
with open('doc2.txt', 'r') as f:
    doc2 = cleanup_html(f.read())


def calculate_similarity_numpy(doc1, doc2):
    pattern = re.compile(r'\W+')
    for i in range(ITERATIONS):
        words1 = [w.strip().lower() for w in pattern.sub(' ', doc1).split()]
        words2 = [w.strip().lower() for w in pattern.sub(' ', doc2).split()]
        counted1 = collections.Counter(words1)
        counted2 = collections.Counter(words2)
        words = set(words1 + words2)
        freq1 = np.ndarray((len(words),))
        freq2 = np.ndarray((len(words),))
        for i, word in enumerate(words):
            freq1[i] = counted1[word] / len(words1)
            freq2[i] = counted2[word] / len(words2)
        len1 = np.sqrt(freq1.dot(freq1))
        len2 = np.sqrt(freq2.dot(freq2))
        normalized1 = (freq1 / len1)
        normalized2 = (freq2 / len2)
        similarity = normalized1.dot(normalized2)
    return similarity


def calculate_similarity_python(doc1, doc2):
    pattern = re.compile(r'\W+')
    for i in range(ITERATIONS):
        words1 = [w.strip().lower() for w in pattern.sub(' ', doc1).split()]
        words2 = [w.strip().lower() for w in pattern.sub(' ', doc2).split()]
        counted1 = collections.Counter(words1)
        counted2 = collections.Counter(words2)
        words = set(words1 + words2)
        freq1 = []
        freq2 = []
        for word in words:
            freq1.append(counted1[word] / len(words1))
            freq2.append(counted2[word] / len(words2))
        len1 = math.sqrt(sum(v * v for v in freq1))
        len2 = math.sqrt(sum(v * v for v in freq2))
        similarity = sum(v1 / len1 * v2 / len2 for v1, v2 in zip(freq1, freq2))
    return similarity


if __name__ == '__main__':
    import sys
    from datetime import datetime
    t1 = datetime.now()
    if sys.argv[1] == 'numpy':
        similarity = calculate_similarity_numpy(doc1, doc2)
    elif sys.argv[1] == 'python':
        similarity = calculate_similarity_python(doc1, doc2)
    t2 = datetime.now()
    print('similarity is %r (type %s)' % (similarity, type(similarity)))
    print('calculation took %s' % (t2 - t1))

2_test-files.txt

http://tmp.dbrgn.ch/doc1.txt
http://tmp.dbrgn.ch/doc2.txt

3_results.txt

$ time python 1_test.py python
similarity is 0.20813409689508422 (type <class 'float'>)
calculation took 0:00:21.932151

real    0m22.641s
user    0m22.440s
sys     0m0.077s
$ time python 1_test.py numpy
similarity is 0.20813409689508311 (type <class 'numpy.float64'>)
calculation took 0:00:22.526667

real    0m23.262s
user    0m22.913s
sys     0m0.063s