Testing marisa-trie performance with 2 million filenames (in find.txt, file list is not included)

trietest.py

#!/usr/bin/env python
#-*- coding: UTF-8 -*-

import marisa_trie

def uread(f):
    for line in f:
        yield line.decode('utf8', 'replace')

ds = []
with open('find.txt', 'rb') as f:
    print 'building data structure...'
    #ds = uread(f) # Use raw generator, lazy read
    #ds = list(uread(f)) # Build a list
    ds = marisa_trie.Trie(uread(f)) # Build a trie

    print 'counting... '
    c = 0
    for e in ds:
        c += 1

    print 'done: ' + str(c)