Skip to content

Instantly share code, notes, and snippets.

@sapamja

sapamja/word_lookup.py

Last active August 29, 2015 13:58
Show Gist options
  • Save sapamja/10001898 to your computer and use it in GitHub Desktop.
Save sapamja/10001898 to your computer and use it in GitHub Desktop.
Download ZIP
Profiling words lookup in long sentence.
Raw
word_lookup.py
#!/usr/bin/python
import cProfile
from timeit import Timer
from faker import Faker
def func1(sentence, words):
return any(word in sentence for word in words)
def func2(sentence, words):
for word in words:
if word in sentence:
return True
return False
def func3(sentence, words):
# using set
sets = set(sentence)
return bool(set(words).intersection(sets))
def func4(sentence, words):
# using set
sets = set(sentence)
for word in words:
if word in sets:
return True
return False
def func5(sentence, words):
# using set
sentence, words = set(sentence), set(words)
return not words.isdisjoint(sentence)
s = Faker()
sentence = s.sentence(nb_words=100000).split()
words = 'quidem necessitatibus minus id quos in neque omnis molestias'.split()
func = [ func1, func2, func3, func4, func5 ]
for fun in func:
t = Timer(lambda: fun(sentence, words))
print fun.__name__, cProfile.run('t.timeit(number=1000)')
Raw
word_lookup_output.txt
Output:
func1 5011 function calls in 0.032 seconds
Ordered by: standard name
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 0.032 0.032 <string>:1(<module>)
1000 0.000 0.000 0.032 0.000 exists.py:42(<lambda>)
1000 0.001 0.000 0.032 0.000 exists.py:8(func1)
2000 0.030 0.000 0.030 0.000 exists.py:9(<genexpr>)
1 0.000 0.000 0.000 0.000 timeit.py:143(setup)
1 0.000 0.000 0.032 0.032 timeit.py:178(timeit)
1 0.000 0.000 0.032 0.032 timeit.py:96(inner)
1000 0.000 0.000 0.030 0.000 {any}
1 0.000 0.000 0.000 0.000 {gc.disable}
1 0.000 0.000 0.000 0.000 {gc.enable}
1 0.000 0.000 0.000 0.000 {gc.isenabled}
1 0.000 0.000 0.000 0.000 {globals}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
2 0.000 0.000 0.000 0.000 {time.time}
None
func2 2011 function calls in 0.031 seconds
Ordered by: standard name
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 0.031 0.031 <string>:1(<module>)
1000 0.031 0.000 0.031 0.000 exists.py:11(func2)
1000 0.000 0.000 0.031 0.000 exists.py:42(<lambda>)
1 0.000 0.000 0.000 0.000 timeit.py:143(setup)
1 0.000 0.000 0.031 0.031 timeit.py:178(timeit)
1 0.000 0.000 0.031 0.031 timeit.py:96(inner)
1 0.000 0.000 0.000 0.000 {gc.disable}
1 0.000 0.000 0.000 0.000 {gc.enable}
1 0.000 0.000 0.000 0.000 {gc.isenabled}
1 0.000 0.000 0.000 0.000 {globals}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
2 0.000 0.000 0.000 0.000 {time.time}
None
func3 3011 function calls in 7.079 seconds
Ordered by: standard name
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 7.079 7.079 <string>:1(<module>)
1000 7.069 0.007 7.073 0.007 exists.py:17(func3)
1000 0.004 0.000 7.077 0.007 exists.py:42(<lambda>)
1 0.000 0.000 0.000 0.000 timeit.py:143(setup)
1 0.000 0.000 7.079 7.079 timeit.py:178(timeit)
1 0.002 0.002 7.079 7.079 timeit.py:96(inner)
1 0.000 0.000 0.000 0.000 {gc.disable}
1 0.000 0.000 0.000 0.000 {gc.enable}
1 0.000 0.000 0.000 0.000 {gc.isenabled}
1 0.000 0.000 0.000 0.000 {globals}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
1000 0.004 0.000 0.004 0.000 {method 'intersection' of 'set' objects}
2 0.000 0.000 0.000 0.000 {time.time}
None
func4 2011 function calls in 7.022 seconds
Ordered by: standard name
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 7.022 7.022 <string>:1(<module>)
1000 7.014 0.007 7.014 0.007 exists.py:22(func4)
1000 0.006 0.000 7.020 0.007 exists.py:42(<lambda>)
1 0.000 0.000 0.000 0.000 timeit.py:143(setup)
1 0.000 0.000 7.022 7.022 timeit.py:178(timeit)
1 0.002 0.002 7.022 7.022 timeit.py:96(inner)
1 0.000 0.000 0.000 0.000 {gc.disable}
1 0.000 0.000 0.000 0.000 {gc.enable}
1 0.000 0.000 0.000 0.000 {gc.isenabled}
1 0.000 0.000 0.000 0.000 {globals}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
2 0.000 0.000 0.000 0.000 {time.time}
None
func5 3011 function calls in 7.142 seconds
Ordered by: standard name
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 7.142 7.142 <string>:1(<module>)
1000 7.133 0.007 7.134 0.007 exists.py:30(func5)
1000 0.006 0.000 7.140 0.007 exists.py:42(<lambda>)
1 0.000 0.000 0.000 0.000 timeit.py:143(setup)
1 0.000 0.000 7.142 7.142 timeit.py:178(timeit)
1 0.002 0.002 7.142 7.142 timeit.py:96(inner)
1 0.000 0.000 0.000 0.000 {gc.disable}
1 0.000 0.000 0.000 0.000 {gc.enable}
1 0.000 0.000 0.000 0.000 {gc.isenabled}
1 0.000 0.000 0.000 0.000 {globals}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
1000 0.002 0.000 0.002 0.000 {method 'isdisjoint' of 'set' objects}
2 0.000 0.000 0.000 0.000 {time.time}
None
Raw
words_lookup_preprocess.py
#!/usr/bin/python
import cProfile
from timeit import Timer
from faker import Faker
def func1(sentence, words):
return any(word in sentence for word in words)
def func2(sentence, words):
for word in words:
if word in sentence:
return True
return False
def func3(sentence, words):
return bool(set(words).intersection(sentence))
def func4(sentence, words):
for word in words:
if word in sentence:
return True
return False
def func5(sentence, words):
return not words.isdisjoint(sentence)
s = Faker()
sentence = set(s.sentence(nb_words=100000).split())
words = set('quidem necessitatibus minus id quos in neque omnis molestias'.split())
func = [ func1, func2, func3, func4, func5 ]
for fun in func:
t = Timer(lambda: fun(sentence, words))
print fun.__name__, cProfile.run('t.timeit(number=1000)')
Raw
words_lookup_preprocess.txt
func1 5011 function calls in 0.003 seconds
Ordered by: standard name
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 0.003 0.003 <string>:1(<module>)
1000 0.000 0.000 0.002 0.000 exists.py:36(<lambda>)
1000 0.001 0.000 0.002 0.000 exists.py:8(func1)
2000 0.001 0.000 0.001 0.000 exists.py:9(<genexpr>)
1 0.000 0.000 0.000 0.000 timeit.py:143(setup)
1 0.000 0.000 0.003 0.003 timeit.py:178(timeit)
1 0.000 0.000 0.003 0.003 timeit.py:96(inner)
1000 0.000 0.000 0.001 0.000 {any}
1 0.000 0.000 0.000 0.000 {gc.disable}
1 0.000 0.000 0.000 0.000 {gc.enable}
1 0.000 0.000 0.000 0.000 {gc.isenabled}
1 0.000 0.000 0.000 0.000 {globals}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
2 0.000 0.000 0.000 0.000 {time.time}
None
func2 2011 function calls in 0.001 seconds
Ordered by: standard name
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 0.001 0.001 <string>:1(<module>)
1000 0.000 0.000 0.000 0.000 exists.py:11(func2)
1000 0.000 0.000 0.001 0.000 exists.py:36(<lambda>)
1 0.000 0.000 0.000 0.000 timeit.py:143(setup)
1 0.000 0.000 0.001 0.001 timeit.py:178(timeit)
1 0.000 0.000 0.001 0.001 timeit.py:96(inner)
1 0.000 0.000 0.000 0.000 {gc.disable}
1 0.000 0.000 0.000 0.000 {gc.enable}
1 0.000 0.000 0.000 0.000 {gc.isenabled}
1 0.000 0.000 0.000 0.000 {globals}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
2 0.000 0.000 0.000 0.000 {time.time}
None
func3 3011 function calls in 0.004 seconds
Ordered by: standard name
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 0.004 0.004 <string>:1(<module>)
1000 0.002 0.000 0.004 0.000 exists.py:17(func3)
1000 0.000 0.000 0.004 0.000 exists.py:36(<lambda>)
1 0.000 0.000 0.000 0.000 timeit.py:143(setup)
1 0.000 0.000 0.004 0.004 timeit.py:178(timeit)
1 0.000 0.000 0.004 0.004 timeit.py:96(inner)
1 0.000 0.000 0.000 0.000 {gc.disable}
1 0.000 0.000 0.000 0.000 {gc.enable}
1 0.000 0.000 0.000 0.000 {gc.isenabled}
1 0.000 0.000 0.000 0.000 {globals}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
1000 0.002 0.000 0.002 0.000 {method 'intersection' of 'set' objects}
2 0.000 0.000 0.000 0.000 {time.time}
None
func4 2011 function calls in 0.001 seconds
Ordered by: standard name
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 0.001 0.001 <string>:1(<module>)
1000 0.000 0.000 0.000 0.000 exists.py:20(func4)
1000 0.000 0.000 0.001 0.000 exists.py:36(<lambda>)
1 0.000 0.000 0.000 0.000 timeit.py:143(setup)
1 0.000 0.000 0.001 0.001 timeit.py:178(timeit)
1 0.000 0.000 0.001 0.001 timeit.py:96(inner)
1 0.000 0.000 0.000 0.000 {gc.disable}
1 0.000 0.000 0.000 0.000 {gc.enable}
1 0.000 0.000 0.000 0.000 {gc.isenabled}
1 0.000 0.000 0.000 0.000 {globals}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
2 0.000 0.000 0.000 0.000 {time.time}
None
func5 3011 function calls in 0.001 seconds
Ordered by: standard name
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 0.001 0.001 <string>:1(<module>)
1000 0.000 0.000 0.001 0.000 exists.py:26(func5)
1000 0.000 0.000 0.001 0.000 exists.py:36(<lambda>)
1 0.000 0.000 0.000 0.000 timeit.py:143(setup)
1 0.000 0.000 0.001 0.001 timeit.py:178(timeit)
1 0.000 0.000 0.001 0.001 timeit.py:96(inner)
1 0.000 0.000 0.000 0.000 {gc.disable}
1 0.000 0.000 0.000 0.000 {gc.enable}
1 0.000 0.000 0.000 0.000 {gc.isenabled}
1 0.000 0.000 0.000 0.000 {globals}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
1000 0.000 0.000 0.000 0.000 {method 'isdisjoint' of 'set' objects}
2 0.000 0.000 0.000 0.000 {time.time}
None
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment