Skip to content

Instantly share code, notes, and snippets.

@oyvindio

oyvindio/export-tab-sep.py

Created May 8, 2010 12:20
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save oyvindio/394535 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save oyvindio/394535 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
export-tab-sep.py
#!/usr/bin/env python
import os
import sys
import shelve
def exportTabSeparated(filename):
shelf = shelve.open(filename)
dct = dict()
for k in shelf.keys():
dct[k] = shelf[k]
shelf.close()
outFilename = os.path.splitext(os.path.basename(filename))[0] + ".txt"
with open(outFilename, "w") as f:
g = tabSeparatedStringGenerator(dct)
for dummy in range(len(dct.values()[0])):
f.write(g.next() + "\n")
def tabSeparatedStringGenerator(shelf):
for i in range(len(shelf.values()[0])):
yield ''.join([str(v[i]) + "\t" for v in shelf.values()]).strip()
if __name__ == "__main__":
if len(sys.argv) == 1:
print >> sys.stderr, "Invalid arguments %s" % sys.argv
exit(1)
else:
for arg in sys.argv[1:]:
exportTabSeparated(arg)
Raw
pstats
In [3]: s.sort_stats('cumulative').print_stats()
Sat May 8 14:17:22 2010 export-tab-sep-gpfs.pstats
500212 function calls in 3.869 CPU seconds
Ordered by: cumulative time
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 3.870 3.870 <string>:1(<module>)
1 0.001 0.001 3.870 3.870 {execfile}
1 0.034 0.034 3.869 3.869 export-tab-sep.py:2(<module>)
1 0.446 0.446 3.808 3.808 export-tab-sep.py:7(exportTabSeparated)
100001 1.512 0.000 2.068 0.000 export-tab-sep.py:21(tabSeparatedStringGenerator)
4 0.049 0.012 0.895 0.224 /usr/lib/python2.6/shelve.py:117(__getitem__)
4 0.754 0.188 0.754 0.188 {built-in method load}
100000 0.315 0.000 0.315 0.000 {method 'write' of 'file' objects}
100000 0.209 0.000 0.209 0.000 {method 'join' of 'str' objects}
100000 0.177 0.000 0.177 0.000 {method 'strip' of 'str' objects}
100004 0.167 0.000 0.167 0.000 {method 'values' of 'dict' objects}
7 0.000 0.000 0.142 0.020 /usr/lib/python2.6/bsddb/dbutils.py:47(DeadlockWrap)
4 0.000 0.000 0.092 0.023 /usr/lib/python2.6/bsddb/__init__.py:265(__getitem__)
4 0.092 0.023 0.092 0.023 /usr/lib/python2.6/bsddb/__init__.py:267(<lambda>)
1 0.000 0.000 0.047 0.047 /usr/lib/python2.6/shelve.py:100(keys)
1 0.000 0.000 0.047 0.047 /usr/lib/python2.6/bsddb/__init__.py:298(keys)
1 0.047 0.047 0.047 0.047 {method 'keys' of 'DB' objects}
1 0.000 0.000 0.033 0.033 /usr/lib/python2.6/shelve.py:221(open)
1 0.000 0.000 0.033 0.033 /usr/lib/python2.6/shelve.py:216(__init__)
1 0.000 0.000 0.029 0.029 /usr/lib/python2.6/anydbm.py:43(<module>)
5 0.003 0.001 0.029 0.006 {__import__}
1 0.001 0.001 0.027 0.027 /usr/lib/python2.6/dbhash.py:1(<module>)
1 0.026 0.026 0.026 0.026 /usr/lib/python2.6/shelve.py:57(<module>)
1 0.024 0.024 0.026 0.026 /usr/lib/python2.6/bsddb/__init__.py:40(<module>)
2 0.004 0.002 0.004 0.002 {range}
1 0.001 0.001 0.003 0.003 /usr/lib/python2.6/anydbm.py:66(open)
2 0.000 0.000 0.003 0.001 /usr/lib/python2.6/shelve.py:142(close)
2 0.000 0.000 0.003 0.001 /usr/lib/python2.6/bsddb/__init__.py:287(close)
1 0.002 0.002 0.002 0.002 {method 'close' of 'DB' objects}
1 0.000 0.000 0.002 0.002 /usr/lib/python2.6/dbhash.py:18(open)
1 0.000 0.000 0.002 0.002 /usr/lib/python2.6/bsddb/__init__.py:350(hashopen)
1 0.000 0.000 0.002 0.002 /usr/lib/python2.6/bsddb/__init__.py:403(_openDBEnv)
1 0.002 0.002 0.002 0.002 {method 'open' of 'DBEnv' objects}
1 0.001 0.001 0.001 0.001 /usr/lib/python2.6/collections.py:1(<module>)
2 0.000 0.000 0.001 0.000 /home/oyvindio/.local/lib/python2.6/abc.py:78(__new__)
1 0.000 0.000 0.001 0.001 /usr/lib/python2.6/bsddb/dbutils.py:27(<module>)
1 0.000 0.000 0.000 0.000 {method 'open' of 'DB' objects}
1 0.000 0.000 0.000 0.000 {_bsddb.DBEnv}
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/whichdb.py:17(whichdb)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/bsddb/db.py:40(<module>)
2 0.000 0.000 0.000 0.000 /home/oyvindio/.local/lib/python2.6/abc.py:81(<genexpr>)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/weakref.py:6(<module>)
47 0.000 0.000 0.000 0.000 {getattr}
2 0.000 0.000 0.000 0.000 {built-in method __new__ of type object at 0x8228620}
4 0.000 0.000 0.000 0.000 {open}
2 0.000 0.000 0.000 0.000 /usr/lib/python2.6/shelve.py:156(sync)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/dumbdbm.py:22(<module>)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/shelve.py:150(__del__)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/bsddb/__init__.py:342(sync)
1 0.000 0.000 0.000 0.000 {_bsddb.DB}
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/bsddb/__init__.py:194(__init__)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/bsddb/__init__.py:189(_DBWithCursor)
4 0.000 0.000 0.000 0.000 {cPickle.Unpickler}
1 0.000 0.000 0.000 0.000 {method 'read' of 'file' objects}
4 0.000 0.000 0.000 0.000 {cStringIO.StringIO}
1 0.000 0.000 0.000 0.000 /home/oyvindio/.local/lib/python2.6/posixpath.py:94(splitext)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/weakref.py:219(WeakKeyDictionary)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/keyword.py:11(<module>)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/weakref.py:33(WeakValueDictionary)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/dumbdbm.py:34(_Database)
2 0.000 0.000 0.000 0.000 {_struct.unpack}
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/whichdb.py:2(<module>)
6 0.000 0.000 0.000 0.000 /usr/lib/python2.6/bsddb/__init__.py:248(_checkOpen)
2 0.000 0.000 0.000 0.000 /usr/lib/python2.6/bsddb/__init__.py:232(_closeCursors)
3 0.000 0.000 0.000 0.000 {hasattr}
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/bsddb/__init__.py:218(__del__)
1 0.000 0.000 0.000 0.000 {method 'set_flags' of 'DB' objects}
7 0.000 0.000 0.000 0.000 {method 'has_key' of 'dict' objects}
1 0.000 0.000 0.000 0.000 {method 'close' of 'file' objects}
7 0.000 0.000 0.000 0.000 {method 'get' of 'dict' objects}
1 0.000 0.000 0.000 0.000 /home/oyvindio/.local/lib/python2.6/posixpath.py:109(basename)
1 0.000 0.000 0.000 0.000 /home/oyvindio/.local/lib/python2.6/genericpath.py:85(_splitext)
1 0.000 0.000 0.000 0.000 {posix.stat}
1 0.000 0.000 0.000 0.000 {method 'sync' of 'DB' objects}
4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects}
3 0.000 0.000 0.000 0.000 {method 'rfind' of 'str' objects}
2 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/shelve.py:85(Shelf)
4 0.000 0.000 0.000 0.000 {len}
4 0.000 0.000 0.000 0.000 {method 'add' of 'set' objects}
1 0.000 0.000 0.000 0.000 {method 'set_lk_detect' of 'DBEnv' objects}
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/shelve.py:167(BsdDbShelf)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/shelve.py:75(_ClosedDict)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/bsddb/__init__.py:91(_iter_mixin)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/shelve.py:92(__init__)
1 0.000 0.000 0.000 0.000 {method 'set_get_returns_none' of 'DB' objects}
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/weakref.py:198(KeyedRef)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/bsddb/__init__.py:414(_checkflag)
1 0.000 0.000 0.000 0.000 {method 'startswith' of 'str' objects}
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/anydbm.py:45(error)
1 0.000 0.000 0.000 0.000 <string>:1(DBNotFoundError)
1 0.000 0.000 0.000 0.000 /usr/lib/python2.6/shelve.py:209(DbfilenameShelf)
1 0.000 0.000 0.000 0.000 <string>:2(DBKeyEmptyError)
1 0.000 0.000 0.000 0.000 {method '__enter__' of 'file' objects}
1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}
Raw
shell
$ time python -m cProfile -o export-tab-sep-gpfs.pstats export-tab-sep.py listdir-gpfs-2010-04-07-170721.shelf
real 0m3.996s
user 0m2.500s
sys 0m1.364s
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment