Last active
May 28, 2021 12:37
-
-
Save kyamagu/fdf7bb8b27774ce79c71 to your computer and use it in GitHub Desktop.
Python iterator for kv-store
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import lmdb | |
import leveldb | |
def lmdb_reader(filename, **kwargs): | |
with lmdb.open(filename, readonly=True, create=False, **kwargs) as env: | |
with env.begin() as txn: | |
with txn.cursor() as cursor: | |
for key, value in cursor: | |
yield key, value | |
def lmdb_writer(filename, iterator, **kwargs): | |
with lmdb.open(filename, map_size=(1 << 40), create=True) as env: | |
with env.begin(write=True, **kwargs) as txn: | |
for key, value in iterator: | |
txn.put(key, value) | |
def leveldb_reader(filename, **kwargs): | |
"""Read key-value pairs from LevelDB.""" | |
database = leveldb.LevelDB(filename, **kwargs) | |
for key, value in database.RangeIter(): | |
yield key, value |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment