Command line idempotent python script which uses a Sqlite LSM key value store as idempotent store

idempotent.py

#!/usr/bin/env python
from lsm import LSM
import argparse
import time
import sys

parser = argparse.ArgumentParser(description='Idempotent')
parser.add_argument('store', type=str, 
  help='path of the LSM database to use as idempotent store')
parser.add_argument('--save', type=bool, default=False,
  help='save line in store')

args = parser.parse_args()

progressDb = LSM(args.store)

for line in sys.stdin:
  finished = line in progressDb

  if(not finished):
    sys.stdout.write(line)

  if(args.save):
    progressDb[line] = str(int(time.time() * 1000))

requirements.txt

lsm-db