Skip to content

Instantly share code, notes, and snippets.

@User001501
Forked from consindo/multithreaded_sqlite.py
Last active February 5, 2024 13:59
Show Gist options
  • Select an option

  • Save User001501/3053f26100ddf281600668fed347e518 to your computer and use it in GitHub Desktop.

Select an option

Save User001501/3053f26100ddf281600668fed347e518 to your computer and use it in GitHub Desktop.
Multithreaded SQLite Python
from threading import Thread
from multiprocessing import Queue
import sqlite3, sys, time, datetime
class MultiThreadOK(Thread):
def __init__(self, db):
super(MultiThreadOK, self).__init__()
self.db=db
self.reqs=Queue()
self.start()
def run(self):
cnx = sqlite3.Connection(self.db)
cursor = cnx.cursor()
while True:
req = self.reqs.get()
if req=='--close--': break
elif req=='--commit--': cnx.commit()
try:
cursor.executescript(req) if ';' in req else cursor.execute(req)
except sqlite3.OperationalError as err:
self.escribir_error('{0} - Error {1}\n'.format(datetime.datetime.now(), err))
self.escribir_error('{0} - {1}\n'.format(datetime.datetime.now(), req))
except:
self.escribir_error('{0} - Salida'.format(datetime.datetime.now()))
cnx.close()
def execute(self, req):
self.reqs.put(req)
def queries(self):
return self.reqs.qsize()
def empty(self):
return self.reqs.empty()
def select(self, req, results=0):
cnx = sqlite3.Connection(self.db)
cursor = cnx.cursor()
try:
if results == 0:
cursor.execute(req)
ret = [x for x in cursor.fetchall()]
cnx.close()
return ret
else:
cursor.execute(req)
ret = [x for x in cursor.fetchall()[:results]]
cnx.close()
return ret
except:
print("Unexpected error: {0}".format(sys.exc_info()[0]))
cnx.close()
def commit(self):
self.execute("--commit--")
def close(self):
self.execute('--close--')
def escribir_error(self, texto):
with open(os.path.dirname(os.path.abspath(__file__)) + '\\errores.txt','a') as archivo:
archivo.write(texto)
if __name__=='__main__':
db='people.db'
sql=MultiThreadOK(db)
sql.execute("create table people(name,first)")
sql.execute("insert into people (name,first) values('Czabania','George')")
sql.commit()
sql.execute("insert into people (name,first) values('Cooper','Jono')")
sql.commit()
sql.execute("insert into people (name,first) values('Wick','John');insert into people values('Anderson','Thomas');")
sql.commit()
#much more efficient way to do bulk Inserts and Updates to the DB
sql.execute("BEGIN TRANSACTION;INSERT INTO people (name,first) values ('Smith','John');INSERT INTO people values ('Anderson','Thomas');COMMIT;")
for q in sql.select("select * from people"):
print(q)
#wait until all write are done to read all updated data
while sql.queries() > 0:
time.sleep(5)
sql.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment