-
-
Save User001501/3053f26100ddf281600668fed347e518 to your computer and use it in GitHub Desktop.
Multithreaded SQLite Python
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from threading import Thread | |
| from multiprocessing import Queue | |
| import sqlite3, sys, time, datetime | |
| class MultiThreadOK(Thread): | |
| def __init__(self, db): | |
| super(MultiThreadOK, self).__init__() | |
| self.db=db | |
| self.reqs=Queue() | |
| self.start() | |
| def run(self): | |
| cnx = sqlite3.Connection(self.db) | |
| cursor = cnx.cursor() | |
| while True: | |
| req = self.reqs.get() | |
| if req=='--close--': break | |
| elif req=='--commit--': cnx.commit() | |
| try: | |
| cursor.executescript(req) if ';' in req else cursor.execute(req) | |
| except sqlite3.OperationalError as err: | |
| self.escribir_error('{0} - Error {1}\n'.format(datetime.datetime.now(), err)) | |
| self.escribir_error('{0} - {1}\n'.format(datetime.datetime.now(), req)) | |
| except: | |
| self.escribir_error('{0} - Salida'.format(datetime.datetime.now())) | |
| cnx.close() | |
| def execute(self, req): | |
| self.reqs.put(req) | |
| def queries(self): | |
| return self.reqs.qsize() | |
| def empty(self): | |
| return self.reqs.empty() | |
| def select(self, req, results=0): | |
| cnx = sqlite3.Connection(self.db) | |
| cursor = cnx.cursor() | |
| try: | |
| if results == 0: | |
| cursor.execute(req) | |
| ret = [x for x in cursor.fetchall()] | |
| cnx.close() | |
| return ret | |
| else: | |
| cursor.execute(req) | |
| ret = [x for x in cursor.fetchall()[:results]] | |
| cnx.close() | |
| return ret | |
| except: | |
| print("Unexpected error: {0}".format(sys.exc_info()[0])) | |
| cnx.close() | |
| def commit(self): | |
| self.execute("--commit--") | |
| def close(self): | |
| self.execute('--close--') | |
| def escribir_error(self, texto): | |
| with open(os.path.dirname(os.path.abspath(__file__)) + '\\errores.txt','a') as archivo: | |
| archivo.write(texto) | |
| if __name__=='__main__': | |
| db='people.db' | |
| sql=MultiThreadOK(db) | |
| sql.execute("create table people(name,first)") | |
| sql.execute("insert into people (name,first) values('Czabania','George')") | |
| sql.commit() | |
| sql.execute("insert into people (name,first) values('Cooper','Jono')") | |
| sql.commit() | |
| sql.execute("insert into people (name,first) values('Wick','John');insert into people values('Anderson','Thomas');") | |
| sql.commit() | |
| #much more efficient way to do bulk Inserts and Updates to the DB | |
| sql.execute("BEGIN TRANSACTION;INSERT INTO people (name,first) values ('Smith','John');INSERT INTO people values ('Anderson','Thomas');COMMIT;") | |
| for q in sql.select("select * from people"): | |
| print(q) | |
| #wait until all write are done to read all updated data | |
| while sql.queries() > 0: | |
| time.sleep(5) | |
| sql.close() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment