Playing with forking and pipes to SQLite3 to speed up that data pumping.

sqlite_import.py

#!/usr/bin/env python

import os, sys, tempfile
from subprocess import Popen, PIPE, STDOUT

#Name of the pipe, use tempfile to create some random filename usually in /tmp
data_pipe = tempfile.mktemp()

#Create the actual pipe 'file' where the OS knows you wanted a pipe type thing
os.mkfifo( data_pipe, 0644 )


#Create a child process to run in parallel
child = os.fork()

#If child is 0 we are the child
if child == 0:
    #Have the child send the command to sqlite3
    #Create a forked process running sqlite3 with a pipe that we can send commands to
    sqlite3 = Popen(['sqlite3', 'test.db'], stdout=PIPE, stdin=PIPE, stderr=STDOUT)

    #Tell sqlite3 to import from our named pipe
    db = sqlite3.communicate(input='.separator "\\t"\n.import %s test3\n.quit\n' % (data_pipe))[0]
    sys.exit(1)

else:
    #File handle to pipe to write data into table
    data_pipe_write_handle = open(data_pipe,'w')
    #Open a datafile we want to be importing from, this would be some array you already have in memory inside your Python program
    #CHRIS YOU WONT NEED THIS FILE USE YOUR ARRAYS INSTEAD
    data = open('data3', 'r')
    #Have the original program send data down the pipe
    #CHRIS EDIT HERE PUMP YOUR DATA INTO data_pipe_write_handle
    for tuple in data:
        data_pipe_write_handle.write("%d\t%d" % tuple)
    data_pipe_write_handle.close()
    #Wait for SQLite3 to finish importing, waiting on all child processes
    os.wait()
    #Remove the named pipe file we created because its junk and we dont want a clash
    os.unlink(data_pipe)