Created
February 9, 2015 21:50
-
-
Save bbarrows/9695784082d248ba8a4e to your computer and use it in GitHub Desktop.
My importer
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
from os import listdir | |
from os.path import isfile, join | |
from subprocess import Popen, PIPE, STDOUT, call | |
import MySQLdb | |
import MySQLdb.cursors | |
import types | |
import warnings | |
DB_NAME = "SGTEST" | |
SQL_PATH = "/tmp" | |
CREATE_NEW_DB = True | |
tmp_files = [ f for f in listdir(SQL_PATH) if isfile(join(SQL_PATH,f)) ] | |
sql_files = [x for x in tmp_files if x.endswith(".sql")] | |
txt_files = [x for x in tmp_files if x.endswith(".txt") and x[0:x.find('.txt')]+".sql" in sql_files] | |
if CREATE_NEW_DB: | |
try: | |
con = MySQLdb.connect('localhost', 'root', 'password') | |
with con: | |
cur = con.cursor() | |
cur.execute("CREATE DATABASE %s" % DB_NAME) | |
except Exception: | |
con = MySQLdb.connect('localhost', 'root', 'password') | |
with con: | |
cur = con.cursor() | |
cur.execute("DROP DATABASE %s" % DB_NAME) | |
cur.execute("CREATE DATABASE %s" % DB_NAME) | |
con = MySQLdb.connect('localhost', 'root', 'password', DB_NAME) | |
file_order = [] | |
do_first = ["PermitApp.sql", "PermitRole.sql", "Avl.sql", "Country.sql", "Customer.sql", "Fleet.sql", "Account.sql", "User.sql", "UserCustomer.sql", "UserFleet.sql", "VehicleType.sql", "Vehicle.sql"] | |
#for f in do_first: | |
#call("mysql -u root -ppassword %s < %s" % (DB_NAME, f), shell=True) | |
for f in do_first: | |
table_name = f[0:f.find('.sql')] | |
csv_file_name = table_name + ".txt" | |
file_order.append((table_name, csv_file_name)) | |
for sql_file in sql_files: | |
if sql_file not in do_first: | |
table_name = sql_file[0:sql_file.find('.sql')] | |
csv_file_name = table_name + ".txt" | |
file_order.append((table_name, csv_file_name)) | |
for table_name, data_file in file_order: | |
print "mysql -u root -ppassword %s < %s" % (DB_NAME, table_name+".sql") | |
call("mysql -u root -ppassword %s < %s" % (DB_NAME, table_name+".sql"), shell=True) | |
for table_name, data_file in file_order: | |
print "\nImporting file %s" % data_file | |
x = "LOAD DATA INFILE \"%s\" INTO TABLE %s COLUMNS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'ESCAPED BY '\"' LINES TERMINATED BY '\\n';" % (join("/tmp", data_file), table_name) | |
print x | |
with con: | |
cur = con.cursor() | |
result = cur.execute(x) | |
print "RESULT:%s" % result | |
# Check out | |
# http://dev.mysql.com/doc/refman/5.1/en/load-data.html | |
# http://stackoverflow.com/questions/13648055/mysql-doesnt-csv-import-n-as-null | |
# http://dev.mysql.com/doc/refman/5.0/en/mysqlimport.html | |
# mysqlimport --fields-optionally-enclosed-by=""" --fields-terminated-by=, --lines-terminated-by="\r\n" --user=YOUR_USERNAME --password YOUR_DATABASE YOUR_TABLE.csv | |
# mysqlimport --fields-optionally-enclosed-by=""" --fields-terminated-by=, --lines-terminated-by="\n" --user=root --password password SGTEST *.csv | |
# Then I manually run: | |
# mysql -u root -ppassword SGTEST < Rule.out |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment