Python script to import BreachCompilation to PostgreSQL database.

import.py

# Define imports.
from time import time, strftime
from datetime import datetime
from glob import iglob
from os import path, system
from sys import exc_info
from subprocess import Popen, PIPE
from curses import initscr, noecho, cbreak, echo, nocbreak, endwin
from psycopg2 import connect

# Function to format time.
def formatTime(time):
	# Define the time variables.
	day = time // (24 * 3600)
	time = time % (24 * 3600)
	
	hour = time // 3600
	time %= 3600
	
	minutes = time // 60
	time %= 60
	
	seconds = time
	
	# Format and return the time.
	formatTime = "%d:%d:%d:%d" % (day, hour, minutes, seconds)
	return formatTime

# Define local variables.
run = True
complete = False
timeStart = time()
root = path.abspath(path.dirname(__file__))

# Define import variables.
files = list()
fileCheck = 0
lineCheck = 0
totalFiles = 0
totalLines = 1
totalErrors = 0
			
# Define database variables.
con = connect(host="localhost", user="**********", password="**********", dbname="**********")
con.set_client_encoding("UNICODE")
cur = con.cursor()

# Initialize the script.
system("clear")
print("IMPORTER")
print("A program to import credentials into a PostgreSQL database.\n")

# Sanitize user input.
try:
	location = input("Folder Path: ")
except KeyboardInterrupt:	
	# Quit the program.
	system("clear")
	run = False
	quit()
	
if(location[len(location) - 1] == "/"):
	location = location[:len(location) - 1]

# Change to cursor screen.
screen = initscr()
noecho()
cbreak()

# Check for folder.
if(path.isdir(location)):
	# Loop through files recursively.
	for dir in iglob(location + "/**/*", recursive=True):
		# Check for exit.
		try:
			# Check for file.
			if(path.isfile(dir)):
				# Update the information.
				files.append(dir)
				totalFiles += 1
				totalLines += int(Popen(["wc", "-l", dir], stdout=PIPE).communicate()[0].decode().split(" ")[0])
				
				# Output live information.
				screen.clear()
				screen.addstr(0, 0, "INITIALIZING IMPORT...")
				screen.addstr(1, 0, "Getting things read to start importing.")
				screen.addstr(3, 0, "Files: " + str(totalFiles))
				screen.addstr(4, 0, "Lines: " + str(totalLines))
				screen.addstr(5, 0, "")
				screen.refresh()
		except KeyboardInterrupt:
			# Reset the screen
			screen.keypad(0)
			echo()
			nocbreak()
			endwin()

			# Break the loop
			run = False
			break
# Check for file.
elif(path.isfile(location)):
	# Add to array.
	files.append(location)
	totalFiles += 1
	totalLines += int(Popen(["wc", "-l", location], stdout=PIPE).communicate()[0].decode().split(" ")[0])
else:
	# Reset the screen
	screen.keypad(0)
	echo()
	nocbreak()
	endwin()
			
	# Reset the screen.
	print("Cannot find specified location.")
	run = False
	quit();

# Ensure the program is running.
while run:
	# Handle errors.
	try:
		# Loop through all files in array.
		for i in range(0, len(files)):
			lineCount = 0
			lineFile = int(Popen(["wc", "-l", files[i]], stdout=PIPE).communicate()[0].decode().split(" ")[0])
					
			# Loop through lines in files
			file = open(files[i], encoding="latin-1", errors="surrogateescape")
			for line in file:
				try:
					# Increment the counters.
					lineCount += 1
					lineCheck += 1

					# Parse out the credentials.
					line = line.rstrip("\n")
					pos = line.find(":")
					if(pos == -1):
						pos = line.find(";")
						
					# Get the credential variables.
					username = line[:pos]
					password = line[pos+1:]
					
					# Output live information.
					screen.clear()
					screen.addstr(0, 0, "IMPORTING CREDENTIALS...")
					screen.addstr(1, 0, "Importing the data into the database.")
					screen.addstr(3, 0, "FILES")
					screen.addstr(4, 0, "Path: " + files[i])
					screen.addstr(5, 0, "Username: " + username[:40])
					screen.addstr(6, 0, "Password: " + password[:40])
					screen.addstr(7, 0, "Lines: " + str(lineCount) + "/" + str(lineFile))
					screen.addstr(8, 0, "Rate: " + str(round(lineCount / (time() - timeStart), 3)) + " str/s")
					screen.addstr(10, 0, "TOTAL")
					screen.addstr(11, 0, "Files: " + str(fileCheck) + "/" + str(totalFiles))
					screen.addstr(12, 0, "Lines: " + str(lineCheck) + "/" + str(totalLines))
					screen.addstr(13, 0, "Errors: " + str(totalErrors))
					screen.addstr(14, 0, "Rate: " + str(round(lineCheck / (time() - timeStart), 3)) + " str/s")
					screen.addstr(15, 0, "Time: " + formatTime(round(time() - timeStart)))
					screen.addstr(16, 0, "Complete: " + str(round((lineCheck / totalLines) * 100, 3)) + "%")
					screen.addstr(17, 0, "")
					screen.refresh()
					
					# Import to the database.
					cur.execute("INSERT INTO clear(username, password) VALUES(%s, %s);", [username, password])
					con.commit()
				except KeyboardInterrupt:
					raise
				except:
					# Check if the files exists.
					if(path.exists(root + "/errors.log") == False):
						new = True
					else:
						new = False
					
					# Log the error's log.
					log = open(root + "/errors.log", "a")
					if(new):
						log.write("{0:<20}".format("TIME"))
						log.write("{0:<30}".format("FILE PATH"))
						log.write("{0:<10}".format("LINE"))
						log.write("{0:<25}".format("USERNAME"))
						log.write("{0:<25}".format("PASSWORD"))
						log.write("{0:<30}".format("TYPE"))
						log.write("{0:<100}\n".format("INFO"))
					
					# Log the error.
					log.write("{0:<20}".format(strftime("%x") + " " + strftime("%X")))
					log.write("{0:<30}".format(files[i]))
					log.write("{0:<10}".format(str(lineCount)))
					log.write("{0:<25}".format(username[:20]))
					log.write("{0:<25}".format(password[:20]))
					log.write("{0:<30}".format(str(exc_info()[0])))
					log.write("{0:<100}\n".format(str(exc_info()[1])))
					log.close()
					
					# Increment the counters.
					totalErrors += 1
								
					# Resume on error.
					pass
			file.close()
			fileCheck += 1
			
			# Check for end of files array.
			if i == len(files) - 1:
				# Reset the screen
				screen.keypad(0)
				echo()
				nocbreak()
				endwin()

				# Close the database connection.
				cur.close()
				con.close()

				# Break the loop
				complete = True
				run = False
	except KeyboardInterrupt:
		# Reset the screen
		screen.keypad(0)
		echo()
		nocbreak()
		endwin()

		# Close the database connection.
		cur.close()
		con.close()

		# Break the loop
		run = False
		
# Output final information.
system("clear")
if(complete):
	print("IMPORT COMPLETE\n")
else:
	print("IMPORT CANCELED\n")
print("Files: " + str(fileCheck) + "/" + str(totalFiles))
print("Lines: " + str(lineCheck) + "/" + str(totalLines))
print("Errors: " + str(totalErrors))
print("Time: " + formatTime(round(time() - timeStart)))
print("Rate: " + str(round(lineCheck / (time() - timeStart), 3)) + " str/s")
print("Complete: " + str(round((lineCheck / totalLines) * 100, 3)) + "%")

https://www.reddit.com/r/pwned/comments/7hhqfo/combination_of_many_breaches/