dedpulicater.py

import csv

lines = set()
outfile = open('Train.csv.cr.dedupe', 'wb')
outwriter = csv.writer(outfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
counter = 0
with open('Train.csv.CR_cleaned', 'rb') as csvfile:
	reader = csv.reader(csvfile, delimiter=',', quotechar='"', skipinitialspace=True)
	header = reader.next()
	outwriter.writerow(header)
	for row in reader:
		current = row[1] + row[2][:80]
		if current not in lines:
			lines.add(current)
			outwriter.writerow(row)
		counter += 1
		if counter % 1000 == 0:
			print counter