Another useful bit of code for preparing flat files for Hive. Takes in csvs with double quote text delimiters and outputs pipe delimited files.

csvToPipe.py

import os, csv

progDir = '/pathToFolderContainingCSVs/'



for filename in os.listdir(progDir):
    if filename != '.DS_Store':
        with open(progDir+filename, 'rb') as csvfile:
            progReader = csv.reader(csvfile, delimiter=',', quotechar='"')
            with open(progDir+filename[:-4] + "T.txt", 'wb') as f:
                writer = csv.writer(f,  delimiter='\t')
                for row in progReader:
                    writer.writerow(row)