Skip to content

Instantly share code, notes, and snippets.

@mr-karan
Last active June 10, 2025 13:40
Show Gist options
  • Save mr-karan/a77986e66534f7d98e3f4f6aa7281772 to your computer and use it in GitHub Desktop.
Save mr-karan/a77986e66534f7d98e3f4f6aa7281772 to your computer and use it in GitHub Desktop.
Python Function to Split CSV File with headers
# coding: utf-8
import csv
import os
from itertools import tee, islice
LINES_SPLIT = 1000
def chunks(l, n):
"""
Source: https://stackoverflow.com/a/22045226/2369204
"""
it = iter(l)
return iter(lambda: tuple(islice(it, n)), ())
directory = "output"
if not os.path.exists(directory):
os.makedirs(directory)
print("created directory")
files = list(filter(lambda f : f.endswith('csv'), os.listdir(".")))
for file in files:
with open(files[0],'r') as infile:
reader = csv.DictReader(infile)
for counter, rows in enumerate(chunks(reader, LINES_SPLIT), start=1):
headers = rows[0].keys()
print("splitting part {} of {}".format(counter,file))
with open("output/{}_split_{}.{}".format(file.split(".")[0], counter, file.split(".")[1]), mode='w') as out_file:
dict_writer = csv.DictWriter(out_file, headers, quotechar='\"',quoting=csv.QUOTE_ALL, lineterminator="\r\n")
dict_writer.writeheader()
dict_writer.writerows(rows)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment