Last active
September 7, 2017 19:56
-
-
Save KobaKhit/abfef3f4c5ef293bc2aa739b90edeabb to your computer and use it in GitHub Desktop.
Write a large dataframe to csv in chunks
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
df = read.csv("your-df.csv") | |
# Number of items in each chunk | |
elements_per_chunk = 100000 | |
# List of vectors [1] 1:100000, [2] 100001:200000, ... | |
l = split(1:nrow(df), ceiling(seq_along(1:nrow(df))/elements_per_chunk)) | |
# Write large data frame to csv in chunks | |
fname = "inventory-cleaned.csv" | |
count = 1 | |
for(i in l){ | |
print(count) | |
write.table(inventory[i,], | |
file=fname, | |
sep = ",", | |
row.names = FALSE, | |
append = TRUE, | |
col.names=!file.exists(fname)) # dont append column names | |
count = count + 1 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment