chunked-fread.R

# dummy dataset
require(gutenbergr)

d = ggplot2::diamonds
d$txt = gutenberg_download(1184)$text[1:nrow(d)]
filename = "~/Downloads/dat.csv"
readr::write_csv(d, filename)

#-------------------

require(data.table)

chunksize = 1000
rowcount = as.numeric(str_extract(system('wc -l ~/Downloads/dat.csv', intern = TRUE), '[0-9]+'))
field_names = strsplit(readLines(filename, n = 1), ',')[[1]]

d = lapply(seq_len(ceiling(rowcount / chunksize)), function(i){
  x = fread(filename, skip = (i-1) * chunksize, nrows = chunksize, col.names = field_names)
  x[stringr::str_detect(txt, 'Count[^a-z]')]
})

bind_rows(d)