Skip to content

Instantly share code, notes, and snippets.

@gz
Created February 4, 2023 07:08
Show Gist options
  • Save gz/e6cd4cc7512c98b2e2d6b489981793c0 to your computer and use it in GitHub Desktop.
Save gz/e6cd4cc7512c98b2e2d6b489981793c0 to your computer and use it in GitHub Desktop.
some ways to read files in rust, mmap is fastest
use std::time::Instant;
use std::fs::File;
use std::io::{self, prelude::*, BufReader};
let file = File::open("examples/WDC_tickbidask.csv")?;
let mut reader = BufReader::new(file);
let mut line = String::new();
let mut lines = 0;
let start = Instant::now();
while reader.read_line(&mut line).unwrap() > 0 {
//println!("{}", line);
line.clear();
lines += 1;
}
println!("{lines} {:?}", start.elapsed());
use memmap2::Mmap;
let mut file = File::open("examples/WDC_tickbidask.csv")?;
let mmap = unsafe { Mmap::map(&file)? };
let mut lines = 0;
let start = Instant::now();
for line in mmap.split(|c| *c as char == '\n') {
//println!("{}", line);
lines += 1;
}
println!("{:?}", start.elapsed());
let file = File::open("examples/WDC_tickbidask.csv")?;
let reader = BufReader::new(file);
let start = Instant::now();
for line in reader.lines() {
//println!("{}", line?);
}
println!("{:?}", start.elapsed());
let mut rdr = csv::ReaderBuilder::new()
.buffer_capacity(2*1024*1024)
.quoting(false)
.from_path("examples/WDC_tickbidask.csv").unwrap();
let headers = rdr.byte_headers()?.clone();
let mut raw_record = csv::ByteRecord::new();
let start = Instant::now();
while rdr.read_byte_record(&mut raw_record)? {
//let record: TickerData = raw_record.deserialize(Some(&headers))?;
//println!("{:?}", raw_record);
}
println!("{:?}", start.elapsed());
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment