Created
February 4, 2023 07:08
-
-
Save gz/e6cd4cc7512c98b2e2d6b489981793c0 to your computer and use it in GitHub Desktop.
some ways to read files in rust, mmap is fastest
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
use std::time::Instant; | |
use std::fs::File; | |
use std::io::{self, prelude::*, BufReader}; | |
let file = File::open("examples/WDC_tickbidask.csv")?; | |
let mut reader = BufReader::new(file); | |
let mut line = String::new(); | |
let mut lines = 0; | |
let start = Instant::now(); | |
while reader.read_line(&mut line).unwrap() > 0 { | |
//println!("{}", line); | |
line.clear(); | |
lines += 1; | |
} | |
println!("{lines} {:?}", start.elapsed()); | |
use memmap2::Mmap; | |
let mut file = File::open("examples/WDC_tickbidask.csv")?; | |
let mmap = unsafe { Mmap::map(&file)? }; | |
let mut lines = 0; | |
let start = Instant::now(); | |
for line in mmap.split(|c| *c as char == '\n') { | |
//println!("{}", line); | |
lines += 1; | |
} | |
println!("{:?}", start.elapsed()); | |
let file = File::open("examples/WDC_tickbidask.csv")?; | |
let reader = BufReader::new(file); | |
let start = Instant::now(); | |
for line in reader.lines() { | |
//println!("{}", line?); | |
} | |
println!("{:?}", start.elapsed()); | |
let mut rdr = csv::ReaderBuilder::new() | |
.buffer_capacity(2*1024*1024) | |
.quoting(false) | |
.from_path("examples/WDC_tickbidask.csv").unwrap(); | |
let headers = rdr.byte_headers()?.clone(); | |
let mut raw_record = csv::ByteRecord::new(); | |
let start = Instant::now(); | |
while rdr.read_byte_record(&mut raw_record)? { | |
//let record: TickerData = raw_record.deserialize(Some(&headers))?; | |
//println!("{:?}", raw_record); | |
} | |
println!("{:?}", start.elapsed()); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment