Skip to content

Instantly share code, notes, and snippets.

@jmsdnns
Created October 9, 2024 22:28
Show Gist options
  • Save jmsdnns/9dfb37c19929ec762315d0345b2b3233 to your computer and use it in GitHub Desktop.
Save jmsdnns/9dfb37c19929ec762315d0345b2b3233 to your computer and use it in GitHub Desktop.
Reading a file into chunks using Tokio's File tools
// TOTAL CHUNKS: 5
// - chunk: 5242880
// - chunk: 5242880
// - chunk: 5242880
// - chunk: 5242880
// - chunk: 1974756
use tokio::{fs::File, io, io::AsyncReadExt, io::AsyncSeekExt};
pub async fn read_chunk(file_path: &str, start: u64, end: u64) -> io::Result<Vec<u8>> {
let mut file = File::open(file_path).await?;
let mut buffer = vec![0; (end - start) as usize];
file.seek(io::SeekFrom::Start(start)).await?;
file.read_exact(&mut buffer).await?;
Ok(buffer)
}
pub async fn read_chunks(file_path: &str, chunk_size: u64) -> io::Result<Vec<Vec<u8>>> {
let file = File::open(file_path).await.unwrap();
let length = file.metadata().await.unwrap().len();
let num_chunks = (length as f32 / chunk_size as f32).ceil() as u64;
let mut chunks: Vec<Vec<u8>> = Vec::new();
for c in 0..num_chunks {
let start = c * chunk_size;
let remaining = length - start;
let mut take_size = chunk_size;
if remaining < chunk_size {
take_size = remaining;
}
let Ok(chunk) = read_chunk(file_path, start, start + take_size).await else {
panic!("AAAAAHHHHHHHHH");
};
chunks.push(chunk);
}
Ok(chunks)
}
#[tokio::main]
async fn main() {
let path = "cantsleepcrew.mp3";
let chunk_size = 5 * 1024 * 1024;
let chunks = read_chunks(path, chunk_size).await.unwrap();
println!("TOTAL CHUNKS: {}", chunks.len());
for c in chunks.iter() {
println!("- chunk: {}", c.len());
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment