Created
October 9, 2024 22:28
-
-
Save jmsdnns/9dfb37c19929ec762315d0345b2b3233 to your computer and use it in GitHub Desktop.
Reading a file into chunks using Tokio's File tools
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// TOTAL CHUNKS: 5 | |
// - chunk: 5242880 | |
// - chunk: 5242880 | |
// - chunk: 5242880 | |
// - chunk: 5242880 | |
// - chunk: 1974756 | |
use tokio::{fs::File, io, io::AsyncReadExt, io::AsyncSeekExt}; | |
pub async fn read_chunk(file_path: &str, start: u64, end: u64) -> io::Result<Vec<u8>> { | |
let mut file = File::open(file_path).await?; | |
let mut buffer = vec![0; (end - start) as usize]; | |
file.seek(io::SeekFrom::Start(start)).await?; | |
file.read_exact(&mut buffer).await?; | |
Ok(buffer) | |
} | |
pub async fn read_chunks(file_path: &str, chunk_size: u64) -> io::Result<Vec<Vec<u8>>> { | |
let file = File::open(file_path).await.unwrap(); | |
let length = file.metadata().await.unwrap().len(); | |
let num_chunks = (length as f32 / chunk_size as f32).ceil() as u64; | |
let mut chunks: Vec<Vec<u8>> = Vec::new(); | |
for c in 0..num_chunks { | |
let start = c * chunk_size; | |
let remaining = length - start; | |
let mut take_size = chunk_size; | |
if remaining < chunk_size { | |
take_size = remaining; | |
} | |
let Ok(chunk) = read_chunk(file_path, start, start + take_size).await else { | |
panic!("AAAAAHHHHHHHHH"); | |
}; | |
chunks.push(chunk); | |
} | |
Ok(chunks) | |
} | |
#[tokio::main] | |
async fn main() { | |
let path = "cantsleepcrew.mp3"; | |
let chunk_size = 5 * 1024 * 1024; | |
let chunks = read_chunks(path, chunk_size).await.unwrap(); | |
println!("TOTAL CHUNKS: {}", chunks.len()); | |
for c in chunks.iter() { | |
println!("- chunk: {}", c.len()); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment