Last active
February 12, 2018 20:25
-
-
Save cedricpinson/4dfeedab2773cad646ea3cf6f98783fc to your computer and use it in GitHub Desktop.
parser csv data in rust
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// -*- compile-command: "cargo build ; cargo run ../../sample.csv" -*- | |
// This is a comment, and will be ignored by the compiler | |
// You can test this code by clicking the "Run" button over there -> | |
// or if prefer to use your keyboard, you can use the "Ctrl + Enter" shortcut | |
// This code is editable, feel free to hack it! | |
// You can always return to the original code by clicking the "Reset" button -> | |
use std::env; | |
use std::thread; | |
use std::fs::File; | |
use std::io::Read; | |
#[macro_use] | |
extern crate serde_derive; | |
extern crate serde; | |
extern crate serde_json; | |
use serde_json::{Value, Error}; | |
fn parse_test_args(argv: Vec<&str>) -> Vec<String> { | |
argv.iter().map(|&s| s.to_string()).collect::<Vec<String>>() | |
} | |
fn process_worker(data_thread: &Vec<String> ) { | |
let mut material_specular_metalness_off = 0; | |
let mut material_albedo_diffuse = 0; | |
let mut material_specular_metalness = 0; | |
for line in data_thread { | |
if line.len() == 0 { | |
continue; | |
} | |
let data_split: Vec<&str> = line.splitn(2, ',').collect(); | |
let mut model_id = data_split[0]; | |
model_id = &model_id[1 .. model_id.len()-1]; | |
let mut materials = data_split[1]; | |
let materials_striped = &materials[1 .. materials.len()-1].to_string(); | |
let materials2 = (&materials_striped).replace("\"\"","\""); | |
let json_struct: Value = serde_json::from_str(&materials2).unwrap(); | |
let mut materials_only = json_struct.as_object().unwrap().clone(); | |
materials_only.remove("updatedAt"); | |
let mut material_diffuse = false; | |
let mut material_albedo = false; | |
let mut material_metalness = false; | |
let mut material_specular = false; | |
for pair in materials_only.iter() { | |
let data = pair.1; | |
let channels = &data["channels"]; | |
let diffuse_pbr = &channels["DiffusePBR"]; | |
let albedo_pbr = &channels["AlbedoPBR"]; | |
let metalness_pbr = &channels["MetalnessPBR"]; | |
let specular_pbr = &channels["SpecularPBR"]; | |
let enable_diffuse_pbr = diffuse_pbr["enable"].as_bool().unwrap(); | |
let enable_albedo_pbr = albedo_pbr["enable"].as_bool().unwrap(); | |
let enable_metalness_pbr = metalness_pbr["enable"].as_bool().unwrap(); | |
let enable_specular_pbr = specular_pbr["enable"].as_bool().unwrap(); | |
if enable_diffuse_pbr { | |
material_diffuse = true; | |
} | |
if enable_albedo_pbr { | |
material_albedo = true; | |
} | |
if enable_metalness_pbr { | |
material_metalness = true; | |
} | |
if enable_specular_pbr { | |
material_specular = true; | |
} | |
if !enable_metalness_pbr && !enable_specular_pbr { | |
material_specular_metalness_off += 1; | |
println!("model id {}", model_id); | |
break; | |
} | |
if material_albedo && material_diffuse { | |
material_albedo_diffuse += 1; | |
println!("model id {}", model_id); | |
break; | |
} | |
if material_metalness && material_specular { | |
material_specular_metalness += 1; | |
println!("model id {}", model_id); | |
break; | |
} | |
} | |
} | |
println!("case specular/metalness {}, albedo/diffuse {}, specular/metalness off {}", material_specular_metalness, material_albedo_diffuse, material_specular_metalness_off); | |
} | |
// This is the main function | |
fn main() { | |
// The statements here will be executed when the compiled binary is called | |
let mut contents = String::new(); | |
let args: Vec<String> = env::args().collect(); | |
if args.len() > 1 { | |
let mut file = File::open(&args[1]).expect("Unable to open"); | |
file.read_to_string(&mut contents).expect("could not read file"); | |
let lines_contents = contents.split('\n').collect::<Vec<&str>>(); | |
let last_lines = lines_contents.len(); | |
let lines = &lines_contents[1..last_lines]; | |
let nb_lines = lines.len(); | |
let mut children = vec![]; | |
let mut wanted_core = 8; | |
if nb_lines < wanted_core { | |
wanted_core = nb_lines; | |
} | |
let max_core = wanted_core; | |
let nb_lines_per_batch = nb_lines/max_core; | |
for n in 0..max_core { | |
let start = nb_lines_per_batch*n; | |
let end = start + nb_lines_per_batch; | |
println!("processed segment {} - {}-{} : {}", n, start, end, max_core); | |
if end > nb_lines { | |
break; | |
} | |
//let data_segment = &lines[start..end]; | |
let mut data_thread = Vec::new(); | |
for i in start..end { | |
data_thread.push(String::from(lines[i])); | |
} | |
children.push(thread::spawn( move || process_worker(&data_thread))); | |
} | |
for child in children { | |
// Wait for the thread to finish. Returns a result. | |
let _ = child.join(); | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment