Skip to content

Instantly share code, notes, and snippets.

@cedricpinson
Last active February 12, 2018 20:25
Show Gist options
  • Save cedricpinson/4dfeedab2773cad646ea3cf6f98783fc to your computer and use it in GitHub Desktop.
Save cedricpinson/4dfeedab2773cad646ea3cf6f98783fc to your computer and use it in GitHub Desktop.
parser csv data in rust
// -*- compile-command: "cargo build ; cargo run ../../sample.csv" -*-
// This is a comment, and will be ignored by the compiler
// You can test this code by clicking the "Run" button over there ->
// or if prefer to use your keyboard, you can use the "Ctrl + Enter" shortcut
// This code is editable, feel free to hack it!
// You can always return to the original code by clicking the "Reset" button ->
use std::env;
use std::thread;
use std::fs::File;
use std::io::Read;
#[macro_use]
extern crate serde_derive;
extern crate serde;
extern crate serde_json;
use serde_json::{Value, Error};
fn parse_test_args(argv: Vec<&str>) -> Vec<String> {
argv.iter().map(|&s| s.to_string()).collect::<Vec<String>>()
}
fn process_worker(data_thread: &Vec<String> ) {
let mut material_specular_metalness_off = 0;
let mut material_albedo_diffuse = 0;
let mut material_specular_metalness = 0;
for line in data_thread {
if line.len() == 0 {
continue;
}
let data_split: Vec<&str> = line.splitn(2, ',').collect();
let mut model_id = data_split[0];
model_id = &model_id[1 .. model_id.len()-1];
let mut materials = data_split[1];
let materials_striped = &materials[1 .. materials.len()-1].to_string();
let materials2 = (&materials_striped).replace("\"\"","\"");
let json_struct: Value = serde_json::from_str(&materials2).unwrap();
let mut materials_only = json_struct.as_object().unwrap().clone();
materials_only.remove("updatedAt");
let mut material_diffuse = false;
let mut material_albedo = false;
let mut material_metalness = false;
let mut material_specular = false;
for pair in materials_only.iter() {
let data = pair.1;
let channels = &data["channels"];
let diffuse_pbr = &channels["DiffusePBR"];
let albedo_pbr = &channels["AlbedoPBR"];
let metalness_pbr = &channels["MetalnessPBR"];
let specular_pbr = &channels["SpecularPBR"];
let enable_diffuse_pbr = diffuse_pbr["enable"].as_bool().unwrap();
let enable_albedo_pbr = albedo_pbr["enable"].as_bool().unwrap();
let enable_metalness_pbr = metalness_pbr["enable"].as_bool().unwrap();
let enable_specular_pbr = specular_pbr["enable"].as_bool().unwrap();
if enable_diffuse_pbr {
material_diffuse = true;
}
if enable_albedo_pbr {
material_albedo = true;
}
if enable_metalness_pbr {
material_metalness = true;
}
if enable_specular_pbr {
material_specular = true;
}
if !enable_metalness_pbr && !enable_specular_pbr {
material_specular_metalness_off += 1;
println!("model id {}", model_id);
break;
}
if material_albedo && material_diffuse {
material_albedo_diffuse += 1;
println!("model id {}", model_id);
break;
}
if material_metalness && material_specular {
material_specular_metalness += 1;
println!("model id {}", model_id);
break;
}
}
}
println!("case specular/metalness {}, albedo/diffuse {}, specular/metalness off {}", material_specular_metalness, material_albedo_diffuse, material_specular_metalness_off);
}
// This is the main function
fn main() {
// The statements here will be executed when the compiled binary is called
let mut contents = String::new();
let args: Vec<String> = env::args().collect();
if args.len() > 1 {
let mut file = File::open(&args[1]).expect("Unable to open");
file.read_to_string(&mut contents).expect("could not read file");
let lines_contents = contents.split('\n').collect::<Vec<&str>>();
let last_lines = lines_contents.len();
let lines = &lines_contents[1..last_lines];
let nb_lines = lines.len();
let mut children = vec![];
let mut wanted_core = 8;
if nb_lines < wanted_core {
wanted_core = nb_lines;
}
let max_core = wanted_core;
let nb_lines_per_batch = nb_lines/max_core;
for n in 0..max_core {
let start = nb_lines_per_batch*n;
let end = start + nb_lines_per_batch;
println!("processed segment {} - {}-{} : {}", n, start, end, max_core);
if end > nb_lines {
break;
}
//let data_segment = &lines[start..end];
let mut data_thread = Vec::new();
for i in start..end {
data_thread.push(String::from(lines[i]));
}
children.push(thread::spawn( move || process_worker(&data_thread)));
}
for child in children {
// Wait for the thread to finish. Returns a result.
let _ = child.join();
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment