Skip to content

Instantly share code, notes, and snippets.

@Flexi23
Created September 13, 2016 21:24
Show Gist options
  • Save Flexi23/a3ad79709dfd19f96a1c7f0bb0285b77 to your computer and use it in GitHub Desktop.
Save Flexi23/a3ad79709dfd19f96a1c7f0bb0285b77 to your computer and use it in GitHub Desktop.
Web Worker using Transferable ArrayBuffers with Float32 views to parse a tab separated value file. The first lines is expected to contain the titles. Also returns the global minimum, maximum, and average value per column.
onmessage = function(e) {
var arrayBuffers = [];
var float32Arrays = [];
var titles = [];
var reader = new FileReader();
reader.onload = function (progressEvent) {
var before = Date.now();
var lines = this.result.split('\n');
var endsWithNewLine = (lines[lines.length - 1] != "");
var statshtml = (lines.length - (endsWithNewLine ? 2 : 1)) + " samples loaded<br />";
titles = lines[0].split('\t');
var min = []; max = []; avg = [];
for(var title = 0; title < titles.length; title++){
min[title] = Number.POSITIVE_INFINITY;
max[title] = Number.NEGATIVE_INFINITY;
avg[title] = 0;
arrayBuffers[title] = new ArrayBuffer((lines.length - (endsWithNewLine ? 1 : 0))*4);
float32Arrays[title] = new Float32Array(arrayBuffers[title]);
}
var progress = 0;
for (var line = 1; line < (lines.length - (endsWithNewLine ? 2 : 1)); line++) {
var newProgress = Math.floor(100 * line / (lines.length+1));
if(newProgress != progress){
progress = newProgress;
postMessage({statshtml: "parsing " + progress + "%<br/>"});
}
val = lines[line].split('\t');
for(var title = 0; title < titles.length; title++){
var v = Number(val[title]);
if(min[title] > v){
min[title] = v;
}
if(max[title] < v){
max[title] = v;
}
if(avg[title] == undefined){
avg[title] = 0;
}
avg[title] += v /(lines.length-1);
float32Arrays[title][line-1] = v;
}
}
var escapedTitles = [];
for(var title = 0; title < titles.length; title++){
var escapedTitle = titles[title].replace(/"/g,'');
escapedTitles.push(escapedTitle);
statshtml += escapedTitle + ": "
+ min[title] + " - " + max[title] + " ("
+ Math.round(avg[title]*10)/10 + ") <br/>";
}
statshtml += "parsing time [ms]: " + (Date.now() - before) + "<br/>";
postMessage({
statshtml: statshtml,
stats: {min: min, max: max, avg: avg},
titles: escapedTitles,
arrayBuffers: arrayBuffers,
time: Date.now()
}, arrayBuffers);
};
reader.readAsText(e.data);
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment