Created
September 13, 2016 21:24
-
-
Save Flexi23/a3ad79709dfd19f96a1c7f0bb0285b77 to your computer and use it in GitHub Desktop.
Web Worker using Transferable ArrayBuffers with Float32 views to parse a tab separated value file. The first lines is expected to contain the titles. Also returns the global minimum, maximum, and average value per column.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
onmessage = function(e) { | |
var arrayBuffers = []; | |
var float32Arrays = []; | |
var titles = []; | |
var reader = new FileReader(); | |
reader.onload = function (progressEvent) { | |
var before = Date.now(); | |
var lines = this.result.split('\n'); | |
var endsWithNewLine = (lines[lines.length - 1] != ""); | |
var statshtml = (lines.length - (endsWithNewLine ? 2 : 1)) + " samples loaded<br />"; | |
titles = lines[0].split('\t'); | |
var min = []; max = []; avg = []; | |
for(var title = 0; title < titles.length; title++){ | |
min[title] = Number.POSITIVE_INFINITY; | |
max[title] = Number.NEGATIVE_INFINITY; | |
avg[title] = 0; | |
arrayBuffers[title] = new ArrayBuffer((lines.length - (endsWithNewLine ? 1 : 0))*4); | |
float32Arrays[title] = new Float32Array(arrayBuffers[title]); | |
} | |
var progress = 0; | |
for (var line = 1; line < (lines.length - (endsWithNewLine ? 2 : 1)); line++) { | |
var newProgress = Math.floor(100 * line / (lines.length+1)); | |
if(newProgress != progress){ | |
progress = newProgress; | |
postMessage({statshtml: "parsing " + progress + "%<br/>"}); | |
} | |
val = lines[line].split('\t'); | |
for(var title = 0; title < titles.length; title++){ | |
var v = Number(val[title]); | |
if(min[title] > v){ | |
min[title] = v; | |
} | |
if(max[title] < v){ | |
max[title] = v; | |
} | |
if(avg[title] == undefined){ | |
avg[title] = 0; | |
} | |
avg[title] += v /(lines.length-1); | |
float32Arrays[title][line-1] = v; | |
} | |
} | |
var escapedTitles = []; | |
for(var title = 0; title < titles.length; title++){ | |
var escapedTitle = titles[title].replace(/"/g,''); | |
escapedTitles.push(escapedTitle); | |
statshtml += escapedTitle + ": " | |
+ min[title] + " - " + max[title] + " (" | |
+ Math.round(avg[title]*10)/10 + ") <br/>"; | |
} | |
statshtml += "parsing time [ms]: " + (Date.now() - before) + "<br/>"; | |
postMessage({ | |
statshtml: statshtml, | |
stats: {min: min, max: max, avg: avg}, | |
titles: escapedTitles, | |
arrayBuffers: arrayBuffers, | |
time: Date.now() | |
}, arrayBuffers); | |
}; | |
reader.readAsText(e.data); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment