Skip to content

Instantly share code, notes, and snippets.

@oshikryu

oshikryu/datamaker.js

Last active May 2, 2017 00:03
Show Gist options
  • Save oshikryu/3a3dbab3a74b1b68bdf8e303890bc171 to your computer and use it in GitHub Desktop.
Save oshikryu/3a3dbab3a74b1b68bdf8e303890bc171 to your computer and use it in GitHub Desktop.
Download ZIP
CSV export with webworkers
Raw
datamaker.js
// this file expects global variables data and compareData to be available
var regData = data;
var doubleData = data.concat(data);
var doubleCompareData = compareData.concat(compareData);
var millionData = [];
for (var idx=0; idx < 10; idx += 1) {
millionData = millionData.concat(doubleCompareData);
}
Raw
index.html
<html>
<title>
testing csv export
</title>
<head>
<script src="compare_data.js"></script>
<script src="data.js"></script>
<script src="worker.js"></script>
<script src="index.js"></script>
<script src="datamaker.js"></script>
</head>
<body>
<button onClick="exportMe(data)">Click to export 50k data points</button>
<button onClick="exportMe(doubleData)">Click to export 100k data points</button>
<button onClick="exportMe(compareData)">Click to export 50k compare by machine</button>
<button onClick="exportMe(doubleCompareData)">Click to export 100k compare by machine</button>
<button onClick="exportMe(millionData)">Click to export a million compare by machine</button>
<textarea type="textarea">
</textarea>
</body>
</html>
Raw
index.js
var myWorker = new Worker('worker.js');
var csvFile, blob;
var clickStart, clickEnd;
/* This method will use the data from data.js and trigger creation of a csv
* @method exportMe
*/
function exportMe(data) {
clickStart = new Date().getTime();
csvFile = getCSV(data);
}
function getCSV(data) {
console.log('Data length is: ' + data.length);
console.info('Starting call for csv format');
var csvFormatTimeStart = new Date().getTime();
var csvFile = workerMaker('csvFormat', data);
var csvFormatTimeEnd = new Date().getTime();
var csvFormatTime = csvFormatTimeEnd - csvFormatTimeStart;
console.log('csv format takes ' + csvFormatTime + ' ms to run');
}
function getBlob(csvFile) {
console.log('creating blob...')
var blobFormatTimeStart = new Date().getTime();
var blob = workerMaker('blobber', csvFile);
var blobFormatTimeEnd = new Date().getTime();
var blobFormatTime = blobFormatTimeEnd - blobFormatTimeStart;
console.log('blob format takes ' + blobFormatTime + ' ms to run');
}
function workerMaker(type, arg) {
if (window.Worker) {
myWorker.postMessage({type: type, arg: arg});
}
}
function saveFile(blob) {
var uniqTime = new Date().getTime();
var filename = 'my_file_' + uniqTime;
if (navigator.msSaveBlob) { // IE 10+
console.info('Starting call for ' + 'ie download');
var csvFormatTimeStart = new Date().getTime();
var ieFilename = filename + '.csv';
navigator.msSaveBlob(blob, ieFilename);
var csvFormatTimeEnd = new Date().getTime();
var csvFormatTime = csvFormatTimeEnd - csvFormatTimeStart;
console.log('ie download takes ' + csvFormatTime + ' ms to run');
} else {
console.info('Starting call for ' + 'regular download');
var csvFormatTimeStart = new Date().getTime();
var link = document.createElement("a");
if (link.download !== undefined) { // feature detection
// Browsers that support HTML5 download attribute
var url = URL.createObjectURL(blob);
link.setAttribute("href", url);
link.setAttribute("download", filename);
link.style.visibility = 'hidden';
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
}
var csvFormatTimeEnd = new Date().getTime();
var csvFormatTime = csvFormatTimeEnd - csvFormatTimeStart;
console.log('regular download takes ' + csvFormatTime + ' ms to run');
}
clickEnd = new Date().getTime();
console.log('The whole process took: ' + (clickEnd - start) + ' ms');
}
myWorker.onmessage = function(e) {
console.log('Message received from worker');
var response = e.data;
var data = response.data;
var type = response.type;
if (type === 'csvFormat') {
getBlob(data);
} else if (type === 'blobber') {
saveFile(data);
} else {
console.error('you dun goofed')
}
}
Raw
worker.js
if( 'function' === typeof importScripts) {
importScripts('https://cdnjs.cloudflare.com/ajax/libs/d3/4.8.0/d3.min.js')
onmessage = function(e) {
var data = e.data;
var type = data.type;
var arg = data.arg;
console.log('Message received from main script');
switch (type) {
case 'csvFormat':
// start timer -----------------
console.log('Posting message back to main script');
var timeStart = new Date().getTime();
// -----------------------------
var res = d3.csvFormat(arg);
// end timer -------------------
var timeEnd = new Date().getTime();
var timeDiff = timeEnd - timeStart;
console.log('blobbing takes ' + timeDiff + ' ms to run');
// -----------------------------
postMessage({
type: type,
data: res,
});
break;
case 'blobber':
// start timer -----------------
console.log('Posting message back to main script');
var timeStart = new Date().getTime();
// -----------------------------
var blob = new Blob([arg], { type: 'text/csv;charset=utf-8;' });
// end timer -------------------
var timeEnd = new Date().getTime();
var timeDiff = timeEnd - timeStart;
console.log('blobbing takes ' + timeDiff + ' ms to run');
// -----------------------------
postMessage({
type: type,
data: blob,
});
break;
default:
console.error('invalid stuff');
break;
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment