Skip to content

Instantly share code, notes, and snippets.

@TomHumphries

TomHumphries/make-csv-backpressure.js

Last active December 16, 2020 08:07
Show Gist options
  • Save TomHumphries/bd2ff018dc9904e96f37b496b5cf1d89 to your computer and use it in GitHub Desktop.
Save TomHumphries/bd2ff018dc9904e96f37b496b5cf1d89 to your computer and use it in GitHub Desktop.
Download ZIP
Node.js random-walk timeseries CSV generation script
Raw
make-csv-backpressure.js
/**
* Arguments:
* START DATE
* END DATE
* SECONDS BETWEEN POINTS
* NUMBER OF SIGNALS (csv columns)
*
* Example use for 1 month of 60-second-reolution data with 100 columns
* node make-csv-backpressure.js 2020-01-01 2020-02-01 60 100
*/
const fs = require('fs');
const path = require('path');
const args = process.argv.slice(2);
const fromStr = args.shift();
const toStr = args.shift();
let spacingSeconds = +args.shift();
let columnCount = +args.shift();
let start;
let end;
if (fromStr) {
start = new Date(fromStr);
} else {
// set to the start of the current year
start = new Date();
start.setUTCMonth(0);
start.setUTCDate(1);
start.setUTCHours(0, 0, 0, 0);
}
if (toStr) {
end = new Date(toStr);
} else {
// set to the start of the next year
end = new Date();
end.setUTCMonth(12);
end.setUTCDate(1);
end.setUTCHours(0, 0, 0, 0);
}
if (Number.isNaN(spacingSeconds)) {
spacingSeconds = 60 * 60; // no spacing supplied - set to 1 hour
}
// if no column-count is supplied, set it to 1
if (Number.isNaN(columnCount)) columnCount = 1;
// log what's being created to console
console.log(`Creating a timeseries CSV file...
columns: ${columnCount.toLocaleString()}
from: ${start.toISOString()}
to: ${end.toISOString()}
spacing: ${spacingSeconds.toLocaleString()} seconds`)
console.time('CSV created');
createCsv(start, end, columnCount, spacingSeconds)
.then(() => {
console.timeEnd('CSV created');
}).catch((err) => {
console.log('Failed', err);
})
function createCsv(start, end, columnCount, spacingSeconds) {
return new Promise((resolve, reject) => {
const filepath = path.join(__dirname, 'output.csv');
// wipe any existing file
fs.writeFileSync(filepath, '');
const writeStream = fs.createWriteStream(filepath, { flags: 'a' }); // 'a' is append
writeStream.on('error', (err) => {
console.log(err);
return reject(err);
})
// create the header row
const columns = ['timestamp'];
const values = []; // array to track the random walk
for (let i = 0; i < columnCount; i++) {
columns.push(`Series ${i + 1}`);
values.push(0);
}
// write the csv header
writeStream.write(`${columns.join(',')}\r\n`);
// https://nodejs.org/api/stream.html#stream_event_drain
// when the stream can take more data, resume the data-writing
writeStream.on('drain', write);
// start writing data to the stream
write();
function write() {
if (start.valueOf() >= end.valueOf()) {
return resolve();
}
const timestamp = new Date(start);
start.setUTCSeconds(start.getUTCSeconds() + spacingSeconds);
const cells = [];
cells.push(timestamp.toISOString());
for (let i = 0; i < columnCount; i++) {
const value = values[i];
cells.push(value.toPrecision(5));
values[i] += Math.random() - 0.5;
}
// if false, there is backpressure from the stream - stop sending new data
let ok = writeStream.write(`${cells.join(',')}\r\n`,);
if (ok) {
return write();
}
}
})
}
Raw
make-csv.html
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-giJF6kkoqNQ00vy+HMDP7azOuL0xtbfIcaT9wjKHr8RbDVddVHyTfAAsrekwKmP1" crossorigin="anonymous">
<title>CSV Random Walk Generator</title>
</head>
<body>
<div class="container py-5">
<h1 class="fw-light">CSV Generator</h1>
<p class="lead text-muted">Generate <a href="https://en.wikipedia.org/wiki/Random_walk">random walk</a> <a href="https://en.wikipedia.org/wiki/Time_series">time series data</a> in <a href="https://en.wikipedia.org/wiki/Comma-separated_values">comma separated value</a> format.</p>
<div class="row mb-3">
<div class="col-md-3">
<label for="columns" class="form-label">Columns</label>
<input type="number" value="5" min="1" step="1" name="columns" id="columns" class="form-control">
</div>
<div class="col-md-3">
<label for="spacing" class="form-label">Seconds spacing</label>
<input type="number" value="60" min="1" step="1" name="spacing" id="spacing" class="form-control">
</div>
<div class="col-md-3">
<label for="from" class="form-label">From</label>
<input type="datetime-local" name="from" id="from" class="form-control">
</div>
<div class="col-md-3">
<label for="to" class="form-label">To</label>
<input type="datetime-local" name="to" id="to" class="form-control">
</div>
<div class="col pt-3">
<button class="btn btn-success mt-1" onclick="onGenerateClick(false)">Generate + copy to
clipboard</button>
<button class="btn btn-outline-success mt-1" onclick="onGenerateClick(true)">Generate + display below
(slow for large datasets)</button>
</div>
</div>
<div class="mb-3">
<label for="output" class="form-label">CSV Output</label>
<textarea name="output" id="output" rows="15" class="form-control" readonly style="white-space: pre;"></textarea>
</div>
</div>
<script>
// pre-set the date time pickers
let _start = new Date();
_start.setUTCHours(0, 0, 0, 0);
_start.setUTCDate(1);
let _end = new Date(_start);
_end.setUTCHours(24, 0, 0, 0);
document.getElementById("from").value = _start.toISOString().split('Z')[0];
document.getElementById("to").value = _end.toISOString().split('Z')[0];
function onGenerateClick(display) {
let spacingSeconds = document.getElementById("spacing").value;
let columnCount = document.getElementById("columns").value;
let startStr = document.getElementById("from").value
let toStr = document.getElementById("to").value
let start = new Date(startStr);
let end = new Date(toStr);
let csv = createCSV(start, end, spacingSeconds, columnCount);
if (display) {
document.getElementById("output").value = csv;
} else {
navigator.clipboard.writeText(csv);
alert("Copied to clipboard");
}
}
function createCSV(start = new Date(), end = new Date(), spacingSeconds = 60, columnCount = 1) {
// log what's being created to console
console.log(`Creating a timeseries CSV file...
columns: ${columnCount.toLocaleString()}
from: ${start.toISOString()}
to: ${end.toISOString()}
spacing: ${spacingSeconds.toLocaleString()} seconds`)
let csv = '';
const columns = ['timestamp'];
const values = []; // array to track the random walk
for (let i = 0; i < +columnCount; i++) {
columns.push(`Series ${i + 1}`);
values.push(0);
}
csv += `${columns.join(',')}\r\n`;
while (start.valueOf() < end.valueOf()) {
const timestamp = new Date(start);
const cells = [];
cells.push(timestamp.toISOString());
for (let i = 0; i < +columnCount; i++) {
const value = values[i];
cells.push(value.toPrecision(5));
values[i] += Math.random() - 0.5;
}
csv += `${cells.join(',')}\r\n`;
start.setUTCSeconds(start.getUTCSeconds() + +spacingSeconds);
}
return csv;
}
</script>
</body>
</html>
Raw
make-csv.js
/**
* Arguments:
* START DATE
* END DATE
* SECONDS BETWEEN POINTS
* NUMBER OF SIGNALS (csv columns)
*
* Example use for 1 month of 60-second-reolution data with 100 columns
* node make-csv.js 2020-01-01 2020-02-01 60 100
*/
const fs = require('fs');
const path = require('path');
const args = process.argv.slice(2);
const fromStr = args.shift();
const toStr = args.shift();
let spacingSeconds = +args.shift();
let columnCount = +args.shift();
let start;
let end;
if (fromStr) {
start = new Date(fromStr);
} else {
// set to the start of the current year
start = new Date();
start.setUTCMonth(0);
start.setUTCDate(1);
start.setUTCHours(0, 0, 0, 0);
}
if (toStr) {
end = new Date(toStr);
} else {
// set to the start of the next year
end = new Date();
end.setUTCMonth(12);
end.setUTCDate(1);
end.setUTCHours(0, 0, 0, 0);
}
if (Number.isNaN(spacingSeconds)) {
spacingSeconds = 60 * 60; // no spacing supplied - set to 1 hour
}
// if no column-count is supplied, set it to 1
if (Number.isNaN(columnCount)) columnCount = 1;
// log what's being created to console
console.log(`Creating a timeseries CSV file...
columns: ${columnCount.toLocaleString()}
from: ${start.toISOString()}
to: ${end.toISOString()}
spacing: ${spacingSeconds.toLocaleString()} seconds`)
const rows = [];
const columns = ['timestamp'];
const values = []; // array to track the random walk
for (let i = 0; i < columnCount; i++) {
columns.push(`Series ${i + 1}`);
values.push(0);
}
rows.push(`${columns.join(',')}\r\n`);
while (start.valueOf() < end.valueOf()) {
const timestamp = new Date(start);
const cells = [];
cells.push(timestamp.toISOString());
for (let i = 0; i < columnCount; i++) {
const value = values[i];
cells.push(value.toPrecision(5));
values[i] += Math.random() - 0.5;
}
rows.push(`${cells.join(',')}\r\n`);
start.setUTCSeconds(start.getUTCSeconds() + spacingSeconds);
}
fs.writeFileSync(path.join(__dirname, 'output.csv'), rows.join(''));
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment