PaulMougel’s gists

PaulMougel / mime.js

Last active December 28, 2015 04:59 — forked from chmanie/gist:7441515

MIME checking using streams in Node.js

	var stream = require('readable-stream');
	var mmm = require('mmmagic');
	var mimeChecker = new stream.Transform();
	mimeChecker.data = [];
	mimeChecker.mimeFound = false;
	mimeChecker._transform = function (chunk, encoding, done) {
	var self = this;

	if (self.mimeFound) {
	self.push(chunk);

PaulMougel / sample-jsonstream.js

Created November 15, 2013 22:45

Sample stream usage in Node.js: parse a JSON array on-the-fly

	var stream = require('stream');
	var JSONStream = require('JSONStream');

	var inputStream = new stream.PassThrough();
	var jsonStream = JSONStream.parse('*');
	var finalStream = new stream.Writable({ objectMode: true });
	finalStream._write = function (doc, encoding, done) {
	console.log(doc);
	done();
	};

PaulMougel / client.js

Last active May 1, 2024 12:38

File upload in Node.js to an Express server, using streams

	// node: v0.10.21
	// request: 2.27.0
	var request = require('request');
	var fs = require('fs');

	var r = request.post("http://server.com:3000/");
	// See http://nodejs.org/api/stream.html#stream_new_stream_readable_options
	// for more information about the highWaterMark
	// Basically, this will make the stream emit smaller chunks of data (ie. more precise upload state)
	var upload = fs.createReadStream('f.jpg', { highWaterMark: 500 });

PaulMougel / child.js

Created November 19, 2013 08:45

Parent-child communication using messages in Node.js

	process.on('message', function(m) {
	console.log('Just received a message from parent: ' + JSON.stringify(m));
	if (m.exit === "true") process.exit();
	});

PaulMougel / output.txt

Created November 22, 2013 13:02

Example of a failing async.queue usage, resulting in a stack overflow

	$ node -v
	v0.10.16
	$ node index.js

	/root/tmp/node_modules/async/lib/async.js:710
	process: function () {
	^
	RangeError: Maximum call stack size exceeded

PaulMougel / stream_leveldb.js

Last active December 30, 2015 03:08 — forked from leonardonsantos/stream_leveldb.js

Stream the output of a LevelDB into a socket, using a Transform stream

	#!/usr/bin/nodejs

	var levelup = require('levelup');
	var http = require('http');
	var util = require('util');
	var Transform = require('stream').Transform \|\| require('readable-stream').Transform;

	var server = http.createServer(onRequest);

	// Save something in db

PaulMougel / generate_input.sh

Created December 14, 2013 16:34

Buffered Transform stream — simple PassThrough stream that also keeps an internal buffer. When a stream is piped into the BufferStream, the internal buffer is written to the new piped stream; after that the BufferStream behaves like a PassThrough.

dd if=/dev/zero of=input.txt bs=512k count=50

PaulMougel / output.txt

Created December 23, 2013 19:30

Example of async.waterfall, see http://stackoverflow.com/questions/20749359/node-async-waterfall-using-an-array-with-callbacks-that-have-arguments

	~/tmp ❯❯❯ node index.js
	null
	[ {}, {}, {} ]

PaulMougel / csvToMongo.js

Last active September 5, 2016 01:13

MongoDB document insertion from a CSV stream using async.queue

	var csv = require('csv');
	var async = require('async');
	var fs = require('fs');
	var MongoClient = require('mongodb').MongoClient;

	MongoClient.connect('mongodb://localhost:27017/csvdb', function(err, db) {
	if (err) throw err;

	var collection = db.collection('myCSVs');
	var queue = async.queue(collection.insert.bind(collection), 5);

PaulMougel / server.js

Created March 3, 2014 11:24

Simple download app using Express and node.js' streams

	var fs = require('fs');
	var express = require('express');
	var app = express();

	app.get('/', function (req, res) {
	res.attachment();
	fs.createReadStream('test.pdf').pipe(res);
	});

	app.listen(8080);

Paul Mougel PaulMougel