Demonstration of mongoose streaming + longjohn causing sadness

README.md

To reproduce:

• Copy the files below
• npm install
• ./node_modules/.bin/coffee app.coffee

Set "repoProblem" to true to make the problem happen, and to false to make the app run WAY faster and not run out of memory. :)

app.coffee

repoProblem = false

if repoProblem
    longjohn             = require 'longjohn'

ProgressBar = require 'progress'
mongoose = require 'mongoose'
Schema   = mongoose.Schema
ObjectId = Schema.ObjectId

OBJECTS_TO_CREATE = 40000

# Define our schemas
ASchema = new Schema
    _id:     false
    d:       { type: ObjectId }
    e:       { type: String }

BSchema = new Schema
    a:      { type: String }
    b:      { type: String,   enum:    ['a','b'] }
    date:   { type: Date,     default: Date.now }
    date2:  { type: Date }
    k:      [ ASchema ]

# Connect to mongo
mongoose.connect 'mongodb://localhost/mongooseProblem'
mongoose.connection.on 'error', (err) ->
    console.log 'Error: unable to connect to mongodb', err
    process.exit 1

connection = mongoose.createConnection 'mongodb://localhost/mongooseProblem'

B = connection.model "b", BSchema

progressBar = new ProgressBar ':bar (:current/:total) :percent eta: :eta', {total: OBJECTS_TO_CREATE, width: 30}

lastObjects = []

# Create some objects
createObject = (index) ->
    if index is 0
        console.log "Creating objects"

    k = []
    for lastObject in lastObjects
        k.push {
            d: lastObject._id
            e: "e-#{index}"
        }

    b = new B {
        a: "Object: #{index}"
        b: "b"
        date: Date.now()
        date2: Date.now() - (1000 * 60 * 4)
        k: k
    }
    b.save (err, result) ->
        progressBar.tick()
        lastObjects.push result
        while lastObjects.length > 3
            lastObjects.shift()

        if err
            console.log "Error creating object", err
            process.exit 1
        else
            if index < OBJECTS_TO_CREATE
                process.nextTick () -> createObject index + 1
            else
                readObjects()

# Kick things off
createObject 0

# Stream the objects back from the db
readObjects = () ->
    B.count {}, (countErr, totalObjects) ->
        if countErr
            console.log "Error counting objects", countErr
            process.exit 1

        progressBar = new ProgressBar ':bar (:current/:total) :percent eta: :eta', {total: totalObjects, width: 30}

        console.log "Found #{totalObjects} objects"

        # Line X
        cursor = B.find()
        stream = cursor.stream()
        objectIndex = 0
        err = null

        collectionName = B.collection.name

        stream.on 'data', (object) ->
            # Line Y
            stream.pause()
            process.nextTick ->
                # Here's where we'd do something useful with the object.  Lalala...
                progressBar.tick()
                stream.resume()

        stream.on 'error', (streamError) ->
            err = streamError
            stream.destroy()

        stream.on 'close', ->
            if err
                console.log err
            console.log "All done!"
            process.exit 0

package.json

{
  "name": "jwalton-mongoose-problem",
  "version": "0.1.1",
  "private": true,
  "dependencies": {
    "progress": "1.0.1",
    "mongoose": "3.6.19",
    "mongodb": "1.3.19",
    "coffee-script": "1.6.3",
    "longjohn": "0.2.1"
  },
  "engines": {
    "node": "0.8.16",
    "npm": "1.1"
  }
}