Derby.js erorrs we experience in production

errors.txt

#######################################################
#   Error 0 - medium often - !!! Critical !!! - screws each client's data 
#######################################################

ERROR: CORRUPT DATA DETECTED in document games.3d2a3281-e764-4c0a-b449-d2c56477cff6
If you're using redis, delete data for document. Please file an issue if you can recreate this state reliably.
Trace: Internal data corruption - cannot submit
  at /home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/share/lib/server/session.js:665:19
  at /home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/share/lib/server/useragent.js:409:23
  at /home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/livedb/lib/index.js:261:18
  at /home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/local-derby-server/node_modules/livedb-mongo/mongo.js:219:5
  at /home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/local-derby-server/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/cursor.js:163:16
  at commandHandler (/home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/local-derby-server/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/cursor.js:709:16)
  at /home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/local-derby-server/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/db.js:1846:9
  at b (domain.js:183:18)
  at Server.Base._callHandler (/home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/local-derby-server/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/base.js:445:41)
  at /home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/local-derby-server/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/server.js:468:18
  at [object Object].MongoReply.parseBody (/home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/local-derby-server/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/responses/mongo_reply.js:68:5)
  at [object Object].<anonymous> (/home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/local-derby-server/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/server.js:426:20)
  at [object Object].EventEmitter.emit (events.js:95:17)
  at [object Object].<anonymous> (/home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/local-derby-server/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/connection_pool.js:201:13)
  at [object Object].EventEmitter.emit (events.js:98:17)
  at Socket.<anonymous> (/home/ubuntu/www/idg_v2/production/releases/20140915171800/node_modules/local-derby-server/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/connection.js:422:22)
  at Socket.EventEmitter.emit (events.js:95:17)
  at Socket.<anonymous> (_stream_readable.js:746:14)
  at Socket.EventEmitter.emit (events.js:92:17)
  at emitReadable_ (_stream_readable.js:408:10)
  at emitReadable (_stream_readable.js:404:5)
  at readableAddChunk (_stream_readable.js:165:9)
  at Socket.Readable.push (_stream_readable.js:127:10)
  at TCP.onread (net.js:528:21)

#################################### Following errors are fixed now: #######################################

#######################################################
#   Error 1 (VERY often)
#######################################################
  
Error: Can't set headers after they are sent.
  at ServerResponse.OutgoingMessage.setHeader (http.js:691:11)
  at ServerResponse.res.setHeader (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/node_modules/express/node_modules/connect/lib/patch.js:133:22)
  at ServerResponse.header (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/express/lib/response.js:717:10)
  at ServerResponse.send (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/express/lib/response.js:146:12)
  at ServerResponse.json (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/express/lib/response.js:235:15)
  at AsyncGroup.<anonymous> (/home/ubuntu/www/idg_v2/production/releases/20140911205037/src/server/serverController.coffee:111:11)
  at AsyncGroup.wrappedCallback [as cb] (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/lib/Model/events.js:44:17)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/lib/util.js:53:10
  at wrappedCallback (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/lib/Model/events.js:44:17)
  at Array.fetchDocCallback (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/lib/Model/subscriptions.js:119:5)
  at [object Object].Doc._finishSub (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/client/doc.js:526:34)
  at [object Object].Doc._onMessage (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/client/doc.js:287:12)
  at [object Object].Connection.handleMessage (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/client/connection.js:215:37)
  at StreamSocket.socket.onmessage (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/client/connection.js:124:18)
  at StreamSocket.Channel.socket.onmessage (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/lib/Channel.js:19:28)
  at Duplex._write (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/lib/Model/connection.server.js:23:12)
  at doWrite (_stream_writable.js:226:10)
  at writeOrBuffer (_stream_writable.js:216:5)
  at Duplex.Writable.write (_stream_writable.js:183:11)
  at Session._send (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/session.js:325:15)
  at Session._reply (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/session.js:363:8)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/session.js:384:28
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/session.js:601:11
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/useragent.js:95:5
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/node_modules/async/lib/async.js:142:25
  at [object Object].<anonymous> (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer-access/index.js:117:16)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/useragent.js:93:12
  at iterate (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/node_modules/async/lib/async.js:134:13)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/node_modules/async/lib/async.js:145:25
  at [object Object].<anonymous> (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer-access/index.js:117:16)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/useragent.js:93:12
  at iterate (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/node_modules/async/lib/async.js:134:13)
  at Object.async.eachSeries (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/node_modules/async/lib/async.js:150:9)
  at [object Object].UserAgent._runFilters (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/useragent.js:92:9)
  at [object Object].UserAgent.filterDoc (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/useragent.js:100:15)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/useragent.js:178:15
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/node_modules/livedb/lib/index.js:610:7
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/node_modules/livedb/lib/index.js:199:5
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/mongo.js:219:5
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/cursor.js:163:16
  at commandHandler (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/cursor.js:709:16)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/db.js:1846:9
  at b (domain.js:183:18)
  at Server.Base._callHandler (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/base.js:445:41)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/server.js:468:18
  at [object Object].MongoReply.parseBody (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/responses/mongo_reply.js:68:5)
  at [object Object].<anonymous> (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/server.js:426:20)
  at [object Object].EventEmitter.emit (events.js:95:17)
  at [object Object].<anonymous> (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/connection_pool.js:201:13)
  at [object Object].EventEmitter.emit (events.js:98:17)
  at Socket.<anonymous> (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/connection.js:422:22)
  at Socket.EventEmitter.emit (events.js:95:17)
  at Socket.<anonymous> (_stream_readable.js:746:14)
  at Socket.EventEmitter.emit (events.js:92:17)
  at emitReadable_ (_stream_readable.js:408:10)
  at emitReadable (_stream_readable.js:404:5)
  at readableAddChunk (_stream_readable.js:165:9)
  at Socket.Readable.push (_stream_readable.js:127:10)
  at TCP.onread (net.js:528:21)



  
#######################################################
#   Error 2 (average often)
#######################################################

Error: opAcknowledged called from a null state. This should never happen.
  at [object Object].Doc._opAcknowledged (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/client/doc.js:953:11)
  at [object Object].Doc._onMessage (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/client/doc.js:337:14)
  at [object Object].Connection.handleMessage (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/client/connection.js:215:37)
  at StreamSocket.socket.onmessage (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/client/connection.js:124:18)
  at StreamSocket.Channel.socket.onmessage (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/lib/Channel.js:19:28)
  at Duplex._write (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/lib/Model/connection.server.js:23:12)
  at doWrite (_stream_writable.js:226:10)
  at writeOrBuffer (_stream_writable.js:216:5)
  at Duplex.Writable.write (_stream_writable.js:183:11)
  at Session._send (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/session.js:325:15)
  at Session._sendOp (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/session.js:349:8)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/session.js:678:16
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/useragent.js:411:9
  at next (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/index.js:109:25)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/index.js:115:7
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/src/lib/initHooks.coffee:32:18
  at next (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/index.js:112:5)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/index.js:115:7
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/src/lib/initHooks.coffee:32:18
  at next (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/index.js:112:5)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/index.js:115:7
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/src/lib/initHooks.coffee:32:18
  at next (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/index.js:112:5)
  at [object Object].ShareInstance._trigger (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/index.js:119:3)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/useragent.js:156:14
  at process._tickDomainCallback (node.js:459:13)
  


#######################################################
#   Error 3 (not so often)
#######################################################

Error: Document already exists
  at Array.defaultCallback [as 0] (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/lib/Model/events.js:27:40)
  at [object Object].Doc._clearInflightOp (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/client/doc.js:936:17)
  at [object Object].Doc._onMessage (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/client/doc.js:323:16)
  at [object Object].Connection.handleMessage (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/client/connection.js:215:37)
  at StreamSocket.socket.onmessage (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/client/connection.js:124:18)
  at StreamSocket.Channel.socket.onmessage (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/lib/Channel.js:19:28)
  at Duplex._write (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/lib/Model/connection.server.js:23:12)
  at doWrite (_stream_writable.js:226:10)
  at writeOrBuffer (_stream_writable.js:216:5)
  at Duplex.Writable.write (_stream_writable.js:183:11)
  at Session._send (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/session.js:325:15)
  at Session._reply (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/session.js:363:8)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/session.js:384:28
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/session.js:667:25
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/lib/server/useragent.js:409:23
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/racer/node_modules/share/node_modules/livedb/lib/index.js:314:18
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/mongo.js:219:5
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/cursor.js:163:16
  at commandHandler (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/cursor.js:709:16)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/db.js:1846:9
  at b (domain.js:183:18)
  at Server.Base._callHandler (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/base.js:445:41)
  at /home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/server.js:468:18
  at [object Object].MongoReply.parseBody (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/responses/mongo_reply.js:68:5)
  at [object Object].<anonymous> (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/server.js:426:20)
  at [object Object].EventEmitter.emit (events.js:95:17)
  at [object Object].<anonymous> (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/connection_pool.js:201:13)
  at [object Object].EventEmitter.emit (events.js:98:17)
  at Socket.<anonymous> (/home/ubuntu/www/idg_v2/production/releases/20140911205037/node_modules/livedb-mongo/node_modules/mongoskin/node_modules/mongodb/lib/mongodb/connection/connection.js:422:22)
  at Socket.EventEmitter.emit (events.js:95:17)
  at Socket.<anonymous> (_stream_readable.js:746:14)
  at Socket.EventEmitter.emit (events.js:92:17)
  at emitReadable_ (_stream_readable.js:408:10)
  at emitReadable (_stream_readable.js:404:5)
  at readableAddChunk (_stream_readable.js:165:9)
  at Socket.Readable.push (_stream_readable.js:127:10)
  at TCP.onread (net.js:528:21)