Skip to content

Instantly share code, notes, and snippets.

@rgarcia

rgarcia/main.coffee

Last active December 27, 2015 13:18
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save rgarcia/7331721 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save rgarcia/7331721 to your computer and use it in GitHub Desktop.
Download ZIP
node: dangers of mixed objectMode
Raw
main.coffee
{Readable,Transform} = require 'stream'
crypto = require 'crypto'
report = (stream) ->
int = setInterval ->
console.log "#{stream.constructor.name}(#{stream._writableState?.length or ''} #{stream._readableState?.length or ''})"
, 1000
stream.on 'end', -> clearInterval int
# generates a bunch of bytes
class DataGenerator extends Readable
constructor: (@bytes) ->
super objectMode: false
@generated = 0
report @
_read: () ->
buf = crypto.randomBytes(Math.min(256, @bytes-@generated))
@push buf
@generated += buf.length
console.log 'bytes generated:', @generated
if @bytes == @generated
console.log 'done generating bytes'
@push null
# extremely slow consumer
class SlowDataTransformer extends Transform
constructor: ->
# EXHIBIT A: consume the data w/ an incorrectly configured objectmode on the writable side
# super objectMode: true
# EXHIBIT B: consume the data w/ an correctly configured objectmode on the writable side
super
@_writableState.objectMode = false
@_readableState.objectMode = true
report @
_transform: (chunk, encoding, cb) ->
setTimeout () ->
cb null, 1
, 100000
new DataGenerator(100000) # 100k
.pipe(new SlowDataTransformer())
.on('data', console.log)
.on('end', -> console.log 'done')
Raw
output_exhibit_a.txt
bytes generated: 256
bytes generated: 512
bytes generated: 768
bytes generated: 1024
bytes generated: 1280
bytes generated: 1536
bytes generated: 1792
bytes generated: 2048
bytes generated: 2304
bytes generated: 2560
bytes generated: 2816
bytes generated: 3072
bytes generated: 3328
bytes generated: 3584
bytes generated: 3840
bytes generated: 4096
bytes generated: 4352
bytes generated: 4608
bytes generated: 4864
bytes generated: 5120
bytes generated: 5376
bytes generated: 5632
bytes generated: 5888
bytes generated: 6144
bytes generated: 6400
bytes generated: 6656
bytes generated: 6912
bytes generated: 7168
bytes generated: 7424
bytes generated: 7680
bytes generated: 7936
bytes generated: 8192
bytes generated: 8448
bytes generated: 8704
bytes generated: 8960
bytes generated: 9216
bytes generated: 9472
bytes generated: 9728
bytes generated: 9984
bytes generated: 10240
bytes generated: 10496
bytes generated: 10752
bytes generated: 11008
bytes generated: 11264
bytes generated: 11520
bytes generated: 11776
bytes generated: 12032
bytes generated: 12288
bytes generated: 12544
bytes generated: 12800
bytes generated: 13056
bytes generated: 13312
bytes generated: 13568
bytes generated: 13824
bytes generated: 14080
bytes generated: 14336
bytes generated: 14592
bytes generated: 14848
bytes generated: 15104
bytes generated: 15360
bytes generated: 15616
bytes generated: 15872
bytes generated: 16128
bytes generated: 16384
bytes generated: 16640
bytes generated: 16896
bytes generated: 17152
bytes generated: 17408
bytes generated: 17664
bytes generated: 17920
bytes generated: 18176
bytes generated: 18432
bytes generated: 18688
bytes generated: 18944
bytes generated: 19200
bytes generated: 19456
bytes generated: 19712
bytes generated: 19968
bytes generated: 20224
bytes generated: 20480
bytes generated: 20736
bytes generated: 20992
bytes generated: 21248
bytes generated: 21504
bytes generated: 21760
bytes generated: 22016
bytes generated: 22272
bytes generated: 22528
bytes generated: 22784
bytes generated: 23040
bytes generated: 23296
bytes generated: 23552
bytes generated: 23808
bytes generated: 24064
bytes generated: 24320
bytes generated: 24576
bytes generated: 24832
bytes generated: 25088
bytes generated: 25344
bytes generated: 25600
bytes generated: 25856
bytes generated: 26112
bytes generated: 26368
bytes generated: 26624
bytes generated: 26880
bytes generated: 27136
bytes generated: 27392
bytes generated: 27648
bytes generated: 27904
bytes generated: 28160
bytes generated: 28416
bytes generated: 28672
bytes generated: 28928
bytes generated: 29184
bytes generated: 29440
bytes generated: 29696
bytes generated: 29952
bytes generated: 30208
bytes generated: 30464
bytes generated: 30720
bytes generated: 30976
bytes generated: 31232
bytes generated: 31488
bytes generated: 31744
bytes generated: 32000
bytes generated: 32256
bytes generated: 32512
bytes generated: 32768
bytes generated: 33024
bytes generated: 33280
bytes generated: 33536
bytes generated: 33792
bytes generated: 34048
bytes generated: 34304
bytes generated: 34560
bytes generated: 34816
bytes generated: 35072
bytes generated: 35328
bytes generated: 35584
bytes generated: 35840
bytes generated: 36096
bytes generated: 36352
bytes generated: 36608
bytes generated: 36864
bytes generated: 37120
bytes generated: 37376
bytes generated: 37632
bytes generated: 37888
bytes generated: 38144
bytes generated: 38400
bytes generated: 38656
bytes generated: 38912
bytes generated: 39168
bytes generated: 39424
bytes generated: 39680
bytes generated: 39936
bytes generated: 40192
bytes generated: 40448
bytes generated: 40704
bytes generated: 40960
bytes generated: 41216
bytes generated: 41472
bytes generated: 41728
bytes generated: 41984
bytes generated: 42240
bytes generated: 42496
bytes generated: 42752
bytes generated: 43008
bytes generated: 43264
bytes generated: 43520
bytes generated: 43776
bytes generated: 44032
bytes generated: 44288
bytes generated: 44544
bytes generated: 44800
bytes generated: 45056
bytes generated: 45312
bytes generated: 45568
bytes generated: 45824
bytes generated: 46080
bytes generated: 46336
bytes generated: 46592
bytes generated: 46848
bytes generated: 47104
bytes generated: 47360
bytes generated: 47616
bytes generated: 47872
bytes generated: 48128
bytes generated: 48384
bytes generated: 48640
bytes generated: 48896
bytes generated: 49152
bytes generated: 49408
bytes generated: 49664
bytes generated: 49920
bytes generated: 50176
bytes generated: 50432
bytes generated: 50688
bytes generated: 50944
bytes generated: 51200
bytes generated: 51456
bytes generated: 51712
bytes generated: 51968
bytes generated: 52224
bytes generated: 52480
bytes generated: 52736
bytes generated: 52992
bytes generated: 53248
bytes generated: 53504
bytes generated: 53760
bytes generated: 54016
bytes generated: 54272
bytes generated: 54528
bytes generated: 54784
bytes generated: 55040
bytes generated: 55296
bytes generated: 55552
bytes generated: 55808
bytes generated: 56064
bytes generated: 56320
bytes generated: 56576
bytes generated: 56832
bytes generated: 57088
bytes generated: 57344
bytes generated: 57600
bytes generated: 57856
bytes generated: 58112
bytes generated: 58368
bytes generated: 58624
bytes generated: 58880
bytes generated: 59136
bytes generated: 59392
bytes generated: 59648
bytes generated: 59904
bytes generated: 60160
bytes generated: 60416
bytes generated: 60672
bytes generated: 60928
bytes generated: 61184
bytes generated: 61440
bytes generated: 61696
bytes generated: 61952
bytes generated: 62208
bytes generated: 62464
bytes generated: 62720
bytes generated: 62976
bytes generated: 63232
bytes generated: 63488
bytes generated: 63744
bytes generated: 64000
bytes generated: 64256
bytes generated: 64512
bytes generated: 64768
bytes generated: 65024
bytes generated: 65280
bytes generated: 65536
bytes generated: 65792
bytes generated: 66048
bytes generated: 66304
bytes generated: 66560
bytes generated: 66816
bytes generated: 67072
bytes generated: 67328
bytes generated: 67584
bytes generated: 67840
bytes generated: 68096
bytes generated: 68352
bytes generated: 68608
bytes generated: 68864
bytes generated: 69120
bytes generated: 69376
bytes generated: 69632
bytes generated: 69888
bytes generated: 70144
bytes generated: 70400
bytes generated: 70656
bytes generated: 70912
bytes generated: 71168
bytes generated: 71424
bytes generated: 71680
bytes generated: 71936
bytes generated: 72192
bytes generated: 72448
bytes generated: 72704
bytes generated: 72960
bytes generated: 73216
bytes generated: 73472
bytes generated: 73728
bytes generated: 73984
bytes generated: 74240
bytes generated: 74496
bytes generated: 74752
bytes generated: 75008
bytes generated: 75264
bytes generated: 75520
bytes generated: 75776
bytes generated: 76032
bytes generated: 76288
bytes generated: 76544
bytes generated: 76800
bytes generated: 77056
bytes generated: 77312
bytes generated: 77568
bytes generated: 77824
bytes generated: 78080
bytes generated: 78336
bytes generated: 78592
bytes generated: 78848
bytes generated: 79104
bytes generated: 79360
bytes generated: 79616
bytes generated: 79872
bytes generated: 80128
bytes generated: 80384
bytes generated: 80640
bytes generated: 80896
bytes generated: 81152
bytes generated: 81408
bytes generated: 81664
bytes generated: 81920
bytes generated: 82176
bytes generated: 82432
bytes generated: 82688
bytes generated: 82944
bytes generated: 83200
bytes generated: 83456
bytes generated: 83712
bytes generated: 83968
bytes generated: 84224
bytes generated: 84480
bytes generated: 84736
bytes generated: 84992
bytes generated: 85248
bytes generated: 85504
bytes generated: 85760
bytes generated: 86016
bytes generated: 86272
bytes generated: 86528
bytes generated: 86784
bytes generated: 87040
bytes generated: 87296
bytes generated: 87552
bytes generated: 87808
bytes generated: 88064
bytes generated: 88320
bytes generated: 88576
bytes generated: 88832
bytes generated: 89088
bytes generated: 89344
bytes generated: 89600
bytes generated: 89856
bytes generated: 90112
bytes generated: 90368
bytes generated: 90624
bytes generated: 90880
bytes generated: 91136
bytes generated: 91392
bytes generated: 91648
bytes generated: 91904
bytes generated: 92160
bytes generated: 92416
bytes generated: 92672
bytes generated: 92928
bytes generated: 93184
bytes generated: 93440
bytes generated: 93696
bytes generated: 93952
bytes generated: 94208
bytes generated: 94464
bytes generated: 94720
bytes generated: 94976
bytes generated: 95232
bytes generated: 95488
bytes generated: 95744
bytes generated: 96000
bytes generated: 96256
bytes generated: 96512
bytes generated: 96768
bytes generated: 97024
bytes generated: 97280
bytes generated: 97536
bytes generated: 97792
bytes generated: 98048
bytes generated: 98304
bytes generated: 98560
bytes generated: 98816
bytes generated: 99072
bytes generated: 99328
bytes generated: 99584
bytes generated: 99840
bytes generated: 100000
done generating bytes
SlowDataTransformer(391 )
SlowDataTransformer(391 )
SlowDataTransformer(391 )
^C
Raw
output_exhibit_b.txt
bytes generated: 256
bytes generated: 512
bytes generated: 768
bytes generated: 1024
bytes generated: 1280
bytes generated: 1536
bytes generated: 1792
bytes generated: 2048
bytes generated: 2304
bytes generated: 2560
bytes generated: 2816
bytes generated: 3072
bytes generated: 3328
bytes generated: 3584
bytes generated: 3840
bytes generated: 4096
bytes generated: 4352
bytes generated: 4608
bytes generated: 4864
bytes generated: 5120
bytes generated: 5376
bytes generated: 5632
bytes generated: 5888
bytes generated: 6144
bytes generated: 6400
bytes generated: 6656
bytes generated: 6912
bytes generated: 7168
bytes generated: 7424
bytes generated: 7680
bytes generated: 7936
bytes generated: 8192
bytes generated: 8448
bytes generated: 8704
bytes generated: 8960
bytes generated: 9216
bytes generated: 9472
bytes generated: 9728
bytes generated: 9984
bytes generated: 10240
bytes generated: 10496
bytes generated: 10752
bytes generated: 11008
bytes generated: 11264
bytes generated: 11520
bytes generated: 11776
bytes generated: 12032
bytes generated: 12288
bytes generated: 12544
bytes generated: 12800
bytes generated: 13056
bytes generated: 13312
bytes generated: 13568
bytes generated: 13824
bytes generated: 14080
bytes generated: 14336
bytes generated: 14592
bytes generated: 14848
bytes generated: 15104
bytes generated: 15360
bytes generated: 15616
bytes generated: 15872
bytes generated: 16128
bytes generated: 16384
bytes generated: 16640
DataGenerator( 256)
SlowDataTransformer(16384 )
DataGenerator( 256)
SlowDataTransformer(16384 )
DataGenerator( 256)
SlowDataTransformer(16384 )
DataGenerator( 256)
SlowDataTransformer(16384 )
^C
@rgarcia

rgarcia commented Nov 6, 2013

Copy link
Copy Markdown
Author

with objectMode set incorrectly on a writable stream, backpressure does not work.

the reason is that objectMode determines how to tally inbound writes: https://github.com/joyent/node/blob/master/lib/_stream_writable.js#L234

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment