-
-
Save prdn/b8c067c758aab7fa3bf715101086b47c to your computer and use it in GitHub Desktop.
| /*USAGE: | |
| npm install ws lodash async moment crc-32 | |
| mkdir logs | |
| node bfx_test_book.js BTCUSD | |
| */ | |
| const WS = require('ws') | |
| const _ = require('lodash') | |
| const async = require('async') | |
| const fs = require('fs') | |
| const moment = require('moment') | |
| const CRC = require('crc-32') | |
| const pair = process.argv[2] | |
| const conf = { | |
| wshost: 'wss://api.bitfinex.com/ws/2' | |
| } | |
| const logfile = __dirname + '/logs/ws-book-aggr.log' | |
| const BOOK = {} | |
| console.log(pair, conf.wshost) | |
| let connected = false | |
| let connecting = false | |
| let cli | |
| let seq = null | |
| function connect () { | |
| if (connecting || connected) return | |
| connecting = true | |
| cli = new WS(conf.wshost, { /* rejectUnauthorized: false */ }) | |
| cli.on('open', function open () { | |
| console.log('WS open') | |
| connecting = false | |
| connected = true | |
| BOOK.bids = {} | |
| BOOK.asks = {} | |
| BOOK.psnap = {} | |
| BOOK.mcnt = 0 | |
| cli.send(JSON.stringify({ event: 'conf', flags: 65536 + 131072 })) | |
| cli.send(JSON.stringify({ event: 'subscribe', channel: 'book', pair: pair, prec: 'P0', len: 100 })) | |
| }) | |
| cli.on('close', function open () { | |
| seq = null | |
| console.log('WS close') | |
| connecting = false | |
| connected = false | |
| }) | |
| cli.on('message', function (msg) { | |
| msg = JSON.parse(msg) | |
| if (msg.event) return | |
| if (msg[1] === 'hb') { | |
| seq = +msg[2] | |
| return | |
| } else if (msg[1] === 'cs') { | |
| seq = +msg[3] | |
| const checksum = msg[2] | |
| const csdata = [] | |
| const bids_keys = BOOK.psnap['bids'] | |
| const asks_keys = BOOK.psnap['asks'] | |
| for (let i = 0; i < 25; i++) { | |
| if (bids_keys[i]) { | |
| const price = bids_keys[i] | |
| const pp = BOOK.bids[price] | |
| csdata.push(pp.price, pp.amount) | |
| } | |
| if (asks_keys[i]) { | |
| const price = asks_keys[i] | |
| const pp = BOOK.asks[price] | |
| csdata.push(pp.price, -pp.amount) | |
| } | |
| } | |
| const cs_str = csdata.join(':') | |
| const cs_calc = CRC.str(cs_str) | |
| fs.appendFileSync(logfile, '[' + moment().format('YYYY-MM-DDTHH:mm:ss.SSS') + '] ' + pair + ' | ' + JSON.stringify(['cs_string=' + cs_str, 'cs_calc=' + cs_calc, 'server_checksum=' + checksum]) + '\n') | |
| if (cs_calc !== checksum) { | |
| console.error('CHECKSUM_FAILED') | |
| process.exit(-1) | |
| } | |
| return | |
| } | |
| fs.appendFileSync(logfile, '[' + moment().format('YYYY-MM-DDTHH:mm:ss.SSS') + '] ' + pair + ' | ' + JSON.stringify(msg) + '\n') | |
| if (BOOK.mcnt === 0) { | |
| _.each(msg[1], function (pp) { | |
| pp = { price: pp[0], cnt: pp[1], amount: pp[2] } | |
| const side = pp.amount >= 0 ? 'bids' : 'asks' | |
| pp.amount = Math.abs(pp.amount) | |
| if (BOOK[side][pp.price]) { | |
| fs.appendFileSync(logfile, '[' + moment().format() + '] ' + pair + ' | ' + JSON.stringify(pp) + ' BOOK snap existing bid override\n') | |
| } | |
| BOOK[side][pp.price] = pp | |
| }) | |
| } else { | |
| const cseq = +msg[2] | |
| msg = msg[1] | |
| if (!seq) { | |
| seq = cseq - 1 | |
| } | |
| if (cseq - seq !== 1) { | |
| console.error('OUT OF SEQUENCE', seq, cseq) | |
| process.exit() | |
| } | |
| seq = cseq | |
| let pp = { price: msg[0], cnt: msg[1], amount: msg[2] } | |
| if (!pp.cnt) { | |
| let found = true | |
| if (pp.amount > 0) { | |
| if (BOOK['bids'][pp.price]) { | |
| delete BOOK['bids'][pp.price] | |
| } else { | |
| found = false | |
| } | |
| } else if (pp.amount < 0) { | |
| if (BOOK['asks'][pp.price]) { | |
| delete BOOK['asks'][pp.price] | |
| } else { | |
| found = false | |
| } | |
| } | |
| if (!found) { | |
| fs.appendFileSync(logfile, '[' + moment().format() + '] ' + pair + ' | ' + JSON.stringify(pp) + ' BOOK delete fail side not found\n') | |
| } | |
| } else { | |
| let side = pp.amount >= 0 ? 'bids' : 'asks' | |
| pp.amount = Math.abs(pp.amount) | |
| BOOK[side][pp.price] = pp | |
| } | |
| } | |
| _.each(['bids', 'asks'], function (side) { | |
| let sbook = BOOK[side] | |
| let bprices = Object.keys(sbook) | |
| let prices = bprices.sort(function (a, b) { | |
| if (side === 'bids') { | |
| return +a >= +b ? -1 : 1 | |
| } else { | |
| return +a <= +b ? -1 : 1 | |
| } | |
| }) | |
| BOOK.psnap[side] = prices | |
| }) | |
| BOOK.mcnt++ | |
| checkCross(msg) | |
| }) | |
| } | |
| setInterval(function () { | |
| if (connected) return | |
| connect() | |
| }, 3500) | |
| function checkCross (msg) { | |
| let bid = BOOK.psnap.bids[0] | |
| let ask = BOOK.psnap.asks[0] | |
| if (bid >= ask) { | |
| let lm = [moment.utc().format(), 'bid(' + bid + ')>=ask(' + ask + ')'] | |
| fs.appendFileSync(logfile, lm.join('/') + '\n') | |
| console.log(lm.join('/')) | |
| } | |
| } | |
| function saveBook () { | |
| const now = moment.utc().format('YYYYMMDDHHmmss') | |
| fs.writeFileSync(__dirname + "/logs/tmp-ws-book-aggr-" + pair + '-' + now + '.log', JSON.stringify({ bids: BOOK.bids, asks: BOOK.asks})) | |
| } | |
| setInterval(function () { | |
| saveBook() | |
| }, 30000) |
@ko0f just published new version
Hey Paolo, thanks for snippet - it helps with quick start.
I just found small misprint(copy-paste) in name of function:
@prdn https://gist.github.com/prdn/b8c067c758aab7fa3bf715101086b47c#file-bfx_test_book-js-L49
psnap = price snapshot
csdata = checksum data
mcnt = message count
Good old times, back to 70s, every byte on the wire matters.
@prdn There is a bug on bitfinex's order book websocket server. If we configure the websocket to report a message's timestamp, the reported timestamp is client dependent. Very easy to observe in a terminal:
wscat -c wss://api-pub.bitfinex.com/ws/2
{"event":"conf","flags": 32768}
{ "event": "subscribe", "channel": "book", "symbol": "tBTCUSD","freq":"F0","len":"1" }
Then open another terminal and do the same thing. Compare the reported timestamps: some of them are identical from the two terminals, some of them differ from each other even if they represent the same event.
@prdn When investigating the timestamps reported by other major exchanges such as coinbase, gemini, bitstamp, kraken, we found that their timestamps are client-independent: i.e. it is the event timestamp: when the change of the order book happened. It is important when it comes to high-frequency algorithmic trading to know the event timestamp rather than the gateway timestamp. For the benefits of bitfinex attracting more advanced algorithmic traders, perhaps consider the proposal of making the timestamp client-independent. Thank you.
@prdn global timestamping: this is the gold we are looking for. Please let us either know the internal flag or the time that you gladly announce the public availability of it. Our email is [email protected]: we'd be more than happy to be the external beta tester of this global timestamping. Thank you.
@prdn Please let us know how to get the global timestamping. Thanks a lot. We appreciate your time and help.
is the problem resolved? I found both the sequence number & timestamp doesn't provide any value at all, in terms of uniquely identify the message.
Would it be possible to add the global timestamps? Thanks.
It doesn't work, replace line 60 with -
let pp = { price: msg[1][0], cnt: msg[1][1], amount: msg[1][2] }