Last active
March 30, 2023 02:45
-
-
Save prdn/b8c067c758aab7fa3bf715101086b47c to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| /*USAGE: | |
| npm install ws lodash async moment crc-32 | |
| mkdir logs | |
| node bfx_test_book.js BTCUSD | |
| */ | |
| const WS = require('ws') | |
| const _ = require('lodash') | |
| const async = require('async') | |
| const fs = require('fs') | |
| const moment = require('moment') | |
| const CRC = require('crc-32') | |
| const pair = process.argv[2] | |
| const conf = { | |
| wshost: 'wss://api.bitfinex.com/ws/2' | |
| } | |
| const logfile = __dirname + '/logs/ws-book-aggr.log' | |
| const BOOK = {} | |
| console.log(pair, conf.wshost) | |
| let connected = false | |
| let connecting = false | |
| let cli | |
| let seq = null | |
| function connect () { | |
| if (connecting || connected) return | |
| connecting = true | |
| cli = new WS(conf.wshost, { /* rejectUnauthorized: false */ }) | |
| cli.on('open', function open () { | |
| console.log('WS open') | |
| connecting = false | |
| connected = true | |
| BOOK.bids = {} | |
| BOOK.asks = {} | |
| BOOK.psnap = {} | |
| BOOK.mcnt = 0 | |
| cli.send(JSON.stringify({ event: 'conf', flags: 65536 + 131072 })) | |
| cli.send(JSON.stringify({ event: 'subscribe', channel: 'book', pair: pair, prec: 'P0', len: 100 })) | |
| }) | |
| cli.on('close', function open () { | |
| seq = null | |
| console.log('WS close') | |
| connecting = false | |
| connected = false | |
| }) | |
| cli.on('message', function (msg) { | |
| msg = JSON.parse(msg) | |
| if (msg.event) return | |
| if (msg[1] === 'hb') { | |
| seq = +msg[2] | |
| return | |
| } else if (msg[1] === 'cs') { | |
| seq = +msg[3] | |
| const checksum = msg[2] | |
| const csdata = [] | |
| const bids_keys = BOOK.psnap['bids'] | |
| const asks_keys = BOOK.psnap['asks'] | |
| for (let i = 0; i < 25; i++) { | |
| if (bids_keys[i]) { | |
| const price = bids_keys[i] | |
| const pp = BOOK.bids[price] | |
| csdata.push(pp.price, pp.amount) | |
| } | |
| if (asks_keys[i]) { | |
| const price = asks_keys[i] | |
| const pp = BOOK.asks[price] | |
| csdata.push(pp.price, -pp.amount) | |
| } | |
| } | |
| const cs_str = csdata.join(':') | |
| const cs_calc = CRC.str(cs_str) | |
| fs.appendFileSync(logfile, '[' + moment().format('YYYY-MM-DDTHH:mm:ss.SSS') + '] ' + pair + ' | ' + JSON.stringify(['cs_string=' + cs_str, 'cs_calc=' + cs_calc, 'server_checksum=' + checksum]) + '\n') | |
| if (cs_calc !== checksum) { | |
| console.error('CHECKSUM_FAILED') | |
| process.exit(-1) | |
| } | |
| return | |
| } | |
| fs.appendFileSync(logfile, '[' + moment().format('YYYY-MM-DDTHH:mm:ss.SSS') + '] ' + pair + ' | ' + JSON.stringify(msg) + '\n') | |
| if (BOOK.mcnt === 0) { | |
| _.each(msg[1], function (pp) { | |
| pp = { price: pp[0], cnt: pp[1], amount: pp[2] } | |
| const side = pp.amount >= 0 ? 'bids' : 'asks' | |
| pp.amount = Math.abs(pp.amount) | |
| if (BOOK[side][pp.price]) { | |
| fs.appendFileSync(logfile, '[' + moment().format() + '] ' + pair + ' | ' + JSON.stringify(pp) + ' BOOK snap existing bid override\n') | |
| } | |
| BOOK[side][pp.price] = pp | |
| }) | |
| } else { | |
| const cseq = +msg[2] | |
| msg = msg[1] | |
| if (!seq) { | |
| seq = cseq - 1 | |
| } | |
| if (cseq - seq !== 1) { | |
| console.error('OUT OF SEQUENCE', seq, cseq) | |
| process.exit() | |
| } | |
| seq = cseq | |
| let pp = { price: msg[0], cnt: msg[1], amount: msg[2] } | |
| if (!pp.cnt) { | |
| let found = true | |
| if (pp.amount > 0) { | |
| if (BOOK['bids'][pp.price]) { | |
| delete BOOK['bids'][pp.price] | |
| } else { | |
| found = false | |
| } | |
| } else if (pp.amount < 0) { | |
| if (BOOK['asks'][pp.price]) { | |
| delete BOOK['asks'][pp.price] | |
| } else { | |
| found = false | |
| } | |
| } | |
| if (!found) { | |
| fs.appendFileSync(logfile, '[' + moment().format() + '] ' + pair + ' | ' + JSON.stringify(pp) + ' BOOK delete fail side not found\n') | |
| } | |
| } else { | |
| let side = pp.amount >= 0 ? 'bids' : 'asks' | |
| pp.amount = Math.abs(pp.amount) | |
| BOOK[side][pp.price] = pp | |
| } | |
| } | |
| _.each(['bids', 'asks'], function (side) { | |
| let sbook = BOOK[side] | |
| let bprices = Object.keys(sbook) | |
| let prices = bprices.sort(function (a, b) { | |
| if (side === 'bids') { | |
| return +a >= +b ? -1 : 1 | |
| } else { | |
| return +a <= +b ? -1 : 1 | |
| } | |
| }) | |
| BOOK.psnap[side] = prices | |
| }) | |
| BOOK.mcnt++ | |
| checkCross(msg) | |
| }) | |
| } | |
| setInterval(function () { | |
| if (connected) return | |
| connect() | |
| }, 3500) | |
| function checkCross (msg) { | |
| let bid = BOOK.psnap.bids[0] | |
| let ask = BOOK.psnap.asks[0] | |
| if (bid >= ask) { | |
| let lm = [moment.utc().format(), 'bid(' + bid + ')>=ask(' + ask + ')'] | |
| fs.appendFileSync(logfile, lm.join('/') + '\n') | |
| console.log(lm.join('/')) | |
| } | |
| } | |
| function saveBook () { | |
| const now = moment.utc().format('YYYYMMDDHHmmss') | |
| fs.writeFileSync(__dirname + "/logs/tmp-ws-book-aggr-" + pair + '-' + now + '.log', JSON.stringify({ bids: BOOK.bids, asks: BOOK.asks})) | |
| } | |
| setInterval(function () { | |
| saveBook() | |
| }, 30000) |
@prdn There is a bug on bitfinex's order book websocket server. If we configure the websocket to report a message's timestamp, the reported timestamp is client dependent. Very easy to observe in a terminal:
wscat -c wss://api-pub.bitfinex.com/ws/2
{"event":"conf","flags": 32768}
{ "event": "subscribe", "channel": "book", "symbol": "tBTCUSD","freq":"F0","len":"1" }
Then open another terminal and do the same thing. Compare the reported timestamps: some of them are identical from the two terminals, some of them differ from each other even if they represent the same event.
hello,
it's our gateway that applies the timestamp, so depending on the gateway
you're subscribed to you'll see different results.
We have hundreds of gateways
…On Mon, 13 Jan 2020 at 18:08, cryptochassis ***@***.***> wrote:
@prdn <https://github.com/prdn> There is a bug on bitfinex's order book
websocket server. If we configure the websocket to report a message's
timestamp, the reported timestamp is client dependent. Very easy to observe
in a terminal:
wscat -c wss://api-pub.bitfinex.com/ws/2
{"event":"conf","flags": 32768}
{ "event": "subscribe", "channel": "book", "symbol":
"tBTCUSD","freq":"F0","len":"1" }
Then open another terminal and do the same thing. Compare the reported
timestamps: some of them are identical from the two terminals, some of them
differ from each other even if they represent the same event.
—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub
<https://gist.github.com/b8c067c758aab7fa3bf715101086b47c?email_source=notifications&email_token=AA4SZ3I4JKLJZQXUK4J2JGLQ5SUYZA5CNFSM4HYPQLY2YY3PNVWWK3TUL52HS4DFVNDWS43UINXW23LFNZ2KUY3PNVWWK3TUL5UWJTQAF7OAO#gistcomment-3136519>,
or unsubscribe
<https://github.com/notifications/unsubscribe-auth/AA4SZ3MKXDHKYPNMZFXCCLLQ5SUYZANCNFSM4HYPQLYQ>
.
--
--
Paolo Ardoino
@prdn When investigating the timestamps reported by other major exchanges such as coinbase, gemini, bitstamp, kraken, we found that their timestamps are client-independent: i.e. it is the event timestamp: when the change of the order book happened. It is important when it comes to high-frequency algorithmic trading to know the event timestamp rather than the gateway timestamp. For the benefits of bitfinex attracting more advanced algorithmic traders, perhaps consider the proposal of making the timestamp client-independent. Thank you.
we can definitely add global timestamping through another flag, we have it
internally. that flag is mainly used to give you a way to calculate full
latency from the gateway you're connected to.
…On Mon, 13 Jan 2020 at 19:22, cryptochassis ***@***.***> wrote:
@prdn <https://github.com/prdn> When investigating the timestamps
reported by other major exchanges such as coinbase, gemini, bitstamp,
kraken, we found that their timestamps are client-independent: i.e. it is
the event timestamp: when the change of the order book happened. It is
important when it comes to high-frequency algorithmic trading to know the
event timestamp rather than the gateway timestamp. For the benefits of
bitfinex attracting more advanced algorithmic traders, perhaps consider the
proposal of making the timestamp client-independent. Thank you.
—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub
<https://gist.github.com/b8c067c758aab7fa3bf715101086b47c?email_source=notifications&email_token=AA4SZ3N76NKMQOODL4O7S4TQ5S5P7A5CNFSM4HYPQLY2YY3PNVWWK3TUL52HS4DFVNDWS43UINXW23LFNZ2KUY3PNVWWK3TUL5UWJTQAF7OHE#gistcomment-3136626>,
or unsubscribe
<https://github.com/notifications/unsubscribe-auth/AA4SZ3IG76AGCPS7XEIFU43Q5S5P7ANCNFSM4HYPQLYQ>
.
--
--
Paolo Ardoino
@prdn global timestamping: this is the gold we are looking for. Please let us either know the internal flag or the time that you gladly announce the public availability of it. Our email is [email protected]: we'd be more than happy to be the external beta tester of this global timestamping. Thank you.
@prdn Please let us know how to get the global timestamping. Thanks a lot. We appreciate your time and help.
is the problem resolved? I found both the sequence number & timestamp doesn't provide any value at all, in terms of uniquely identify the message.
Would it be possible to add the global timestamps? Thanks.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
psnap= price snapshotcsdata= checksum datamcnt= message countGood old times, back to 70s, every byte on the wire matters.