bfx_test_book.js

/*USAGE:
npm install ws lodash async moment crc-32
mkdir logs
node bfx_test_book.js BTCUSD
*/

const WS = require('ws')
const _ = require('lodash')
const async = require('async')
const fs = require('fs')
const moment = require('moment')
const CRC = require('crc-32')

const pair = process.argv[2]

const conf = {
  wshost: 'wss://api.bitfinex.com/ws/2'
}

const logfile = __dirname + '/logs/ws-book-aggr.log'

const BOOK = {}

console.log(pair, conf.wshost)

let connected = false
let connecting = false
let cli
let seq = null

function connect () {
  if (connecting || connected) return
  connecting = true

  cli = new WS(conf.wshost, { /* rejectUnauthorized: false */ })

  cli.on('open', function open () {
    console.log('WS open')
    connecting = false
    connected = true
    BOOK.bids = {}
    BOOK.asks = {}
    BOOK.psnap = {}
    BOOK.mcnt = 0
    cli.send(JSON.stringify({ event: 'conf', flags: 65536 + 131072 }))
    cli.send(JSON.stringify({ event: 'subscribe', channel: 'book', pair: pair, prec: 'P0', len: 100 }))
  })

  cli.on('close', function open () {
    seq = null
    console.log('WS close')
    connecting = false
    connected = false
  })

  cli.on('message', function (msg) {
    msg = JSON.parse(msg)

    if (msg.event) return
    if (msg[1] === 'hb') {
      seq = +msg[2]
      return
    } else if (msg[1] === 'cs') {
      seq = +msg[3]

      const checksum = msg[2]
      const csdata = []
      const bids_keys = BOOK.psnap['bids']
      const asks_keys = BOOK.psnap['asks']

      for (let i = 0; i < 25; i++) {
        if (bids_keys[i]) {
          const price = bids_keys[i]
          const pp = BOOK.bids[price]
          csdata.push(pp.price, pp.amount)
        }
        if (asks_keys[i]) {
          const price = asks_keys[i]
          const pp = BOOK.asks[price]
          csdata.push(pp.price, -pp.amount)
        }
      }

      const cs_str = csdata.join(':')
      const cs_calc = CRC.str(cs_str)

      fs.appendFileSync(logfile, '[' + moment().format('YYYY-MM-DDTHH:mm:ss.SSS') + '] ' + pair + ' | ' + JSON.stringify(['cs_string=' + cs_str, 'cs_calc=' + cs_calc, 'server_checksum=' + checksum]) + '\n')
      if (cs_calc !== checksum) {
        console.error('CHECKSUM_FAILED')
        process.exit(-1)
      }
      return
    }

    fs.appendFileSync(logfile, '[' + moment().format('YYYY-MM-DDTHH:mm:ss.SSS') + '] ' + pair + ' | ' + JSON.stringify(msg) + '\n')

    if (BOOK.mcnt === 0) {
      _.each(msg[1], function (pp) {
        pp = { price: pp[0], cnt: pp[1], amount: pp[2] }
        const side = pp.amount >= 0 ? 'bids' : 'asks'
        pp.amount = Math.abs(pp.amount)
        if (BOOK[side][pp.price]) {
          fs.appendFileSync(logfile, '[' + moment().format() + '] ' + pair + ' | ' + JSON.stringify(pp) + ' BOOK snap existing bid override\n')
        }
        BOOK[side][pp.price] = pp
      })
    } else {
      const cseq = +msg[2]
      msg = msg[1]

      if (!seq) {
        seq = cseq - 1
      }

      if (cseq - seq !== 1) {
        console.error('OUT OF SEQUENCE', seq, cseq)
        process.exit()
      }

      seq = cseq

      let pp = { price: msg[0], cnt: msg[1], amount: msg[2] }

      if (!pp.cnt) {
        let found = true

        if (pp.amount > 0) {
          if (BOOK['bids'][pp.price]) {
            delete BOOK['bids'][pp.price]
          } else {
            found = false
          }
        } else if (pp.amount < 0) {
          if (BOOK['asks'][pp.price]) {
            delete BOOK['asks'][pp.price]
          } else {
            found = false
          }
        }

        if (!found) {
          fs.appendFileSync(logfile, '[' + moment().format() + '] ' + pair + ' | ' + JSON.stringify(pp) + ' BOOK delete fail side not found\n')
        }
      } else {
        let side = pp.amount >= 0 ? 'bids' : 'asks'
        pp.amount = Math.abs(pp.amount)
        BOOK[side][pp.price] = pp
      }
    }

    _.each(['bids', 'asks'], function (side) {
      let sbook = BOOK[side]
      let bprices = Object.keys(sbook)

      let prices = bprices.sort(function (a, b) {
        if (side === 'bids') {
          return +a >= +b ? -1 : 1
        } else {
          return +a <= +b ? -1 : 1
        }
      })

      BOOK.psnap[side] = prices
    })

    BOOK.mcnt++
    checkCross(msg)
  })
}

setInterval(function () {
  if (connected) return
  connect()
}, 3500)

function checkCross (msg) {
  let bid = BOOK.psnap.bids[0]
  let ask = BOOK.psnap.asks[0]
  if (bid >= ask) {
    let lm = [moment.utc().format(), 'bid(' + bid + ')>=ask(' + ask + ')']
    fs.appendFileSync(logfile, lm.join('/') + '\n')
    console.log(lm.join('/'))
  }
}

function saveBook () {
  const now = moment.utc().format('YYYYMMDDHHmmss')
  fs.writeFileSync(__dirname + "/logs/tmp-ws-book-aggr-" + pair + '-' + now + '.log', JSON.stringify({ bids: BOOK.bids, asks: BOOK.asks}))
}

setInterval(function () {
  saveBook()
}, 30000)

hello, it's our gateway that applies the timestamp, so depending on the gateway you're subscribed to you'll see different results. We have hundreds of gateways

…

On Mon, 13 Jan 2020 at 18:08, cryptochassis ***@***.***> wrote: @prdn <https://github.com/prdn> There is a bug on bitfinex's order book websocket server. If we configure the websocket to report a message's timestamp, the reported timestamp is client dependent. Very easy to observe in a terminal: wscat -c wss://api-pub.bitfinex.com/ws/2 {"event":"conf","flags": 32768} { "event": "subscribe", "channel": "book", "symbol": "tBTCUSD","freq":"F0","len":"1" } Then open another terminal and do the same thing. Compare the reported timestamps: some of them are identical from the two terminals, some of them differ from each other even if they represent the same event. — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <https://gist.github.com/b8c067c758aab7fa3bf715101086b47c?email_source=notifications&email_token=AA4SZ3I4JKLJZQXUK4J2JGLQ5SUYZA5CNFSM4HYPQLY2YY3PNVWWK3TUL52HS4DFVNDWS43UINXW23LFNZ2KUY3PNVWWK3TUL5UWJTQAF7OAO#gistcomment-3136519>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AA4SZ3MKXDHKYPNMZFXCCLLQ5SUYZANCNFSM4HYPQLYQ> .

-- -- Paolo Ardoino

@prdn When investigating the timestamps reported by other major exchanges such as coinbase, gemini, bitstamp, kraken, we found that their timestamps are client-independent: i.e. it is the event timestamp: when the change of the order book happened. It is important when it comes to high-frequency algorithmic trading to know the event timestamp rather than the gateway timestamp. For the benefits of bitfinex attracting more advanced algorithmic traders, perhaps consider the proposal of making the timestamp client-independent. Thank you.

we can definitely add global timestamping through another flag, we have it internally. that flag is mainly used to give you a way to calculate full latency from the gateway you're connected to.

…

On Mon, 13 Jan 2020 at 19:22, cryptochassis ***@***.***> wrote: @prdn <https://github.com/prdn> When investigating the timestamps reported by other major exchanges such as coinbase, gemini, bitstamp, kraken, we found that their timestamps are client-independent: i.e. it is the event timestamp: when the change of the order book happened. It is important when it comes to high-frequency algorithmic trading to know the event timestamp rather than the gateway timestamp. For the benefits of bitfinex attracting more advanced algorithmic traders, perhaps consider the proposal of making the timestamp client-independent. Thank you. — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <https://gist.github.com/b8c067c758aab7fa3bf715101086b47c?email_source=notifications&email_token=AA4SZ3N76NKMQOODL4O7S4TQ5S5P7A5CNFSM4HYPQLY2YY3PNVWWK3TUL52HS4DFVNDWS43UINXW23LFNZ2KUY3PNVWWK3TUL5UWJTQAF7OHE#gistcomment-3136626>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AA4SZ3IG76AGCPS7XEIFU43Q5S5P7ANCNFSM4HYPQLYQ> .

-- -- Paolo Ardoino

@prdn global timestamping: this is the gold we are looking for. Please let us either know the internal flag or the time that you gladly announce the public availability of it. Our email is [email protected]: we'd be more than happy to be the external beta tester of this global timestamping. Thank you.

@prdn Please let us know how to get the global timestamping. Thanks a lot. We appreciate your time and help.

is the problem resolved? I found both the sequence number & timestamp doesn't provide any value at all, in terms of uniquely identify the message.

Would it be possible to add the global timestamps? Thanks.