bitmex_and_binance.py

# IMPORTS
import pandas as pd
import math
import os.path
import time
from bitmex import bitmex
from binance.client import Client
from datetime import timedelta, datetime
from dateutil import parser
from tqdm import tqdm_notebook #(Optional, used for progress-bars)

### API
bitmex_api_key = '[REDACTED]'    #Enter your own API-key here
bitmex_api_secret = '[REDACTED]' #Enter your own API-secret here
binance_api_key = '[REDACTED]'    #Enter your own API-key here
binance_api_secret = '[REDACTED]' #Enter your own API-secret here

### CONSTANTS
binsizes = {"1m": 1, "5m": 5, "1h": 60, "1d": 1440}
batch_size = 750
bitmex_client = bitmex(test=False, api_key=bitmex_api_key, api_secret=bitmex_api_secret)
binance_client = Client(api_key=binance_api_key, api_secret=binance_api_secret)


### FUNCTIONS
def minutes_of_new_data(symbol, kline_size, data, source):
    if len(data) > 0:  old = parser.parse(data["timestamp"].iloc[-1])
    elif source == "binance": old = datetime.strptime('1 Jan 2017', '%d %b %Y')
    elif source == "bitmex": old = bitmex_client.Trade.Trade_getBucketed(symbol=symbol, binSize=kline_size, count=1, reverse=False).result()[0][0]['timestamp']
    if source == "binance": new = pd.to_datetime(binance_client.get_klines(symbol=symbol, interval=kline_size)[-1][0], unit='ms')
    if source == "bitmex": new = bitmex_client.Trade.Trade_getBucketed(symbol=symbol, binSize=kline_size, count=1, reverse=True).result()[0][0]['timestamp']
    return old, new

def get_all_binance(symbol, kline_size, save = False):
    filename = '%s-%s-data.csv' % (symbol, kline_size)
    if os.path.isfile(filename): data_df = pd.read_csv(filename)
    else: data_df = pd.DataFrame()
    oldest_point, newest_point = minutes_of_new_data(symbol, kline_size, data_df, source = "binance")
    delta_min = (newest_point - oldest_point).total_seconds()/60
    available_data = math.ceil(delta_min/binsizes[kline_size])
    if oldest_point == datetime.strptime('1 Jan 2017', '%d %b %Y'): print('Downloading all available %s data for %s. Be patient..!' % (kline_size, symbol))
    else: print('Downloading %d minutes of new data available for %s, i.e. %d instances of %s data.' % (delta_min, symbol, available_data, kline_size))
    klines = binance_client.get_historical_klines(symbol, kline_size, oldest_point.strftime("%d %b %Y %H:%M:%S"), newest_point.strftime("%d %b %Y %H:%M:%S"))
    data = pd.DataFrame(klines, columns = ['timestamp', 'open', 'high', 'low', 'close', 'volume', 'close_time', 'quote_av', 'trades', 'tb_base_av', 'tb_quote_av', 'ignore' ])
    data['timestamp'] = pd.to_datetime(data['timestamp'], unit='ms')
    if len(data_df) > 0:
        temp_df = pd.DataFrame(data)
        data_df = data_df.append(temp_df)
    else: data_df = data
    data_df.set_index('timestamp', inplace=True)
    if save: data_df.to_csv(filename)
    print('All caught up..!')
    return data_df

def get_all_bitmex(symbol, kline_size, save = False):
    filename = '%s-%s-data.csv' % (symbol, kline_size)
    if os.path.isfile(filename): data_df = pd.read_csv(filename)
    else: data_df = pd.DataFrame()
    oldest_point, newest_point = minutes_of_new_data(symbol, kline_size, data_df, source = "bitmex")
    delta_min = (newest_point - oldest_point).total_seconds()/60
    available_data = math.ceil(delta_min/binsizes[kline_size])
    rounds = math.ceil(available_data / batch_size)
    if rounds > 0:
        print('Downloading %d minutes of new data available for %s, i.e. %d instances of %s data in %d rounds.' % (delta_min, symbol, available_data, kline_size, rounds))
        for round_num in tqdm_notebook(range(rounds)):
            time.sleep(1)
            new_time = (oldest_point + timedelta(minutes = round_num * batch_size * binsizes[kline_size]))
            data = bitmex_client.Trade.Trade_getBucketed(symbol=symbol, binSize=kline_size, count=batch_size, startTime = new_time).result()[0]
            temp_df = pd.DataFrame(data)
            data_df = data_df.append(temp_df)
    data_df.set_index('timestamp', inplace=True)
    if save and rounds > 0: data_df.to_csv(filename)
    print('All caught up..!')
    return data_df

thank you my friend. This is very useful, avoid to read tons of strange API docs from BITMEX.

hello bro: i get the error:

ModuleNotFoundError: No module named 'bitmex'

hello bro: i get the error:

ModuleNotFoundError: No module named 'bitmex'

You need to install the Bitmex module first, which you can do by using pip install bitmex in the terminal

Thanks a lot for this! Could this be expanded to the FTX exchange as well?

hi, thanx for your great effort
i have these error

C:\Users\Rayan tech\Desktop\new bot\bitmex - binance data\bitmex_and_binance.py:65: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0
Please use tqdm.notebook.tqdm instead of tqdm.tqdm_notebook
for round_num in tqdm_notebook(range(rounds)):
Traceback (most recent call last):
File "C:\Users\Rayan tech\AppData\Local\Programs\Python\Python38-32\lib\site-packages\tqdm\notebook.py", line 96, in status_printer
pbar = IProgress(min=0, max=total)
NameError: name 'IProgress' is not defined

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "c:\Users\Rayan tech\Desktop\new bot\bitmex - binance data\bitmex_and_binance.py", line 76, in
data = get_all_bitmex("XBTUSD", "5m", save = True)
File "c:\Users\Rayan tech\Desktop\new bot\bitmex - binance data\bitmex_and_binance.py", line 65, in get_all_bitmex
for round_num in tqdm_notebook(range(rounds)):
File "C:\Users\Rayan tech\AppData\Local\Programs\Python\Python38-32\lib\site-packages\tqdm_init_.py", line 28, in tqdm_notebook
return _tqdm_notebook(*args, **kwargs)
File "C:\Users\Rayan tech\AppData\Local\Programs\Python\Python38-32\lib\site-packages\tqdm\notebook.py", line 208, in init
self.container = self.status_printer(
File "C:\Users\Rayan tech\AppData\Local\Programs\Python\Python38-32\lib\site-packages\tqdm\notebook.py", line 103, in status_printer
raise ImportError(
ImportError: FloatProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
Exception ignored in: <function tqdm.del at 0x0B065970>
Traceback (most recent call last):
File "C:\Users\Rayan tech\AppData\Local\Programs\Python\Python38-32\lib\site-packages\tqdm\std.py", line 1077, in del
self.close()
File "C:\Users\Rayan tech\AppData\Local\Programs\Python\Python38-32\lib\site-packages\tqdm\notebook.py", line 240, in close
self.sp(bar_style='danger')
AttributeError: 'tqdm_notebook' object has no attribute 'sp'

Hi nistrup, this is an amazing code. It has saved me so much time. If you find the time, please add a function for binance futures data too. It would be a life saver

trying to run thru visual code, adde API keys, dind save anything, runned in terminal couple times

Hi I would like to try this within Node.JS, which package has the same features? Are you using the data from Futures?

• https://github.com/jaggedsoft/node-binance-api#readme
• https://github.com/Ashlar/binance-api-node

Hello, I am getting this error, can you help me to solve this? Thank you

... get_all_binance("BTCUSDT", "5m", False) File "<stdin>", line 21 get_all_binance("BTCUSDT", "5m", False) ^ SyntaxError: invalid syntax

Hello, I am getting this error, can you help me to solve this? Thank you

... get_all_binance("BTCUSDT", "5m", False) File "<stdin>", line 21 get_all_binance("BTCUSDT", "5m", False) ^ SyntaxError: invalid syntax

try setting the save attr like this:

data = get_all_binance("BTCUSDT", "5m", save=False)

For future reference - the prerequisite package: simplejson, which is needed by the Bitmex API wrapper, requires Python 3.7 to function.
simplejson is not compatible with Python 3.9 at this time.
In addendum, the wrapper used for the Binance connection is python-binance by Sam McHardy so be sure to install this wrapper.

The Bitmex function doesn't seem to work. Binance works though.

> data = get_all_bitmex("BTCUSDT", "5m", save=True)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "bitmex_and_binance_data_downloader.py", line 65, in get_all_bitmex
    if os.path.isfile(filename): data_df = pd.read_csv(filename)
  File "bitmex_and_binance_data_downloader.py", line 37, in minutes_of_new_data
    elif source == "bitmex": old = bitmex_client.Trade.Trade_getBucketed(symbol=symbol, binSize=kline_size, count=1, reverse=False).result()[0][0]['timestamp']
  File "C:\Program Files\Python37\lib\site-packages\bravado\http_future.py", line 271, in result
    swagger_result = self._get_swagger_result(incoming_response)
  File "C:\Program Files\Python37\lib\site-packages\bravado\http_future.py", line 124, in wrapper
    return func(self, *args, **kwargs)
  File "C:\Program Files\Python37\lib\site-packages\bravado\http_future.py", line 303, in _get_swagger_result
    self.request_config.response_callbacks,
  File "C:\Program Files\Python37\lib\site-packages\bravado\http_future.py", line 353, in unmarshal_response
    raise_on_expected(incoming_response)
  File "C:\Program Files\Python37\lib\site-packages\bravado\http_future.py", line 422, in raise_on_expected
    swagger_result=http_response.swagger_result)
bravado.exception.HTTPBadRequest: 400 Bad Request: {'error': {'message': 'This request has expired - `expires` is in the past. Current time: 1656901326', 'name': 'HTTPError'}}

Getting this annoying message

C:\Users\Paul\anaconda3\lib\site-packages\dateparser\date_parser.py:35: PytzUsageWarning: The localize method is no longer necessary, as this time zone supports the fold attribute (PEP 495). For more details on migrating to a PEP 495-compliant implementation, see https://pytz-deprecation-shim.readthedocs.io/en/latest/migration.html
date_obj = stz.localize(date_obj)

Thanks Bro, its amazing