ccxt fetch entire historical

ccxt_fetch_historical_dataset.py

import os
import sys
import time
import pandas as pd
from pandas import ExcelWriter


root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.append(root + '/python')

import ccxt  # noqa: E402

msec = 1000
minute = 60 * msec
hold = 30

exchange = ccxt.binance({
    'rateLimit': 10000,
    'enableRateLimit': True,
    # 'verbose': True,
})

def run():
    from_datetime = '2017-01-01 00:00:00'
    from_timestamp = exchange.parse8601(from_datetime)
    data = []
    now = exchange.milliseconds()

    while from_timestamp < now:
        writer = ExcelWriter('1min.xlsx')
        try:
            print(from_timestamp)
            print(exchange.milliseconds(), 'Fetching candles starting from', exchange.iso8601(from_timestamp), from_timestamp)
            candles = exchange.fetch_ohlcv('BTC/USDT', '1m', from_timestamp)
            print(from_timestamp)
            #print(candles, from_timestamp)     
            print(exchange.milliseconds(), 'Fetched', len(candles), 'candles')
            first = candles[0][0]
            last = candles[-1][0]
            print('First candle epoch', first, exchange.iso8601(first))
            print('Last candle epoch', last, exchange.iso8601(last))
            from_timestamp = candles[-1][0] + minute
            data += candles

            df = pd.DataFrame(data, columns=['Timestamp','Open','High','Low','Close', 'Volume'])
            df['Timestamp'] = pd.DataFrame(df['Timestamp'].apply(exchange.iso8601))
            save_excel = df.to_excel(writer, sheet_name='2017_CURRENT')
            writer.save()
            print('1m candles saved')
        except (ccxt.ExchangeError, ccxt.AuthenticationError, ccxt.ExchangeNotAvailable, ccxt.RequestTimeout) as error:

            print('Got an error', type(error).__name__, error.args, ', retrying in', hold, 'seconds...')
            time.sleep(hold)
if __name__ == "__main__":      
    run()