manual_binance_historical_klines_to_csv.py

import time
import dateparser
import pytz
import json
import csv
import datetime
from dateutil.rrule import rrule, MONTHLY
from binance.client import Client
import os
import sys

pathname = os.path.dirname(sys.argv[0])
full_path = '{}/'.format(str(os.path.abspath(pathname)))

def grab_currencies_list():
	currencies_list = list()
	info = Client(None,None)
	pair_query = info.get_all_tickers()
	list_preference = input('Which base currencies would you like to grab data for?(eg. BTC or ETH,BNB or BTC,ETH,BNB,USDT) ')
	if list_preference.lower().replace(' ','') in ['btc']:
		for x in range(0,len(pair_query)):
			symbol = pair_query[x]['symbol']
			if symbol[-3:] == 'BTC':
				currencies_list.append(symbol)
		return currencies_list
	elif list_preference.lower().replace(' ','') in ['eth']:
		for x in range(0,len(pair_query)):
			symbol = pair_query[x]['symbol']
			if symbol[-3:] == 'ETH':
				currencies_list.append(symbol)
		return currencies_list
	elif list_preference.lower().replace(' ','') in ['bnb']:
		for x in range(0,len(pair_query)):
			symbol = pair_query[x]['symbol']
			if symbol[-3:] == 'BNB':
				currencies_list.append(symbol)
		return currencies_list
	elif list_preference.lower().replace(' ','') in ['usdt']:
		for x in range(0,len(pair_query)):
			symbol = pair_query[x]['symbol']
			if symbol[-4:] == 'USDT':
				currencies_list.append(symbol)
		return currencies_list
	elif list_preference.lower().replace(' ','') in ['btc,eth','eth,btc']:
		for x in range(0,len(pair_query)):
			symbol = pair_query[x]['symbol']
			if symbol[-3:] in ['BTC','ETH']:
				currencies_list.append(symbol)
		return currencies_list
	elif list_preference.lower().replace(' ','') in ['btc,bnb','bnb,btc']:
		for x in range(0,len(pair_query)):
			symbol = pair_query[x]['symbol']
			if symbol[-3:] in ['BTC','BNB']:
				currencies_list.append(symbol)
		return currencies_list
	elif list_preference.lower().replace(' ','') in ['eth,bnb','bnb,eth']:
		for x in range(0,len(pair_query)):
			symbol = pair_query[x]['symbol']
			if symbol[-3:] in ['ETH','BNB']:
				currencies_list.append(symbol)
		return currencies_list
	elif list_preference.lower().replace(' ','') in ['btc,usdt','usdt,btc']:
		for x in range(0,len(pair_query)):
			symbol = pair_query[x]['symbol']
			if symbol[-3:] in ['BTC']:
				currencies_list.append(symbol)
			elif symbol[-4:] in ['USDT']:
				currencies_list.append(symbol)
		return currencies_list
	elif list_preference.lower().replace(' ','') in ['eth,usdt','usdt,eth']:
		for x in range(0,len(pair_query)):
			symbol = pair_query[x]['symbol']
			if symbol[-3:] in ['ETH']:
				currencies_list.append(symbol)
			elif symbol[-4:] in ['USDT']:
				currencies_list.append(symbol)
		return currencies_list
	elif list_preference.lower().replace(' ','') in ['bnb,usdt','usdt,bnb']:
		for x in range(0,len(pair_query)):
			symbol = pair_query[x]['symbol']
			if symbol[-3:] in ['BNB']:
				currencies_list.append(symbol)
			elif symbol[-4:] in ['USDT']:
				currencies_list.append(symbol)
		return currencies_list
	elif list_preference.lower().replace(' ','') in ['btc,eth,usdt','btc,usdt,eth','usdt,btc,eth','usdt,eth,btc','eth,usdt,btc','eth,btc,usdt']:
		for x in range(0,len(pair_query)):
			symbol = pair_query[x]['symbol']
			if symbol[-3:] in ['BTC','ETH']:
				currencies_list.append(symbol)
			elif symbol[-4:] in ['USDT']:
				currencies_list.append(symbol)
		return currencies_list
	elif list_preference.lower().replace(' ','') in ['bnb,eth,usdt','bnb,usdt,eth','usdt,bnb,eth','usdt,eth,bnb','eth,usdt,bnb','eth,bnb,usdt']:
		for x in range(0,len(pair_query)):
			symbol = pair_query[x]['symbol']
			if symbol[-3:] in ['BNB','ETH']:
				currencies_list.append(symbol)
			elif symbol[-4:] in ['USDT']:
				currencies_list.append(symbol)
		return currencies_list
	elif list_preference.lower().replace(' ','') in ['btc,eth,usdt,bnb','btc,usdt,eth,bnb','btc,bnb,eth,usdt','btc,bnb,usdt,eth','btc,usdt,bnb,eth','btc,eth,bnb,usdt','usdt,eth,btc,bnb','usdt,btc,eth,bnb','usdt,btc,bnb,eth','usdt,eth,bnb,btc','usdt,bnb,btc,eth','usdt,bnb,eth,btc','eth,usdt,btc,bnb','eth,btc,usdt,bnb','eth,btc,bnb,usdt','eth,usdt,bnb,btc','eth,bnb,usdt,btc','eth,bnb,btc,usdt','bnb,btc,eth,usdt','bnb,btc,usdt,eth','bnb,eth,btc,usdt','bnb,eth,usdt,btc','bnb,usdt,eth,btc','bnb,usdt,btc,eth']:
			for x in range(0,len(pair_query)):
				symbol = pair_query[x]['symbol']
				currencies_list.append(symbol)
			return currencies_list
def grab_kline_interval():
	kline_interval = input('What Kline Interal would you prefer? Options: 1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h ')
	if kline_interval in ['1m','3m','5m','15m','30m','1h','2h','4h','6h','8h','12h']:
		return kline_interval
	else:
		print ('{} is an invalid option, please try again'.format(str(kline_interval)))
		kline_interval_2 = input('What Kline Interal would you prefer? Options: 1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h ')
		if kline_interval_2 in ['1m','3m','5m','15m','30m','1h','2h','4h','6h','8h','12h']:
			return kline_interval_2


def create_directories(pair_list):
	main_directory_finder = [x[0] for x in os.walk(full_path)]
	historical_price_data_directory = '{}historical_price_data'.format(str(full_path))
	if historical_price_data_directory not in main_directory_finder:
		os.makedirs(historical_price_data_directory)
	pair_directory_finder = [x[0] for x in os.walk('{}historical_price_data/'.format(str(full_path)))]
	for x in range(0,len(pair_list)):
		symbol = pair_list[x]
		if '{}/{}'.format(str(historical_price_data_directory),str(symbol)) not in pair_directory_finder:
			os.makedirs('{}historical_price_data/{}'.format(str(full_path),str(symbol)))

def date_to_milliseconds(date_str):
	epoch = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc)
	d = dateparser.parse(date_str)
	if d.tzinfo is None or d.tzinfo.utcoffset(d) is None:
		d = d.replace(tzinfo=pytz.utc)
	return int((d - epoch).total_seconds() * 1000.0)


def interval_to_milliseconds(interval):
	ms = None
	seconds_per_unit = {'m': 60,'h': 60 * 60,'d': 24 * 60 * 60,'w': 7 * 24 * 60 * 60}
	unit = interval[-1]
	if unit in seconds_per_unit:
		try:
			ms = int(interval[:-1]) * seconds_per_unit[unit] * 1000
		except ValueError:
			pass
	return ms


def get_historical_klines(symbol, interval, start_str, end_str=None):
	client = Client(None, None)
	output_data = []
	limit = 500
	timeframe = interval_to_milliseconds(interval)
	start_ts = date_to_milliseconds(start_str)
	end_ts = None
	if end_str:
		end_ts = date_to_milliseconds(end_str)
	idx = 0
	symbol_existed = False
	while True:
		temp_data = client.get_klines(symbol=symbol,interval=interval,limit=limit,startTime=start_ts,endTime=end_ts)
		if not symbol_existed and len(temp_data):
			symbol_existed = True
		if symbol_existed:
			output_data += temp_data
			start_ts = temp_data[len(temp_data) - 1][0] + timeframe
		else:
			start_ts += timeframe
		idx += 1
		if len(temp_data) < limit:
			break
		if idx % 3 == 0:
			time.sleep(1)
	return output_data

# pair_list = grab_currencies_list()
pair_list = ['ETHBTC', 'LTCBTC', 'BNBBTC', 'NEOBTC', 'BCCBTC', 'GASBTC', 'MCOBTC', 'WTCBTC', 'LRCBTC', 'QTUMBTC', 'YOYOBTC', 'OMGBTC', 'ZRXBTC', 'STRATBTC', 'SNGLSBTC', 'BQXBTC', 'KNCBTC', 'FUNBTC', 'SNMBTC', 'IOTABTC', 'LINKBTC', 'XVGBTC', 'SALTBTC', 'MDABTC', 'MTLBTC', 'SUBBTC', 'EOSBTC', 'SNTBTC', 'ETCBTC', 'MTHBTC', 'ENGBTC', 'DNTBTC', 'ZECBTC', 'BNTBTC', 'ASTBTC', 'DASHBTC', 'OAXBTC', 'BTGBTC', 'EVXBTC', 'REQBTC', 'VIBBTC', 'TRXBTC', 'POWRBTC', 'ARKBTC', 'XRPBTC', 'MODBTC', 'ENJBTC', 'STORJBTC', 'KMDBTC', 'RCNBTC', 'NULSBTC', 'RDNBTC', 'XMRBTC', 'DLTBTC', 'AMBBTC', 'BATBTC', 'BCPTBTC', 'ARNBTC', 'GVTBTC', 'CDTBTC', 'GXSBTC', 'POEBTC', 'QSPBTC', 'BTSBTC', 'XZCBTC', 'LSKBTC', 'TNTBTC', 'FUELBTC', 'MANABTC', 'BCDBTC', 'DGDBTC', 'ADXBTC', 'ADABTC', 'PPTBTC', 'CMTBTC', 'XLMBTC', 'CNDBTC', 'LENDBTC', 'WABIBTC', 'TNBBTC', 'WAVESBTC', 'GTOBTC', 'ICXBTC', 'OSTBTC', 'ELFBTC', 'AIONBTC', 'NEBLBTC', 'BRDBTC', 'EDOBTC', 'WINGSBTC', 'NAVBTC', 'LUNBTC', 'APPCBTC', 'VIBEBTC', 'RLCBTC', 'INSBTC', 'PIVXBTC', 'IOSTBTC', 'STEEMBTC', 'NANOBTC', 'VIABTC', 'BLZBTC', 'AEBTC', 'NCASHBTC', 'POABTC', 'ZILBTC', 'ONTBTC', 'STORMBTC', 'XEMBTC', 'WANBTC', 'WPRBTC', 'QLCBTC', 'SYSBTC', 'GRSBTC', 'CLOAKBTC', 'GNTBTC', 'LOOMBTC', 'REPBTC', 'TUSDBTC', 'ZENBTC', 'SKYBTC', 'CVCBTC', 'THETABTC', 'IOTXBTC', 'QKCBTC', 'AGIBTC', 'NXSBTC', 'DATABTC', 'SCBTC', 'NPXSBTC', 'KEYBTC', 'NASBTC', 'MFTBTC', 'DENTBTC', 'ARDRBTC', 'HOTBTC', 'VETBTC', 'DOCKBTC', 'POLYBTC', 'PHXBTC', 'HCBTC', 'GOBTC', 'PAXBTC', 'RVNBTC', 'DCRBTC']
interval = grab_kline_interval()
create_needed_directories = create_directories(pair_list)

for x in range(0,len(pair_list)):
	symbol = str(pair_list[x]).replace(' ','')
	start_date = datetime.date(2017,7,1)
	end_date = datetime.datetime.now()
	dates = [date for date in rrule(MONTHLY, dtstart=start_date, until=end_date)]
	for x in range(0,len(dates)):
		date_splitter = str(dates[x]).split('-')
		year = date_splitter[0]
		numerical_month = date_splitter[1]
		month_abbreviation_dict = {'01':'Jan','02':'Feb','03':'Mar','04':'Apr','05':'May','06':'Jun','07':'Jul','08':'Aug','09':'Sept','10':'Oct','11':'Nov','12':'Dec'}
		for num,abbr in month_abbreviation_dict.items():
			if numerical_month==num:
				calendar_month = abbr
		klines_date = '{}, {}'.format(str(calendar_month),str(year))
		csv_month = '{}-{}-'.format(str(year),str(numerical_month))
		if numerical_month in ['01','03','05','07','08','10','12']:
			day = range(1,32)
		elif numerical_month in ['04','06','09','11']:
			day = range(1,31)
		elif numerical_month in ['02'] and year not in ['2020','2024','2028','2032','2036','2040','2044']:
			day = range(1,29)
		else:
			day = range(1,30)
		for y in day:
			next_day = y+1
			start = '{} {}'.format(str(y),str(klines_date))
			if y == day[-1]:
				next_day = 1
				if int(numerical_month) in range(1,11):
					next_numerical_month = int(numerical_month)+1
				else:
					next_numerical_month = 1
				next_numerical_month_abbreviation_dict = {'1':'Jan','2':'Feb','3':'Mar','4':'Apr','5':'May','6':'Jun','7':'Jul','8':'Aug','9':'Sept','10':'Oct','11':'Nov','12':'Dec'}
				for key,val in next_numerical_month_abbreviation_dict.items():
					if str(next_numerical_month)==key:
						calendar_month = val
			end = '{} {}'.format(str(next_day),str(klines_date))
			print ('symbol {} start {} end {}'.format(str(symbol),str(start),str(end)))
			klines = get_historical_klines(symbol, interval, start, end)
			if len(klines)>0:
				titles = ('Date','Open','High','Low','Close','Volume')
				if y in range(1,10):
					csv_day = '0{}'.format(str(y))
					print (str(csv_day))
				else:
					csv_day = str(y)
				with open('{}historical_price_data/{}/{}{}_{}.csv'.format(str(full_path),str(symbol),str(csv_month),str(csv_day),str(interval)), 'a') as f:
						writer = csv.writer(f)
						writer.writerow(titles)
				history_list = list()
				for x in range(0,(len(klines)-1)):
					open_epoch_timestamp = float(klines[x][0])/1000
					open_time = datetime.datetime.utcfromtimestamp(open_epoch_timestamp).strftime('%Y-%m-%d %H:%M:%S.%f')
					open_price = klines[x][1]
					high = klines[x][2]
					low = klines[x][3]
					close_price = klines[x][4]
					volume = klines[x][5]
					close_epoch_timestamp = float(klines[x][6])/1000
					close_time = datetime.datetime.utcfromtimestamp(close_epoch_timestamp).strftime('%Y-%m-%d %H:%M:%S.%f')
					quote_asset_volume = klines[x][7]
					number_of_trades = klines[x][8]
					taker_buy_base_asset_volume = klines[x][9]
					taker_buy_quote_asset_volume = klines[x][10]
					fields = (open_time,open_price,high,low,close_price,volume)
					history_list.append(fields)
					with open('{}historical_price_data/{}/{}{}_{}.csv'.format(str(full_path),str(symbol),str(csv_month),str(csv_day),str(interval)), 'a') as f:
						writer = csv.writer(f)
						writer.writerow(fields)