Skip to content

Instantly share code, notes, and snippets.

@nagishin

nagishin/OHLCV DataFrameをjson変換.ipynb

Created October 29, 2018 01:40
Show Gist options
  • Save nagishin/e9e43c9fe4dcd2d5b72224c0404cc19f to your computer and use it in GitHub Desktop.
Save nagishin/e9e43c9fe4dcd2d5b72224c0404cc19f to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
OHLCV DataFrameをjson変換.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
"# coding: utf-8\n",
"from datetime import datetime, timedelta\n",
"import time, calendar, pytz, requests\n",
"import pandas as pd\n",
"\n",
"ALLOWED_PERIOD = {\n",
" \"1m\": [\"1m\", 1, 1], \"3m\": [\"1m\", 3, 3],\n",
" \"5m\": [\"5m\", 1, 5], \"15m\": [\"5m\", 3, 15], \"30m\": [\"5m\", 6, 30],\n",
" \"1h\": [\"1h\", 1, 60], \"2h\": [\"1h\", 2, 120],\n",
" \"3h\": [\"1h\", 3, 180], \"4h\": [\"1h\", 4, 240],\n",
" \"6h\": [\"1h\", 6, 360], \"12h\": [\"1h\", 12, 720],\n",
" \"1d\": [\"1d\", 1, 1440],\n",
" # not support yet '3d', '1w', '2w', '1m'\n",
"}\n",
"\n",
"#---------------------------------------------------------------------\n",
"# 指定された時間足のOHLCVデータをBitMEX REST APIより取得\n",
"#---------------------------------------------------------------------\n",
"# [@param]\n",
"# period =\"1m\" 時間足(1m, 3m, 5m, 15m, 30m, 1h, 2h, 3h, 4h, 6h, 12h, 1d)\n",
"# symbol =\"XBTUSD\" 通貨ペア\n",
"# count =1000 取得期間 ※\n",
"# reverse =True 並び順(True:新->古, False:古->新)\n",
"# partial =False 最新未確定足を取得するか(True:含む, False:含まない)\n",
"# tstype =\"UTMS\" 時刻形式(以下の5パターン)\n",
"# \"UTMS\":UnixTime(ミリ秒), \"UTS\":UnixTime(秒), \"DT\":datetime,\n",
"# \"STMS\":日付文字列(%Y-%m-%dT%H:%M:%S.%fZ), \"STS\":日付文字列(%Y-%m-%dT%H:%M:%S)\n",
"# [return]\n",
"# DataFrame : [[timestamp, open, high, low, close, volume], ・・・]\n",
"# List : [[timestamp, open, high, low, close, volume], ・・・]\n",
"# ※戻り値データ型 (List or DataFrame) によって取得関数を2パターンに分けてある\n",
"#---------------------------------------------------------------------\n",
"# ※ 1m, 5m, 1h, 1dを基準にmergeしているため、それ以外を取得する場合は件数に注意\n",
"# (3m : 1m * 3期間より、3mを10期間取得するなら、APIでは 10 * 3 = 30件になる)\n",
"# APIは1回でMAX10000件までの取得としている\n",
"# 30mや4h、6h、12hなどを取得する場合、件数を大きくするとAPI制限にかかる可能性がある\n",
"#---------------------------------------------------------------------\n",
"# [usage] lst_ohlcv = fetch_ohlcv_lst(period=\"15m\", count=1000, tstype=\"DT\")\n",
"# df_ohlcv = fetch_ohlcv_df(period=\"5m\", count=1000, partial=True)\n",
"#---------------------------------------------------------------------\n",
"# ListでOHLCVを取得\n",
"def fetch_ohlcv_lst(period=\"1m\", symbol=\"XBTUSD\", count=1000, reverse=True, partial=False, tstype=\"UTMS\"):\n",
" df = fetch_ohlcv_df(period, symbol, count, reverse, partial, tstype)\n",
" if df is None:\n",
" return None\n",
" return [[int(x[0]) if tstype==\"UTS\" or tstype==\"UTMS\" else x[0], x[1], x[2], x[3], x[4], int(x[5])] for x in df.values.tolist()]\n",
"\n",
"# DataFrameでOHLCVを取得\n",
"def fetch_ohlcv_df(period=\"1m\", symbol=\"XBTUSD\", count=1000, reverse=True, partial=False, tstype=\"UTMS\"):\n",
" if period not in ALLOWED_PERIOD:\n",
" return None\n",
" period_params = ALLOWED_PERIOD[period]\n",
" need_count = (count + 1) * period_params[1] # マージ状況により、不足が発生する可能性があるため、多めに取得\n",
"\n",
" # REST APIリクエストでOHLCVデータ取得\n",
" df_ohlcv = __get_ohlcv_paged(symbol=symbol, period=period_params[0], count=need_count)\n",
"\n",
" # DataFrame化して指定時間にリサンプリング\n",
" if period_params[1] > 1:\n",
" minutes = ALLOWED_PERIOD[period][2]\n",
" offset = str(minutes) + \"T\"\n",
" if 60 <= minutes < 1440:\n",
" offset = str(minutes / 60) + \"H\"\n",
" elif 1440 <= minutes:\n",
" offset = str(minutes / 1440) + \"D\"\n",
" df_ohlcv = df_ohlcv.resample(offset).agg({\n",
" \"timestamp\": \"first\",\n",
" \"open\": \"first\",\n",
" \"high\": \"max\",\n",
" \"low\": \"min\",\n",
" \"close\": \"last\",\n",
" \"volume\": \"sum\",\n",
" })\n",
" # 未確定の最新足を除去\n",
" if partial == False:\n",
" df_ohlcv = df_ohlcv.iloc[:-1]\n",
" # マージした結果、余分に取得している場合、古い足から除去\n",
" if len(df_ohlcv) > count:\n",
" df_ohlcv = df_ohlcv.iloc[len(df_ohlcv)-count:]\n",
" # index解除\n",
" df_ohlcv.reset_index(inplace=True)\n",
" # timestampを期間終わり時刻にするため、datetimeをシフト\n",
" df_ohlcv[\"datetime\"] += timedelta(minutes=ALLOWED_PERIOD[period][2])\n",
" # timestamp変換\n",
" __convert_timestamp(df_ohlcv, tstype)\n",
" # datetime列を削除\n",
" df_ohlcv.drop(\"datetime\", axis=1, inplace=True)\n",
" # 並び順を反転\n",
" if reverse == True:\n",
" df_ohlcv = df_ohlcv.iloc[::-1]\n",
" # indexリセット\n",
" df_ohlcv.reset_index(inplace=True, drop=True)\n",
" return df_ohlcv\n",
"\n",
"# private\n",
"def __convert_timestamp(df_ohlcv, timestamp=\"UTMS\"):\n",
" if timestamp == \"UTS\":\n",
" df_ohlcv[\"timestamp\"] = pd.Series([int(dt.timestamp()) for dt in df_ohlcv[\"datetime\"]])\n",
" elif timestamp == \"UTMS\":\n",
" df_ohlcv[\"timestamp\"] = pd.Series([int(dt.timestamp()) * 1000 for dt in df_ohlcv[\"datetime\"]])\n",
" elif timestamp == \"DT\":\n",
" df_ohlcv[\"timestamp\"] = df_ohlcv[\"datetime\"]\n",
" elif timestamp == \"STS\":\n",
" df_ohlcv[\"timestamp\"] = pd.Series([dt.strftime(\"%Y-%m-%dT%H:%M:%S\") for dt in df_ohlcv[\"datetime\"]])\n",
" elif timestamp == \"STMS\":\n",
" df_ohlcv[\"timestamp\"] = pd.Series([dt.strftime(\"%Y-%m-%dT%H:%M:%S.%fZ\") for dt in df_ohlcv[\"datetime\"]])\n",
" else:\n",
" df_ohlcv[\"timestamp\"] = df_ohlcv[\"datetime\"]\n",
"\n",
"def __get_ohlcv_paged(symbol=\"XBTUSD\", period=\"1m\", count=1000):\n",
" ohlcv_list = []\n",
" utc_now = datetime.now(pytz.utc)\n",
" to_time = int(utc_now.timestamp())\n",
" #to_time = int(time.mktime(utc_now.timetuple()))\n",
" from_time = to_time - ALLOWED_PERIOD[period][2] * 60 * count\n",
" start = from_time\n",
" end = to_time\n",
" if count > 10000:\n",
" end = from_time + ALLOWED_PERIOD[period][2] * 60 * 10000\n",
" while start <= to_time:\n",
" ohlcv_list += __fetch_ohlcv_list(symbol=symbol, period=period, start=start, end=end)\n",
" start = end + ALLOWED_PERIOD[period][2] * 60\n",
" end = start + ALLOWED_PERIOD[period][2] * 60 * 10000\n",
" if end > to_time:\n",
" end = to_time\n",
" df_ohlcv = pd.DataFrame(ohlcv_list,\n",
" columns=[\"timestamp\", \"open\", \"high\", \"low\", \"close\", \"volume\"])\n",
" df_ohlcv[\"datetime\"] = pd.to_datetime(df_ohlcv[\"timestamp\"], unit=\"s\")\n",
" df_ohlcv = df_ohlcv.set_index(\"datetime\")\n",
" df_ohlcv.index = df_ohlcv.index.tz_localize(\"UTC\")\n",
" return df_ohlcv\n",
"\n",
"def __fetch_ohlcv_list(symbol=\"XBTUSD\", period=\"1m\", start=0, end=0):\n",
" param = {\"period\": ALLOWED_PERIOD[period][2], \"from\": start, \"to\": end}\n",
" url = \"https://www.bitmex.com/api/udf/history?symbol=XBTUSD&resolution={period}&from={from}&to={to}\".format(**param)\n",
" res = requests.get(url)\n",
" data = res.json()\n",
" return [list(ohlcv) for ohlcv in zip(data[\"t\"], data[\"o\"], data[\"h\"], data[\"l\"], data[\"c\"], data[\"v\"])]\n"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"① df_ohlcv は [DataFrame]\n"
]
},
{
"data": {
"text/plain": [
"pandas.core.frame.DataFrame"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>timestamp</th>\n",
" <th>open</th>\n",
" <th>high</th>\n",
" <th>low</th>\n",
" <th>close</th>\n",
" <th>volume</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1540776840</td>\n",
" <td>6405.0</td>\n",
" <td>6405.5</td>\n",
" <td>6405</td>\n",
" <td>6405.0</td>\n",
" <td>72254</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1540776900</td>\n",
" <td>6405.0</td>\n",
" <td>6405.5</td>\n",
" <td>6405</td>\n",
" <td>6405.5</td>\n",
" <td>40434</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1540776960</td>\n",
" <td>6405.5</td>\n",
" <td>6405.5</td>\n",
" <td>6405</td>\n",
" <td>6405.0</td>\n",
" <td>37376</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1540777020</td>\n",
" <td>6405.0</td>\n",
" <td>6405.5</td>\n",
" <td>6405</td>\n",
" <td>6405.0</td>\n",
" <td>9063</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1540777080</td>\n",
" <td>6405.0</td>\n",
" <td>6405.0</td>\n",
" <td>6405</td>\n",
" <td>6405.0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" timestamp open high low close volume\n",
"0 1540776840 6405.0 6405.5 6405 6405.0 72254\n",
"1 1540776900 6405.0 6405.5 6405 6405.5 40434\n",
"2 1540776960 6405.5 6405.5 6405 6405.0 37376\n",
"3 1540777020 6405.0 6405.5 6405 6405.0 9063\n",
"4 1540777080 6405.0 6405.0 6405 6405.0 0"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
"② df_ohlcv['close'] は [Series]\n"
]
},
{
"data": {
"text/plain": [
"pandas.core.series.Series"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"0 6405.0\n",
"1 6405.5\n",
"2 6405.0\n",
"3 6405.0\n",
"4 6405.0\n",
"Name: close, dtype: float64"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
"③ df_ohlcv['close'].values は [ndarray]\n"
]
},
{
"data": {
"text/plain": [
"numpy.ndarray"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"array([6405. , 6405.5, 6405. , 6405. , 6405. ])"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
"④ df_ohlcv['close'].values.tolist() は [list]\n"
]
},
{
"data": {
"text/plain": [
"list"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"[6405.0, 6405.5, 6405.0, 6405.0, 6405.0]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# DataFrameにてOHLCV取得\n",
"df_ohlcv = fetch_ohlcv_df(period=\"1m\", count=5, reverse=False, partial=True, tstype=\"UTS\")\n",
"print(\"① df_ohlcv は [DataFrame]\")\n",
"display(type(df_ohlcv), df_ohlcv)\n",
"\n",
"print(\"\\n\")\n",
"\n",
"# DataFrameの列はSeries\n",
"print(\"② df_ohlcv['close'] は [Series]\")\n",
"display(type(df_ohlcv[\"close\"]), df_ohlcv[\"close\"])\n",
"\n",
"print(\"\\n\")\n",
"\n",
"# Seriesのvaluesはndarray\n",
"print(\"③ df_ohlcv['close'].values は [ndarray]\")\n",
"display(type(df_ohlcv[\"close\"].values), df_ohlcv[\"close\"].values)\n",
"\n",
"print(\"\\n\")\n",
"\n",
"# ndarrayのtolist()はlist\n",
"print(\"④ df_ohlcv['close'].values.tolist() は [list]\")\n",
"display(type(df_ohlcv[\"close\"].values.tolist()), df_ohlcv[\"close\"].values.tolist())"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"dict"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"{'t': array([1540776840, 1540776900, 1540776960, 1540777020, 1540777080],\n",
" dtype=int64),\n",
" 'o': array([6405. , 6405. , 6405.5, 6405. , 6405. ]),\n",
" 'h': array([6405.5, 6405.5, 6405.5, 6405.5, 6405. ]),\n",
" 'l': array([6405, 6405, 6405, 6405, 6405], dtype=int64),\n",
" 'c': array([6405. , 6405.5, 6405. , 6405. , 6405. ]),\n",
" 'v': array([72254, 40434, 37376, 9063, 0], dtype=int64)}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"<class 'numpy.ndarray'> [6405. 6405.5 6405. 6405. 6405. ]\n",
"\n",
"\n"
]
},
{
"data": {
"text/plain": [
"dict"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"{'t': [1540776840, 1540776900, 1540776960, 1540777020, 1540777080],\n",
" 'o': [6405.0, 6405.0, 6405.5, 6405.0, 6405.0],\n",
" 'h': [6405.5, 6405.5, 6405.5, 6405.5, 6405.0],\n",
" 'l': [6405, 6405, 6405, 6405, 6405],\n",
" 'c': [6405.0, 6405.5, 6405.0, 6405.0, 6405.0],\n",
" 'v': [72254, 40434, 37376, 9063, 0]}"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"<class 'list'> [6405.0, 6405.5, 6405.0, 6405.0, 6405.0]\n"
]
}
],
"source": [
"import json\n",
"\n",
"# 辞書型に変換(value:ndarray)\n",
"dict_ohlcv_array = {\n",
" \"t\" : df_ohlcv[\"timestamp\"].values,\n",
" \"o\" : df_ohlcv[\"open\"].values,\n",
" \"h\" : df_ohlcv[\"high\"].values,\n",
" \"l\" : df_ohlcv[\"low\"].values,\n",
" \"c\" : df_ohlcv[\"close\"].values,\n",
" \"v\" : df_ohlcv[\"volume\"].values,\n",
"}\n",
"display(type(dict_ohlcv_array), dict_ohlcv_array)\n",
"print(type(dict_ohlcv_array[\"c\"]), dict_ohlcv_array[\"c\"])\n",
"\n",
"print(\"\\n\")\n",
"\n",
"# 辞書型に変換(valueはlist)\n",
"dict_ohlcv_list = {\n",
" \"t\" : df_ohlcv[\"timestamp\"].values.tolist(),\n",
" \"o\" : df_ohlcv[\"open\"].values.tolist(),\n",
" \"h\" : df_ohlcv[\"high\"].values.tolist(),\n",
" \"l\" : df_ohlcv[\"low\"].values.tolist(),\n",
" \"c\" : df_ohlcv[\"close\"].values.tolist(),\n",
" \"v\" : df_ohlcv[\"volume\"].values.tolist(),\n",
"}\n",
"display(type(dict_ohlcv_list), dict_ohlcv_list)\n",
"print(type(dict_ohlcv_list[\"c\"]), dict_ohlcv_list[\"c\"])\n",
"\n",
"# json形式でファイル保存\n",
"with open(\"ohlcv_list.json\", mode=\"w\") as f:\n",
" json.dump(dict_ohlcv_list, f, indent=4)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.6"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment