A script to export all FTX history

scrape_ftx.py

import pandas as pd
import time
import requests
import time
import hmac
from requests import Request
import sys
import json

api_key = input("Enter api key:")
api_secret = input("Enter api secret:")

def get(url, subaccount=None):
    ts = int(time.time() * 1000)
    request = Request('GET', url)
    prepared = request.prepare()
    signature_payload = f'{ts}{prepared.method}{prepared.path_url}'
    if prepared.body:
        signature_payload += prepared.body
    signature_payload = signature_payload.encode()
    signature = hmac.new(api_secret.encode(), signature_payload, 'sha256').hexdigest()

    headers = {
        "accept": "application/json",
        "Content-Type": "application/json",
        "FTX-KEY": f"{api_key}",
        "FTX-SIGN": f"{signature}",
        "FTX-TS": f"{ts}",
    }
    
    if subaccount is not None:
        headers["FTX-SUBACCOUNT"] = subaccount

    r = requests.get(url, headers=headers)
    if r.status_code != 200:
        print("Request got error", r.json(), "status", r.status_code)
        sys.exit("")
    return r

def get_start_end(url, subaccount=None):
    end_time = int(time.time())
    df = pd.DataFrame()
    while True:
        shape_before = df.shape[0]
        req_url = url + f"?start_time=0&end_time={end_time}"
        l = get(req_url, subaccount=subaccount).json()["result"]
        if "deposit" in url:
            for x in l:
                if "address" in x.keys():
                    x["address"] = str(x["address"])
        df_new = pd.DataFrame(l)
        df = pd.concat([df, df_new])
        df = df.drop_duplicates()
        shape_after = df.shape[0]
        print(end_time, req_url, df_new.shape[0], shape_after - shape_before)
        if shape_after == shape_before:
            break
        end_time = pd.to_datetime(df["time"]).min().timestamp()
    return df

subaccounts = [None] + [x["nickname"] for x in get("https://ftx.com/api/subaccounts").json()["result"]]
print("Found subaccounts", subaccounts)

for subaccount in subaccounts:
    if subaccount is None:
        subaccount_prefix = ""
    else:
        subaccount_prefix = f"subaccount_{subaccount}_"

    # account details
    account_info = get("https://ftx.com/api/account", subaccount=subaccount).json()["result"]
    json.dump(account_info, open(subaccount_prefix + "account_info.json", "w"))

    # balances
    balances = get("https://ftx.com/api/wallet/all_balances", subaccount=subaccount).json()["result"]
    json.dump(balances, open(subaccount_prefix + "balances.json", "w"))

    # Deposits
    deposit_history = get_start_end("https://ftx.com/api/wallet/deposits", subaccount=subaccount)
    deposit_history.to_csv(subaccount_prefix + "deposit_history.csv")

    # Withdrawals
    withdrawal_history = get_start_end("https://ftx.com/api/wallet/withdrawals", subaccount=subaccount)
    withdrawal_history.to_csv(subaccount_prefix + "withdrawal_history.csv")

    # borrow history
    borrow_history = get_start_end("https://ftx.com/api/spot_margin/borrow_history", subaccount=subaccount)
    borrow_history.to_csv(subaccount_prefix + "borrow_history.csv")

    # lend history
    lending_history = get_start_end("https://ftx.com/api/spot_margin/lending_history", subaccount=subaccount)
    lending_history.to_csv(subaccount_prefix + "lending_history.csv")

    # referral history
    referral_history = get_start_end("https://ftx.com/api/referral_rebate_history", subaccount=subaccount)
    referral_history.to_csv(subaccount_prefix + "referral_history.csv")

    # fill history
    fills = get_start_end("https://ftx.com/api/fills", subaccount=subaccount)
    fills.to_csv(subaccount_prefix + "fills.csv")

    # funding payments
    funding_payments = get_start_end("https://ftx.com/api/funding_payments", subaccount=subaccount)
    funding_payments.to_csv(subaccount_prefix + "funding_payments.csv")