A script to export all FTX history

scrape_ftx.py

import pandas as pd
import time
import requests
import time
import hmac
from requests import Request
import sys
import json
import os

try:
    import pandas as pd
except Exception as e:
    print("You need to install pandas! Try `python3 -m pip install pandas`")
    raise e


if not os.path.exists("./data"):
    os.mkdir("./data")

api_key = input("Enter api key:")
api_secret = input("Enter api secret:")

def get(url, subaccount=None):
    ts = int(time.time() * 1000)
    request = Request('GET', url)
    prepared = request.prepare()
    signature_payload = f'{ts}{prepared.method}{prepared.path_url}'
    if prepared.body:
        signature_payload += prepared.body
    signature_payload = signature_payload.encode()
    signature = hmac.new(api_secret.encode(), signature_payload, 'sha256').hexdigest()

    headers = {
        "accept": "application/json",
        "Content-Type": "application/json",
        "FTX-KEY": f"{api_key}",
        "FTX-SIGN": f"{signature}",
        "FTX-TS": f"{ts}",
    }
    
    if subaccount is not None:
        headers["FTX-SUBACCOUNT"] = subaccount

    r = requests.get(url, headers=headers)
    if r.status_code != 200:
        print("Request got error", r.json(), "status", r.status_code)
        sys.exit("")
    return r

def get_start_end(url, subaccount=None):
    end_time = int(time.time())
    df = pd.DataFrame()
    while True:
        shape_before = df.shape[0]
        req_url = url + f"?start_time=0&end_time={end_time}"
        l = get(req_url, subaccount=subaccount).json()["result"]
        for x in l:
            for k in x.keys():
                if isinstance(x[k], dict):
                    x[k] = str(x[k])
        df_new = pd.DataFrame(l)
        df = pd.concat([df, df_new])
        df = df.drop_duplicates()
        shape_after = df.shape[0]
        print(end_time, req_url, df_new.shape[0], shape_after - shape_before)
        if shape_after == shape_before:
            break
        end_time = pd.to_datetime(df["time"]).min().timestamp()
    return df

subaccounts = [None] + [x["nickname"] for x in get("https://ftx.com/api/subaccounts").json()["result"]]
print("Found subaccounts", subaccounts)

for subaccount in subaccounts:
    if subaccount is None:
        subaccount_prefix = "./data/"
    else:
        subaccount_prefix = f"./data/subaccount_{subaccount}_"

    # account details
    try:
        account_info = get("https://ftx.com/api/account", subaccount=subaccount).json()["result"]
        json.dump(account_info, open(subaccount_prefix + "account_info.json", "w"))
    except Exception as e:
        print("WARNING: On pulling", "account details got exception", e, "skipping")

    # balances
    try:
        balances = get("https://ftx.com/api/wallet/all_balances", subaccount=subaccount).json()["result"]
        json.dump(balances, open(subaccount_prefix + "balances.json", "w"))
    except Exception as e:
        print("WARNING: On pulling", "balances got exception", e, "skipping")

    # Deposits
    try:
        deposit_history = get_start_end("https://ftx.com/api/wallet/deposits", subaccount=subaccount)
        deposit_history.to_csv(subaccount_prefix + "deposit_history.csv")
    except Exception as e:
        print("WARNING: On pulling", "Deposits got exception", e, "skipping")

    # Withdrawals
    try:
        withdrawal_history = get_start_end("https://ftx.com/api/wallet/withdrawals", subaccount=subaccount)
        withdrawal_history.to_csv(subaccount_prefix + "withdrawal_history.csv")
    except Exception as e:
        print("WARNING: On pulling", "Withdrawals got exception", e, "skipping")

    # borrow history
    try:
        borrow_history = get_start_end("https://ftx.com/api/spot_margin/borrow_history", subaccount=subaccount)
        borrow_history.to_csv(subaccount_prefix + "borrow_history.csv")
    except Exception as e:
        print("WARNING: On pulling", "borrow history got exception", e, "skipping")

    # lend history
    try:
        lending_history = get_start_end("https://ftx.com/api/spot_margin/lending_history", subaccount=subaccount)
        lending_history.to_csv(subaccount_prefix + "lending_history.csv")
    except Exception as e:
        print("WARNING: On pulling", "lend history got exception", e, "skipping")

    # referral history
    try:
        referral_history = get_start_end("https://ftx.com/api/referral_rebate_history", subaccount=subaccount)
        referral_history.to_csv(subaccount_prefix + "referral_history.csv")
    except Exception as e:
        print("WARNING: On pulling", "referral history got exception", e, "skipping")

    # fill history
    try:
        fills = get_start_end("https://ftx.com/api/fills", subaccount=subaccount)
        fills.to_csv(subaccount_prefix + "fills.csv")
    except Exception as e:
        print("WARNING: On pulling", "fill history got exception", e, "skipping")

    # funding payments
    try:
        funding_payments = get_start_end("https://ftx.com/api/funding_payments", subaccount=subaccount)
        funding_payments.to_csv(subaccount_prefix + "funding_payments.csv")
    except Exception as e:
        print("WARNING: On pulling", "funding payments got exception", e, "skipping")

hey, I got an error on line:
df = df.drop_duplicates()
" Unhashable type: "dict" "
It's running the fix:
df = df.loc[df.astype('str').drop_duplicates().index]

do you know which line from the main script generates this (eg is it fills/borrow/lending/etc)? Sorry, I don't have much FTX account history so probably didn't catch all corner cases.

yeah it's 53
df = df.drop_duplicates()
i didn't have the current revision

Thank you for this!
Note for others: To download for different subaccounts change the headers to:

headers = {
        "accept": "application/json",
        "Content-Type": "application/json",
        "FTX-KEY": f"{api_key}",
        "FTX-SIGN": f"{signature}",
        "FTX-TS": f"{ts}",
        "FTX-SUBACCOUNT": "subaccount_here"
    }

(and change some filenames otherwise they overlap)

Now the code is frozen though, so above fix isnt a good solution.

I added looping through subaccounts, skipping pulls that throw errors, and better error messages, etc...

Thank you @sbarratt .

Thank you @sbarratt.

Line 1 seems superfluous because it overlaps lines 11-15, and line 1 provides more detailed error handling.