gistfile1.txt

import sys
import time
import numpy as np
import pandas as pd
from utils.utility import Utility
import ccxt

class DebtComposition(Utility):

    def __init__(self,conf):        
        super().__init__(conf=conf)
    
    def prepare_output(self):
        
        self.log("running composition",False)

        #refresh futures currencies and addresses
        self.update_perps_ccy_to_address_dict()
        self.update_futures_ccy_to_address_dict()

        dfMainnet = self.get_debt_pool('ethereum')
        dfMainnet.fillna(0, inplace=True)
        self.out(dfMainnet.copy(), "output/data-mainnet.csv")

        dfOptimism = self.get_debt_pool('optimism')
        dfOptimism.fillna(0, inplace=True)
        self.out(dfOptimism.copy(), "output/data-optimism.csv")

        dfCombined = pd.concat([dfMainnet, dfOptimism], axis=0)
        self.out(dfCombined)

        self.log("ran successful",False)

    def out(self, df, outFile="output/data.csv"):
        cols= ['supply','cap','loans_ccy','shorts_ccy','wrappers_ccy','futures_skew_ccy','futures_debt_usd','perps_skew_ccy','perps_debt_usd','eth_wrapper_ccy_legacy','debt_in_usd','position_in_usd','position_in_ccy']
        combinedDF = df.groupby(by=['currencyKey'])[cols].agg(sum)
        df.index = df["currencyKey"]
        priceDict = df["price"].to_dict()
        combinedDF["price"] = combinedDF.index.map(priceDict)
        combinedDF["debt_in_percent"] = combinedDF["position_in_usd"].abs()/combinedDF["position_in_usd"].abs().sum()*1e2
        combinedDF["debt_in_percent"] = combinedDF["debt_in_percent"].round(2)
        combinedDF["user_debt_hedge_in_usd"] =  combinedDF["position_in_usd"] / combinedDF["debt_in_usd"].sum() * 100
        combinedDF["user_debt_hedge_in_usd"] = combinedDF["user_debt_hedge_in_usd"].round(2)
        combinedDF["position_type"]          = combinedDF["position_in_usd"].apply(lambda a: "long" if a > 0 else "short")
        combinedDF["user_debt_hedge_with_correlation_in_usd"] = self.compute_hedge_with_correlation(combinedDF)
        combinedDF["timestamp"]              = int(time.time())
        combinedDF.to_csv(outFile)

    def compute_hedge_with_correlation(self,df):

        df["user_debt_hedge_with_correlation_in_usd"] = df["user_debt_hedge_in_usd"]
        
        #Incorporates stables into sUSD
        stableCurrencyKeys = ['sEUR','sCHF','sKRW','sJPY','sGBP','sAUD','sXAG','sXAU',"sUSD"]
        stablesTotal       = df.loc[df.index.isin(stableCurrencyKeys),"user_debt_hedge_in_usd"].sum()
        df.loc[df.index.isin(stableCurrencyKeys),"user_debt_hedge_with_correlation_in_usd"] = 0
        df.loc["sUSD","user_debt_hedge_with_correlation_in_usd"] = stablesTotal

        #collapse rest of cryptos into ETH
        allCurrencyKeys = set(df.index)
        majorCurrencyKeys = set(['sETH','sBTC'])
        stableCurrencyKeys = set(stableCurrencyKeys)
        nonMajorCurrencyKeys = allCurrencyKeys - majorCurrencyKeys - stableCurrencyKeys - {'sETHBTC'} # accounted for seperately
        nonMajorTotal = df.loc[df.index.isin(nonMajorCurrencyKeys),"user_debt_hedge_in_usd"].sum()
        df.loc[df.index.isin(nonMajorCurrencyKeys),"user_debt_hedge_with_correlation_in_usd"] = 0
                
        #incorporates correlation between eth and btc
        #creates correlation column
        correlation                                   = self.compute_eth_btc_correlation()

        ethHedge    = df.loc['sETH',"user_debt_hedge_in_usd"]
        btcHedge    = df.loc['sBTC',"user_debt_hedge_in_usd"]
        ethbtcHedge = df.loc['sETHBTC',"user_debt_hedge_in_usd"]
        
        df.loc['sETH',"user_debt_hedge_with_correlation_in_usd"]    = ethHedge + btcHedge * correlation + (1-correlation) * ethbtcHedge + nonMajorTotal        
        df.loc['sBTC',"user_debt_hedge_with_correlation_in_usd"]    = 0
        df.loc['sETHBTC',"user_debt_hedge_with_correlation_in_usd"] = 0
        df.loc['sUSD',"user_debt_hedge_with_correlation_in_usd"]    = 100 - (df["user_debt_hedge_with_correlation_in_usd"].abs().sum() - df.loc['sUSD',"user_debt_hedge_with_correlation_in_usd"])
        
        return df["user_debt_hedge_with_correlation_in_usd"]
        
                
    def get_debt_pool(self,chain):
        df = self.get_synth_caps(chain=chain)
        df["loans_ccy"], df["shorts_ccy"] = zip(*(df["currencyKeyHex"].apply(lambda currencyKeyHex: self.get_multi_collateral(currencyKeyHex,chain))))
        df["wrappers_ccy"]                = df["currencyKey"].apply(lambda currencyKey: self.get_wrappr_issued_synth(currencyKey,chain)) / df["price"]
        df["futures_skew_ccy"]            = df["currencyKey"].apply(lambda currencyKey: self.get_futures_skew(currencyKey,chain)) 
        df["futures_debt_usd"]            = df["currencyKey"].apply(lambda currencyKey: self.get_futures_debt(currencyKey,chain)) 
        df["perps_skew_ccy"]              = df["currencyKey"].apply(lambda currencyKey: self.get_perps_skew(currencyKey,chain)) 
        df["perps_debt_usd"]              = df["currencyKey"].apply(lambda currencyKey: self.get_perps_debt(currencyKey,chain))
        df["eth_wrapper_ccy_legacy"]      = 0
        if chain == 'mainnet':
            issuedETH, issuedUSD = self.get_legacy_wrappr_synths()
            df.loc[df.currencyKey=='sETH','eth_wrapper_ccy_legacy'] = issuedETH
            df.loc[df.currencyKey=='sUSD','eth_wrapper_ccy_legacy'] = issuedUSD
        df["debt_in_usd"]     = (df["supply"] - df["loans_ccy"] - df["shorts_ccy"]- df["wrappers_ccy"] - df["eth_wrapper_ccy_legacy"])*df["price"] + df["futures_debt_usd"] + df["perps_debt_usd"]
        df["position_in_ccy"] = (df["supply"] - df["loans_ccy"] - df["shorts_ccy"]- df["wrappers_ccy"] - df["eth_wrapper_ccy_legacy"] + df["futures_skew_ccy"] + df["perps_skew_ccy"])
        df["position_in_usd"] = df["position_in_ccy"] * df["price"]
        df["chain"] = chain
        return df
            
    def get_futures_skew(self,currencyKey,chain):
        if not currencyKey in self.futuresCcyToAddress.keys() or chain =='ethereum':
            return 0
        futuresContractAddress = self.futuresCcyToAddress[currencyKey]
        dataContract           = self.get_snx_contract(contractName='FuturesMarketData', chain=chain)
        data = dataContract.functions.marketDetails(futuresContractAddress).call()
        return data[6][-1] / 1e18

    def get_futures_debt(self,currencyKey,chain):
        if not currencyKey in self.futuresCcyToAddress.keys() or chain =='ethereum':
            return 0
        contract = self.get_contract_from_address(self.futuresCcyToAddress[currencyKey],'optimism')
        return contract .functions.marketDebt().call()[0]/1e18

    def get_perps_skew(self,currencyKey,chain):
        if not currencyKey in self.perpsCcyToAddress.keys() or chain =='ethereum':
            return 0        
        w3                  = self.get_w3(chain)
        functionSignature   = w3.keccak(text='marketSkew()')[:4].hex()
        hexstr              = w3.eth.call({'to':self.perpsCcyToAddress[currencyKey],'data':functionSignature}).hex()
        return int.from_bytes(bytes.fromhex(hexstr[2:]),byteorder='big',signed=True)/1e18 #hex str to signed int

    def get_perps_debt(self,currencyKey,chain):
        if not currencyKey in self.perpsCcyToAddress.keys() or chain =='ethereum':
            return 0        
        w3                  = self.get_w3(chain)
        functionSignature   = w3.keccak(text='marketDebt()')[:4].hex()
        hexstr              = w3.eth.call({'to':self.perpsCcyToAddress[currencyKey],'data':functionSignature}).hex()[:66]
        return int(hexstr,16)/1e18
            
    def get_synth_caps(self,chain):
        w3                      = self.get_w3(chain)
        #get synth list
        utilsContract           = self.get_snx_contract(contractName='SynthUtil',chain=chain)
        synthSupplyList         = utilsContract.functions.synthsTotalSupplies().call()
        synthDF                 = pd.DataFrame(synthSupplyList).T
        synthDF.columns         = ['currencyKeyHex','supply','cap']        
        synthDF["currencyKey"]  = synthDF["currencyKeyHex"].str.decode('utf-8').str.replace('\x00','')
        synthDF                 = synthDF.query("supply > 0").copy()
        synthDF["price"]        = synthDF["cap"] / synthDF["supply"]
        synthDF["cap"]          = synthDF["cap"]  / 1e18
        synthDF["supply"]       = synthDF["supply"] / 1e18
        #add futures keys
        if not chain == 'ethereum':
            futuresCurrencyKeyHex     = self.get_futures_markets(chain)
            for currencyKeyHex in futuresCurrencyKeyHex:
                currencyKey = w3.toText(currencyKeyHex).replace('\x00','')
                if not currencyKey in synthDF["currencyKey"].to_list():
                    price       = self.get_underlying_price_for_futures_markets(currencyKey,chain)
                    synthDF     = synthDF.append({'currencyKeyHex':currencyKeyHex,
                                                  'supply':0,
                                                  'cap':0,
                                                  'currencyKey':currencyKey,
                                                  'price':price},
                                                 ignore_index=True)    
            for currencyKey in self.perpsCcyToAddress.keys():
                if not currencyKey in synthDF["currencyKey"].to_list():
                    price       = self.get_underlying_price_for_futures_markets(currencyKey,chain)
                    synthDF     = synthDF.append({'currencyKeyHex': w3.toHex(text=currencyKey),
                                                  'supply':0,
                                                  'cap':0,
                                                  'currencyKey':currencyKey,
                                                  'price':price},
                                                 ignore_index=True)    

        return synthDF
    
    def get_underlying_price_for_futures_markets(self,currencyKey,chain):
        w3     = self.get_w3(chain)
        ticker = currencyKey[1:]
        tickerHex = w3.toHex(text=ticker)
        exchangerContract = self.get_snx_contract(contractName='ExchangeRates', chain=chain)
        return exchangerContract.functions.rateForCurrency(tickerHex).call()/1e18
    
    def get_futures_markets(self,chain):
        currencyKeyList = list()
        marketAddressList    = self.get_futures_markets_addresses()
        for marketAddress in marketAddressList:
            contract = self.get_contract_from_address(address=marketAddress,
                                                      chain=chain)
            currencyKeyList.append(contract.functions.marketKey().call())
        return currencyKeyList
          
    def get_wrappr_issued_synth(self,currencyKey,chain):
        ticker = currencyKey[1:].lower()
        if not  ticker in self.conf["factoryWrappers"][chain].keys():
            return 0
        contract = self.get_contract_from_address(address=self.conf["factoryWrappers"][chain][ticker], chain=chain)
        return contract.functions.totalIssuedSynths().call() / 1e18
    
    def get_multi_collateral(self,currencyKeyHex,chain):
        contract = self.get_snx_contract(contractName = 'CollateralManagerState',chain = chain)
        longs, shorts = contract.functions.totalIssuedSynths(currencyKeyHex).call()
        return longs / 1e18 , shorts / 1e18
    
    def get_legacy_wrappr_synths(self):
        wrapprContract = self.get_snx_contract(contractName="EtherWrapper", chain='mainnet')
        return wrapprContract.functions.sETHIssued().call()/1e18, wrapprContract.functions.sUSDIssued().call()/1e18
        
    def compute_eth_btc_correlation(self):
        exchange = ccxt.binanceus()
        for symbol in ['ETH/USDT','BTC/USDT']:            
            ohlcvs = exchange.fetch_ohlcv(symbol, '1d')
            tempDF = pd.DataFrame(ohlcvs)
            tempDF = tempDF[tempDF.columns[:2]]            
            tempDF.columns = ['timestamp',symbol]
            tempDF[symbol] = tempDF[symbol].apply(np.log).diff(1)
            tempDF[symbol].dropna(inplace=True)
            if not 'df' in vars():
                df = tempDF.copy()
            else:
                df = pd.merge_asof(left=df,right=tempDF,left_on='timestamp',right_on='timestamp')
        df.dropna(inplace=True)                                    
        return df['ETH/USDT'].corr(df["BTC/USDT"])    
    
    def run_auto(self):

        while True:
            try:

                #get data
                self.prepare_output()
                                
                #uploads data file
                time.sleep(60*10*1)
                                
            except KeyboardInterrupt:
                break
            
            except:
                self.logger.exception('issue seen with controller restarting bot')
                time.sleep(60*5)
                sys.exit(3)
                
#%%
if __name__ == '__main__':
    from utils.utility import parse_config
    conf = parse_config(r"config/conf.yaml")
    self = DebtComposition(conf)
    self.prepare_output()