javipus · June 2, 2021 12:37
diff --git a/dummy_keys.py b/dummy_keys.py
 API_KEYS = {
    'infura': "YOUR_INFURA_KEY",
    'thegraph': "YOUR_THEGRAPH_KEY",
    'etherscan': "YOUR_ETHERSCAN_KEY",
 }
diff --git a/get_usd_prices.py b/get_usd_prices.py
 from collections import OrderedDict
 import json
 import time
 import warnings
 import requests
 import argparse
 from web3 import Web3
 from web3.exceptions import ABIFunctionNotFound
 from gql import gql, Client
 from gql.transport.requests import RequestsHTTPTransport

 from keys import API_KEYS

 # Hardcoded unit conversion in case the ERC-20 interface doesn't implement `decimals`
 wei = 18
 units = {
    'WBTC': 8,
    'USDC': 6,
 }

 # see on etherscan https://etherscan.io/address/0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f
 uniswap_factory_address = "0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f"

 # h/t https://towardsdatascience.com/exploring-decentraland-marketplace-sales-with-thegraph-and-graphql-2f5e8e7199b5
 # Select your transport with a defined url endpoint
 transport = RequestsHTTPTransport(
    url="https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v2")
 # Create a GraphQL client using the defined transport
 client = Client(transport=transport, fetch_schema_from_transport=True)

 # Use infura to fetch and call smart contracts via HTTP
 web3 = Web3(Web3.HTTPProvider(
    f'https://mainnet.infura.io/v3/{API_KEYS["infura"]}'))


 def load_tokens(tokens_file):
    with open(tokens_file, 'r') as f:
        tokens = f.read().splitlines()
    return tokens


 def get_reserves(token0, token1, cache='reserves.json'):
    _token0, _token1, pool = get_uniswap_pool(token0, token1)
    reserve0, reserve1, blockTimeStampLast = pool.functions.getReserves().call()
    decimals0, decimals1 = map(get_decimals, (_token0, _token1))
    ret = OrderedDict({
        _token0: reserve0 / 10**decimals0,
        _token1: reserve1 / 10**decimals1,
    })

    # TODO switch to csv cache so you don't have to parse the whole file every time you want to append
    if cache:
        with open(cache, 'r') as f:
            data = json.load(f)
        data[f'{_token0}/{_token1}'] = ret
        with open(cache, 'w') as f:
            json.dump(data, f)

    return ret


 def get_marginal_price(token0, token1):
    """Marginal price of `token0` in units of `token1`"""
    _token0, _token1, pool = get_uniswap_pool(token0, token1)
    reserve0, reserve1, blockTimeStampLast = pool.functions.getReserves().call()
    decimals0, decimals1 = map(get_decimals, (_token0, _token1))
    reserve0 /= 10**decimals0
    reserve1 /= 10**decimals1
    return reserve0 / reserve1, f"{_token0}/{_token1}"


 def get_uniswap_pool(token0, token1):
    """
    Get contract object representing the Uniswap pool between `token0` and `token1`.
    @param token0, token1: String e.g. "DAI" or "WETH". Token symbols are searched on Uniswap's subgraph. The contract with that symbol and highest number of transactions is returned.
    @return (tokenA, tokenB, pool): `(tokenA, tokenB)` are the pool tokens _in the order they are defined by the contract_. `pool` is the contract object.
    """
    address0, address1 = map(get_token_address, (token0, token1))
    uniswap_factory = get_contract_from_address(uniswap_factory_address)
    pool_address = uniswap_factory.functions.getPair(address0, address1).call()
    pool = get_contract_from_address(pool_address)
    _address0, _address1 = pool.functions.token0().call(), pool.functions.token1().call()
    if _address0 == address0 and \
            _address1 == address1:
        return (token0, token1, pool)
    elif _address0 == address1 and \
            _address1 == address0:
        # print(
        #     f"WARNING: You requested {token0}/{token1} but this pool is {token1}/{token0}")
        return (token1, token0, pool)
    else:
        raise Exception("Pool symbols don't match")


 def get_decimals(token, graphql_client=client, web3_provider=web3):
    contract = get_contract_from_address(get_token_address(
        token, graphql_client=graphql_client), web3_provider=web3_provider)
    try:
        return contract.functions.decimals().call()
    except ABIFunctionNotFound:
        warnings.warn(
            f"{token} ERC20 contract has no `decimals` function. Defaulting to {units.get(token, wei)}.")
        return units.get(token, wei)


 def get_token_address(token, graphql_client=client):
    # graphQL magic
    # TODO use graphene, it's safer
    query = gql(f"""
  {{
    tokens(where: {{symbol:"{token}"}}, orderBy:txCount, orderDirection: desc, first: 1){{
      id,
      txCount,
      symbol,
    }}
  }}""")
    result = client.execute(query)
    assert len(result['tokens']
               ) == 1, f"Found more than one token with name {token}"
    # TODO addresses are not checksum
    # I don't know if this is TheGraph's or Uniswap's fault
    # I'm fixing it myself but this is a hack and it's not safe
    return Web3.toChecksumAddress(result['tokens'][0]['id'])


 def get_contract_from_address(address, web3_provider=web3):
    """Get web3 contract object from address. Calls etherscan's API to retrieve ABI."""
    abi = get_abi(address)
    contract = web3.eth.contract(address=address, abi=abi)
    return contract


 def get_abi(address):
    # TODO may need to work around API limit of 5 calls / sec
    response = requests.get(
        f'https://api.etherscan.io/api?module=contract&action=getabi&apikey={API_KEYS["etherscan"]}&address={address}')
    rjson = response.json()
    if rjson['status'] == '1' and rjson['message'] == 'OK':
        return rjson['result']
    print(rjson)
    raise Exception(rjson['message'])


 def main(tokens, reference_token):
    prices = {}
    # Fetch prices
    for token in tokens:
        try:
            print(f"\n\nFetching {token} price in units of {reference_token}...")
            price, pool = get_marginal_price(token, reference_token)
            if pool == f"{token}/{reference_token}":
                price = 1/price
            elif pool == f"{reference_token}/{token}":
                pass
            else:
                raise ValueError(
                    f"Requested pool was {token}/{reference_token} but got {pool}.")
            prices[token] = price
            time.sleep(2)  # gentle on the APIs
        except Exception as e:
            warnings.warn(f"\tFailed to fetch price. Error: {e}")

    return prices


 parser = argparse.ArgumentParser()

 parser.add_argument("-t", "--tokens", default="./tokens.txt", type=str,
                    help="File containing list of token symbols, with one symbol per line", dest="input_file")
 parser.add_argument("-r", "--ref", default='USDC', type=str,
                    help="Calculate prices with respect to this reference token", dest="reference_token")
 parser.add_argument("-o", "--output", default="./prices.csv", type=str,
                    help="Saves output to a file with this name", dest="output_file")

 if __name__ == "__main__":
    args = parser.parse_args()

    with open(args.input_file, 'r') as f:
        tokens = f.read().splitlines()

    prices = main(tokens, args.reference_token)

    with open(args.output_file, 'w') as f:
        f.write(f'asset,{args.reference_token}_price')
        f.write('\n')
        f.writelines([f'{k},{v}\n' for k, v in prices.items()])
diff --git a/requirements.txt b/requirements.txt
 aniso8601==7.0.0
 argon2-cffi==20.1.0
 async-generator==1.10
 attrs==20.3.0
 autobahn==21.3.1
 Automat==20.2.0
 autopep8==1.5.6
 backcall @ file:///home/ktietz/src/ci/backcall_1611930011877/work
 base58==2.1.0
 bitarray==1.2.2
 bleach==3.3.0
 certifi==2020.12.5
 cffi==1.14.5
 chainlink-feeds==0.2.9
 chardet==4.0.0
 configparser==5.0.2
 constantly==15.1.0
 cryptography==3.4.7
 cryptowatch-sdk==0.0.14
 cycler==0.10.0
 cytoolz==0.11.0
 dateparser==1.0.0
 decorator @ file:///home/ktietz/src/ci/decorator_1611930055503/work
 defusedxml==0.7.1
 entrypoints==0.3
 eth-abi==2.1.1
 eth-account==0.5.4
 eth-hash==0.3.1
 eth-keyfile==0.5.1
 eth-keys==0.3.3
 eth-rlp==0.2.1
 eth-typing==2.2.2
 eth-utils==1.10.0
 gql==2.0.0
 graphene==2.1.8
 graphql-core==2.3.2
 graphql-relay==2.0.1
 hexbytes==0.2.1
 hyperlink==21.0.0
 idna==2.10
 incremental==21.3.0
 ipfshttpclient==0.7.0a1
 ipykernel @ file:///tmp/build/80754af9/ipykernel_1607452791405/work/dist/ipykernel-5.3.4-py3-none-any.whl
 ipython @ file:///tmp/build/80754af9/ipython_1614616449711/work
 ipython-genutils @ file:///tmp/build/80754af9/ipython_genutils_1606773439826/work
 ipywidgets==7.6.3
 jedi==0.18.0
 Jinja2==2.11.3
 jsonschema==3.2.0
 jupyter-client @ file:///tmp/build/80754af9/jupyter_client_1601311786391/work
 jupyter-core @ file:///tmp/build/80754af9/jupyter_core_1612213314396/work
 jupyterlab-pygments==0.1.2
 jupyterlab-widgets==1.0.0
 kiwisolver==1.3.1
 lru-dict==1.1.7
 MarkupSafe==1.1.1
 marshmallow==3.11.1
 matplotlib==3.3.4
 mistune==0.8.4
 multiaddr==0.0.9
 nbclient==0.5.3
 nbconvert==6.0.7
 nbformat==5.1.3
 nest-asyncio==1.5.1
 netaddr==0.8.0
 notebook==6.3.0
 numpy==1.20.1
 packaging==20.9
 pandas==1.2.3
 pandocfilters==1.4.3
 parsimonious==0.8.1
 parso==0.8.1
 pexpect @ file:///tmp/build/80754af9/pexpect_1605563209008/work
 pickleshare @ file:///tmp/build/80754af9/pickleshare_1606932040724/work
 Pillow==8.1.2
 plotly==4.14.3
 prometheus-client==0.9.0
 promise==2.3
 prompt-toolkit @ file:///tmp/build/80754af9/prompt-toolkit_1616415428029/work
 protobuf==3.15.5
 ptyprocess @ file:///tmp/build/80754af9/ptyprocess_1609355006118/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
 pycodestyle==2.7.0
 pycparser==2.20
 pycryptodome==3.10.1
 Pygments @ file:///tmp/build/80754af9/pygments_1615143339740/work
 pyOpenSSL==20.0.1
 pyparsing==2.4.7
 pyrsistent==0.17.3
 python-binance==0.7.9
 python-coinmarketcap==0.2
 python-dateutil @ file:///home/ktietz/src/ci/python-dateutil_1611928101742/work
 python-dotenv==0.15.0
 pytz==2021.1
 PyYAML==5.4.1
 pyzmq==20.0.0
 regex==2021.3.17
 requests==2.25.1
 requests-cache==0.5.2
 retrying==1.3.3
 rlp==2.0.1
 Rx==1.6.1
 scipy==1.6.1
 seaborn==0.11.1
 Send2Trash==1.5.0
 service-identity==18.1.0
 six @ file:///tmp/build/80754af9/six_1605205306277/work
 terminado==0.9.4
 testpath==0.4.4
 toml==0.10.2
 toolz==0.11.1
 tornado @ file:///tmp/build/80754af9/tornado_1606942317143/work
 traitlets @ file:///home/ktietz/src/ci/traitlets_1611929699868/work
 Twisted==21.2.0
 txaio==21.2.1
 tzlocal==2.1
 ujson==4.0.2
 urllib3==1.26.3
 varint==1.0.2
 wcwidth @ file:///tmp/build/80754af9/wcwidth_1593447189090/work
 web3==5.17.0
 webencodings==0.5.1
 websocket-client==0.58.0
 websockets==8.1
 widgetsnbextension==3.5.1
 zope.interface==5.3.0
	API_KEYS = {
	'infura': "YOUR_INFURA_KEY",
	'thegraph': "YOUR_THEGRAPH_KEY",
	'etherscan': "YOUR_ETHERSCAN_KEY",
	}
	from collections import OrderedDict
	import json
	import time
	import warnings
	import requests
	import argparse
	from web3 import Web3
	from web3.exceptions import ABIFunctionNotFound
	from gql import gql, Client
	from gql.transport.requests import RequestsHTTPTransport

	from keys import API_KEYS

	# Hardcoded unit conversion in case the ERC-20 interface doesn't implement `decimals`
	wei = 18
	units = {
	'WBTC': 8,
	'USDC': 6,
	}

	# see on etherscan https://etherscan.io/address/0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f
	uniswap_factory_address = "0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f"

	# h/t https://towardsdatascience.com/exploring-decentraland-marketplace-sales-with-thegraph-and-graphql-2f5e8e7199b5
	# Select your transport with a defined url endpoint
	transport = RequestsHTTPTransport(
	url="https://api.thegraph.com/subgraphs/name/uniswap/uniswap-v2")
	# Create a GraphQL client using the defined transport
	client = Client(transport=transport, fetch_schema_from_transport=True)

	# Use infura to fetch and call smart contracts via HTTP
	web3 = Web3(Web3.HTTPProvider(
	f'https://mainnet.infura.io/v3/{API_KEYS["infura"]}'))


	def load_tokens(tokens_file):
	with open(tokens_file, 'r') as f:
	tokens = f.read().splitlines()
	return tokens


	def get_reserves(token0, token1, cache='reserves.json'):
	_token0, _token1, pool = get_uniswap_pool(token0, token1)
	reserve0, reserve1, blockTimeStampLast = pool.functions.getReserves().call()
	decimals0, decimals1 = map(get_decimals, (_token0, _token1))
	ret = OrderedDict({
	_token0: reserve0 / 10**decimals0,
	_token1: reserve1 / 10**decimals1,
	})

	# TODO switch to csv cache so you don't have to parse the whole file every time you want to append
	if cache:
	with open(cache, 'r') as f:
	data = json.load(f)
	data[f'{_token0}/{_token1}'] = ret
	with open(cache, 'w') as f:
	json.dump(data, f)

	return ret


	def get_marginal_price(token0, token1):
	"""Marginal price of `token0` in units of `token1`"""
	_token0, _token1, pool = get_uniswap_pool(token0, token1)
	reserve0, reserve1, blockTimeStampLast = pool.functions.getReserves().call()
	decimals0, decimals1 = map(get_decimals, (_token0, _token1))
	reserve0 /= 10**decimals0
	reserve1 /= 10**decimals1
	return reserve0 / reserve1, f"{_token0}/{_token1}"


	def get_uniswap_pool(token0, token1):
	"""
	Get contract object representing the Uniswap pool between `token0` and `token1`.
	@param token0, token1: String e.g. "DAI" or "WETH". Token symbols are searched on Uniswap's subgraph. The contract with that symbol and highest number of transactions is returned.
	@return (tokenA, tokenB, pool): `(tokenA, tokenB)` are the pool tokens _in the order they are defined by the contract_. `pool` is the contract object.
	"""
	address0, address1 = map(get_token_address, (token0, token1))
	uniswap_factory = get_contract_from_address(uniswap_factory_address)
	pool_address = uniswap_factory.functions.getPair(address0, address1).call()
	pool = get_contract_from_address(pool_address)
	_address0, _address1 = pool.functions.token0().call(), pool.functions.token1().call()
	if _address0 == address0 and \
	_address1 == address1:
	return (token0, token1, pool)
	elif _address0 == address1 and \
	_address1 == address0:
	# print(
	# f"WARNING: You requested {token0}/{token1} but this pool is {token1}/{token0}")
	return (token1, token0, pool)
	else:
	raise Exception("Pool symbols don't match")


	def get_decimals(token, graphql_client=client, web3_provider=web3):
	contract = get_contract_from_address(get_token_address(
	token, graphql_client=graphql_client), web3_provider=web3_provider)
	try:
	return contract.functions.decimals().call()
	except ABIFunctionNotFound:
	warnings.warn(
	f"{token} ERC20 contract has no `decimals` function. Defaulting to {units.get(token, wei)}.")
	return units.get(token, wei)


	def get_token_address(token, graphql_client=client):
	# graphQL magic
	# TODO use graphene, it's safer
	query = gql(f"""
	{{
	tokens(where: {{symbol:"{token}"}}, orderBy:txCount, orderDirection: desc, first: 1){{
	id,
	txCount,
	symbol,
	}}
	}}""")
	result = client.execute(query)
	assert len(result['tokens']
	) == 1, f"Found more than one token with name {token}"
	# TODO addresses are not checksum
	# I don't know if this is TheGraph's or Uniswap's fault
	# I'm fixing it myself but this is a hack and it's not safe
	return Web3.toChecksumAddress(result['tokens'][0]['id'])


	def get_contract_from_address(address, web3_provider=web3):
	"""Get web3 contract object from address. Calls etherscan's API to retrieve ABI."""
	abi = get_abi(address)
	contract = web3.eth.contract(address=address, abi=abi)
	return contract


	def get_abi(address):
	# TODO may need to work around API limit of 5 calls / sec
	response = requests.get(
	f'https://api.etherscan.io/api?module=contract&action=getabi&apikey={API_KEYS["etherscan"]}&address={address}')
	rjson = response.json()
	if rjson['status'] == '1' and rjson['message'] == 'OK':
	return rjson['result']
	print(rjson)
	raise Exception(rjson['message'])


	def main(tokens, reference_token):
	prices = {}
	# Fetch prices
	for token in tokens:
	try:
	print(f"\n\nFetching {token} price in units of {reference_token}...")
	price, pool = get_marginal_price(token, reference_token)
	if pool == f"{token}/{reference_token}":
	price = 1/price
	elif pool == f"{reference_token}/{token}":
	pass
	else:
	raise ValueError(
	f"Requested pool was {token}/{reference_token} but got {pool}.")
	prices[token] = price
	time.sleep(2) # gentle on the APIs
	except Exception as e:
	warnings.warn(f"\tFailed to fetch price. Error: {e}")

	return prices


	parser = argparse.ArgumentParser()

	parser.add_argument("-t", "--tokens", default="./tokens.txt", type=str,
	help="File containing list of token symbols, with one symbol per line", dest="input_file")
	parser.add_argument("-r", "--ref", default='USDC', type=str,
	help="Calculate prices with respect to this reference token", dest="reference_token")
	parser.add_argument("-o", "--output", default="./prices.csv", type=str,
	help="Saves output to a file with this name", dest="output_file")

	if __name__ == "__main__":
	args = parser.parse_args()

	with open(args.input_file, 'r') as f:
	tokens = f.read().splitlines()

	prices = main(tokens, args.reference_token)

	with open(args.output_file, 'w') as f:
	f.write(f'asset,{args.reference_token}_price')
	f.write('\n')
	f.writelines([f'{k},{v}\n' for k, v in prices.items()])
	aniso8601==7.0.0
	argon2-cffi==20.1.0
	async-generator==1.10
	attrs==20.3.0
	autobahn==21.3.1
	Automat==20.2.0
	autopep8==1.5.6
	backcall @ file:///home/ktietz/src/ci/backcall_1611930011877/work
	base58==2.1.0
	bitarray==1.2.2
	bleach==3.3.0
	certifi==2020.12.5
	cffi==1.14.5
	chainlink-feeds==0.2.9
	chardet==4.0.0
	configparser==5.0.2
	constantly==15.1.0
	cryptography==3.4.7
	cryptowatch-sdk==0.0.14
	cycler==0.10.0
	cytoolz==0.11.0
	dateparser==1.0.0
	decorator @ file:///home/ktietz/src/ci/decorator_1611930055503/work
	defusedxml==0.7.1
	entrypoints==0.3
	eth-abi==2.1.1
	eth-account==0.5.4
	eth-hash==0.3.1
	eth-keyfile==0.5.1
	eth-keys==0.3.3
	eth-rlp==0.2.1
	eth-typing==2.2.2
	eth-utils==1.10.0
	gql==2.0.0
	graphene==2.1.8
	graphql-core==2.3.2
	graphql-relay==2.0.1
	hexbytes==0.2.1
	hyperlink==21.0.0
	idna==2.10
	incremental==21.3.0
	ipfshttpclient==0.7.0a1
	ipykernel @ file:///tmp/build/80754af9/ipykernel_1607452791405/work/dist/ipykernel-5.3.4-py3-none-any.whl
	ipython @ file:///tmp/build/80754af9/ipython_1614616449711/work
	ipython-genutils @ file:///tmp/build/80754af9/ipython_genutils_1606773439826/work
	ipywidgets==7.6.3
	jedi==0.18.0
	Jinja2==2.11.3
	jsonschema==3.2.0
	jupyter-client @ file:///tmp/build/80754af9/jupyter_client_1601311786391/work
	jupyter-core @ file:///tmp/build/80754af9/jupyter_core_1612213314396/work
	jupyterlab-pygments==0.1.2
	jupyterlab-widgets==1.0.0
	kiwisolver==1.3.1
	lru-dict==1.1.7
	MarkupSafe==1.1.1
	marshmallow==3.11.1
	matplotlib==3.3.4
	mistune==0.8.4
	multiaddr==0.0.9
	nbclient==0.5.3
	nbconvert==6.0.7
	nbformat==5.1.3
	nest-asyncio==1.5.1
	netaddr==0.8.0
	notebook==6.3.0
	numpy==1.20.1
	packaging==20.9
	pandas==1.2.3
	pandocfilters==1.4.3
	parsimonious==0.8.1
	parso==0.8.1
	pexpect @ file:///tmp/build/80754af9/pexpect_1605563209008/work
	pickleshare @ file:///tmp/build/80754af9/pickleshare_1606932040724/work
	Pillow==8.1.2
	plotly==4.14.3
	prometheus-client==0.9.0
	promise==2.3
	prompt-toolkit @ file:///tmp/build/80754af9/prompt-toolkit_1616415428029/work
	protobuf==3.15.5
	ptyprocess @ file:///tmp/build/80754af9/ptyprocess_1609355006118/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl
	pyasn1==0.4.8
	pyasn1-modules==0.2.8
	pycodestyle==2.7.0
	pycparser==2.20
	pycryptodome==3.10.1
	Pygments @ file:///tmp/build/80754af9/pygments_1615143339740/work
	pyOpenSSL==20.0.1
	pyparsing==2.4.7
	pyrsistent==0.17.3
	python-binance==0.7.9
	python-coinmarketcap==0.2
	python-dateutil @ file:///home/ktietz/src/ci/python-dateutil_1611928101742/work
	python-dotenv==0.15.0
	pytz==2021.1
	PyYAML==5.4.1
	pyzmq==20.0.0
	regex==2021.3.17
	requests==2.25.1
	requests-cache==0.5.2
	retrying==1.3.3
	rlp==2.0.1
	Rx==1.6.1
	scipy==1.6.1
	seaborn==0.11.1
	Send2Trash==1.5.0
	service-identity==18.1.0
	six @ file:///tmp/build/80754af9/six_1605205306277/work
	terminado==0.9.4
	testpath==0.4.4
	toml==0.10.2
	toolz==0.11.1
	tornado @ file:///tmp/build/80754af9/tornado_1606942317143/work
	traitlets @ file:///home/ktietz/src/ci/traitlets_1611929699868/work
	Twisted==21.2.0
	txaio==21.2.1
	tzlocal==2.1
	ujson==4.0.2
	urllib3==1.26.3
	varint==1.0.2
	wcwidth @ file:///tmp/build/80754af9/wcwidth_1593447189090/work
	web3==5.17.0
	webencodings==0.5.1
	websocket-client==0.58.0
	websockets==8.1
	widgetsnbextension==3.5.1
	zope.interface==5.3.0