Ingest Crypto Currency Data into Elasticsearch using the coinmarketcap API

crypto_currency_ingest_elasticsearch.py

## Resources:
# https://pypi.python.org/pypi/coinmarketcap/
# https://coinmarketcap.com/api/

import json
import requests

from coinmarketcap import Market
c = Market()

# set elasticsearch endpoint
elasticsearchEndpoint = 'http://es.domain.com/indexname/currencyname/'

# api call to coinmarketcap.com
data = c.ticker('ripple', limit=1, convert='USD')
data_dict = data[0]

# variables and formatting to data types
marketCapUSD = data_dict['market_cap_usd']
priceUSD = data_dict['price_usd']
lastUpdated = data_dict['last_updated']
curName = data_dict['name']
volumeUSD24h = data_dict['24h_volume_usd']
percentageChange7d = data_dict['percent_change_7d']
curSymbol = data_dict['symbol']
curRank = data_dict['rank']
percentageChange1h = data_dict['percent_change_1h']
totalSupply = data_dict['total_supply']
priceBTC = data_dict['price_btc']
availableSupply = data_dict['available_supply']
percentageChange24h = data_dict['percent_change_24h']
curId = data_dict['id']

# setting the payload for elasticsearch
payload = json.dumps(
    {
        'market_cap_usd': float(marketCapUSD),
        'price_usd': float(priceUSD),
        'last_updated': int(lastUpdated),
        'name': str(curName),
        '24h_volume_usd': float(volumeUSD24h),
        'percent_change_7d': float(percentageChange7d),
        'symbol': str(curSymbol),
        'rank': int(curRank),
        'percent_change_1h': float(percentageChange1h),
        'total_supply': float(totalSupply),
        'price_btc': float(priceBTC),
        'available_supply': float(availableSupply),
        'percent_change_24h': float(percentageChange24h),
        'id': str(curId)
    }
)

# put request to elasticsearch with the updated time as the docid, so if the api was not updated, it will overwrite the docid
requests.put(elasticsearchEndpoint + lastUpdated, data=payload)

# creating a csv of the data
log_data = open('ripple.csv', 'a')
log_data.write(
    marketCapUSD + ', ' +
    priceUSD + ', ' +
    lastUpdated + ', ' +
    curName + ', ' +
    volumeUSD24h + ', ' +
    percentageChange7d + ', ' +
    curSymbol + ', ' +
    curRank + ', ' +
    percentageChange1h + ', ' +
    totalSupply + ', ' +
    priceBTC + ', ' +
    availableSupply + ', ' +
    percentageChange24h + ', ' +
    curId +
    '\n'
)

log_data.close()

# print a curl command to test the output
print('curl -XGET ' + elasticsearchEndpoint + lastUpdated + '?pretty')

## ==== example output: ==== ##

## $ python crypto_currency_ingest_elasticsearch.py
## curl -XGET http://es.domain.com/coins/ripple/1498546468?pretty

## $ curl -XGET http://es.domain.com/coins/ripple/1498546468?pretty
## {
##   "_index" : "coins",
##   "_type" : "ripple",
##   "_id" : "1498546468",
##   "_version" : 1,
##  "found" : true,
##   "_source" : {
##     "last_updated" : 1498546468,
##     "symbol" : "XRP",
##     "rank" : 3,
##     "percent_change_24h" : -8.19,
##     "price_btc" : 1.1016E-4,
##     "id" : "ripple",
##     "market_cap_usd" : 1.0313900175E10,
##     "price_usd" : 0.269353,
##     "name" : "Ripple",
##     "24h_volume_usd" : 2.70005E8,
##     "percent_change_7d" : -12.45,
##     "percent_change_1h" : -2.42,
##     "total_supply" : 9.9994608423E10,
##     "available_supply" : 3.829138779E10
##   }
## }