Skip to content

Instantly share code, notes, and snippets.

@shokoe

shokoe/aws-spot-prices-regions.json - Grafana Dashboard

Created March 21, 2017 20:04
Show Gist options
  • Save shokoe/336dd1f431f83d716c248a1465489f27 to your computer and use it in GitHub Desktop.
Save shokoe/336dd1f431f83d716c248a1465489f27 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
aws-spot-prices-regions.json - Grafana Dashboard
{
"__inputs": [
{
"name": "DS_GRAPHITE",
"label": "graphite",
"description": "",
"type": "datasource",
"pluginId": "graphite",
"pluginName": "Graphite"
}
],
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "4.1.2"
},
{
"type": "panel",
"id": "graph",
"name": "Graph",
"version": ""
},
{
"type": "datasource",
"id": "graphite",
"name": "Graphite",
"version": "1.0.0"
}
],
"annotations": {
"list": []
},
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"hideControls": false,
"id": null,
"links": [],
"refresh": false,
"rows": [
{
"collapse": false,
"height": "350px",
"panels": [
{
"aliasColors": {
"general.ondemand": "#DEDAF7",
"us-east-1.general.ondemand": "#DEDAF7"
},
"bars": false,
"datasource": "${DS_GRAPHITE}",
"decimals": 2,
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"id": 1,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": false,
"show": true,
"sort": "current",
"sortDesc": false,
"total": true,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"maxDataPoints": "",
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/ondemand/",
"fill": 1,
"linewidth": 0,
"zindex": -1
}
],
"span": 9,
"stack": false,
"steppedLine": true,
"targets": [
{
"hide": false,
"refId": "A",
"target": "aliasByNode(summarize(keepLastValue(aws.spot_price_region.$region.linux.$env.ondemand.$type, 5000), '1h', 'avg', false), 2, 4, 5)",
"textEditor": false
},
{
"hide": false,
"refId": "B",
"target": "aliasByNode(exclude(summarize(keepLastValue(aws.spot_price_region.$region.linux.$env.$zone.$type, 5000), '1h', 'avg', false), 'ondemand'), 2, 4, 5)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Hourly avg prices - $type",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": {
"general.ondemand": "#DEDAF7"
},
"bars": false,
"datasource": "${DS_GRAPHITE}",
"editable": true,
"error": false,
"fill": 0,
"grid": {},
"id": 3,
"legend": {
"alignAsTable": true,
"avg": false,
"current": true,
"max": false,
"min": false,
"show": true,
"sort": null,
"sortDesc": null,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"maxDataPoints": "",
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/ondemand/",
"fill": 1,
"linewidth": 0,
"zindex": -1
}
],
"span": 3,
"stack": false,
"steppedLine": false,
"targets": [
{
"hide": false,
"refId": "A",
"target": "aliasByNode(integral(summarize(keepLastValue(aws.spot_price_region.$region.linux.$env.ondemand.$type, 5000), '1h', 'avg', false)), 4, 5)",
"textEditor": false
},
{
"hide": false,
"refId": "B",
"target": "aliasByNode(integral(exclude(summarize(keepLastValue(aws.spot_price_region.$region.linux.$env.$zone.$type, 5000), '1h', 'avg', false), 'ondemand')), 4, 5)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Aggregated cost - $type",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": true,
"title": "Instance - avg price",
"titleSize": "h6"
},
{
"collapse": false,
"height": "450px",
"panels": [
{
"aliasColors": {
"general.ondemand": "#DEDAF7"
},
"bars": false,
"datasource": "${DS_GRAPHITE}",
"editable": true,
"error": false,
"fill": 0,
"grid": {},
"id": 2,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 0,
"links": [],
"maxDataPoints": "",
"nullPointMode": "null",
"percentage": false,
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/ondemand/",
"fill": 1,
"linewidth": 0,
"points": false,
"zindex": -1
}
],
"span": 9,
"stack": false,
"steppedLine": true,
"targets": [
{
"hide": false,
"refId": "A",
"target": "aliasByNode(keepLastValue(aws.spot_price_region.$region.linux.$env.ondemand.$type, 1440), 4, 5)",
"textEditor": false
},
{
"hide": false,
"refId": "B",
"target": "aliasByNode(consolidateBy(exclude(aws.spot_price_region.$region.linux.$env.$zone.$type, 'ondemand'), 'max'), 4, 5)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Discrete data",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": true,
"title": "Instance - discrete data",
"titleSize": "h6"
},
{
"collapse": false,
"height": "250px",
"panels": [
{
"aliasColors": {
"general.ondemand": "#DEDAF7"
},
"bars": false,
"datasource": "${DS_GRAPHITE}",
"decimals": 2,
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"id": 4,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": false,
"show": true,
"sort": null,
"sortDesc": null,
"total": true,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"maxDataPoints": "",
"minSpan": 2,
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"repeat": "compare",
"seriesOverrides": [
{
"alias": "/ondemand/",
"fill": 1,
"linewidth": 0,
"zindex": -1
}
],
"span": 6,
"stack": false,
"steppedLine": true,
"targets": [
{
"hide": false,
"refId": "A",
"target": "aliasByNode(summarize(keepLastValue(aws.spot_price_region.$region.linux.$env.ondemand.$compare, 5000), '1h', 'avg', false), 4, 5)",
"textEditor": false
},
{
"hide": false,
"refId": "B",
"target": "aliasByNode(exclude(summarize(keepLastValue(aws.spot_price_region.$region.linux.$env.$zone.$compare, 5000), '1h', 'avg', false), 'ondemand'), 4, 5)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Hourly avg prices - $compare",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": true,
"title": "Compare - avg price",
"titleSize": "h6"
},
{
"collapse": false,
"height": "250px",
"panels": [
{
"aliasColors": {
"general.ondemand": "#DEDAF7"
},
"bars": false,
"datasource": "${DS_GRAPHITE}",
"editable": true,
"error": false,
"fill": 0,
"grid": {},
"id": 8,
"legend": {
"alignAsTable": true,
"avg": false,
"current": true,
"max": false,
"min": false,
"show": true,
"sort": null,
"sortDesc": null,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"maxDataPoints": "",
"minSpan": 2,
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"repeat": "compare",
"seriesOverrides": [
{
"alias": "general.ondemand",
"fill": 1,
"linewidth": 0,
"zindex": -1
}
],
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"hide": false,
"refId": "A",
"target": "aliasByNode(integral(summarize(keepLastValue(aws.spot_price_region.$region.linux.$env.ondemand.$compare, 5000), '1h', 'avg', false)), 4, 5)",
"textEditor": false
},
{
"hide": false,
"refId": "B",
"target": "aliasByNode(integral(exclude(summarize(keepLastValue(aws.spot_price_region.$region.linux.$env.$zone.$compare, 5000), '1h', 'avg', false), 'ondemand')), 4, 5)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Aggregated cost - m3_2xlarge",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": true,
"title": "Compare - aggregated price",
"titleSize": "h6"
}
],
"schemaVersion": 14,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"allFormat": "glob",
"allValue": null,
"current": {},
"datasource": "${DS_GRAPHITE}",
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"multiFormat": "glob",
"name": "region",
"options": [],
"query": "aws.spot_price_region.*",
"refresh": 1,
"refresh_on_load": false,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allFormat": "glob",
"allValue": null,
"current": {},
"datasource": "${DS_GRAPHITE}",
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"multiFormat": "glob",
"name": "env",
"options": [],
"query": "aws.spot_price_region.$region.linux.*",
"refresh": 1,
"refresh_on_load": false,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allFormat": "glob",
"allValue": null,
"current": {},
"datasource": "${DS_GRAPHITE}",
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"multiFormat": "glob",
"name": "zone",
"options": [],
"query": "aws.spot_price_region.$region.linux.$env.*",
"refresh": 1,
"refresh_on_load": false,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allFormat": "glob",
"allValue": null,
"current": {},
"datasource": "${DS_GRAPHITE}",
"hide": 0,
"includeAll": false,
"label": null,
"multi": false,
"multiFormat": "glob",
"name": "type",
"options": [],
"query": "aws.spot_price_region.$region.linux.$env.$zone.*",
"refresh": 1,
"refresh_on_load": false,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allFormat": "glob",
"allValue": null,
"current": {},
"datasource": "${DS_GRAPHITE}",
"hide": 0,
"includeAll": false,
"label": null,
"multi": true,
"multiFormat": "glob",
"name": "compare",
"options": [],
"query": "aws.spot_price_region.$region.linux.$env.$zone.*",
"refresh": 1,
"refresh_on_load": false,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-2d",
"now": false,
"to": "now"
},
"timepicker": {
"collapse": false,
"enable": true,
"notice": false,
"now": true,
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"status": "Stable",
"time_options": [
"24h",
"2d",
"7d",
"14d",
"30d",
"60d",
"90d"
],
"type": "timepicker"
},
"timezone": "utc",
"title": "AWS Spot Prices Regions",
"version": 3
}
Raw
ec2py_pri_regions.py
#!/usr/bin/python
# curl 'http://a0.awsstatic.com/pricing/1/ec2/ri-v2/linux-unix-shared.min.js' 2>/dev/null | sed '1,5d; 6s#^callback(##; $s#);$##;' | jsonlint --format --nonstrict | json.sh -l -b | less
import urllib2
import json
import re
urls = ['http://a0.awsstatic.com/pricing/1/ec2/previous-generation/linux-od.min.js', 'http://a0.awsstatic.com/pricing/1/ec2/linux-od.min.js']
ri_url = 'http://a0.awsstatic.com/pricing/1/ec2/ri-v2/linux-unix-shared.min.js'
ru = urllib2.urlopen(ri_url).read()
ru = ru.split("(")[1].split(")")[0]
ru = re.sub(r"(\w*):", r'"\1":', ru)
rnj = json.loads(ru)
ri_url = 'http://a0.awsstatic.com/pricing/1/ec2/previous-generation/ri-v2/linux-unix-shared.min.js'
ru = urllib2.urlopen(ri_url).read()
ru = ru.split("(")[1].split(")")[0]
ru = re.sub(r"(\w*):", r'"\1":', ru)
roj = json.loads(ru)
print "region","gen","name","os","cost","stor_type","stor_count","stor_size","ecu","vcpu","mem","ri1upno_up","ri1upno_h","ri1part_up","ri1part_h","ri1all_up","ri1all_h","ri3part_up","ri3part_h","ri3all_up","ri3all_h"
gen = 1
for u in urls:
u = urllib2.urlopen(u).read()
u = u.split("(")[1].split(")")[0]
u = re.sub(r"(\w*):", r'"\1":', u)
j = json.loads(u)
#print json.dumps(j, indent=1)
for reg in j["config"]["regions"]:
## # limit region
## if reg["region"] != "us-east-1":
## continue
for fam in reg["instanceTypes"]:
for ins in fam["sizes"]:
# search for ri
ri1upno_up = "NA"
ri1upno_h = "NA"
ri1part_up = "NA"
ri1part_h = "NA"
ri1all_up = "NA"
ri1all_h = "NA"
ri3part_up = "NA"
ri3part_h = "NA"
ri3all_up = "NA"
ri3all_h = "NA"
for ri_j in [rnj, roj]:
for rreg in ri_j["config"]["regions"]:
if rreg["region"] == reg["region"]:
#print rreg["region"]
for rname in rreg["instanceTypes"]:
if rname["type"] == ins["size"]:
#print rname["type"]
#ricost = rname["terms"][0]["onDemandHourly"][0]["prices"]["USD"]
for term in rname["terms"]:
if term["term"] == "yrTerm1":
for pay in term["purchaseOptions"]:
if pay["purchaseOption"] == "noUpfront":
for xxx in pay["valueColumns"]:
if xxx["name"] == "upfront":
ri1upno_up = xxx["prices"]["USD"]
if xxx["name"] == "effectiveHourly":
ri1upno_h = xxx["prices"]["USD"]
if pay["purchaseOption"] == "partialUpfront":
for xxx in pay["valueColumns"]:
if xxx["name"] == "upfront":
ri1part_up = xxx["prices"]["USD"]
if xxx["name"] == "effectiveHourly":
ri1part_h = xxx["prices"]["USD"]
if pay["purchaseOption"] == "allUpfront":
for xxx in pay["valueColumns"]:
if xxx["name"] == "upfront":
ri1all_up = xxx["prices"]["USD"]
if xxx["name"] == "effectiveHourly":
ri1all_h = xxx["prices"]["USD"]
if term["term"] == "yrTerm3":
for pay in term["purchaseOptions"]:
if pay["purchaseOption"] == "partialUpfront":
for xxx in pay["valueColumns"]:
if xxx["name"] == "upfront":
ri3part_up = xxx["prices"]["USD"]
if xxx["name"] == "effectiveHourly":
ri3part_h = xxx["prices"]["USD"]
if pay["purchaseOption"] == "allUpfront":
for xxx in pay["valueColumns"]:
if xxx["name"] == "upfront":
ri3all_up = xxx["prices"]["USD"]
if xxx["name"] == "effectiveHourly":
ri3all_h = xxx["prices"]["USD"]
region = reg["region"]
family = fam["type"]
ecu = ins["ECU"]
if re.compile("variable").match(ecu):
ecu = "var"
mem = ins["memoryGiB"]
name = ins["size"]
storage = ins["storageGB"]
re_ssd = re.compile("SSD")
if re.compile(".*x.*SSD.*").match(storage):
stor_type = "SSD"
stor_count = storage.split(" ")[0]
stor_size = storage.split(" ")[2]
elif re.compile("ebsonly").match(storage):
stor_type = "EBS"
stor_count = "NA"
stor_size = "NA"
elif re.compile("60 SSD").match(storage):
stor_type = "SSD"
stor_count = 1
stor_size = 60
else:
stor_type = "LOC"
stor_count = storage.split(" ")[0]
stor_size = storage.split(" ")[2]
os = ins["valueColumns"][0]["name"]
cost = ins["valueColumns"][0]["prices"]["USD"]
vcpu = ins["vCPU"]
#print region,family,ecu,mem,name,storage,os,cost,vcpu
print region,gen,name,os,cost,stor_type,stor_count,stor_size,ecu,vcpu,mem,ri1upno_up,ri1upno_h,ri1part_up,ri1part_h,ri1all_up,ri1all_h,ri3part_up,ri3part_h,ri3all_up,ri3all_h
gen = gen + 1
Raw
graphite_spot_price.sh
#!/bin/bash
# requires:
# pip install awscli
# apt-get install netcat-traditional
# /usr/local/bin/ec2py_pri_regions.py
# crontab line:
# 22 02 * * * /usr/local/bin/graphite_spot_price.sh
host=10.167.158.150
port=2003
netcat=$(which nc)
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
log_base="/var/log/graphite_spot_price"
log_hostname=`hostname`
log_out(){
echo -e "`date +'%d/%m %H:%M:%S'` ($log_hostname) (pid $$) $log_ident -- $1" | tee -a ${log_base}.log
}
log_pipe() {
[ ! -z "$1" ] && p="$1: " || p=""
while read data; do
echo -e "`date +'%d/%m %H:%M:%S'` ($log_hostname) (pid $$) $log_ident -- ${p}$data" | tee -a ${log_base}.log
done
}
od_prices=`ec2py_pri_regions.py`
prefix='aws.spot_price_region'
inerval=0.01
# for history data feed fro specific date you can use dayz like this:
# dayz='Thu Mar 28 17:05:52 UTC 2016'
dayz=`date -d "-2 day"`
doday=$((`date -d "$dayz" +%s`/(24*60*60)*(24*60*60)))
while [[ $doday < `date +%s` ]]; do
log_out "starting new scrape for '`date -d @$doday`'"
s=`date +%s`
# you can limit regions hear
for R in `aws ec2 describe-regions | awk '{print $3}'`; do
log_out " scraping region $R"
rs=`date +%s`
for C in "Linux/UNIX (Amazon VPC)" "Linux/UNIX"; do
aws --region $R ec2 describe-spot-price-history\
--start-time `date +"%Y-%m-%dT%H:%M:%S.000Z" -d @$doday`\
--end-time `date +"%Y-%m-%dT%H:%M:%S.000Z" -d @$(($doday+86400))`\
--product-descriptions "$C"\
--output text |\
sed 's#[()]##g; s#[ /]#_#g' |\
sed "s#Linux_UNIX_Amazon_VPC#linux.vpc#; s#Linux_UNIX[^_]#linux.classic #;" |\
while read A z t d p T; do
echo "${prefix}.${R}.${d}.${z}.${t//./_}" $p `date -d "$T" +%s`
sleep $inerval
done | $netcat -q0 $host $port
done
# on-demand
for i in `seq 0 23`; do
echo "$od_prices" | egrep "^\<$R\>" | awk '{print $3" "$5}' |\
while read t p; do
T=`date +%s -d @$(($doday+($i*60*60)))`
echo "${prefix}.${R}.linux.general.ondemand.${t/./_}" $p $T | $netcat -q0 $host $port
sleep $inerval
done
done
sleep 0.1
log_out " finished region $R (`date -d @$((\`date +%s\`-$rs)) | awk '{print $3-1"d " $4}'`)"
done
log_out "finished scrape for '`date -d @$doday`' (`date -d @$((\`date +%s\`-$s)) | awk '{print $3-1"d " $4}'`)"
doday=$(($doday+86400))
done
exit 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment