Created
March 21, 2018 18:36
-
-
Save shokoe/02e4e4265b528f09ef6862382eb5de15 to your computer and use it in GitHub Desktop.
Export aws billing info to grafana. Uses AWS billing info from S3. Just activate logs in billing and change the 'f' var in the script.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin | |
netcat=$(which nc) | |
host='<carbon IP>' | |
port='2003' | |
nc_cmd="$netcat -q0 $host $port" | |
get_month(){ | |
f="<your id>-aws-billing-detailed-line-items-with-resources-and-tags-${1}" | |
cd /tmp | |
aws s3 cp s3://billingdata.sightera.com/${f}.csv.zip /tmp/ | |
unzip /tmp/${f}.csv.zip | |
cd - | |
cat /tmp/${f}.csv |\ | |
sed 's#""#"EMPTY"#g; s#","#^#g; s#^"##; s#"$##;' |\ | |
gawk -F ^ ' | |
BEGIN{ | |
# the PT (product tag) hash defines one user tag per product to limit the main multi-dimention array | |
PT["Amazon Elastic Compute Cloud"] = "user:type" | |
PT["Amazon Simple Storage Service"] = "user:buckMain" | |
# examples for adding products without a dedicated tag | |
PT["Amazon CloudFront"] = "zamburama" | |
#PT["Amazon CloudFront"] = "" | |
} | |
NR==1 { | |
# this is for using column headers instead of numbers | |
for (i=1; i<=NF; i++) { | |
f[$i] = i | |
} | |
} | |
NR!=1{ | |
# default tag | |
TAG="user:type" | |
if ( PT[$f["ProductName"]] in f ) TAG=PT[$f["ProductName"]] | |
# this if limits metrics to PT array | |
if ( PT[$f["ProductName"]] ){ | |
# sort input a bit more | |
gsub(/ /, "_") | |
gsub(/[()]/, "") | |
# UsageType include instance type wich has periods | |
gsub(/\./, "-", $f["UsageType"]) | |
# get timestamp | |
gsub(/[^0-9]/, " ", $f["UsageStartDate"]) | |
D=mktime($f["UsageStartDate"]) | |
# main monster hash | |
clus[D][$f["ProductName"]][$f["UsageType"]][$f["Operation"]][$f[TAG]]+=$f["Cost"] | |
} | |
} | |
END{ | |
for (d in clus){ | |
for (p in clus[d]){ | |
for (u in clus[d][p]){ | |
for (o in clus[d][p][u]){ | |
for (t in clus[d][p][u][o]){ | |
printf "aws_billing.%s %.2f %d\n", p"."u"."o"."t, clus[d][p][u][o][t], d | |
# extract and report instances specific data | |
if ( u ~ /BoxUsage:/ || u ~ /^HeavyUsage:/ || u ~ /^SpotUsage:/ ){ | |
# extract instance types | |
split(u,z,":") | |
# and family | |
split(z[2],y,"-") | |
# report omdemand | |
if ( u ~ /BoxUsage:/ || u ~ /^HeavyUsage:/ ){ | |
printf "aws_billing_instances.%s %.2f %d\n", p"."u"."o"."y[1]"."z[2]".ondemand."t, clus[d][p][u][o][t], d | |
} | |
# report spot | |
if ( u ~ /^SpotUsage:/ ){ | |
split(u,z,":") | |
printf "aws_billing_instances.%s %.2f %d\n", p"."u"."o"."y[1]"."z[2]".spot."t, clus[d][p][u][o][t], d | |
} | |
} | |
} | |
} | |
} | |
} | |
} | |
} | |
' | $nc_cmd | |
rm -f /tmp/${f}.csv &>/dev/null | |
} | |
get_month `date -d "-1 hour" +%Y-%m` | |
exit | |
#for i in `seq 14`; do | |
# D=`date -d "-$i month" +%Y-%m` | |
# echo $D | |
# get_month $D | $nc_cmd | |
#done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment