Last active
September 28, 2021 12:17
-
-
Save shokoe/167fc3235a449dccab1b5a6c637fee46 to your computer and use it in GitHub Desktop.
Export aws billing info to grafana. Uses AWS billing info from S3. Just activate logs in billing and change the 'f' var in the script.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin | |
netcat=$(which nc) | |
host='<carbon IP>' | |
port='2003' | |
nc_cmd="$netcat -q0 $host $port" | |
get_month(){ | |
f="<your id>-aws-billing-detailed-line-items-with-resources-and-tags-${1}" | |
cd /tmp | |
aws s3 cp s3://<billing bucket>/${f}.csv.zip /tmp/ | |
unzip /tmp/${f}.csv.zip | |
cd - | |
cat /tmp/${f}.csv |\ | |
sed 's#""#"EMPTY"#g; s#","#^#g; s#^"##; s#"$##;' |\ | |
gawk -F ^ ' | |
BEGIN{ | |
# the PT (product tag) hash defines one user tag per product to limit the main multi-dimention array | |
PT["Amazon Elastic Compute Cloud"] = "user:type" | |
PT["Amazon Simple Storage Service"] = "user:buckMain" | |
# examples for adding products without a dedicated tag | |
PT["Amazon CloudFront"] = "zamburama" | |
#PT["Amazon CloudFront"] = "" | |
} | |
NR==1 { | |
# this is for using column headers instead of numbers | |
for (i=1; i<=NF; i++) { | |
f[$i] = i | |
} | |
} | |
NR!=1{ | |
# default tag | |
TAG="user:type" | |
if ( PT[$f["ProductName"]] in f ) TAG=PT[$f["ProductName"]] | |
# this if limits metrics to PT array | |
if ( PT[$f["ProductName"]] ){ | |
# sort input a bit more | |
gsub(/ /, "_") | |
gsub(/[()]/, "") | |
# UsageType include instance type wich has periods | |
gsub(/\./, "-", $f["UsageType"]) | |
# get timestamp | |
gsub(/[^0-9]/, " ", $f["UsageStartDate"]) | |
D=mktime($f["UsageStartDate"]) | |
# main monster hash | |
clus[D][$f["ProductName"]][$f["UsageType"]][$f["Operation"]][$f[TAG]]+=$f["Cost"] | |
} | |
} | |
END{ | |
for (d in clus){ | |
for (p in clus[d]){ | |
for (u in clus[d][p]){ | |
for (o in clus[d][p][u]){ | |
for (t in clus[d][p][u][o]){ | |
printf "aws_billing.%s %.2f %d\n", p"."u"."o"."t, clus[d][p][u][o][t], d | |
# extract and report instances specific data | |
if ( u ~ /BoxUsage:/ || u ~ /^HeavyUsage:/ || u ~ /^SpotUsage:/ ){ | |
# extract instance types | |
split(u,z,":") | |
# and family | |
split(z[2],y,"-") | |
# report omdemand | |
if ( u ~ /BoxUsage:/ || u ~ /^HeavyUsage:/ ){ | |
printf "aws_billing_instances.%s %.2f %d\n", p"."u"."o"."y[1]"."z[2]".ondemand."t, clus[d][p][u][o][t], d | |
} | |
# report spot | |
if ( u ~ /^SpotUsage:/ ){ | |
split(u,z,":") | |
printf "aws_billing_instances.%s %.2f %d\n", p"."u"."o"."y[1]"."z[2]".spot."t, clus[d][p][u][o][t], d | |
} | |
} | |
} | |
} | |
} | |
} | |
} | |
} | |
' | $nc_cmd | |
rm -f /tmp/${f}.csv &>/dev/null | |
} | |
get_month `date -d "-1 hour" +%Y-%m` | |
exit | |
#for i in `seq 14`; do | |
# D=`date -d "-$i month" +%Y-%m` | |
# echo $D | |
# get_month $D | $nc_cmd | |
#done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment