Ingests CSV data from http://api.bitcoincharts.com/v1/csv/ into a local InfluxDB server

process.sh

#!/usr/bin/env bash

curl -s -i -XPOST 'http://127.0.0.1:8086/query' --data-urlencode "q=CREATE DATABASE order_history" > /dev/null

for file in *.gz; do
  echo "extracting ${file}..."
  unpigz -k $file
  file2=${file::-3}
  pair=${file2::-4}
  exchange=${pair::-3}
  currency=${pair: -3}

  echo "formatting ${file2}..."
  sed -E -i'' "s/(.*),(.*),(.*)/rate,exchange=${exchange},currency=${currency} id=\"$(uuidgen)\",price=\2,volume=\3 \1000000000/g" ${file2}

  echo "splitting ${file2}..."
  split -a 5 -l 5000 ${file2} splat

  echo "ingesting for exchange ${exchange} and currency ${currency} into db ${db}..."
  for part in splat*; do
    printf "ingesting part %s...\r" ${part}
    curl -s -i -XPOST "http://127.0.0.1:8086/write?db=order_history" --data-binary @${part} > /dev/null
  done

  echo
  echo "cleaning up..."
  rm splat*
  rm ${file2}
done