Parse EC2 spot prices; store in an elasticsearch DB as a timeseries dataset.

ec2_spot_prices.js

require('http').globalAgent.maxSockets = 10

setInterval(main, (5 * 60 * 1000) - 6) // rinse-and-repeat every 5 minutes
main() // kick start!

function main () {
  const req = require('request-promise')
  const co = require('co')
  const oO = require('eyes').inspector({maxLength: 99999})

  const urls = {
    spot: 'https://spot-price.s3.amazonaws.com/spot.js',
    spotblocks_linux: 'https://spot-price.s3.amazonaws.com/spotblocks-generic.js',
    spotblocks_windows: 'https://spot-price.s3.amazonaws.com/spotblocks-windows.js'
  }

  co(function * run () {
    const timestamp = Date.now()
    oO(timestamp)

    for (let group in urls) {
      oO(group, new Date())
      const url = urls[group]
      const data = yield getData(url)
      const prices = parseData(group, timestamp, data)
      yield postPrices(prices)
    }

    oO('COMPLETED!', ((Date.now() - timestamp) / 1000) + 's')
  })

  function scrub (a) {
    return a.replace(/\W+/g, '_')
  }

  function * getData (url) {
    const resp = yield req(url)
    return JSON.parse(resp.substring(9, resp.length - 2))
  }

  function parseData (group, timestamp, resp) {
    const prices = []

    resp.config.regions.forEach((i1) => {
      const region = scrub(i1.region)

      i1.instanceTypes.forEach((i2) => {
        const type = scrub(i2.type)

        i2.sizes.forEach((i3) => {
          const size = scrub(i3.size)

          i3.valueColumns.forEach((i4) => {
            const price = +(i4.prices.USD)
            if (!price) return
            const name = scrub(i4.name)

            prices.push({ group, region, type, size, price_usd: price, name, timestamp })
          })
        })
      })
    })

    return prices
  }

  function * postPrice (body) {
    try {
      yield req.post({uri: 'http://192.168.100.1:9200/aws/ec2_spot/', body, json: true, timeout: 60000})
    } catch (err) {
      oO(err)
      return
    }
  }

  function * postPrices (prices) {
    let i = 0
    for (let body of prices) {
      if (++i % 40 === 0) {
        yield postPrice(body) // pause
      } else {
        postPrice(body).next() // fire and forget
      }
    }
  }
}

mappings.json

{
  "mappings": {
    "ec2_spot": {
      "properties": {
        "group": {
          "type": "string"
        },
        "name": {
          "type": "string"
        },
        "price_usd": {
          "type": "double"
        },
        "region": {
          "type": "string"
        },
        "size": {
          "type": "string"
        },
        "timestamp": {
          "type": "date",
          "format": "epoch_millis"
        },
        "type": {
          "type": "string"
        }
      }
    }
  }
}

I think/hope I got all the memory leaks ironed out :)