farm_water_storage.py

import fiona
from shapely.geometry import shape
import glob
import pickle
import urllib2
import pandas as pd
from StringIO import StringIO
import os

def get_dams():
    water_bodies = fiona.open("/g/data/fj4/scratch/water_bodies/water_bodies_wgs84.shp")

    prop_dams = {}

    shps = glob.glob("/g/data/xc0/project/NSW_Water_Compliance/data/1_shp_in/wgs84_dissolve/*.shp")
    for shp in shps:
        polyf = fiona.open(shp)
        poly = shape(polyf[0]['geometry'])
        dams = []
        for water_body in water_bodies:
            wb = shape(water_body['geometry'])
            #if poly.intersects(wb):
            if poly.contains(wb):
                dams.append(water_body['properties']['FID'])
        prop_dams[shp] = dams

    with open('property_dams.pickle', 'wb') as handle:
        pickle.dump(prop_dams, handle, protocol=pickle.HIGHEST_PROTOCOL)

def aux():
    prop_dams = pickle.load(open("property_dams.pickle", "rb" ))
    print(prop_dams)

    prop_df = None
    for prop, dams in prop_dams.iteritems():
        prop = os.path.splitext(os.path.basename(prop))[0]
        print(prop)
       
        res = None
        for dam_id in dams:
            url = get_url(dam_id)
            print(url)
            response = urllib2.urlopen(url)
            out = StringIO()
            out.write(response.read())
            out.seek(0)
            df = process_csv(out)

            if res is None:
                res = df
            else:
                res = pd.concat([res, df], axis=1)

        if res is None:
            continue
        res.interpolate(inplace=True)
        res[prop] = res.iloc[:, :].sum(axis=1)

        if prop_df is None:
            prop_df = res[prop]
        else:
            prop_df = pd.concat([prop_df, res[prop]], axis=1)

    prop_df.interpolate(inplace=True)
    prop_df = prop_df.resample('M').mean()
    prop_df.interpolate(inplace=True)

    prop_df.to_csv("onsite_storage.csv", float_format='%.3f')

def get_url(uid):
    return "https://data.dea.ga.gov.au/projects/WaterBodies/feature_info/{:04d}/{:06d}.csv".format(uid//100, uid)

def process_csv(fname):
   df = pd.read_csv(fname,parse_dates=True,index_col=0)
   df = df.loc['2014-1-1':]
   
   return df.iloc[:, [1]] * 625

#get_dams()
aux()