Skip to content

Instantly share code, notes, and snippets.

@jsignell

jsignell/Web scrape to NetCDF.ipynb

Last active July 17, 2016 14:16
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save jsignell/86c7507d8b2c7591f911c6207525e886 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save jsignell/86c7507d8b2c7591f911c6207525e886 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
Web scrape to NetCDF.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import os\n",
"import numpy as np\n",
"import pandas as pd\n",
"import xarray as xr\n",
"from mechanize import Browser\n",
"\n",
"out_path = '~/erddapData/MaricopaCounty/'\n",
"tr = [y+'-01-01' for y in [str(y) for y in range(1980, 2017)]]\n",
"tr.append(str(pd.datetime.utcnow().date()))\n",
"\n",
"meta = pd.read_csv(\"~/MaricopaCounty/sensor_ids.csv\")\n",
"meta['DEV_ID'] = meta['DEV_ID'].astype(int).astype(str)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"array(['Rptr. Batt.', 'Precip.', 'Stream PT', 'Non Subm PT', 'Dewpoint',\n",
" 'Humidity', 'Temperature', 'Bubbler', 'Peak Wind', 'Wind Spd/Dir',\n",
" 'Wind Dir.', 'Solar Rad.', 'Pressure', 'Status', 'Flasher',\n",
" 'Ave. Wind', 'Radar', 'Water Temp.'], dtype=object)"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"types = meta.DEV_TYPE.unique()\n",
"types"
]
},
{
"cell_type": "code",
"execution_count": 99,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>DEV_ID</th>\n",
" <th>DEV_NAME</th>\n",
" <th>DEV_TYPE</th>\n",
" <th>DEV_DATE</th>\n",
" <th>STA_LAT_DMS</th>\n",
" <th>STA_LONG_DMS</th>\n",
" <th>STA_ELEV</th>\n",
" <th>STA_LOC</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>358</td>\n",
" <td>Mt. Union Repeater</td>\n",
" <td>Rptr. Batt.</td>\n",
" <td>4/28/1992 0:00:00</td>\n",
" <td>34 24 54.3</td>\n",
" <td>112 24 16.9</td>\n",
" <td>7980.0</td>\n",
" <td>Top of Mt. Union, 10 mi. SSE of Prescott</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>754</td>\n",
" <td>Sacaton Peak</td>\n",
" <td>Rptr. Batt.</td>\n",
" <td>6/11/1996 0:00:00</td>\n",
" <td>33 00 09.5</td>\n",
" <td>111 40 26.8</td>\n",
" <td>2720.0</td>\n",
" <td>4.5 miles E. of the SR387 / I-10 junction</td>\n",
" </tr>\n",
" <tr>\n",
" <th>113</th>\n",
" <td>4757</td>\n",
" <td>Salt/67th Flow</td>\n",
" <td>Status</td>\n",
" <td>7/14/2008 0:00:00</td>\n",
" <td>33 23 51.8</td>\n",
" <td>112 12 12.6</td>\n",
" <td>975.0</td>\n",
" <td>Salt River at 67th Avenue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>142</th>\n",
" <td>4807</td>\n",
" <td>ACDC @ Dreamy Draw</td>\n",
" <td>Status</td>\n",
" <td>11/4/2009 0:00:00</td>\n",
" <td>33 32 50.6</td>\n",
" <td>112 03 28.5</td>\n",
" <td>1235.0</td>\n",
" <td>Arizona Canal near Northern Ave.</td>\n",
" </tr>\n",
" <tr>\n",
" <th>145</th>\n",
" <td>4812</td>\n",
" <td>ACDC @ 16th St.</td>\n",
" <td>Status</td>\n",
" <td>11/4/2009 0:00:00</td>\n",
" <td>33 32 31.5</td>\n",
" <td>112 02 54.5</td>\n",
" <td>1240.0</td>\n",
" <td>ACDC @ 16th St.</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" DEV_ID DEV_NAME DEV_TYPE DEV_DATE STA_LAT_DMS \\\n",
"0 358 Mt. Union Repeater Rptr. Batt. 4/28/1992 0:00:00 34 24 54.3 \n",
"1 754 Sacaton Peak Rptr. Batt. 6/11/1996 0:00:00 33 00 09.5 \n",
"113 4757 Salt/67th Flow Status 7/14/2008 0:00:00 33 23 51.8 \n",
"142 4807 ACDC @ Dreamy Draw Status 11/4/2009 0:00:00 33 32 50.6 \n",
"145 4812 ACDC @ 16th St. Status 11/4/2009 0:00:00 33 32 31.5 \n",
"\n",
" STA_LONG_DMS STA_ELEV STA_LOC \n",
"0 112 24 16.9 7980.0 Top of Mt. Union, 10 mi. SSE of Prescott \n",
"1 111 40 26.8 2720.0 4.5 miles E. of the SR387 / I-10 junction \n",
"113 112 12 12.6 975.0 Salt River at 67th Avenue \n",
"142 112 03 28.5 1235.0 Arizona Canal near Northern Ave. \n",
"145 112 02 54.5 1240.0 ACDC @ 16th St. "
]
},
"execution_count": 99,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"id1s = [f[0:-3] for f in os.listdir(out_path)]\n",
"unprocessed = [s for s in meta.index if meta.loc[s, 'DEV_ID'] not in id1s]\n",
"meta.iloc[unprocessed].head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"for j in unprocessed:\n",
" name = meta.loc[j, 'DEV_TYPE']\n",
" id1 = meta.loc[j, 'DEV_ID']\n",
"\n",
" L = []\n",
" for i in range(len(tr)-1):\n",
" L0 = fill_form(id1, tr[i], tr[i+1])\n",
" while len(L0) == 0:\n",
" L0 = fill_form(id1, tr[i], tr[i+1])\n",
" if len(L0) > 16:\n",
" if len(L) == 0:\n",
" L = L0[0:-2]\n",
" else:\n",
" L.extend(L0[14:-2])\n",
"\n",
" units, s = get_data(L, name)\n",
"\n",
" if (name=='Temperature' or name=='Pressure' or name=='Water Temp.') and s.shape[1] > 1:\n",
" s = s.iloc[:,0].to_frame()\n",
" units = [units[0]]\n",
" elif name == 'Precip.' and s.shape[1] > 1:\n",
" s = s.iloc[:,1].to_frame()\n",
" units = [units[1]]\n",
"\n",
" # get lat to a float\n",
" lat = to_decimal(*[float(lat) for lat in meta.loc[j,'STA_LAT_DMS'].split()])\n",
"\n",
" # get lon to a float and we know it is meant to be in the US so make it negative\n",
" lon = -to_decimal(*[float(lon) for lon in meta.loc[j,'STA_LONG_DMS'].split()])\n",
"\n",
" # get elevation as a float\n",
" elev = float(meta.loc[j,'STA_ELEV'])\n",
"\n",
" ds = to_ds(s, lat, lon, elev, units, name)\n",
" ds.attrs.update(meta.loc[j].to_dict())\n",
" ds.to_netcdf('{out_path}{id1}.nc'.format(out_path=out_path, id1=id1), format='netCDF4', engine='netcdf4')\n",
" print id1"
]
},
{
"cell_type": "code",
"execution_count": 47,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def fill_form(id1, start, end):\n",
" br = Browser()\n",
" br.open(\"http://alert.fcd.maricopa.gov/showrpts_mc.html\")\n",
"\n",
" for i, f in enumerate(br.forms()):\n",
" if i==0:\n",
" continue\n",
" else:\n",
" br.form =f\n",
"\n",
" br.form.set_value(id1, name=\"ID1\", type=\"text\") \n",
"\n",
" #set start time and date\n",
" br.form.set_value([start[5:7]], name=\"ms\")\n",
" br.form.set_value([start[8:10]], name=\"ds\")\n",
" br.form.set_value([start[0:4]], name=\"ys\")\n",
" br.form.set_value([\"00:00:00\"], name=\"hs\")\n",
"\n",
" #set end time and date\n",
" br.form.set_value([end[5:7]], name=\"ME\")\n",
" br.form.set_value([end[8:10]], name=\"DE\")\n",
" br.form.set_value([end[0:4]], name=\"YE\")\n",
" br.form.set_value([\"24:00:00\"], name=\"HE\")\n",
" response = br.submit() # submit current form\n",
" L = [l for l in response.readlines()]\n",
" return(L)\n",
"\n",
"def get_data(L, name):\n",
" s = pd.DataFrame([l.split() for l in L[14:-2]])\n",
" cols = [l.strip() for l in L[13].split(' ') if l.strip()]\n",
" cols.extend(range(s.shape[1]-len(cols)))\n",
" s.columns = cols\n",
" \n",
" # data are in mountain time with no daylight savings so convert to UTC\n",
" dt_index = pd.DatetimeIndex(s['Date']+' '+s['Time'])+pd.DateOffset(hours=7)\n",
" s = s.set_index(dt_index).drop(['Date','Time'], axis=1)\n",
" s.index.name='time'\n",
" \n",
" # change the name of the data variable from units to the actual name\n",
" units = list(s.columns)\n",
" \n",
" if name == 'Stream PT' or name=='Non Subm PT' or name=='Bubbler' or name=='Radar':\n",
" if len(units)==1 and units[0]=='feet':\n",
" s.columns = ['stage']\n",
" elif units[0]=='feet':\n",
" s.columns = ['stage', 'discharge']\n",
" units[1]='cfs'\n",
" \n",
" else:\n",
" name = L[7].strip()\n",
" s.columns = [name]*len(s.columns)\n",
" \n",
" # convert to float\n",
" s = s.astype('float')\n",
" s = s.sort_index()\n",
" return(units, s)\n",
" \n",
"def to_decimal(degree, minute, second):\n",
" return(degree+(minute/60.)+(second/3600.))\n",
"\n",
"def to_ds(s, lat, lon, elev, units, name):\n",
" \n",
" ds = s.to_xarray()\n",
" ds['lat'] = lat\n",
" ds['lon'] = lon\n",
" ds['elev'] = elev\n",
" ds.set_coords(['time','lat','lon', 'elev'], inplace=True)\n",
" if s.shape[0] < 20:\n",
" chunk=1\n",
" else:\n",
" chunk = s.shape[0]/20\n",
" \n",
" if name == 'Precip.':\n",
" attrs = [{'standard_name': 'rainfall'}]\n",
" elif name == 'Temperature':\n",
" attrs = [{'standard_name': 'air_temperature'}]\n",
" elif name == 'Dewpoint':\n",
" attrs = [{'standard_name': 'dew_point_temperature'}]\n",
" elif name == 'Stream PT'or name == 'Non Subm PT' or name=='Bubbler' or name=='Radar':\n",
" attrs = [{'standard_name': 'water_elevation'}, \n",
" {'standard_name':'discharge'}]\n",
" elif name == 'Humidity':\n",
" attrs = [{'standard_name': 'relative_humidity'}]\n",
" elif (name == 'Ave. Wind' or name == 'Wind Spd/Dir') and units[0] == 'miles per hour':\n",
" attrs = [{'standard_name': 'wind_speed'}]\n",
" elif name == 'Peak Wind':\n",
" attrs = [{'standard_name': 'max_wind_speed'}]\n",
" elif name == 'Wind Dir.':\n",
" attrs = [{'standard_name': 'wind_from_direction'}]\n",
" elif name == 'Solar Rad.':\n",
" attrs = [{'standard_name': 'solar_radiation'}]\n",
" elif name == 'Pressure':\n",
" attrs = [{'standard_name': 'air_pressure'}]\n",
" elif name == 'Water Temp.':\n",
" attrs = [{'standard_name': 'water_temperature'}]\n",
" \n",
" for i, u in enumerate(units):\n",
" if u == 'degrees F' or u == 'degreesF':\n",
" attrs[i].update({'units': 'degF'})\n",
" elif u == 'inches':\n",
" attrs[i].update({'units': 'inch'})\n",
" else:\n",
" attrs[i].update({'units': u})\n",
" \n",
" for i, v in enumerate(ds.data_vars.keys()):\n",
" ds[v].attrs.update(attrs[i])\n",
" ds[v].encoding.update({'dtype': np.double,'chunksizes':(chunk,),'zlib': True})\n",
" ds.lat.attrs.update({'units': 'degrees_north',\n",
" 'axis': 'Y',\n",
" 'long_name': 'latitude',\n",
" 'standard_name': 'latitude'})\n",
" ds.lat.encoding.update({'dtype': np.double,'chunksizes':(chunk,),'zlib': True})\n",
" ds.lon.attrs.update({'units': 'degrees_east',\n",
" 'axis': 'X',\n",
" 'long_name': 'longitude',\n",
" 'standard_name': 'longitude'})\n",
" ds.lon.encoding.update({'dtype': np.double,'chunksizes':(chunk,),'zlib': True})\n",
"\n",
" ds.elev.attrs.update({'units': 'feet',\n",
" 'axis': 'Z',\n",
" 'long_name': 'elevation',\n",
" 'standard_name': 'elevation'})\n",
" ds.elev.encoding.update({'dtype': np.double,'chunksizes':(chunk,),'zlib': True})\n",
" ds.time.encoding.update({'units':'seconds since 1970-01-01', \n",
" 'calendar':'gregorian',\n",
" 'dtype': np.double,'chunksizes':(chunk,),'zlib': True})\n",
"\n",
" ds.attrs.update({ 'institution': 'Data from Flood Control District of Maricopa County, hosted by Princeton University',\n",
" 'references': 'http://alert.fcd.maricopa.gov/showrpts_mc.html',\n",
" 'featureType': 'timeSeries',\n",
" 'Conventions': 'CF-1.6',\n",
" 'history': 'Created by Princeton University Hydrometeorology Group at {now} '.format(now=pd.datetime.now()),\n",
" 'author': 'jsignell@princeton.edu'})\n",
" return(ds)"
]
},
{
"cell_type": "code",
"execution_count": 85,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"['04/03/1994 22:04:42 107.3094628000.0\\n']"
]
},
"execution_count": 85,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"drop = [l for l in L if '107.3094628000.0' in l]\n",
"print drop\n",
"L.remove(drop[0])"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.11"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment