Created
December 5, 2012 05:44
-
-
Save cynici/4212722 to your computer and use it in GitHub Desktop.
Get weather data from Wunderground stations listed in input JSON and output to a CSV file
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # -*- coding: utf-8 -*- | |
| import os, sys | |
| from optparse import OptionParser | |
| import string | |
| import urllib2 | |
| import simplejson | |
| import logging | |
| from datetime import datetime | |
| import time | |
| help_text = """%prog [options] JSON_file ... | |
| Get weather data from Wunderground stations listed in input JSON file(s) | |
| and output in a CSV file. Details: http://www.wunderground.com/weather/api/ | |
| JSON format: | |
| [{ | |
| "c": "AO", | |
| "l": "/q/zmw:00000.1.66116", | |
| "name": "AO Noqui Angola", | |
| "type": "city", | |
| "tz": "Africa/Luanda", | |
| "tzs": "WAT", | |
| "zmw": "00000.1.66116" | |
| }] | |
| CSV format: | |
| Station_id,Longitude,Latitude,Name,yyyy-mm-dd HH:MM:SS,temp_c,rh_pct,w_deg,w_ms | |
| W_66160,13.224200,-8.797928,Angola Luanda,2012-07-31 06:06:56,19.8,-999,0,1.03""" | |
| geolookup_url = """http://api.wunderground.com/api/b80d370b52a348da/geolookup/conditions%(l)s.json""" | |
| def format_line(w): | |
| '''Format query response data from dictionary to a output line''' | |
| d = {} | |
| l = w["location"] | |
| o = w["current_observation"] | |
| if "station_id" in o and o["station_id"]: | |
| wmo = o["station_id"] | |
| if "wmo" and l["wmo"]: | |
| wmo = l["wmo"] | |
| if "country" in l and l["country"]: | |
| cc = l['country'] | |
| if "country_iso3166" in l and l["country_iso3166"]: | |
| cc = l["country_iso3166"] | |
| if "city" in l and l["city"]: | |
| city = l["city"] | |
| if "lat" in l and l["lat"]: | |
| d["lat"] = l["lat"] | |
| if "lon" in l and l["lon"]: | |
| d["lon"] = l["lon"] | |
| for pkey in ["display_location", "observation_location"]: | |
| if pkey not in o: | |
| continue | |
| pdict = o[pkey] | |
| if "latitude" in pdict and pdict["latitude"]: | |
| d["lat"] = pdict["latitude"] | |
| if "longitude" in pdict and pdict["longitude"]: | |
| d["lon"] = pdict["longitude"] | |
| if "country" in pdict and pdict["country"]: | |
| cc = pdict["country"] | |
| if "country_iso3166" in pdict and pdict["country_iso3166"]: | |
| cc = pdict["country_iso3166"] | |
| if "city" in pdict and pdict["city"]: | |
| city = pdict["city"] | |
| if "lat" not in d or "lon" not in d or not cc or not city: | |
| raise ValueError("Missing key data: lat, lon, country or city.") | |
| d["id"] = "W_"+wmo | |
| d["name"] = ("%s %s"%(cc, city)).replace(',','') | |
| if "observation_epoch" in o and o["observation_epoch"]: | |
| key = "observation_epoch" | |
| else: | |
| if "local_epoch" in o and o["local_epoch"]: | |
| key = "local_epoch" | |
| else: | |
| raise ValueError("No timestamp") | |
| dt = datetime.utcfromtimestamp(int(o[key])) | |
| d["datestr"] = dt.strftime("%Y-%m-%d %H:00:00") | |
| if "temp_c" in o: | |
| d["temp_c"] = str(o["temp_c"]) | |
| else: | |
| d["temp_c"] = "-999" | |
| if "relative_humidity" in o and o["relative_humidity"].upper() != "N/A": | |
| d["rh_pct"] = o["relative_humidity"].replace('%','') | |
| else: | |
| d["rh_pct"] = "-999" | |
| if "wind_degrees" in o: | |
| d["w_deg"] = str(o["wind_degrees"]) | |
| else: | |
| d["w_deg"] = "-999" | |
| if "wind_kph" in o: | |
| d["w_ms"] = str(o["wind_kph"]/3.6) | |
| else: | |
| d["w_ms"] = "-999" | |
| return "%(id)s,%(lon)s,%(lat)s,%(name)s,%(datestr)s,%(temp_c)s,%(rh_pct)s,%(w_deg)s,%(w_ms)s" % d | |
| def main(argv=None): | |
| if argv is None: | |
| argv = sys.argv | |
| debuglevelD = { | |
| 'debug': logging.DEBUG, | |
| 'info': logging.INFO, | |
| 'warning': logging.WARNING, | |
| 'error': logging.ERROR, | |
| 'critical': logging.CRITICAL, | |
| } | |
| scriptdir = os.path.dirname(__file__) | |
| scriptname = os.path.basename(__file__) | |
| scriptnoext, scriptext = os.path.splitext(__file__) | |
| defvals = { | |
| 'output': datetime.utcnow().strftime('$HOME/afisdata/non-ftp/w_wunderground/wg-%%Y%%m%%d-%%H%%M-%s.txt'%os.getpid()), | |
| 'interval': 20, # seconds, free API limit 10 calls per minute | |
| } | |
| parser = OptionParser(help_text) | |
| parser.add_option("-o", "--output", dest="output", type="string", help="Output specification (%s)"%defvals['output']) | |
| parser.add_option("--interval", dest="interval", type="int", help="Interval between API calls (%s seconds)"%defvals['interval'], metavar='SECS') | |
| parser.add_option("-l", "--loglevel", dest="loglevel", type="string", help="Verbosity %s"%debuglevelD.keys(), metavar='LOGLEVEL') | |
| parser.set_defaults(**defvals) | |
| (options, args) = parser.parse_args() | |
| if options.loglevel: | |
| if options.loglevel not in debuglevelD: raise AssertionError("Verbosity level must be one of: %s"%debuglevelD.keys()) | |
| dbglvl = debuglevelD[options.loglevel] | |
| else: | |
| dbglvl = logging.WARNING | |
| logger = logging.getLogger() | |
| logger.setLevel(dbglvl) | |
| ch = logging.StreamHandler() | |
| ch.setFormatter( logging.Formatter('%(asctime)s %(lineno)d %(name)s %(funcName)s - %(levelname)s - %(message)s') ) | |
| ch.setLevel(dbglvl) | |
| logger.addHandler(ch) | |
| jsonfiles = [] | |
| for f in args: | |
| if os.path.exists(f): | |
| jsonfiles.append(f) | |
| else: | |
| if os.path.exists(os.path.join(scriptdir, f)): | |
| jsonfiles.append(os.path.join(scriptdir, f)) | |
| else: | |
| parser.error("Input JSON file not found: %s" % f) | |
| if not jsonfiles: | |
| parser.error("Requires at least one input JSON file.") | |
| ofh = None | |
| linecount = 0 | |
| for f in jsonfiles: | |
| station_list = simplejson.loads(open(f).read().decode("ISO-8859-1")) | |
| logger.debug("Found %s stations in %s" % (len(station_list), f)) | |
| for d in station_list: | |
| try: | |
| if '/q/' not in d['l']: | |
| # Some locations are like "/global/ZB.html" which are invalid | |
| logger.debug("Skipped %s %s" % (f, d['l'])) | |
| continue | |
| if linecount and options.interval > 0: | |
| logger.debug("Going to sleep ...") | |
| time.sleep(options.interval) | |
| url = geolookup_url % dict(l=d['l']) | |
| logger.debug("GET %s ..."%(url)) | |
| weather = simplejson.loads(urllib2.urlopen(url).read().decode("ISO-8859-1")) | |
| line = format_line(weather) | |
| if "-999" in line or ",," in line: | |
| logger.warning("Skipped %s %s : %s" % (f, url, line)) | |
| continue | |
| except Exception, err: | |
| logger.warning("Failed %s %s : %s"%(f, url, err)) | |
| continue | |
| if ofh is None: | |
| ofh = open(os.path.expandvars(options.output),"w+") | |
| ofh.write(line+'\n') | |
| ofh.flush() | |
| linecount += 1 | |
| if ofh: ofh.close() | |
| logger.info("Wrote %s lines to %s" % (linecount, options.output)) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
| """ | |
| http://autocomplete.wunderground.com/aq?query=A&c=CL | |
| { "RESULTS": [ { "name": "Antofagasta, Chile", "type": "city", "c": "CL", "zmw": "00000.1.85442", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.1.85442" }, { "name": "Arica, Chile", "type": "city", "c": "CL", "zmw": "00000.1.85406", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.1.85406" }, { "name": "Angol, Chile", "type": "city", "c": "CL", "zmw": "00000.5.85703", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.5.85703" }, { "name": "Ancud, Chile", "type": "city", "c": "CL", "zmw": "00000.10.85799", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.10.85799" }, { "name": "Arauco, Chile", "type": "city", "c": "CL", "zmw": "00000.6.85682", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.6.85682" }, { "name": "Almirante Schroeders, Chile", "type": "city", "c": "CL", "zmw": "00000.7.85934", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.7.85934" }, { "name": "Alto Palena, Chile", "type": "city", "c": "CL", "zmw": "00000.1.85836", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.1.85836" }, { "name": "ANF, Chile", "type": "city", "c": "CL", "zmw": "00000.5.85442", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.5.85442" }, { "name": "ARI, Chile", "type": "city", "c": "CL", "zmw": "00000.5.85406", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.5.85406" }, { "name": "ARR, Chile", "type": "city", "c": "CL", "zmw": "00000.5.85864", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.5.85864" }, { "name": "Arturo Merino Benitez International, Chile", "type": "city", "c": "CL", "zmw": "00000.7.85574", "tz": "America/Santiago", "tzs": "CLST", "l": "/q/zmw:00000.7.85574" } ] } | |
| http://api.wunderground.com/api/b80d370b52a348da/geolookup/conditions/q/zmw:00000.1.68335.json | |
| { | |
| "response": { | |
| "version": "0.1" | |
| ,"termsofService": "http://www.wunderground.com/weather/api/d/terms.html" | |
| ,"features": { | |
| "geolookup": 1 | |
| , | |
| "conditions": 1 | |
| } | |
| } | |
| , "location": { | |
| "type":"INTLCITY", | |
| "country":"ZA", | |
| "country_iso3166":"ZA", | |
| "country_name":"South Africa", | |
| "state":"", | |
| "city":"Taung", | |
| "tz_short":"SAST", | |
| "tz_long":"Africa/Johannesburg", | |
| "lat":"-27.54999924", | |
| "lon":"24.77000046", | |
| "zip":"00000", | |
| "magic":"1", | |
| "wmo":"68335", | |
| "l":"/q/zmw:00000.1.68335", | |
| "requesturl":"global/stations/68335.html", | |
| "wuiurl":"http://www.wunderground.com/global/stations/68335.html", | |
| "nearby_weather_stations": { | |
| "airport": { | |
| "station": [ | |
| { "city":"Taung", "state":"", "country":"ZA", "icao":"", "lat":"-27.54999924", "lon":"24.77000046" } | |
| ] | |
| } | |
| , | |
| "pws": { | |
| "station": [ | |
| ] | |
| } | |
| } | |
| } | |
| , "current_observation": { | |
| "image": { | |
| "url":"http://icons-ak.wxug.com/graphics/wu2/logo_130x80.png", | |
| "title":"Weather Underground", | |
| "link":"http://www.wunderground.com" | |
| }, | |
| "display_location": { | |
| "full":"Taung, South Africa", | |
| "city":"Taung", | |
| "state":"", | |
| "state_name":"South Africa", | |
| "country":"ZA", | |
| "country_iso3166":"ZA", | |
| "zip":"00000", | |
| "latitude":"-27.54999924", | |
| "longitude":"24.77000046", | |
| "elevation":"1100.00000000" | |
| }, | |
| "observation_location": { | |
| "full":"Taung, ", | |
| "city":"Taung", | |
| "state":"", | |
| "country":"ZA", | |
| "country_iso3166":"ZA", | |
| "latitude":"-27.54999924", | |
| "longitude":"24.77000046", | |
| "elevation":" ft" | |
| }, | |
| "estimated": { | |
| }, | |
| "station_id":"68335", | |
| "observation_time":"Last Updated on December 4, 8:00 AM SAST", | |
| "observation_time_rfc822":"Tue, 04 Dec 2012 08:00:00 +0200", | |
| "observation_epoch":"1354600800", | |
| "local_time_rfc822":"Tue, 04 Dec 2012 13:13:34 +0200", | |
| "local_epoch":"1354619614", | |
| "local_tz_short":"SAST", | |
| "local_tz_long":"Africa/Johannesburg", | |
| "local_tz_offset":"+0200", | |
| "weather":"", | |
| "temperature_string":"69 F (21 C)", | |
| "temp_f":69, | |
| "temp_c":21, | |
| "relative_humidity":"51%", | |
| "wind_string":"From the West at 2 MPH", | |
| "wind_dir":"West", | |
| "wind_degrees":260, | |
| "wind_mph":2, | |
| "wind_gust_mph":0, | |
| "wind_kph":4, | |
| "wind_gust_kph":0, | |
| "pressure_mb":"", | |
| "pressure_in":"", | |
| "pressure_trend":"+", | |
| "dewpoint_string":"55 F (13 C)", | |
| "dewpoint_f":55, | |
| "dewpoint_c":13, | |
| "heat_index_string":"NA", | |
| "heat_index_f":"NA", | |
| "heat_index_c":"NA", | |
| "windchill_string":"NA", | |
| "windchill_f":"NA", | |
| "windchill_c":"NA", | |
| "feelslike_string":"69 F (21 C)", | |
| "feelslike_f":"69", | |
| "feelslike_c":"21", | |
| "visibility_mi":"", | |
| "visibility_km":"", | |
| "solarradiation":"", | |
| "UV":"-1", | |
| "precip_1hr_string":" in ( mm)", | |
| "precip_1hr_in":"", | |
| "precip_1hr_metric":"", | |
| "precip_today_string":" in ( mm)", | |
| "precip_today_in":"", | |
| "precip_today_metric":"", | |
| "icon":"clear", | |
| "icon_url":"http://icons-ak.wxug.com/i/c/k/clear.gif", | |
| "forecast_url":"http://www.wunderground.com/global/stations/68335.html", | |
| "history_url":"http://www.wunderground.com/history/airport/68335/2012/12/4/DailyHistory.html", | |
| "ob_url":"http://www.wunderground.com/cgi-bin/findweather/getForecast?query=-27.54999924,24.77000046" | |
| } | |
| } | |
| """ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment