get all agency ideas from ideascale

#!/usr/bin/python

import json, urllib2

'''
retrieves all ideas from all categories for all agencies, and
normalizes category names from unique numerical ids to shared category
names so they can be compared across agencies. each idea in the list
contains the following keys:

data structure:

agency_ideas = {each_agency : {
                               'transparency': [idea_objects...], 
                               'participation' = [idea_objects...], 
                               'collaboration' : [idea_objects...], 
                               'innovation' = [idea_objects...] 
                               }

each idea_object has the following keys:

"voteCount", "author", "authorInfo" : {"voteCount", "url", "id":,
"name":, "ideaCount": }, "url", "text", "tags", "id", "commentCount",
"authorId", "timestamp", "title", "myVote", "categoryID"

note: the text of the comments is not included in what's returned by
the API.

'''

# API keys for each agency's ideascale site
agencies = {
    "usaid": "your key here...",
    "comm":"your key here...",
    "dod": "your key here...",
    "ed": "your key here...",
    "energy": "your key here...",
    "nasa":"your key here...",   
    'dot': "your key here...",
    "int": "your key here...",
    "va": "your key here...",
    "treas": "your key here...",
    "gsa": "your key here...",
    "opm": "your key here...",
    "labor": "your key here...",
    "doj": "your key here...",
    "ssa": "your key here...",
    "state": "your key here...",
    "nsf": "your key here...",
    "hud": "your key here...",
    "epa": "your key here...",
    "sba": "your key here...",
    "dhs": "your key here...",
    "nrc": "your key here...",
    "ostp": "your key here...",
}

# category ids for each category for each agency (the category IDs are
# unique for each agency, but the categories they map to are the
# same).
cat_id = {
    'usaid': {11832: 'transparency' , 11833: 'participation', 11834:'collaboration', 11835: 'innovation', 11836: 'site_feedback'}, 
    'comm': {11860: 'transparency', 11861: 'participation', 11862: 'collaboration', 11863: 'innovation', 11864: 'site_feedback'}, 
    'dod': {11865: 'transparency', 11866: 'participation', 11867: 'collaboration', 11868: 'innovation', 11869: 'site_feedback'}, 
    'ed': {11870: 'transparency', 11871: 'participation', 11872: 'collaboration', 11873: 'innovation', 11874: 'site_feedback'}, 
    'energy': {11808: 'transparency', 11809: 'participation', 11810: 'collaboration', 11811: 'innovation', 11812: 'site_feedback'}, 
    'nasa': {11571: 'transparency', 11572: 'participation', 11573: 'collaboration', 11928: 'innovation', 11929: 'site_feedback'}, 
    'dot': {11908: 'transparency', 11909: 'participation', 11910: 'collaboration', 11911: 'innovation', 11912: 'site_feedback'}, 
    'int': {11883: 'transparency', 11884: 'participation', 11885: 'collaboration', 11886: 'innovation', 11887: 'site_feedback'}, 
    'va': {11918: 'transparency', 11919: 'participation', 11920: 'collaboration', 11921: 'innovation', 11922: 'site_feedback'}, 
    'treas': {11913: 'transparency', 11914: 'participation', 11915: 'collaboration', 11916: 'innovation', 11917: 'site_feedback'}, 
    'gsa': {11827: 'transparency', 11828: 'participation', 11829: 'collaboration', 11830: 'innovation', 11831: 'site_feedback'}, 
    'opm': {11580: 'transparency', 11581: 'participation', 11582: 'collaboration', 11937: 'innovation', 11938: 'site_feedback'}, 
    'labor': {11893: 'transparency', 11894: 'participation', 11895: 'collaboration', 11896: 'innovation', 11897: 'site_feedback'}, 
    'doj': {11888: 'transparency', 11889: 'participation', 11890: 'collaboration', 11891: 'innovation', 11892: 'site_feedback'}, 
    'ssa': {11653: 'transparency', 11654: 'participation', 11655: 'collaboration', 11941: 'innovation', 11942: 'site_feedback'}, 
    'state': {11903: 'transparency', 11904: 'participation', 11905: 'collaboration', 11906: 'innovation', 11907: 'site_feedback'}, 
    'nsf': {11577: 'transparency', 11578: 'participation', 11579: 'collaboration', 11930: 'innovation', 11931: 'site_feedback'}, 
    'hud': {11878: 'transparency', 11879: 'participation', 11880: 'collaboration', 11881: 'innovation', 11882: 'site_feedback'}, 
    'epa': {11813: 'transparency', 11814: 'participation', 11815: 'collaboration', 11816: 'innovation', 11817: 'site_feedback'}, 
    'sba': {11650: 'transparency', 11651: 'participation', 11652: 'collaboration', 11939: 'innovation', 11940: 'site_feedback'}, 
    'dhs': {11923: 'transparency', 11924: 'participation', 11925: 'collaboration', 11926: 'innovation', 11927: 'site_feedback'}, 
    'nrc': {11583: 'transparency', 11584: 'participation', 11585: 'collaboration', 11935: 'innovation', 11936: 'site_feedback'}, 
    'ostp': {12295: 'transparency', 12296: 'participation', 12297: 'collaboration', 12299: 'innovation', 122300: 'site_feedback'}, 
}

agency_ideas = {}
for agency in agencies.keys():
    key = agencies[agency]
    api_base_url = "http://api.ideascale.com/akira/api/ideascale.ideas.getRecentIdeas"
    
    ideas = {}
    for category_id, category_name in cat_id[agency].iteritems():
        arguments = "?categoryID=%s&apiKey=%s" % (category_id, key)
        api_call = api_base_url+arguments
        url = urllib2.urlopen(api_call)
        js = json.loads(url.read())
        ideas[category_name] = js['response']['ideas']
    agency_ideas[agency] = ideas

save = open('agency_data.json', 'w')
json.dump(agency_ideas, save)
save.close()

gistfile2.txt