export_data.py

# -*- coding: utf-8 -*-
import csv, json, codecs, sys
from HTMLParser import HTMLParser as html
import html2text

reload(sys)
sys.setdefaultencoding('utf-8')

html = html()
__ = html2text.HTML2Text()
__.ignore_links = True
__.ignore_emphasis = True
__.ignore_images = True
__.ignore_tables = True
__.body_width = 0

posts = codecs.open('posts_data.json', 'r', encoding='utf-8')
posts = html.unescape(''.join(posts.readline()))
posts = json.loads(posts)

# categories = open('categories.json', 'r')
# categories = json.load(categories)

# markers = open('markers.json', 'r')
# markers = json.load(markers)
html_posts = [642,665,538,1492,661,765,1395,1416,1921,1480,1924,895,490,1363,
           247,848,479,646,512,629,251,1436,654,650,658,784,632,469,636,466,
           764,885,897,670,830,1369]

# keys
keys = [
   "ID",
   "category",
   "post_title",
   "post_status",
   "post_content",
   "post_content_filtered",
   "post_author",
   "post_name",
   "post_date_gmt",
   "post_date",
   "post_modified_gmt",
   "post_modified",

   "image01",
   "image02",
   "image03",
   "image04",
   "image05",
   "image06",
   "image07",
   "image08",
   "image09",
   "image10",
   "image11",
   "image12",
   "image13",
   "image14",
   "image15",
   "image16",
   "image17",
   "image18",
   "image19",
   "image20",
   "image21",
   "image22",
   "image23",
   "image24",
   "image25",
   "image26",

   "marker_id",
   "user_city",
   "_thumbnail_id",
   # "_mp_geojson",
   "geocode_longitude",
   "geocode_latitude",
   "geocode_address",
   "geocode_viewport",
   "_geocode_country",
   "_geocode_city",
   "_streetview_pitch",
   "_streetview_heading",
   "_streetview",

   "_views"
]

rows = []
rows_to_validate = []
for post in posts.values():
    imgs = post['images'].values()
    len_imgs = len(post['images'])
    num_imgs = 0
    row_data = dict.fromkeys(keys)
    for key in keys:
        if post.has_key(key):
            if 'category' == key:
                if len(post[key]) == 1:
                    post[key] = post[key][0]['name']
                elif len(post[key]) > 1:
                    post[key] = ', '.join([c['name'] for c in post[key]])
                else:
                    print 'Post sem categoria: %s (%s)' % (post['post_title'], 
                            post['post_name'])
                    post[key] = ''
            if 'post_content' == key:
                post[key] = __.handle(post[key])    
            row_data[key] = post[key]
        elif post['meta'].has_key(key):
            row_data[key] = post['meta'][key]['meta_value']
        elif 'image' in key:
            if num_imgs < len_imgs:
                row_data[key] = imgs[num_imgs]['guid']
                num_imgs+=1
               
    rows.append(row_data)
    if post['ID'] in html_posts:
        rows_to_validate.append(row_data)

with open('ushahidi.json', 'w') as jsonfile:
    jsonfile.write(json.dumps(rows))

with open('ushahidi_validate.json', 'w') as jsonfile:
    jsonfile.write(json.dumps(rows_to_validate))

with open('ushahidi.csv', 'w') as csvfile:
    writer = csv.DictWriter(csvfile, fieldnames=keys, delimiter='|')
    writer.writeheader()

    for row in rows:
        writer.writerow(row)

    csvfile.close()