Dump ParkAPI database.

dump.py

#!/usr/bin/env python3

import psycopg2 as psql
from sys import argv
import configparser
import os
from urllib.parse import urlparse
from tqdm import tqdm
import csv

APP_ROOT = os.path.realpath(os.path.dirname(__file__))

DEFAULT_CONFIGURATION = {
    "port": 5000,
    "host": "::1",
    "debug": False,
    "live_scrape": True,
    "database_uri": "postgres:///park_api",
}

if __name__ == "__main__":
  
  if len(argv) < 3:
    print("Usage: {0} <City> <year>".format(argv[0]))
    exit(1)
  
  print("loading config...")
  
  config_path = os.path.join(APP_ROOT, "config.ini")
  try:
      config_file = open(config_path)
  except (OSError, FileNotFoundError) as e:
      print("Failed load configuration: %s" % e)
      exit(1)
  config = configparser.ConfigParser(DEFAULT_CONFIGURATION, strict=False)
  config.read_file(config_file)
  db_conf = urlparse(config["production"].get("database_uri"))
  
  print("connecting to database...")
  
  with psql.connect(database=db_conf.path[1:],
          user=db_conf.username,
          password=db_conf.password,
          host=db_conf.hostname,
          port=db_conf.port) as db:
    
    cursor = db.cursor()
    
    print("precalculating...")
    
    cursor.execute("select count(city) from parkapi where city = '{0}' and extract(year from timestamp_downloaded) = '{1}'".format(argv[1], argv[2]))
    rowcount = cursor.fetchone()[0]
    
    print("fetching from database...")
    
    cursor.execute("select timestamp_downloaded, city, data from parkapi where city = '{0}' and extract(year from timestamp_downloaded) = '{1}'".format(argv[1], argv[2]))
    
    table = []
    for i in tqdm(range(rowcount)):
      table.append(cursor.fetchone())
        
    print("refining data...")
    
    data = {}
    for row in tqdm(table):
      for lot in row[2]['lots']:
        try:
          data[lot['id']].append((row[2]['last_downloaded'], lot['free']))
        except KeyError:
          data[lot['id']] = [(row[2]['last_downloaded'], lot['free'])]
    
    print("writing data...")
    
    for city in tqdm(data.keys()):
      with open("{0}-{1}.{2}.csv".format(city.replace('\n', '-nl-'), argv[2], os.getpid()), "w") as outfile:
        writer = csv.writer(outfile)
        writer.writerows(data[city])