gistfile1.py

import os
import sys
import pkg_resources
import paste.script.appinstall
from paste.deploy import loadapp
import paste.fixture
import time
import csv
import cProfile
import requests

here_dir = os.path.dirname(os.path.abspath(__file__))
conf_dir = here_dir

sys.path.insert(0, conf_dir)
pkg_resources.working_set.add_entry(conf_dir)
pkg_resources.require('Paste')
pkg_resources.require('PasteScript')

def config_abspath(file_path):
    if os.path.isabs(file_path):
        return file_path
    return os.path.join(conf_dir, file_path)

config_path = config_abspath('development.ini')

cmd = paste.script.appinstall.SetupCommand('setup-app')
cmd.run([config_path])

wsgiapp = loadapp('config:datahub.ini', relative_to=conf_dir)
app = paste.fixture.TestApp(wsgiapp)


out = open("output times", "w+")
csv_out = csv.writer(out)


def run_url(url, output, line_num):

    try:
        if url[0] == url[-1] == '"':
            url = url[1:-1]

        start_time = time.time()
        res = app.get(url, status=[200], extra_environ={'REMOTE_USER': 'visitor'})
        length = time.time() - start_time
        output.writerow([url.strip(), 1, length, "", line_num])
    except paste.fixture.AppError:
        output.writerow([url.strip(), 0, 0, "app error", line_num])
    except KeyboardInterrupt:
        raise
    except:
        import traceback
        traceback.print_exc()
        output.writerow([url.strip(), 0, 0, "unknown error", line_num])


def run_all(lines):
    lines_done = 0
    for line in open("urls.txt"):
        if 'rest' in line:
            continue
        run_url(line, csv_out, lines_done)
        if lines_done > lines:
            break
        lines_done += 1
        print lines_done
        if lines_done % 1000 == 0:
            import pdb; pdb.set_trace()
    


#cProfile.runctx( command, globals(), locals(), filename="ckan.profile" )

run_all(10000000000)