davidread

#!/usr/bin/env bash
set -ex

# NB_USER=$USER
# NB_UID=$USER_UID
# USER_UID=1001
# GROUP=staff

# # Add the $USER user
# useradd -g $GROUP -u $USER_UID -d /home/$USER $USER

davidread

FROM jupyter/datascience-notebook

# Install Anaconda
#RUN conda install anaconda

# Install Jupyter Dashboard
#RUN pip install jupyter_dashboards
#RUN jupyter dashboards quick-setup --sys-prefix
#RUN jupyter nbextension enable jupyter_dashboards --py --sys-prefix

davidread

Keybase proof

I hereby claim:

• I am davidread on github.
• I am davidread (https://keybase.io/davidread) on keybase.
• I have a public key ASCLfxHooMvjCtVRSqJRX7n4AQOgYplbm4wzd_JNMCnI7wo

To claim this, I am signing this object:

davidread

'''Load some test CSV using pgloader'''

import argparse
import subprocess
from pprint import pprint
import os.path

import sqlalchemy
import messytables

davidread

>>> for pkg in model.Session.query(model.Package).filter_by(state='active'): print pkg.name, '\t\t', ' '.join([t.name for t in pkg.get_tags()])
...
africa-powermining-projects-database 		mining power sub-saharan africa
africa-the-electicity-supply-system 		Africa Electricity OSeMOSYS
biomass-per-district-2002 		Biomass District Uganda
bla
cameroon-hydro-power-projects 		hydro power plant
climatescope-mapping-the-global-frontiers-for-clean-energy-investment-2016 		renewable energy
dar-es-salaam-raw
demand-for-distributed-renewable-energy-generation-in-pakistan 		distributed renewable energy iesco islamabad pakistan solar tail-ended-grid

davidread

(ckan)vagrant@precise64:/vagrant/src/ckan$ paster --plugin=ckan db init -c test-core.ini
Traceback (most recent call last):
  File "/home/vagrant/ckan/bin/paster", line 11, in <module>
    sys.exit(run())
  File "/home/vagrant/ckan/local/lib/python2.7/site-packages/paste/script/command.py", line 102, in run
    invoke(command, command_name, options, args[1:])
  File "/home/vagrant/ckan/local/lib/python2.7/site-packages/paste/script/command.py", line 141, in invoke
    exit_code = runner.run(args)
  File "/home/vagrant/ckan/local/lib/python2.7/site-packages/paste/script/command.py", line 236, in run
    result = self.command()

davidread

SELECT substring( R.url from '.*://([^/]*)' ) as hostname, count(substring( R.url from '.*://([^/]*)' )) from resource R
JOIN resource_group RG ON R.resource_group_id = RG.id
JOIN Package P ON P.id = RG.package_id
WHERE R.state='active' and P.state='active'
GROUP BY hostname
ORDER BY count desc;
                                                                                    hostname                                                                                     | count
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------
 www.gov.uk                                                                                                                                                                      | 20151
 www.ons.gov.uk                                                                                                                                                            

davidread

$ paster govuk_publications --config=/var/ckan/ckan.ini scrape
...
After 2387/2387 pages:

Publications:
        Created: 94880 ['consultations/gda-of-hitachi-ge-nuclear-energy-ltds-uk-advanced-boiling-water-reactor', 'consultations/postgraduate-doctoral-loans', 'consulta...
        Unchanged: 484 ['publications/the-ombudsmans-annual-report-and-accounts-2015-16', 'publications/rg1-8nh-kingfisher-colours-limited-environmental-permit-applicati...
        Updated: 83 ['publications/oil-and-gas-public-statements-relating-to-2014-operations', 'statistics/tabulation-tool-questionnaire-statistical-notice', 'publicat...
        Error - Incomplete publication - title: 7 ['statistics/womens-smoking-status-at-time-of-delivery-in-england-october-2014-to-december-2014', 'statistics/summary-hospital-level-mortality-indic...
        Error - Publication redirect: 1 ['publications/preventing-illegal-working-guidance-for-employers-october-2013']

davidread

1. Register for Microsoft account at https://signup.live.com/signup
2. Email verification - click link (logs you in)
3. Register on Azure Data Market at https://datamarket.azure.com/register
4. Go to: https://datamarket.azure.com/dataset/datagovuk/metofficeweatheropendata click 'Sign up' then 'agree', then 'sign up'.

Use Web i/f to download CSV:
At: https://datamarket.azure.com/dataset/explore/0f2cba12-e5cf-4c6d-83c9-83114d44387a click 'Explore', 'Three Hourly Forecast' and 'Download Excel (CSV)'

or by API:
Go to: https://datamarket.azure.com/account for account key

davidread

>>> requests.get('http://uk-air.defra.gov.uk/datastore/pcb/2010_TOMPs_PCB_Data.xlsx', headers={'User-agent': 'python-requests'})
<Response [403]>
>>> requests.get('http://uk-air.defra.gov.uk/datastore/pcb/2010_TOMPs_PCB_Data.xlsx', headers={'User-agent': 'python-request'})
<Response [200]>