Created
June 11, 2015 07:44
-
-
Save mattfullerton/7d04fff53d0ce49e8c18 to your computer and use it in GitHub Desktop.
Download all resources from a CKAN
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import urllib, urllib2, json | |
baseurl = 'https://yourckanurl.com' | |
apikey = None # Set to something else if using private datasets | |
print 'Downloading ' + baseurl + "/api/3/action/current_package_list_with_resources..." | |
request = urllib2.Request(baseurl +'/api/3/action/current_package_list_with_resources') | |
if apikey != None: | |
request.add_header('Authorization', apikey) | |
jsonurl = urllib2.urlopen(request, "{}") | |
packagelist = json.loads(jsonurl.read()) | |
packagelist = packagelist['result'] | |
for package in packagelist: | |
if ('resources' in package): | |
resources = package['resources'] | |
for file in resources: | |
if (file['url'] not in [None, '']): | |
if '://' not in file['url']: | |
filepath = baseurl + file['url'] | |
else: | |
filepath = file['url'] | |
print 'Downloading file ' + filepath | |
try: | |
downloadfile = urllib.URLopener() | |
downloadfile.retrieve(filepath, filepath.split('/')[-1]) | |
except: | |
print 'ERROR! Failed to download file ' + filepath |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment