Created
June 30, 2016 06:08
-
-
Save neuromusic/ef7672f2142efcb22adb015bac20d029 to your computer and use it in GitHub Desktop.
downloading from crcns
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "code", | |
| "execution_count": 1, | |
| "metadata": { | |
| "collapsed": true | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "import requests" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": false, | |
| "scrolled": false | |
| }, | |
| "outputs": [ | |
| { | |
| "name": "stdout", | |
| "output_type": "stream", | |
| "text": [ | |
| "{'content-length': '3340', 'x-powered-by': 'PHP/5.3.3', 'set-cookie': 'crcns_nersc_download=id6582key607477388', 'accept-ranges': 'bytes', 'server': 'Apache/2.2.3 (CentOS)', 'connection': 'close', 'date': 'Thu, 30 Jun 2016 05:15:40 GMT', 'content-type': 'text/html; charset=UTF-8'}\n", | |
| "\n", | |
| "<html>\n", | |
| "<head>\n", | |
| "<title>CRCNS.org data download</title>\n", | |
| "</head>\n", | |
| "<body>\n", | |
| "<h2>CRCNS.org data download</h2>\n", | |
| "<p>Logged in as jkiggins (Justin Kiggins). <a href=\"/project/crcns/download/index.php?cmd=logout\">logout</a>.<br />\n", | |
| "Contents of alm-1/datafiles/data_structure_files:</p>\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles\">..</a><br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM210861.tar.gz\">data_structure_ANM210861.tar.gz</a> (568625369)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM210862.tar.gz\">data_structure_ANM210862.tar.gz</a> (423681526)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM210863.tar.gz\">data_structure_ANM210863.tar.gz</a> (478127420)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM214427.tar.gz\">data_structure_ANM214427.tar.gz</a> (713036422)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM214429.tar.gz\">data_structure_ANM214429.tar.gz</a> (1064032176)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM214430.tar.gz\">data_structure_ANM214430.tar.gz</a> (607337575)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM218453.tar.gz\">data_structure_ANM218453.tar.gz</a> (651067621)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM218457.tar.gz\">data_structure_ANM218457.tar.gz</a> (829973651)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM218693.tar.gz\">data_structure_ANM218693.tar.gz</a> (415317775)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219030.tar.gz\">data_structure_ANM219030.tar.gz</a> (716106206)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219031.tar.gz\">data_structure_ANM219031.tar.gz</a> (663861496)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219033.tar.gz\">data_structure_ANM219033.tar.gz</a> (828558913)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219036.tar.gz\">data_structure_ANM219036.tar.gz</a> (751800829)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219037.tar.gz\">data_structure_ANM219037.tar.gz</a> (926004948)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219038.tar.gz\">data_structure_ANM219038.tar.gz</a> (1040644533)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219247.tar.gz\">data_structure_ANM219247.tar.gz</a> (704992587)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219248.tar.gz\">data_structure_ANM219248.tar.gz</a> (1210463535)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219253.tar.gz\">data_structure_ANM219253.tar.gz</a> (743796292)<br />\n", | |
| "<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM221977.tar.gz\">data_structure_ANM221977.tar.gz</a> (392405930)<br />\n", | |
| "\n", | |
| "</body>\n", | |
| "</html> \n", | |
| "\n" | |
| ] | |
| } | |
| ], | |
| "source": [ | |
| "crcns_username = ''\n", | |
| "crcns_password = ''\n", | |
| "\n", | |
| "URL = 'https://portal.nersc.gov/project/crcns/download/index.php'\n", | |
| "\n", | |
| "path = 'alm-1/datafiles/data_structure_files'\n", | |
| "\n", | |
| "login_data = dict(\n", | |
| " username=crcns_username,\n", | |
| " password=crcns_password,\n", | |
| " fn=path,\n", | |
| " submit='Login' \n", | |
| " )\n", | |
| "\n", | |
| "with requests.Session() as s:\n", | |
| " r = s.post(URL,data=login_data)\n", | |
| " print r.headers\n", | |
| " print r.text" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "collapsed": false | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "datafile = 'alm-1/datafiles/data_structure_files/data_structure_ANM218457.tar.gz'\n", | |
| "login_data.update(fn=datafile)\n", | |
| "\n", | |
| "with requests.Session() as s:\n", | |
| " local_filename = login_data['fn'].split('/')[-1]\n", | |
| " r = s.post(URL,data=login_data,stream=True)\n", | |
| " print r.headers\n", | |
| " with open(local_filename, 'wb') as f:\n", | |
| " for chunk in r.iter_content(chunk_size=1024):\n", | |
| " if chunk:\n", | |
| " f.write(chunk)\n", | |
| " print local_filename" | |
| ] | |
| } | |
| ], | |
| "metadata": { | |
| "kernelspec": { | |
| "display_name": "Python 2", | |
| "language": "python", | |
| "name": "python2" | |
| }, | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 2 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython2", | |
| "version": "2.7.11" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 0 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment