Skip to content

Instantly share code, notes, and snippets.

@cfriedline

cfriedline/Untitled0.ipynb-4.json

Last active August 29, 2015 14:02
Show Gist options
  • Save cfriedline/76d5deab3d7ca73e41e9 to your computer and use it in GitHub Desktop.
Save cfriedline/76d5deab3d7ca73e41e9 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
Untitled0.ipynb-4.json
{
"metadata": {
"name": "",
"signature": "sha256:08fbbaacbb2f7ed6a72dbda30b0539a89fe5f1e1bb540c869d19c9976d4feba8"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "code",
"collapsed": false,
"input": [
"from IPython.parallel import Client\n",
"import pandas"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 23
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rc = Client(profile=\"huge\")\n",
"dview = rc[:]\n",
"lview = rc.load_balanced_view()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 24
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"with dview.sync_imports():\n",
" import pandas"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"importing pandas on engine(s)\n"
]
}
],
"prompt_number": 25
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"def get_df(name):\n",
" store = pandas.HDFStore(\"store.h5\")\n",
" df = store[name].ix[0:5,0:3]\n",
" store.close()\n",
" return df, type(df)\n",
"dview['get_df'] = get_df"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 41
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#Fails"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"print get_df('ass')"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"( S0 S2 S16\n",
"0 5.326002 0.000000 -0.294884\n",
"1 -0.363137 2.922360 -0.294884\n",
"2 0.000000 0.000000 0.000000\n",
"3 -0.363137 0.000000 0.000000\n",
"4 0.000000 0.000000 0.000000\n",
"5 -0.363137 -0.618853 0.000000, <class 'pandas.core.frame.DataFrame'>)\n"
]
}
],
"prompt_number": 42
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"print lview.apply(get_df, 'ass').get()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"ename": "RemoteError",
"evalue": "KeyError('No object named ass in the file')",
"output_type": "pyerr",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mKeyError\u001b[0m Traceback (most recent call last)\u001b[1;32m<string>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m",
"\u001b[1;32m<ipython-input-41-cdc5a19131dc>\u001b[0m in \u001b[0;36mget_df\u001b[1;34m(name)\u001b[0m",
"\u001b[1;32m/data7/cfriedline/anaconda/envs/conda/lib/python2.7/site-packages/pandas/io/pytables.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m",
"\u001b[0;32m 414\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[0;32m 415\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[1;32m--> 416\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[0m\u001b[0;32m 417\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[0;32m 418\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m__setitem__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[1;32m/data7/cfriedline/anaconda/envs/conda/lib/python2.7/site-packages/pandas/io/pytables.pyc\u001b[0m in \u001b[0;36mget\u001b[1;34m(self, key)\u001b[0m",
"\u001b[0;32m 617\u001b[0m \u001b[0mgroup\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_node\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[0;32m 618\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mgroup\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[1;32m--> 619\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'No object named %s in the file'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[0m\u001b[0;32m 620\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_read_group\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[0;32m 621\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m",
"\u001b[1;31mKeyError\u001b[0m: 'No object named ass in the file'"
]
}
],
"prompt_number": 43
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#Works"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"print get_df('pca_maf')"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"( L0 L2 L6\n",
"0 1 -1 0\n",
"1 0 1 1\n",
"2 -1 -1 0\n",
"3 0 -1 0\n",
"4 -1 -1 -1\n",
"5 0 0 -1, <class 'pandas.core.frame.DataFrame'>)\n"
]
}
],
"prompt_number": 46
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"print lview.apply(get_df, 'pca_maf').get()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"( L0 L2 L6\n",
"0 1 -1 0\n",
"1 0 1 1\n",
"2 -1 -1 0\n",
"3 0 -1 0\n",
"4 -1 -1 -1\n",
"5 0 0 -1, <class 'pandas.core.frame.DataFrame'>)\n"
]
}
],
"prompt_number": 47
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": []
}
],
"metadata": {}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment