Skip to content

Instantly share code, notes, and snippets.

@Qwlouse

Qwlouse/Query_Sacred_MongoDB_Demo.ipynb

Last active November 12, 2015 23:00
Show Gist options
  • Save Qwlouse/0132e8deaa19ffd9980d to your computer and use it in GitHub Desktop.
Save Qwlouse/0132e8deaa19ffd9980d to your computer and use it in GitHub Desktop.
Download ZIP
Very basic demo on how to query a MongoDB collection created by Sacred
Display the source blob
Display the rendered blob
Raw
Query_Sacred_MongoDB_Demo.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import pymongo\n",
"from pandas.io.json import json_normalize\n",
"import gridfs"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"client = pymongo.MongoClient()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"runs = client.nascence.default.runs\n",
"fs = gridfs.GridFS(client.nascence, collection='default')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def get_results(filt, project=None):\n",
" project_dict = {'result': True, '_id': False}\n",
" if project is None:\n",
" project_dict['config'] = True\n",
" elif isinstance(project, dict):\n",
" for k, v in project.items():\n",
" project_dict[k] = v\n",
" else:\n",
" for k in project:\n",
" project_dict[k] = True\n",
" \n",
" return json_normalize(runs.find(filt, project_dict).sort('result', direction=pymongo.DESCENDING))"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"project = {'config', 'status'}"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>config.act_func</th>\n",
" <th>config.batch_size</th>\n",
" <th>config.binary</th>\n",
" <th>config.filename</th>\n",
" <th>config.in_size</th>\n",
" <th>config.layer_size</th>\n",
" <th>config.learning_rate</th>\n",
" <th>config.max_epochs</th>\n",
" <th>config.net_type</th>\n",
" <th>config.num_layers</th>\n",
" <th>config.seed</th>\n",
" <th>config.t_bias</th>\n",
" <th>result</th>\n",
" <th>status</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>rel</td>\n",
" <td>20</td>\n",
" <td>False</td>\n",
" <td>maktuba-filtered-5.0.h5</td>\n",
" <td>44</td>\n",
" <td>8</td>\n",
" <td>0.000590</td>\n",
" <td>100</td>\n",
" <td>plain</td>\n",
" <td>2</td>\n",
" <td>638087870</td>\n",
" <td>-1</td>\n",
" <td>0.500602</td>\n",
" <td>COMPLETED</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>rel</td>\n",
" <td>20</td>\n",
" <td>False</td>\n",
" <td>maktuba-filtered-5.0.h5</td>\n",
" <td>44</td>\n",
" <td>8</td>\n",
" <td>0.000135</td>\n",
" <td>100</td>\n",
" <td>plain</td>\n",
" <td>5</td>\n",
" <td>771747020</td>\n",
" <td>-1</td>\n",
" <td>0.499323</td>\n",
" <td>COMPLETED</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>rel</td>\n",
" <td>20</td>\n",
" <td>False</td>\n",
" <td>maktuba-filtered-5.0.h5</td>\n",
" <td>44</td>\n",
" <td>8</td>\n",
" <td>0.000101</td>\n",
" <td>100</td>\n",
" <td>plain</td>\n",
" <td>2</td>\n",
" <td>737253126</td>\n",
" <td>-1</td>\n",
" <td>0.497999</td>\n",
" <td>COMPLETED</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>rel</td>\n",
" <td>20</td>\n",
" <td>False</td>\n",
" <td>maktuba-filtered-5.0.h5</td>\n",
" <td>44</td>\n",
" <td>32</td>\n",
" <td>0.000171</td>\n",
" <td>100</td>\n",
" <td>plain</td>\n",
" <td>10</td>\n",
" <td>470379278</td>\n",
" <td>-1</td>\n",
" <td>0.497488</td>\n",
" <td>COMPLETED</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>rel</td>\n",
" <td>20</td>\n",
" <td>False</td>\n",
" <td>maktuba-filtered-5.0.h5</td>\n",
" <td>44</td>\n",
" <td>16</td>\n",
" <td>0.001008</td>\n",
" <td>100</td>\n",
" <td>plain</td>\n",
" <td>2</td>\n",
" <td>453090728</td>\n",
" <td>-1</td>\n",
" <td>0.490946</td>\n",
" <td>COMPLETED</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>rel</td>\n",
" <td>20</td>\n",
" <td>False</td>\n",
" <td>maktuba-filtered-5.0.h5</td>\n",
" <td>44</td>\n",
" <td>8</td>\n",
" <td>0.001263</td>\n",
" <td>100</td>\n",
" <td>plain</td>\n",
" <td>1</td>\n",
" <td>586890871</td>\n",
" <td>-1</td>\n",
" <td>0.488610</td>\n",
" <td>COMPLETED</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>rel</td>\n",
" <td>20</td>\n",
" <td>False</td>\n",
" <td>maktuba-filtered-5.0.h5</td>\n",
" <td>44</td>\n",
" <td>16</td>\n",
" <td>0.001749</td>\n",
" <td>100</td>\n",
" <td>plain</td>\n",
" <td>1</td>\n",
" <td>784002041</td>\n",
" <td>-1</td>\n",
" <td>0.486264</td>\n",
" <td>COMPLETED</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>rel</td>\n",
" <td>20</td>\n",
" <td>False</td>\n",
" <td>maktuba-filtered-5.0.h5</td>\n",
" <td>44</td>\n",
" <td>8</td>\n",
" <td>0.063038</td>\n",
" <td>100</td>\n",
" <td>plain</td>\n",
" <td>10</td>\n",
" <td>536664405</td>\n",
" <td>-1</td>\n",
" <td>0.483417</td>\n",
" <td>COMPLETED</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>rel</td>\n",
" <td>20</td>\n",
" <td>False</td>\n",
" <td>maktuba-filtered-5.0.h5</td>\n",
" <td>44</td>\n",
" <td>8</td>\n",
" <td>0.045924</td>\n",
" <td>100</td>\n",
" <td>plain</td>\n",
" <td>10</td>\n",
" <td>286268469</td>\n",
" <td>-1</td>\n",
" <td>0.482362</td>\n",
" <td>COMPLETED</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>rel</td>\n",
" <td>20</td>\n",
" <td>False</td>\n",
" <td>maktuba-filtered-5.0.h5</td>\n",
" <td>44</td>\n",
" <td>8</td>\n",
" <td>0.011928</td>\n",
" <td>100</td>\n",
" <td>plain</td>\n",
" <td>1</td>\n",
" <td>846158399</td>\n",
" <td>-1</td>\n",
" <td>0.482240</td>\n",
" <td>COMPLETED</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" config.act_func config.batch_size config.binary config.filename \\\n",
"0 rel 20 False maktuba-filtered-5.0.h5 \n",
"1 rel 20 False maktuba-filtered-5.0.h5 \n",
"2 rel 20 False maktuba-filtered-5.0.h5 \n",
"3 rel 20 False maktuba-filtered-5.0.h5 \n",
"4 rel 20 False maktuba-filtered-5.0.h5 \n",
"5 rel 20 False maktuba-filtered-5.0.h5 \n",
"6 rel 20 False maktuba-filtered-5.0.h5 \n",
"7 rel 20 False maktuba-filtered-5.0.h5 \n",
"8 rel 20 False maktuba-filtered-5.0.h5 \n",
"9 rel 20 False maktuba-filtered-5.0.h5 \n",
"\n",
" config.in_size config.layer_size config.learning_rate config.max_epochs \\\n",
"0 44 8 0.000590 100 \n",
"1 44 8 0.000135 100 \n",
"2 44 8 0.000101 100 \n",
"3 44 32 0.000171 100 \n",
"4 44 16 0.001008 100 \n",
"5 44 8 0.001263 100 \n",
"6 44 16 0.001749 100 \n",
"7 44 8 0.063038 100 \n",
"8 44 8 0.045924 100 \n",
"9 44 8 0.011928 100 \n",
"\n",
" config.net_type config.num_layers config.seed config.t_bias result \\\n",
"0 plain 2 638087870 -1 0.500602 \n",
"1 plain 5 771747020 -1 0.499323 \n",
"2 plain 2 737253126 -1 0.497999 \n",
"3 plain 10 470379278 -1 0.497488 \n",
"4 plain 2 453090728 -1 0.490946 \n",
"5 plain 1 586890871 -1 0.488610 \n",
"6 plain 1 784002041 -1 0.486264 \n",
"7 plain 10 536664405 -1 0.483417 \n",
"8 plain 10 286268469 -1 0.482362 \n",
"9 plain 1 846158399 -1 0.482240 \n",
"\n",
" status \n",
"0 COMPLETED \n",
"1 COMPLETED \n",
"2 COMPLETED \n",
"3 COMPLETED \n",
"4 COMPLETED \n",
"5 COMPLETED \n",
"6 COMPLETED \n",
"7 COMPLETED \n",
"8 COMPLETED \n",
"9 COMPLETED "
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"get_results({'config.act_func':'rel'}, project)[:10]"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Retrieve a file stored in the database as an artifact\n",
"best = runs.find({'status':'COMPLETED'}).sort('result', pymongo.DESCENDING)[0]\n",
"\n",
"with open('best_net.h5', 'wb') as f:\n",
" f.write(fs.get(best['artifacts'][0]).read())"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.4.3"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment