andrew_ms.ipynb.json

{
 "metadata": {
  "name": "",
  "signature": "sha256:29933bf423b0f63d5ae9b0a7597a52fbd50ec1ac016b537876b45d0a0cf0dacc"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dirs = [\"cmds\", \"results\"]\n",
      "for d in dirs:\n",
      "    if not os.path.exists(d):\n",
      "        os.mkdir(d)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mscall_file = \"/home/cfriedline/ms_lob_sim.txt\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mscall_data = open(mscall_file).read()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mscall_cmd = mscall_data.strip().split(\">\")[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def get_qsub_header(sim_num):\n",
      "    header = '''\n",
      "#!/bin/bash\n",
      "#$ -cwd\n",
      "#$ -j yes\n",
      "#$ -n ms%d\n",
      "#$ -V\n",
      "''' % sim_num\n",
      "    return header\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "profile=\"huge\"\n",
      "from IPython.parallel import Client"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rc = Client(profile=profile)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dview = rc[:]\n",
      "lview = rc.load_balanced_view()\n",
      "len(rc)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 35,
       "text": [
        "800"
       ]
      }
     ],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ms = \"/home/cfriedline/data7/src/ms/msdir/ms\"\n",
      "mscall_cmd = mscall_cmd.replace(\"./ms\", ms)\n",
      "dview['mscall_cmd'] = mscall_cmd"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 42
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "with dview.sync_imports():\n",
      "    import os\n",
      "    import time"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "importing os on engine(s)\n",
        "importing time on engine(s)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 49
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def run_ms(i):\n",
      "    start = time.time()\n",
      "    out_dir = \"/home/cfriedline/data7/ipython/notebooks/ms/results/\"\n",
      "    out = os.path.join(out_dir, \"output_%d.txt\" % i)\n",
      "    cmd = \"%s > %s\" % (mscall_cmd, out)\n",
      "    !$cmd\n",
      "    return out, \"%.5f\" % (time.time() - start)\n",
      "dview[\"run_ms\"] = run_ms"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 56
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test = lview.apply_async(run_ms, 0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 57
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test.r"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 58,
       "text": [
        "('/home/cfriedline/data7/ipython/notebooks/ms/results/output_0.txt',\n",
        " '174.68680')"
       ]
      }
     ],
     "prompt_number": 58
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "    ('/home/cfriedline/data7/ipython/notebooks/ms/results/output_0.txt',\n",
      "     '174.68680')"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "total = 3082000"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 69
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "results = []\n",
      "for i in xrange(total):\n",
      "    results.append(lview.apply_async(run_ms, i))\n",
      "    if i %1000 == 0:\n",
      "        print \"at %d\" % i"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ready = 0\n",
      "for x in results:\n",
      "    if x.ready():\n",
      "        ready += 1\n",
      "print \"%d/%d\" % (ready, len(results))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "22827/47736\n"
       ]
      }
     ],
     "prompt_number": 74
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "results2 = lview.map_async(run_ms, range[47736, total])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}