NewHaven.io NLP Tutorial

.gitignore

.ipynb*
build*
venv*
refs*
*.pyc
tmp*

README.md

Kyle's NLP Tutorial from the NHV.io meetup

NewHaven.io NLP tutorial.ipynb

{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "NewHaven.io NLP tutorial"
     ]
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Getting started"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "You don't need to install anything to view this notebook, but\n",
      "you do if you want to run or alter it. If you want to do that,\n",
      "first, you'll need to install some dependencies. I like to use\n",
      "[pip](http://www.pip-installer.org/en/latest/) and \n",
      "[virtualenv](http://www.virtualenv.org/en/latest/).\n",
      "\n",
      "```shell\n",
      "git clone https://gist.github.com/9662971.git nhvio-nlp-tutorial\n",
      "cd nhvio-nlp-tutorial\n",
      "virtualenv venv\n",
      ". ./venv/bin/activate\n",
      "pip install tornado ipython jinja2 nltk pyzmq matplotlib\n",
      ". ./venv/bin/activate\n",
      "ipython notebook --pylab inline\n",
      "```\n",
      "\n",
      "Then you can open this notebook and run the code. Note, depending\n",
      "on your operating system, you'll have some library depend"
     ]
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Sentence tokenization"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from nltk.tokenize import sent_tokenize"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "hello = \"\"\"Hello Nhv.io peeps. This is a presentation on NLTK.\n",
      "I hope you like it. Joel, would you get me a beer please?\"\"\"\n",
      "sent_tokenize(hello)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "['Hello Nhv.io peeps.',\n",
        " 'This is a presentation on NLTK.',\n",
        " 'I hope you like it.',\n",
        " 'Joel, would you get me a beer please?']"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Word tokenization"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from nltk.tokenize import word_tokenize, wordpunct_tokenize"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print word_tokenize(hello)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "['Hello', 'Nhv.io', 'peeps.', 'This', 'is', 'a', 'presentation', 'on', 'NLTK.', 'I', 'hope', 'you', 'like', 'it.', 'Joel', ',', 'would', 'you', 'get', 'me', 'a', 'beer', 'please', '?']\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print wordpunct_tokenize(hello)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "['Hello', 'Nhv', '.', 'io', 'peeps', '.', 'This', 'is', 'a', 'presentation', 'on', 'NLTK', '.', 'I', 'hope', 'you', 'like', 'it', '.', 'Joel', ',', 'would', 'you', 'get', 'me', 'a', 'beer', 'please', '?']\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Part-of-speech Tagging"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from nltk.tag import pos_tag"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pos_tag(word_tokenize(\"They refuse to permit us to obtain the refuse permit\"))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "ImportError",
       "evalue": "No module named numpy.core.multiarray",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-7-1577d4823e6c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpos_tag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mword_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"They refuse to permit us to obtain the refuse permit\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[0;32m/Users/kljensen/mnt/e-Syncd/Current/nhv.io-nlp-gist/venv/lib/python2.7/site-packages/nltk/tag/__init__.pyc\u001b[0m in \u001b[0;36mpos_tag\u001b[0;34m(tokens)\u001b[0m\n\u001b[1;32m     97\u001b[0m     \u001b[0;34m:\u001b[0m\u001b[0mrtype\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m     \"\"\"\n\u001b[0;32m---> 99\u001b[0;31m     \u001b[0mtagger\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_POS_TAGGER\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    100\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtagger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/Users/kljensen/mnt/e-Syncd/Current/nhv.io-nlp-gist/venv/lib/python2.7/site-packages/nltk/data.pyc\u001b[0m in \u001b[0;36mload\u001b[0;34m(resource_url, format, cache, verbose, logic_parser, fstruct_parser)\u001b[0m\n\u001b[1;32m    603\u001b[0m     \u001b[0;31m# Load the resource.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    604\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mformat\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'pickle'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 605\u001b[0;31m         \u001b[0mresource_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource_url\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    606\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mformat\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'yaml'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    607\u001b[0m         \u001b[0;32mimport\u001b[0m \u001b[0myaml\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;31mImportError\u001b[0m: No module named numpy.core.multiarray"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Tag\t\tMeaning\t\tExamples\n",
      "ADJ\t    adjective\tnew, good, high, special, big, local\n",
      "ADV\t\tadverb\t\treally, already, still, early, now\n",
      "CNJ\t\tconjunction\tand, or, but, if, while, although\n",
      "DET\t\tdeterminer\tthe, a, some, most, every, no\n",
      "FW\t\tforeign \tdolce, ersatz, esprit, quo, maitre\n",
      "MOD\t\tmodal verb\twill, can, would, may, must, should\n",
      "N\t\tnoun\t\tyear, home, costs, time, education\n",
      "NUM\t\tnumber\t\ttwenty-four, fourth, 1991, 14:24\n",
      "PRO\t\tpronoun\t\the, their, her, its, my, I, us\n",
      "P\t\tprepos.\t\ton, of, at, with, by, into, under\n",
      "TO\t\tword to\t\tto\n",
      "UH\t\tinterje.\tah, bang, ha, whee, hmpf, oops\n",
      "V\t\tverb\t\tis, has, get, do, make, see, run\n",
      "EX\t\texistential\t\t\tthere, there's\n",
      "NP\t\tproper noun\t\t\tAlison, Africa, April, Washington\n",
      "VD\t\tpast tense\t\t\tsaid, took, told, made, asked\n",
      "VG\t\tp.participle\t\tmaking, going, playing, working\n",
      "VN\t\tpast participle\t\tgiven, taken, begun, sung\n",
      "WH\t\twh determiner\t\twho, which, when, what, where, how"
     ]
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Chunking and named entity identification"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from nltk.chunk import ne_chunk"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sentence = \"My name is Kyle Jensen and I live in New Haven, Connecticut.\"\n",
      "print ne_chunk(pos_tag(word_tokenize(sentence)))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Stop word removal"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from nltk.corpus import stopwords\n",
      "english_stopwords = stopwords.words('english')\n",
      "sentence = \"There are cars that are better than your car. Seriously, dude. Some good cars out there.\"\n",
      "words = [\n",
      "    word.lower()\n",
      "    for word in word_tokenize(sentence)\n",
      "    if word.lower() not in set(stopwords.words('english'))\n",
      "]\n",
      "print words\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "['cars', 'better', 'car.', 'seriously', ',', 'dude.', 'good', 'cars', '.']\n"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import re\n",
      "nonchars = re.compile(r\"[^a-zA-Z]\")\n",
      "\n",
      "# Removing non-characters\n",
      "words = [nonchars.sub('', word) for word in words]\n",
      "\n",
      "# Removing the empty string\n",
      "words = [word for word in words if len(word) > 0]\n",
      "\n",
      "print words"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "['cars', 'better', 'car', 'seriously', 'dude', 'good', 'cars']\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Stemming"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from nltk.stem.snowball import SnowballStemmer\n",
      "stemmer = SnowballStemmer('english')\n",
      "print [stemmer.stem(word) for word in words]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[u'car', u'better', u'car', u'serious', u'dude', u'good', u'car']\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Lemmatization"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from nltk.stem.wordnet import WordNetLemmatizer\n",
      "lmtzr = WordNetLemmatizer()\n",
      "lmtzr.lemmatize('cars')\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "'car'"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "lmtzr.lemmatize('feet')\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 12,
       "text": [
        "'foot'"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "lmtzr.lemmatize('run')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 13,
       "text": [
        "'run'"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "NLTK comes with useful corpora"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from nltk.corpus import wordnet\n",
      "wordnet.synsets('motorcar')\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 14,
       "text": [
        "[Synset('car.n.01')]"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "wordnet.synset('car.n.01').lemma_names"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 15,
       "text": [
        "['car', 'auto', 'automobile', 'machine', 'motorcar']"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for synset in wordnet.synsets('car'):\n",
      "    print synset.lemma_names"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "['car', 'auto', 'automobile', 'machine', 'motorcar']\n",
        "['car', 'railcar', 'railway_car', 'railroad_car']\n",
        "['car', 'gondola']\n",
        "['car', 'elevator_car']\n",
        "['cable_car', 'car']\n"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Corpus exploration"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from nltk.corpus import movie_reviews"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "movie_reviews.fileids()[1000:1010]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 18,
       "text": [
        "['pos/cv000_29590.txt',\n",
        " 'pos/cv001_18431.txt',\n",
        " 'pos/cv002_15918.txt',\n",
        " 'pos/cv003_11664.txt',\n",
        " 'pos/cv004_11636.txt',\n",
        " 'pos/cv005_29443.txt',\n",
        " 'pos/cv006_15448.txt',\n",
        " 'pos/cv007_4968.txt',\n",
        " 'pos/cv008_29435.txt',\n",
        " 'pos/cv009_29592.txt']"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "len(movie_reviews.words())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 19,
       "text": [
        "1583820"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "movie_reviews.words()[:10]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 20,
       "text": [
        "['plot', ':', 'two', 'teen', 'couples', 'go', 'to', 'a', 'church', 'party']"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from nltk import Text\n",
      "review_text = Text(movie_reviews.words())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "review_text.concordance('camera')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Building index...\n",
        "Displaying 25 of 333 matches:"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "tims being brutalized and killed on camera . i ' m talking about films like \" \n",
        "grapher darius khondji ' s stunning camera work guiding viewers into the bowel\n",
        " , \" double team \" ) never lets the camera rest for a second , and that ' s no\n",
        "features some of the most inventive camera work i have ever seen , but raimi n\n",
        " the special effects and outrageous camera angles * rule * \" knock off \" to th\n",
        " beginning and a sequence where the camera pans over top of a bunch of rooms i\n",
        "and grace . it ' s obvious that the camera loves her and she is very photogeni\n",
        " howard deutch , who was behind the camera for two of the best films of ' 80s \n",
        "n the motives of the man behind the camera . the conclusion of \" bully \" offer\n",
        "his movie : lots of shots where the camera is the eyes of the predator ( croc \n",
        "during the opening credits , as the camera pans over the streets and byways of\n",
        "s and does a lot of mugging for the camera . tom is easily one of the most lif\n",
        "bay book of directing and keeps his camera spinning . we get an endless parade\n",
        "h an urge to reach out and hold the camera still since \" armageddon . \" and wh\n",
        "anish into the cinematography . the camera , by the way , sometimes behaves so\n",
        " know how to shoot a flick -- great camera angles , shifting viewpoints , wick\n",
        " this work . he ' s back behind the camera for the first time in 11 years . it\n",
        " i noticed it is badly filmed , the camera is so shaky you cannot even tell wh\n",
        "ection is also not that great . his camera angles and sense of style are all i\n",
        " mcnaughton knows how to handle the camera , and he keeps the campy fillings o\n",
        "e ' s simply thrown in front of the camera as a pretty face . at first , he ' \n",
        " and stiles directly addressing the camera . this is the second film in a row \n",
        "it ' s all sound effects and tricky camera - work , along with a muddled , dis\n",
        " tries to offshoot this with flashy camera techniques , but pretty soon this h\n",
        " one shot , actress has cloths on , camera angle changes , actress seems to ha\n"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "review_text.collocations()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Building collocations list\n",
        "special effects; new york; star wars; high school; science fiction;\n",
        "even though; hong kong; star trek; pulp fiction; phantom menace; van\n",
        "damme; united states; starship troopers; box office; jackie chan;\n",
        "years ago; supporting cast; blair witch; boogie nights; las vegas"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print review_text.common_contexts(['sucked'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Building word-context index...\n",
        ",_( ,_in also_rocks are_into are_out be_into been_into eagerly_up\n",
        "ending_. get_into get_under gets_through gets_to getting_into\n",
        "immediately_into is_into it_me just_an just_my magically_into"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "None\n"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from nltk.probability import FreqDist\n",
      "english_stopwords = set(stopwords.words('english'))\n",
      "dist = FreqDist(stemmer.stem(w) for w in movie_reviews.words() if w not in english_stopwords and len(w) > 3)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 25
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from collections import Counter\n",
      "counts = Counter(stemmer.stem(w) for w in movie_reviews.words() if w not in english_stopwords and len(w) > 3)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "counts.most_common(10)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 27,
       "text": [
        "[(u'film', 11199),\n",
        " (u'movi', 6977),\n",
        " (u'like', 4137),\n",
        " (u'charact', 3881),\n",
        " (u'make', 3243),\n",
        " (u'time', 3047),\n",
        " (u'scene', 2671),\n",
        " (u'even', 2615),\n",
        " (u'good', 2476),\n",
        " (u'play', 2382)]"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dist.plot(25)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "ValueError",
       "evalue": "The plot function requires the matplotlib package (aka pylab). See http://matplotlib.sourceforge.net/",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-28-0be8e1cdbbb9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[0;32m/Users/kljensen/mnt/e-Syncd/Current/nhv.io-nlp-gist/venv/lib/python2.7/site-packages/nltk/probability.pyc\u001b[0m in \u001b[0;36mplot\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    290\u001b[0m             \u001b[0;32mimport\u001b[0m \u001b[0mpylab\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    291\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mImportError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 292\u001b[0;31m             raise ValueError('The plot function requires the matplotlib package (aka pylab). '\n\u001b[0m\u001b[1;32m    293\u001b[0m                          'See http://matplotlib.sourceforge.net/')\n\u001b[1;32m    294\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;31mValueError\u001b[0m: The plot function requires the matplotlib package (aka pylab). See http://matplotlib.sourceforge.net/"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Text classification"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from nltk.classify import NaiveBayesClassifier\n",
      "import nltk.classify.util\n",
      "def word_feats(words):\n",
      "    return dict([(stemmer.stem(word), True) for word in words])\n",
      " \n",
      "negids = movie_reviews.fileids('neg')\n",
      "posids = movie_reviews.fileids('pos')\n",
      " \n",
      "negfeats = [(word_feats(movie_reviews.words(fileids=[f])), 'neg') for f in negids]\n",
      "posfeats = [(word_feats(movie_reviews.words(fileids=[f])), 'pos') for f in posids]\n",
      " \n",
      "negcutoff = len(negfeats)*3/4\n",
      "poscutoff = len(posfeats)*3/4\n",
      " \n",
      "trainfeats = negfeats[:negcutoff] + posfeats[:poscutoff]\n",
      "testfeats = negfeats[negcutoff:] + posfeats[poscutoff:]\n",
      "print 'train on %d instances, test on %d instances' % (len(trainfeats), len(testfeats))\n",
      " \n",
      "classifier = NaiveBayesClassifier.train(trainfeats)\n",
      "print 'accuracy:', nltk.classify.util.accuracy(classifier, testfeats)\n",
      "classifier.show_most_informative_features()\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "Topic modeling & unsupervised learning"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def doc2words(doc):\n",
      "    \"\"\" Turns a piece of text into a list of stemmed words\n",
      "    \"\"\"\n",
      "    return [\n",
      "        stemmer.stem(word.lower())\n",
      "        for sentence in sent_tokenize(doc)\n",
      "        for word in word_tokenize(sentence)\n",
      "        if word.lower() not in english_stopwords\n",
      "        and len(word) > 3\n",
      "    ]\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 63
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "doc2words('stink kyle jensen likes')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 64,
       "text": [
        "[u'stink', u'kyle', u'jensen', u'like']"
       ]
      }
     ],
     "prompt_number": 64
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "negids = movie_reviews.fileids('neg')\n",
      "posids = movie_reviews.fileids('pos')\n",
      "negids[:5]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 69,
       "text": [
        "['neg/cv000_29416.txt',\n",
        " 'neg/cv001_19502.txt',\n",
        " 'neg/cv002_17424.txt',\n",
        " 'neg/cv003_12683.txt',\n",
        " 'neg/cv004_12641.txt']"
       ]
      }
     ],
     "prompt_number": 69
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def get_reviews():\n",
      "    return (doc2words(movie_reviews.raw(fileids=[f])) for f in (negids + posids))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 96
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from gensim.corpora.dictionary import Dictionary\n",
      "dictionary = Dictionary()\n",
      "dictionary.add_documents(get_reviews())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 84
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dictionary.filter_extremes(keep_n=10000, no_below=25)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 109
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dictionary.items()[:10]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 172,
       "text": [
        "[(0, 'woodi'),\n",
        " (2, 'four'),\n",
        " (3, 'sleep'),\n",
        " (4, 'mansion'),\n",
        " (5, 'captain'),\n",
        " (6, 'hate'),\n",
        " (7, 'payoff'),\n",
        " (9, 'whose'),\n",
        " (10, 'accus'),\n",
        " (11, 'accur')]"
       ]
      }
     ],
     "prompt_number": 172
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print dictionary"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Dictionary(3031 unique tokens)\n"
       ]
      }
     ],
     "prompt_number": 110
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "corpus = [dictionary.doc2bow(words) for words in get_reviews()]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 111
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print corpus[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(8, 1), (21, 1), (49, 1), (54, 2), (61, 1), (62, 1), (103, 1), (113, 1), (128, 1), (133, 1), (147, 2), (154, 1), (156, 1), (160, 1), (176, 2), (178, 1), (190, 1), (202, 2), (219, 1), (258, 2), (260, 1), (280, 1), (284, 1), (316, 1), (332, 1), (337, 2), (343, 1), (384, 1), (396, 1), (399, 1), (440, 2), (443, 1), (459, 1), (485, 1), (496, 1), (499, 1), (546, 1), (557, 1), (613, 2), (619, 2), (624, 1), (663, 1), (665, 1), (676, 1), (679, 1), (681, 1), (684, 1), (711, 1), (718, 1), (748, 1), (749, 1), (751, 1), (763, 1), (788, 1), (794, 1), (799, 1), (806, 1), (828, 3), (829, 1), (849, 1), (874, 1), (876, 1), (891, 1), (906, 1), (914, 1), (919, 1), (960, 2), (969, 1), (982, 1), (999, 1), (1001, 1), (1003, 1), (1010, 1), (1016, 1), (1039, 2), (1042, 1), (1081, 1), (1090, 1), (1134, 1), (1137, 1), (1152, 1), (1161, 2), (1218, 1), (1254, 1), (1297, 2), (1310, 1), (1312, 1), (1329, 1), (1335, 1), (1346, 2), (1440, 2), (1461, 2), (1479, 1), (1510, 1), (1550, 1), (1556, 3), (1577, 1), (1637, 1), (1639, 4), (1713, 1), (1736, 1), (1771, 2), (1773, 2), (1815, 1), (1823, 2), (1838, 1), (1848, 2), (1861, 2), (1865, 1), (1885, 2), (1928, 1), (1934, 5), (2031, 1), (2034, 2), (2045, 2), (2063, 1), (2083, 1), (2092, 2), (2106, 1), (2120, 1), (2128, 1), (2132, 2), (2148, 1), (2169, 1), (2175, 1), (2187, 1), (2221, 1), (2233, 1), (2280, 1), (2286, 2), (2296, 2), (2308, 1), (2330, 1), (2356, 2), (2377, 1), (2409, 1), (2420, 1), (2452, 1), (2462, 1), (2479, 1), (2495, 1), (2504, 1), (2505, 1), (2562, 1), (2604, 1), (2611, 1), (2612, 1), (2669, 1), (2674, 1), (2722, 1), (2755, 1), (2767, 1), (2778, 1), (2781, 1), (2782, 1), (2790, 1), (2794, 1), (2809, 1), (2812, 1), (2825, 1), (2836, 1), (2850, 2), (2875, 1), (2876, 1), (2881, 2), (2930, 2), (2950, 1), (2958, 2), (2988, 3), (3002, 2), (3006, 2)]\n"
       ]
      }
     ],
     "prompt_number": 112
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from gensim import corpora\n",
      "corpora.MmCorpus.serialize('review-corpos.mm', corpus) "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 113
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from gensim import models\n",
      "tfidf = models.TfidfModel(corpus)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 102
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print tfidf[corpus[0]]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(68, 0.03321354131787013), (88, 0.06266319209820685), (104, 0.01830461619329953), (114, 0.03654732805905633), (132, 0.07961009365343), (158, 0.08948060357910728), (177, 0.13721186173159686), (188, 0.046086675434065916), (320, 0.041258416891399884), (323, 0.10955188296118333), (351, 0.06130986508293805), (364, 0.01870800705325282), (469, 0.06436838482726705), (492, 0.06860593086579843), (505, 0.03995311545779118), (523, 0.0184800914133219), (568, 0.036990369533285404), (644, 0.054386254177717204), (725, 0.11220129327379119), (776, 0.07441911901830385), (828, 0.048335241719980056), (866, 0.06386605155381113), (957, 0.015689551966740243), (973, 0.030943987633586918), (985, 0.048041734232301014), (1111, 0.18919243575022485), (1134, 0.1410579816963094), (1200, 0.23164657991864324), (1296, 0.03316890938729051), (1307, 0.18477875685576434), (1327, 0.030132358973154492), (1353, 0.1616868244892916), (1473, 0.07484582588309108), (1517, 0.0901987042475063), (1541, 0.02668930966473223), (1637, 0.05517058967972435), (1652, 0.02631209099330154), (1690, 0.049145195178394926), (1710, 0.060537448978381835), (1719, 0.0203535331262223), (1827, 0.0913716628147444), (1878, 0.0520664056931873), (1966, 0.049145195178394926), (2030, 0.06337612284748723), (2058, 0.19983774332346013), (2135, 0.0656828458076175), (2176, 0.0456858314073722), (2361, 0.015558731023699156), (2458, 0.04078785480885019), (2530, 0.10150700738765359), (2586, 0.05407009835676713), (2786, 0.049145195178394926), (2791, 0.04558687307514643), (2814, 0.08694424829877206), (2830, 0.0563235092537373), (2858, 0.044717780662422615), (2909, 0.06709060661417918), (2939, 0.039512183878941655), (2943, 0.035041210389422775), (2969, 0.041489557698814045), (2995, 0.06892339531019098), (3011, 0.014833276502503892), (3012, 0.058282517543734855), (3013, 0.026462118963186815), (3226, 0.035570337866115305), (3240, 0.042655676363344185), (3311, 0.04898498904300013), (3382, 0.042912800106185506), (3490, 0.04529292541018279), (3545, 0.1152598130181202), (3573, 0.0350993102976483), (3772, 0.07572779794864065), (3837, 0.031107205780674847), (3900, 0.016837797833000746), (3959, 0.045966374568349004), (3977, 0.16166934901799643), (3979, 0.044158839349194186), (4084, 0.05582117598028138), (4143, 0.024356596014313305), (4169, 0.027426405149832456), (4254, 0.08180680572024529), (4287, 0.0826762009406576), (4290, 0.05100085546711966), (4299, 0.0548531252353318), (4321, 0.01377265666211892), (4353, 0.07731734289396043), (4413, 0.08332678724121464), (4458, 0.16799886294018573), (4494, 0.09238937842788217), (4526, 0.025614552025296153), (4566, 0.04712571410619459), (4594, 0.03523566899198956), (4598, 0.021639583917461673), (4618, 0.029907145380514498), (4779, 0.039080838532664836), (4784, 0.043173299749448654), (4822, 0.10721493744459044), (4829, 0.08026826749688563), (4876, 0.07528191166768487), (4906, 0.04326089858702295), (4965, 0.03845090929225731), (4969, 0.07222792755132884), (5078, 0.07130038343266835), (5112, 0.04509935212375315), (5199, 0.06488376716694892), (5262, 0.0322833191915891), (5284, 0.04388516460362112), (5299, 0.06924602200372969), (5315, 0.0291668753560378), (5376, 0.055987211296320415), (5397, 0.036459770029254275), (5513, 0.040212063464203196), (5533, 0.051930040177150874), (5633, 0.04810977219729793), (5702, 0.09238937842788217), (5800, 0.07863657279741554), (5860, 0.06487071810072297), (5992, 0.03208239489752419), (6006, 0.04308608596295253), (6020, 0.05108506683632286), (6031, 0.031933025776905564), (6033, 0.07174102317310797), (6035, 0.023462850559021208), (6041, 0.0808081381242002), (6053, 0.04198587083602046), (6115, 0.06829346606978345), (6155, 0.019834603361224553), (6220, 0.09991887166173007), (6279, 0.15523771436855552), (6346, 0.031012272085297846), (6373, 0.09707959287168128), (6406, 0.05363174864867619), (6540, 0.08616606603126344), (6544, 0.03238454953608691), (6635, 0.06411562848582601), (6644, 0.05074317516375204), (6746, 0.026424505483711017), (6751, 0.04381421696052577), (6780, 0.10726349729735238), (6809, 0.105124468445211), (6877, 0.050237641970205096), (6881, 0.12394998463634806), (6897, 0.0534842304636885), (6959, 0.026015396051989874), (6970, 0.05048879854510776), (6984, 0.05517058967972435), (6996, 0.042068370349814754), (7067, 0.05924121148375677), (7083, 0.0913716628147444), (7091, 0.0520664056931873), (7130, 0.044907649139863545), (7156, 0.0913716628147444), (7173, 0.018454928309782563), (7264, 0.027268935240081765), (7326, 0.08514140724654133), (7385, 0.03139476305281453), (7485, 0.05065606580439642), (7492, 0.08813051791809741), (7493, 0.07166445061012744), (7508, 0.060666313387837206), (7577, 0.030314388731398953), (7664, 0.020243151175427757), (7682, 0.0318340674446798), (7737, 0.029464245194000562), (7752, 0.11220129327379119), (7805, 0.03581011243951182), (7842, 0.035041210389422775), (7888, 0.09346213181412381), (7893, 0.01651591885872429), (8087, 0.036990369533285404), (8102, 0.08962497556974573), (8160, 0.05924121148375677), (8220, 0.10123115839360085), (8221, 0.02774514460656239), (8243, 0.08943556132484523), (8274, 0.04310240937356585), (8306, 0.029773234155786064), (8356, 0.04478360156738431), (8540, 0.09238937842788217), (8624, 0.06109302025262445), (8668, 0.06130986508293805), (8672, 0.018205037040833053), (8750, 0.057547577802201724), (8823, 0.11887727407567765), (8835, 0.061750796661787566), (8856, 0.0797093261836572), (8982, 0.03115484350939692), (9005, 0.051130961536482285), (9017, 0.04198587083602046), (9053, 0.04274101457869298), (9105, 0.20301401477530717), (9143, 0.048335241719980056), (9201, 0.08220063920973036)]\n"
       ]
      }
     ],
     "prompt_number": 105
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "lsi = models.LsiModel(corpus, id2word=dictionary, num_topics=20) "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 118
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "lsi.print_topics(5)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 120,
       "text": [
        "['0.125*\"well\" + 0.124*\"thing\" + 0.120*\"realli\" + 0.117*\"know\" + 0.116*\"could\" + 0.113*\"year\" + 0.111*\"perform\" + 0.110*\"peopl\" + 0.109*\"plot\" + 0.109*\"littl\"',\n",
        " '-0.699*\"alien\" + -0.210*\"star\" + 0.157*\"love\" + -0.143*\"action\" + -0.140*\"seri\" + -0.138*\"effect\" + 0.137*\"life\" + -0.116*\"planet\" + -0.113*\"trek\" + -0.108*\"special\"',\n",
        " '-0.331*\"life\" + -0.273*\"alien\" + 0.194*\"star\" + 0.194*\"jacki\" + 0.189*\"plot\" + -0.156*\"world\" + 0.149*\"realli\" + -0.142*\"anim\" + -0.134*\"love\" + 0.131*\"action\"',\n",
        " '0.444*\"star\" + -0.421*\"alien\" + 0.203*\"war\" + 0.200*\"trek\" + 0.181*\"effect\" + 0.166*\"anim\" + 0.142*\"special\" + -0.138*\"thing\" + -0.132*\"realli\" + 0.128*\"disney\"',\n",
        " '-0.292*\"jacki\" + -0.266*\"perform\" + 0.212*\"know\" + -0.189*\"action\" + 0.182*\"anim\" + -0.164*\"role\" + 0.149*\"realli\" + 0.148*\"disney\" + 0.145*\"peopl\" + 0.144*\"horror\"']"
       ]
      }
     ],
     "prompt_number": 120
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "lsi_corpus = lsi[corpus]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 121
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for doc in lsi_corpus:\n",
      "    print doc\n",
      "    break"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(0, 7.7194011926300252), (1, 0.6529465744999059), (2, 1.248558031997143), (3, -1.7252988776312437), (4, 2.5462866349529332), (5, -0.50825679782313038), (6, 0.72239397757338786), (7, 0.87968837656580767), (8, 0.25308329994781548), (9, -0.2072915015975603), (10, 1.1457960802818181), (11, -0.23647907672736168), (12, 0.22755479304609985), (13, -0.55646554449801755), (14, 0.87429702330756542), (15, -0.87738785361778193), (16, 0.56706417881093385), (17, -0.86872005432337274), (18, 0.55197691426923179), (19, 0.55460152214544656)]\n"
       ]
      }
     ],
     "prompt_number": 123
    }
   ],
   "metadata": {}
  }
 ]
}

requirements.txt

BeautifulSoup==3.2.1
Flask==0.9
Flask-Login==0.1.2
Flask-Mail==0.7.4
Flask-Principal==0.3.3
Flask-SQLAlchemy==0.16
Flask-Script==0.5.3
Flask-Security==1.5.4
Flask-Testing==0.4
Flask-WTF==0.8.2
Jinja2==2.6
Pygments==1.5
SQLAlchemy==0.8.0b2
Twisted==13.1.0
WTForms==1.0.3
Werkzeug==0.8.3
amqp==1.0.6
anyjson==0.3.3
argparse==1.2.1
billiard==2.7.3.19
blinker==1.2
celery==3.0.13
celery-with-redis==3.0
certifi==0.0.8
chardet==1.0.1
csvkit==0.4.3
distribute==0.6.28
git-remote-helpers==0.1.0
honcho==0.2.0
ipython==0.13.1
itsdangerous==0.17
jrnl==0.3.0
kombu==2.5.4
nose==1.2.1
oauthlib==0.1.3
openpyxl==1.5.7
parsedatetime==0.8.7
passlib==1.6.1
psycopg2==2.4.6
py-bcrypt==0.2
pyasn1==0.1.3
python-dateutil==1.5
redis==2.7.2
requests==0.13.1
requests-cache==0.1.3
rsa==3.1.1
simplejson==2.6.2
twill==0.9
vboxapi==1.0
viff==1.0
virtualenv==1.7.1.2
virtualenvwrapper==3.0.1
wsgiref==0.1.2
xlrd==0.7.1
zope.interface==4.0.5