DocumentClassifier.ipynb

{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "##Classifier code"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import requests\n",
      "from collections import defaultdict\n",
      "\n",
      "class Classifier(object):\n",
      "    \n",
      "    def __init__(self, mltUrl=\"http://localhost:8983/solr/mlt\",\n",
      "                idField=\"Id\",\n",
      "                mltFields=\"Title Body\",\n",
      "                tagField=\"Tags\",\n",
      "                filterQuery=\"Tags:*\",\n",
      "                numNearest=10):\n",
      "        self.mltUrl = mltUrl\n",
      "        self.idField = idField\n",
      "        self.mltFields = mltFields if type(mltFields) == str  else \" \".join(mltFields)\n",
      "        self.tagField = tagField\n",
      "        self.filterQuery = filterQuery\n",
      "        self.numNearest = numNearest\n",
      "        self.sess = requests.Session()\n",
      "        \n",
      "        \n",
      "    def classifyDoc(self,docId,\n",
      "                  method=\"best\" #or \"sorted\" or \"details\"\n",
      "                ):\n",
      "        #send the MLT query to Solr\n",
      "        params = {\"q\": self.idField + \":\" + docId,\n",
      "                  \"mlt.fl\": self.mltFields,\n",
      "                  \"fl\": self.tagField,\n",
      "                  \"fq\": self.filterQuery,\n",
      "                  \"rows\": self.numNearest,\n",
      "                  \"wt\":\"json\"\n",
      "                  }\n",
      "        resp = sess.get(url=self.mltUrl,params=params)\n",
      "        \n",
      "        #Perform error checking\n",
      "        if resp.status_code != 200:\n",
      "            raise IOError(\"HTTP Status \" + str(resp.status_code))\n",
      "        json = resp.json()\n",
      "        if int(json[\"match\"][\"numFound\"]) == 0:\n",
      "            raise RuntimeError(\"no document with that id\")\n",
      "        if int(json[\"response\"][\"numFound\"]) == 0:\n",
      "            raise RuntimeError(\"no interesting terms in document\")  \n",
      "            \n",
      "        #If no errors, then collect and count tags for each similar document\n",
      "        tagDict = defaultdict(int)\n",
      "        for tagList in json[\"response\"][\"docs\"] :\n",
      "            for tag in tagList[self.tagField].split(' '):\n",
      "                tagDict[tag] += 1\n",
      "                \n",
      "        #Return the best tag, all of the tags sorted best \n",
      "        #to worst, or the list of tags and their count\n",
      "        if method == \"best\":\n",
      "            return max(tagDict, key=tagDict.get)\n",
      "        elif method == \"sorted\":\n",
      "            return sorted(tagDict, key=lambda x : tagDict[x], reverse=True)\n",
      "        elif method == \"details\":\n",
      "            return tagDict\n",
      "        "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 273
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "##Build a classifier"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "c = Classifier(mltUrl=\"http://localhost:8983/solr/mlt\",\n",
      "                idField=\"Id\",\n",
      "                mltFields=\"Title Body\",\n",
      "                tagField=\"Tags\",\n",
      "                filterQuery=\"Tags:*\",\n",
      "                numNearest=10)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 274
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "And as an example, use it to find the plausible tags for a document."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print c.classifyDoc(\"8723\",method=\"sorted\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[u'star-trek', u'holodeck', u'star-trek-tng', u'technology', u'magical-transportation', u'harry-potter', u'weapon', u'time-travel', u'diagon-alley']\n"
       ]
      }
     ],
     "prompt_number": 292
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "##Test the accuracy of the classifier code\n",
      "\n",
      "First create a function that tests the classifier."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def classifierTester(q=\"*:*\"):\n",
      "    sess = requests.Session()\n",
      "    #retrieve all documents that have Tags and match the q argument\n",
      "    resp = sess.get(url=\"http://localhost:8983/solr/select\",params={\"q\":q,\"fq\":\"Tags:*\",\"fl\":\"Id Tags\",\"rows\":\"9999999\",\"wt\":\"json\"})\n",
      "    docs = resp.json()[\"response\"][\"docs\"]\n",
      "    \n",
      "    #classify each document and count the number of matches\n",
      "    count = 0\n",
      "    hitCount = 0\n",
      "    for doc in docs :\n",
      "        count += 1\n",
      "        try:\n",
      "            if c.classifyDoc(doc[\"Id\"]) in doc[\"Tags\"].split(' ') :\n",
      "                hitCount += 1\n",
      "        except Exception:\n",
      "            pass\n",
      "    print \"{0} out of {1} correct. That's {2}%\".format(hitCount,count,100*float(hitCount)/count)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 285
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "    Perform various tests:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#All questions that have tags\n",
      "classifierTester()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "4041 out of 5805 correct. That's 69.6124031008%\n"
       ]
      }
     ],
     "prompt_number": 288
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Tags present in 10 or more questions\n",
      "classifierTester(q=\"Tags:(harry-potter story-identification star-trek star-wars comics movie marvel-comics lord-of-the-rings dc-comics doctor-who futurama star-trek-tng tv x-men time-travel stargate books magic avengers the-matrix aliens novel a-song-of-ice-and-fire george-r-r-martin video-games short-stories batman technology game-of-thrones suggested-order superman stargate-sgstar-trek-voyager voldemort battlestar-galactica dune robots fantasy-genre star-trek-dstolkien plot canon alien-franchise fringe wolverine borg vampire rings-of-power thor horcrux weapon star-trek-tos green-lantern firefly the-walking-dead spider-man cartoon jedi isaac-asimov zombie magical-creatures spaceship languages terminator science powers the-new-star-trek-enterprise the-hunger-games buffy hogwarts dark-knight-rises ftl-drive space tv-series the-legend-of-korra history-of young-adult supernatural spells continuity phantom-menace robert-a-heinlein twilight avatar-the-last-airbender darth-vader prometheus avengers-vs-x-men religion science-fiction-genre the-hobbit hard-sci-fi larry-niven history magical-theory magical-items clones wheel-of-time middle-earth anime character-identification sith iron-man horror super-hero stargate-atlantis known-space enders-game my-little-pony transformers orson-scott-card physics computers races my-little-pony-fim elves warp stargate-universe captain-america frank-herbert john-carter the-force back-to-the-future babylon-klingon music the-incredible-hulk luke-skywalker the-hulk animals warfare neal-stephenson gandalf extended-universe h-p-lovecraft star-trek-q indiana-jones biology inception philip-k-dick tron-legacy inheritance-cycle terra-nova hitchhikers-guide space-exploration farscape character-development werewolf the-clone-wars alternate-history sauron star-trek-data highlander parallel-universe dcau cthulhu-mythos terminology blade-runner vulcan ghost tron christopher-paolini wolverine-and-the-xmen obi-wan-kenobi mass-effect neil-gaiman good-against-evil jk-rowling authors urban-fantasy terry-pratchett names paradox influences yoda han-solo warhammer40k economics alien-c-3po mistborn ringworld discworld online-resources apocalypse timeline star-trek-real-world society snow-crash the-flash quidditch brandon-sanderson eureka arthur-c-clarke)\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "3966 out of 5297 correct. That's 74.8725693789%\n"
       ]
      }
     ],
     "prompt_number": 293
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Tags present in 50 or more questions\n",
      "classifierTester(q=\"Tags:(harry-potter story-identification star-trek star-wars comics movie marvel-comics lord-of-the-rings dc-comics doctor-who futurama star-trek-tng tv x-men time-travel stargate books magic avengers the-matrix aliens novel a-song-of-ice-and-fire george-r-r-martin video-games short-stories batman technology game-of-thrones suggested-order superman stargate-sgstar-trek-voyager voldemort battlestar-galactica dune robots fantasy-genre)\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "3556 out of 4457 correct. That's 79.784608481%\n"
       ]
      }
     ],
     "prompt_number": 294
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Tags present in 100 or more questions\n",
      "classifierTester(q=\"Tags:(harry-potter story-identification star-trek star-wars comics movie marvel-comics lord-of-the-rings dc-comics doctor-who futurama star-trek-tng tv x-men time-travel stargate books magic avengers the-matrix aliens)\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "3323 out of 4042 correct. That's 82.2117763483%\n"
       ]
      }
     ],
     "prompt_number": 296
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Tags present in 500 or more questions\n",
      "classifierTester(q=\"Tags:(harry-potter story-identification star-trek star-wars)\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2136 out of 2344 correct. That's 91.1262798635%\n"
       ]
      }
     ],
     "prompt_number": 297
    }
   ],
   "metadata": {}
  }
 ]
}