Skip to content

Instantly share code, notes, and snippets.

@fnielsen

fnielsen/Nielsen2014Python_case.ipynb

Last active January 27, 2016 19:48
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save fnielsen/e07d388d2d8d4d4e3519 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save fnielsen/e07d388d2d8d4d4e3519 to your computer and use it in GitHub Desktop.
Download ZIP
Nielsen2014Python_case.ipynb
Display the source blob
Display the rendered blob
Raw
Nielsen2014Python_case.ipynb
{
"metadata": {
"name": "",
"signature": "sha256:b715e6a82defdc0fb73551197edeab91474dff90db7d955ecf5565b9f7890524"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Import packages to extended the functionality of basic Python\n",
"\n",
"# Let Python 2 behave like Python 3\n",
"from __future__ import division, unicode_literals, print_function\n",
"\n",
"# Utility data structures\n",
"from collections import Counter\n",
"\n",
"# Plotting\n",
"import matplotlib.pyplot as plt\n",
"import matplotlib.cm as cm\n",
"% matplotlib inline\n",
"\n",
"# Natural language processing package\n",
"import nltk\n",
"import nltk.corpus\n",
"\n",
"# Numerical \n",
"import numpy as np\n",
"\n",
"# Operating system functions\n",
"import os\n",
"\n",
"# Data analysis package\n",
"import pandas as pd\n",
"\n",
"# Kenneth Reitz' module to download data from the Web. (instead of urllib)\n",
"import requests"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 4
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Specify a URL to download information from the Semantic MediaWiki 'WikiLit'\n",
"# Such a query can be constructed on the web site and the resulting URL copy-and-pasted here.\n",
"url = (\"http://wikilit.referata.com/\" \n",
" \"wiki/Special:Ask/\"\n",
" \"-5B-5BCategory:Publications-5D-5D/\" \n",
" \"-3FHas-20author%3DAuthor(s)/-3FYear/\"\n",
" \"-3FPublished-20in/-3FAbstract/-3FHas-20topic%3DTopic(s)/\" \n",
" \"-3FHas-20domain%3DDomain(s)/\" \n",
" \"format%3D-20csv/limit%3D-20600/offset%3D0\")"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 2
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Download and read data as comma-separated values (CSV) information into a Pandas DataFrame\n",
"documents = pd.read_csv(url)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 3
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"filename = os.path.expanduser('~/data/dtu02819/wikilit.csv')"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 6
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Write to a comma-separated values file at the local file system\n",
"documents.to_csv(filename)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 7
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Read the comma-separated values file from the local file system\n",
"documents = pd.read_csv(filename, index_col=0)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 12
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"documents.head()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"html": [
"<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Unnamed: 0.1</th>\n",
" <th>Author(s)</th>\n",
" <th>Year</th>\n",
" <th>Published in</th>\n",
" <th>Abstract</th>\n",
" <th>Topic(s)</th>\n",
" <th>Domain(s)</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td> 'Wikipedia, the free encyclopedia' as a role m...</td>\n",
" <td> Gordon M\u00fcller-Seitz,Guido Reger</td>\n",
" <td> 2010</td>\n",
" <td> International Journal of Technology Management</td>\n",
" <td> Accounts of open source software (OSS) develop...</td>\n",
" <td> Contributor motivation,Policies and governance...</td>\n",
" <td> Information systems</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td> A 'resource review' of Wikipedia</td>\n",
" <td> Cormac Lawler</td>\n",
" <td> 2006</td>\n",
" <td> Counselling &amp; Psychotherapy Research</td>\n",
" <td> The article offers information on Wikipedia, a...</td>\n",
" <td> Miscellaneous topics</td>\n",
" <td> Information systems</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td> A Persian web page classifier applying a combi...</td>\n",
" <td> Mojgan Farhoodi,Alireza Yari,Maryam Mahmoudi</td>\n",
" <td> 2009</td>\n",
" <td> International Journal of Information Studies</td>\n",
" <td> There are many automatic classification method...</td>\n",
" <td> Text classification</td>\n",
" <td> Computer science</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td> A Wikipedia literature review</td>\n",
" <td> Owen S. Martin</td>\n",
" <td> 2010</td>\n",
" <td> ArXiv</td>\n",
" <td> This paper was originally designed as a litera...</td>\n",
" <td> Literature review</td>\n",
" <td> Mathematics</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td> A Wikipedia matching approach to contextual ad...</td>\n",
" <td> Alexander N. Pak,Chin-Wan Chung</td>\n",
" <td> 2010</td>\n",
" <td> World Wide Web</td>\n",
" <td> Contextual advertising is an important part of...</td>\n",
" <td> Other information retrieval topics</td>\n",
" <td> Computer science</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"metadata": {},
"output_type": "pyout",
"prompt_number": 14,
"text": [
" Unnamed: 0.1 \\\n",
"0 'Wikipedia, the free encyclopedia' as a role m... \n",
"1 A 'resource review' of Wikipedia \n",
"2 A Persian web page classifier applying a combi... \n",
"3 A Wikipedia literature review \n",
"4 A Wikipedia matching approach to contextual ad... \n",
"\n",
" Author(s) Year \\\n",
"0 Gordon M\u00fcller-Seitz,Guido Reger 2010 \n",
"1 Cormac Lawler 2006 \n",
"2 Mojgan Farhoodi,Alireza Yari,Maryam Mahmoudi 2009 \n",
"3 Owen S. Martin 2010 \n",
"4 Alexander N. Pak,Chin-Wan Chung 2010 \n",
"\n",
" Published in \\\n",
"0 International Journal of Technology Management \n",
"1 Counselling & Psychotherapy Research \n",
"2 International Journal of Information Studies \n",
"3 ArXiv \n",
"4 World Wide Web \n",
"\n",
" Abstract \\\n",
"0 Accounts of open source software (OSS) develop... \n",
"1 The article offers information on Wikipedia, a... \n",
"2 There are many automatic classification method... \n",
"3 This paper was originally designed as a litera... \n",
"4 Contextual advertising is an important part of... \n",
"\n",
" Topic(s) Domain(s) \n",
"0 Contributor motivation,Policies and governance... Information systems \n",
"1 Miscellaneous topics Information systems \n",
"2 Text classification Computer science \n",
"3 Literature review Mathematics \n",
"4 Other information retrieval topics Computer science "
]
}
],
"prompt_number": 14
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Example on word tokenization of the first sentence in the first abstract\n",
"sentences = nltk.sent_tokenize(documents.ix[0, 'Abstract'])\n",
"\n",
"# Show the sentences as a Python list of strings\n",
"print(nltk.word_tokenize(sentences[0]))"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"['Accounts', 'of', 'open', 'source', 'software', '(', 'OSS', ')', 'development', 'projects', 'frequently', 'stress', 'their', 'democratic', ',', 'sometimes', 'even', 'anarchic', 'nature', ',', 'in', 'contrast', 'to', 'for-profit', 'organisations', '.']\n"
]
}
],
"prompt_number": 16
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Tokenize all the text and count tokens\n",
"\n",
"# A extra attribute to contain the new data in the documents object\n",
"documents.data = []\n",
"\n",
"# Token counter\n",
"token_counts = Counter()\n",
"\n",
"# Iterate over all documents and all sentence and all words\n",
"for abstract in documents['Abstract']:\n",
" datum = {}\n",
" datum['sentences'] = nltk.sent_tokenize(abstract)\n",
" datum['tokenlist'] = [word.lower() for sent in datum['sentences'] \n",
" for word in nltk.word_tokenize(sent)]\n",
" token_counts.update(datum['tokenlist'])\n",
" documents.data.append(datum)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 17
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# The five most common tokens in the entire WikiLit corpus\n",
"token_counts.most_common(5)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 18,
"text": [
"[('the', 4866), ('of', 3826), (',', 3785), ('.', 3585), ('and', 2856)]"
]
}
],
"prompt_number": 18
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Read stopword list ('the', 'a', 'for')\n",
"stopwords = nltk.corpus.stopwords.words('english')\n",
"relevant_tokens = {token: count for token, count in token_counts.items()\n",
" if count > 2 and token not in stopwords and token.isalpha()}"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 19
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Show the most common tokens in the reduced token set\n",
"Counter(relevant_tokens).most_common(5)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 20,
"text": [
"[('wikipedia', 1428),\n",
" ('information', 486),\n",
" ('knowledge', 341),\n",
" ('articles', 279),\n",
" ('online', 265)]"
]
}
],
"prompt_number": 20
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Exclude the work 'wikipedia'\n",
"relevant_tokens.pop('wikipedia', 0)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 21,
"text": [
"1428"
]
}
],
"prompt_number": 21
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Construct a dense document-term matrix with word counts in the elements\n",
"# as a Numpy matrix\n",
"tokens = relevant_tokens.keys() # as list\n",
"M = np.asmatrix(np.zeros([len(documents), len(tokens)]))\n",
"for n in range(len(documents)):\n",
" for m, token in enumerate(tokens):\n",
" M[n, m] = documents.data[n]['tokenlist'].count(token)\n",
"M.shape"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 22,
"text": [
"(525, 2899)"
]
}
],
"prompt_number": 22
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Value of the element in the first row in the column corresponding to the word 'software'.\n",
"M[0, tokens.index('software')]"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 23,
"text": [
"1.0"
]
}
],
"prompt_number": 23
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Plot part of the matrix as an image\n",
"plt.imshow(M[:100, :100], cmap=cm.gray_r, interpolation='nearest')\n",
"plt.xlabel('Tokens')\n",
"plt.ylabel('Documents')\n",
"plt.show()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "display_data",
"png": "iVBORw0KGgoAAAANSUhEUgAAAQoAAAEJCAYAAABsX2lnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGd5JREFUeJzt3XtQVOf5B/DvCsaaxgRUWEYxv6UqICAIap1xbFwki8UG\ng5qodSTrrZexNZpkai/TFGinAuM4wUydtukY3KEZjG0zlsl4SYnFmqoxFiSpsSGjbKHIYiuQaESB\n5f39kWHDyuXAnrN73rP7/cw47i67nOfsLs953st5j0kIIUBENIJxegdARPJjoiAiRUwURKSIiYKI\nFDFREJEiJgoiUhTQRHHixAkkJiZi9uzZKC0tDeSmiUgFU6DmUbjdbiQkJKC6uhrTp0/HwoULUVlZ\niTlz5gRi80SkQnigNnThwgXMmjULFosFALB+/Xr8+c9/9koUJpMpUOEQ0X1GqhkClihaWlowY8YM\nz/3Y2Fi8++67g563dOlSWK1WAIDVavXclllhYSEKCwv1DmPUAhGv0+kEAM+Bwdfn9NPrPe6Psf//\n0X4fZf9O1NTUoKamxnO/qKhoxOcHLFGMtlqwWq1Sv8G+GssfRTAYzX4a4b3oj9EIsY7F/QdhpUQR\nsM7M6dOno7m52XO/ubkZsbGxgdo8EakQsIpiwYIF+Pjjj+F0OjFt2jS8/vrrqKysHPQ8IzQ17jea\nmGU6IgXrezySQFd0Mr/HA5sco40zYKMeAHD8+HHs2rULbrcbW7duxY9//GPvYEymETtUiHwVak2/\nkQyVKJT+9gJWUQBATk4OcnJyArlJItJAQCsKJawoiPSh9LfHKdxEpCigTQ899bdRAbZTg5UvnXQy\nkrE/hRUFESkyVB+FjJlWFqyYjEmW7zT7KIhINUNVFEQ0NLWVCSsKIlKNiYKIFLHpQQHj7467kTp0\nZek09DdfO7XZ9CAi1UKmotBr+DBUjmT3U7vfvryeQ8S+Y0VBRKqFTEVBwU/visLI1SMrCiJSjRUF\nkY4OHToEANi0adOgnwWyQmFFQUSqMVEQkaKgaHro3YlFZHRsehCRakFRURiBkYfOjMLXylLPz0aW\n7wUrCiJSLaQrivuzOfs6KFSxoiAi1YJ+Fe6R2oCsGgJLq/a4LO36UMKKgogUMVEQkSJDdWbquUKS\nEelRorNZMDb9Fy3S+4JF7MwkItWkrSj0PBoGertEgTTUpRdZURCRatJWFEQUOKwoiEg1Q0+4CpUe\ndvadhDYZvuesKIhIkeaJorm5GZmZmUhOTkZKSgpefvllAEB7eztsNhvi4+ORnZ2Nzs5OrTdNRH6i\neWemy+WCy+XCvHnzcPv2bcyfPx9Hjx5FeXk5pk6dit27d6O0tBQdHR0oKSnxDoadmV5kKDkpNAS8\nMzMmJgbz5s0DADz00EOYM2cOWlpaUFVVBbvdDgCw2+04evSo1psmIj/xa2em0+lEXV0dFi1ahLa2\nNpjNZgCA2WxGW1vbkK8pLCz03LZarbpPbdUTK4nRkWGFKr227wuTyeS5XVBQMLrX+Gsexe3bt7F0\n6VK8+OKLyMvLQ2RkJDo6Ojw/nzx5Mtrb272DYdODfMBEMTYDE0X/35vS355fKoqenh6sWbMG+fn5\nyMvLA/B5FeFyuRATE4PW1lZER0f7Y9OqGPFDJ31XKPPHdtQkvtG8B74cjDXvoxBCYOvWrUhKSsKu\nXbs8j69cuRIOhwMA4HA4PAmEiOSnedPjnXfewWOPPYbU1FRPiVNcXIyvfvWrWLt2LZqammCxWHDk\nyBFERER4ByNR04MjDvrw99GUhhbwpseSJUvQ19c35M+qq6u13hwRBQBnZhKRIp49SlJi0y+wePYo\nEalm6IoiVI467KTTjpHey0B+v1lREJFqhq4oZGWko5YvZN2/Q4cOeW5v2rRp1K8L5JH7/m1xFW4i\nChqsKIiIFQURqSdtonA6nV5tYb1/j2zborHhZ6OOtImCiOTBREFEitiZSRTC+ptjcXFx7MwkInUM\nfQEgIho7XybMsaIgIkUh00ch67TjYBEqJ+hpjVO4iShohEwfBY90/uWv93cslYoRq0ajTAJjRUFE\nipgoiEhR0HdmytLJJkscRENhZyYRqRb0FQVRKOkfbgXGNuTKioKIVGNFQcMKVL+KEYc1tSJL3xUr\nCiJSjRWFCrIcDcj/gv2zZkVBRKoxURCRIjY9SDdal/NG7BQd6T3gJQWJyFBYUZBfBPtZn/6kR8cp\nKwoiUk2xorh9+zYmTpyIsLAwfPTRR/joo4+Qk5OD8ePHax8MKwpNBPtQ3v202t9Qe98GUl1RPPbY\nY7h37x5aWlqwfPlyVFRUjOlK0URkfIqJQgiBBx98EG+88Qa2b9+OP/zhD/jnP/+p+IvdbjfS09OR\nm5sLAGhvb4fNZkN8fDyys7PR2dmpPnoaksViMexRsaamxuvEptHQan9H83u0vjShUS51OKo+inPn\nzuG1117DN77xDQBAX1+f4mv279+PpKQkmEwmAEBJSQlsNhsaGhqQlZWFkpISFWETUSApJoqysjIU\nFxdj1apVSE5OxtWrV5GZmTnia/7zn//g2LFj2LZtm6fdU1VVBbvdDgCw2+04evSoBuETUSAoLq7b\n1taGqqoqz/2ZM2diyZIlI77mueeew969e/Hpp596/R6z2QwAMJvNaGtrG/K1hYWFnttWq1X3Zcz9\nJZQ7zkYiy/L1w9H689Lr8x9rE08xURQXF2Pt2rWKj/V78803ER0djfT09GEDMZlMnibJ/QYmCiLy\nj/sPwkVFRSM+f9hEcfz4cRw7dgwtLS149tlnPU2IW7dujTg0evbsWVRVVeHYsWO4e/cuPv30U+Tn\n58NsNsPlciEmJgatra2Ijo4e464FF1YSQ/PXgUKPCk726mgshu2jmDZtGubPn48vfelLmD9/vuff\nypUrcfLkyWF/4Z49e9Dc3IzGxkYcPnwYy5YtQ0VFBVauXAmHwwEAcDgcyMvL035viMgvFCdc9fT0\n+Dy56vTp09i3bx+qqqrQ3t6OtWvXoqmpCRaLBUeOHEFERIR3MJxwRaSq+vHXmpmKieKdd95BUVER\nnE4nent7Pb/02rVrow5itJgoiAyaKBISElBWVoaMjAyEhYV5Hp86deqogxgtJgoKNbKMfin97SmO\nekRERCAnJ0fToIjIWBQTRWZmJn7wgx9g9erVmDBhgufxjIwMvwZGRPJQbHpYrdYh5zz89a9/1T4Y\nNj2IdKG6jyKQmCiI9KH6NHOXy4WtW7fi61//OgDgww8/xMGDB7WLUCL9Z/IZ4Ww+okBSTBSbNm1C\ndnY2rl+/DgCYPXs2XnrpJb8HRkTyUOzM/N///od169Z5TgsfP348wsMVXxYQY5kiO5p1GX0dopJl\niEsmerwneq+9KfuU7UOHDgGATwtPKVYUDz30EG7evOm5f/78eTzyyCNj3hARGZdiZ+Y//vEP7Nix\nA5cvX0ZycjL++9//4o9//CPS0tK0DyYEOzP1PgrKTpZqTetqQZb96qd6wtX8+fNx+vRpNDQ0QAiB\nhIQEvyysS0TyUkwUvb29OHbsmOdcj5MnT8JkMuH5558PRHxEJAHFpkdOTg4mTpyIuXPnYty4L7o0\nCgoKtA8mSJoebE6Q0ahuerS0tOD999/XNCgiMhbFUY/s7OwRF6qhwfqXfWc1QUqMMsFPsaJYvHgx\nVq1ahb6+Pk8npslk8lo4l4iCm2IfhcViQVVVFVJSUrz6KPwSTJD0UQxFtuEw0oes3wPV53o8+uij\nSE5O9nuSICJ5KVYUdrsdjY2NyMnJwQMPPPD5i/w0PBrMFYUaHEUhf1M96hEXF4e4uDh0d3eju7sb\nQohhr8lBRMGJ61GQ4cjazjcy1RXFUNcZNZlMOHXqlLrIiMgwFBPF3r17Pbfv3r2LP/3pT9KcZk5E\ngeFT02PhwoV47733tA+GTY+gxiaD//nruh6KpUF7e7vndl9fHy5evMjJVkQhZlQTrvpHOcLDw2Gx\nWFBQUIAlS5ZoHwwrClLB16OpbPxdeQ013K66ojDCPHQi8i/FiuLAgQPYsGEDIiMjAQAdHR2orKzE\n9u3btQ+GFQWRLlRf1yMtLQ319fVej82bNw+XLl3SJsKBwTBREOlCddOjr68PfX19nnM93G43enp6\ntIvQYDidmkKRYqJYvnw51q9fj+985zsQQuC3v/2t52JARBQaFJsebrcbr7zyCt5++20AgM1mw7Zt\n2xAWFqZ9MAZoerCioGCkybVH7927h4aGBgBAYmKi31bhNkKiMJL7R6xkS2ycgCUP1X0UNTU1sNvt\n+L//+z8AQFNTExwOB5YuXapdlEQkNcWKIiMjA5WVlUhISAAANDQ0YP369aitrdU+GD9UFGouo0bG\n0v9ZA/y8x0r1Cle9vb2eJAEA8fHx6O3t1SY6IjIExYpi8+bNCAsLw8aNGyGEwGuvvYa+vj68+uqr\nw76ms7MT27Ztw+XLl2EymVBeXo7Zs2dj3bp1+Pe//w2LxYIjR44gIiLCOxj2UdB92I8RGKo7M+/e\nvYsDBw7g73//OwDga1/7GrZv344JEyYM+xq73Y6lS5diy5Yt6O3txWeffYZf/vKXmDp1Knbv3o3S\n0lJ0dHR4rpA+2mAp9DBRBIYmox43btwAAERHRytu8JNPPkF6ejquXbvm9XhiYiJOnz4Ns9kMl8sF\nq9WKf/3rX2MKlsjoZE18Po96CCFQVFSEX/3qV3C73QCAsLAw7NixAz/72c+GXTezsbERUVFR2Lx5\nM+rr6zF//nyUlZWhra0NZrMZAGA2m9HW1jbk6wsLCz23rVaroc8CJJJVTU2N19m2isQw9u3bJx5/\n/HFx7do1z2NXr14VNptN7Nu3b7iXiffee0+Eh4eLCxcuCCGE2Llzp/jpT38qIiIivJ4XGRk56LUj\nhEMUFBobG0VjY6PeYQyi9Lc37E/T0tLEjRs3Bj1+48YNkZaWNuwvbG1tFRaLxXP/zJkzYsWKFSIx\nMVG0trYKIYS4fv26SEhIGHOwsikvLxfl5eVjeo2sXxQKbUp/e8MOj/b29iIqKmrQ41FRUSMOj8bE\nxGDGjBmemZzV1dVITk5Gbm4uHA4HAMDhcCAvL2/0ZQ8R6WrYzsz09HTU1dUN+aKRfgYA9fX12LZt\nG7q7uzFz5kyUl5fD7XZj7dq1aGpq4vAokWR8HvUICwvDgw8+OOSLurq6/DLpiomCSB8+j3r0j3QQ\nEfHKw0SkiFfyoaCh91ohvkymknUC1v1YURCRIiYKIlLEpgcFDT2bG75uf6Smh0zNElYURKTIp4sU\n+4vJZEJjYyMAObIoGYvenZlGpnqFKyIi6SoKicKhICJTe3+g/lO9h1pOIZAxs6IgItVYUWhgpKMC\nkRGwoiAi1ZgoiEgRmx5ExKYHEanHREGacDqdgy6KTL6R8b1koiAiRTwpjDSh9QlRvuAU7tEZeO2c\ngbdHwoqCiBSxohjAiEcko5ymPBSt45J1P8fK3/vhy+9nRUFEipgoiEgRJ1wNYMSmB5EWOOGKiFRj\nRaGzQ4cOeW5v2rRJtzhIeyN1Jvd/1v3Dk3pXsKwoiEg1VhRkCKPpP2Ifk+9YURCRaiFTUfBoQzQ8\nVhREpFrIVBREgaZ2LVWuwk1EhsJEQUSK2PQgIn2aHsXFxUhOTsbcuXOxYcMG3Lt3D+3t7bDZbIiP\nj0d2djY6Ozv9sWki8gPNKwqn04lly5bhypUrmDBhAtatW4cVK1bg8uXLmDp1Knbv3o3S0lJ0dHSg\npKTEO5gQrCg4bEu+UNPR2d/JCnzR0RrwiuLhhx/G+PHjcefOHfT29uLOnTuYNm0aqqqqYLfbAQB2\nux1Hjx7VetNE5Cd+6aN45ZVX8MILL2DixIlYvnw5KioqEBkZiY6ODgCAEAKTJ0/23PcEYzKhoKDA\nc99qtfIyfUR+UFNT41VZFBUVjVhRaJ4orl69itzcXJw5cwaPPPIInn76aaxZswY7duzwSgyTJ09G\ne3u7dzAh2PQgkkHAmx4XL17E4sWLMWXKFISHh2P16tU4d+4cYmJi4HK5AACtra2Ijo7WetNEhlFY\nWDjqFbBloHmiSExMxPnz59HV1QUhBKqrq5GUlITc3Fw4HA4AgMPhQF5entabJiI/0XwV7rS0NDzz\nzDNYsGABxo0bh4yMDHz729/GrVu3sHbtWhw8eBAWiwVHjhzRetNE5CeccEWkEV/O7ehf4Uzv1c14\nrgcRqRb0FYXsF8Eh7XDymu9YURCRarykIAFQv3aCDPxRRfi7IjXK+86KgogUBX0fxViwjUvDCfa+\nLvZREJFqTBREpMhQTY9gL/9IHVmbjrLGNRCbHkSkmqGGR2XNxr4wyrCYPxmxQhxYHQCji92f+xeo\n95AVBREpMlQfhZEZ8eipRqjtr9Gxj4KIVGNF4QdG6OWmL2hV/fjye2SpvFhREJFqTBREpIhNDwMY\na1NGlnI20IzY5JNlmJxNDyJSjRVFgBjxKG/EmMk3rCiISDVpKwpZVieWXSCP+mNpT48mLlYs8mBF\nQUSqSVtRUGCFytFdtpERjnoQUdBgoiAiRWx6kF/o0ZTxZ7PCyOdx3G+o94lNDyJSjRWFwcl61CJj\nYUVBRKqxohij/uEsQP8hLfIm29CnkbCiICLVDLUKtwxYRRgL+3C0wYqCiBRJlygG9gEYhdFiNlq8\ngPFiNlq8SpgoNGC0mMcab01NjV/20el0DrqgzkgxKCksLPT862exWHRpdhjtO6FEukRBRPJhZyYp\n8lcHrtZH+v41TEh70s2jICJ9jJQKpKooJMpZRDQA+yiISBETBREpYqIgIkVSJYoTJ04gMTERs2fP\nRmlpqd7hDNLc3IzMzEwkJycjJSUFL7/8MgCgvb0dNpsN8fHxyM7ORmdnp86RDuZ2u5Geno7c3FwA\n8sfc2dmJp556CnPmzEFSUhLeffddqWMuLi5GcnIy5s6diw0bNuDevXtSxztW0iQKt9uN73//+zhx\n4gQ+/PBDVFZW4sqVK3qH5WX8+PF46aWXcPnyZZw/fx4HDhzAlStXUFJSApvNhoaGBmRlZaGkpETv\nUAfZv38/kpKSPCNLsse8c+dOrFixAleuXMH777+PxMREaWN2Op343e9+h9raWnzwwQdwu904fPiw\ntPH6REji7NmzYvny5Z77xcXFori4WMeIlD355JPiL3/5i0hISBAul0sIIURra6tISEjQOTJvzc3N\nIisrS5w6dUo88cQTQgghdcydnZ0iLi5u0OOyxnzz5k0RHx8v2tvbRU9Pj3jiiSfEW2+9JW28vpCm\nomhpacGMGTM892NjY9HS0qJjRCNzOp2oq6vDokWL0NbWBrPZDAAwm81oa2vTOTpvzz33HPbu3Ytx\n4774uGWOubGxEVFRUdi8eTMyMjLwrW99C5999pm0MU+ePBkvvPACHn30UUybNg0RERGw2WzSxusL\naRKFkSZb3b59G2vWrMH+/fsxadIkr5+ZTCap9uXNN99EdHQ00tPTh52nIlvMvb29qK2txfbt21Fb\nW4svf/nLg8p2mWK+evUqysrK4HQ6cf36ddy+fRu///3vvZ4jU7y+kCZRTJ8+Hc3NzZ77zc3NiI2N\n1TGiofX09GDNmjXIz89HXl4egM+PFi6XCwDQ2tqK6OhoPUP0cvbsWVRVVSEuLg7f/OY3cerUKeTn\n50sdc2xsLGJjY7Fw4UIAwFNPPYXa2lrExMRIGfPFixexePFiTJkyBeHh4Vi9ejXOnTsnbby+kCZR\nLFiwAB9//DGcTie6u7vx+uuvY+XKlXqH5UUIga1btyIpKQm7du3yPL5y5Uo4HA4AgMPh8CQQGezZ\nswfNzc1obGzE4cOHsWzZMlRUVEgdc0xMDGbMmIGGhgYAQHV1NZKTk5GbmytlzImJiTh//jy6urog\nhEB1dTWSkpKkjdcnOveReDl27JiIj48XM2fOFHv27NE7nEHOnDkjTCaTSEtLE/PmzRPz5s0Tx48f\nFzdv3hRZWVli9uzZwmaziY6ODr1DHVJNTY3Izc0VQgjpY7506ZJYsGCBSE1NFatWrRKdnZ1Sx1xa\nWiqSkpJESkqKeOaZZ0R3d7fU8Y6VVCeFEZGcpGl6EJG8mCiISBETBREpYqIgIkVSLVxD+rl58yYe\nf/xxAIDL5UJYWBiioqJgMplw4cIFhId/8VWxWCyora3F5MmT9QqXAoyJggAAU6ZMQV1dHQCgqKgI\nkyZNwvPPPz/kc408w5B8w6YHDUkIgbfffhvp6elITU3F1q1b0d3d7fWcrq4u5OTk4ODBg7hz5w62\nbNmCRYsWISMjA1VVVQA+X/B29erVyMnJQXx8PH74wx8C+Pxs4U2bNmHu3LlITU1FWVlZwPeRRo8V\nBQ3p7t272Lx5M06dOoVZs2bBbrfj17/+NXbu3AkAuHXrFtatWwe73Y6NGzfiJz/5CbKysvDqq6+i\ns7MTixYt8jRl6uvrcenSJTzwwANISEjAjh070NbWhuvXr+ODDz4AAHzyySe67SspY0VBQ3K73fjK\nV76CWbNmAQDsdjv+9re/Afi82njyySexZcsWbNy4EQDw1ltvoaSkBOnp6cjMzMS9e/fQ1NQEk8mE\nrKwsTJo0CRMmTEBSUhKampowc+ZMXLt2Dc8++yxOnjyJhx9+WLd9JWVMFDSsgZN2B942mUxYsmQJ\njh8/7vX8N954A3V1dairq4PT6URiYiIAYMKECZ7nhIWFobe3FxEREaivr4fVasVvfvMbbNu2zc97\nQ2owUdCQwsLC4HQ6cfXqVQBARUWF14WAfv7znyMyMhLf+973AADLly/3LA0IwNMxOtQZAkII3Lx5\nE263G6tXr8YvfvEL1NbW+nFvSC0mChrSxIkTUV5ejqeffhqpqakIDw/Hd7/7XQBfjHrs378fXV1d\n+NGPfoQXX3wRPT09SE1NRUpKCgoKCjzPvX+UxGQyoaWlBZmZmUhPT0d+fr6xl4kLATwpjIgUsaIg\nIkVMFESkiImCiBQxURCRIiYKIlLEREFEiv4fm1zMoGsjTjIAAAAASUVORK5CYII=\n",
"text": [
"<matplotlib.figure.Figure at 0x4bde990>"
]
}
],
"prompt_number": 24
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"def nmf(M, components=5, iterations=500):\n",
" \"\"\"Factorize matrix with non-negative matrix factorization.\"\"\"\n",
" # Initialize to matrices\n",
" W = np.asmatrix(np.random.random(([M.shape[0], components])))\n",
" H = np.asmatrix(np.random.random(([components, M.shape[1]])))\n",
" for n in range(0, iterations):\n",
" H = np.multiply(H, (W.T * M) / (W.T * W * H + 0.001))\n",
" W = np.multiply(W, (M * H.T) / (W * (H * H.T) + 0.001))\n",
" return (W, H)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 25
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Perform the actual computation\n",
"W, H = nmf(M, iterations=50, components=3)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 26
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"W.max(), H.max()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 27,
"text": [
"(15.094951905539212, 1.8585767533145232)"
]
}
],
"prompt_number": 27
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# Show the results in some format - This could be written nicer, using Jinja2\n",
"for component in range(W.shape[1]):\n",
" print(\"=\" * 80)\n",
" print(\"COMPONENT %d: \" % (component + 1,))\n",
" indices = (-H[component, :]).getA1().argsort()\n",
" print(\" - \".join([tokens[i] for i in indices[:6] ]))\n",
" print(\"-\")\n",
" indices = (-W[:, component]).getA1().argsort()\n",
" print(\"\\n\".join([documents.ix[i, 0][:80] for i in indices[:5] ]))"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"================================================================================\n",
"COMPONENT 1: \n",
"authors - content - community - number - articles - analysis\n",
"-\n",
"Wikipedia - a quantitative analysis\n",
"Open content and value creation\n",
"Sharing knowledge and building communities: a narrative of the formation, develo\n",
"Extracting content holes by comparing community-type content with Wikipedia\n",
"An analysis of open content systems\n",
"================================================================================\n",
"COMPONENT 2: \n",
"knowledge - document - clustering - linkage - topic - algorithm\n",
"-\n",
"Exploiting external/domain knowledge to enhance traditional text mining using gr\n",
"Wikitology: a novel hybrid knowledge base derived from Wikipedia\n",
"The WikiID: an alternative approach to the body of knowledge\n",
"Breaking the knowledge acquisition bottleneck through conversational knowledge m\n",
"Extracting lexical semantic knowledge from Wikipedia and Wiktionary\n",
"================================================================================\n",
"COMPONENT 3: \n",
"information - web - use - students - search - results\n",
"-\n",
"Gender differences in information behavior concerning Wikipedia, an unorthodox i\n",
"How and why do college students use Wikipedia?\n",
"Where does the information come from? Information source use patterns in Wikiped\n",
"What is the quality of surgery-related information on the Internet? Lessons lear\n",
"Reliability of Wikipedia as a medication information source for pharmacy student\n"
]
}
],
"prompt_number": 28
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 137
}
],
"metadata": {}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment