bag-of-word.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vectorizer = CountVectorizer(min_df=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "min_dfより出現回数が少ない単語は無視される"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
      "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
      "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
      "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
      "        strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
      "        tokenizer=None, vocabulary=None)\n"
     ]
    }
   ],
   "source": [
    "print(vectorizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "7つの単語をトークンとして抽出する"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['disk', 'format', 'hard', 'how', 'my', 'problems', 'to']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content = [\"How to format my hard disk\", \" Hard disk format problems\", ]\n",
    "X = vectorizer.fit_transform(content)\n",
    "vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "単語の出現回数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 1]\n",
      " [1 1]\n",
      " [1 1]\n",
      " [1 0]\n",
      " [1 0]\n",
      " [0 1]\n",
      " [1 0]]\n"
     ]
    }
   ],
   "source": [
    "print(X.toarray().transpose())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text01 = \"This is a toy post about machine learning.Actually, it contains not much interesting stuff.\"\n",
    "text02 = \"Imaging databases can get huge.\"\n",
    "text03 = \"Most imaging databases safe images permanently.\"\n",
    "text04 = \"Imaging databases store data.\"\n",
    "text05 = \"Imaging databases store data. Imaging databases store data. Imaging databases store data.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "posts = [text01, text02, text03, text04, text05]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "文書数と単語数を表示する"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#samples: 5, #features: 25\n"
     ]
    }
   ],
   "source": [
    "X_train = vectorizer.fit_transform(posts)\n",
    "num_samples, num_features = X_train.shape\n",
    "print(\"#samples: %d, #features: %d\" % (num_samples, num_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['about',\n",
       " 'actually',\n",
       " 'can',\n",
       " 'contains',\n",
       " 'data',\n",
       " 'databases',\n",
       " 'get',\n",
       " 'huge',\n",
       " 'images',\n",
       " 'imaging',\n",
       " 'interesting',\n",
       " 'is',\n",
       " 'it',\n",
       " 'learning',\n",
       " 'machine',\n",
       " 'most',\n",
       " 'much',\n",
       " 'not',\n",
       " 'permanently',\n",
       " 'post',\n",
       " 'safe',\n",
       " 'store',\n",
       " 'stuff',\n",
       " 'this',\n",
       " 'toy']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x25 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 2 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_post = \"imaging databases\"\n",
    "new_post_vec = vectorizer.transform([new_post])\n",
    "new_post_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 5)\t1\n",
      "  (0, 9)\t1\n"
     ]
    }
   ],
   "source": [
    "print(new_post_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_post_vec.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "新しい文書と既存の文書間でユークリッド距離を計算する"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import scipy as sp\n",
    "def dist_raw(v1, v2):\n",
    "    delta = v1 - v2\n",
    "    return sp.linalg.norm(delta.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Post 0 with dist=4.00: This is a toy post about machine learning.Actually, it contains not much interesting stuff.\n",
      "=== Post 1 with dist=1.73: Imaging databases can get huge.\n",
      "=== Post 2 with dist=2.00: Most imaging databases safe images permanently.\n",
      "=== Post 3 with dist=1.41: Imaging databases store data.\n",
      "=== Post 4 with dist=5.10: Imaging databases store data. Imaging databases store data. Imaging databases store data.\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "best_doc = None\n",
    "best_dist = sys.maxsize\n",
    "best_i = None\n",
    "for i in range(0, len(posts)):\n",
    "    post = posts[i]\n",
    "    if post == new_post:\n",
    "        continue\n",
    "    post_vec = X_train.getrow(i)\n",
    "    d = dist_raw(post_vec, new_post_vec)\n",
    "    print(\"=== Post %i with dist=%.2f: %s\" % (i, d, post))\n",
    "    if d < best_dist:\n",
    "        best_dist = d\n",
    "        best_i = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best post is 3 with dist=1.41\n"
     ]
    }
   ],
   "source": [
    "print(\"Best post is %i with dist=%.2f\" % (best_i, best_dist))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}