Loading comprehensive company list of Nasdaq, Nyse and Amex to Sqlite

NasdaqCompaniesLoader.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading comprehensive company list of Nasdaq, Nyse and Amex to Sqlite\n",
    "#### Data\n",
    "[Nasdaq company list](http://www.nasdaq.com/screening/company-list.aspx)\n",
    "#### Pre-requirements\n",
    "- Downloaded company lists from Nasdaq website (url above), and stored under the path \"./data/nasdaq/\"\n",
    "- Downloaded the module for handling sqlite from here: https://gist.github.com/shotahorii/274b832570205e439bd54a3a8ce0bde6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sqlitehandler import SqliteHandler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load the company lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./data/nasdaq/companylist_nasdaq.csv', 'rb') as f:\n",
    "    df_nasdaq = pd.read_csv(f).drop('Unnamed: 8', axis=1)\n",
    "    \n",
    "with open('./data/nasdaq/companylist_nyse.csv', 'rb') as f:\n",
    "    df_nyse = pd.read_csv(f).drop('Unnamed: 8', axis=1)\n",
    "    \n",
    "with open('./data/nasdaq/companylist_amex.csv', 'rb') as f:\n",
    "    df_amex = pd.read_csv(f).drop('Unnamed: 8', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_companies = pd.concat([df_nasdaq, df_nyse, df_amex]).drop_duplicates().reset_index(drop=True)\n",
    "# duplicates are the companies in 2 of Nasdaq, Nyse or Amex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_companies = df_companies.rename(columns={'Summary Quote': 'SummaryQuote'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Symbol</th>\n",
       "      <th>Name</th>\n",
       "      <th>LastSale</th>\n",
       "      <th>MarketCap</th>\n",
       "      <th>IPOyear</th>\n",
       "      <th>Sector</th>\n",
       "      <th>industry</th>\n",
       "      <th>SummaryQuote</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PIH</td>\n",
       "      <td>1347 Property Insurance Holdings, Inc.</td>\n",
       "      <td>7.4596</td>\n",
       "      <td>$44.44M</td>\n",
       "      <td>2014</td>\n",
       "      <td>Finance</td>\n",
       "      <td>Property-Casualty Insurers</td>\n",
       "      <td>http://www.nasdaq.com/symbol/pih</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TURN</td>\n",
       "      <td>180 Degree Capital Corp.</td>\n",
       "      <td>1.56</td>\n",
       "      <td>$48.55M</td>\n",
       "      <td>n/a</td>\n",
       "      <td>Finance</td>\n",
       "      <td>Finance/Investors Services</td>\n",
       "      <td>http://www.nasdaq.com/symbol/turn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FLWS</td>\n",
       "      <td>1-800 FLOWERS.COM, Inc.</td>\n",
       "      <td>10.15</td>\n",
       "      <td>$663.15M</td>\n",
       "      <td>1999</td>\n",
       "      <td>Consumer Services</td>\n",
       "      <td>Other Specialty Stores</td>\n",
       "      <td>http://www.nasdaq.com/symbol/flws</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Symbol                                    Name LastSale MarketCap IPOyear  \\\n",
       "0    PIH  1347 Property Insurance Holdings, Inc.   7.4596   $44.44M    2014   \n",
       "1   TURN                180 Degree Capital Corp.     1.56   $48.55M     n/a   \n",
       "2   FLWS                 1-800 FLOWERS.COM, Inc.    10.15  $663.15M    1999   \n",
       "\n",
       "              Sector                    industry  \\\n",
       "0            Finance  Property-Casualty Insurers   \n",
       "1            Finance  Finance/Investors Services   \n",
       "2  Consumer Services      Other Specialty Stores   \n",
       "\n",
       "                        SummaryQuote  \n",
       "0   http://www.nasdaq.com/symbol/pih  \n",
       "1  http://www.nasdaq.com/symbol/turn  \n",
       "2  http://www.nasdaq.com/symbol/flws  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_companies.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Store into the database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "db = SqliteHandler('nasdaq')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "db.df2table('companies', df_companies)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}