pulling_free_us_stock_market_data\list_puller

list_puller.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "import logging\n",
    "\n",
    "import datetime as dt\n",
    "from datetime import date\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import string\n",
    "import re\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_or_create_dir(directory):\n",
    "\n",
    "    if not check_exists(directory):\n",
    "        os.makedirs(directory)\n",
    "        \n",
    "        \n",
    "def check_exists(path):\n",
    "\n",
    "    return os.path.exists(path)\n",
    "\n",
    "\n",
    "def get_latest_file(dir):\n",
    "    ls_files = [f for f in os.listdir(dir) if os.path.isfile(os.path.join(dir, f))]\n",
    "    ls_files.sort(key=lambda x: dt.datetime.strptime(x.split('_')[0], '%Y%m%d'))\n",
    "\n",
    "    latest_file = ls_files[0] if len(ls_files) > 0 else None\n",
    "\n",
    "    return latest_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_eod_http(dic_dir, dic_args):\n",
    "    dic_url = {'NYSE': 'http://eoddata.com/stocklist/NYSE/',\n",
    "               'NASDAQ': 'https://www.eoddata.com/stocklist/NASDAQ/'}\n",
    "\n",
    "    today = date.today()\n",
    "    creation_date = today.replace(day=1).strftime('%Y%m%d')\n",
    "\n",
    "    output_file = creation_date + \"_\" + dic_args['composite_symbol'] + \".csv\"\n",
    "\n",
    "    dic_dir['components_list_dir'] = os.path.join(dic_dir['base'], dic_args['components_list_dir'])\n",
    "\n",
    "    for param in re.findall(r'\\{(.*?)\\}', dic_dir['components_list_dir']):\n",
    "        dic_dir['components_list_dir'] = dic_dir['components_list_dir'].replace('{'+param+'}', dic_args[param])\n",
    "\n",
    "    exchange_data_path = os.path.join(dic_dir['components_list_dir'], output_file)\n",
    "\n",
    "    if not check_exists(exchange_data_path):\n",
    "        ls_record = []\n",
    "\n",
    "        ls_alpha = list(string.ascii_uppercase)\n",
    "\n",
    "        for alpha in ls_alpha:\n",
    "            url = dic_url[dic_args['composite_symbol']] + alpha + \".html\"\n",
    "\n",
    "            resp = requests.get(url)\n",
    "            site = resp.content\n",
    "\n",
    "            soup = BeautifulSoup(site, 'html.parser')\n",
    "            table = soup.find('table', {'class': 'quotes'})\n",
    "\n",
    "            for row in table.findAll('tr')[1:]:\n",
    "\n",
    "                try:\n",
    "                    symbol = row.findAll('td')[0].text.rstrip()\n",
    "                    company = row.findAll('td')[1].text.rstrip()\n",
    "\n",
    "                    ls_record.append([symbol, company])\n",
    "\n",
    "                except Exception as e:\n",
    "                    print(e)\n",
    "                    pass\n",
    "\n",
    "        ls_cols = ['entity_symbol', 'company_name']\n",
    "\n",
    "        df = pd.DataFrame(ls_record, columns=ls_cols)\n",
    "\n",
    "        if not df.empty:\n",
    "            df['region'] = dic_args['region']\n",
    "            df['composite_symbol'] = dic_args['composite_symbol']\n",
    "            df['pr_key'] = df['entity_symbol'] + '_' + df['composite_symbol']\n",
    "\n",
    "            check_or_create_dir(dic_dir['components_list_dir'])\n",
    "\n",
    "            df.to_csv(exchange_data_path, sep=';', quotechar='\"', date_format='%Y-%m-%d %H:%M:%S.%f', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_wiki_http(dic_dir, dic_args):\n",
    "    dic_url = {'SP500': 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies'}\n",
    "\n",
    "    url = dic_url[dic_args['composite_symbol']]\n",
    "\n",
    "    today = date.today()\n",
    "    creation_date = today.replace(day=1).strftime('%Y%m%d')\n",
    "\n",
    "    output_file = creation_date + \"_\" + dic_args['composite_symbol'] + \".csv\"\n",
    "\n",
    "    dic_dir['components_list_dir'] = os.path.join(dic_dir['base'], dic_args['components_list_dir'])\n",
    "\n",
    "    for param in re.findall(r'\\{(.*?)\\}', dic_dir['components_list_dir']):\n",
    "        dic_dir['components_list_dir'] = dic_dir['components_list_dir'].replace('{'+ param+'}', dic_args[param])\n",
    "\n",
    "    index_data_path = os.path.join(dic_dir['components_list_dir'], output_file)\n",
    "\n",
    "    if not check_exists(index_data_path):\n",
    "        resp = requests.get(url)\n",
    "        site = resp.content\n",
    "\n",
    "        soup = BeautifulSoup(site, 'lxml')\n",
    "        table = soup.find('table', {'class': 'wikitable sortable'})\n",
    "\n",
    "        ls_record = []\n",
    "        ls_cols = []\n",
    "\n",
    "        for row in table.findAll('tr')[1:]:\n",
    "\n",
    "            try:\n",
    "                symbol = row.findAll('td')[0].text.rstrip('\\n')\n",
    "                name = row.findAll('td')[1].text.rstrip('\\n')\n",
    "                gics_sector = row.findAll('td')[3].text.rstrip('\\n')\n",
    "                gics_sub_sector = row.findAll('td')[4].text.rstrip('\\n')\n",
    "\n",
    "                ls_record.append([symbol, name, gics_sector, gics_sub_sector])\n",
    "\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "                pass\n",
    "\n",
    "        ls_cols = ['entity_symbol', 'company_name', 'sector', 'sub_sector']\n",
    "\n",
    "        df = pd.DataFrame(ls_record, columns=ls_cols)\n",
    "\n",
    "        if not df.empty:\n",
    "            df['region'] = dic_args['region']\n",
    "            df['composite_symbol'] = dic_args['composite_symbol']\n",
    "            df['pr_key'] = df['entity_symbol'] + '_' + df['composite_symbol']\n",
    "\n",
    "            check_or_create_dir(dic_dir['components_list_dir'])\n",
    "\n",
    "            df.to_csv(index_data_path, sep=';', quotechar='\"', date_format='%Y-%m-%d %H:%M:%S.%f', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if sys.platform == \"win32\":\n",
    "    base_dir = r\"\\\\dir\"  # dir for win machines\n",
    "else:\n",
    "    base_dir = \"/dir\"  # dir for non-win machines\n",
    "    \n",
    "    \n",
    "dic_args_set = {\n",
    "\n",
    "    'args1': {'func': 'US_list_pull',\n",
    "              'region': 'US',\n",
    "              'data_cat': 'list',\n",
    "              'data_subcat': 'exchange_components_list',\n",
    "              'composite_symbol': 'NYSE',\n",
    "              'data_source': 'eod_http',\n",
    "              'components_list_dir': os.path.join('{region}', '{data_cat}', '{data_subcat}',\n",
    "                                                  '{composite_symbol}', '{data_source}')},\n",
    "    \n",
    "    'args2': {'func': 'US_list_pull',\n",
    "              'region': 'US',\n",
    "              'data_cat': 'list',\n",
    "              'data_subcat': 'exchange_components_list',\n",
    "              'composite_symbol': 'NASDAQ',\n",
    "              'data_source': 'eod_http',\n",
    "              'components_list_dir': os.path.join('{region}', '{data_cat}', '{data_subcat}',\n",
    "                                                  '{composite_symbol}', '{data_source}')},\n",
    "    \n",
    "    'args3': {'func': 'US_list_pull',\n",
    "              'region': 'US',\n",
    "              'data_cat': 'list',\n",
    "              'data_subcat': 'index_components_list',\n",
    "              'composite_symbol': 'SP500',\n",
    "              'data_source': 'wiki_http',\n",
    "              'components_list_dir': os.path.join('{region}', '{data_cat}', '{data_subcat}',\n",
    "                                                  '{composite_symbol}', '{data_source}')},\n",
    "    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dic_args in dic_args_set.values():\n",
    "    creation_date = date.today().strftime('%Y%m%d')\n",
    "    log_file = creation_date + \"_\" + dic_args['func'] + \".log\"\n",
    "\n",
    "    logs_dir = os.path.join(base_dir, '_logs')\n",
    "\n",
    "    dic_dir = {'base': base_dir,\n",
    "               'logs': logs_dir}\n",
    "\n",
    "    check_or_create_dir(dic_dir['logs'])\n",
    "\n",
    "    log_path = os.path.join(dic_dir['logs'], log_file)\n",
    "\n",
    "    logging.basicConfig(filename=log_path, level=logging.INFO)\n",
    "\n",
    "    log_str = \"{}({}, {}, {}, {})\".format(dic_args['func'], dic_args['region'], dic_args['data_cat'],\n",
    "                                          dic_args['data_subcat'], dic_args['data_source'])\n",
    "    \n",
    "    try:\n",
    "        print(\"\\n\" + \"STARTING: \" + log_str)\n",
    "        logging.info(\"\\n\" + \"STARTING: \" + log_str)\n",
    "\n",
    "        if dic_args['data_subcat'] == 'exchange_components_list':\n",
    "        \n",
    "            get_eod_http(dic_dir, dic_args)\n",
    "            \n",
    "        elif dic_args['data_subcat'] == 'index_components_list':\n",
    "            \n",
    "            get_wiki_http(dic_dir, dic_args)\n",
    "            \n",
    "\n",
    "        print(\"COMPLETED: \" + log_str)\n",
    "        logging.info(\"COMPLETED: \" + log_str)\n",
    "\n",
    "    except Exception as e:\n",
    "        print(str(e))\n",
    "        logging.error(str(e))\n",
    "\n",
    "        print(\"FAILED: \" + log_str)\n",
    "        logging.error(\"FAILED: \" + log_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}