EMSS2020

apt.txt

chromium-chromedriver

collect_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "import time\n",
    "from collections import defaultdict\n",
    "from functools import partial\n",
    "from time import perf_counter\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from helium import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scroll_helium(driver):\n",
    "    def get_height():\n",
    "        return driver.find_elements_by_xpath('/html/body/ytd-app')[0].size['height']\n",
    "    \n",
    "    last_height = get_height()\n",
    "\n",
    "    while True:\n",
    "        press(END)\n",
    "        \n",
    "        try:\n",
    "            wait_until(lambda : get_height() > last_height, timeout_secs=5)\n",
    "        except TimeoutException:\n",
    "            break\n",
    "        \n",
    "        last_height = get_height()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_text(text_list, text):\n",
    "    for text_elem in text_list:\n",
    "        aria = text_elem.get_attribute('aria-label')\n",
    "        if aria is None:\n",
    "            continue\n",
    "            \n",
    "        if text.lower() in aria.lower():\n",
    "            return aria\n",
    "        \n",
    "    return Exception()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Headless Helium Scrapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scroll_until_found(func):\n",
    "    press(PAGE_DOWN)\n",
    "    return func()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def headless_download_stats(channel, link=None):\n",
    "    \n",
    "    path_safe = \"0123456789abcdefghijklmnopqrstuvwxyz\"\n",
    "    \n",
    "    start = perf_counter()\n",
    "    \n",
    "    driver = start_chrome(headless=True)\n",
    "    \n",
    "    if link is None:\n",
    "        link = \"https://www.youtube.com/channel/\"\n",
    "        url = f'{link}{channel}/'\n",
    "    else:\n",
    "        url = link\n",
    "    \n",
    "    go_to(url)\n",
    "    \n",
    "    click(Text(\"VIDEOS\"))\n",
    "    \n",
    "    name_raw = driver.find_elements_by_class_name('ytd-channel-name')[0].text\n",
    "    name = ''.join(char for char in name_raw if char.lower() in path_safe)\n",
    "    print(f\"Scrapping for {name} started\")\n",
    "    \n",
    "    wait_until(lambda : len(driver.find_elements_by_xpath('//*[@id=\"icon-label\"]')) != 0)\n",
    "    scroll_helium(driver)\n",
    "\n",
    "    videos = driver.find_elements_by_xpath('//*[@id=\"video-title\"]')\n",
    "    video_data = defaultdict(list)\n",
    "    \n",
    "    video_data['title'] = [video.text for video in videos]\n",
    "    video_data['link'] = [video.get_attribute('href') for video in videos]\n",
    "    video_data['channel'] = [name] * len(videos)\n",
    "\n",
    "    for video_link in tqdm(video_data['link']):\n",
    "        go_to(video_link)\n",
    "        wait_until(lambda : len(driver.find_elements_by_xpath('//*[@id=\"date\"]')) != 0)\n",
    "\n",
    "        try:\n",
    "            views_raw = driver.find_element_by_class_name('view-count').text\n",
    "            views = int(views_raw.split()[0].replace(\",\", \"\"))\n",
    "        except:\n",
    "            views = np.nan\n",
    "\n",
    "        video_data[\"views_count\"].append(views)\n",
    "        \n",
    "        \n",
    "        try:\n",
    "            video_date = driver.find_elements_by_xpath('//*[@id=\"date\"]')[0].text[1:]\n",
    "        except:\n",
    "            video_date = \"\"\n",
    "            \n",
    "        video_data[\"date\"].append(video_date)\n",
    "        \n",
    "\n",
    "        wait_until(lambda : len(driver.find_elements_by_xpath('//*[@id=\"text\"]')) >= 7)\n",
    "\n",
    "        text_in_page = driver.find_elements_by_xpath('//*[@id=\"text\"]')\n",
    "\n",
    "        try:\n",
    "            likes_raw = find_text(text_in_page, 'like')\n",
    "            likes = int(likes_raw.split()[0].replace(\",\", \"\"))\n",
    "        except:\n",
    "            likes = np.nan\n",
    "            \n",
    "        video_data[\"likes\"].append(likes)\n",
    "\n",
    "        try:\n",
    "            dislikes_raw = find_text(text_in_page, 'dislike')\n",
    "            dislikes = int(dislikes_raw.split()[0].replace(\",\", \"\"))\n",
    "        except:\n",
    "            dislikes = np.nan\n",
    "            \n",
    "        video_data[\"dislikes\"].append(dislikes)\n",
    "\n",
    "        try:\n",
    "            wait_until(lambda : scroll_until_found(lambda : len(driver.find_elements_by_class_name('count-text')) >= 1),\n",
    "                       timeout_secs=40, interval_secs=1)\n",
    "            comments_count_raw = driver.find_elements_by_class_name('count-text')[0].text\n",
    "            comments_count = int(comments_count_raw.split()[0].replace(\",\", \"\"))\n",
    "        except:\n",
    "            comments_count = np.nan\n",
    "            \n",
    "        video_data[\"comments_count\"].append(comments_count)\n",
    "\n",
    "    kill_browser()\n",
    "    \n",
    "    elapsed = perf_counter() - start\n",
    "    print(f\"Time Elapsed: {elapsed:2.2f}s\")\n",
    "    \n",
    "    df = pd.DataFrame(video_data)\n",
    "    \n",
    "    df.to_csv(f'{name}_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scrapping for PBSInfiniteSeries started\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c8abe45ce294737ba8ebc3141ea3bf8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=65.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1028.67\n"
     ]
    }
   ],
   "source": [
    "channel = \"UCs4aHmggTfFrpkPcWSaBN9g\" # PBS Infinite Series Channel ID\n",
    "try:\n",
    "    headless_download_stats(channel)\n",
    "except:\n",
    "    kill_browser()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}

environment.yml

name: EMSS-Castano

channels:
  - conda-forge

dependencies:
  - python=3.7
  - numpy
  - pandas
  - scipy
  - seaborn
  - matplotlib
  - selenium
  - tqdm
  - pymc3
  - mkl-service
  - pip:
    - helium

inference.ipynb

paper_figures.ipynb

pbs_data.csv