{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Automatic 301 Redirect Mapping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## library imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import csv\n",
    "import time\n",
    "import requests\n",
    "import re\n",
    "from goose3 import Goose\n",
    "import spacy\n",
    "nlp = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## User inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "domain = \"dakessianlaw.com\"\n",
    "sf_crawl = \"C:/Users/paul.Shapiro/Desktop/live-website-source.csv\"\n",
    "csv_output = \"C:/Users/paul.Shapiro/Desktop/301-redirect-matching.csv\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract historic URLs from Wayback Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['http://dakessianlaw.com/', 'http://www.dakessianlaw.com/about-our-firm', 'http://www.dakessianlaw.com/aop.php', 'http://www.dakessianlaw.com/attorney_profile.php', 'http://www.dakessianlaw.com/attorney_profile.php?attorney=aleen', 'http://www.dakessianlaw.com/attorney_profile.php?attorney=marty', 'http://www.dakessianlaw.com/attorney_profile.php?attorney=shiraz', 'http://www.dakessianlaw.com/case-evaluation/', 'http://www.dakessianlaw.com/common/files/rebuild.html', 'http://www.dakessianlaw.com/contact-us/', 'http://www.dakessianlaw.com/contact_us.php', 'http://dakessianlaw.com/demo/contact_thanks.php', 'http://www.dakessianlaw.com/different.php', 'http://www.dakessianlaw.com/documents/Marty-Dakessian-VCard.vcf', 'http://www.dakessianlaw.com/documents/Ruben-Sislyan-VCard.vcf', 'http://www.dakessianlaw.com/index.php', 'http://www.dakessianlaw.com/news-information', 'http://www.dakessianlaw.com/news-information/2013/', 'http://www.dakessianlaw.com/news-information/2013/april/', 'http://www.dakessianlaw.com/news-information/2013/april/dakessian-quoted-in-la-daily-journal-regarding-f/', 'http://www.dakessianlaw.com/news-information/2013/june/', 'http://www.dakessianlaw.com/news-information/2013/june/judge-awards-firm-1-2-million-in-fees-in-taxatio/', 'http://www.dakessianlaw.com/news-information/2015/', 'http://www.dakessianlaw.com/news-information/2015/january/', 'http://www.dakessianlaw.com/news-information/2015/january/award-winning-lawyer-marty-dakessian-launches-ca/', 'http://www.dakessianlaw.com/news-information/2016/', 'http://www.dakessianlaw.com/news-information/2016/february/', 'http://www.dakessianlaw.com/news-information/2016/february/california-franchise-tax-board-recognizes-taxpay/', 'http://www.dakessianlaw.com/news-information/2016/february/marty-dakessian-launches-three-lawyer-california', 'http://www.dakessianlaw.com/news-information/2016/january/', 'http://www.dakessianlaw.com/news-information/2016/january/attorney-dakessian-talks-about-court-upholding-t/', 'http://www.dakessianlaw.com/news-information/2016/january/california-supreme-court-rules-against-taxpayers/', 'http://www.dakessianlaw.com/news-information/2016/march/', 'http://www.dakessianlaw.com/news-information/2016/march/attorney-marty-dakessian-opens-boutique-californ', 'http://www.dakessianlaw.com/news-information/categories/articles/', 'http://www.dakessianlaw.com/news-information/categories/firm-news/', 'http://www.dakessianlaw.com/news-information/categories/press-releases/', 'http://www.dakessianlaw.com/news-information/categories/tax-law/', 'http://www.dakessianlaw.com/news-information/categories/tax-updates/', 'http://www.dakessianlaw.com/our-lawyers', 'http://www.dakessianlaw.com/our-lawyers/marty-dakessian', 'http://www.dakessianlaw.com/our-lawyers/ruben-sislyan', 'http://www.dakessianlaw.com/privacy-policy/', 'http://www.dakessianlaw.com/site-map/', 'http://www.dakessianlaw.com/site_map.php', 'http://www.dakessianlaw.com/tax-controversy-litigation/', 'http://www.dakessianlaw.com/tax-controversy-litigation/california-s-unique-tax-system/', 'http://www.dakessianlaw.com/video-center/']\n"
     ]
    }
   ],
   "source": [
    "wayback_cdx_api = \"https://web.archive.org/cdx/search/cdx?url=\" + domain + \"&matchType=domain&fl=original&collapse=urlkey&filter=mimetype:text/html&output=json\"\n",
    "r = requests.get(wayback_cdx_api)\n",
    "wayback_urls = []\n",
    "for x in r.json()[1:]:\n",
    "    non_html = re.search('png|js|css|jpg|svg|jpeg|gif|xml|rss|PNG|JS|CSS|JPG|SVG|JPEG|GIF|XML|RSS|txt|TXT|ico|ICO|pdf|PDF|woff|woff2|WOFF|WOFF2', x[0])\n",
    "    if non_html is None:\n",
    "        wayback_urls.append(re.sub(':80', '', x[0]))\n",
    "print(wayback_urls)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find archives of historic URLs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['http://web.archive.org/web/20181209204837/https://dakessianlaw.com/', 'http://web.archive.org/web/20160502012159/http://www.dakessianlaw.com:80/about-our-firm/', 'http://web.archive.org/web/20080413022830/http://www.dakessianlaw.com:80/aop.php', 'http://web.archive.org/web/20081025053133/http://www.dakessianlaw.com:80/attorney_profile.php', 'http://web.archive.org/web/20080328080114/http://www.dakessianlaw.com:80/attorney_profile.php?attorney=aleen', 'http://web.archive.org/web/20080328080118/http://www.dakessianlaw.com:80/attorney_profile.php?attorney=marty', 'http://web.archive.org/web/20080328080123/http://www.dakessianlaw.com:80/attorney_profile.php?attorney=shiraz', 'http://web.archive.org/web/20160402065011/http://www.dakessianlaw.com:80/case-evaluation/', 'http://web.archive.org/web/20160611034022/http://www.dakessianlaw.com:80/common/files/rebuild.html', 'http://web.archive.org/web/20160415134238/http://www.dakessianlaw.com:80/contact-us/', 'http://web.archive.org/web/20080407230256/http://www.dakessianlaw.com:80/contact_us.php', 'http://web.archive.org/web/20080413022843/http://www.dakessianlaw.com:80/different.php', 'http://web.archive.org/web/20160401213157/http://www.dakessianlaw.com:80/documents/Marty-Dakessian-VCard.vcf', 'http://web.archive.org/web/20071210235115/http://www.dakessianlaw.com:80/index.php', 'http://web.archive.org/web/20160502012703/http://www.dakessianlaw.com:80/news-information/', 'http://web.archive.org/web/20160401213313/http://www.dakessianlaw.com:80/news-information/2013/', 'http://web.archive.org/web/20160401213203/http://www.dakessianlaw.com:80/news-information/2013/april/', 'http://web.archive.org/web/20160401213209/http://www.dakessianlaw.com:80/news-information/2013/april/dakessian-quoted-in-la-daily-journal-regarding-f/', 'http://web.archive.org/web/20160401213319/http://www.dakessianlaw.com:80/news-information/2013/june/', 'http://web.archive.org/web/20160401213410/http://www.dakessianlaw.com:80/news-information/2013/june/judge-awards-firm-1-2-million-in-fees-in-taxatio/', 'http://web.archive.org/web/20160401213540/http://www.dakessianlaw.com:80/news-information/2015/', 'http://web.archive.org/web/20160401213416/http://www.dakessianlaw.com:80/news-information/2015/january/', 'http://web.archive.org/web/20160401213332/http://www.dakessianlaw.com:80/news-information/2015/january/award-winning-lawyer-marty-dakessian-launches-ca/', 'http://web.archive.org/web/20160401213423/http://www.dakessianlaw.com:80/news-information/2016/', 'http://web.archive.org/web/20160401213216/http://www.dakessianlaw.com:80/news-information/2016/february/', 'http://web.archive.org/web/20160401164049/http://www.dakessianlaw.com:80/news-information/2016/february/california-franchise-tax-board-recognizes-taxpay/', 'http://web.archive.org/web/20160402065029/http://www.dakessianlaw.com:80/news-information/2016/february/marty-dakessian-launches-three-lawyer-california/', 'http://web.archive.org/web/20160401213221/http://www.dakessianlaw.com:80/news-information/2016/january/', 'http://web.archive.org/web/20160401164053/http://www.dakessianlaw.com:80/news-information/2016/january/attorney-dakessian-talks-about-court-upholding-t/', 'http://web.archive.org/web/20160401164121/http://www.dakessianlaw.com:80/news-information/2016/january/california-supreme-court-rules-against-taxpayers/', 'http://web.archive.org/web/20160401213434/http://www.dakessianlaw.com:80/news-information/2016/march/', 'http://web.archive.org/web/20160419141903/http://www.dakessianlaw.com:80/news-information/2016/march/attorney-marty-dakessian-opens-boutique-californ/', 'http://web.archive.org/web/20160403074728/http://www.dakessianlaw.com:80/news-information/categories/articles/', 'http://web.archive.org/web/20160403074411/http://www.dakessianlaw.com:80/news-information/categories/firm-news/', 'http://web.archive.org/web/20160403074514/http://www.dakessianlaw.com:80/news-information/categories/press-releases/', 'http://web.archive.org/web/20160403074348/http://www.dakessianlaw.com:80/news-information/categories/tax-law/', 'http://web.archive.org/web/20160403074526/http://www.dakessianlaw.com:80/news-information/categories/tax-updates/', 'http://web.archive.org/web/20160502012830/http://www.dakessianlaw.com:80/our-lawyers/', 'http://web.archive.org/web/20160502012708/http://www.dakessianlaw.com:80/our-lawyers/marty-dakessian/', 'http://web.archive.org/web/20160509233730/http://www.dakessianlaw.com:80/our-lawyers/ruben-sislyan/', 'http://web.archive.org/web/20160402064955/http://www.dakessianlaw.com:80/privacy-policy/', 'http://web.archive.org/web/20160422072426/http://www.dakessianlaw.com:80/site-map/', 'http://web.archive.org/web/20080418094240/http://www.dakessianlaw.com:80/site_map.php', 'http://web.archive.org/web/20160505155006/http://www.dakessianlaw.com:80/tax-controversy-litigation/', 'http://web.archive.org/web/20160422072429/http://www.dakessianlaw.com:80/tax-controversy-litigation/california-s-unique-tax-system/', 'http://web.archive.org/web/20160403074353/http://www.dakessianlaw.com:80/video-center/']\n"
     ]
    }
   ],
   "source": [
    "archive_urls = []\n",
    "for x in wayback_urls:\n",
    "    r = requests.get(\"https://archive.org/wayback/available?url=\" + x)\n",
    "    try:\n",
    "        archive_urls.append(r.json()['archived_snapshots']['closest']['url'])\n",
    "    except KeyError:\n",
    "        continue\n",
    "print(archive_urls)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare ScreamingFrog Crawl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['https://dakessianlaw.com/', 'https://dakessianlaw.com/what-we-do/', 'https://dakessianlaw.com/our-results/', 'https://dakessianlaw.com/news/', 'https://dakessianlaw.com/contact-us/', 'https://dakessianlaw.com/faqs/', 'https://dakessianlaw.com/about-us/', 'https://dakessianlaw.com/privacy/', 'https://dakessianlaw.com/our-team/', 'https://dakessianlaw.com/news/marty-dakessian-named-tax-mvp-in-2017-by-law360/', 'https://dakessianlaw.com/news/california-franchise-tax-board-recognizes-taxpayers-will-appeal-gillette-ruling/', 'https://dakessianlaw.com/news/lawmakers-tax-experts-debate-future-of-boe/', 'https://dakessianlaw.com/news/marty-dakessian-honored-as-a-2016-southern-california-super-lawyer/', 'https://dakessianlaw.com/news/former-nfl-player-sues-california-tax-board-over-residency-issue/', 'https://dakessianlaw.com/news/attorney-dakessian-talks-about-court-upholding-tax-law-that-helps-ca-businesses/', 'https://dakessianlaw.com/2017/?post_type=news', 'https://dakessianlaw.com/news/bill-to-overhaul-california-boe-racing-through-legislature/', 'https://dakessianlaw.com/news/california-supreme-court-rules-against-taxpayers-in-multistate-tax-compact-apportionment-case/', 'https://dakessianlaw.com/category/tax-law/', 'https://dakessianlaw.com/2013/?post_type=news', 'https://dakessianlaw.com/2016/?post_type=news', 'https://dakessianlaw.com/news/successful-challenge-of-los-angeles-business-tax-conferred-significant-public-benefit-court-rules/', 'https://dakessianlaw.com/news/marty-dakessian-named-tax-mvp-in-2016-by-law360/', 'https://dakessianlaw.com/2019/?post_type=news', 'https://dakessianlaw.com/category/articles/', 'https://dakessianlaw.com/news/new-irs-chief-may-help-reform-embattled-agency/', 'https://dakessianlaw.com/news/retroactive-tax-laws-muddying-businesses-future-decisions/', 'https://dakessianlaw.com/news/agency-denies-ftbs-petition-to-rehear-case-involving-football-star-keyshawn-johnson/', 'https://dakessianlaw.com/2018/?post_type=news', 'https://dakessianlaw.com/news/surprise-ruling-says-some-local-taxes-need-no-vote/', 'https://dakessianlaw.com/news/through-the-looking-glass-reflections-on-the-mtc/', 'https://dakessianlaw.com/news/california-governor-appoints-head-of-new-tax-agency/', 'https://dakessianlaw.com/news/a-new-law-reorganizes-californias-tax-system/', 'https://dakessianlaw.com/news/marty-dakessian-featured-in-bloomberg-bna-report-about-las-hotel-tax-on-airbnb/', 'https://dakessianlaw.com/news/attorney-marty-dakessian-opens-boutique-california-tax-law-firm/', 'https://dakessianlaw.com/news/dakessian-quoted-in-la-daily-journal-regarding-ftbs-retroactively-elimination-of-tax-incentive/', 'https://dakessianlaw.com/news/marty-dakessian-launches-three-lawyer-california-tax-boutique/', 'https://dakessianlaw.com/news/appellate-lawyer-zareh-jaltorossian-joins-award-winning-los-angeles-tax-boutique-dakessian-law/', 'https://dakessianlaw.com/news/irs-pick-to-face-tough-questions-on-agencys-direction/', 'https://dakessianlaw.com/news/san-diego-airport-fights-fees-in-hertz-suit-against-port/', 'https://dakessianlaw.com/news/judge-awards-firm-1-2-million-in-fees-in-taxation-case/', 'https://dakessianlaw.com/news/californias-tax-system-overhaul-creates-more-questions-than-answers-for-businesses/', 'https://dakessianlaw.com/category/firm-news-press/', 'https://dakessianlaw.com/news/wayfair-implementation-bill-clears-first-committee-with-unanimous-vote/', 'https://dakessianlaw.com/news/award-winning-lawyer-marty-dakessian-launches-california-tax-boutique/', 'https://dakessianlaw.com/news/ftb-riding-high-after-summary-judgment-win-in-harley-commerce-clause-case/']\n"
     ]
    }
   ],
   "source": [
    "original_urls = []\n",
    "with open(sf_crawl, \"r\", encoding='utf8') as csv_file:\n",
    "    csv_reader = csv.reader(csv_file, delimiter=',')\n",
    "    for lines in csv_reader:\n",
    "        non_html = re.search('png|js|css|jpg|svg|jpeg|gif|xml|rss|PNG|JS|CSS|JPG|SVG|JPEG|GIF|XML|RSS|txt|TXT|ico|ICO|pdf|PDF|woff|woff2|WOFF|WOFF2|pd|PD', lines[0])\n",
    "        if non_html is None:\n",
    "            original_urls.append(lines[0])\n",
    "del original_urls[0:2]\n",
    "print(original_urls)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract body text of historic URLs\n",
    "\n",
    "*This may take some time*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://web.archive.org/web/20181209204837/https://dakessianlaw.com/\n",
      "http://web.archive.org/web/20160502012159/http://www.dakessianlaw.com:80/about-our-firm/\n",
      "http://web.archive.org/web/20080413022830/http://www.dakessianlaw.com:80/aop.php\n",
      "http://web.archive.org/web/20081025053133/http://www.dakessianlaw.com:80/attorney_profile.php\n",
      "http://web.archive.org/web/20080328080114/http://www.dakessianlaw.com:80/attorney_profile.php?attorney=aleen\n",
      "http://web.archive.org/web/20080328080118/http://www.dakessianlaw.com:80/attorney_profile.php?attorney=marty\n",
      "http://web.archive.org/web/20080328080123/http://www.dakessianlaw.com:80/attorney_profile.php?attorney=shiraz\n",
      "http://web.archive.org/web/20160402065011/http://www.dakessianlaw.com:80/case-evaluation/\n",
      "http://web.archive.org/web/20160611034022/http://www.dakessianlaw.com:80/common/files/rebuild.html\n",
      "http://web.archive.org/web/20160415134238/http://www.dakessianlaw.com:80/contact-us/\n",
      "http://web.archive.org/web/20080407230256/http://www.dakessianlaw.com:80/contact_us.php\n",
      "http://web.archive.org/web/20080413022843/http://www.dakessianlaw.com:80/different.php\n",
      "http://web.archive.org/web/20160401213157/http://www.dakessianlaw.com:80/documents/Marty-Dakessian-VCard.vcf\n",
      "http://web.archive.org/web/20071210235115/http://www.dakessianlaw.com:80/index.php\n",
      "http://web.archive.org/web/20160502012703/http://www.dakessianlaw.com:80/news-information/\n",
      "http://web.archive.org/web/20160401213313/http://www.dakessianlaw.com:80/news-information/2013/\n",
      "http://web.archive.org/web/20160401213203/http://www.dakessianlaw.com:80/news-information/2013/april/\n",
      "http://web.archive.org/web/20160401213209/http://www.dakessianlaw.com:80/news-information/2013/april/dakessian-quoted-in-la-daily-journal-regarding-f/\n",
      "http://web.archive.org/web/20160401213319/http://www.dakessianlaw.com:80/news-information/2013/june/\n",
      "http://web.archive.org/web/20160401213410/http://www.dakessianlaw.com:80/news-information/2013/june/judge-awards-firm-1-2-million-in-fees-in-taxatio/\n",
      "http://web.archive.org/web/20160401213540/http://www.dakessianlaw.com:80/news-information/2015/\n",
      "http://web.archive.org/web/20160401213416/http://www.dakessianlaw.com:80/news-information/2015/january/\n",
      "http://web.archive.org/web/20160401213332/http://www.dakessianlaw.com:80/news-information/2015/january/award-winning-lawyer-marty-dakessian-launches-ca/\n",
      "http://web.archive.org/web/20160401213423/http://www.dakessianlaw.com:80/news-information/2016/\n",
      "http://web.archive.org/web/20160401213216/http://www.dakessianlaw.com:80/news-information/2016/february/\n",
      "http://web.archive.org/web/20160401164049/http://www.dakessianlaw.com:80/news-information/2016/february/california-franchise-tax-board-recognizes-taxpay/\n",
      "http://web.archive.org/web/20160402065029/http://www.dakessianlaw.com:80/news-information/2016/february/marty-dakessian-launches-three-lawyer-california/\n",
      "http://web.archive.org/web/20160401213221/http://www.dakessianlaw.com:80/news-information/2016/january/\n",
      "http://web.archive.org/web/20160401164053/http://www.dakessianlaw.com:80/news-information/2016/january/attorney-dakessian-talks-about-court-upholding-t/\n",
      "http://web.archive.org/web/20160401164121/http://www.dakessianlaw.com:80/news-information/2016/january/california-supreme-court-rules-against-taxpayers/\n",
      "http://web.archive.org/web/20160401213434/http://www.dakessianlaw.com:80/news-information/2016/march/\n",
      "http://web.archive.org/web/20160419141903/http://www.dakessianlaw.com:80/news-information/2016/march/attorney-marty-dakessian-opens-boutique-californ/\n",
      "http://web.archive.org/web/20160403074728/http://www.dakessianlaw.com:80/news-information/categories/articles/\n",
      "http://web.archive.org/web/20160403074411/http://www.dakessianlaw.com:80/news-information/categories/firm-news/\n",
      "http://web.archive.org/web/20160403074514/http://www.dakessianlaw.com:80/news-information/categories/press-releases/\n",
      "http://web.archive.org/web/20160403074348/http://www.dakessianlaw.com:80/news-information/categories/tax-law/\n",
      "http://web.archive.org/web/20160403074526/http://www.dakessianlaw.com:80/news-information/categories/tax-updates/\n",
      "http://web.archive.org/web/20160502012830/http://www.dakessianlaw.com:80/our-lawyers/\n",
      "http://web.archive.org/web/20160502012708/http://www.dakessianlaw.com:80/our-lawyers/marty-dakessian/\n",
      "http://web.archive.org/web/20160509233730/http://www.dakessianlaw.com:80/our-lawyers/ruben-sislyan/\n",
      "http://web.archive.org/web/20160402064955/http://www.dakessianlaw.com:80/privacy-policy/\n",
      "http://web.archive.org/web/20160422072426/http://www.dakessianlaw.com:80/site-map/\n",
      "http://web.archive.org/web/20080418094240/http://www.dakessianlaw.com:80/site_map.php\n",
      "http://web.archive.org/web/20160505155006/http://www.dakessianlaw.com:80/tax-controversy-litigation/\n",
      "http://web.archive.org/web/20160422072429/http://www.dakessianlaw.com:80/tax-controversy-litigation/california-s-unique-tax-system/\n",
      "http://web.archive.org/web/20160403074353/http://www.dakessianlaw.com:80/video-center/\n"
     ]
    }
   ],
   "source": [
    "archive_text = []\n",
    "for x in archive_urls:\n",
    "    g = Goose()\n",
    "    article = g.extract(url=x)\n",
    "    print(x)\n",
    "    archive_text.append({'archive_url': x, 'archive_text': article.title + \" \" + article.cleaned_text})\n",
    "    time.sleep(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract body text of current URLs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://dakessianlaw.com/\n",
      "https://dakessianlaw.com/what-we-do/\n",
      "https://dakessianlaw.com/our-results/\n",
      "https://dakessianlaw.com/news/\n",
      "https://dakessianlaw.com/contact-us/\n",
      "https://dakessianlaw.com/faqs/\n",
      "https://dakessianlaw.com/about-us/\n",
      "https://dakessianlaw.com/privacy/\n",
      "https://dakessianlaw.com/our-team/\n",
      "https://dakessianlaw.com/news/marty-dakessian-named-tax-mvp-in-2017-by-law360/\n",
      "https://dakessianlaw.com/news/california-franchise-tax-board-recognizes-taxpayers-will-appeal-gillette-ruling/\n",
      "https://dakessianlaw.com/news/lawmakers-tax-experts-debate-future-of-boe/\n",
      "https://dakessianlaw.com/news/marty-dakessian-honored-as-a-2016-southern-california-super-lawyer/\n",
      "https://dakessianlaw.com/news/former-nfl-player-sues-california-tax-board-over-residency-issue/\n",
      "https://dakessianlaw.com/news/attorney-dakessian-talks-about-court-upholding-tax-law-that-helps-ca-businesses/\n",
      "https://dakessianlaw.com/2017/?post_type=news\n",
      "https://dakessianlaw.com/news/bill-to-overhaul-california-boe-racing-through-legislature/\n",
      "https://dakessianlaw.com/news/california-supreme-court-rules-against-taxpayers-in-multistate-tax-compact-apportionment-case/\n",
      "https://dakessianlaw.com/category/tax-law/\n",
      "https://dakessianlaw.com/2013/?post_type=news\n",
      "https://dakessianlaw.com/2016/?post_type=news\n",
      "https://dakessianlaw.com/news/successful-challenge-of-los-angeles-business-tax-conferred-significant-public-benefit-court-rules/\n",
      "https://dakessianlaw.com/news/marty-dakessian-named-tax-mvp-in-2016-by-law360/\n",
      "https://dakessianlaw.com/2019/?post_type=news\n",
      "https://dakessianlaw.com/category/articles/\n",
      "https://dakessianlaw.com/news/new-irs-chief-may-help-reform-embattled-agency/\n",
      "https://dakessianlaw.com/news/retroactive-tax-laws-muddying-businesses-future-decisions/\n",
      "https://dakessianlaw.com/news/agency-denies-ftbs-petition-to-rehear-case-involving-football-star-keyshawn-johnson/\n",
      "https://dakessianlaw.com/2018/?post_type=news\n",
      "https://dakessianlaw.com/news/surprise-ruling-says-some-local-taxes-need-no-vote/\n",
      "https://dakessianlaw.com/news/through-the-looking-glass-reflections-on-the-mtc/\n",
      "https://dakessianlaw.com/news/california-governor-appoints-head-of-new-tax-agency/\n",
      "https://dakessianlaw.com/news/a-new-law-reorganizes-californias-tax-system/\n",
      "https://dakessianlaw.com/news/marty-dakessian-featured-in-bloomberg-bna-report-about-las-hotel-tax-on-airbnb/\n",
      "https://dakessianlaw.com/news/attorney-marty-dakessian-opens-boutique-california-tax-law-firm/\n",
      "https://dakessianlaw.com/news/dakessian-quoted-in-la-daily-journal-regarding-ftbs-retroactively-elimination-of-tax-incentive/\n",
      "https://dakessianlaw.com/news/marty-dakessian-launches-three-lawyer-california-tax-boutique/\n",
      "https://dakessianlaw.com/news/appellate-lawyer-zareh-jaltorossian-joins-award-winning-los-angeles-tax-boutique-dakessian-law/\n",
      "https://dakessianlaw.com/news/irs-pick-to-face-tough-questions-on-agencys-direction/\n",
      "https://dakessianlaw.com/news/san-diego-airport-fights-fees-in-hertz-suit-against-port/\n",
      "https://dakessianlaw.com/news/judge-awards-firm-1-2-million-in-fees-in-taxation-case/\n",
      "https://dakessianlaw.com/news/californias-tax-system-overhaul-creates-more-questions-than-answers-for-businesses/\n",
      "https://dakessianlaw.com/category/firm-news-press/\n",
      "https://dakessianlaw.com/news/wayfair-implementation-bill-clears-first-committee-with-unanimous-vote/\n",
      "https://dakessianlaw.com/news/award-winning-lawyer-marty-dakessian-launches-california-tax-boutique/\n",
      "https://dakessianlaw.com/news/ftb-riding-high-after-summary-judgment-win-in-harley-commerce-clause-case/\n"
     ]
    }
   ],
   "source": [
    "current_text = []\n",
    "for x in original_urls:\n",
    "    g = Goose()\n",
    "    article = g.extract(url=x)\n",
    "    print(x)\n",
    "    current_text.append({'current_url': x, 'current_text': article.title + \" \" + article.cleaned_text})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create similarity matrix\n",
    "*This will likely take a while*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "archive = pd.DataFrame.from_dict(archive_text)\n",
    "current = pd.DataFrame.from_dict(current_text)\n",
    "all_urls = []\n",
    "for index, current_row in current.iterrows():\n",
    "    for index, archive_row in archive.iterrows(): \n",
    "        similarity = nlp(current_row[\"current_text\"]).similarity(nlp(archive_row['archive_text']))\n",
    "        all_urls.append({'archive_url': archive_row['archive_url'], 'current_url': current_row['current_url'], 'similarity': similarity })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fix wayback URLs and add to dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for x in all_urls:\n",
    "    x.update({'old_url': re.sub(':80', '', re.sub('http:\\/\\/web\\.archive\\.org\\/web\\/[0-9]*\\/', '', x['archive_url']))})\n",
    "all_urls_pd = pd.DataFrame.from_dict(all_urls)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write to CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_urls_pd.to_csv(csv_output, encoding='utf-8')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}