MatthewCaseres

setwd("~/GitHub/LongMortalityTables/data-raw/2015VBT/2015VBT_RR_ALB")

library(dplyr)
library(readxl)
library(stringr)
library(tidyr)

#You need to set your working directory to the file with all of the excel files
file.list <- list.files(path = ".", pattern='*.xls')

MatthewCaseres

#digits then space then . then lastname comma space firstname then stop before you hit another number
regexName = re.compile(r'\d+\.\s[^,]+,\s[^0-9]+')

def scrapeNames(path):
    #Use pdf reader
    pdfFileObj=open(path, "rb")
    pdfReader = PyPDF2.PdfFileReader(pdfFileObj)
    #Iterate over pages and extract all names
    maxPage = pdfReader.numPages
    allNames = []

MatthewCaseres

def formatName(nameString):
    nameString = re.sub(r'^[0-9]+\.\s','',nameString) #Remove Leading Numbers
    nameString = re.sub('\n','',nameString) #Remove newline
    nameString = nameString.rstrip() #Remove trailing whitespace
    namePair = nameString.split(", ", 1) #Split strings
    return namePair
  
  #Convert all names like so
  [formatName(name) for name in rawNames]

MatthewCaseres

examFolders = ["edu-names-2018-modified/" + examPath for examPath in os.listdir("edu-names-2018-modified") if "Exam" in examPath] #Full path for exam folders

examFiles = []
for examFolder in examFolders:
    for resultFile in os.listdir(examFolder):
        if "names" in resultFile:
            examFiles.append(examFolder + "/" + resultFile)

MatthewCaseres

#Find exam name from file path
regexExamName = re.compile(r'Exam\s[A-Z]*')
#Find exam date from file path
regexDate = re.compile(r'[0-9]{4}-[0-9]{2}')

allNames = []
#examFiles is the list of all paths to all PDF files
for file in examFiles:
    examInfo = regexExamName.search(file).group()
    monthInfo = regexDate.search(file).group()

MatthewCaseres

import pandas as pd
df = pd.DataFrame(allNames, columns = ['first', 'last', 'exam', 'date'])
df.groupby(['exam'])['exam'].count().sort_values(ascending=False)

MatthewCaseres

allUrls = [
    'https://web.archive.org/web/20130620104832/reddit.com/r/fatpeoplehate',
    'https://web.archive.org/web/20130621034233/reddit.com/r/fatpeoplehate',
    'https://web.archive.org/web/20130723124459/reddit.com/r/fatpeoplehate',
    'https://web.archive.org/web/20130804223405/http://www.reddit.com/r/fatpeoplehate/',
    'https://web.archive.org/web/20130919070919/reddit.com/r/fatpeoplehate',
    'https://web.archive.org/web/20131007083812/http://www.reddit.com/r/fatpeoplehate/',
    'https://web.archive.org/web/20131101210726/http://www.reddit.com/r/fatpeoplehate/',
    'https://web.archive.org/web/20131115222834/http://www.reddit.com/r/fatpeoplehate/',
    'https://web.archive.org/web/20131208005710/http://www.reddit.com/r/fatpeoplehate/',

MatthewCaseres

data = []

def insertData(url):
    res = requests.get(url)
    res.raise_for_status()
    soup = bs4.BeautifulSoup(res.text, 'html.parser')

    titles = [title.string for title in soup.select('a.title')]
    users = [username.string for username in soup.select('p.tagline>a')]
    titleLinks = [titleLink.get('href') for titleLink in soup.select('a.title')]

MatthewCaseres

reqCodes = [requests.get(post[3]).status_code for post in data]
df = pd.DataFrame(data, columns=['title', 'author', 'titleLink', 'commentLink'])
df['reqCode'] = reqCodes
df.to_csv("posts.csv", index=False)

MatthewCaseres

allComments = []
allUsers = []

liveLinks = posts[posts['reqCode'] == 200]['commentLink'].tolist()
for url in liveLinks:
    res = requests.get(url)
    res.raise_for_status()
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    allComments += [comment.string for comment in soup.select('div.commentarea .usertext-body div.md>p') if not comment.string == '[deleted]']
    allUsers += [user.string for user in soup.select('.author')]