Script to scrape S1 Document URLs for any company

sec_s1_scraper.py

from bs4 import BeautifulSoup
import requests
import pandas as pd


"""
This script prints the URL of the S1 Document given the company name and the path of the cik lookup, more to be added

make sure the dependencies are installed. Runs on python 2.7. 

run from terminal: python sec_scraper.py <path of csv> <company name in quotes>

example: python sec_scraper.py ~/Downloads/cik_ticker.csv 'Facebook Inc'

"""

def find_original_s1_url(url):
    """
    finds the url for the original s1
    """
    r = requests.get(url)
    data = r.text
    soup = BeautifulSoup(data)
    s1_url = soup.find_all('a',href=True)[-5]['href']
    return 'https://www.sec.gov' + s1_url


def get_original_s1_document_url(url):
    """
    finds the url for the s1 document
    """
    original_s1_url = find_original_s1_url(url)
    return 'https://www.sec.gov' + BeautifulSoup(requests.get(original_s1_url).text).find_all('a',href=True)[8]['href']


def get_cik(fn,name):
    """
    gets the cik number given the lookup csv filename and the name of company
    """
    cik_lookup_df = pd.read_csv(fn,delimiter='|')
    cik_lookup_df.columns = [col.lower() for col in cik_lookup_df.columns]
    return cik_lookup_df[cik_lookup_df.name == name]['cik'].values[0]


def create_main_url(cik):
    """
    creates the main url from the cik
    """
    cik = '000' + str(cik)
    return 'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=' + cik + '&type=S-1&dateb=&owner=exclude&count=100'



import argparse
parser = argparse.ArgumentParser()
parser.add_argument("csv_filename")
parser.add_argument("company_name")

args = parser.parse_args()

cik = get_cik(args.csv_filename, args.company_name)

main_url = create_main_url(cik)

print get_original_s1_document_url(main_url)