Terminal Procedures Downloader

termproc.py

#!/usr/bin/env python
import re, os, sys, errno, requests, urllib2
from bs4 import BeautifulSoup
from PyPDF2 import PdfFileMerger, PdfFileReader

DOWNLOAD_PATH = "pdf"

IFR = False

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def download_file(src, dst, ident=None):
    """Downloads a file from src to dst"""
    response = urllib2.urlopen(src)
    path = os.path.join(DOWNLOAD_PATH, dst)

    if ident is not None:
        path = os.path.join(DOWNLOAD_PATH, ident, dst)

    if not path.startswith('/'):
        path = os.path.join(os.getcwd(), path)

    if os.path.exists(path):
        return False

    if not os.path.exists(os.path.dirname(path)):
        mkdir_p(os.path.dirname(path))

    print("Downloading to '%s'" % (path))

    with open(path, 'w') as fd:
        fd.write(response.read())
    return True

class TerminalProcedures:


    def __init__(self):
        self.unique = set()
        self.url = "https://www.faa.gov/air_traffic/flight_info/aeronav/digital_products/dtpp/search/results/?cycle=1604&ident="

    def _parseListings(self, page):
        soup = BeautifulSoup(page)
        table = soup.select('#resultsTable')
        if table:
            table = table[0]
            available = {}
            for row in table.findAll('tr'):
                link = row.find('a')
                if link:
                    href = link.get('href')
                    name = str(link.text)
                    if href.endswith('.pdf'):
                        available[href] = name
            return available
        return {}

    def getListings(self, ident):
        url = "%s%s" % (self.url, ident)
        req = requests.get(url)

        if req.ok is True:
            return self._parseListings(req.text)

    @staticmethod
    def getFileName(inpt):
        return inpt.replace(' ', '_').replace('/','-').lower() + ".pdf"

    @staticmethod
    def getAllForIdent(ident):
        ident = ident.upper()
        tp = TerminalProcedures()
        listings = tp.getListings(ident)
        files = []
        unique = []
        for href in listings:
            if listings[href] == 'ALTERNATE MINIMUMS':
                continue

            if listings[href] == 'TAKEOFF MINIMUMS':
                continue

            if 'HOT SPOT' in listings[href]:
                continue

            if IFR is False:
                if 'RNAV' in listings[href]: continue
                if 'ILS OR LOC' in listings[href]: continue

            if href not in unique:
                print("\033[33;01m%s\033[0m (%s)" % (listings[href], href))
                filename = TerminalProcedures.getFileName(listings[href])
                download_file(href, filename, ident)
                files.append(os.path.join(DOWNLOAD_PATH, ident, filename))
                unique.append(href)
        return files

def is_valid_ident(ident):
    regex = re.compile('^[K]?[A-Z0-9]{3}$', re.IGNORECASE)
    return regex.match(ident)

def merge_airports(files, remove_duplicates=True):
    merger = PdfFileMerger()
    added = []
    for ident in files:
        for fname in files[ident]:
            if fname not in added:
                merger.append(PdfFileReader(file(fname, 'rb')))
                added.append(fname)
    name = "output_%s.pdf" % (str('_'.join(files.keys())))
    merger.write(name)

if __name__ == '__main__':
    AGGREGATE = '-a' in sys.argv
    files = {}
    for (i, arg) in enumerate(sys.argv):
        if is_valid_ident(arg):
            files[arg.upper()] = TerminalProcedures.getAllForIdent(arg)

    if AGGREGATE:
        merge_airports(files)