warmer.py

#!/usr/bin/python
import argparse
import urllib2
import logging as log

import re

from multiprocessing import Process
import threading


def warmsuburl(url):
    r = urllib2.Request(url=args.host + url)
    r.add_header('Host', args.hostaddr)
    try:
        urllib2.urlopen(r)
        log.info("warmed : %s", args.host + url)
    except urllib2.HTTPError as e:
        print "HTTPError %s: %s" % (e.code, args.host + url)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('host',
                        help='Target Host')
    parser.add_argument('-u', '--url',
                        required=True,
                        help='url to warm')
    parser.add_argument('-a', '--hostaddr',
                        help='hostaddress')
    parser.add_argument('-s', '--subdironly',
                        action='store_true',
                        help='stay on this subdirectory [does not work on rootdirs]')

    parser.add_argument("-d", "--debug", action='store_true', help="increase output verbosity")
    args = parser.parse_args()

    if args.debug:
        log.basicConfig(format='%(levelname)s:%(message)s', level=log.DEBUG)

    if not args.hostaddr:
        args.hostaddr = args.host

    log.info("host: %s", args.host)
    log.info("hostaddr: %s", args.hostaddr)
    log.info("url : %s", args.url)

    r = urllib2.Request(url=args.host + args.url)
    r.add_header('Host', args.hostaddr)
    response = urllib2.urlopen(r)
    html = response.read()

    # get all relative URL's in HTML
    suburls = re.findall(r'href=[\'"]?(\/[^\'" >]+)', html)

    # search and unique URL's
    if args.subdironly:
        suburls = filter(lambda url: args.url in url, suburls)

    suburls = set(suburls)

    print "found %s URL's to warm in %s" % (len(suburls) ,args.host + args.url)
    for suburl in suburls:
        log.info("found : %s", suburl)
        p = Process(target=warmsuburl, args=(suburl,))
        p.start()
    p.join()