Generate urls for wrk lua script

get-wrk-script.py

#!/usr/bin/env python
#-*-coding: utf-8 -*-

from BeautifulSoup import BeautifulSoup
from urlparse import urlparse
import urllib2
import json
import sys
import re

if len(sys.argv) < 2 :
    print 'Usage : python %s URL' % sys.argv[0]
    exit(-1)

http = urllib2.build_opener()
http.addheaders = [('User-agent', 'โปรแกรมเมอร์เกรด B')]
links = []
try :
    page = http.open(sys.argv[1])
    soup = BeautifulSoup(page.read())
    url = urlparse(sys.argv[1])
    patt = re.compile('^http*|^\/\/')
    site_url = url.scheme + '://' + url.netloc
    url_patt = re.compile( '^' + site_url +'\/*')

    if url.query:
        links.append(url.path + '?' + url.query)

    # css
    for l in soup.findAll('link'):
        href = l.get('href')
        if (l.get('rel') == 'stylesheet' or l.get('rel') == 'shortcut icon') \
                and url_patt.match(href) \
                or not patt.match(href):
            links.append(href.replace(site_url,''))

    # img
    for s in soup.findAll('img'):
        src = s.get('src')
        if url_patt.match(src) and \
                not patt.match(src):
            links.append(src.replace(site_url,''))

    # script
    for s in soup.findAll('script'):
        src = s.get('src')
        if src and (url_patt.match(src) or not patt.match(src)):
            links.append(src.replace(site_url,''))

    urls_text = json.dumps(links, sort_keys=False, indent=4, ensure_ascii=False)
    urls_text = urls_text.replace('[','{')
    urls_text = urls_text.replace(']','}')
    print 'local urls =', urls_text
    print '''
local count = 1

request = function()
    local url = urls[count]
    count = count + 1
    if count > #urls then
        count = 1
    end
    return wrk.format(
        'GET',
        url
    )
end
    '''

except ValueError:
    print 'Invalid URL'

$ python get-wrk-script.py http://example.com/index.html > index.lua
$ cat index.lua
local urls = {
    "/css/bootstrap.min.css", 
    "/css/bootstrap-theme.min.css", 
    "/imgs/img1.jpg", 
    "/js/jquery.min.js", 
    "/js/bootstrap.min.js"
}

local count = 1

request = function()
    local url = urls[count]
    count = count + 1
    if count > #urls then
        count = 1
    end
    return wrk.format(
        'GET',
        url
    )
end

$ wrk -s index.lua -c1k -t4 -d20s http://example.com/index.html
Running 20s test @ http://127.0.0.1:8080/poll.html
  4 threads and 1000 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    32.13ms   27.83ms 188.69ms   83.31%
    Req/Sec     1.79k     1.07k    6.76k    69.88%
  138122 requests in 20.10s, 811.94MB read
  Socket errors: connect 0, read 1077, write 1, timeout 0
Requests/sec:   6872.79
Transfer/sec:     40.40MB