a simple script for loadtesting solr using it's logs

README.rst

there are many loadtesting tools here.

apache_load_test.js

replays a single apache log piped into it delaying requests to occur at the same rate they did. takes a single argument, the host to make the request to. A cookie can be copied into line 22 to make authenticated requests

multi-apache

takes a list of files on standard in takes one or two regexes as arguments. with one regex it will cut all matching lines with two regexes it will cut from the first occurance of the first to the first occurance of the second calls apache_load_test.js with this cut and the host as the test. version of the filename

apache_load_test.js

#!/usr/bin/env node
var sys = require("sys");
var http = require("http");
var request = require("request");

// this is what you'll be testing
var host = process.argv[2];
var port = 80;
var regex = /"GET (\S+)/;

//open stdin
process.stdin.resume();
process.stdin.setEncoding('utf8');
var data = '';

var lines_processed = 0;
var requests_started = 0;
var requests_done = 0;


var jar = request.jar();
//jar.add(request.cookie("sessionid=SESSIONID COOKIE GOES HERE TO TEST LOGGED IN"));

//split into lines and async
process.stdin.on('data', function (chunk) {
    var foo = data + chunk;
    var lines = foo.split('\n');
    data = lines.pop();
    console.log(lines.length);
    lines.forEach( function(line) {
        lines_processed += 1;
        console.log("processed: " + lines_processed + "started: " + requests_started + " done: " + requests_done);
        process.nextTick(function() {make_request(line);});
        console.log("calling back");
        });
});

//make sure the line is a request, extract the url and make it
var make_request = function make_request (line) {
    var match  =  regex.exec(line);
    if(match) {
        requests_started += 1;
        console.log("processed: " + lines_processed + "started: " + requests_started + " done: " + requests_done);
        var url = match[1];
        console.log(match);
        console.log(url)
        url = "http://" + host + url;
        console.log("Making request to: " + url);
        request({url: url, jar: jar }, function(err, resp, body){
            requests_done += 1;
            console.log("processed: " + lines_processed + "started: " + requests_started + " done: " + requests_done);
            if(err) {console.log(err);}
            console.log('MADE REQUEST TO: '+ url);
            console.log('STATUS: ' + resp.statusCode);
            //console.log('BODY: ' + body);
            });
    }
}

lite_load.js

#!/usr/bin/env node
var sys = require("sys");
var http = require("http");

// this is what you'll be testing
var host = "www.test.wsbtv.com" //process.argv[2];
var conc = 100 //process.argv[3];
var port = 80;
var path = '/';

var success = 0;
var error = 0;
var counter = 0

var make_request = function make_request (client) {
    var request = client.request('GET', path, { 'host': host});
    request.end();
    request.on('response', function(response) {
        if(response.statusCode != 200){
            error++;
            console.log('STATUS: ' + response.statusCode + ' ' + error);
        }
        counter++;
        if (counter%100 == 0){
            console.log("counter: " + counter);
        }
        make_request(client);
    });

}

for(var i=0; i<conc; i++){
    console.log("starting a client");
    var client = http.createClient(port, host);
    //process.nextTick(function() { make_request(client); });
    make_request(client);
}

media_load.js

#!/usr/bin/env node
var fs = require("fs");
var sys = require("sys");
var http = require("http");

// this is what you'll be testing
var host = "test.media.cmgdigital.com" //process.argv[2];
var conc = 100 //process.argv[3];
var port = 80;
var path = '/shared';
var filename = 'img.txt';

var success = 0;
var error = 0;
var counter = 0;
var urls = [];

var path_prefix = "/shared/lt/lt_cache";
var file_txt = fs.readFileSync(filename, 'ascii');

var trim = function trim(s){ return s.replace(/(^\s|\s$)/g, '');}

file_txt.split('\n').forEach(function for_line(line){
  var parts = trim(line).split(',');
  var img_path = trim(parts[0]);
  var width = parseInt(parts[1]);
  var height = parseInt(parts[2]);

  for(var i=1;i<width;i++){
    for(var j=1;j<height;j++){
      urls[urls.length] = path_prefix + "/resize/" + i + "x" + j + "/" + img_path;
      urls[urls.length] = path_prefix + "/thumbnail/" + i + "/" + img_path;
      //urls[urls.length] = path_prefix + "/thumbnail/x" +  j + "/" + img_path;
    }
    console.log(i);
 }
});

console.log("finished building urls: " + urls.length);

var make_request = function make_request (client) {
    // this will just error when it runs out of urls
    if( !urls.length){return;};
    var path = urls.pop()
    var request = client.request('GET', path, { 'host': host});
    request.end();
    request.on('response', function(response) {
        if(response.statusCode != 200){
            error++;
            console.log('STATUS: ' + response.statusCode + ' ' + error + ' PATH: ' + path);
        }
        counter++;
        if (counter%100 == 0){
            console.log("counter: " + counter);
        }
        make_request(client);
    });

}

for(var i=0; i<conc; i++){
    console.log("starting a client");
    var client = http.createClient(port, host);
    //process.nextTick(function() { make_request(client); });
    make_request(client);
}

multi_apache.sh

#!/bin/bash
# To invoke this cd into the directory containing the log files you want to replay
# pipe the names of them into this script
# the script takes a single argument which is what it should search for to slice the logs
# I normally mount the apache log directory onto the box I want to run scripts from
# It also depends on being able to access apache_load_test.js from path which is this called
#
# for example if you mount the log dir into a subdir of this repo and want to run between 14:00 and 16:00
# $ ls *access_log | ../multi_apache.sh "\[18/Sep/2012:1[45]:"




match=$1
endmatch=$2

scriptdir=`dirname $0`

while read line
do
   #choose whether to use cat or bzcat
   cat=cat
   if [ `echo $line|grep "bz2"` ]
     then
       cat=bzcat
   fi

   begin=`$cat $line | grep -nm 1 $match  | cut -d ":" -f 1`
   # either grab all between first and last match if there
   # is one or between the first match and the last endmatch
   if [ $endmatch ]
     then
       end=`$cat $line | grep -n $endmatch  | cut -d ":" -f 1`
     else
       end=`$cat $line | grep -n $match  |tail -1 |cut -d ":" -f 1`
   fi
   site=www.test.`echo $line|cut -d '.' -f 1`.com
   $cat $line| sed "$begin,$end!d"  | $scriptdir/apache_load_test.js $site &
   echo $! >> load_test.pid
done

solr_load_test.js

#!/usr/bin/env node
var sys = require("sys");
var http = require("http");

// this is what you'll be testing
var host = 'localhost';
var port = 8080;
var solr = http.createClient(port, host);
var core_path = '/solr/medley';
var regex = /path=(\/\w+\/)\sparams=\{(.+)\}/g;

//open stdin
process.stdin.resume();
process.stdin.setEncoding('utf8');
var data = '';

//split into lines and async
process.stdin.on('data', function (chunk) {
    foo = data + chunk;
    var lines = foo.split('\n');
    data = lines.pop();
    console.log(lines.length);
    lines.forEach( function(line) {
        process.nextTick(function() {make_request(line);});
        console.log("calling back");
        });
});

//make sure the line is a request, extract the url and make it
var make_request = function make_request (line) {
    console.log("called the shit");
    var match  =  regex.exec(line);
    if(match) {
        var url = core_path + match[1] + '?' + match[2];
        var request = solr.request('GET', url);
        request.end();
    }
}

time_log.sh

#!/bin/bash

ARGS=2
E_BADARGS=85

if [ $# -ne "$ARGS" ]
then
    echo "Usage: `basename $0` filename seconds"
    exit $E_BADARGS
fi

LINES=`wc -l $1 | cut -d' ' -f'1'`

DELAY=$(echo "scale=4; $2 / $LINES" | bc -q 2>/dev/null)

while read p; do
  echo $p
  sleep $DELAY
done < $1

timed_load_test.js

#!/usr/bin/env node
var sys = require("util");
var http = require("http");
var request = require("request");
var moment = require("moment");

var time_offset = null;

// this is what you'll be testing
var host = process.argv[2];
var port = 80;
var regex = /"GET (\S+)/;
var time_regex = /\[([^\]]+)\]/;
var time_format = "DD/MMM/YYYY:H:mm:ss ZZ";

//open stdin
process.stdin.resume();
process.stdin.setEncoding('utf8');
var data = '';

var jar = request.jar();
//uncomment and add a cookie to test logged in
//jar.add(request.cookie("sessionid=COOKIE HERE"));

//split into lines and async
process.stdin.on('data', function (chunk) {
    var foo = data + chunk;
    var lines = foo.split('\n');
    data = lines.pop();
    lines.forEach( function(line) {
        delay = get_delay(line);
        if(delay) {
            setTimeout(function() {make_request(line);}, delay);
            console.log("delaying: " + delay);
        }
        });
});

var get_delay = function get_delay(line){
    var match = time_regex.exec(line);
    if(match) {
        file_time = moment(match[1], time_format).valueOf();
        // this may happen more than once but it shouldn't hurt
        // this breaks streaming live files too
        if(!time_offset){
            var offset = moment().valueOf() - file_time;
            // will break if we try to play logs from the future.
            time_offset = (offset < 1) ? 0 : offset;
            console.log("SETTING START OFFSET TO " + time_offset);
            return 1;
        }
        var delay = file_time + time_offset - moment().valueOf();
        return (delay < 1) ? 1 : delay;
    }
};




//make sure the line is a request, extract the url and make it
var make_request = function make_request (line) {
    var match  =  regex.exec(line);
    if(match) {
        var url = match[1];
        url = "http://" + host + url;
        //console.log("Making request to: " + url);
        request({url: url, jar: jar }, function(err, resp, body){
            if(err) {console.log(err);}
            console.log('URL: '+ url + " STATUS: " + resp.statusCode);
            //console.log('BODY: ' + body);
            });
    }
}