Lognit performance tests

perf.pl

# File: perf.pl
# Description: lognit performance tests
#              this script outputs a csv file with metrics
# Usage: perl perf.pl
# Dependencies (binaries): jmap, pidstat
# Notes: Tested under perl 5.12.4


use feature qw(say);
use IO::Handle;


my $lognit_path='/opt/intelie/lognit';

#replace with your jmap
my $jmap_path='/usr/lib/jvm/java-6-sun-1.6.0.26/bin';

#rate index log line id
my $id;

#total docs indexed 
my $total_docs;


sub start_lognit {
    `sudo $lognit_path/bin/./lognit restart`;
}

sub get_pid {
    substr(`ps aux | grep -v grep | grep lognit | grep jmx | awk '{print \$2}'`, 0, -1);
}

sub get_index_data {
    ('indexMB' => substr(`du -s $lognit_path/data/index | awk '{print \$1/1024}'`, 0, -1));
}

sub get_store_data {
    ('storeMB' => substr(`du -s $lognit_path/data/store | awk '{print \$1/1024}'`, 0, -1));
}

sub get_heap_data {
    my $pid = get_pid;
    ('heapuseMB' => substr(`sudo $jmap_path/./jmap -heap $pid | grep -E "used  " | awk '{print \$3}' | head -n4 | awk '{s += \$1/1048576} END {print s}'`, 0, -1),
     'heapmaxMB' => substr(`sudo $jmap_path/./jmap -heap $pid | grep MaxHeapSize | awk '{print \$3/1048576}'`, 0, -1));
}

sub get_total_memory_data {
    ('TotalMem' => substr(`free -b | grep Mem: | awk '{print \$2/1048576}'`, 0, -1));
}

sub get_indexing_rate_data {
    my $rate = 0;
    my $filename =  substr(`date +"%Y_%m_%d"`, 0, -1) . '.stderrout.log';
    my $line = substr(`grep 'in this session' $lognit_path/logs/$filename | tail -1`, 0, -1);
    if ($line =~ /(?<id>\d*) .* (?<docs>\d*) docs .*\(\+(?<rate>.*)\)$/g) {
        if ($id != $+{id}) {
            $id = $+{id};
            $rate = $+{rate};
            $total_docs = $+{docs};
        }
    }
    return ('IndexingRate' => $rate, 'TotalDocs' => $total_docs) ;
}

sub get_file_descriptors_data {
    my $pid = get_pid;
    ('FileDescriptors' => substr(`sudo ls -l /proc/$pid/fd | wc -l`, 0, -1));
}

sub get_pidstat_data {
    my $pid = shift;
    my $pidstat_data = `pidstat -hdru -p $pid 1 1`;
    my @pidstat_lines = split('\n', $pidstat_data);

    my @pidstat_keys = split(' ', @pidstat_lines[2]);
    #getting rid of '#' from pidstat output
    my @keys = @pidstat_keys[1..@pidstat_keys-1]; 
    my @values = split(' ', @pidstat_lines[3]);

    map {$_.strip} @keys;
    map {$_.strip} @values;

    my %data;
    @data{@keys} = @values;

    delete $data{Command};

    return %data;
}

sub get_data {
     my $data = {get_pidstat_data(get_pid), 
              get_index_data, 
              get_store_data, 
              get_heap_data,
              get_total_memory_data,
              get_indexing_rate_data,
              get_file_descriptors_data};
     return $data;
}

sub run {
    my $data = get_data;

    my $io = IO::Handle->new();
    open(my $fh, '>>output.csv');
    $io->fdopen($fh, 'w');

    #header of csv
    $io->say(join(',', keys(%$data)));
 
    while (1) {
        $data = get_data;
        $io->say(join(',', values(%$data)));
        $io->flush;
        sleep 5;
    }
}

start_lognit;
run

plot.R

# File: plot.R
# Description: Plotting lognit performance tests.
#              It is supposed to be called from plot.sh

data <- read.csv(file="output.csv", sep=",", head=TRUE)


for (column in names(data)) {
    if (column != "Time" && column != "PID" ) {
        plot(data[,c("Time", column)], type="o", col="blue")

    }
}

plot.sh

#!/bin/bash
# File: plot.sh
# Description: run to generate graphs from output.csv file.
# Usage: sh plot.sh
# Dependencies (binaries): R

R CMD BATCH plot.R