Performance Testing Scripts

envinfo

#!/bin/sh

echo "CPU Model:             `grep -m1 'model name' /proc/cpuinfo | cut -d' ' -f3- | tr -s ' '`"
echo "CPU/Core Speed (MHz):  `grep -m1 'cpu MHz' /proc/cpuinfo | cut -d' ' -f3` MHz"
echo "RAM:                   `grep 'MemTotal' /proc/meminfo | awk '{ print $2,$3 }'`"
echo "Operating System:      `uname -sr`"
echo "Interconnect:          N/A"
echo "g++ version:           `g++ --version | head -1`"
echo "javac version:         `javac -version 2>&1`"
echo "icpc version:          `icpc  --version | head -1`"
echo "/usr/bin/time version: `/usr/bin/time --version 2>&1`"

perfstat

#!/bin/sh

# Provided under the MIT License:
#
# Copyright (c) 2015 Andrew Kallmeyer <[email protected]>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

# Usage "[program] (args)" [output file]
#
# Runs program $NUM_RUNS times and prints a table of timing data obtained using
# /usr/bin/time. The table columns are as follows:
# #  | Elapsed Time | User Time | System Time |   Memory   | CPU%
#
# The output of the program is saved to the output file if specified
#
# At the end it will also calculate the average, standard deviation, and
# confidence interval (using $Z as a Z-Score) for each column. Replace the value
# in $Z with the appropriate value for your number of runs and confidence level

#95%, one-sided confidence interval using a Student T Distribution for 5 runs
Z=2.132
NUM_RUNS=5

################################################################################

outfile=/dev/null
time_format="Elapsed Time: %e\nUser Time: %U\nSystem Time: %S\nCPU%%: %P\nmem: %M"

elapsed=()
user=()
system=()
mem=()
cpu=()

if [ -n "$2" ]; then
    outfile=$2
fi

echo "#  | Elapsed Time | User Time | System Time |   Memory   | CPU%"
for i in `seq 1 $NUM_RUNS`; do
    #Run the program once
    time_output=`/usr/bin/time --format="$time_format" $1 2>&1 >> $outfile`
    echo -e "\n----------------------------------------\n" >> $outfile 

    #Separate the fields of the time output
    elapsed_i=`echo "$time_output" | grep 'Elapsed Time' | awk '{ print $NF }'`
    user_i=`echo "$time_output" | grep 'User Time' | awk '{ print $NF }'`
    system_i=`echo "$time_output" | grep 'System Time' | awk '{ print $NF }'`
    mem_i=`echo "$time_output" | grep 'mem' | awk '{ print $NF }'`
    cpu_i=`echo "$time_output" | grep 'CPU%' | awk '{ print $NF }'`

    #Append to the arrays
    elapsed=("${elapsed[@]}" "$elapsed_i")
    user=("${user[@]}" "$user_i")
    system=("${system[@]}" "$system_i")
    mem=("${mem[@]}" "$mem_i")
    cpu=("${cpu[@]}" "${cpu_i::${#cpu_i}-1}")

    #Print the table row
    printf "%-2s | %-12s | %-9s | %-11s | %-10s | %s\n" $i $elapsed_i $user_i $system_i $mem_i $cpu_i
done

echo "-----------------------------------------------------------"
#Generate plusses for use with dc later
plusses=`awk "function r(str,n,rep,i) { for(;i<n;i++) rep = rep str; return rep } BEGIN { print r(\"+ \",$NUM_RUNS-1) }"`

#compute the average of each column
elapsed_ave=`echo "3 k ${elapsed[@]} $plusses ${#elapsed[@]}.0 / p" | dc`
user_ave=`echo "3 k ${user[@]} $plusses ${#user[@]}.0 / p" | dc`
system_ave=`echo "3 k ${system[@]} $plusses ${#system[@]}.0 / p" | dc`
mem_ave=`echo "3 k ${mem[@]} $plusses ${#mem[@]}.0 / p" | dc`
cpu_ave=`echo "3 k ${cpu[@]} $plusses ${#cpu[@]}.0 / p" | dc`

#used to generate the dc string for computing the standard deviation
generate_sd_function() {
    arr=("${!1}")
    math="3 k"
    for i in ${arr[@]}; do
        math="$math $i $2 - d *"
    done

    math="$math $3 ${#arr[@]}.0 / v p"
}

#compute the standard deviations
generate_sd_function elapsed[@] "$elapsed_ave" "$plusses"
elapsed_sd=`echo "$math" | dc`
generate_sd_function user[@] "$user_ave" "$plusses"
user_sd=`echo "$math" | dc`
generate_sd_function system[@] "$system_ave" "$plusses"
system_sd=`echo "$math" | dc`
generate_sd_function mem[@] "$mem_ave" "$plusses"
mem_sd=`echo "$math" | dc`
generate_sd_function cpu[@] "$cpu_ave" "$plusses"
cpu_sd=`echo "$math" | dc`

#Compute the confidence intervals
elapsed_ci=`echo "3 k $Z $elapsed_sd * $NUM_RUNS v / p" | dc`
user_ci=`echo "3 k $Z $user_sd * $NUM_RUNS v / p" | dc`
system_ci=`echo "3 k $Z $system_sd * $NUM_RUNS v / p" | dc`
mem_ci=`echo "3 k $Z $mem_sd * $NUM_RUNS v / p" | dc`
cpu_ci=`echo "3 k $Z $cpu_sd * $NUM_RUNS v / p" | dc`

#Print the aggregated data
printf "av | %-12s | %-9s | %-11s | %-10s | %s\n" $elapsed_ave $user_ave $system_ave $mem_ave "$cpu_ave%"
printf "sd | %-12s | %-9s | %-11s | %-10s | %s\n" $elapsed_sd $user_sd $system_sd $mem_sd "$cpu_sd%"
printf "ci | %-12s | %-9s | %-11s | %-10s | %s\n" $elapsed_ci $user_ci $system_ci $mem_ci "$cpu_ci%"