Scripts for XDP benchmarking

ethtool_stats.pl

#!/usr/bin/perl -w

=head1 NAME

 ethtool_stats.pl - Sample changing adapter statistics from ethtool -S

=head1 SYNOPSIS

 ethtool_stats.pl --dev DEVICE [options]

 options:
    --dev	Ethernet adapter(s)/device(s) to get stats from.
                (specify --dev more times to sample multiple devices)
    --count	How many seconds sampling will run (default: infinite)
    --sec	Sets sample interval in seconds (default: 1.0 sec)
    --all	List all zero stats
    --help	Brief usage/help message.
    --man	Full documentation.


=head1 DESCRIPTION

 This script shows ethtool (-S|--statistics) stats, but only stats
 that changes.  And then reports stats per sec.

 Created this script because some driver, e.g. mlx5, report false
 stats via ifconfig.

=cut

use strict;
use warnings FATAL => 'all';
use Data::Dumper;
use Pod::Usage;
use Getopt::Long;
use Time::HiRes;

my @DEV    = ();
my $debug  = 0;
my $dumper = 0;
my $help   = 0;
my $man    = 0;
my $all    = 0;
my $count  = 0;
my $delay  = 1;

GetOptions (
    'dev=s'    => \@DEV,
    'count=s'  => \$count,
    'sec=s'    => \$delay,
    'all!'     => \$all,
    'debug=s'  => \$debug,
    'dumper!'  => \$dumper,
    'help|?'   => sub { Getopt::Long::HelpMessage(-verbose => 1) },
    'man'      => \$man,
    ) or pod2usage(2);
pod2usage(-exitstatus => 0, -verbose => 2) if $man;
pod2usage(-exitstatus => 1, -verbose => 1) unless scalar @DEV;

my %STATS;

sub collect_stats($) {
    # Parse ethtool stats and return key=value hash
    my $device = shift;
    my %hash;
    open(ETHTOOL, "sudo /usr/sbin/ethtool -S $device |");
    $hash{timestamp} = Time::HiRes::time();
    while (defined(my $line = <ETHTOOL>)) {
	chomp($line);
	if ($line =~ m/\s*(.+):\s?(\d+)/) {
	    my $key   = $1;
	    my $value = $2;
	    $hash{$key} = $value;
	    print "PARSED: $line -- key:$key val:$value\n" if ($debug > 2);
	} else {
	    print "WARN: could not parse line:\"$line\"\n" if ($debug > 1);
	}
    }
    close(ETHTOOL)
	|| die "ERR: Ethtool --statistics failed on device:$device $!";
    return \%hash;
}

# Example
sub traverse_hash(%) {
    my $hash = shift;
    while(my ($key, $value) = each %$hash) {
	print "key:$key val:$value\n";
    }
}
# Example sort
sub traverse_hash_sorted(%) {
    my $hash = shift;
    my @keys = (sort keys %$hash);
    foreach my $key (@keys) {
	print "key:$key val:$hash->{$key}\n";
    }
}

sub difference($$$) {
    my ($device, $stat, $prev)= @_;
    my $something_changed = 0;
    if (!defined($prev)) {
	return 0;
    }
    # The sleep function might not be accurate enough, and this
    # program also add some delay, thus calculate sampling period by
    # highres timestamps
    my $period = $stat->{timestamp} - $prev->{timestamp};
    print "timestamp $stat->{timestamp} - $prev->{timestamp} = $period\n"
	if $debug;
    if (($period > $delay * 2) || ($period < ($delay / 2))) {
	print " ***WARN***: Sample period ($delay) not accurate ($period)\n";
    }
    delete $prev->{timestamp};

    my @keys = (sort keys %$prev);
    foreach my $key (@keys) {
	my $value_now  = $stat->{$key};
	my $value_prev = $prev->{$key};
	my $diff = ($value_now - $value_prev) / $period;
	next if (($diff == 0) && !$all);
	# Round off number
	$diff = sprintf("%.0f", $diff);
	my $pretty = $diff;
	# Add thousands comma separators (use Number::Format instead?)
	$pretty =~ s/(\d{1,3}?)(?=(\d{3})+$)/$1,/g;
	# Right-justify via printf
	printf("Ethtool(%-8s) stat: %12d (%15s) <= %s /sec\n",
	       $device, $diff, $pretty, $key);
	$something_changed++;
    }
    return $something_changed;
}

sub stats_loop() {
    my $collect = $count + 1; # First round was empty (+1)
    my %prev = ();
    my %stats = ();

    # count == 0 is infinite
    while ( ($count == 0) ? 1 : $collect-- ) {
	print "\nShow adapter(s) (" . join(' ', @DEV) .
	    ") statistics (ONLY that changed!)\n";
	my $changes = 0;
	foreach my $device (@DEV){
		$stats{$device} = collect_stats($device);
		$changes += difference($device,
				       $stats{$device}, $prev{$device});
	}
	if (!scalar keys %prev) {
	    print " ***NOTE***: Collecting stats for next round ($delay sec)\n";
	} elsif (!$changes) {
	    print " ***WARN***: No counters changed\n" ;
	}
	%prev = %stats;
	Time::HiRes::sleep($delay);
    }
}
stats_loop();

#my $HASH = collect_stats($DEV);
#traverse_hash_sorted($HASH);

#print Dumper($DEV) if $dumper;
#print Dumper(\%STATS) if $dumper;
#print Dumper($HASH) if $dumper;

__END__

=head1 AUTHOR

 Jesper Dangaard Brouer <[email protected]>

=cut

mmwatch

#!/usr/bin/env python3

import sys
import time
import subprocess
import datetime
import re


digits_re = re.compile("([0-9eE.+]*)")
to = 2.0
CLS='\033[2J\033[;H'
digit_chars = set('0123456789.')


def isfloat(v):
    try:
        float(v)
    except ValueError:
        return False
    return True

def total_seconds(td):
    return (td.microseconds + (td.seconds + td.days * 24. * 3600) * 10**6) / 10**6

def main(cmd):
    prevp = []
    prevt = None

    while True:
        t0 = datetime.datetime.now()
        out = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).communicate()[0]

        p = digits_re.split(out.decode())

        if len(prevp) != len(p):
            s = p
        else:
            s = []
            i = 0
            for i, (n, o) in enumerate(zip(p, prevp)):
                if isfloat(n) and isfloat(o) and float(n) > float(o):
                    td = t0 - prevt
                    v = (float(n) - float(o)) / total_seconds(td)
                    if v > 1000000000:
                        v, suffix = v / 1000000000., 'g'
                    elif v > 1000000:
                        v, suffix = v / 1000000., 'm'
                    elif v > 1000:
                        v, suffix = v / 1000.,'k'
                    else:
                        suffix = ''
                    s.append('\x1b[7m')
                    s.append('%*s' % (len(n), '%.1f%s/s' % (v, suffix)))
                    s.append('\x1b[0m')
                else:
                    s.append(n)
            s += n[i:]

        prefix = "%sEvery %.1fs: %s\t\t%s" % (CLS, to, ' '.join(cmd), t0)
        sys.stdout.write(prefix + '\n\n' + ''.join(s).rstrip() + '\n')
        sys.stdout.flush()

        prevt = t0
        prevp = p
        time.sleep(to)

if __name__ == '__main__':
    try:
        main(sys.argv[1:])
    except KeyboardInterrupt:
        print('Interrupted')
        sys.exit(0)
    except SystemExit:
        os._exit(0)

read_route_table.sh

#!/bin/bash

VERB=add

if [[ "$1" == "-d" ]]; then
    VERB=del
    shift
fi

GATEWAY="$1"

if [ -z "$GATEWAY" ]; then
    echo "Usage: $0 [-d] <gateway> < iplist.txt" >&2
    exit 1
fi

while read line; do
    echo "route $VERB $line via $GATEWAY"
done | ip -b -

udp_for_benchmarks.py

# Modified from trex, to also vary UDP dport when running multiple streams
from trex_stl_lib.api import *

# Tunable example
#
#trex>profile -f stl/udp_for_benchmarks.py
#
#Profile Information:
#
#
#General Information:
#Filename:         stl/udp_for_benchmarks.py
#Stream count:          1
#
#Specific Information:
#Type:             Python Module
#Tunables:         ['stream_count = 1', 'direction = 0', 'packet_len = 64']
#
#trex>start -f stl/udp_for_benchmarks.py -t  packet_len=128 --port 0
#

class STLS1(object):
    '''
    Generalization of udp_1pkt_simple, can specify number of streams and packet length
    '''
    def create_stream (self, packet_len, stream_count):
        packets = []
        for i in range(stream_count):
            base_pkt = Ether()/IP(src="16.0.0.1",dst="48.0.0.1")/UDP(dport=12+i,sport=1025)
            base_pkt_len = len(base_pkt)
            base_pkt /= 'x' * max(0, packet_len - base_pkt_len)
            packets.append(STLStream(
                packet = STLPktBuilder(pkt = base_pkt),
                mode = STLTXCont()
                ))
        return packets

    def get_streams (self, direction = 0, packet_len = 64, stream_count = 1, **kwargs):
        # create 1 stream
        return self.create_stream(packet_len - 4, stream_count)


# dynamic load - used for trex console or simulator
def register():
    return STLS1()

udp_multi_ip.py

# Modified from trex, to also vary UDP dport when running multiple streams
from trex_stl_lib.api import *
import random

# Quick and dirty martian avoidance: Just don't generate these octets
ALLOWED_OCTETS = list(range(1,254))
ALLOWED_OCTETS.remove(10)
ALLOWED_OCTETS.remove(100)
ALLOWED_OCTETS.remove(127)
ALLOWED_OCTETS.remove(169)
ALLOWED_OCTETS.remove(172)
ALLOWED_OCTETS.remove(192)
ALLOWED_OCTETS.remove(224)
ALLOWED_OCTETS.remove(240)

# Tunable example
#
#trex>profile -f stl/udp_for_benchmarks.py
#
#Profile Information:
#
#
#General Information:
#Filename:         stl/udp_for_benchmarks.py
#Stream count:          1
#
#Specific Information:
#Type:             Python Module
#Tunables:         ['stream_count = 1', 'direction = 0', 'packet_len = 64']
#
#trex>start -f stl/udp_for_benchmarks.py -t  packet_len=128 --port 0
#

class STLS1(object):
    '''
    Generalization of udp_1pkt_simple, can specify number of streams and packet length
    '''
    def create_stream (self, packet_len, stream_count, port_count):
        packets = []
        if port_count < 1:
            port_count = 1

        if stream_count < port_count:
            stream_count = port_count

        for i in range(stream_count):
            port = 12 + (i % port_count)
            dst = "128.%d.%d.%d" % tuple([random.choice(ALLOWED_OCTETS) for i in range(3)])
            base_pkt = Ether()/IP(src="10.70.1.2",dst=dst)/UDP(dport=port,sport=1025)
            base_pkt_len = len(base_pkt)
            base_pkt /= 'x' * max(0, packet_len - base_pkt_len)
            packets.append(STLStream(
                packet = STLPktBuilder(pkt = base_pkt),
                mode = STLTXCont()
                ))
        return packets

    def get_streams (self, direction = 0, packet_len = 64, stream_count = 1, port_count = 1, **kwargs):
        # create 1 stream
        return self.create_stream(packet_len - 4, stream_count, port_count)


# dynamic load - used for trex console or simulator
def register():
    return STLS1()