binarytemple · August 7, 2014 13:33
diff --git a/plotiostat.py b/plotiostat.py
 #!/bin/env python
 # -*- coding: utf-8 -*-
 #
 # -----------------------------------------------------------------------------
 # plotiostat.py, Copyright Bjoern Olausson
 # -----------------------------------------------------------------------------
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 2 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 # GNU General Public License for more details.
 #
 # To view the license visit
 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
 # or write to
 # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301 USA
 # -----------------------------------------------------------------------------
 # -----------------------------------------------------------------------------
 #
 # This script is intended to plot the output from "iostat" from the
 # sysstat tools (http://pagesperso-orange.fr/sebastien.godard/)
 #

 __version__ = '1.6.0'
 __version_info__ = tuple([int(num) for num in __version__.split('.')])

 import sys
 import pylab
 import csv
 import re
 import os
 import pprint
 import itertools
 import numpy as np
 from datetime import datetime
 from optparse import OptionParser, OptionGroup
 from matplotlib.ticker import AutoMinorLocator
 from math import ceil, sqrt
 from glob import glob


 def sort_nicely(l):
    """ Sort the given list in the way that humans expect.
    http://www.codinghorror.com/blog/2007/12/
    sorting-for-humans-natural-sort-order.html"""
    convert = lambda text: int(text) if text.isdigit() else text
    alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
    l.sort(key=alphanum_key)
    return l


 def smooth(x, window_len=10, window='flat'):
    """http://www.scipy.org/Cookbook/SignalSmooth

    smooth the data using a window with requested size.

    This method is based on the convolution of a scaled window with the signal.
    The signal is prepared by introducing reflected copies of the signal
    (with the window size) in both ends so that transient parts are minimized
    in the begining and end part of the output signal.

    input:
        x: the input signal
        window_len: the dimension of the smoothing window; should be an odd
                    integer
        window: the type of window from 'flat', 'hanning', 'hamming',
                'bartlett', 'blackman'
            flat window will produce a moving average smoothing.

    output:
        the smoothed signal

    example:

    t=linspace(-2,2,0.1)
    x=sin(t)+randn(len(t))*0.1
    y=smooth(x)

    see also:

    numpy.hanning, numpy.hamming, numpy.bartlett, numpy.blackman,
    numpy.convolve scipy.signal.lfilter

    TODO: the window parameter could be the window itself
    TODO: if an array instead of a string
    """

    if x.ndim != 1:
        raise ValueError("smooth only accepts 1 dimension arrays.")

    if x.size < window_len:
        print x.size, window_len
        raise ValueError("Input vector needs to be bigger than window size.")

    if window_len < 3:
        return x

    if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
        raise ValueError(
            "Window is on of 'flat', 'hanning', 'hamming',"
            "'bartlett', 'blackman'")

    s = np.r_[2 * x[0] - x[window_len - 1::-1], x,
              2 * x[-1] - x[-1:-window_len:-1]]

    # print(len(s))
    if window == 'flat':  # moving average
        w = np.ones(window_len, 'd')
    else:
        w = eval('np.' + window + '(window_len)')

    y = np.convolve(w / w.sum(), s, mode='same')
    return y[window_len:-window_len + 1]


 usage = "usage: %prog -f logfile_*"
 opa = OptionParser(usage=usage, version=__version__)

 opa.add_option("-f", action="store", dest="logfiles", metavar="FILE",
               help="iostot log file(s). Quote strings containing wildcards")

 opa.add_option("-a", action="store_true", dest="annotate",
               help="Annotate max. peak for each dataset")

 opa.add_option("-s", action="store", dest="smoothwin", metavar="INT",
               type="int", help="Windows size to smooth plot")

 opa.add_option("-d", action="store", dest="device", metavar="STR",
               type="str", help="Storage device to plot (sda, sda1, sdc, ...)")

 opa.add_option("-c", action="append", dest="category", metavar="STR",
               type="str",
               help="Device category to plot(rrqm / s, avgrq - sz, ...)."
               "Use ALL to plot all.")

 opa.add_option("-p", action="append", dest="cpu_category", metavar="STR",
               type="str", help="CPU category to plot (%user, % nice, ...)."
               "Use ALL to plot all.")

 opa.add_option("-i", action="store_true", dest="interactive", default=False,
               help="Show interactive plot")

 opa.add_option("-t", action="store_true", dest="timestamp", default=False,
               help="Add timestamp instead of numbers as X tick label."
               "iostat must be invoked with S_TIME_FORMAT=ISO")

 opa.add_option("-n", action="store", dest="tsint", metavar="INT",
               default=0, help="Number of timestamp ticks on the X axis")

 opa.add_option("-r", action="store", dest="plotrange", metavar="START:STOP",
               type="str", default="0:0",
               help="Data range to plot. Use numpy array synthax")

 (options, args) = opa.parse_args()

 if options.logfiles is None:
    opa.error('No logfile specified')

 if options.device is None:
    opa.error('No device specified')

 LOGFILES = sort_nicely(glob(options.logfiles))
 ANNOTATE = options.annotate
 SMOOTHWIN = options.smoothwin
 DEVICE = options.device
 CATEGORY = options.category
 CPU_CATEGORY = options.cpu_category
 LOGNAMES = [os.path.basename(name) for name in LOGFILES]
 INTERACTIVE = options.interactive
 TIMESTAMP = options.timestamp
 TSINT = options.tsint
 PS = int(options.plotrange.split(":")[0])
 PE = int(options.plotrange.split(":")[1])

 # Plot title
 TITLE = "%(F)s (smoothwindow: %(S)s)" % {"F": LOGNAMES[0], "S": SMOOTHWIN}

 print "Plotting the content of:"
 pprint.pprint(LOGNAMES)

 # Prepare the plot
 fig = pylab.figure(figsize=(15, 15))
 minorLocator = AutoMinorLocator()
 pylab.title(TITLE)

 # pylab.suptitle("xxxxxxx", fontsize=18)
 # pylab.figtext(0.5,0.99,'xxxxxxx',
 #                fontsize=18, ha='center')

 # RegEx to catch the timestamp "2013-03-02T11:33:00+0100"
 Dre = "[0-9]{4}-[0-9]{2}-[0-9]{2}"
 Tre = "[0-9]{2}\:[0-9]{2}\:[0-9]{2}"
 Zre = "[0-9]{4}"
 RT = re.compile(
    r"^(%(d)s)T(%(t)s)\+(%(z)s)$" % {"d": Dre, "t": Tre, "z": Zre})

 AV_CAT = []
 CPU_AV_CAT = []
 TS = []

 ## --------- Start sourcing files --------- ##

 for FILE, FILENAME in itertools.izip(LOGFILES, LOGNAMES):
    with open(FILE, 'rb') as f:
        for line in f:
            linelist = line.split()
            # Get the timestamp. Timestamp has to be in ISO format!
            # export the environment variable S_TIME_FORMAT=ISO
            if TIMESTAMP:
                TS_LINE = RT.match(line)
                if TS_LINE:
                    date = TS_LINE.group(1).replace("-", ".")
                    time = TS_LINE.group(2)
                    TS.append("%(d)s\n%(t)s" % {"d": date, "t": time})

            # Get the average CPU stats
            if CPU_CATEGORY:
                try:
                    if linelist[0] == "avg-cpu:" and len(CPU_AV_CAT) == 0:
                        CPU_AV_CAT = linelist[1:]
                        if "ALL" in CPU_CATEGORY:
                            CPU_CATEGORY = CPU_AV_CAT

                    if linelist[0] == "avg-cpu:":
                        # if "ALL" in CPU_CATEGORY:
                            # CPU_CATEGORY = CPU_AV_CAT
                        linelist = f.next().split()
                        CPU_COLLUMNS = [
                            CPU_AV_CAT.index(c) for c in CPU_CATEGORY]
                        cpu_values = [linelist[i] for i in CPU_COLLUMNS]
                        try:
                            Y2 = np.vstack(
                                (Y2, np.array(cpu_values, dtype=float)))
                        except (NameError), e:
                            Y2 = np.array(cpu_values, dtype=float)
                except (IndexError), e:
                    pass

            #-----------------------------------------------------------------#
            # Get the device stats
            try:
                if linelist[0] == "Device:" and len(AV_CAT) == 0:
                    AV_CAT = linelist
                    if "ALL" in CATEGORY:
                        CATEGORY = AV_CAT[1:]
                    COLLUMNS = [AV_CAT.index(c) for c in CATEGORY]
            except (IndexError), e:
                pass

            # Store the selected properties in values_np
            try:
                if linelist[0] == DEVICE:
                    values = [linelist[i] for i in COLLUMNS]
                    try:
                        Y = np.vstack((Y, np.array(values, dtype=float)))
                    except (NameError), e:
                        Y = np.array(values, dtype=float)
            except (IndexError), e:
                pass

        if len(TS) == 0 and TIMESTAMP != 0:
            print "%(f)s containts no timestep!" % {"f": FILE}
            sys.exit(1)

        try:
            NUMPROP = Y.shape[1]
        except NameError, e:
            print "Device \"%(d)s\" not found" % {"d": DEVICE}
            sys.exit(1)

        if CPU_CATEGORY:
            try:
                NUMCPUPROP = Y2.shape[1]
            except NameError, e:
                print "CPU properties not found"
                sys.exit(1)

        if SMOOTHWIN is not None:
            print "Smoothing windows size: %(s)s" % {"s": SMOOTHWIN}
            # Smooth device data
            for N in range(NUMPROP):
                ONEPROP = Y[:, N]
                YSO = smooth(ONEPROP, int(SMOOTHWIN))
                try:
                    YS = np.vstack((YS, YSO))
                except Exception, e:
                    YS = YSO
            Y = YS.transpose()
            # Clear YS, Y2S so we can reuse it
            YS = ""

            # Smooth CPU data
            for N in range(NUMCPUPROP):
                ONEPROP = Y2[:, N]
                YSO = smooth(ONEPROP, int(SMOOTHWIN))
                try:
                    YS = np.vstack((YS, YSO))
                except Exception, e:
                    YS = YSO
            Y2 = YS.transpose()
            # Clear YS so we can reuse it for the next file
            YS = ""

 ## --------- Done sourcing files --------- ##

 if PS or PE:
    Y = Y[PS:PE]

 MINYMAXY = [np.amin(Y) - np.amin(Y) * 0.05, np.amax(Y) + np.amax(Y) * 0.05]

 if '%util' in CATEGORY and len(CATEGORY) > 1:
    UTIL_INDEX = CATEGORY.index("%util")
    del CATEGORY[UTIL_INDEX]

    try:
        CPU_CATEGORY.append("%util")
    except AttributeError, e:
        CPU_CATEGORY = ["%util"]

    UTIL_Y = np.reshape(Y[:, UTIL_INDEX], (-1, 1))

    try:
        Y2 = np.hstack((Y2, UTIL_Y))
    except NameError, e:
        Y2 = np.array(UTIL_Y)

    if PS or PE:
        Y2 = Y2[PS:PE]

    Y = np.delete(Y, UTIL_INDEX, 1)
    # sys.exit(0)

 ax1 = fig.add_subplot(1, 1, 1)
 # pylab.xlim([0, X[-1] + X[-1] * 0.05])
 ax1.set_ylim(MINYMAXY)
 ax1.set_xlabel("Time")
 ax1.set_ylabel(CATEGORY)
 ax1.tick_params(axis='x', width=2,
                length=8, color='black', bottom='on',
                direction='out')
 ax1.xaxis.set_minor_locator(minorLocator)
 ax1.tick_params(axis='x', which='minor', width=1,
                length=4, color='b', bottom='on',
                direction='out')

 if ANNOTATE:
    # Annotate peak for each dataset
    for dataset in range(Y.shape[1]):
        maxvaly = np.amax(Y[:, dataset])
        maxvalx = np.argmax(Y[:, dataset])

        if TIMESTAMP:
            # print TS[2814]
            # print TS[2905]
            ANNO_TXT = "%(y)s (%(x)s)" % {
                "y": maxvaly, "x": TS[maxvalx].replace("\n", " ")}
        else:
            ANNO_TXT = "%(y)s (%(x)s)" % {"y": maxvaly, "x": maxvalx}

        ax1.annotate(ANNO_TXT, xy=(maxvalx, maxvaly),  xycoords='data',
                     xytext=(-15, 10), textcoords='offset points',
                     arrowprops=dict(facecolor='black', shrink=0.05),
                     horizontalalignment='center', verticalalignment='bottom',
                     )

 ax1.plot(Y)
 leg1 = ax1.legend(CATEGORY, loc="upper right", prop={'size': 15})
 # set the linewidth of the legend objects
 for i in range(len(leg1.legendHandles)):
    leg1.legendHandles[i].set_linewidth(2.0)

 if CPU_CATEGORY:
    ax2 = ax1.twinx()
    ax2.set_ylim((0, 110))
    ax2.set_xlabel("Time")
    ax2.set_ylabel("%")
    ax2.tick_params(axis='x', width=2,
                    length=8, color='black', bottom='on',
                    direction='out')
    ax2.xaxis.set_minor_locator(minorLocator)
    ax2.tick_params(axis='x', which='minor', width=1,
                    length=4, color='b', bottom='on',
                    direction='out')

    if ANNOTATE:
        # Annotate peak for each dataset
        for dataset in range(Y2.shape[1]):
            maxvaly = np.amax(Y2[:, dataset])
            maxvalx = np.argmax(Y2[:, dataset])

            if TIMESTAMP:
                # print TS[2814]
                # print TS[2905]
                ANNO_TXT = "%(y)s (%(x)s)" % {
                    "y": maxvaly, "x": TS[maxvalx].replace("\n", " ")}
            else:
                ANNO_TXT = "%(y)s (%(x)s)" % {"y": maxvaly, "x": maxvalx}

            ax2.annotate(ANNO_TXT, xy=(maxvalx, maxvaly),  xycoords='data',
                         xytext=(-15, 10), textcoords='offset points',
                         arrowprops=dict(facecolor='black', shrink=0.05),
                         horizontalalignment='center',
                         verticalalignment='bottom',
                         )

    ax2.plot(Y2, linestyle=":")
    leg2 = ax2.legend(CPU_CATEGORY, loc="center right", prop={'size': 15})
    # set the linewidth of the legend objects
    for i in range(len(leg2.legendHandles)):
        leg2.legendHandles[i].set_linewidth(2.0)

 if TIMESTAMP:
    if TSINT != 0:
        ax1.xaxis.set_major_locator(pylab.MaxNLocator(TSINT))
    TSloc = ax1.xaxis.get_ticklocs()
    TSint = [TS[int(i)] for i in TSloc[:-1]]
    ax1.set_xticklabels(TSint, ha="center")
    for tick in ax1.xaxis.iter_ticks():
        tick[0].label2On = False
        tick[0].label1On = True
        tick[0].label1.set_rotation(90)
    ax1.tick_params(axis='x', width=2,
                    length=8, color='black', bottom='on',
                    direction='out')
    # ax1.tick_params(axis='x', direction='out')

 # pylab.subplots_adjust(
 #    left=0.05, right=0.995, bottom=0.05, top=0.9, wspace=0.1, hspace=0.2)
 pylab.tight_layout()

 nf = "%(n)s" % {"n": os.path.splitext(LOGNAMES[0])[0]}
 nc = "%(n)s" % {"n": "-".join(CATEGORY).replace("/", "")}

 if CPU_CATEGORY:
    ncc = "%(n)s" % {"n": "-".join(CPU_CATEGORY).replace("/", "").replace("%", "")}
    imgname = "%(a)s_-_%(b)s_-_%(c)s.png" % {"a": nf,
                                            "b": nc,
                                            "c": ncc,
                                            }
 else:
    imgname = "%(a)s_-_%(b)s.png" % {"a": nf,
                                    "b": nc,
                                    }

 # You might want to add bbox_inches='tight' to pylab.savefig
 pylab.savefig(imgname)

 print '''
 IOSTAT HELP:
 Reports
 The iostat command generates three types of reports, the CPU Utilization report, the Device Utilization report and the Network Filesystem report.

 CPU Utilization Report:
 The first report generated by the iostat command is the CPU Utilization Report. For multiprocessor systems, the CPU values are global averages among all processors. The report has the following format:
 %user
 Show the percentage of CPU utilization that occurred while executing at the user level (application).
 %nice
 Show the percentage of CPU utilization that occurred while executing at the user level with nice priority.
 %system
 Show the percentage of CPU utilization that occurred while executing at the system level (kernel).
 %iowait
 Show the percentage of time that the CPU or CPUs were idle during which the system had an outstanding disk I/O request.
 %steal
 Show the percentage of time spent in involuntary wait by the virtual CPU or CPUs while the hypervisor was servicing another virtual processor.
 %idle
 Show the percentage of time that the CPU or CPUs were idle and the system did not have an outstanding disk I/O request.
 Device Utilization Report
 The second report generated by the iostat command is the Device Utilization Report. The device report provides statistics on a per physical device or partition basis. Block devices for which statistics are to be displayed may be entered on the command line. Partitions may also be entered on the command line providing that option -x is not used. If no device nor partition is entered, then statistics are displayed for every device used by the system, and providing that the kernel maintains statistics for it. If the ALL keyword is given on the command line, then statistics are displayed for every device defined by the system, including those that have never been used. The report may show the following fields, depending on the flags used:

 Device:
 This column gives the device (or partition) name, which is displayed as hdiskn with 2.2 kernels, for the nth device. It is displayed as devm-n with 2.4 kernels, where m is the major number of the device, and n a distinctive number. With newer kernels, the device name as listed in the /dev directory is displayed.
 tps
 Indicate the number of transfers per second that were issued to the device. A transfer is an I/O request to the device. Multiple logical requests can be combined into a single I/O request to the device. A transfer is of indeterminate size.
 Blk_read/s
 Indicate the amount of data read from the device expressed in a number of blocks per second. Blocks are equivalent to sectors with kernels 2.4 and later and therefore have a size of 512 bytes. With older kernels, a block is of indeterminate size.
 Blk_wrtn/s
 Indicate the amount of data written to the device expressed in a number of blocks per second.
 Blk_read
 The total number of blocks read.
 Blk_wrtn
 The total number of blocks written.
 kB_read/s
 Indicate the amount of data read from the device expressed in kilobytes per second.
 kB_wrtn/s
 Indicate the amount of data written to the device expressed in kilobytes per second.
 kB_read
 The total number of kilobytes read.
 kB_wrtn
 The total number of kilobytes written.
 MB_read/s
 Indicate the amount of data read from the device expressed in megabytes per second.
 MB_wrtn/s
 Indicate the amount of data written to the device expressed in megabytes per second.
 MB_read
 The total number of megabytes read.
 MB_wrtn
 The total number of megabytes written.
 rrqm/s
 The number of read requests merged per second that were queued to the device.
 wrqm/s
 The number of write requests merged per second that were queued to the device.
 r/s
 The number of read requests that were issued to the device per second.
 w/s
 The number of write requests that were issued to the device per second.
 rsec/s
 The number of sectors read from the device per second.
 wsec/s
 The number of sectors written to the device per second.
 rkB/s
 The number of kilobytes read from the device per second.
 wkB/s
 The number of kilobytes written to the device per second.
 rMB/s
 The number of megabytes read from the device per second.
 wMB/s
 The number of megabytes written to the device per second.
 avgrq-sz
 The average size (in sectors) of the requests that were issued to the device.
 avgqu-sz
 The average queue length of the requests that were issued to the device.
 await
 The average time (in milliseconds) for I/O requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.
 svctm
 The average service time (in milliseconds) for I/O requests that were issued to the device. Warning! Do not trust this field any more. This field will be removed in a future sysstat version.
 %util
 Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%.
 Network Filesystem report
 The Network Filesystem (NFS) report provides statistics for each mounted network filesystem. The report shows the following fields:

 Filesystem:
 This columns shows the hostname of the NFS server followed by a colon and by the directory name where the network filesystem is mounted.
 rBlk_nor/s
 Indicate the number of blocks read by applications via the read(2) system call interface. A block has a size of 512 bytes.
 wBlk_nor/s
 Indicate the number of blocks written by applications via the write(2) system call interface.
 rBlk_dir/s
 Indicate the number of blocks read from files opened with the O_DIRECT flag.
 wBlk_dir/s
 Indicate the number of blocks written to files opened with the O_DIRECT flag.
 rBlk_svr/s
 Indicate the number of blocks read from the server by the NFS client via an NFS READ request.
 wBlk_svr/s
 Indicate the number of blocks written to the server by the NFS client via an NFS WRITE request.
 rkB_nor/s
 Indicate the number of kilobytes read by applications via the read(2) system call interface.
 wkB_nor/s
 Indicate the number of kilobytes written by applications via the write(2) system call interface.
 rkB_dir/s
 Indicate the number of kilobytes read from files opened with the O_DIRECT flag.
 wkB_dir/s
 Indicate the number of kilobytes written to files opened with the O_DIRECT flag.
 rkB_svr/s
 Indicate the number of kilobytes read from the server by the NFS client via an NFS READ request.
 wkB_svr/s
 Indicate the number of kilobytes written to the server by the NFS client via an NFS WRITE request.
 rMB_nor/s
 Indicate the number of megabytes read by applications via the read(2) system call interface.
 wMB_nor/s
 Indicate the number of megabytes written by applications via the write(2) system call interface.
 rMB_dir/s
 Indicate the number of megabytes read from files opened with the O_DIRECT flag.
 wMB_dir/s
 Indicate the number of megabytes written to files opened with the O_DIRECT flag.
 rMB_svr/s
 Indicate the number of megabytes read from the server by the NFS client via an NFS READ request.
 wMB_svr/s
 Indicate the number of megabytes written to the server by the NFS client via an NFS WRITE request.
 ops/s
 Indicate the number of operations that were issued to the filesystem per second.
 rops/s
 Indicate the number of 'read' operations that were issued to the filesystem per second.
 wops/s
 Indicate the number of 'write' operations that were issued to the filesystem per second.
 '''

 if INTERACTIVE:
    pylab.show()
	#!/bin/env python
	# -- coding: utf-8 --
	#
	# -----------------------------------------------------------------------------
	# plotiostat.py, Copyright Bjoern Olausson
	# -----------------------------------------------------------------------------
	# This program is free software; you can redistribute it and/or modify
	# it under the terms of the GNU General Public License as published by
	# the Free Software Foundation; either version 2 of the License, or
	# (at your option) any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# To view the license visit
	# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
	# or write to
	# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
	# 02110-1301 USA
	# -----------------------------------------------------------------------------
	# -----------------------------------------------------------------------------
	#
	# This script is intended to plot the output from "iostat" from the
	# sysstat tools (http://pagesperso-orange.fr/sebastien.godard/)
	#

	__version__ = '1.6.0'
	__version_info__ = tuple([int(num) for num in __version__.split('.')])

	import sys
	import pylab
	import csv
	import re
	import os
	import pprint
	import itertools
	import numpy as np
	from datetime import datetime
	from optparse import OptionParser, OptionGroup
	from matplotlib.ticker import AutoMinorLocator
	from math import ceil, sqrt
	from glob import glob


	def sort_nicely(l):
	""" Sort the given list in the way that humans expect.
	http://www.codinghorror.com/blog/2007/12/
	sorting-for-humans-natural-sort-order.html"""
	convert = lambda text: int(text) if text.isdigit() else text
	alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
	l.sort(key=alphanum_key)
	return l


	def smooth(x, window_len=10, window='flat'):
	"""http://www.scipy.org/Cookbook/SignalSmooth

	smooth the data using a window with requested size.

	This method is based on the convolution of a scaled window with the signal.
	The signal is prepared by introducing reflected copies of the signal
	(with the window size) in both ends so that transient parts are minimized
	in the begining and end part of the output signal.

	input:
	x: the input signal
	window_len: the dimension of the smoothing window; should be an odd
	integer
	window: the type of window from 'flat', 'hanning', 'hamming',
	'bartlett', 'blackman'
	flat window will produce a moving average smoothing.

	output:
	the smoothed signal

	example:

	t=linspace(-2,2,0.1)
	x=sin(t)+randn(len(t))*0.1
	y=smooth(x)

	see also:

	numpy.hanning, numpy.hamming, numpy.bartlett, numpy.blackman,
	numpy.convolve scipy.signal.lfilter

	TODO: the window parameter could be the window itself
	TODO: if an array instead of a string
	"""

	if x.ndim != 1:
	raise ValueError("smooth only accepts 1 dimension arrays.")

	if x.size < window_len:
	print x.size, window_len
	raise ValueError("Input vector needs to be bigger than window size.")

	if window_len < 3:
	return x

	if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
	raise ValueError(
	"Window is on of 'flat', 'hanning', 'hamming',"
	"'bartlett', 'blackman'")

	s = np.r_[2 * x[0] - x[window_len - 1::-1], x,
	2 * x[-1] - x[-1:-window_len:-1]]

	# print(len(s))
	if window == 'flat': # moving average
	w = np.ones(window_len, 'd')
	else:
	w = eval('np.' + window + '(window_len)')

	y = np.convolve(w / w.sum(), s, mode='same')
	return y[window_len:-window_len + 1]


	usage = "usage: %prog -f logfile_*"
	opa = OptionParser(usage=usage, version=__version__)

	opa.add_option("-f", action="store", dest="logfiles", metavar="FILE",
	help="iostot log file(s). Quote strings containing wildcards")

	opa.add_option("-a", action="store_true", dest="annotate",
	help="Annotate max. peak for each dataset")

	opa.add_option("-s", action="store", dest="smoothwin", metavar="INT",
	type="int", help="Windows size to smooth plot")

	opa.add_option("-d", action="store", dest="device", metavar="STR",
	type="str", help="Storage device to plot (sda, sda1, sdc, ...)")

	opa.add_option("-c", action="append", dest="category", metavar="STR",
	type="str",
	help="Device category to plot(rrqm / s, avgrq - sz, ...)."
	"Use ALL to plot all.")

	opa.add_option("-p", action="append", dest="cpu_category", metavar="STR",
	type="str", help="CPU category to plot (%user, % nice, ...)."
	"Use ALL to plot all.")

	opa.add_option("-i", action="store_true", dest="interactive", default=False,
	help="Show interactive plot")

	opa.add_option("-t", action="store_true", dest="timestamp", default=False,
	help="Add timestamp instead of numbers as X tick label."
	"iostat must be invoked with S_TIME_FORMAT=ISO")

	opa.add_option("-n", action="store", dest="tsint", metavar="INT",
	default=0, help="Number of timestamp ticks on the X axis")

	opa.add_option("-r", action="store", dest="plotrange", metavar="START:STOP",
	type="str", default="0:0",
	help="Data range to plot. Use numpy array synthax")

	(options, args) = opa.parse_args()

	if options.logfiles is None:
	opa.error('No logfile specified')

	if options.device is None:
	opa.error('No device specified')

	LOGFILES = sort_nicely(glob(options.logfiles))
	ANNOTATE = options.annotate
	SMOOTHWIN = options.smoothwin
	DEVICE = options.device
	CATEGORY = options.category
	CPU_CATEGORY = options.cpu_category
	LOGNAMES = [os.path.basename(name) for name in LOGFILES]
	INTERACTIVE = options.interactive
	TIMESTAMP = options.timestamp
	TSINT = options.tsint
	PS = int(options.plotrange.split(":")[0])
	PE = int(options.plotrange.split(":")[1])

	# Plot title
	TITLE = "%(F)s (smoothwindow: %(S)s)" % {"F": LOGNAMES[0], "S": SMOOTHWIN}

	print "Plotting the content of:"
	pprint.pprint(LOGNAMES)

	# Prepare the plot
	fig = pylab.figure(figsize=(15, 15))
	minorLocator = AutoMinorLocator()
	pylab.title(TITLE)

	# pylab.suptitle("xxxxxxx", fontsize=18)
	# pylab.figtext(0.5,0.99,'xxxxxxx',
	# fontsize=18, ha='center')

	# RegEx to catch the timestamp "2013-03-02T11:33:00+0100"
	Dre = "[0-9]{4}-[0-9]{2}-[0-9]{2}"
	Tre = "[0-9]{2}\:[0-9]{2}\:[0-9]{2}"
	Zre = "[0-9]{4}"
	RT = re.compile(
	r"^(%(d)s)T(%(t)s)\+(%(z)s)$" % {"d": Dre, "t": Tre, "z": Zre})

	AV_CAT = []
	CPU_AV_CAT = []
	TS = []

	## --------- Start sourcing files --------- ##

	for FILE, FILENAME in itertools.izip(LOGFILES, LOGNAMES):
	with open(FILE, 'rb') as f:
	for line in f:
	linelist = line.split()
	# Get the timestamp. Timestamp has to be in ISO format!
	# export the environment variable S_TIME_FORMAT=ISO
	if TIMESTAMP:
	TS_LINE = RT.match(line)
	if TS_LINE:
	date = TS_LINE.group(1).replace("-", ".")
	time = TS_LINE.group(2)
	TS.append("%(d)s\n%(t)s" % {"d": date, "t": time})

	# Get the average CPU stats
	if CPU_CATEGORY:
	try:
	if linelist[0] == "avg-cpu:" and len(CPU_AV_CAT) == 0:
	CPU_AV_CAT = linelist[1:]
	if "ALL" in CPU_CATEGORY:
	CPU_CATEGORY = CPU_AV_CAT

	if linelist[0] == "avg-cpu:":
	# if "ALL" in CPU_CATEGORY:
	# CPU_CATEGORY = CPU_AV_CAT
	linelist = f.next().split()
	CPU_COLLUMNS = [
	CPU_AV_CAT.index(c) for c in CPU_CATEGORY]
	cpu_values = [linelist[i] for i in CPU_COLLUMNS]
	try:
	Y2 = np.vstack(
	(Y2, np.array(cpu_values, dtype=float)))
	except (NameError), e:
	Y2 = np.array(cpu_values, dtype=float)
	except (IndexError), e:
	pass

	#-----------------------------------------------------------------#
	# Get the device stats
	try:
	if linelist[0] == "Device:" and len(AV_CAT) == 0:
	AV_CAT = linelist
	if "ALL" in CATEGORY:
	CATEGORY = AV_CAT[1:]
	COLLUMNS = [AV_CAT.index(c) for c in CATEGORY]
	except (IndexError), e:
	pass

	# Store the selected properties in values_np
	try:
	if linelist[0] == DEVICE:
	values = [linelist[i] for i in COLLUMNS]
	try:
	Y = np.vstack((Y, np.array(values, dtype=float)))
	except (NameError), e:
	Y = np.array(values, dtype=float)
	except (IndexError), e:
	pass

	if len(TS) == 0 and TIMESTAMP != 0:
	print "%(f)s containts no timestep!" % {"f": FILE}
	sys.exit(1)

	try:
	NUMPROP = Y.shape[1]
	except NameError, e:
	print "Device \"%(d)s\" not found" % {"d": DEVICE}
	sys.exit(1)

	if CPU_CATEGORY:
	try:
	NUMCPUPROP = Y2.shape[1]
	except NameError, e:
	print "CPU properties not found"
	sys.exit(1)

	if SMOOTHWIN is not None:
	print "Smoothing windows size: %(s)s" % {"s": SMOOTHWIN}
	# Smooth device data
	for N in range(NUMPROP):
	ONEPROP = Y[:, N]
	YSO = smooth(ONEPROP, int(SMOOTHWIN))
	try:
	YS = np.vstack((YS, YSO))
	except Exception, e:
	YS = YSO
	Y = YS.transpose()
	# Clear YS, Y2S so we can reuse it
	YS = ""

	# Smooth CPU data
	for N in range(NUMCPUPROP):
	ONEPROP = Y2[:, N]
	YSO = smooth(ONEPROP, int(SMOOTHWIN))
	try:
	YS = np.vstack((YS, YSO))
	except Exception, e:
	YS = YSO
	Y2 = YS.transpose()
	# Clear YS so we can reuse it for the next file
	YS = ""

	## --------- Done sourcing files --------- ##

	if PS or PE:
	Y = Y[PS:PE]

	MINYMAXY = [np.amin(Y) - np.amin(Y) * 0.05, np.amax(Y) + np.amax(Y) * 0.05]

	if '%util' in CATEGORY and len(CATEGORY) > 1:
	UTIL_INDEX = CATEGORY.index("%util")
	del CATEGORY[UTIL_INDEX]

	try:
	CPU_CATEGORY.append("%util")
	except AttributeError, e:
	CPU_CATEGORY = ["%util"]

	UTIL_Y = np.reshape(Y[:, UTIL_INDEX], (-1, 1))

	try:
	Y2 = np.hstack((Y2, UTIL_Y))
	except NameError, e:
	Y2 = np.array(UTIL_Y)

	if PS or PE:
	Y2 = Y2[PS:PE]

	Y = np.delete(Y, UTIL_INDEX, 1)
	# sys.exit(0)

	ax1 = fig.add_subplot(1, 1, 1)
	# pylab.xlim([0, X[-1] + X[-1] * 0.05])
	ax1.set_ylim(MINYMAXY)
	ax1.set_xlabel("Time")
	ax1.set_ylabel(CATEGORY)
	ax1.tick_params(axis='x', width=2,
	length=8, color='black', bottom='on',
	direction='out')
	ax1.xaxis.set_minor_locator(minorLocator)
	ax1.tick_params(axis='x', which='minor', width=1,
	length=4, color='b', bottom='on',
	direction='out')

	if ANNOTATE:
	# Annotate peak for each dataset
	for dataset in range(Y.shape[1]):
	maxvaly = np.amax(Y[:, dataset])
	maxvalx = np.argmax(Y[:, dataset])

	if TIMESTAMP:
	# print TS[2814]
	# print TS[2905]
	ANNO_TXT = "%(y)s (%(x)s)" % {
	"y": maxvaly, "x": TS[maxvalx].replace("\n", " ")}
	else:
	ANNO_TXT = "%(y)s (%(x)s)" % {"y": maxvaly, "x": maxvalx}

	ax1.annotate(ANNO_TXT, xy=(maxvalx, maxvaly), xycoords='data',
	xytext=(-15, 10), textcoords='offset points',
	arrowprops=dict(facecolor='black', shrink=0.05),
	horizontalalignment='center', verticalalignment='bottom',
	)

	ax1.plot(Y)
	leg1 = ax1.legend(CATEGORY, loc="upper right", prop={'size': 15})
	# set the linewidth of the legend objects
	for i in range(len(leg1.legendHandles)):
	leg1.legendHandles[i].set_linewidth(2.0)

	if CPU_CATEGORY:
	ax2 = ax1.twinx()
	ax2.set_ylim((0, 110))
	ax2.set_xlabel("Time")
	ax2.set_ylabel("%")
	ax2.tick_params(axis='x', width=2,
	length=8, color='black', bottom='on',
	direction='out')
	ax2.xaxis.set_minor_locator(minorLocator)
	ax2.tick_params(axis='x', which='minor', width=1,
	length=4, color='b', bottom='on',
	direction='out')

	if ANNOTATE:
	# Annotate peak for each dataset
	for dataset in range(Y2.shape[1]):
	maxvaly = np.amax(Y2[:, dataset])
	maxvalx = np.argmax(Y2[:, dataset])

	if TIMESTAMP:
	# print TS[2814]
	# print TS[2905]
	ANNO_TXT = "%(y)s (%(x)s)" % {
	"y": maxvaly, "x": TS[maxvalx].replace("\n", " ")}
	else:
	ANNO_TXT = "%(y)s (%(x)s)" % {"y": maxvaly, "x": maxvalx}

	ax2.annotate(ANNO_TXT, xy=(maxvalx, maxvaly), xycoords='data',
	xytext=(-15, 10), textcoords='offset points',
	arrowprops=dict(facecolor='black', shrink=0.05),
	horizontalalignment='center',
	verticalalignment='bottom',
	)

	ax2.plot(Y2, linestyle=":")
	leg2 = ax2.legend(CPU_CATEGORY, loc="center right", prop={'size': 15})
	# set the linewidth of the legend objects
	for i in range(len(leg2.legendHandles)):
	leg2.legendHandles[i].set_linewidth(2.0)

	if TIMESTAMP:
	if TSINT != 0:
	ax1.xaxis.set_major_locator(pylab.MaxNLocator(TSINT))
	TSloc = ax1.xaxis.get_ticklocs()
	TSint = [TS[int(i)] for i in TSloc[:-1]]
	ax1.set_xticklabels(TSint, ha="center")
	for tick in ax1.xaxis.iter_ticks():
	tick[0].label2On = False
	tick[0].label1On = True
	tick[0].label1.set_rotation(90)
	ax1.tick_params(axis='x', width=2,
	length=8, color='black', bottom='on',
	direction='out')
	# ax1.tick_params(axis='x', direction='out')

	# pylab.subplots_adjust(
	# left=0.05, right=0.995, bottom=0.05, top=0.9, wspace=0.1, hspace=0.2)
	pylab.tight_layout()

	nf = "%(n)s" % {"n": os.path.splitext(LOGNAMES[0])[0]}
	nc = "%(n)s" % {"n": "-".join(CATEGORY).replace("/", "")}

	if CPU_CATEGORY:
	ncc = "%(n)s" % {"n": "-".join(CPU_CATEGORY).replace("/", "").replace("%", "")}
	imgname = "%(a)s_-_%(b)s_-_%(c)s.png" % {"a": nf,
	"b": nc,
	"c": ncc,
	}
	else:
	imgname = "%(a)s_-_%(b)s.png" % {"a": nf,
	"b": nc,
	}

	# You might want to add bbox_inches='tight' to pylab.savefig
	pylab.savefig(imgname)

	print '''
	IOSTAT HELP:
	Reports
	The iostat command generates three types of reports, the CPU Utilization report, the Device Utilization report and the Network Filesystem report.

	CPU Utilization Report:
	The first report generated by the iostat command is the CPU Utilization Report. For multiprocessor systems, the CPU values are global averages among all processors. The report has the following format:
	%user
	Show the percentage of CPU utilization that occurred while executing at the user level (application).
	%nice
	Show the percentage of CPU utilization that occurred while executing at the user level with nice priority.
	%system
	Show the percentage of CPU utilization that occurred while executing at the system level (kernel).
	%iowait
	Show the percentage of time that the CPU or CPUs were idle during which the system had an outstanding disk I/O request.
	%steal
	Show the percentage of time spent in involuntary wait by the virtual CPU or CPUs while the hypervisor was servicing another virtual processor.
	%idle
	Show the percentage of time that the CPU or CPUs were idle and the system did not have an outstanding disk I/O request.
	Device Utilization Report
	The second report generated by the iostat command is the Device Utilization Report. The device report provides statistics on a per physical device or partition basis. Block devices for which statistics are to be displayed may be entered on the command line. Partitions may also be entered on the command line providing that option -x is not used. If no device nor partition is entered, then statistics are displayed for every device used by the system, and providing that the kernel maintains statistics for it. If the ALL keyword is given on the command line, then statistics are displayed for every device defined by the system, including those that have never been used. The report may show the following fields, depending on the flags used:

	Device:
	This column gives the device (or partition) name, which is displayed as hdiskn with 2.2 kernels, for the nth device. It is displayed as devm-n with 2.4 kernels, where m is the major number of the device, and n a distinctive number. With newer kernels, the device name as listed in the /dev directory is displayed.
	tps
	Indicate the number of transfers per second that were issued to the device. A transfer is an I/O request to the device. Multiple logical requests can be combined into a single I/O request to the device. A transfer is of indeterminate size.
	Blk_read/s
	Indicate the amount of data read from the device expressed in a number of blocks per second. Blocks are equivalent to sectors with kernels 2.4 and later and therefore have a size of 512 bytes. With older kernels, a block is of indeterminate size.
	Blk_wrtn/s
	Indicate the amount of data written to the device expressed in a number of blocks per second.
	Blk_read
	The total number of blocks read.
	Blk_wrtn
	The total number of blocks written.
	kB_read/s
	Indicate the amount of data read from the device expressed in kilobytes per second.
	kB_wrtn/s
	Indicate the amount of data written to the device expressed in kilobytes per second.
	kB_read
	The total number of kilobytes read.
	kB_wrtn
	The total number of kilobytes written.
	MB_read/s
	Indicate the amount of data read from the device expressed in megabytes per second.
	MB_wrtn/s
	Indicate the amount of data written to the device expressed in megabytes per second.
	MB_read
	The total number of megabytes read.
	MB_wrtn
	The total number of megabytes written.
	rrqm/s
	The number of read requests merged per second that were queued to the device.
	wrqm/s
	The number of write requests merged per second that were queued to the device.
	r/s
	The number of read requests that were issued to the device per second.
	w/s
	The number of write requests that were issued to the device per second.
	rsec/s
	The number of sectors read from the device per second.
	wsec/s
	The number of sectors written to the device per second.
	rkB/s
	The number of kilobytes read from the device per second.
	wkB/s
	The number of kilobytes written to the device per second.
	rMB/s
	The number of megabytes read from the device per second.
	wMB/s
	The number of megabytes written to the device per second.
	avgrq-sz
	The average size (in sectors) of the requests that were issued to the device.
	avgqu-sz
	The average queue length of the requests that were issued to the device.
	await
	The average time (in milliseconds) for I/O requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.
	svctm
	The average service time (in milliseconds) for I/O requests that were issued to the device. Warning! Do not trust this field any more. This field will be removed in a future sysstat version.
	%util
	Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%.
	Network Filesystem report
	The Network Filesystem (NFS) report provides statistics for each mounted network filesystem. The report shows the following fields:

	Filesystem:
	This columns shows the hostname of the NFS server followed by a colon and by the directory name where the network filesystem is mounted.
	rBlk_nor/s
	Indicate the number of blocks read by applications via the read(2) system call interface. A block has a size of 512 bytes.
	wBlk_nor/s
	Indicate the number of blocks written by applications via the write(2) system call interface.
	rBlk_dir/s
	Indicate the number of blocks read from files opened with the O_DIRECT flag.
	wBlk_dir/s
	Indicate the number of blocks written to files opened with the O_DIRECT flag.
	rBlk_svr/s
	Indicate the number of blocks read from the server by the NFS client via an NFS READ request.
	wBlk_svr/s
	Indicate the number of blocks written to the server by the NFS client via an NFS WRITE request.
	rkB_nor/s
	Indicate the number of kilobytes read by applications via the read(2) system call interface.
	wkB_nor/s
	Indicate the number of kilobytes written by applications via the write(2) system call interface.
	rkB_dir/s
	Indicate the number of kilobytes read from files opened with the O_DIRECT flag.
	wkB_dir/s
	Indicate the number of kilobytes written to files opened with the O_DIRECT flag.
	rkB_svr/s
	Indicate the number of kilobytes read from the server by the NFS client via an NFS READ request.
	wkB_svr/s
	Indicate the number of kilobytes written to the server by the NFS client via an NFS WRITE request.
	rMB_nor/s
	Indicate the number of megabytes read by applications via the read(2) system call interface.
	wMB_nor/s
	Indicate the number of megabytes written by applications via the write(2) system call interface.
	rMB_dir/s
	Indicate the number of megabytes read from files opened with the O_DIRECT flag.
	wMB_dir/s
	Indicate the number of megabytes written to files opened with the O_DIRECT flag.
	rMB_svr/s
	Indicate the number of megabytes read from the server by the NFS client via an NFS READ request.
	wMB_svr/s
	Indicate the number of megabytes written to the server by the NFS client via an NFS WRITE request.
	ops/s
	Indicate the number of operations that were issued to the filesystem per second.
	rops/s
	Indicate the number of 'read' operations that were issued to the filesystem per second.
	wops/s
	Indicate the number of 'write' operations that were issued to the filesystem per second.
	'''

	if INTERACTIVE:
	pylab.show()
No results found