Jula log stat

logstat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import re\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = 'AppLogs'\n",
    "\n",
    "def files():\n",
    "    for file in os.listdir(path):\n",
    "        yield open(os.path.join(path, file), 'rb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "names = ['type', 'proc', 'user', 'date', 'data']\n",
    "converters = {name: str.strip for name in names}\n",
    "df = pd.concat([pd.read_csv(file, sep='|', names=names, converters=converters) for file in files()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_values(data):\n",
    "    if '[PaymentMethodValidator] Time spent to call fraudService:' not in data:\n",
    "        return None\n",
    "    match = re.search('\\[PaymentMethodValidator\\] Time spent to call fraudService: \\'([0-9\\.]+)\\' sec;', data)\n",
    "    return float(match.groups()[0])\n",
    "\n",
    "times = df[(df.date > '2015/08/18 12:23') & (df.date < '2015/08/18 12:43')].data.map(get_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    714.000000\n",
       "mean       1.810340\n",
       "std        0.622092\n",
       "min        0.901000\n",
       "5%         1.179650\n",
       "25%        1.367250\n",
       "50%        1.652500\n",
       "75%        2.054500\n",
       "95%        3.069100\n",
       "max        6.725000\n",
       "Name: data, dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "times.dropna()\n",
    "\n",
    "times.describe(percentiles=[.05, .25, .5, .75, .95])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f9ce8413ac8>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAEACAYAAABMEua6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGJBJREFUeJzt3W+MZvV53vHvBeutjUFsXVfLglGXF2CykSVw4pVbFHxM\nAEHVAnkTg2QZktRCIrWRK1dZKrWgxFrjSnYttbItJeBdp2Fb6igUHC3hT/Y0i6IusctgzLABLI/E\numabyEBNaat1uPtizpZhPDPPmfObZ+7nd/b6SKM9v3POM+eaZ+CaM/c8s6uIwMzMxuG07ABmZrZx\nXOpmZiPiUjczGxGXupnZiLjUzcxGxKVuZjYia5a6pHdKOiJpTtK8pM93+++SdEzSU93btUsec4ek\nFyQdlXT1tD8AMzN7iya9Tl3SGRHxhqQtwBPAZ4FfBn4SEV9adu4u4D7gQ8B5wGPARRHx5jTCm5nZ\n200cv0TEG93mVuB04JVurRVOvx44EBEnImIBeBHYvQE5zcysh4mlLuk0SXPAceBQRDzbHfqUpKcl\n3SNpW7fvXODYkocfY/GO3czMNkGfO/U3I+IS4H3A5ZIa4KvABcAlwI+AL671LjYgp5mZ9bCl74kR\n8ZqkPwZ+MSLak/sl/R7wULf8IXD+koe9r9v3NpJc9GZmA0TESqPvt52w6hvwXmBbt/0u4M9Y/CHp\nOUvO+QxwX7e9C5hjcf5+AfB9uh/GLnu/sdZ1Z/0NuCs7w6mY3fnz35w/PX9MOmfSnfoOYL+k01gc\n1fx+RDwu6RuSLmFxtPID4NbuavOS7gfmgZ8Ct0WXZGR2ZgcosDM7QKGd2QEK7cwOUGhndoBCO7MD\nTNuapR4RzwAfXGH/J9Z4zF5gb3k0MzNbL/9G6TD7sgMU2JcdoNC+7ACF9mUHKLQvO0ChfdkBpm3i\nLx9N5aJSxKRhv5mZvU2f7vSd+gDdyzqrVHN2cP5szj/7XOpmZiPi8YuZWSU8fjEzO8W41AeoeS5X\nc3Zw/mzOP/tc6mZmI+KZuplZJTxTNzM7xbjUB6h5LldzdnD+bM4/+1zqZmYj4pm6mVklPFM3MzvF\nuNQHqHkuV3N2cP5szj/7XOpmZiPimbqZWSU8UzczO8W41AeoeS5Xc3Zw/mzOP/tc6mZmI+KZuplZ\nJTxTNzM7xbjUB6h5LldzdnD+bM4/+9YsdUnvlHRE0pykeUmf7/a/R9Kjkp6X9IikbUsec4ekFyQd\nlXT1tD8AMzN7y8SZuqQzIuINSVuAJ4DPAtcBfx0R/1rSbwF/OyL2SNoF3Ad8CDgPeAy4KCLeXPY+\nPVM3M1unDZmpR8Qb3eZW4HTgFRZLfX+3fz9wQ7d9PXAgIk5ExALwIrB7/dHNzGyIiaUu6TRJc8Bx\n4FBEPAtsj4jj3SnHge3d9rnAsSUPP8biHfuoTHMuJyk2+21aH8s01D4Tdf5ctefvY8ukE7rRySWS\nzgb+RNJHlx2fVAwrHpO0D1jolq8CcxHRdsea7n3P5JrF52OK7/9Qd5mm+7PdwPXJ7ZNrpT+fXnvt\n9crrbvsWFi3Qw7pepy7pXwL/G/gnQBMRL0vaweId/MWS9nRh7u7Ofxi4MyKOLHs/nqmvYvEL5Gbe\nPAt/LszqUDxTl/Reda9skfQu4CrgKeBB4ObutJuBB7rtB4EbJW2VdAFwIfDk8A/BzMzWY9JMfQfw\np91M/QjwUEQ8DtwNXCXpeeCKbk1EzAP3A/PAQeC2yPiV1Smrey7XZgcoUvdz7/zZas/fx5oz9Yh4\nBvjgCvt/DFy5ymP2Ans3JJ2Zma2L/+6XGeOZupmtpnimbmZmdXGpD1D3XK7NDlCk7ufe+bPVnr8P\nl7qZ2Yh4pj5jPFM3s9V4pm5mdopxqQ9Q91yuzQ5QpO7n3vmz1Z6/D5e6mdmIeKY+YzxTN7PVeKZu\nZnaKcakPUPdcrs0OUKTu5975s9Wevw+XupnZiHimPmM8Uzez1XimbmZ2inGpD1D3XK7NDlCk7ufe\n+bPVnr8Pl7qZ2Yh4pj5jPFM3s9V4pm5mdopxqQ9Q91yuzQ5QpO7n3vmz1Z6/D5e6mdmIeKY+YzxT\nN7PVeKZuZnaKcakPUPdcrs0OUKTu5975s9Wev481S13S+ZIOSXpW0vckfbrbf5ekY5Ke6t6uXfKY\nOyS9IOmopKun/QGYmdlb1pypSzoHOCci5iSdCXwHuAH4VeAnEfGlZefvAu4DPgScBzwGXBQRby47\nzzP1VXimbmarKZ6pR8TLETHXbb8OPMdiWQOs9I6vBw5ExImIWABeBHavN7iZmQ3Te6YuaSdwKfBf\nu12fkvS0pHskbev2nQscW/KwY7z1RWA06p7LtdkBitT93Dt/ttrz97Glz0nd6OWbwO0R8bqkrwK/\n3R3+HeCLwG+s8vAVZwmS9gEL3fJVYC4i2u5YAzCra+ASSVN8/ycv07AZ6+zn02uvvV553W3fwqIF\nepj4OnVJ7wC+BRyMiC+vcHwn8FBEfEDSni7M3d2xh4E7I+LIssd4pr4Kz9TNbDXFM3VJAu4B5pcW\nuqQdS077FeCZbvtB4EZJWyVdAFwIPDkkvJmZrd+kmfplwMeBjy57+eIXJH1X0tPAR4DPAETEPHA/\nMA8cBG6LjF9ZnbK653JtdoAidT/3zp+t9vx9rDlTj4gnWLn4D67xmL3A3sJcZmY2gP/ulzVI+vvA\nz2/yZX/XM3UzW0mf7uz16pdT1xmfgIt/HXb9dHOu9/Bp8NebcykzGyWX+ppOE3xiK9y+9e37W956\neeBGuvw1OPzOKbzjJVqmk31zSGqWvLS0Os6fq/b8ffgv9DIzGxGX+iBNdoACTXaAIrXfZTl/rtrz\n9+FSNzMbEZf6IG12gAJtdoAitb/O2Plz1Z6/D5e6mdmIuNQHabIDFGiyAxSpfSbq/Llqz9+HS93M\nbERc6oO02QEKtNkBitQ+E3X+XLXn78OlbmY2Ii71QZrsAAWa7ABFap+JOn+u2vP34VI3MxsRl/og\nbXaAAm12gCK1z0SdP1ft+ftwqZuZjYhLfZAmO0CBJjtAkdpnos6fq/b8fbjUzcxGxKU+SJsdoECb\nHaBI7TNR589Ve/4+XOpmZiPiUh+kyQ5QoMkOUKT2majz56o9fx8udTOzEXGpD9JmByjQZgcoUvtM\n1Plz1Z6/jzVLXdL5kg5JelbS9yR9utv/HkmPSnpe0iOSti15zB2SXpB0VNLV0/4AzMzsLZPu1E8A\nn4mInwc+DPympJ8D9gCPRsRFwOPdGkm7gI8Bu4BrgK9IGuF3A012gAJNdoAitc9EnT9X7fn7WLNw\nI+LliJjrtl8HngPOA64D9nen7Qdu6LavBw5ExImIWABeBHZPIbeZma2g9120pJ3ApcARYHtEHO8O\nHQe2d9vnAseWPOwYi18ERqbNDlCgzQ5QpPaZqPPnqj1/H1v6nCTpTOAPgdsj4ieS/v+xiAhJscbD\nVzwmaR+w0C1fBeZOfmt08onPXsOZXby2+7Pbzdyy9fLjQ9dMOD6d9aw831577fXb1932LSxaoAdF\nrNXHIOkdwLeAgxHx5W7fUaCJiJcl7QAORcTFkvZ0Ye7uznsYuDMijix7nxERYsZJZ30NPncr3L5J\nV7z8NTh89ipfB6dE1PC5MLN+3Tnp1S8C7gHmTxZ650Hg5m77ZuCBJftvlLRV0gXAhcCTQ8Kbmdn6\nTZqpXwZ8HPiopKe6t2uAu4GrJD0PXNGtiYh54H5gHjgI3BaTvhWoUpsdoECbHaBI7TNR589Ve/4+\n1pypR8QTrF78V67ymL3A3sJcZmY2wAhfQ74ZmuwABZrsAEVqf52x8+eqPX8fLnUzsxFxqQ/SZgco\n0GYHKFL7TNT5c9Wevw+XupnZiLjUB2myAxRosgMUqX0m6vy5as/fh0vdzGxEXOqDtNkBCrTZAYrU\nPhN1/ly15+/DpW5mNiIu9UGa7AAFmuwARWqfiTp/rtrz9+FSNzMbEZf6IG12gAJtdoAitc9EnT9X\n7fn7cKmbmY2IS32QJjtAgSY7QJHaZ6LOn6v2/H241M3MRsSlPkibHaBAmx2gSO0zUefPVXv+Plzq\nZmYj4lIfpMkOUKDJDlCk9pmo8+eqPX8fLnUzsxFxqQ/SZgco0GYHKFL7TNT5c9Wevw+XupnZiLjU\nB2myAxRosgMUqX0m6vy5as/fh0vdzGxEXOqDtNkBCrTZAYrUPhN1/ly15+9jYqlLulfScUnPLNl3\nl6Rjkp7q3q5dcuwOSS9IOirp6mkFNzOzn9XnTv3rwDXL9gXwpYi4tHs7CCBpF/AxYFf3mK9IGuF3\nA012gAJNdoAitc9EnT9X7fn7mFi4EXEYeGWFQ1ph3/XAgYg4ERELwIvA7qKEZmbWW8ld9KckPS3p\nHknbun3nAseWnHMMOK/gGjOqzQ5QoM0OUKT2majz56o9fx9bBj7uq8Bvd9u/A3wR+I1Vzo2Vdkra\nByx0y1eBuZPfGp184rPXcGYXr+3+7HYzt2y9/PjQNROOT2c9K8+31157/fZ1t30LixboQRErdu7b\nT5J2Ag9FxAfWOiZpTxfm7u7Yw8CdEXFk2WMiIlYa38wU6ayvweduhds36YqXvwaHz17l6+CUiBo+\nF2bWrzsHjV8k7Viy/BXg5CtjHgRulLRV0gXAhcCTQ65hZmbr1+cljQeAPwfeL+klSb8OfEHSdyU9\nDXwE+AxARMwD9wPzwEHgtujzrUB12uwABdrsAEVqn4k6f67a8/cxcaYeETetsPveNc7fC+wtCWVm\nZsOM8DXkm6HJDlCgyQ5QpPbXGTt/rtrz9+FSNzMbEZf6IG12gAJtdoAitc9EnT9X7fn7cKmbmY2I\nS32QJjtAgSY7QJHaZ6LOn6v2/H241M3MRsSlPkibHaBAmx2gSO0zUefPVXv+PlzqZmYj4lIfpMkO\nUKDJDlCk9pmo8+eqPX8fLnUzsxFxqQ/SZgco0GYHKFL7TNT5c9Wevw+XupnZiLjUB2myAxRosgMU\nqX0m6vy5as/fh0vdzGxEXOqDtNkBCrTZAYrUPhN1/ly15+/DpW5mNiIu9UGa7AAFmuwARWqfiTp/\nrtrz9+FSNzMbEZf6IG12gAJtdoAitc9EnT9X7fn7cKmbmY2IS32QJjtAgSY7QJHaZ6LOn6v2/H24\n1M3MRsSlPkibHaBAmx2gSO0zUefPVXv+PiaWuqR7JR2X9MySfe+R9Kik5yU9ImnbkmN3SHpB0lFJ\nV08ruJmZ/aw+d+pfB65Ztm8P8GhEXAQ83q2RtAv4GLCre8xXJI3wu4EmO0CBJjtAkdpnos6fq/b8\nfUws3Ig4DLyybPd1wP5uez9wQ7d9PXAgIk5ExALwIrB7Y6KamdkkQ++it0fE8W77OLC92z4XOLbk\nvGPAeQOvMcPa7AAF2uwARWqfiTp/rtrz97Gl9B1EREiKtU5ZaaekfcBCt3wVmDv5rdHJJz57DWd2\n8druz243c8vWy48PXTPh+HTWEz5/0/BRyP/8eu31rK+77VtYtEAPipj8/7OkncBDEfGBbn0UaCLi\nZUk7gEMRcbGkPV2Yu7vzHgbujIgjy95fRIT6BMwknfU1+NytcPsmXfHy1+Dw2at8HZwSsdnXq+Fz\nbzaL+nTn0PHLg8DN3fbNwANL9t8oaaukC4ALgScHXsPMzNapz0saDwB/Drxf0kuSfg24G7hK0vPA\nFd2aiJgH7gfmgYPAbdHnW4HqtNkBCrTZAYrUPhN1/ly15+9j4kw9Im5a5dCVq5y/F9hbEsrMzIYZ\n4WvIN0OTHaBAkx2gSO2vM3b+XLXn78OlbmY2Ii71QdrsAAXa7ABFap+JOn+u2vP34VI3MxsRl/og\nTXaAAk12gCK1z0SdP1ft+ftwqZuZjYhLfZA2O0CBNjtAkdpnos6fq/b8fbjUzcxGxKU+SJMdoECT\nHaBI7TNR589Ve/4+XOpmZiPiUh+kzQ5QoM0OUKT2majz56o9fx8udTOzEXGpD9JkByjQZAcoUvtM\n1Plz1Z6/D5e6mdmIuNQHabMDFGizAxSpfSbq/Llqz9+HS93MbERc6oM02QEKNNkBitQ+E3X+XLXn\n78OlbmY2Ii71QdrsAAXa7ABFap+JOn+u2vP34VI3MxsRl/ogTXaAAk12gCK1z0SdP1ft+ftwqZuZ\njYhLfZA2O0CBNjtAkdpnos6fq/b8fWwpebCkBeB/An8DnIiI3ZLeA/xH4O8BC8CvRsSrhTnNzKyH\n0jv1AJqIuDQidnf79gCPRsRFwOPdemSa7AAFmuwARWqfiTp/rtrz97ER4xctW18H7O+29wM3bMA1\nzMysh424U39M0rclfbLbtz0ijnfbx4HthdeYQW12gAJtdoAitc9EnT9X7fn7KJqpA5dFxI8k/V3g\nUUlHlx6MiJAUKz1Q0j4WZ+4ArwJzJ781OvnEZ6/hzC5e2/3Z7WZu2Xr58aFrJhwfx3pWPr9eez3r\n6277FhYt0IMiVuzcdZN0J/A68EkW5+wvS9oBHIqIi5edGxGxfGwzc6SzvgafuxVu36QrXv4aHD57\n8RugzSI2+3o1fO7NZlGf7hw8fpF0hqSzuu13A1cDzwAPAjd3p90MPDD0GmZmtj4lM/XtwGFJc8AR\n4FsR8QhwN3CVpOeBK7r1yLTZAQq02QGQFJv1lv2xLlf7TNf5Z9/gmXpE/AC4ZIX9PwauLAllY1fS\ntS39X5bpKY+devwbpYM02QEKNNkBCjXZAYrU/jpp5599LnUzsxFxqQ/SZgco0GYHKNRmByhS+0zX\n+WefS93MbERc6oM02QEKNNkBCjXZAYrUPtN1/tnnUjczGxGX+iBtdoACbXaAQm12gCK1z3Sdf/a5\n1M3MRsSlPkiTHaBAkx2gUJMdoEjtM13nn30udTOzEXGpD9JmByjQZgco1GYHKFL7TNf5Z59L3cxs\nRFzqgzTZAQo02QEKNdkBitQ+03X+2edSNzMbEZf6IG12gAJtdoBCbXaAIrXPdJ1/9rnUzcxGxKU+\nSJMdoECTHaBQkx2gSO0zXeeffYP/5SOzGmz2P2nnf1TbsvlOfZA2O0CBNjtAoXad58cmvk1W+0zX\n+WefS93MbERc6oM02QEKNNkBCjXZAYrUPtN1/tnnmbrZBvIM37JN5U5d0jWSjkp6QdJvTeMaudrs\nAAXa7ACF2uwAE0yaux/qcc7GzfA3Wu0z6drz97HhpS7pdODfAdcAu4CbJP3cRl8n11x2gAI1Zwfn\nT3dJdoBCteefaBp36ruBFyNiISJOAP8BuH4K10n0anaAAjVnB+dPty07QKHa8080jVI/D3hpyfpY\nt8/MzKZsGj8ozRn2TcXfvAn/9v/Af/6/b9//7BnwZ29s/PWe/lsb/z6XW5j+JaZqITtAoYXsAKV2\nZgcotHO1A2P5IbciNvbjkPRh4K6IuKZb3wG8GRFfWHLOiIrfzGzzTPpiMI1S3wL8JfDLwH8HngRu\niojnNvRCZmb2MzZ8/BIRP5X0T4E/AU4H7nGhm5ltjg2/Uzczszyb+tcE1P5LSZLulXRc0jPZWdZL\n0vmSDkl6VtL3JH06O9N6SHqnpCOS5iTNS/p8dqYhJJ0u6SlJD2VnWS9JC5K+2+V/MjvPekjaJumb\nkp7r/vv5cHamviS9v3vOT769ttb/v5t2p979UtJfAlcCPwT+gspm7ZJ+CXgd+EZEfCA7z3pIOgc4\nJyLmJJ0JfAe4obLn/4yIeKP7uc0TwGcj4onsXOsh6Z8BvwCcFRHXZedZD0k/AH4hIn6cnWW9JO0H\n/ktE3Nv99/PuiHgtO9d6STqNxf7cHREvrXTOZt6pV/9LSRFxGHglO8cQEfFyRMx1268DzwHn5qZa\nn4g4+TLSrSz+vKaqcpH0PuAfAr8H1Pp3tlSXW9LZwC9FxL2w+HO/Ggu9cyXw/dUKHTa31P1LSTNC\n0k7gUuBIbpL1kXSapDngOHAoIuazM63TvwH+OfBmdpCBAnhM0rclfTI7zDpcAPyVpK9L+m+SflfS\nGdmhBroRuG+tEzaz1P0T2RnQjV6+Cdze3bFXIyLejIhLgPcBl9f0lzNJ+kfA/4iIp6jwbrdzWURc\nClwL/GY3jqzBFuCDwFci4oPA/wL25EZaP0lbgX8M/Ke1ztvMUv8hcP6S9fks3q3bJpH0DuAPgX8f\nEQ9k5xmq+9b5j4FfzM6yDv8AuK6bSx8ArpD0jeRM6xIRP+r+/Cvgj1gcqdbgGHAsIv6iW3+TxZKv\nzbXAd7rnf1WbWerfBi6UtLP7ivMx4MFNvP4pTZKAe4D5iPhydp71kvReSdu67XcBVwFP5abqLyL+\nRUScHxEXsPgt9J9GxCeyc/Ul6QxJZ3Xb7wauBqp4FVhEvAy8JOmibteVwLOJkYa6icUbgjVt2j+S\nMYZfSpJ0APgI8HckvQT8q4j4enKsvi4DPg58V9LJMrwjIh5OzLQeO4D93U//TwN+PyIeT85UorZx\n5HbgjxbvDdgC/EFEPJIbaV0+BfxBd0P5feDXkvOsS/eF9Epg4s8y/MtHZmYj4n+j1MxsRFzqZmYj\n4lI3MxsRl7qZ2Yi41M3MRsSlbmY2Ii51M7MRcambmY3I/wPsB1oUEWEIsQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9cc1da7d68>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "times.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}