Skip to content

Instantly share code, notes, and snippets.

@Kelvinrr

Kelvinrr/logparse.ipynb

Created August 24, 2017 20:43
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save Kelvinrr/32f9ea028c5c99500ace0601923e664e to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save Kelvinrr/32f9ea028c5c99500ace0601923e664e to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
logparse.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 266,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import os\n",
"import re\n",
"import smtplib\n",
"from email.mime.text import MIMEText\n",
"from email.mime.multipart import MIMEMultipart"
]
},
{
"cell_type": "code",
"execution_count": 267,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"def parse_logs(filename):\n",
" with open(filename) as f:\n",
" content = f.readlines() \n",
" \n",
" content = [x.strip() for x in content] \n",
" results = []\n",
" \n",
" for line in content: \n",
" try:\n",
" data = {}\n",
" data['passed'] = bool(re.search(\"passed\", line, flags=re.IGNORECASE))\n",
" data['name'] = re.search(\"isis3_[A-z]+|[A-z]+_app_[A-z]+\", line, flags=re.IGNORECASE).group(0)\n",
" data['test_id'] = int(re.search(\"#[0-9]+\", line, flags=re.IGNORECASE).group(0)[1:])\n",
" data['time'] = float(re.search(\"[0-9]+\\.[0-9]+\", line, flags=re.IGNORECASE).group(0))\n",
" results.append(data)\n",
" except Exception as e:\n",
" pass\n",
" \n",
" return pd.DataFrame(results)"
]
},
{
"cell_type": "code",
"execution_count": 268,
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"unittest_logs = parse_logs('unittest.log')\n",
"apptest_logs = parse_logs('apptest.log')"
]
},
{
"cell_type": "code",
"execution_count": 269,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style>\n",
" .dataframe thead tr:only-child th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>passed</th>\n",
" <th>test_id</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>280</th>\n",
" <td>isis3_unit_test_IException</td>\n",
" <td>False</td>\n",
" <td>281</td>\n",
" <td>0.30</td>\n",
" </tr>\n",
" <tr>\n",
" <th>301</th>\n",
" <td>isis3_unit_test_SqlQuery</td>\n",
" <td>False</td>\n",
" <td>302</td>\n",
" <td>1.30</td>\n",
" </tr>\n",
" <tr>\n",
" <th>302</th>\n",
" <td>isis3_unit_test_DatabaseFactory</td>\n",
" <td>False</td>\n",
" <td>303</td>\n",
" <td>0.30</td>\n",
" </tr>\n",
" <tr>\n",
" <th>303</th>\n",
" <td>isis3_unit_test_Database</td>\n",
" <td>False</td>\n",
" <td>304</td>\n",
" <td>1.30</td>\n",
" </tr>\n",
" <tr>\n",
" <th>305</th>\n",
" <td>isis3_unit_test_SqlRecord</td>\n",
" <td>False</td>\n",
" <td>306</td>\n",
" <td>1.28</td>\n",
" </tr>\n",
" <tr>\n",
" <th>327</th>\n",
" <td>isis3_unit_test_MosaicSceneWidget</td>\n",
" <td>False</td>\n",
" <td>328</td>\n",
" <td>1.36</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name passed test_id time\n",
"280 isis3_unit_test_IException False 281 0.30\n",
"301 isis3_unit_test_SqlQuery False 302 1.30\n",
"302 isis3_unit_test_DatabaseFactory False 303 0.30\n",
"303 isis3_unit_test_Database False 304 1.30\n",
"305 isis3_unit_test_SqlRecord False 306 1.28\n",
"327 isis3_unit_test_MosaicSceneWidget False 328 1.36"
]
},
"execution_count": 269,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"unittest_logs[~unittest_logs.passed]"
]
},
{
"cell_type": "code",
"execution_count": 270,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style>\n",
" .dataframe thead tr:only-child th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>passed</th>\n",
" <th>test_id</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>223</th>\n",
" <td>fits_app_test_infoAll</td>\n",
" <td>False</td>\n",
" <td>554</td>\n",
" <td>0.32</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name passed test_id time\n",
"223 fits_app_test_infoAll False 554 0.32"
]
},
"execution_count": 270,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"apptest_logs[~apptest_logs.passed]"
]
},
{
"cell_type": "code",
"execution_count": 278,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"TOTAL TIME = 288.92999999999995\n",
"AVERAGE TIME = 0.8755454545454544\n"
]
}
],
"source": [
"print(\"TOTAL TIME = {}\".format(unittest_logs['time'].sum()))\n",
"print(\"AVERAGE TIME = {}\".format(unittest_logs['time'].mean()))"
]
},
{
"cell_type": "code",
"execution_count": 279,
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style>\n",
" .dataframe thead tr:only-child th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>passed</th>\n",
" <th>test_id</th>\n",
" <th>time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>280</th>\n",
" <td>isis3_unit_test_IException</td>\n",
" <td>False</td>\n",
" <td>281</td>\n",
" <td>0.30</td>\n",
" </tr>\n",
" <tr>\n",
" <th>301</th>\n",
" <td>isis3_unit_test_SqlQuery</td>\n",
" <td>False</td>\n",
" <td>302</td>\n",
" <td>1.30</td>\n",
" </tr>\n",
" <tr>\n",
" <th>302</th>\n",
" <td>isis3_unit_test_DatabaseFactory</td>\n",
" <td>False</td>\n",
" <td>303</td>\n",
" <td>0.30</td>\n",
" </tr>\n",
" <tr>\n",
" <th>303</th>\n",
" <td>isis3_unit_test_Database</td>\n",
" <td>False</td>\n",
" <td>304</td>\n",
" <td>1.30</td>\n",
" </tr>\n",
" <tr>\n",
" <th>305</th>\n",
" <td>isis3_unit_test_SqlRecord</td>\n",
" <td>False</td>\n",
" <td>306</td>\n",
" <td>1.28</td>\n",
" </tr>\n",
" <tr>\n",
" <th>327</th>\n",
" <td>isis3_unit_test_MosaicSceneWidget</td>\n",
" <td>False</td>\n",
" <td>328</td>\n",
" <td>1.36</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name passed test_id time\n",
"280 isis3_unit_test_IException False 281 0.30\n",
"301 isis3_unit_test_SqlQuery False 302 1.30\n",
"302 isis3_unit_test_DatabaseFactory False 303 0.30\n",
"303 isis3_unit_test_Database False 304 1.30\n",
"305 isis3_unit_test_SqlRecord False 306 1.28\n",
"327 isis3_unit_test_MosaicSceneWidget False 328 1.36"
]
},
"execution_count": 279,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"failed = results[~results.passed]\n",
"failed"
]
},
{
"cell_type": "code",
"execution_count": 295,
"metadata": {},
"outputs": [],
"source": [
"msg = MIMEMultipart(\"TEST ATTACH\")\n",
"msg['Subject'] = 'EMAIL TEST'\n",
"msg['From'] = '[email protected]'\n",
"msg['To'] = '[email protected]'\n",
"\n",
"unittest_logs['type'] = 'unit'\n",
"apptest_logs['type'] = 'app'\n",
"\n",
"df = unittest_logs[~unittest_logs.passed].append(apptest_logs[~apptest_logs.passed])\n",
"df = df.set_index(['type', 'test_id'])\n",
"\n",
"html = df.to_html().replace(\"\\n\", \"\").replace('<th></th>', '').replace('</tr> <tr>', '')\n",
"attachment = MIMEText(html, 'html')\n",
"msg.attach(attachment)"
]
},
{
"cell_type": "code",
"execution_count": 296,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(221, b'2.0.0 closing connection b12sm2961152qkc.56 - gsmtp')"
]
},
"execution_count": 296,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"server = smtplib.SMTP('smtp.gmail.com',587) #port 465 or 587\n",
"server.ehlo()\n",
"server.starttls()\n",
"server.ehlo()\n",
"server.login('[email protected]','Axel1105')\n",
"server.sendmail(msg['From'], [msg['From']], msg.as_string())\n",
"server.quit()"
]
},
{
"cell_type": "code",
"execution_count": 306,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'<table border=\"1\" class=\"dataframe\"> <thead> <tr style=\"text-align: right;\"> <th></th> <th></th> <th>name</th> <th>passed</th> <th>time</th> </tr> <tr> <th>type</th> <th>test_id</th> <th></th> <th></th> <th></th> </tr> </thead> <tbody> <tr> <th rowspan=\"6\" valign=\"top\">unit</th> <th>281</th> <td>isis3_unit_test_IException</td> <td>False</td> <td>0.30</td> </tr> <tr> <th>302</th> <td>isis3_unit_test_SqlQuery</td> <td>False</td> <td>1.30</td> </tr> <tr> <th>303</th> <td>isis3_unit_test_DatabaseFactory</td> <td>False</td> <td>0.30</td> </tr> <tr> <th>304</th> <td>isis3_unit_test_Database</td> <td>False</td> <td>1.30</td> </tr> <tr> <th>306</th> <td>isis3_unit_test_SqlRecord</td> <td>False</td> <td>1.28</td> </tr> <tr> <th>328</th> <td>isis3_unit_test_MosaicSceneWidget</td> <td>False</td> <td>1.36</td> </tr> <tr> <th>app</th> <th>554</th> <td>fits_app_test_infoAll</td> <td>False</td> <td>0.32</td> </tr> </tbody></table>'"
]
},
"execution_count": 306,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.to_html().replace(\"\\n\", \"\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<table border=\"1\" class=\"dataframe\"> <thead> <tr style=\"text-align: right;\"> <th></th> <th></th> <th>name</th> <th>passed</th> <th>time</th> </tr> <tr> <th>type</th> <th>test_id</th> <th></th> <th></th> <th></th> </tr> </thead> <tbody> <tr> <th rowspan=\"6\" valign=\"top\">unit</th> <th>281</th> <td>isis3_unit_test_IException</td> <td>False</td> <td>0.30</td> </tr> <tr> <th>302</th> <td>isis3_unit_test_SqlQuery</td> <td>False</td> <td>1.30</td> </tr> <tr> <th>303</th> <td>isis3_unit_test_DatabaseFactory</td> <td>False</td> <td>0.30</td> </tr> <tr> <th>304</th> <td>isis3_unit_test_Database</td> <td>False</td> <td>1.30</td> </tr> <tr> <th>306</th> <td>isis3_unit_test_SqlRecord</td> <td>False</td> <td>1.28</td> </tr> <tr> <th>328</th> <td>isis3_unit_test_MosaicSceneWidget</td> <td>False</td> <td>1.36</td> </tr> <tr> <th>app</th> <th>554</th> <td>fits_app_test_infoAll</td> <td>False</td> <td>0.32</td> </tr> </tbody></table>"
]
},
{
"cell_type": "code",
"execution_count": 297,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'<table border=\"1\" class=\"dataframe\"> <thead> <tr style=\"text-align: right;\"> <th>name</th> <th>passed</th> <th>time</th> <th>type</th> <th>test_id</th> </tr> </thead> <tbody> <tr> <th rowspan=\"6\" valign=\"top\">unit</th> <th>281</th> <td>isis3_unit_test_IException</td> <td>False</td> <td>0.30</td> <th>302</th> <td>isis3_unit_test_SqlQuery</td> <td>False</td> <td>1.30</td> <th>303</th> <td>isis3_unit_test_DatabaseFactory</td> <td>False</td> <td>0.30</td> <th>304</th> <td>isis3_unit_test_Database</td> <td>False</td> <td>1.30</td> <th>306</th> <td>isis3_unit_test_SqlRecord</td> <td>False</td> <td>1.28</td> <th>328</th> <td>isis3_unit_test_MosaicSceneWidget</td> <td>False</td> <td>1.36</td> <th>app</th> <th>554</th> <td>fits_app_test_infoAll</td> <td>False</td> <td>0.32</td> </tr> </tbody></table>'"
]
},
"execution_count": 297,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"html"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment