Skip to content

Instantly share code, notes, and snippets.

@akiross

akiross/Pipfile

Last active December 5, 2018 11:45
Show Gist options
  • Save akiross/c5cd4485e6abbb5e6b7dd09df6cfb6c1 to your computer and use it in GitHub Desktop.
Save akiross/c5cd4485e6abbb5e6b7dd09df6cfb6c1 to your computer and use it in GitHub Desktop.
Download ZIP
Basic json-log viewer with filtering
Raw
logview.py
# Copyright (c) 2018 Alessandro "AkiRoss" Re
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""Visualize log files by filtering and comparing events.
Requires flask, jinja2 and gevent."""
import os
import sys
import json
import jinja2
import argparse
from itertools import zip_longest
from gevent.pywsgi import WSGIServer
from flask import Flask, Response, render_template_string
log_view_template = r'''
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<title>Logview {{ fname }}</title>
<style>
div.views {
display: flex;
}
ul.view {
display: flex;
flex-direction: column;
flex-grow: 1;
}
.w-40 { width: 40% !important; }
.view.left {
background-color: #eeeeff;
}
.view.right {
background-color: #eeffee;
}
</style>
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
<script>
function loadURL() {
var lsel = document.getElementById("leftSel").value;
var lloc = document.getElementById("leftSelSub").value;
var rsel = document.getElementById("rightSel").value;
var rloc = document.getElementById("rightSelSub").value;
var url = new URL(window.location);
window.location = new URL("/"+lsel+"/"+lloc+"/"+rsel+"/"+rloc, url.origin);
}
</script>
</head>
<body>
<table class="table table-striped">
<thead class="thead-dark">
<tr>
<th colspan="3" class="text-center">Inspecting {{ fname }}</th>
</tr>
{% if tag_keys is not none %}
<tr>
<th class="w-50 text-center">
<select id="leftSel" class="custom-select mr-sm-2 w-40">
{% for key in tag_keys %}
<option value="{{ key }}">{{ key }}</option>
{% endfor %}
</select>
<select id="leftSelSub" class="custom-select mr-sm-2 w-40">
{% for key in location_keys %}
<option value="{{ key }}">{{ key }}</option>
{% endfor %}
</select>
</th>
<th class="text-center">
<button type="button" class="btn btn-primary" onclick="loadURL();">Reload</button>
</th>
<th class="text-center">
<select id="rightSel" class="custom-select mr-sm-2 w-40">
{% for key in tag_keys %}
<option value="{{ key }}">{{ key }}</option>
{% endfor %}
</select>
<select id="rightSelSub" class="custom-select mr-sm-2 w-40">
{% for key in location_keys %}
<option value="{{ key }}">{{ key }}</option>
{% endfor %}
</select>
</th>
</tr>
{% endif %}
</thead>
{% for litem, citem, ritem in items %}
<tr>
<td>{{ litem }}</td>
<td>{{ citem }}</td>
<td>{{ ritem }}</td>
</tr>
{% endfor %}
</table>
</body>
</html>
'''
app = Flask(__name__)
def get_first(l, default=None):
"""Return item at position 0 or default if not present."""
try:
return l[0]
except IndexError:
return default
def dget(d, key, sep='/'):
"""d is a dict, key is foo/bar/baz, return d['foo']['bar']['baz']."""
def _rget(d, k):
if len(k) == 1:
return d[k[0]]
return _rget(d[k[0]], k[1:])
return _rget(d, key.split(sep))
def lr_selector(iterable, left_cond, right_cond):
left, right = [], []
# Use a single iterator
iterator = iter(iterable)
def _next_heads():
"""Extract at least one item for each list."""
for item in iterator:
if left_cond(item):
left.append(item)
if right_cond(item):
right.append(item)
if left and right:
break # We got at least one item per list, stop here
def _make_picker(target):
"""Pick the next element from a list, filling the list first."""
def _next(default=None, peek=False):
if not target:
_next_heads() # Try to pop the next element
if not target:
return default # Nothing left, return default
if peek:
return target[0] # Just peek the value
else:
v, target[:] = target[0], target[1:] # Pop head
return v
return _next
return _make_picker(left), _make_picker(right)
def stream_template(template_code, **context):
app.update_template_context(context)
# t = app.jinja_env.Template(template_code)
t = jinja2.Template(template_code)
rv = t.stream(context)
rv.enable_buffering(5)
return rv
def make_viewer(args):
"""Build a log viewer."""
print("Building logview with arguments:", args)
@app.route("/", defaults={'ltag': None, 'rtag': None, 'lloc': None, 'rloc': None})
@app.route("/<ltag>/<rtag>/", strict_slashes=False, defaults={'lloc': None, 'rloc': None})
@app.route("/<ltag>/<lloc>/<rtag>/<rloc>/", strict_slashes=False)
def logview(ltag, lloc, rtag, rloc):
fname = args.log_file
# TODO use generator for data streaming
# see http://flask.pocoo.org/docs/1.0/patterns/streaming/
# Get timestamps
res = args.resolution
# Merge by timestamps
fill_value = ''
def left_cond(row):
row_tags = dget(row, args.first_path)
row_loc = dget(row, args.second_path)
tag_cond = ltag is not None and ltag in row_tags
loc_cond = lloc is None or lloc == row_loc
return tag_cond and loc_cond
def right_cond(row):
row_tags = dget(row, args.first_path)
row_loc = dget(row, args.second_path)
tag_cond = rtag is not None and rtag in row_tags
loc_cond = lloc is None or rloc == row_loc
return tag_cond and loc_cond
def extract_data(item):
if item is None:
return None
time = dget(item, args.time_path)
message = dget(item, args.message_path)
return (time, message)
fp = open(fname, 'rt')
js = map(json.loads, fp) # Convert each line to json object
next_left, next_right = lr_selector(js, left_cond, right_cond)
# Merge lines by time
def _generate_rows():
while True:
l = extract_data(next_left(peek=True))
r = extract_data(next_right(peek=True))
if r is None and l is None:
fp.close()
break
elif r is None or l is not None and l[0]//res < r[0]//res:
yield (l[1], l[0], fill_value)
next_left()
elif l is None or r is not None and l[0]//res > r[0]//res:
yield (fill_value, r[0], r[1])
next_right()
else:
yield (l[1], l[0], r[1])
next_left()
next_right()
# Stream response
return Response(stream_template(log_view_template,
fname=fname,
items=_generate_rows(),
ltag=ltag,
rtag=rtag))
# Render template with rows
return render_template_string(log_view_template,
fname=fname,
items=merged_rows,
ltag=ltag,
rtag=rtag)
return logview
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--resolution', type=int, default=1000,
help='Timestamp resolution to match columns')
parser.add_argument('--time-path', default='context/time')
parser.add_argument('--message-path', default='message')
# TODO: arbitrary length
parser.add_argument('--first-path', default='context/tags')
parser.add_argument('--second-path', default='context/location/filename')
parser.add_argument('log_file', help='Log file to process')
args = parser.parse_args()
make_viewer(args)
server = WSGIServer(('', 5000), app)
server.serve_forever()
Raw
Pipfile
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[dev-packages]
neovim = "*"
[packages]
flask = "*"
gevent = "*"
"jinja2" = "*"
greenlet = "*"
[requires]
python_version = "3.7"
Raw
Pipfile.lock
{
"_meta": {
"hash": {
"sha256": "8f0f48695208bbce1b66edbcaa5f7a788f02305da313ded19ca04ac9b271f87f"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.7"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"click": {
"hashes": [
"sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
"sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
],
"version": "==7.0"
},
"flask": {
"hashes": [
"sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48",
"sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05"
],
"index": "pypi",
"version": "==1.0.2"
},
"gevent": {
"hashes": [
"sha256:1f277c5cf060b30313c5f9b91588f4c645e11839e14a63c83fcf6f24b1bc9b95",
"sha256:298a04a334fb5e3dcd6f89d063866a09155da56041bc94756da59db412cb45b1",
"sha256:30e9b2878d5b57c68a40b3a08d496bcdaefc79893948989bb9b9fab087b3f3c0",
"sha256:33533bc5c6522883e4437e901059fe5afa3ea74287eeea27a130494ff130e731",
"sha256:3f06f4802824c577272960df003a304ce95b3e82eea01dad2637cc8609c80e2c",
"sha256:419fd562e4b94b91b58cccb3bd3f17e1a11f6162fca6c591a7822bc8a68f023d",
"sha256:4ea938f44b882e02cca9583069d38eb5f257cc15a03e918980c307e7739b1038",
"sha256:51143a479965e3e634252a0f4a1ea07e5307cf8dc773ef6bf9dfe6741785fb4c",
"sha256:5bf9bd1dd4951552d9207af3168f420575e3049016b9c10fe0c96760ce3555b7",
"sha256:6004512833707a1877cc1a5aea90fd182f569e089bc9ab22a81d480dad018f1b",
"sha256:640b3b52121ab519e0980cb38b572df0d2bc76941103a697e828c13d76ac8836",
"sha256:6951655cc18b8371d823e81c700883debb0f633aae76f425dfeb240f76b95a67",
"sha256:71eeb8d9146e8131b65c3364bb760b097c21b7b9fdbec91bf120685a510f997a",
"sha256:7c899e5a6f94d6310352716740f05e41eb8c52d995f27fc01e90380913aa8f22",
"sha256:8465f84ba31aaf52a080837e9c5ddd592ab0a21dfda3212239ce1e1796f4d503",
"sha256:99de2e38dde8669dd30a8a1261bdb39caee6bd00a5f928d01dfdb85ab0502562",
"sha256:9fa4284b44bc42bef6e437488d000ae37499ccee0d239013465638504c4565b7",
"sha256:a1beea0443d3404c03e069d4c4d9fc13d8ec001771c77f9a23f01911a41f0e49",
"sha256:a66a26b78d90d7c4e9bf9efb2b2bd0af49234604ac52eaca03ea79ac411e3f6d",
"sha256:a94e197bd9667834f7bb6bd8dff1736fab68619d0f8cd78a9c1cebe3c4944677",
"sha256:ac0331d3a3289a3d16627742be9c8969f293740a31efdedcd9087dadd6b2da57",
"sha256:d26b57c50bf52fb38dadf3df5bbecd2236f49d7ac98f3cf32d6b8a2d25afc27f",
"sha256:fd23b27387d76410eb6a01ea13efc7d8b4b95974ba212c336e8b1d6ab45a9578"
],
"index": "pypi",
"version": "==1.3.7"
},
"greenlet": {
"hashes": [
"sha256:000546ad01e6389e98626c1367be58efa613fa82a1be98b0c6fc24b563acc6d0",
"sha256:0d48200bc50cbf498716712129eef819b1729339e34c3ae71656964dac907c28",
"sha256:23d12eacffa9d0f290c0fe0c4e81ba6d5f3a5b7ac3c30a5eaf0126bf4deda5c8",
"sha256:37c9ba82bd82eb6a23c2e5acc03055c0e45697253b2393c9a50cef76a3985304",
"sha256:51503524dd6f152ab4ad1fbd168fc6c30b5795e8c70be4410a64940b3abb55c0",
"sha256:8041e2de00e745c0e05a502d6e6db310db7faa7c979b3a5877123548a4c0b214",
"sha256:81fcd96a275209ef117e9ec91f75c731fa18dcfd9ffaa1c0adbdaa3616a86043",
"sha256:853da4f9563d982e4121fed8c92eea1a4594a2299037b3034c3c898cb8e933d6",
"sha256:8b4572c334593d449113f9dc8d19b93b7b271bdbe90ba7509eb178923327b625",
"sha256:9416443e219356e3c31f1f918a91badf2e37acf297e2fa13d24d1cc2380f8fbc",
"sha256:9854f612e1b59ec66804931df5add3b2d5ef0067748ea29dc60f0efdcda9a638",
"sha256:99a26afdb82ea83a265137a398f570402aa1f2b5dfb4ac3300c026931817b163",
"sha256:a19bf883b3384957e4a4a13e6bd1ae3d85ae87f4beb5957e35b0be287f12f4e4",
"sha256:a9f145660588187ff835c55a7d2ddf6abfc570c2651c276d3d4be8a2766db490",
"sha256:ac57fcdcfb0b73bb3203b58a14501abb7e5ff9ea5e2edfa06bb03035f0cff248",
"sha256:bcb530089ff24f6458a81ac3fa699e8c00194208a724b644ecc68422e1111939",
"sha256:beeabe25c3b704f7d56b573f7d2ff88fc99f0138e43480cecdfcaa3b87fe4f87",
"sha256:d634a7ea1fc3380ff96f9e44d8d22f38418c1c381d5fac680b272d7d90883720",
"sha256:d97b0661e1aead761f0ded3b769044bb00ed5d33e1ec865e891a8b128bf7c656"
],
"index": "pypi",
"version": "==0.4.15"
},
"itsdangerous": {
"hashes": [
"sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19",
"sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749"
],
"version": "==1.1.0"
},
"jinja2": {
"hashes": [
"sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd",
"sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4"
],
"index": "pypi",
"version": "==2.10"
},
"markupsafe": {
"hashes": [
"sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432",
"sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b",
"sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9",
"sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af",
"sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834",
"sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd",
"sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d",
"sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7",
"sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b",
"sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3",
"sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c",
"sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2",
"sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7",
"sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36",
"sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1",
"sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e",
"sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1",
"sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c",
"sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856",
"sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550",
"sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492",
"sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672",
"sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401",
"sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6",
"sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6",
"sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c",
"sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd",
"sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1"
],
"version": "==1.1.0"
},
"werkzeug": {
"hashes": [
"sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c",
"sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b"
],
"version": "==0.14.1"
}
},
"develop": {
"greenlet": {
"hashes": [
"sha256:000546ad01e6389e98626c1367be58efa613fa82a1be98b0c6fc24b563acc6d0",
"sha256:0d48200bc50cbf498716712129eef819b1729339e34c3ae71656964dac907c28",
"sha256:23d12eacffa9d0f290c0fe0c4e81ba6d5f3a5b7ac3c30a5eaf0126bf4deda5c8",
"sha256:37c9ba82bd82eb6a23c2e5acc03055c0e45697253b2393c9a50cef76a3985304",
"sha256:51503524dd6f152ab4ad1fbd168fc6c30b5795e8c70be4410a64940b3abb55c0",
"sha256:8041e2de00e745c0e05a502d6e6db310db7faa7c979b3a5877123548a4c0b214",
"sha256:81fcd96a275209ef117e9ec91f75c731fa18dcfd9ffaa1c0adbdaa3616a86043",
"sha256:853da4f9563d982e4121fed8c92eea1a4594a2299037b3034c3c898cb8e933d6",
"sha256:8b4572c334593d449113f9dc8d19b93b7b271bdbe90ba7509eb178923327b625",
"sha256:9416443e219356e3c31f1f918a91badf2e37acf297e2fa13d24d1cc2380f8fbc",
"sha256:9854f612e1b59ec66804931df5add3b2d5ef0067748ea29dc60f0efdcda9a638",
"sha256:99a26afdb82ea83a265137a398f570402aa1f2b5dfb4ac3300c026931817b163",
"sha256:a19bf883b3384957e4a4a13e6bd1ae3d85ae87f4beb5957e35b0be287f12f4e4",
"sha256:a9f145660588187ff835c55a7d2ddf6abfc570c2651c276d3d4be8a2766db490",
"sha256:ac57fcdcfb0b73bb3203b58a14501abb7e5ff9ea5e2edfa06bb03035f0cff248",
"sha256:bcb530089ff24f6458a81ac3fa699e8c00194208a724b644ecc68422e1111939",
"sha256:beeabe25c3b704f7d56b573f7d2ff88fc99f0138e43480cecdfcaa3b87fe4f87",
"sha256:d634a7ea1fc3380ff96f9e44d8d22f38418c1c381d5fac680b272d7d90883720",
"sha256:d97b0661e1aead761f0ded3b769044bb00ed5d33e1ec865e891a8b128bf7c656"
],
"index": "pypi",
"version": "==0.4.15"
},
"msgpack": {
"hashes": [
"sha256:102802a9433dcf36f939b632cce9dea87310b2f163bb37ffc8bc343677726e88",
"sha256:3055c44f39833b6edb27fd48028dc7822d1fd75bfeef8a2434caed8d62bb24ee",
"sha256:3b7fd45c8e9e537640f541d3699b1773cf5cb9345d4a75f93baa8f055084e59c",
"sha256:64abc6bf3a2ac301702f5760f4e6e227d0fd4d84d9014ef9a40faa9d43365259",
"sha256:6e962c4adc7970af5a3d6a4f9bb87c617b1bd041fd9ab42355a263d421017ed9",
"sha256:72259661a83f8b08ef6ee83927ce4937f841226735824af5b10a536d886eeb36",
"sha256:78e297c3996fd9f35090fbddd1c148c2a71e0d6024500bcf3af90a4b9698bc19",
"sha256:85f1342b9d7549dd3daf494100d47a3dc7daae703cdbfc2c9ee7bbdc8a492cba",
"sha256:8ce9f88b6cb75d74eda2a5522e5c2e5ec0f17fd78605d6502abb61f46b306865",
"sha256:8d0af8d64198e4b4f942a15ea9cb0dd9c4a0bd3e4e2ba57425e108bdbd4c3a0f",
"sha256:9936ce3a530ca78db60b6631003b5f4ba383cfb1d9830a27d1b5c61857226e2f",
"sha256:b688721df31c4bad6f508fb262719eb7e4a3532024c66d3c44ad6a4704519dda",
"sha256:c28478328e9cd868ce54e8465eae9fa3605790450c66cc7e8bc416526917ef6e",
"sha256:cb4e228f3d93779a1d77a1e9d72759b79dfa2975c1a5bd2a090eaa98239fa4b1",
"sha256:d03d0b6e4adf5bd1cbf7a81a20a56c883351947a57b7b85235181b057adf1120",
"sha256:d2b179faebd278e5f4e255a6bbc7ccb467f02ed5c4c00c8a68dc926002223a20",
"sha256:f1a8f7bd84be103979a73da57be3cb929d702a656162ee466597b816fa9eec97"
],
"version": "==0.6.0"
},
"neovim": {
"hashes": [
"sha256:a6a0e7a5b4433bf4e6ddcbc5c5ff44170be7d84259d002b8e8d8fb4ee78af60f"
],
"index": "pypi",
"version": "==0.3.1"
},
"pynvim": {
"hashes": [
"sha256:dd881595055869c2de770517d403faf40d31aa991db2472a1843ff17db47b0fb"
],
"version": "==0.3.1"
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment