check current block head level from multiple sources

headcheck.py

#!/bin/env python36

import requests
import json
from time import sleep
from datetime import timedelta, datetime as dt
from requests.exceptions import Timeout, HTTPError, ConnectionError
from concurrent.futures import TimeoutError, ThreadPoolExecutor

nodes = [
        ["Local", 'http://127.0.0.1:8732/chains/main/blocks/head/header'],
        ["TZBeta", 'https://rpc.tzbeta.net/chains/main/blocks/head/header'],
]

##
##      ["Obsidian", 'https://tezos-api.obsidian.systems/api/v1/NetXdQprcVkpaWU/head'],
##

def getLevel(node):

        name = node[0]
        url = node[1]

        try:
                resp = requests.get(url, timeout=10)
                resp.raise_for_status()
        except Timeout:
                return [name, 0, dt.now(), "(T)"]
        except HTTPError as e:
                return [name, 0, dt.now(), "(E) {}".format(e.response.status_code)]
        except ConnectionError as e:
                return [name, 0, dt.now(), "(CE)"]
        else:

                js = resp.json()

                #print("JSON: %s", js)
                #print("Level: {}".format(js['level']))

                # Make datetime object from Tezos timestamp, 2019-07-02T19:06:30Z
                mdt = dt.strptime(js['timestamp'], "%Y-%m-%dT%H:%M:%SZ")

                # returning tuple
                return [name, js['level'], mdt]

def getLevels():

        global nodes

        levels = []

        with ThreadPoolExecutor(max_workers=3) as exec:

                results = exec.map(getLevel, nodes, timeout=15)

                try:
                        # Get node levels
                        for i in results:
                                levels.append(i)

                except TimeoutError as te:
                        print("Timeout Error: {}".format(te))
                        print("DEBUG: {}".format(levels))

        return levels

def getStatus(lvls):

        # "my level" should always be index 0
        myLevel = lvls.pop(0)

        # Highest level, of remaining data points
        highLevel = max(lvls, key=lambda x: x[1])

        # Logic
        status = ""

        if (myLevel[1] == highLevel[1]):
                status = "Sync"
        elif (myLevel[1] < highLevel[1]):

                # Check if timestamps within 10s
                tenSeconds = timedelta(seconds=10)

                if ((myLevel[2] + tenSeconds) >= highLevel[2]):

                        # This is ok. Local node within 10s
                        status = "S-10"

                else:
                        # Probably bad, need to alarm
                        # Ideally would inform baker/endorser to RPC to remote
                        status = "NonS"
        else:
                status = "High"

        return status

def main():

        global nodes

        # print table header
        titles = ['status'] + list(map(lambda x: x[0], nodes))
        print('| '.join(str(x).ljust(12) for x in titles))

        try:
                while True:

                        levelStats = getLevels()
                        if (len(levelStats) < 2):
                                print("-- Unable to fetch enough stats")
                        else:
                                status = getStatus(levelStats.copy())

                                # Copy/Append potential timeout/error message for pretty print
                                for i, d in enumerate(levelStats):
                                        if (levelStats[i][1] == 0):
                                                levelStats[i][1] = "{} {}".format(str(levelStats[i][1]), levelStats[i][3])
                                                del(levelStats[i][3])

                                # combine status plus levels
                                data = [status] + list(map(lambda x: x[1], levelStats))

                                # print table data
                                print('| '.join(str(x).ljust(12) for x in data))

                        sleep(30)

        except KeyboardInterrupt:
                print("Exiting...")

if __name__ == "__main__":
        main()