NemのSupernodeの一覧を取得するPythonスクリプト

nem-sn-json.py

import json
import sys
import datetime
import math
import requests
import bs4

"""
https://supernodes.nem.io/をスクレイピングしてSupernodeの一覧を格納した辞書オブジェクトとjsonオブジェクトを作ります。
引数で取得件数を決められます。省略すると全部取ります。
Python 3.6、RequestsとBeautiful Soup4とlxmlが必要です。
"""

ARGVS = sys.argv
ARGC = len(ARGVS)

URL = 'https://supernodes.nem.io/'
try:
    RESP = requests.get(URL, timeout=30)
except requests.exceptions.ConnectTimeout:
    print('Timeout')
except requests.exceptions.ConnectionError:
    print('Connection Error')
except requests.exceptions.HTTPError:
    print('HTTP Error')
PARSETEXT = bs4.BeautifulSoup(RESP.text, "lxml")
TD = PARSETEXT.select("td")
NOW = datetime.datetime.utcnow()

# supernode list dictionary
NIS = {"Datetime": NOW.isoformat(timespec='seconds') + 'Z'}
# supernode csv text
NISCSV = NOW.isoformat(timespec='seconds') + 'Z'
i = 0
if ARGC < 2:
    MAXINDEX = len(TD) / 4
else:
    MAXINDEX = int(ARGVS[1])
while i < (MAXINDEX * 4):
    NIS.update({int(TD[i + 0].getText()): {"Address": TD[i + 1].getText(),
                                           "Name": TD[i + 2].getText(),
                                           "Status": TD[i + 3].getText()}})
    NISCSV += ',' + TD[i + 1].getText()
    i += 4

# print(NIS)
# print(NISCSV)
# print()
# NISという辞書オブジェクトに取得した日時(UTC)とSupernodeの一覧が格納されています
# NISCSVという文字列オブジェクトに取得した日時(UTC)とSupernodeのアドレスの一覧が格納されています

i = 1
while i < (MAXINDEX+1):
    if (i in NIS) and (NIS[i]["Status"] == "Active"):
        URL = 'http://' + NIS[i]["Address"] + ':7890/node/extended-info'
        try:
            RESP = requests.get(URL, timeout=30)
        except requests.exceptions.ConnectTimeout:
            print('  Timeout', file=sys.stderr)
        except requests.exceptions.Timeout:
            print('  Timeout', file=sys.stderr)
        except requests.exceptions.ConnectionError:
            print('  Connection Error', file=sys.stderr)
        except requests.exceptions.HTTPError:
            print('  HTTP Error', file=sys.stderr)
        NISINFOJSON = json.loads(RESP.text)
        NIS[i]["Version"] = NISINFOJSON["nisInfo"]["version"]
        NIS[i]["Uptime"] = NISINFOJSON["nisInfo"]["currentTime"] - \
            NISINFOJSON["nisInfo"]["startTime"]  # Uptime is 'second'
    i += 1

# jsonオブジェクトに変換します
NISJSON = json.dumps(NIS, indent=2)
# print(NISJSON)

sample.csv

sample.json

{
  "Datetime": "2018-01-27T06:21:06Z",
  "1": {
    "Address": "176.9.68.110",
    "Name": "HugeAlice2",
    "Status": "Active",
    "Version": "0.6.95-BETA",
    "Uptime": 2925808
  },
  "2": {
    "Address": "176.9.20.180",
    "Name": "HugeAlice3",
    "Status": "Active",
    "Version": "0.6.95-BETA",
    "Uptime": 2924737
  },
  "3": {
    "Address": "199.217.118.114",
    "Name": "HugeAlice4",
    "Status": "Active",
    "Version": "0.6.95-BETA",
    "Uptime": 2823510
  },
  "4": {
    "Address": "108.61.182.27",
    "Name": "Alice5",
    "Status": "Active",
    "Version": "0.6.95-BETA",
    "Uptime": 3009313
  },
  "5": {
    "Address": "108.61.168.86",
    "Name": "Alice6",
    "Status": "Active",
    "Version": "0.6.95-BETA",
    "Uptime": 3009312
  },
  "6": {
    "Address": "104.238.161.61",
    "Name": "Alice7",
    "Status": "Active",
    "Version": "0.6.95-BETA",
    "Uptime": 3009311
  },
  "7": {
    "Address": "88.99.192.82",
    "Name": "HugeAlice",
    "Status": "Active",
    "Version": "0.6.95-BETA",
    "Uptime": 3011448
  },
  "8": {
    "Address": "san.nem.ninja",
    "Name": "san",
    "Status": "Active",
    "Version": "0.6.95-BETA",
    "Uptime": 1271158
  },
  "9": {
    "Address": "go.nem.ninja",
    "Name": "go",
    "Status": "Active",
    "Version": "0.6.95-BETA",
    "Uptime": 1271144
  },
  "10": {
    "Address": "hachi.nem.ninja",
    "Name": "hachi",
    "Status": "Active",
    "Version": "0.6.95-BETA",
    "Uptime": 1271145
  }
}