Skip to content

Instantly share code, notes, and snippets.

@yoursunny

yoursunny/.gitignore

Last active February 4, 2021 03:10
Show Gist options
  • Save yoursunny/f9087dc7499178eabca5fe7ea5d15203 to your computer and use it in GitHub Desktop.
Save yoursunny/f9087dc7499178eabca5fe7ea5d15203 to your computer and use it in GitHub Desktop.
Download ZIP
The Reality of NDN Video Streaming https://yoursunny.com/t/2021/NDN-video-reality/
Raw
.gitignore
*.mmdb
*.ndjson
.vscode
__pycache__
Raw
README.md

The Reality of NDN Video Streaming

This gist contains code and input data from blog article The Reality of NDN Video Streaming.

To run this script:

  1. pipenv install
  2. Download MaxMind GeoLite2-ASN.mmdb and GeoLite2-City.mmdb
  3. ./process.sh
Raw
decode.py
#!/usr/bin/python
import datetime
import ipaddress
import json
import sys
import urllib.parse
import geoip2.database
import ndjson
geoip2asn = geoip2.database.Reader("GeoLite2-ASN.mmdb")
geoip2city = geoip2.database.Reader("GeoLite2-City.mmdb")
writer = ndjson.writer(sys.stdout)
date0 = datetime.datetime(2021, 1, 1, tzinfo=datetime.timezone.utc).timestamp()
date1 = datetime.datetime(2021, 2, 1, tzinfo=datetime.timezone.utc).timestamp()
for j in ndjson.reader(sys.stdin):
try:
r = json.loads(urllib.parse.unquote(j["request"]["uri"][1:]))
if j["ts"] < date0 or j["ts"] >= date1:
continue
r["ts"] = j["ts"]
ip = urllib.parse.urlsplit("//" + j["request"]["remote_addr"]).hostname
subnet = ipaddress.IPv4Network((ip, 24), strict=False)
asn = geoip2asn.asn(ip)
city = geoip2city.city(ip)
r["ip"] = str(subnet.network_address)
r["asn"] = asn.autonomous_system_number
r["location"] = [city.location.longitude, city.location.latitude]
r["city_accuracy"] = city.city.geoname_id is not None
r["country"] = city.country.iso_code
r["continent"] = city.continent.code
writer.writerow(r)
except Exception:
pass
Raw
group.py
#!/usr/bin/python
import sys
from collections import defaultdict
from typing import Any, Dict, List, Set
import ndjson
from ndn.encoding.name import Name
from testbed import find_router
Record = Dict[str, Any]
class Video(dict):
def __init__(self, name: Name.FormalName):
self.name = name
self["name"] = Name.to_str(name)
self["fetch"] = []
self["playback"] = []
def add_fetch_record(self, j: Record):
j["n"] = Name.to_str(Name.from_str(j["n"])[len(self.name):])
self["fetch"].append(j)
def add_playback_record(self, j: Record):
del j["n"]
self["playback"].append(j)
class Session(dict):
SESS_KEYS = ["sess", "site", "remote", "ip", "asn",
"location", "city_accuracy", "country", "continent"]
def __init__(self):
self.fetch: List[Record] = []
self.playback: List[Record] = []
def add_record(self, j: Record):
for KEY in Session.SESS_KEYS:
self[KEY] = j.pop(KEY, None)
if j["a"] == "P":
del j["a"]
self.playback.append(j)
else:
self.fetch.append(j)
def tag_router(self):
router = find_router(self["remote"])
self["router"] = router
self["router_dist"] = None if router is None \
else router.geodistance(self["location"])
def bin_videos(self):
videos: Dict[bytes, Video] = dict()
unaccounted: List[Record] = []
for j in self.playback:
name = Name.from_str(j["n"])[:-1]
video = videos.setdefault(Name.to_bytes(name), Video(name))
video.add_playback_record(j)
for j in self.fetch:
name = Name.from_str(j["n"])
found = False
for video in videos.values():
if Name.is_prefix(video.name, name):
video.add_fetch_record(j)
found = True
break
if not found:
unaccounted.append(j)
self["V"] = list(videos.values())
self["F"] = unaccounted
sessions: defaultdict[str, Session] = defaultdict(lambda: Session())
for j in ndjson.reader(sys.stdin):
sessions[j["sess"]].add_record(j)
for s in sessions.values():
s.tag_router()
s.bin_videos()
ndjson.dump(sessions.values(), sys.stdout)
Raw
Pipfile
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[pipenv]
allow_prereleases = true
[packages]
geoip2 = "~=4.1.0"
geopy = "~=2.1.0"
ndjson = "~=0.3.1"
python-ndn = "==0.3a1.post1"
[dev-packages]
autopep8 = "*"
isort = "*"
[requires]
python_version = "3.9"
[scripts]
lint = "bash -c \"autopep8 -i *.py && isort *.py\""
Raw
Pipfile.lock
{
"_meta": {
"hash": {
"sha256": "22aa8c3e8beee0a97ba5a481d7088abb79b11bf99784eeb0399b0d63a0eb4f9c"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.9"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"aiohttp": {
"hashes": [
"sha256:0b795072bb1bf87b8620120a6373a3c61bfcb8da7e5c2377f4bb23ff4f0b62c9",
"sha256:0d438c8ca703b1b714e82ed5b7a4412c82577040dadff479c08405e2a715564f",
"sha256:16a3cb5df5c56f696234ea9e65e227d1ebe9c18aa774d36ff42f532139066a5f",
"sha256:1edfd82a98c5161497bbb111b2b70c0813102ad7e0aa81cbeb34e64c93863005",
"sha256:2406dc1dda01c7f6060ab586e4601f18affb7a6b965c50a8c90ff07569cf782a",
"sha256:2858b2504c8697beb9357be01dc47ef86438cc1cb36ecb6991796d19475faa3e",
"sha256:2a7b7640167ab536c3cb90cfc3977c7094f1c5890d7eeede8b273c175c3910fd",
"sha256:3228b7a51e3ed533f5472f54f70fd0b0a64c48dc1649a0f0e809bec312934d7a",
"sha256:328b552513d4f95b0a2eea4c8573e112866107227661834652a8984766aa7656",
"sha256:39f4b0a6ae22a1c567cb0630c30dd082481f95c13ca528dc501a7766b9c718c0",
"sha256:3b0036c978cbcc4a4512278e98e3e6d9e6b834dc973206162eddf98b586ef1c6",
"sha256:3ea8c252d8df5e9166bcf3d9edced2af132f4ead8ac422eac723c5781063709a",
"sha256:41608c0acbe0899c852281978492f9ce2c6fbfaf60aff0cefc54a7c4516b822c",
"sha256:59d11674964b74a81b149d4ceaff2b674b3b0e4d0f10f0be1533e49c4a28408b",
"sha256:5e479df4b2d0f8f02133b7e4430098699450e1b2a826438af6bec9a400530957",
"sha256:684850fb1e3e55c9220aad007f8386d8e3e477c4ec9211ae54d968ecdca8c6f9",
"sha256:6ccc43d68b81c424e46192a778f97da94ee0630337c9bbe5b2ecc9b0c1c59001",
"sha256:6d42debaf55450643146fabe4b6817bb2a55b23698b0434107e892a43117285e",
"sha256:710376bf67d8ff4500a31d0c207b8941ff4fba5de6890a701d71680474fe2a60",
"sha256:756ae7efddd68d4ea7d89c636b703e14a0c686688d42f588b90778a3c2fc0564",
"sha256:77149002d9386fae303a4a162e6bce75cc2161347ad2ba06c2f0182561875d45",
"sha256:78e2f18a82b88cbc37d22365cf8d2b879a492faedb3f2975adb4ed8dfe994d3a",
"sha256:7d9b42127a6c0bdcc25c3dcf252bb3ddc70454fac593b1b6933ae091396deb13",
"sha256:8389d6044ee4e2037dca83e3f6994738550f6ee8cfb746762283fad9b932868f",
"sha256:9c1a81af067e72261c9cbe33ea792893e83bc6aa987bfbd6fdc1e5e7b22777c4",
"sha256:c1e0920909d916d3375c7a1fdb0b1c78e46170e8bb42792312b6eb6676b2f87f",
"sha256:c68fdf21c6f3573ae19c7ee65f9ff185649a060c9a06535e9c3a0ee0bbac9235",
"sha256:c733ef3bdcfe52a1a75564389bad4064352274036e7e234730526d155f04d914",
"sha256:c9c58b0b84055d8bc27b7df5a9d141df4ee6ff59821f922dd73155861282f6a3",
"sha256:d03abec50df423b026a5aa09656bd9d37f1e6a49271f123f31f9b8aed5dc3ea3",
"sha256:d2cfac21e31e841d60dc28c0ec7d4ec47a35c608cb8906435d47ef83ffb22150",
"sha256:dcc119db14757b0c7bce64042158307b9b1c76471e655751a61b57f5a0e4d78e",
"sha256:df3a7b258cc230a65245167a202dd07320a5af05f3d41da1488ba0fa05bc9347",
"sha256:df48a623c58180874d7407b4d9ec06a19b84ed47f60a3884345b1a5099c1818b",
"sha256:e1b95972a0ae3f248a899cdbac92ba2e01d731225f566569311043ce2226f5e7",
"sha256:f326b3c1bbfda5b9308252ee0dcb30b612ee92b0e105d4abec70335fab5b1245",
"sha256:f411cb22115cb15452d099fec0ee636b06cf81bfb40ed9c02d30c8dc2bc2e3d1"
],
"markers": "python_version >= '3.6'",
"version": "==3.7.3"
},
"async-timeout": {
"hashes": [
"sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
"sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
],
"markers": "python_full_version >= '3.5.3'",
"version": "==3.0.1"
},
"attrs": {
"hashes": [
"sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
"sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==20.3.0"
},
"certifi": {
"hashes": [
"sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
"sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
],
"version": "==2020.12.5"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"version": "==3.0.4"
},
"dataclasses": {
"hashes": [
"sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f",
"sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84"
],
"version": "==0.6"
},
"geographiclib": {
"hashes": [
"sha256:12bd46ee7ec25b291ea139b17aa991e7ef373e21abd053949b75c0e9ca55c632",
"sha256:51cfa698e7183792bce27d8fb63ac8e83689cd8170a730bf35e1a5c5bf8849b9"
],
"version": "==1.50"
},
"geoip2": {
"hashes": [
"sha256:57d8d15de2527e0697bbef44fc16812bba709f03a07ef99297bd56c1df3b1efd",
"sha256:707025542ef076bd8fd80e97138bebdb7812527b2a007d141a27ad98b0370fff"
],
"index": "pypi",
"version": "==4.1.0"
},
"geopy": {
"hashes": [
"sha256:4db8a2b79a2b3358a7d020ea195be639251a831a1b429c0d1b20c9f00c67c788",
"sha256:892b219413e7955587b029949af3a1949c6fbac9d5ad17b79d850718f6a9550f"
],
"index": "pypi",
"version": "==2.1.0"
},
"idna": {
"hashes": [
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==2.10"
},
"maxminddb": {
"hashes": [
"sha256:47e86a084dd814fac88c99ea34ba3278a74bc9de5a25f4b815b608798747c7dc"
],
"markers": "python_version >= '3.6'",
"version": "==2.0.3"
},
"multidict": {
"hashes": [
"sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a",
"sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93",
"sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632",
"sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656",
"sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79",
"sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7",
"sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d",
"sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5",
"sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224",
"sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26",
"sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea",
"sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348",
"sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6",
"sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76",
"sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1",
"sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f",
"sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952",
"sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a",
"sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37",
"sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9",
"sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359",
"sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8",
"sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da",
"sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3",
"sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d",
"sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf",
"sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841",
"sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d",
"sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93",
"sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f",
"sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647",
"sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635",
"sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456",
"sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda",
"sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5",
"sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281",
"sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80"
],
"markers": "python_version >= '3.6'",
"version": "==5.1.0"
},
"ndjson": {
"hashes": [
"sha256:839c22275e6baa3040077b83c005ac24199b94973309a8a1809be962c753a410",
"sha256:bf9746cb6bb1cb53d172cda7f154c07c786d665ff28341e4e689b796b229e5d6"
],
"index": "pypi",
"version": "==0.3.1"
},
"pycryptodomex": {
"hashes": [
"sha256:15c03ffdac17731b126880622823d30d0a3cc7203cd219e6b9814140a44e7fab",
"sha256:20fb7f4efc494016eab1bc2f555bc0a12dd5ca61f35c95df8061818ffb2c20a3",
"sha256:28ee3bcb4d609aea3040cad995a8e2c9c6dc57c12183dadd69e53880c35333b9",
"sha256:305e3c46f20d019cd57543c255e7ba49e432e275d7c0de8913b6dbe57a851bc8",
"sha256:3547b87b16aad6afb28c9b3a9cd870e11b5e7b5ac649b74265258d96d8de1130",
"sha256:3642252d7bfc4403a42050e18ba748bedebd5a998a8cba89665a4f42aea4c380",
"sha256:404faa3e518f8bea516aae2aac47d4d960397199a15b4bd6f66cad97825469a0",
"sha256:42669638e4f7937b7141044a2fbd1019caca62bd2cdd8b535f731426ab07bde1",
"sha256:4632d55a140b28e20be3cd7a3057af52fb747298ff0fd3290d4e9f245b5004ba",
"sha256:4a88c9383d273bdce3afc216020282c9c5c39ec0bd9462b1a206af6afa377cf0",
"sha256:4ce1fc1e6d2fd2d6dc197607153327989a128c093e0e94dca63408f506622c3e",
"sha256:55cf4e99b3ba0122dee570dc7661b97bf35c16aab3e2ccb5070709d282a1c7ab",
"sha256:5e486cab2dfcfaec934dd4f5d5837f4a9428b690f4d92a3b020fd31d1497ca64",
"sha256:65ec88c8271448d2ea109d35c1f297b09b872c57214ab7e832e413090d3469a9",
"sha256:6c95a3361ce70068cf69526a58751f73ddac5ba27a3c2379b057efa2f5338c8c",
"sha256:73240335f4a1baf12880ebac6df66ab4d3a9212db9f3efe809c36a27280d16f8",
"sha256:7651211e15109ac0058a49159265d9f6e6423c8a81c65434d3c56d708417a05b",
"sha256:7b5b7c5896f8172ea0beb283f7f9428e0ab88ec248ce0a5b8c98d73e26267d51",
"sha256:836fe39282e75311ce4c38468be148f7fac0df3d461c5de58c5ff1ddb8966bac",
"sha256:871852044f55295449fbf225538c2c4118525093c32f0a6c43c91bed0452d7e3",
"sha256:892e93f3e7e10c751d6c17fa0dc422f7984cfd5eb6690011f9264dc73e2775fc",
"sha256:934e460c5058346c6f1d62fdf3db5680fbdfbfd212722d24d8277bf47cd9ebdc",
"sha256:9736f3f3e1761024200637a080a4f922f5298ad5d780e10dbb5634fe8c65b34c",
"sha256:a1d38a96da57e6103423a446079ead600b450cf0f8ebf56a231895abf77e7ffc",
"sha256:a385fceaa0cdb97f0098f1c1e9ec0b46cc09186ddf60ec23538e871b1dddb6dc",
"sha256:a7cf1c14e47027d9fb9d26aa62e5d603994227bd635e58a8df4b1d2d1b6a8ed7",
"sha256:a9aac1a30b00b5038d3d8e48248f3b58ea15c827b67325c0d18a447552e30fc8",
"sha256:b696876ee583d15310be57311e90e153a84b7913ac93e6b99675c0c9867926d0",
"sha256:bef9e9d39393dc7baec39ba4bac6c73826a4db02114cdeade2552a9d6afa16e2",
"sha256:c885fe4d5f26ce8ca20c97d02e88f5fdd92c01e1cc771ad0951b21e1641faf6d",
"sha256:d2d1388595cb5d27d9220d5cbaff4f37c6ec696a25882eb06d224d241e6e93fb",
"sha256:d2e853e0f9535e693fade97768cf7293f3febabecc5feb1e9b2ffdfe1044ab96",
"sha256:d62fbab185a6b01c5469eda9f0795f3d1a5bba24f5a5813f362e4b73a3c4dc70",
"sha256:f20a62397e09704049ce9007bea4f6bad965ba9336a760c6f4ef1b4192e12d6d",
"sha256:f81f7311250d9480e36dec819127897ae772e7e8de07abfabe931b8566770b8e"
],
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==3.9.9"
},
"pygtrie": {
"hashes": [
"sha256:43205559d28863358dbbf25045029f58e2ab357317a59b11f11ade278ac64692"
],
"version": "==2.4.2"
},
"python-ndn": {
"hashes": [
"sha256:40cb44c311cfc78a4e7c21f1133a90dd1fdd1a228c2ea57d98e11c560895b515",
"sha256:c0b846d544aa9e535b55dd24f73e6be5712195e3bfc7dc12741b6900bc7cdc0c"
],
"index": "pypi",
"version": "==0.3a1.post1"
},
"requests": {
"hashes": [
"sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
"sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==2.25.1"
},
"typing-extensions": {
"hashes": [
"sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
"sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
"sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
],
"version": "==3.7.4.3"
},
"urllib3": {
"hashes": [
"sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80",
"sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
"version": "==1.26.3"
},
"yarl": {
"hashes": [
"sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e",
"sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434",
"sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366",
"sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3",
"sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec",
"sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959",
"sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e",
"sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c",
"sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6",
"sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a",
"sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6",
"sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424",
"sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e",
"sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f",
"sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50",
"sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2",
"sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc",
"sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4",
"sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970",
"sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10",
"sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0",
"sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406",
"sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896",
"sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643",
"sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721",
"sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478",
"sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724",
"sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e",
"sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8",
"sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96",
"sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25",
"sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76",
"sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2",
"sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2",
"sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c",
"sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a",
"sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71"
],
"markers": "python_version >= '3.6'",
"version": "==1.6.3"
}
},
"develop": {
"autopep8": {
"hashes": [
"sha256:d21d3901cb0da6ebd1e83fc9b0dfbde8b46afc2ede4fe32fbda0c7c6118ca094"
],
"index": "pypi",
"version": "==1.5.4"
},
"isort": {
"hashes": [
"sha256:c729845434366216d320e936b8ad6f9d681aab72dc7cbc2d51bedc3582f3ad1e",
"sha256:fff4f0c04e1825522ce6949973e83110a6e907750cd92d128b0d14aaaadbffdc"
],
"index": "pypi",
"version": "==5.7.0"
},
"pycodestyle": {
"hashes": [
"sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367",
"sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==2.6.0"
},
"toml": {
"hashes": [
"sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
"sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
],
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==0.10.2"
}
}
}
Raw
process.sh
#!/bin/bash
set -eo pipefail
if ! [[ -f testbed.ndjson ]]; then
pipenv run python testbed.py > testbed.ndjson
fi
# routers by continent
jq -sc 'group_by(.continent)[] | [.[0].continent, (.|length), (.|map(select(.wss))|length)]' testbed.ndjson > testbed-continent.ndjson
if ! [[ -f logs.ndjson ]]; then
xzcat ndnts-video-beacon.log.xz | pipenv run python decode.py > logs.ndjson
fi
if ! [[ -f sessions.ndjson ]]; then
pipenv run python group.py < logs.ndjson > sessions.ndjson
fi
# log entries by type
jq -sc 'group_by(.a)[] | [.[0].a, (.|length)]' logs.ndjson > log-type.ndjson
# session video count
jq -sc 'group_by(.V|length)[] | [(.[0].V|length), (.|length)]' sessions.ndjson > session-videos.ndjson
# session location accuracy
jq -sc 'group_by(.city_accuracy)[] | [(.[0].city_accuracy), (.|length)]' sessions.ndjson > session-city-accuracy.ndjson
# session continent and top country
jq -sc 'group_by(.continent)[] | [.[0].continent, (.|length), (.|group_by(.country)|sort_by(.|length)|.[-1] | (.[0].country, (.|length)))]' sessions.ndjson > continent-country.ndjson
# playback duration
jq -sc 'map(.V[].playback | length) | group_by(.)[] | [5*(.[0]), (.|length)]' sessions.ndjson > duration.ndjson
# interval resolution by playback time
jq -sc '. as $a | range(map(.V[].playback | length) | max) | [(. as $i | [null,240,360,480,720,1080] | .[] as $r | [$a[].V[].playback | select((.|length)>$i and .[$i].r==$r)] | length)]' sessions.ndjson > resolution-time.ndjson
# interval resolution by continent
jq -sc '. as $a | map(.continent) | unique[] as $continent | [$continent, ([null,240,360,480,720,1080] | .[] as $r | [$a[] | select(.continent==$continent) | .V[].playback[] | select(.r==$r)] | length)]' sessions.ndjson > resolution-continent.ndjson
# startup latency CDF by continent
jq -sc '. as $a | map(.continent) | unique[] | . as $continent | [$continent, ([$a[] | select(.continent==$continent) | .V[].playback[-1].tl | select(.!=null)] | sort)]' sessions.ndjson > startup-continent.ndjson
# router distance - avg estimated bandwidth - median resolution
jq -c '
def avg: . | map(select(.!=null)) | length as $len | if $len==0 then null else (.|add/$len) end;
def median: . | map(select(.!=null)) | sort | length as $len | if $len==0 then null else .[$len/2 | floor] end;
select((.V|length)>0 and .router_dist!=null) | [.continent, .country, .router_dist, ([.V[].playback[].be] | avg), ([.V[].playback[].r] | median)]' sessions.ndjson | jq -sc '. | sort[]' > router-distance.ndjson
Raw
testbed-nodes.json
{
"UNIVH2C": {
"ws-tls": false,
"neighbors": [
"WU",
"LACL",
"UFBA",
"PADUA",
"COPELABS",
"URJC"
],
"hr_radius": "15.7232",
"name": "University Hassan II Casablanca",
"ip_addresses": [
"196.200.164.99"
],
"backbone": false,
"fch-enabled": false,
"site": "http://196.200.164.99:80/",
"prefix": "ndn:/ndn/ma/univh2c",
"ndn-up": false,
"https": "https://196.200.164.99:443/",
"memsize": "unknown",
"position": [
33.5,
-7.2
],
"shortname": "UNIVH2C",
"hr_angle": "3.03408"
},
"AWS-US-W-2": {
"ws-tls": false,
"neighbors": [],
"name": "AWS US West 2 Region, Oregon",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
44.1,
-123.1
],
"shortname": "AWS-US-W-2"
},
"MINHO": {
"ws-tls": false,
"neighbors": [
"AVEIRO",
"CORUNA",
"COPELABS",
"BASEL",
"URJC",
"PADUA"
],
"hr_radius": "16.2060",
"name": "University oh Minho",
"ip_addresses": [
"193.136.9.206"
],
"backbone": false,
"fch-enabled": true,
"site": "http://netlab.gcom.di.uminho.pt:80/",
"prefix": "ndn:/ndn/pt/uminho",
"ndn-up": true,
"https": "https://netlab.gcom.di.uminho.pt:443/",
"memsize": "4G",
"position": [
41.8,
-8.2
],
"shortname": "MINHO",
"hr_angle": "2.62083"
},
"MUMBAI_AWS": {
"ws-tls": true,
"neighbors": [
"WU",
"AFA",
"UUM"
],
"hr_radius": "12.8934",
"name": "MUMBAI gateway at AWS in Mumbai, India",
"ip_addresses": [
"13.126.132.174"
],
"backbone": false,
"fch-enabled": true,
"site": "http://mumbai.testbed.named-data.net:80/",
"prefix": "ndn:/ndn/aws/mumbai",
"ndn-up": true,
"https": "https://mumbai.testbed.named-data.net:443/",
"memsize": "1G",
"position": [
19.0,
73.2
],
"shortname": "MUMBAI_AWS",
"hr_angle": "2.95081"
},
"MSU": {
"ws-tls": false,
"neighbors": [
"UUM",
"SRRU",
"UCLA",
"TNO",
"ANYANG"
],
"hr_radius": "19.6533",
"name": "Mahasarakham University, Thailand",
"ip_addresses": [
"202.28.34.249"
],
"backbone": false,
"fch-enabled": false,
"site": "http://msu.testbed.named-data.net:80/",
"prefix": "ndn:/ndn/th/ac/msu",
"ndn-up": false,
"https": "https://msu.testbed.named-data.net:443/",
"memsize": "unknown",
"position": [
17.3,
103.7
],
"shortname": "MSU",
"hr_angle": "3.64135"
},
"AVEIRO": {
"ws-tls": false,
"neighbors": [
"WU",
"URJC",
"MINHO",
"COPELABS",
"AFA"
],
"hr_radius": "16.2060",
"name": "University Aveiro",
"ip_addresses": [
"193.136.92.155"
],
"backbone": false,
"fch-enabled": true,
"site": "http://selficn.av.it.pt:80/",
"prefix": "ndn:/ndn/pt/it/av",
"ndn-up": true,
"https": "https://selficn.av.it.pt:443/",
"memsize": "4G",
"position": [
40.6,
-9.0
],
"shortname": "AVEIRO",
"hr_angle": "2.62081"
},
"BASEL": {
"ws-tls": true,
"neighbors": [
"BERN",
"TNO",
"MINHO",
"GOETTINGEN",
"PADUA",
"LIP6",
"URJC"
],
"hr_radius": "13.2717",
"name": "University of Basel",
"ip_addresses": [
"192.43.193.111"
],
"backbone": false,
"fch-enabled": true,
"site": "http://dmi-ndn-testbed1.dmi.unibas.ch:80/",
"prefix": "ndn:/ndn/ch/unibas",
"ndn-up": true,
"https": "https://dmi-ndn-testbed1.dmi.unibas.ch:443/",
"memsize": "2G",
"position": [
47.55858,
7.5836
],
"shortname": "BASEL",
"hr_angle": "2.41932"
},
"WU": {
"ws-tls": true,
"neighbors": [
"MML1",
"BERN",
"QUB",
"WU2",
"MUMBAI_AWS",
"AVEIRO",
"UNIVH2C",
"UFBA",
"CORUNA",
"UCLACS",
"UCLA",
"URJC",
"ARIZONA",
"MEMPHIS",
"UIUC"
],
"hr_radius": "19.4268",
"name": "Washington University in St. Louis",
"ip_addresses": [
"128.252.153.194"
],
"backbone": false,
"fch-enabled": true,
"site": "http://wundngw.arl.wustl.edu:80/",
"_real_position": [
38.6490701,
-90.3033471
],
"prefix": "ndn:/ndn/edu/wustl",
"ndn-up": true,
"https": "https://wundngw.arl.wustl.edu:443/",
"memsize": "2G",
"position": [
38.1490701,
-90.3033471
],
"shortname": "WU",
"hr_angle": "4.31067"
},
"NEU": {
"ws-tls": false,
"neighbors": [
"QUB",
"SAVI",
"MICHIGAN",
"MEMPHIS"
],
"hr_radius": "19.9001",
"name": "Northeastern University",
"ip_addresses": [
"129.10.224.42"
],
"backbone": false,
"fch-enabled": true,
"site": "http://neu.testbed.named-data.net:80/",
"_real_position": [
42.3398783,
-71.0887358
],
"prefix": "ndn:/ndn/edu/neu",
"ndn-up": true,
"https": "https://neu.testbed.named-data.net:443/",
"memsize": "unknown",
"position": [
42.7,
-71.0887358
],
"shortname": "NEU",
"hr_angle": "2.13894"
},
"AWS-EU-W-2": {
"ws-tls": false,
"neighbors": [],
"name": "AWS EU West 2 Region, London",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
51.5,
0.1
],
"shortname": "AWS-EU-W-2"
},
"AWS-EU-W-1": {
"ws-tls": false,
"neighbors": [],
"name": "AWS EU West 1 Region, Ireland",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
53.5,
-7.0
],
"shortname": "AWS-EU-W-1"
},
"AWS-AP-SE-1": {
"ws-tls": false,
"neighbors": [],
"name": "AWS AP SouthEast 1 Region, Singapore",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
1.4,
103.8
],
"shortname": "AWS-AP-SE-1"
},
"AWS-US-W-1": {
"ws-tls": false,
"neighbors": [],
"name": "AWS US West 1 Region, N. California",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
41.1,
-121.4
],
"shortname": "AWS-US-W-1"
},
"UASLP": {
"ws-tls": false,
"neighbors": [
"UFBA",
"ARIZONA",
"MEMPHIS"
],
"hr_radius": "19.0473",
"name": "Universidad Aut\u00f3noma de San Luis Potos\u00ed",
"ip_addresses": [
"148.224.92.99"
],
"backbone": false,
"fch-enabled": true,
"site": "http://uaslp.testbed.named-data.net:80/",
"prefix": "ndn:/ndn/mx/uaslp",
"ndn-up": false,
"https": "https://uaslp.testbed.named-data.net:443/",
"memsize": "8B",
"position": [
22.0,
-100.5
],
"shortname": "UASLP",
"hr_angle": "2.96488"
},
"UIUC": {
"ws-tls": false,
"neighbors": [
"MICHIGAN",
"CSU",
"WU",
"PADUA"
],
"hr_radius": "29.2553",
"name": "University of Illinois, Urbana-Champaign",
"ip_addresses": [
"72.36.112.82"
],
"backbone": false,
"fch-enabled": true,
"site": "http://ndnx.cs.illinois.edu:80/",
"_real_position": [
40.1026039,
-88.23171080000002
],
"prefix": "ndn:/ndn/edu/illinois",
"ndn-up": false,
"https": "https://ndnx.cs.illinois.edu:443/",
"memsize": "8G",
"position": [
40.6026039,
-88.23171080000002
],
"shortname": "UIUC",
"hr_angle": "2.98231"
},
"COPELABS": {
"ws-tls": false,
"neighbors": [
"UNIVH2C",
"AFA",
"UFBA",
"AVEIRO",
"MINHO",
"LIP6",
"URJC",
"PADUA"
],
"hr_radius": "16.2060",
"name": "COPELABS, University Lusofona",
"ip_addresses": [
"193.137.75.171"
],
"backbone": false,
"fch-enabled": false,
"site": "http://copelabs.testbed.named-data.net:80/",
"prefix": "ndn:/ndn/pt/ulusofona/copelabs",
"ndn-up": true,
"https": "https://copelabs.testbed.named-data.net:443/",
"memsize": "1G",
"position": [
38.6,
-9.0
],
"shortname": "COPELABS",
"hr_angle": "2.62082"
},
"AWS-AP-SE-2": {
"ws-tls": false,
"neighbors": [],
"name": "AWS AP SouthEast 2 Region, Sydney",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
-33.9,
151.0
],
"shortname": "AWS-AP-SE-2"
},
"AWS-AP-NE-1": {
"ws-tls": false,
"neighbors": [],
"name": "AWS AP NorthEast 1 Region, Tokyo",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"_real_position": [
35.6,
139.9
],
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
35.0,
140.9
],
"shortname": "AWS-AP-NE-1"
},
"PADUA": {
"ws-tls": false,
"neighbors": [
"UNIVH2C",
"AFA",
"MINHO",
"GOETTINGEN",
"COPELABS",
"URJC",
"UIUC",
"BASEL"
],
"hr_radius": "13.5903",
"name": "University of Padua",
"ip_addresses": [
"147.162.114.18"
],
"backbone": false,
"fch-enabled": true,
"site": "http://ndnnode.math.unipd.it:80/",
"prefix": "ndn:/ndn/it/unipd",
"ndn-up": true,
"https": "https://ndnnode.math.unipd.it:443/",
"memsize": "unknown",
"position": [
45.35858,
11.7836
],
"shortname": "PADUA",
"hr_angle": "1.13934"
},
"LIP6": {
"ws-tls": true,
"neighbors": [
"MML2",
"SAVI",
"QUB",
"LACL",
"BERN",
"TNO",
"COPELABS",
"BASEL",
"URJC",
"MICHIGAN"
],
"hr_radius": "29.2553",
"name": "University Pierre et Marie Curie, Sorbonne Universities LIP6",
"ip_addresses": [
"132.227.62.163",
"2001:660:3302:282c:160::163"
],
"backbone": false,
"fch-enabled": true,
"site": "http://ndnhub.ipv6.lip6.fr:80/",
"_real_position": [
48.86,
2.34
],
"prefix": "ndn:/ndn/fr/lip6",
"ndn-up": true,
"https": "https://ndnhub.ipv6.lip6.fr:443/",
"memsize": "4G",
"position": [
48.86,
2.34
],
"shortname": "LIP6",
"hr_angle": "2.62295"
},
"ANYANG": {
"ws-tls": true,
"neighbors": [
"SRRU",
"MSU",
"OSAKA",
"WASEDA",
"UCLA"
],
"hr_radius": "14.3378",
"name": "Anyang University",
"ip_addresses": [
"210.114.89.49"
],
"backbone": false,
"fch-enabled": false,
"site": "http://anyang.testbed.named-data.net:80/",
"prefix": "ndn:/ndn/kr/anyang",
"ndn-up": true,
"https": "https://anyang.testbed.named-data.net:443/",
"memsize": "48G",
"position": [
37.7,
127.0
],
"shortname": "ANYANG",
"hr_angle": "2.99956"
},
"UFBA": {
"ws-tls": true,
"neighbors": [
"WU",
"UNIVH2C",
"UASLP",
"MEMPHIS",
"COPELABS"
],
"hr_radius": "29.2553",
"name": "Federal University of Bahia, Brazil",
"ip_addresses": [
"200.128.51.61"
],
"backbone": false,
"fch-enabled": true,
"site": "http://ufba.testbed.named-data.net:80/",
"prefix": "ndn:/ndn/br/ufba",
"ndn-up": true,
"https": "https://ufba.testbed.named-data.net:443/",
"memsize": "unknown",
"position": [
-3.7,
-49.6
],
"shortname": "UFBA",
"hr_angle": "3.51898"
},
"URJC": {
"ws-tls": true,
"neighbors": [
"UNIVH2C",
"AVEIRO",
"CORUNA",
"MINHO",
"COPELABS",
"PADUA",
"BASEL",
"WU",
"LIP6"
],
"hr_radius": "13.6888",
"name": "Universidad Rey Juan Carlos (King Juan Carlos University)",
"ip_addresses": [
"193.147.79.41"
],
"backbone": false,
"fch-enabled": true,
"site": "http://insula.gsyc.es:80/",
"_real_position": [
40.3359,
-3.8732
],
"prefix": "ndn:/ndn/es/urjc",
"ndn-up": true,
"https": "https://insula.gsyc.es:443/",
"memsize": "24G",
"position": [
41.3359,
-3.8732
],
"shortname": "URJC",
"hr_angle": "1.17169"
},
"AWS-SA-E-1": {
"ws-tls": false,
"neighbors": [],
"name": "AWS SA East 1 Region, Sao Paolo",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
-23.2,
-45.0
],
"shortname": "AWS-SA-E-1"
},
"LACL": {
"ws-tls": false,
"neighbors": [
"BERN",
"LIP6",
"UNIVH2C"
],
"hr_radius": "12.5144",
"name": "Laboratoire D'Algorithmique, Complexite et Logique (LACL), Universite Paris-Est Creteil",
"ip_addresses": [
"193.48.143.172"
],
"backbone": false,
"fch-enabled": true,
"site": "http://ndn.lacl.fr:80/",
"_real_position": [
48.5,
2.21
],
"prefix": "ndn:/ndn/fr/lacl",
"ndn-up": true,
"https": "https://ndn.lacl.fr:443/",
"memsize": "unknown",
"position": [
47.5,
2.5
],
"shortname": "LACL",
"hr_angle": "2.62296"
},
"MICHIGAN": {
"ws-tls": true,
"neighbors": [
"SAVI",
"CSU",
"UIUC",
"MEMPHIS",
"NEU",
"LIP6"
],
"hr_radius": "17.7738",
"name": "University Michigan",
"ip_addresses": [
"198.111.224.197"
],
"backbone": false,
"fch-enabled": true,
"site": "http://michigan.testbed.named-data.net:80/",
"_real_position": [
42.1026039,
-80.23171080000002
],
"prefix": "ndn:/ndn/edu/umich",
"ndn-up": true,
"https": "https://michigan.testbed.named-data.net:443/",
"memsize": "4G",
"position": [
42.5026039,
-83.4
],
"shortname": "MICHIGAN",
"hr_angle": "2.98018"
},
"AFA": {
"ws-tls": true,
"neighbors": [
"AVEIRO",
"MUMBAI_AWS",
"PADUA",
"COPELABS",
"BERN"
],
"hr_radius": "10.3354",
"name": "AFA Systems",
"ip_addresses": [
"93.42.157.82"
],
"backbone": false,
"fch-enabled": true,
"site": "http://93-42-157-82.ip87.fastwebnet.it:80/",
"prefix": "ndn:/ndn/it/afasystems",
"ndn-up": true,
"https": "https://93-42-157-82.ip87.fastwebnet.it:443/",
"memsize": "4G",
"position": [
42.1,
14.5
],
"shortname": "AFA",
"hr_angle": "1.12640"
},
"MML1": {
"ws-tls": true,
"neighbors": [
"WU",
"MEMPHIS"
],
"hr_radius": "14.1225",
"name": "Athens University of Economics and Business, Greece",
"ip_addresses": [
"195.251.234.11"
],
"backbone": false,
"fch-enabled": true,
"site": "http://mmlab-aueb-1.mmlab.edu.gr:80/",
"_real_position": [
37.9942,
23.7326
],
"prefix": "ndn:/ndn/gr/edu/mmlab1",
"ndn-up": true,
"https": "http://mmlab-aueb-1.mmlab.edu.gr:443/",
"memsize": "2G",
"position": [
38.9942,
24.7326
],
"shortname": "MML1",
"hr_angle": "3.01596"
},
"CORUNA": {
"ws-tls": true,
"neighbors": [
"URJC",
"MINHO",
"WU"
],
"hr_radius": "13.6888",
"name": "Universidade da Coruna",
"ip_addresses": [
"193.144.50.247"
],
"backbone": false,
"fch-enabled": true,
"site": "http://ndn.gtec.udc.es:80/",
"prefix": "ndn:/ndn/es/udc",
"ndn-up": true,
"https": "https://ndn.gtec.udc.es:443/",
"memsize": "2G",
"position": [
43.3,
-8.2
],
"shortname": "CORUNA",
"hr_angle": "1.17180"
},
"WASEDA": {
"ws-tls": false,
"neighbors": [
"OSAKA",
"ANYANG",
"ARIZONA"
],
"hr_radius": "19.3583",
"name": "Waseda University",
"ip_addresses": [
"133.9.67.98"
],
"backbone": false,
"fch-enabled": true,
"site": "http://ndn-tb.nz.comm.waseda.ac.jp:80/",
"_real_position": [
35.7092,
139.7193
],
"prefix": "ndn:/ndn/jp/waseda",
"ndn-up": true,
"https": "https://ndn-tb.nz.comm.waseda.ac.jp:443/",
"memsize": "unknown",
"position": [
35.8092,
139.7193
],
"shortname": "WASEDA",
"hr_angle": "2.99935"
},
"CSU": {
"ws-tls": false,
"neighbors": [
"ARIZONA",
"UCLA",
"MICHIGAN",
"UIUC"
],
"hr_radius": "14.1056",
"name": "Colorado State University",
"ip_addresses": [
"129.82.138.48"
],
"backbone": false,
"fch-enabled": true,
"site": "http://mccoy.netsec.colostate.edu:80/",
"prefix": "ndn:/ndn/edu/colostate",
"ndn-up": true,
"https": "https://mccoy.netsec.colostate.edu:443/",
"memsize": "4G",
"position": [
40.5747175,
-105.0848293
],
"shortname": "CSU",
"hr_angle": "2.99266"
},
"TNO": {
"ws-tls": false,
"neighbors": [
"MML2",
"QUB",
"GOETTINGEN",
"LIP6",
"BASEL",
"MSU"
],
"hr_radius": "12.0148",
"name": "Netherlands Organization for Applied Scientific Research",
"ip_addresses": [
"134.221.116.42"
],
"backbone": false,
"fch-enabled": false,
"site": "http://tno.testbed.named-data.net:80/",
"prefix": "ndn:/ndn/nl/tno",
"ndn-up": true,
"https": "https://tno.testbed.named-data.net:443/",
"memsize": "4G",
"position": [
52.05858,
4.0
],
"shortname": "TNO",
"hr_angle": "2.46031"
},
"BERN": {
"ws-tls": false,
"neighbors": [
"WU",
"LACL",
"LIP6",
"BASEL",
"AFA"
],
"hr_radius": "13.2717",
"name": "University of Bern",
"ip_addresses": [
"130.92.70.172"
],
"backbone": false,
"fch-enabled": false,
"site": "http://os-float172.cnds.unibe.ch:80/",
"prefix": "ndn:/ndn/ch/unibe",
"ndn-up": true,
"https": "https://os-float172.cnds.unibe.ch:443/",
"memsize": "unknown",
"position": [
46.8,
8.5836
],
"shortname": "BERN",
"hr_angle": "2.41933"
},
"AWS-US-E-2": {
"ws-tls": false,
"neighbors": [],
"name": "AWS US East 2 Region, Ohio",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"_real_position": [
40.0,
-83.0
],
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
40.5,
-82.0
],
"shortname": "AWS-US-E-2"
},
"AWS-US-E-1": {
"ws-tls": false,
"neighbors": [],
"name": "AWS US East 1 Region, N. Virginia",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
38.0,
-78.5
],
"shortname": "AWS-US-E-1"
},
"AWS-AP-NE-2": {
"ws-tls": false,
"neighbors": [],
"name": "AWS AP NorthEast 2 Region, Seoul",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"_real_position": [
37.3,
127.0
],
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
37.0,
128.0
],
"shortname": "AWS-AP-NE-2"
},
"MML2": {
"ws-tls": true,
"neighbors": [
"LIP6",
"TNO"
],
"hr_radius": "14.1226",
"name": "Athens University of Economics and Business, Greece",
"ip_addresses": [
"195.251.234.11"
],
"backbone": false,
"fch-enabled": true,
"site": "http://mmlab-aueb-2.mmlab.edu.gr:80/",
"prefix": "ndn:/ndn/gr/edu/mmlab1",
"ndn-up": true,
"https": "http://mmlab-aueb-2.mmlab.edu.gr:443/",
"memsize": "2G",
"position": [
37.9942,
23.7326
],
"shortname": "MML2",
"hr_angle": "3.01597"
},
"AWS-EU-C-1": {
"ws-tls": false,
"neighbors": [],
"name": "AWS EU Central 1 Region, Frankfurt",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"_real_position": [
50.1,
8.7
],
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
51.0,
9.5
],
"shortname": "AWS-EU-C-1"
},
"ARIZONA": {
"ws-tls": true,
"neighbors": [
"UASLP",
"UCLACS",
"UCLA",
"CSU",
"MEMPHIS",
"WU",
"WASEDA",
"OSAKA"
],
"hr_radius": "16.2305",
"name": "The University of Arizona",
"ip_addresses": [
"128.196.203.36"
],
"backbone": false,
"fch-enabled": true,
"site": "http://hobo.cs.arizona.edu:80/",
"prefix": "ndn:/ndn/edu/arizona",
"ndn-up": true,
"https": "https://hobo.cs.arizona.edu:443/",
"memsize": "8G",
"position": [
32.2289614,
-110.9483226
],
"shortname": "ARIZONA",
"hr_angle": "2.97033"
},
"AWS-CA-C-1": {
"ws-tls": false,
"neighbors": [],
"name": "AWS CA Central 1 Region, Toronto",
"ip_addresses": [
""
],
"backbone": false,
"fch-enabled": false,
"site": "http://0.0.0.0:80/",
"_real_position": [
43.6,
-79.4
],
"ndn-up": false,
"https": "https://0.0.0.0:443/",
"memsize": "unknown",
"position": [
44.6,
-78.4
],
"shortname": "AWS-CA-W-1"
},
"SRRU": {
"ws-tls": false,
"neighbors": [
"MEMPHIS",
"UUM",
"MSU",
"ANYANG",
"OSAKA"
],
"hr_radius": "12.8769",
"name": "Surindra Rajabhat University, Thailand",
"ip_addresses": [
"202.29.30.31"
],
"backbone": false,
"fch-enabled": false,
"site": "http://srru.testbed.named-data.net:80/",
"prefix": "ndn:/ndn/th/ac/srru",
"ndn-up": true,
"https": "https://srru.testbed.named-data.net:443/",
"memsize": "8G",
"position": [
15.3,
103.7
],
"shortname": "SRRU",
"hr_angle": "3.63761"
},
"MEMPHIS": {
"ws-tls": true,
"neighbors": [
"MML1",
"SRRU",
"UASLP",
"UFBA",
"ARIZONA",
"WU",
"WU2",
"MICHIGAN",
"NEU"
],
"hr_radius": "15.2647",
"name": "University of Memphis",
"ip_addresses": [
"141.225.11.173"
],
"backbone": false,
"fch-enabled": true,
"site": "http://titan.cs.memphis.edu:80/",
"prefix": "ndn:/ndn/edu/memphis",
"ndn-up": true,
"https": "https://titan.cs.memphis.edu:443/",
"memsize": "4G",
"position": [
35.1186099,
-89.93700109999999
],
"shortname": "MEMPHIS",
"hr_angle": "2.98360"
},
"OSAKA": {
"ws-tls": false,
"neighbors": [
"UUM",
"SRRU",
"ANYANG",
"GOETTINGEN",
"WASEDA",
"ARIZONA"
],
"hr_radius": "19.3573",
"name": "Osaka University",
"ip_addresses": [
"133.1.17.51"
],
"backbone": false,
"fch-enabled": true,
"site": "http://ndn.ist.osaka-u.ac.jp:80/",
"_real_position": [
34.72,
135.5193
],
"prefix": "ndn:/ndn/jp/ac/osaka-u",
"ndn-up": true,
"https": "https://ndn.ist.osaka-u.ac.jp:443/",
"memsize": "16G",
"position": [
34.3,
135.7193
],
"shortname": "OSAKA",
"hr_angle": "3.53687"
},
"UCLA": {
"ws-tls": true,
"neighbors": [
"ARIZONA",
"WU",
"UCLACS",
"CSU",
"MSU",
"ANYANG"
],
"hr_radius": "16.5866",
"name": "University of California, Los Angeles",
"ip_addresses": [
"131.179.196.48"
],
"backbone": false,
"fch-enabled": true,
"site": "http://suns.cs.ucla.edu:80/",
"_real_position": [
34.0728225,
-118.4387398
],
"prefix": "ndn:/ndn/edu/ucla",
"ndn-up": true,
"https": "https://suns.cs.ucla.edu:443/",
"memsize": "2G",
"position": [
36.5,
-130
],
"shortname": "UCLA",
"hr_angle": "2.99911"
},
"UUM": {
"ws-tls": true,
"neighbors": [
"MSU",
"SRRU",
"MUMBAI_AWS",
"OSAKA"
],
"hr_radius": "14.8351",
"name": "Universiti Utara Malaysia",
"ip_addresses": [
"203.80.17.88"
],
"backbone": false,
"fch-enabled": true,
"site": "http://uum.testbed.named-data.net:80/",
"prefix": "ndn:/ndn/my/edu/uum",
"ndn-up": true,
"https": "https://uum.testbed.named-data.net:443/",
"memsize": "16G",
"position": [
6.6,
100.3
],
"shortname": "UUM",
"hr_angle": "3.44434"
},
"QUB": {
"ws-tls": true,
"neighbors": [
"WU",
"NEU",
"TNO",
"LIP6"
],
"hr_radius": "17.8582321576696",
"name": "Queen's University Belfast, UK",
"ip_addresses": [
"80.76.206.92"
],
"backbone": false,
"fch-enabled": true,
"site": "http://ndn.qub.ac.uk:80/",
"prefix": "ndn:/ndn/uk/ac/qub",
"ndn-up": true,
"https": "https://ndn.qub.ac.uk:443/",
"memsize": "8G",
"position": [
54.5973,
-5.9301
],
"shortname": "QUB",
"hr_angle": "2.42743127401714"
},
"SAVI": {
"ws-tls": true,
"neighbors": [
"NEU",
"MICHIGAN",
"LIP6"
],
"hr_radius": "19.5605",
"name": "SAVI, University of Toronto",
"ip_addresses": [
"142.150.208.242"
],
"backbone": false,
"fch-enabled": true,
"site": "http://savi.testbed.named-data.net:80/",
"prefix": "ndn:/ndn/ca/utoronto",
"ndn-up": true,
"https": "https://savi.testbed.named-data.net:443/",
"memsize": "8G",
"position": [
43.6,
-79.4
],
"shortname": "SAVI",
"hr_angle": "4.04925"
},
"UCLACS": {
"ws-tls": true,
"neighbors": [
"UCLA",
"ARIZONA",
"WU"
],
"hr_radius": "16.5866",
"name": "University of California, Los Angeles",
"ip_addresses": [
"131.179.196.46"
],
"backbone": false,
"fch-enabled": false,
"site": "http://spurs.cs.ucla.edu:80/",
"_real_position": [
34.0728225,
-118.4387398
],
"prefix": "ndn:/ndn/edu/ucla/cs",
"ndn-up": true,
"https": "https://spurs.cs.ucla.edu:443/",
"memsize": "2G",
"position": [
35.5,
-120
],
"shortname": "UCLACS",
"hr_angle": "2.99910"
}
}
Raw
testbed.py
#!/usr/bin/python
import ipaddress
import json
import re
import sys
import urllib.parse
from typing import Any, Dict, List, Optional
import geoip2.database
import ndjson
from geopy.distance import geodesic
Record = Dict[str, Any]
geoip2asn = geoip2.database.Reader("GeoLite2-ASN.mmdb")
class TestbedRouter(dict):
def __init__(self, j: Record):
self.router: str = urllib.parse.urlsplit(j["site"]).hostname
self["host"] = self.router
self.wss: bool = j["ws-tls"]
(lon, lat) = j["_real_position"] if "_real_position" in j \
else j["position"]
self.lonlat = [lon, lat]
self["location"] = [lat, lon]
for ipAddr in j["ip_addresses"]:
try:
ip = str(ipaddress.IPv4Address(ipAddr))
self["ip"] = ip
self["asn"] = geoip2asn.asn(ip).autonomous_system_number
break
except Exception:
pass
def geodistance(self, latlon: List[float]) -> float:
return geodesic(reversed(latlon), self.lonlat).meters
testbed: Dict[str, TestbedRouter] = dict()
with open("testbed-nodes.json") as testbedNodesData:
for j in json.load(testbedNodesData).values():
try:
router = TestbedRouter(j)
testbed[router.router] = router
except Exception:
continue
reWss = re.compile(r"wss:[^)]+")
def find_router(remote: Optional[str]) -> Optional[TestbedRouter]:
if remote is None:
return None
m = reWss.search(remote)
if m is None:
return None
return testbed.get(urllib.parse.urlsplit(m[0]).hostname)
if __name__ == "__main__":
geoip2city = geoip2.database.Reader("GeoLite2-City.mmdb")
writer = ndjson.writer(sys.stdout)
for router in testbed.values():
router["wss"] = router.wss
try:
city = geoip2city.city(router["ip"])
router["country"] = city.country.iso_code
router["continent"] = city.continent.code
except Exception:
router["country"] = None
router["continent"] = None
continue
writer.writerow(router)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment