Test time to DNS query and connect to various domains

domains.txt

google.com
facebook.com
youtube.com
twitter.com
instagram.com
linkedin.com
microsoft.com
apple.com
wikipedia.org
googletagmanager.com
youtu.be
en.wikipedia.org
plus.google.com
pinterest.com
play.google.com
vimeo.com
maps.google.com
adobe.com
goo.gl
wordpress.com
bit.ly
itunes.apple.com
github.com
wordpress.org
blogspot.com
docs.google.com
player.vimeo.com
amazon.com
mozilla.org
yahoo.com
tumblr.com
drive.google.com
support.google.com
apps.apple.com
europa.eu
flickr.com
gravatar.com
reddit.com
whatsapp.com
nytimes.com
apache.org
sites.google.com
amazonaws.com
nih.gov
qq.com
support.microsoft.com
w3.org
soundcloud.com
t.co
forbes.com
medium.com
github.io
sourceforge.net
api.whatsapp.com
support.apple.com
ec.europa.eu
theguardian.com
cnn.com
www.ncbi.nlm.nih.gov
archive.org
zoom.us
dropbox.com
baidu.com
macromedia.com
creativecommons.org
virginmedia.com
policies.google.com
bbc.co.uk
who.int
issuu.com
weebly.com
forms.gle
paypal.com
spotify.com
l.facebook.com
m.facebook.com
tools.google.com
tinyurl.com
cloudflare.com
cdc.gov
httpd.apache.org
bbc.com
get.adobe.com
php.net
youtube-nocookie.com
reuters.com
washingtonpost.com
vk.com
wsj.com
developers.google.com
opera.com
bloomberg.com
web.archive.org
msn.com
imdb.com
open.spotify.com
harvard.edu
gnu.org
sciencedirect.com
accounts.google.com
weibo.com
oracle.com
wikimedia.org
bing.com
mit.edu
wp.com
slideshare.net
businessinsider.com
ibm.com
stanford.edu
w.soundcloud.com
wiley.com
list-manage.com
windows.microsoft.com
wix.com
nginx.org
live.com
nature.com
un.org
telegraph.co.uk
wpa.qq.com
go.com
cpanel.net
wa.me
office.com
doi.org
nasa.gov
t.me
usatoday.com
time.com
dailymail.co.uk
cpanel.com
cnet.com
nginx.com
eventbrite.com
surveymonkey.com
huffingtonpost.com
bitly.com
ted.com
validator.w3.org
myspace.com
cnbc.com
researchgate.net
springer.com
www.gov.uk
ebay.com
bp.blogspot.com
npr.org
wired.com
chrome.google.com
mysql.com
onlinelibrary.wiley.com
aol.com
mp.weixin.qq.com
i0.wp.com
blogger.com
hp.com
mail.google.com
cloudfront.net
i1.wp.com
i2.wp.com
independent.co.uk
amzn.to
techcrunch.com
imgur.com
debian.org
sitemaps.org
hugedomains.com
berkeley.edu
etsy.com
commons.wikimedia.org
arnebrachhold.de
themeforest.net
fpdownload.macromedia.com
sina.com.cn
googleusercontent.com
gc.ca
link.springer.com
addtoany.com
latimes.com
dailymotion.com
behance.net
yelp.com
loc.gov
amazon.co.uk
ft.com
ietf.org
outlook.com
free.fr
beian.gov.cn
youku.com
nationalgeographic.com
cornell.edu
s3.amazonaws.com
unsplash.com
theverge.com
washington.edu
mailchimp.com
theatlantic.com
baike.baidu.com
statcounter.com
digg.com
scribd.com
xinhuanet.com
google.de
prnewswire.com
akismet.com
cbsnews.com
kickstarter.com
skype.com
about.com
google.co.uk
ca.gov
jimdo.com
fda.gov
taobao.com
goodreads.com
livejournal.com
1.bp.blogspot.com
launchpad.net
samsung.com
fb.com
mailchi.mp
aboutads.info
squarespace.com
tripadvisor.com
stackoverflow.com
abcnews.go.com
twitch.tv
shopify.com
princeton.edu
sciencemag.org
statista.com
163.com
godaddy.com
typepad.com
nbcnews.com
huffpost.com
webmd.com
fr.wikipedia.org
g.page
change.org
stumbleupon.com
foxnews.com
tandfonline.com
allaboutcookies.org
v.qq.com
economist.com
networkadvertising.org
oup.com
eur-lex.europa.eu
unesco.org
tools.ietf.org
whitehouse.gov
marriott.com
addthis.com
bandcamp.com
usnews.com
deviantart.com
usda.gov
sagepub.com
hbr.org
2.bp.blogspot.com
4.bp.blogspot.com
eepurl.com
3.bp.blogspot.com
mashable.com
myshopify.com
msdn.microsoft.com
translate.google.com
pbs.org
feedburner.com
finance.yahoo.com
buzzfeed.com
webs.com
columbia.edu
booking.com
giphy.com
pixabay.com
sohu.com
de.wikipedia.org
marketwatch.com
vice.com
quora.com
m.youtube.com
epa.gov
umich.edu
dx.doi.org
business.site
news.bbc.co.uk
yale.edu
businesswire.com
sciencedaily.com
hubspot.com
vox.com
netflix.com
redhat.com
psu.edu
iso.org
tripod.com
cisco.com
zdnet.com
books.google.com
disqus.com
noaa.gov
fb.me
gofundme.com
wikihow.com
state.gov
cbc.ca
geocities.com
podcasts.apple.com
intel.com
upenn.edu
feeds.feedburner.com
vkontakte.ru
worldbank.org
guardian.co.uk
cambridge.org
sun.com
example.com
abc.net.au
tiktok.com
arcgis.com
indiatimes.com
plesk.com
docs.microsoft.com
engadget.com
britannica.com
yandex.ru
upload.wikimedia.org
or.kr
newyorker.com
fastcompany.com
line.me
investopedia.com
dribbble.com
academia.edu
mayoclinic.org
ox.ac.uk
android.com
web.facebook.com
fortune.com
psychologytoday.com
wisc.edu
people.com.cn
alibaba.com
inc.com
jigsaw.w3.org
plos.org
money.cnn.com
umn.edu
gouv.fr
youronlinechoices.com
zendesk.com
gizmodo.com
theconversation.com
nypost.com
bmj.com
scientificamerican.com
meetup.com
photobucket.com
wpengine.com
constantcontact.com
entrepreneur.com
web.whatsapp.com
weforum.org
oreilly.com
apnews.com
healthline.com
deloitte.com
uci.edu
academic.oup.com
jhu.edu
telegram.me
ieee.org
dev.mysql.com
cmu.edu
aboutcookies.org
elegantthemes.com
picasaweb.google.com
gotowebinar.com
amazon.de
ucla.edu
wikia.com
oecd.org
cam.ac.uk
apa.org
typeform.com
appspot.com
irs.gov
arxiv.org
dell.com
biomedcentral.com
es.wikipedia.org
indiegogo.com
elsevier.com
chicagotribune.com
newsweek.com
news.yahoo.com
uk.com
doubleclick.net
ftc.gov
utexas.edu
i.imgur.com
google.ca
nps.gov
slate.com
fandom.com
uchicago.edu
afternic.com
dw.com
steampowered.com
ubuntu.com
qz.com
arstechnica.com
azurewebsites.net
pubmed.ncbi.nlm.nih.gov
sharepoint.com
mckinsey.com
bizjournals.com
pewresearch.org
openstreetmap.org
bitnami.com
kiev.ua
si.edu
uiuc.edu
player.youku.com
us.com
sfgate.com
forms.office.com
box.com
ed.gov
mobile.twitter.com
bbb.org
evernote.com
house.gov
patreon.com
python.org
getpocket.com
pnas.org
scholar.google.com
cbslocal.com
xing.com
store.steampowered.com
thelancet.com
mirror.co.uk
linktr.ee
discord.gg
ikea.com
merriam-webster.com
walmart.com
altervista.org
fao.org
howstuffworks.com
qc.ca
business.facebook.com
thesun.co.uk
salesforce.com
yahoo.co.jp
livescience.com
wiktionary.org
netscape.com
spiegel.de
googleblog.com
googleapis.com
about.me
enable-javascript.com
withgoogle.com
blogspot.co.uk
aliexpress.com
purdue.edu
usc.edu
teamviewer.com
over-blog.com
hilton.com
ow.ly
fc2.com
smh.com.au
google.fr
chinadaily.com.cn
jamanetwork.com
nyu.edu
census.gov
indeed.com
politico.com
senate.gov
gob.mx
go.id
trustpilot.com
pexels.com
google.es
mediafire.com
parallels.com
ning.com
jotform.com
venturebeat.com
proofpoint.com
nydailynews.com
espn.com
help.opera.com
amazon-adsystem.com
gitlab.com
usgs.gov
earthlink.net
on.ca
joomla.org
google.co.jp
govt.nz
msu.edu
variety.com
barnesandnoble.com
rt.com
widgets.wp.com
coursera.org
acs.org
jstor.org
duke.edu
gartner.com
docker.com
calendly.com
trello.com
nejm.org
elpais.com
bls.gov
hhs.gov
medicalnewstoday.com
postgresql.org
pcmag.com
jd.com
m.me
thetimes.co.uk
privacyshield.gov
instructables.com
fbcdn.net
google.it
aliyun.com
northwestern.edu
themeisle.com
canva.com
globenewswire.com
imageshack.us
automattic.com
umd.edu
ufl.edu
icio.us
aljazeera.com
ucsd.edu
so.com
zend.com
rollingstone.com
slack.com
angelfire.com
del.icio.us
itu.int
microsoftonline.com
lifehacker.com
in.ua
express.co.uk
mozilla.com
slashdot.org
developer.apple.com
nvidia.com
nist.gov
mitre.org
w3schools.com
unc.edu
newscientist.com
eff.org
sky.com
theglobeandmail.com
vmware.com
stackexchange.com
airbnb.com
foursquare.com
cdn.shopify.com
timeanddate.com
sec.gov
moodle.org
dot.gov
www.gov.cn
news.com.au
talktalk.co.uk
shutterstock.com
openssl.org
apachefriends.org
pan.baidu.com
sakura.ne.jp
jquery.com
mixcloud.com
scmp.com
mystrikingly.com
namecheap.com
login.microsoftonline.com
matterport.com
unicef.org
drupal.org
thehill.com
news.google.com
siemens.com
kernel.org
readthedocs.io
ok.ru
freebsd.org
wp.me
is.gd
weather.com
www.nhs.uk
ru.wikipedia.org
hollywoodreporter.com
prezi.com
history.com
getbootstrap.com
go.th
technologyreview.com
tmall.com
my.matterport.com
huawei.com
mail.ru
mdpi.com
digitaltrends.com
ssrn.com
illinois.edu
libsyn.com
chron.com
symantec.com
woocommerce.com
spb.ru
rutgers.edu
arizona.edu
stripe.com
freepik.com
wufoo.com
thedailybeast.com
gallup.com
dmoz.org
moz.com
webex.com
force.com
techradar.com
nike.com
boston.com
anchor.fm
livestream.com
phys.org
metro.co.uk
ucdavis.edu
opensource.org
iana.org
utoronto.ca
today.com
nymag.com
frontiersin.org
ustream.tv
icann.org
lemonde.fr
ny.gov
fcc.gov
paypal.me
justice.gov
treasury.gov
twimg.com
thenextweb.com
naver.com
dan.com
udemy.com
amazon.co.jp
google.co.in
virginia.edu
techtarget.com
uol.com.br
archives.gov
uspto.gov
asu.edu
blogs.wsj.com
prweb.com
podbean.com
admin.ch
go.kr
vic.gov.au
alexa.com
themegrill.com
pwc.com
khanacademy.org
bc.ca
smithsonianmag.com
rs6.net
canada.ca
1688.com
verisign.com
fbi.gov
pinterest.ca
usa.gov
businessweek.com
frontier.com
platform.twitter.com
gmail.com
yolasite.com
osu.edu
hatena.ne.jp
lonelyplanet.com
google.cn
playstation.com
dropcatch.com
ubc.ca
flic.kr
herokuapp.com
dictionary.com
colorado.edu
cctv.com
acm.org
kde.org
bilibili.com
adweek.com
ign.com
perl.org
urbandictionary.com
uber.com
google.com.au
service.gov.uk
list-manage1.com
att.com
java.com
bigcartel.com
brookings.edu
imf.org
r20.rs6.net
en.m.wikipedia.org
de-de.facebook.com
colorlib.com
snapchat.com
bu.edu
tamu.edu
360.cn
faqs.org
observer.com
google.ru
bostonglobe.com
discovery.com
nokia.com
wustl.edu
atlassian.com
discord.com
fedoraproject.org
windows.net
ucl.ac.uk
ico.org.uk
autodesk.com
salon.com
greenpeace.org
phpbb.com
marketingplatform.google.com
mapquest.com
congress.gov
dhs.gov
ethz.ch
bund.de
office365.com
cancer.org
tufts.edu
oxfordjournals.org
rfc-editor.org
photos.google.com
bitbucket.org
theregister.co.uk
gumroad.com
nikkei.com
com.com
images.google.com
indiana.edu
army.mil
energy.gov
people.com
pcworld.com
schneier.com
target.com
thoughtco.com
gutenberg.org
consumerreports.org
last.fm
heart.org
mlb.com
euronews.com
g.co
lww.com
fiverr.com
cpan.org
fool.com
elsevierhealth.com
qualtrics.com
thefreedictionary.com
filesusr.com
nba.com
undp.org
sap.com
globo.com
ncsu.edu
eurekalert.org
vanityfair.com
va.gov
dol.gov
search.yahoo.com
hatenablog.com
thestar.com
google.nl
asus.com
e.issuu.com
novell.com
500px.com
pitt.edu
computerworld.com
consent.youtube.com
technet.microsoft.com
google.com.br
chinanews.com
cutt.ly
standard.co.uk
feedly.com
voanews.com
udel.edu
delicious.com
storage.googleapis.com
blog.google
hootsuite.com
dropboxusercontent.com
digital.com
prestashop.com
intuit.com
utah.edu
crunchbase.com
wunderground.com
wa.gov
smugmug.com
mercurynews.com
youronlinechoices.eu
houzz.com
haxx.se
akamaihd.net
t.qq.com
cygwin.com
netdna-ssl.com
medscape.com
nielsen.com
tiny.cc
accuweather.com
steamcommunity.com
lenovo.com
pastebin.com
jiathis.com
cia.gov
irishtimes.com
ey.com
miit.gov.cn
sba.gov
outlook.office365.com
narod.ru
amazon.ca
pbs.twimg.com
ebay.co.uk
nasdaq.com
usembassy.gov
ap.org
navy.mil
blogspot.ca
dedecms.com
coe.int
duckduckgo.com
cell.com
tinypic.com
gamespot.com
neilpatel.com
theregister.com
patch.com
seattletimes.com
garmin.com
hyatt.com
ucsb.edu
lh3.googleusercontent.com
medlineplus.gov
sony.com
en.wiktionary.org
ibtimes.com
speedtest.net
zoho.com
snopes.com
washingtontimes.com
lefigaro.fr
iheart.com
ndtv.com
fifa.com
jimdofree.com
ibb.co
nsw.gov.au
amazon.fr
perfectdomain.com
ed.ac.uk
globalnews.ca
ameblo.jp
craigslist.org
icloud.com
wikidot.com
cancer.gov
pbase.com
it.wikipedia.org
diigo.com
imagemagick.org
athemes.com
openldap.org
1drv.ms
ups.com
ilo.org
nobelprize.org
usps.com
staticflickr.com
fb.watch
georgetown.edu
billboard.com
china.com.cn
accenture.com
ihg.com
broadcom.com
jetpack.wordpress.com
lego.com
ea.com
semanticscholar.org
disney.com
homestead.com
gob.es
blogspot.de
spreaker.com
securityfocus.com
eventbrite.co.uk
accor.com
secureserver.net
curl.se
xbox.com
scoop.it
redcross.org
msdn.com
worldometers.info
music.apple.com
hawaii.edu
caltech.edu
douban.com
ccc.de
zhihu.com
azure.microsoft.com
iop.org
es.wordpress.org
cuny.edu
americanexpress.com
hexun.com
hrw.org
startribune.com
seekingalpha.com
rakuten.co.jp
brown.edu
imageshack.com
foxbusiness.com
liveinternet.ru
vatican.va
nyc.gov
ehow.com
gimp.org
fedex.com
amd.com
panasonic.com
france24.com
www.law.cornell.edu
shutterfly.com
brightcove.net
searchengineland.com
players.brightcove.net
git-scm.com
thebalance.com
fema.gov
glassdoor.com
vt.edu
gatech.edu
japantimes.co.jp
gob.ar
msnbc.com
mac.com
pinimg.com
webnode.com
olympic.org
osha.gov
foodnetwork.com
deadline.com
buzzfeednews.com
gv.at
mcafee.com
redbull.com
aarp.org
verizon.com
us.org
stitcher.com
kaspersky.com
wired.co.uk

web_connect_test.py

#!/usr/bin/env python3
import socket
import sys
import time

import dns.resolver
import numpy

NAMESERVERS = [None, "1.1.1.1", "8.8.8.8"]


def timeit(function):
    def wrap(*args, **kwargs):
        start = time.time()
        value = function(*args, **kwargs)
        return value, (time.time() - start) * 1000

    return wrap


@timeit
def dns_lookup(resolver, domain):
    return resolver.resolve(domain, "a")[0].address


def domains(offset=0, limit=None):
    with open("domains.txt") as fp:
        for i, row in enumerate(fp):
            if i < offset:
                continue
            yield row.strip()
            if limit and i >= offset + limit - 1:
                break


def main():
    if len(sys.argv) == 2:
        limit = int(sys.argv[1])
    else:
        limit = 100

    ips = set()
    resolver = dns.resolver.Resolver()

    for nameserver in NAMESERVERS:
        dns_times = []
        if nameserver:
            resolver.nameservers = [nameserver]
        else:
            nameserver = resolver.nameservers[0]
        for domain in domains(limit=limit):
            try:
                ip, dns_duration = dns_lookup(resolver, domain)
                ips.add(ip)
            except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
                continue
            dns_times.append(dns_duration)
        print(f"{nameserver:15s} {len(dns_times)} lookups")
        array = numpy.array(dns_times)
        print(f"P01 {numpy.percentile(array, 1):8.02f}\nP50 {numpy.percentile(array, 50):8.02f}\nP95 {numpy.percentile(array, 95):8.02f}\nP99 {numpy.percentile(array, 99):8.02f}\n")


    tcp_times = []
    for ip in sorted(ips):
        try:
            tcp_times.append(tcp_connect((ip, 80))[1])
        except (OSError, socket.timeout):
            pass
    print(f"tcp connect     {len(tcp_times)} connections")
    array = numpy.array(tcp_times)
    print(f"P01 {numpy.percentile(array, 1):8.02f}\nP50 {numpy.percentile(array, 50):8.02f}\nP95 {numpy.percentile(array, 95):8.02f}\nP99 {numpy.percentile(array, 99):8.02f}")


@timeit
def tcp_connect(address):
    socket.create_connection(address, timeout=0.5)


if __name__ == "__main__":
    sys.exit(main())