Detects websites with brave-leder-verifications in their DNS records. Note that such a ledger is no guarantee that the website is infact a verified Brave publisher.

brave_publisher_dns_crawler.py

# Download and unzip Alexa Top 1 Million websites
# http://s3.amazonaws.com/alexa-static/top-1m.csv.zip
# Run this script.

from os.path import isfile
from urllib.request import urlopen
import concurrent.futures
import socket
import dns.resolver
import time

socket.setdefaulttimeout(20)

def download_robot(domain):
    try:
        if any(['brave-ledger-verification=' in str(_) for _ in dns.resolver.query(domain, 'TXT').rrset]):
            print("DETECTED: {0}".format(domain), flush=True)
        else:
            pass
    except:
        pas

executor = concurrent.futures.ThreadPoolExecutor(max_workers=18)

with open('top-1m.csv') as file:
    for num, line in enumerate(file, 1):
        domain = line.split(',')[1].rstrip()
        ok = executor.submit(download_robot, domain)