Check compression in use

compression.py

#!/usr/bin/env python
from multiprocessing.dummy import Pool
from collections import defaultdict
import requests
import logging


def uses_compression(domain):
    logging.info(domain)
    try:
        r = requests.get("https://{}".format(domain), verify=False, timeout=2)
        return 'Content-Encoding' in r.headers
    except:
        return "Unknown"


def main():
    compression = defaultdict(int)
    domains = [domain.rstrip() for domain in open("domains").readlines()]
    p = Pool(100)
    results = p.map(uses_compression, domains)
    for result in results:
        compression[result] += 1
    print(compression)


if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)
    logging.getLogger("requests").setLevel(logging.WARNING)
    main()