Check if any domains in Alexa top 1 million list is under any CDN and if so, which one is it

checkDomainForCDN.py

package main

import (
	"sync"
	//"strconv"
	"io"
	"net/http"
        "archive/zip"
        "path/filepath"
	"bufio"
	"strings"
	"os"
	"fmt"
	//"net"
	"github.com/miekg/dns"
	"github.com/fatih/color"
	//"log"
)	
var count=1
func unzip(archive, target string) error {
	reader, err := zip.OpenReader(archive)
	if err != nil {
		return err
	}

	if err := os.MkdirAll(target, 0755); err != nil {
		return err
	}
	for _, file := range reader.File {
		path := filepath.Join(target, file.Name)
		if file.FileInfo().IsDir() {
			os.MkdirAll(path, file.Mode())
			continue
		}
		fileReader, err := file.Open()
		if err != nil {
			return err
		}
		defer fileReader.Close()
		targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
		if err != nil {
			return err
		}
		defer targetFile.Close()
		if _, err := io.Copy(targetFile, fileReader); err != nil {
			return err
		}
	}
	return nil
}
func downloadFromUrl(url string) {
	tokens := strings.Split(url, "/")
	fileName := tokens[len(tokens)-1]
	fmt.Println("Downloading", url, "to", fileName)

	// TODO: check file existence first with io.IsExist
	output, err := os.Create(fileName)
	if err != nil {
		fmt.Println("Error while creating", fileName, "-", err)
		return
	}
	defer output.Close()
	response, err := http.Get(url)
	if err != nil {
		fmt.Println("Error while downloading", url, "-", err)
		return
	}
	defer response.Body.Close()
	n, err := io.Copy(output, response.Body)
	if err != nil {
		fmt.Println("Error while downloading", url, "-", err)
		return
	}
	fmt.Println(n, "bytes downloaded.")
}
func readLines(path string) ([]string, error) {
  file, err := os.Open(path)
  if err != nil {
    return nil, err
  }
  defer file.Close()

  var lines []string
  scanner := bufio.NewScanner(file)
  for scanner.Scan() {
    lines = append(lines, scanner.Text())
  }
  return lines, scanner.Err()
}

func checkDomain(urlChan chan string) {
	for domainName := range urlChan {
		c := new(dns.Client)
		m := new(dns.Msg)
		m.SetQuestion(domainName+".", dns.TypeSOA)
		m.RecursionDesired = true
		m.Authoritative = true
		r, _, err := c.Exchange(m, "8.8.8.8:53")
		if err==nil {
			for _, ans := range r.Answer {
				if mx, ok := ans.(*dns.SOA); ok {
					var qryAns=mx.String()
					s := strings.Split(qryAns,"SOA")
					s1 := strings.TrimSpace(s[1])
					if strings.Contains(s1,"ns.cloudflare.com") {
						//fmt.Printf("%s | %s | %s\n", strconv.Itoa(count), domainName,color.BlueString("Cloudflare CDN"))
						fmt.Printf("%s | %s\n", domainName,color.BlueString("Cloudflare CDN"))
						//return domainName
					} else if strings.Contains(s1,"alibabadns.com") {
						//fmt.Printf("%s | %s | %s\n", strconv.Itoa(count), domainName,color.BlueString("Alibaba CDN"))
						fmt.Printf("%s | %s\n", domainName,color.BlueString("Alibaba CDN"))
						//return domainName
					} else if strings.Contains(s1,"awsdns-hostmaster.amazon.com") {
						//fmt.Printf("%s | %s | %s\n", strconv.Itoa(count), domainName,color.BlueString("Amazon CloudFront"))
						fmt.Printf("%s | %s\n", domainName,color.BlueString("Amazon CloudFront"))
						//return domainName
					} else if strings.Contains(s1,"a248.e.akamai.net") {
						//fmt.Printf("%s | %s | %s\n", strconv.Itoa(count), domainName,color.BlueString("Akamai CDN"))
						fmt.Printf("%s | %s\n", domainName,color.BlueString("Akamai CDN"))
						//return domainName
					} else if strings.Contains(s1,"secure.footprint.net") {
						//fmt.Printf("%s | %s | %s\n", strconv.Itoa(count), domainName,color.BlueString("Level 3 CDN"))
						fmt.Printf("%s | %s\n", domainName,color.BlueString("Level 3 CDN"))
						//return domainName
					} else if strings.Contains(s1,"unbouncepages.com") {
						//fmt.Printf("%s | %s | %s\n", strconv.Itoa(count), domainName,color.BlueString("Unbounce CDN"))
						fmt.Printf("%s | %s\n", domainName,color.BlueString("Unbounce CDN"))
						//return domainName
					} else if (strings.Contains(s1,"aspnetcdn.com") || strings.Contains(s1,"msecnd.net")) {
						//fmt.Printf("%s | %s | %s\n", strconv.Itoa(count), domainName,color.BlueString("Azure CDN"))
						fmt.Printf("%s | %s\n", domainName,color.BlueString("Azure CDN"))
						//return domainName
					//} else if strings.Contains(s1,"dns-admin.google.com") {
					} else if strings.Contains(s1,"appspot.com") {
						//fmt.Printf("%s | %s | %s\n", strconv.Itoa(count), domainName,color.BlueString("Google Cloud CDN"))
						fmt.Printf("%s | %s\n", domainName,color.BlueString("Google Cloud CDN"))
						//return domainName
					} //else {
					//	fmt.Println(strconv.Itoa(count)+" | "+domainName+" | "+s1)
					//}	
				}
			}
		}	
		count+=1
	}
	//_=err
	//return ""
}
func main() {
	var domainList []string
	var downloadUrl="http://s3.amazonaws.com/alexa-static/top-1m.csv.zip"
	tokens := strings.Split(downloadUrl, "/")
	fileName := tokens[len(tokens)-1]
	if _, err := os.Stat("top-1m.csv.zip"); os.IsNotExist(err) {
		fmt.Println("[*] Downloading", downloadUrl, "to", fileName)
		downloadFromUrl(downloadUrl)
	}
	pwd, err := os.Getwd()
	if err == nil {
		if _, err := os.Stat("top-1m.csv"); os.IsNotExist(err) {
			fmt.Println("[*] Uncompressing ",fileName)
			fmt.Println(unzip(pwd+"/"+fileName, pwd))
		}
	}
	var pFilename = "top-1m.csv"
	lines, err2 := readLines(pFilename)
	for _, v := range lines {
		v=strings.TrimSpace(v)
		if len(v)>0 {
			var s1=strings.Split(v,",")
			domainList = append(domainList, s1[1])
		}
	}		
	_ = err2

	var workersCount=10
	var wg sync.WaitGroup
	urlChan := make(chan string)
	wg.Add(workersCount)
	
	for i := 0; i < workersCount; i++ {
		go func() {
			checkDomain(urlChan)
			wg.Done()
		}()
	}
	for _, each := range domainList {
		urlChan <- each
	}
	close(urlChan)
	os.Exit(3)

	//m.SetQuestion("miek.nl.", dns.TypeMX)
	//m.SetQuestion("medium.com.", dns.TypeCNAME)
	//m.SetQuestion("medium.com.", dns.TypeA)

	/*
	r, _, err := c.Exchange(m, "173.245.58.61:53")
	if err!=nil {
		fmt.Println(err)
	}
	fmt.Println(len(r.Answer))
	for _, ans1 := range r.Answer {
		if mx, ok := ans1.(*dns.A); ok {
			var qryAns=mx.String()
			fmt.Println(qryAns)
		}
	}
	*/
	//_=err
}