download pcaps

pcap_download.go

package main
import (
	"fmt"
	"strings"
	"github.com/cavaliercoder/grab"
	"github.com/gocolly/colly/v2"
)
func main() {
	domain := "www.malware-traffic-analysis.net"
	c := colly.NewCollector(
		colly.AllowedDomains(domain),
		colly.MaxDepth(4),
		colly.Async(),
	)
	client := grab.NewClient()
	c.Limit(&colly.LimitRule{DomainGlob: "*", Parallelism: 4})
	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
		link := e.Attr("href")
		absoluteLink := e.Request.AbsoluteURL(link)

		if strings.Contains(absoluteLink, "pcap") && strings.Contains(absoluteLink, "zip") {
			//fmt.Println(absoluteLink)
			req, err := grab.NewRequest(".", absoluteLink)
			if err != nil {
				fmt.Println(err)
			}
			resp := client.Do(req)
			if err := resp.Err(); err != nil {
				fmt.Println(err)
			}

			fmt.Println("Download saved to", resp.Filename)
		}
		e.Request.Visit(link)
	})
	c.Visit("https://" + domain)
	c.Wait()
}