Skip to content

Instantly share code, notes, and snippets.

@scottcagno
Created August 31, 2016 17:41
Show Gist options
  • Select an option

  • Save scottcagno/fc8bc4bb068e2e984ee64d36c6fd1e7c to your computer and use it in GitHub Desktop.

Select an option

Save scottcagno/fc8bc4bb068e2e984ee64d36c6fd1e7c to your computer and use it in GitHub Desktop.
HTTP Scraper by URL in Golang (Not tested, just wrote in here)
type mu sync.RWMutex
func ScrapeFromURL(ptrn []byte, urls ...string) [][]byte {
mu.Lock()
defer mu.Unlock()
var data [][]byte
for _, url := range urls {
res, err := http.Get(url)
if err != nil {
log.Fatal(err)
return nil
}
body, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
log.Fatal(err)
return nil
}
if !bytes.Contains(body, ptrn) {
continue
}
data = append(data, body)
}
return data
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment