Skip to content

Instantly share code, notes, and snippets.

@dimm999
Created November 25, 2015 06:35
Show Gist options
  • Save dimm999/caa70e860d1a2b2d229c to your computer and use it in GitHub Desktop.
Save dimm999/caa70e860d1a2b2d229c to your computer and use it in GitHub Desktop.
Proxy list parser on Go
package main
import (
"fmt"
"github.com/opesun/goquery"
"io/ioutil"
"net/http"
"os"
"strconv"
"time"
)
func check(e error) {
if e != nil {
panic(e)
}
}
func main() {
for {
f, err := os.OpenFile("proxylist.txt", os.O_APPEND|os.O_WRONLY, 0600)
check(err)
defer f.Close()
for i := 0; i < 11; i++ {
res, err := http.Get("http://proxy-list.org/russian/search.php?search=&country=RU&type=any&port=any&ssl=any&reset=Reset+search&p=" + strconv.Itoa(i))
if nil != err {
fmt.Println(err)
continue
}
text, err := ioutil.ReadAll(res.Body)
res.Body.Close()
check(err)
result, err := goquery.ParseString(string(text))
for _, v := range result.Find(".proxy").HtmlAll() {
if v == "Прокси" {
continue
}
if _, err = f.WriteString(v + "\n"); err != nil {
panic(err)
}
}
}
res, err := http.Get("http://proxy-ip-list.com/fresh-proxy-list.html")
if nil != err {
fmt.Println(err)
} else {
text, err := ioutil.ReadAll(res.Body)
res.Body.Close()
check(err)
result, err := goquery.ParseString(string(text))
i := 0
for _, v := range result.Find(".table_body tr td:first").HtmlAll() {
i++
if i == 1 {
if _, err = f.WriteString(v + "\n"); err != nil {
panic(err)
}
}
if i == 5 {
i = 0
}
}
}
fmt.Println("ok")
time.Sleep(10 * time.Minute)
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment