proxy.go

package main

import (
	"bufio"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"sync"
	"time"
)

func main() {
	servers := getServers("./proxies/combined.txt")

	workers := fanOut(servers, 5000)

	file, err := os.Create("./proxies/out.txt")
	if err != nil {
		log.Fatal("File creation error:", err)
	}
	defer file.Close()

	for ip := range merge(workers...) {
		fmt.Fprintln(file, ip)
	}

	// this is counting stuff
	// var total time.Duration
	// for _, duration := range times {
	// 	total += duration
	// }
	// mean := int(total/time.Second) / len(times)
	// fmt.Println("Average response for a working server was:", mean, "seconds")
	// fmt.Println("There were", len(times), "working proxy servers.")
}

func getServers(filePath string) chan string {
	servers := make(chan string)
	go func() {
		file, err := os.Open(filePath)
		if err != nil {
			log.Fatal(err)
		}
		defer file.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			servers <- scanner.Text()
		}

		if err := scanner.Err(); err != nil {
			log.Println("Scanner error:", err)
		}

		close(servers)
	}()

	return servers
}

func fanOut(in chan string, n int) []chan string {
	var channels []chan string
	for i := 0; i < n; i++ {
		channels = append(channels, testProxy(in))
	}

	return channels
}

func testProxy(servers chan string) chan string {
	out := make(chan string)

	go func() {
		for server := range servers {
			address := "http://" + server
			log.Println("Checking proxy for address:", address)

			proxyURL, err := url.Parse(address)
			if err != nil {
				log.Println("Parsing error:", err)
				continue
			}

			proxyClient := &http.Client{
				Transport: &http.Transport{
					Proxy:           http.ProxyURL(proxyURL),
					IdleConnTimeout: time.Second,
				},
				Timeout: 15 * time.Second,
			}

			_, err = proxyClient.Get("https://sci-organizer.science/")
			if err != nil {
				log.Println("Connection error:", err)
				continue
			}

			// log.Println("Server responded with:", response.Status)
			out <- server
		}
		close(out)
	}()

	return out

	// start := time.Now()

	// timesMutex.Lock()
	// times = append(times, time.Since(start))
	// timesMutex.Unlock()

	// serversMutex.Lock()
	// workingServers = append(workingServers, ip)
	// serversMutex.Unlock()
}

func merge(channels ...chan string) chan string {
	var wg sync.WaitGroup
	out := make(chan string)

	output := func(channel chan string) {
		for n := range channel {
			out <- n
		}
		wg.Done()
	}

	wg.Add(len(channels))
	for _, channel := range channels {
		go output(channel)
	}

	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}