The provided code is a Golang implementation designed to make HTTP requests with a specific focus on bypassing certain security checks, particularly the TLS fingerprinting that some services use to detect and block bots.
The CustomTransport struct is used to override the default behavior of the HTTP client's transport mechanism. This allows for the setting of a custom User-Agent for every request, which can be essential in bypassing some basic bot detection mechanisms.
A notable feature of the code is its use of the github.com/refraction-networking/utls package, which is a fork of Go's standard TLS library. This package allows for more granular control over the TLS handshake process, enabling the user to mimic the TLS fingerprints of popular browsers. This is a crucial part of bypassing advanced bot detection mechanisms that look at the TLS fingerprint to determine if the client is a real browser or a bot.
Additionally, instead of connecting to the domain name, the code resolves the IP address of the domain and connects directly to it. This can sometimes help bypass DNS-based challenges or blocks.
In essence, this code is designed to mimic the behavior and appearance of a genuine browser as closely as possible, both in terms of the headers it sends and the TLS handshake process. The use of uTLS for custom TLS fingerprinting is the primary strategy for bypassing advanced bot detection mechanisms based on TLS fingerprint analysis.
package main
import (
"context"
cbrotli "github.com/google/brotli/go/cbrotli"
tls "github.com/refraction-networking/utls"
"io"
"net"
"net/http"
"net/http/cookiejar"
"time"
)
// CustomTransport is a custom transport for http client
type CustomTransport struct {
Transport http.RoundTripper
UserAgent string
}
type CustomRequest struct {
*http.Request
Client *http.Client
}
// RoundTrip adds the User-Agent header to every request
func (c *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req.Header.Set("User-Agent", c.UserAgent)
return c.Transport.RoundTrip(req)
}
func GenerateIfModifiedSinceHeader() string {
// If-Modified-Since: Thu, 17 Aug 2023 06:57:33 GMT
ifModified := time.Now().AddDate(0, 0, 1).Format(http.TimeFormat)
return ifModified
}
func GetIpFromHost(host string) (string, error) {
ips, err := net.LookupIP(host)
if err != nil {
return "", err
}
return ips[0].String(), nil
}
// NewHttpClient is a function for creating a new http client
func NewHttpClient() *http.Client {
config := tls.Config{ServerName: "webapi.computeruniverse.net"}
addr, err := GetIpFromHost(config.ServerName)
if err != nil {
panic(err)
}
dialConn, err := net.DialTimeout("tcp", addr+":443", 10*time.Second)
if err != nil {
panic(err)
}
tlsConn := tls.Client(dialConn, &config)
userAgent := "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0"
jar, err := cookiejar.New(nil)
if err != nil {
panic(err)
}
return &http.Client{
Timeout: 10 * time.Second,
Transport: &CustomTransport{
Transport: &http.Transport{
DisableKeepAlives: true,
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
return tlsConn, nil
},
},
UserAgent: userAgent,
},
Jar: jar,
}
}
func (req *CustomRequest) AddHeaders() {
// Add headers to request
req.Header.Set("If-Modified-Since", GenerateIfModifiedSinceHeader())
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.7,ru;q=0.3")
req.Header.Set("Accept-Encoding", "gzip, deflate, br")
req.Header.Set("DNT", "1")
req.Header.Set("Connection", "keep-alive")
// Sec headers
req.Header.Set("Sec-Fetch-Dest", "document")
req.Header.Set("Sec-Fetch-Mode", "navigate")
req.Header.Set("Sec-Fetch-Site", "none")
req.Header.Set("Sec-Fetch-User", "?1")
req.Header.Set("Upgrade-Insecure-Requests", "1")
}
func NewCustomRequest(client *http.Client, method, url string, body io.Reader) (*CustomRequest, error) {
req, err := http.NewRequest(method, url, body)
if err != nil {
return nil, err
}
cReq := &CustomRequest{
Request: req,
Client: client,
}
cReq.AddHeaders()
return cReq, nil
}
func (cReq *CustomRequest) Do() (*http.Response, error) {
return cReq.Client.Do(cReq.Request)
}
func main() {
// Open and print computerunverse.net page
client := NewHttpClient()
cReq, err := NewCustomRequest(client, "GET", "https://webapi.computeruniverse.net/api/products/90925281/variable", nil)
if err != nil {
panic(err)
}
// Выполняем запрос
resp, err := cReq.Do()
if err != nil {
panic(err)
}
defer resp.Body.Close()
reader := cbrotli.NewReader(resp.Body)
bodyBytes, err := io.ReadAll(reader)
if err != nil {
panic(err)
}
print(string(bodyBytes))
}