A simple web crawler

crawler.go

package main

import (
	"bytes"
  
	"crypto/tls"
  
	"fmt"
  
	"log"
  
	"net/http/httptrace"
  
	"time"
  
	"net/http"

	"github.com/gocolly/colly"
)

func main() {
  
	colly.NewCollector(
		colly.AllowedDomains("https://convocation.unilag.edu.ng"),
	)

	loginURL := "https://convocation.unilag.edu.ng/access"
	loginData := map[string]string{
		"username": "xxxx", 
		"password": "xxxx", 
	}

	statusCode, err := submitForm(loginURL, loginData)
	if err != nil {
		fmt.Printf("Error submitting login form: %v\n", err)
		return
	}
	fmt.Printf("Login form submission status: %d\n", statusCode)

	}

// submitForm submits a form using HTTP POST and returns the status code
func submitForm(url string, formData map[string]string) (int, error) {
	form := ""
	for key, value := range formData {
		form += fmt.Sprintf("%s=%s&", key, value)
	}
	form = form[:len(form)-1]

	req, err := http.NewRequest("POST", url, bytes.NewBufferString(form))
	if err != nil {
		return 0, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
  
  // cater for TLS handshake timeout errors with verbose logging

	trace := &httptrace.ClientTrace{
		TLSHandshakeStart: func() { log.Println("TLS handshake started") },
		TLSHandshakeDone:  func(state tls.ConnectionState, err error) { log.Println("TLS handshake done") },
	}
	req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))

	client := &http.Client{Timeout: 60 * time.Second}
	resp, err := client.Do(req)
	if err != nil {
		return 0, err
	}
	defer resp.Body.Close()

	return resp.StatusCode, nil
}