find_dead_links.go

package main

import (
        "encoding/json"
        "fmt"
        "log"
        "os"
        "regexp"
        "strings"

        "github.com/gocolly/colly/v2"
)

type LinkError struct {
        Link       string   `json:"link"`
        StatusCode int      `json:"code"`
        Pages      []string `json:"pages"`
}

var invalidRegex = regexp.MustCompile(".*(zh-hans|tutorials|solutions|articles|archive).*")

func valid(link string) bool {
        return !invalidRegex.MatchString(link)
}

func main() {

        var pages = map[string][]string{}

        // Instantiate default collector
        c := colly.NewCollector(
                colly.AllowedDomains("developer.algorand.com", "developer.algorand.org"),
                colly.MaxDepth(2),
        )

        // On every a element which has href attribute call callback
        c.OnHTML("a[href]", func(e *colly.HTMLElement) {
                relative := strings.Trim(e.Attr("href"), " ")
                link := e.Request.AbsoluteURL(relative)
                if valid(link) {
                        currpage := e.Request.URL.String()
                        pages[link] = append(pages[link], currpage)
                        c.Visit(link)
                }
        })

        errpages := map[string]int{}
        c.OnError(func(r *colly.Response, err error) {
                if r.StatusCode != 0 {
                        if valid(r.Request.URL.String()) {
                                errpages[r.Request.URL.String()] = r.StatusCode
                                fmt.Printf("Error on %s: %d, %s\n", r.Request.URL.String(), r.StatusCode, err.Error())
                        }
                }
        })

        // Start scraping on https://en.wikipedia.org
        c.Visit("https://developer.algorand.com/")

        c.Wait()

        elist := []LinkError{}
        for link, statuscode := range errpages {

                p := map[string]bool{}
                distinct := []string{}
                for _, page := range pages[link] {
                        if _, ok := p[page]; !ok {
                                p[page] = true
                                distinct = append(distinct, page)
                        }
                }

                elist = append(elist, LinkError{
                        Link:       link,
                        StatusCode: statuscode,
                        Pages:      distinct,
                })
        }

        f, err := os.Create("doc_errors.json")
        if err != nil {
                log.Fatalf("Failed to create json file: %+v", err)
        }

        b, err := json.Marshal(elist)
        if err != nil {
                log.Fatalf("Failed to marshal elist: %+v", err)
        }

        f.Write(b)
        f.Close()
}