Golang KartRider patches crawler

main.go

package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
)

var (
	hostname   string
	patchDir   string
	startPatch int
)

func GetFiles(version int) ([]string, error) {
	var contents string = ""

	localDir := patchDir + strconv.Itoa(version) + "/"
	remoteDir := hostname + localDir

	if binaryContents, err := ioutil.ReadFile(localDir + "files.nfo2"); err != nil && len(contents) > 0 {
		contents = string(binaryContents)
	} else {

		res, err := http.Get(remoteDir + "files.nfo2")
		if err != nil {
			return nil, err
		}

		if res.StatusCode != 200 {
			return nil, errors.New("File not found: " + remoteDir + "files.nfo2")
		}

		os.MkdirAll(localDir, 0755)

		binaryContents, err := ioutil.ReadAll(res.Body)
		res.Body.Close()
		contents = string(binaryContents)

		ioutil.WriteFile(localDir+"files.nfo2", binaryContents, 0755)

	}

	lines := strings.Split(contents, "\r\n")

	l := make([]string, 0, len(lines))
	l = append(l, remoteDir+"files.nfo2") // Add main files.nfo2 thing

	for _, line := range lines {
		if strings.HasPrefix(line, `"`) {
			// Get 'last' quote
			filename := line[1:]
			lastQuoteIndex := strings.Index(filename, `"`)

			filename = filename[:lastQuoteIndex]

			// Replace backslashes with forwardslashes
			filename = strings.Replace(filename, "\\", "/", -1)

			// GZ compressed files
			filename += ".gz"

			downloadURL := remoteDir + filename

			l = append(l, downloadURL)
		}
	}

	return l, nil

}

func main() {
	fmt.Println("KartRider patch downloader")

	flag.StringVar(&hostname, "hostname", "http://kart.dn.nexoncdn.co.kr/", "Hostname of the download location")
	flag.StringVar(&patchDir, "patchdir", "patch/", "Directory of the patchfiles")
	flag.IntVar(&startPatch, "startpatch", 3089, "Patch to start scanning from")
	flag.Parse()

	downloads := make([]string, 0)

	os.MkdirAll(patchDir, 0755)

	failCounter := 0
	foundDownload := false
	for i := startPatch; ; i++ {
		if l, err := GetFiles(i); err != nil {
			if foundDownload && failCounter >= 20 {
				fmt.Println("End!")
				break
			} else {
				fmt.Println("Not found: ", err)
				failCounter++
			}
		} else {
			fmt.Println("Added files for version ", i)
			foundDownload = true
			downloads = append(downloads, l...)
			failCounter = 0
		}
	}

	ioutil.WriteFile("todownload.txt", []byte(strings.Join(downloads, "\n")), 0755)
}