MDWiki Markdown Downloader

mdwiki-download.go

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
)

var client = &http.Client{}
var host, outDir *string
var urlRegexp *regexp.Regexp

func main() {
	host = flag.String("u", "", "host of the mdwiki site, e.g. http://mdwiki.local")
	outDir = flag.String("o", ".", "folder to download markdown files to")
	flag.Parse()

	if *host == "" {
		log.Fatal("missing url arg (-u)")
	}
	if _, err := url.Parse(*host); err != nil {
		log.Fatal("invalid url arg (-u)")
	}

	urlRegexp = regexp.MustCompile(`\(([^)]+)\)`)

	err := downloadFile("", "navigation.md")

	if err != nil {
		log.Fatal(err)
	}

	fmt.Println("Done!")
}

func downloadFile(directory, fileName string) error {

	writeDir := fmt.Sprintf("%s/%s", *outDir, directory)
	os.MkdirAll(writeDir, 0700)
	if _, err := os.Stat(writeDir + "/" + fileName); !os.IsNotExist(err) {
		return nil // already downloaded, likely reverse link
	}

	navUrl := fmt.Sprintf("%s/%s/%s", *host, directory, fileName)

	req, err := http.NewRequest(http.MethodGet, navUrl, nil)
	if err != nil {
		return err
	}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != 200 {
		return nil
	}

	body, _ := ioutil.ReadAll(resp.Body)
	err = ioutil.WriteFile(writeDir+"/"+fileName, body, 0666)
	if err != nil {
		return err
	}

	if directory != "" {
		fmt.Printf("saved %s to %s\n", fileName, directory)
	} else {
		fmt.Printf("saved %s\n", fileName)
	}

	if !strings.HasSuffix(fileName, ".md") {
		return nil
	}

	links := urlRegexp.FindAll(body, -1)
	for _, link := range links {
		link := string(link[1 : len(link)-1])
		div := strings.LastIndex(link, "/")
		if div == -1 {
			if err := downloadFile(directory, link); err != nil {
				return err
			}
		} else {
			dir := directory + "/" + link[:div]
			if err := downloadFile(dir, link[div+1:]); err != nil {
				return err
			}
		}
	}
	return nil
}