Caddy folder downloader

README.md

go run . -auth username:password https://domain.com/directory/to/download/

cadgrab.go

package main

import (
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sync"

	"github.com/cheggaaa/pb"
)

type File struct {
	Name      string
	Size      uint64
	URL       string
	ModTime   string
	Mode      uint32
	IsDir     bool
	IsSymlink bool
}

type config struct {
	*url.URL
	BasicAuth string
	Progress  *pb.Pool
}

func (c config) auth() string {
	return base64.StdEncoding.EncodeToString([]byte(c.BasicAuth))
}

func (c config) GET(url string) (*http.Response, error) {
	rq, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatalln("Failed to make a new request:", err)
	}

	rq.Header.Set("Accept", "application/json")

	if c.BasicAuth != "" {
		rq.Header.Set("Authorization", "Basic "+c.auth())
	}

	return http.DefaultClient.Do(rq)
}

func main() {
	config := flags()

	pwd, err := os.Getwd()
	if err != nil {
		log.Fatalln("Failed to get PWD:", err)
	}

	// GET
	r, err := config.GET(config.String())
	if err != nil {
		log.Fatalln("Failed to make a new request:", err)
	}
	defer r.Body.Close()

	var files []File
	if err := json.NewDecoder(r.Body).Decode(&files); err != nil {
		log.Fatalln("Failed to decode JSON:", err)
	}

	var dir = filepath.Join(pwd, config.Path)
	if err := os.MkdirAll(dir, os.ModeDir|os.ModePerm); err != nil {
		log.Fatalln("Failed to mkdirAll on ", dir+":", err)
	}

	var wg sync.WaitGroup

	for _, f := range files {
		downloadFile(config, dir, f, &wg)
	}

	if err := config.Progress.Start(); err != nil {
		log.Println("Can't start the progress bar:", err)
	} else {
		defer config.Progress.Stop()
	}

	wg.Wait()
}

func downloadFile(c config, dir string, f File, wg *sync.WaitGroup) {
	p := filepath.Join(dir, f.Name)
	w, err := os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm)
	if err != nil {
		log.Println("Failed to create file at", p, err)
		return
	}

	bar := pb.New64(int64(f.Size))
	bar.Units = pb.U_BYTES
	bar.AlwaysUpdate = true
	c.Progress.Add(bar)

	wg.Add(1)
	go func() {
		defer w.Close()
		defer wg.Done()
		download(c, f, w, bar)
	}()
}

func download(c config, f File, to io.Writer, pb *pb.ProgressBar) {
	r, err := c.GET(c.String() + f.Name)
	if err != nil {
		log.Println("Cannot download", f.Name, ":", err)
		return
	}
	defer r.Body.Close()

	rd := pb.NewProxyReader(r.Body)

	_, err = io.Copy(to, rd)
	if err != nil {
		log.Println("Cannot write:", err)
	}
}

func basicAuth(pair string) string {
	return base64.StdEncoding.EncodeToString([]byte(pair))
}

func flags() config {
	var c config

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: cadgrab -auth 'username:password' URL")
	}

	flag.StringVar(&c.BasicAuth, "auth", "", "username:password")
	flag.Parse()

	URL := flag.Arg(0)

	if URL == "" {
		flag.Usage()
		os.Exit(2)
	}

	u, err := url.Parse(URL)
	if err != nil {
		log.Fatalln("Failed to parse given URL:", err)
	}

	c.URL = u

	c.Progress = pb.NewPool()
	return c
}