tar archive and zstandard compression of a directory

thing.go

package main

import (
	"archive/tar"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/klauspost/compress/zstd"
	"github.com/urfave/cli/v2"
)

func sendDir(c *cli.Context) error {
	if c.NArg() != 1 {
		return errors.New("path to directory and where to send to required")
	}

	src := c.Args().Get(0)
	// host := c.Args().Get(1)

	// client := http.Client{
	// 	Timeout: 10 * time.Second,
	// }

	// eg := errgroup.WithContext(context.Background())
	// _ = eg

	// req, err := http.NewRequest(http.MethodPost, "http://"+host+":8711/send-dir")

	out, err := os.Create("output.tar.zst")
	if err != nil {
		return fmt.Errorf("failed to create output file: %w", err)
	}
	defer func() {
		if err := out.Close(); err != nil {
			log.Printf("failed to close output file: %s", err.Error())
		}
	}()

	archiveReader, archiveWriter := io.Pipe()
	compressReader, compressWriter := io.Pipe()

	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		defer wg.Done()
		if _, err := io.Copy(out, compressReader); err != nil {
			log.Printf("failed to write to file: %s", err.Error())
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		defer compressWriter.Close()
		if err := compress(archiveReader, compressWriter); err != nil {
			log.Printf("failed to compress: %s", err.Error())
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		defer archiveWriter.Close()
		if err := archive(src, archiveWriter); err != nil {
			log.Print("failed to archive directory: %w", err)
		}
	}()

	log.Print("Sending...")
	wg.Wait()
	log.Print("EVERYTHING IS DONE!!!")

	return nil
}

func compress(in io.Reader, out io.Writer) error {
	w, err := zstd.NewWriter(out)
	if err != nil {
		return fmt.Errorf("failed to create zstd writer: %w", err)
	}
	log.Print("About to start compressing...")
	defer log.Print("Compressing done")
	if _, err := io.Copy(w, in); err != nil {
		w.Close()
		return fmt.Errorf("failure copying compressed stream: %w", err)
	}
	if err := w.Close(); err != nil {
		return fmt.Errorf("failed to flush compressed stream: %w", err)
	}
	return nil
}

func archive(src string, out io.Writer) error {
	if abs, err := filepath.Abs(src); err != nil {
		return fmt.Errorf("failed to get absolute path of %q: %w", src, err)
	} else {
		src = abs
	}
	if fi, err := os.Stat(src); err != nil {
		return fmt.Errorf("could not stat %q: %w", src, err)
	} else if !fi.IsDir() {
		return fmt.Errorf("is not a directory: %q", src)
	}

	tw := tar.NewWriter(out)
	defer tw.Close()

	log.Print("About to start archiving...")
	defer log.Print("Archiving done")
	return filepath.Walk(src, func(file string, fi os.FileInfo, err error) error {
		// return on any error
		if err != nil {
			return err
		}

		if !fi.Mode().IsRegular() {
			return nil
		}

		// create a new dir/file header
		header, err := tar.FileInfoHeader(fi, fi.Name())
		if err != nil {
			return err
		}

		// update the name to correctly reflect the desired destination when untaring
		header.Name = strings.TrimPrefix(strings.Replace(file, src, "", -1), string(filepath.Separator))

		// write the header
		if err := tw.WriteHeader(header); err != nil {
			return err
		}

		// open files for taring
		f, err := os.Open(file)
		if err != nil {
			return err
		}
		defer func() {
			if err := f.Close(); err != nil {
				log.Printf("Failed to close file %q whilst archiving: %s", file, err.Error())
			}
		}()

		// copy file data into tar writer
		if _, err := io.Copy(tw, f); err != nil {
			return err
		}

		return nil
	})
}

func checksum(in io.Reader) ([]byte, error) {
	h := sha256.New()
	if _, err := io.Copy(h, in); err != nil {
		return nil, err
	}
	return h.Sum(nil), nil
}