Hightail (formerly Yousendit) basic console client in golang

hitail.go

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/alexflint/go-arg"
	"github.com/cheggaaa/pb"
	"github.com/nu7hatch/gouuid"
	"github.com/xiconet/utils"
	"io"
	"io/ioutil"
	"math"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	ospath "path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"menteslibres.net/gosexy/to"
	"menteslibres.net/gosexy/yaml"
)

const (
	host     = "spaces.hightail.com"
	cfg_file = "C:/Users/ARC/.config/hightail.yml"
)

var (
	auth_url     = fmt.Sprintf("https://api.%s/api/v1/auth", host)
	base_url     = fmt.Sprintf("https://folders.%s", host)
	download_url = fmt.Sprintf("https://download.%s/api/v1/download", host)
	upload_url   = fmt.Sprintf("https://upload.%s/api/v1/upload", host)
	cookies      = map[string]string{}
)

type LoginData struct {
	SessionID string `json:"sessionId"`
	User      struct {
		Id             string `json:"id"`
		Email          string `json "email"`
		Name           string `json:"name"`
		Verified       bool   `json:"verified"`
		IsNative       bool   `json:"isNative"`
		Status         string `json:"status"`
		InletType      string `json:"inletType"`
		AuthType       string `json:"authType"`
		OrganizationId string `json:"organizationId"`
		EarlyAccess    bool   `json:"earlyAccess"`
		UpdatedAt      int    `json:"updatedAt"`
		CreatedAt      int    `json:"createdAt"`
		Active         bool   `json:"active"`
		Native         bool   `json:"native"`
	} `json:"user"`
}

type Node struct {
	CreateDate           string   `json:"createDate"`
	EffectivePermissions []string `json:"effectivePermissions"`
	FolderType           string   `json:"folderType"`
	Id                   string   `json:"id"`
	IsDeleted            bool     `json:"isDeleted"`
	IsDirectory          bool     `json:"isDirectory"`
	ModifyDate           string   `json:"modifyDate"`
	Name                 string   `json:"name"`
	OwnerId              string   `json:"ownerId"`
	PhysicalFileSize     string   `json:"physicalFileSize"`
	ParentId             string   `json:"parentId"`
	Revision             string   `json:"revision"`
}

type Folder struct {
	Children []Node `json:"children"`
	Id       string `json:"id"`
}

type ByName []Node

func (a ByName) Len() int           { return len(a) }
func (a ByName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByName) Less(i, j int) bool { return a[i].Name < a[j].Name }

type NewDir struct {
	GeneratedFileId string `json:"generatedFileId"`
	IsDeleted       bool   `json:"isDeleted"`
	Revision        string `json:"revision"`
}

type UpResp struct {
	ClientCreatedDate string `json:"clientCreatedDate"`
	ClientUpdatedDate string `json:"clientUpdatedDate"`
	CreatedDate       string `json:"createdDate"`
	CreatorId         string `json:"creatorId"`
	FolderId          string `json:"folderId"`
	Id                string `json:"id"`
	LogicalFileId     string `json:"logicalFileId"`
	Name              string `json:"name"`
	Region            string `json:"region"`
	Revision          string `json:"revision"`
	RevisionStr       string `json:"revisionStr"`
	Size              string `json:"size"`
	UfId              string `json:"ufId"`
	UpdatedDate       string `json:"updatedDate"`
}

type ReqData struct {
	Directory   bool   `json:"directory"`
	Id          string `json:"id,omitempty"`
	Name        string `json:"name,omitempty"`
	NewFilename string `json:"newFilename,omitempty"`
	ParentId    string `json:"parentId,omitempty"`
}

func userCfg(user string) (email, pwd string) {
	cfg, err := yaml.Open(cfg_file)
	if err != nil {
		panic(err)
	}
	if user == "current_user" {
		user = to.String(cfg.Get("users", "current_user"))
	}
	pwd = to.String(cfg.Get(user, "password"))
	email = to.String(cfg.Get(user, "email"))
	return

}

func apiReq(method, uri string, data interface{}) (error, string) {
	var req *http.Request
	var err error
	if method == "GET" {
		req, err = http.NewRequest(method, uri, nil)
		if err != nil {
			panic(err)
		}
	}
	if data != nil {
		js, err := json.Marshal(data)
		if err != nil {
			panic(err)
		}
		req, err = http.NewRequest(method, uri, strings.NewReader(string(js)))
		if err != nil {
			panic(err)
		}
		req.Header.Set("Content-Type", "application/json")
	}
	if req == nil {
		req, err = http.NewRequest(method, uri, nil)
		if err != nil {
			panic(err)
		}
	}
	if !(strings.Contains(uri, "login")) {
		for k, v := range cookies {
			req.Header.Add("Cookie", fmt.Sprintf("%s=%s", k, v))
		}
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}
	if resp.StatusCode != 200 {
		fmt.Println("error: bad server status", resp.Status)
		fmt.Println(string(body))
	}
	return nil, string(body)
}

func login(email, pwd string) {
	data := map[string]string{"email": email, "password": pwd}
	uri, _ := url.Parse(auth_url + "/login")
	err, body := apiReq("POST", uri.String(), data)
	if err != nil {
		fmt.Println("login request error:", err)
		os.Exit(1)
	}
	ld := LoginData{}
	err = json.Unmarshal([]byte(body), &ld)
	if err != nil {
		fmt.Println("json error:", err)
		fmt.Println(body)
		os.Exit(1)
	}
	cookies["sessionId"] = ld.SessionID
	cookies["userId"] = ld.User.Id
}

func listFolder(folderId string) (f Folder) {
	uri, _ := url.Parse(base_url)
	uri.Path += fmt.Sprintf("/api/v1/hfsEdge/children/%s", folderId)
	err, body := apiReq("GET", uri.String(), nil)
	if err != nil {
		fmt.Println("apiReq error:", err)
		fmt.Println(body)
		return
	}
	err = json.Unmarshal([]byte(body), &f)
	if err != nil {
		fmt.Println("json error:", err)
		fmt.Println("body:", body)
		os.Exit(1)
	}
	return
}

func treeList(folderId string, i, depth int) {
	idt := strings.Repeat(" ", 2*i)
	node := listFolder(folderId)
	sort.Sort(ByName(node.Children))
	for _, c := range node.Children {
		if !c.IsDirectory {
			pfs, _ := strconv.Atoi(c.PhysicalFileSize)
			fsize, _ := utils.NiceBytes(int64(pfs))
			sfmt := fmt.Sprintf("%%s%%-%ds %%10s\n", 67-2*i)
			fmt.Printf(sfmt, idt, c.Name, fsize)
		} else if depth == 0 || i < depth {
			fmt.Printf("%s%s/\n", idt, c.Name)
			treeList(c.Id, i+1, depth)
		}
	}
}

func treeSize(folderID string, usedSpace int64) int64 {
	node := listFolder(folderID)
	for _, c := range node.Children {
		if !c.IsDirectory {
			pfs, _ := strconv.Atoi(c.PhysicalFileSize)
			usedSpace += int64(pfs)
		} else {
			usedSpace = treeSize(c.Id, usedSpace)
		}
	}
	return usedSpace
}

func pathToID(path, rootID string) (found bool, n Node) {
	if path == "/" {
		return true, Node{Id: "0", IsDirectory: true}
	}
	for _, p := range strings.Split(path, "/") {
		found = false
		children := listFolder(rootID).Children
		for _, c := range children {
			if c.Name == p {
				found = true
				rootID = c.Id
				n = c
				break
			}
		}
		if !found {
			fmt.Println("error: path not found:", p)
		}
	}
	return
}

func createFolder(folder_name, parent_id string) (nd NewDir) {
	uri, _ := url.Parse(base_url + "/api/v1/hfsEdge/create")
	data := ReqData{Name: folder_name, Directory: true, ParentId: parent_id}
	err, body := apiReq("POST", uri.String(), data)
	if err != nil {
		fmt.Println("error:", err)
	}
	err = json.Unmarshal([]byte(body), &nd)
	if err != nil {
		fmt.Println(err)
		fmt.Println(body)
	}
	return
}

func download(n Node, dest string) int64 {
	if dest == "" {
		dest = n.Name
	}
	uri, _ := url.Parse(download_url)
	uri.Path += fmt.Sprintf("/link/HIGHTAIL_FILE/%s/%s/%s", n.Id, n.Revision, n.Name)
	fmt.Println("url:", uri.String())
	req, err := http.NewRequest("GET", uri.String(), nil)
	if err != nil {
		panic(err)
	}
	for k, v := range cookies {
		req.Header.Add("Cookie", fmt.Sprintf("%s=%s", k, v))
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		fmt.Println("request error:", err)
		fmt.Printf("request: %+v\n", req)
		os.Exit(1)
	}
	if resp.StatusCode != 200 {
		fmt.Println("error: bad server status:", resp.Status)
		os.Exit(1)
	}
	defer resp.Body.Close()
	out, err := os.Create(dest)
	if err != nil {
		panic(err)
	}
	defer out.Close()
	var srcSize, fl int64
	cl := resp.Header.Get("Content-Length")
	if cl != "" {
		i, err := strconv.Atoi(cl)
		if err != nil {
			fmt.Println(err)
		}
		srcSize = int64(i)
	} else {
		pfs, err := strconv.Atoi(n.PhysicalFileSize)
		if err != nil {
			fmt.Println(err)
		}
		fl = int64(pfs)
		srcSize = fl
	}
	src := resp.Body
	bar := pb.New(int(srcSize)).SetUnits(pb.U_BYTES).SetRefreshRate(time.Millisecond * 10)
	if srcSize >= 1024*1024 {
		bar.ShowSpeed = true
	}
	bar.Start()
	writer := io.MultiWriter(out, bar)
	m, err := io.Copy(writer, src)
	if err != nil {
		fmt.Println("Error while downloading from:", uri.String(), "-", err)
		return 0
	}
	bar.Finish()
	return m
}

func downsync(obj Node, localPath string, tBytes int64) int64 {
	err := os.MkdirAll(localPath, 0777)
	if err != nil {
		fmt.Println("error: could not create new folder", localPath, ":", err)
		os.Exit(1)
	}
	for _, c := range listFolder(obj.Id).Children {
		if !c.IsDirectory {
			tBytes += download(c, ospath.Join(localPath, c.Name))
		} else {
			tBytes = downsync(c, ospath.Join(localPath, c.Name), tBytes)
		}
	}
	return tBytes
}

//func GetFileContentType tries to detect a file's mime type
func GetFileContentType(out *os.File) (string, error) {
	// Only the first 512 bytes are used to sniff the content type.
	buffer := make([]byte, 512)
	_, err := out.Read(buffer)
	if err != nil {
		return "", err
	}
	// The net/http DectectContentType function returns a valid
	// "application/octet-stream" if no other match is found.
	contentType := http.DetectContentType(buffer)
	return contentType, nil
}

func makeChunk(fh *os.File, offset int64) []byte {
	chunksize := int64(5 * 1024 * 1024)
	p := make([]byte, chunksize)
	//fmt.Println("offset:", offset)
	n, _ := fh.ReadAt(p, offset)
	return p[:n]
}

func uploadRequest(uri string, params interface{}, chunk []byte) (*http.Request, error) {
	body := &bytes.Buffer{}
	ck := bytes.NewBuffer(chunk)
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile("file", "blob")
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, ck)
	for key, val := range params.(map[string]string) {
		_ = writer.WriteField(key, val)
	}
	err = writer.Close()
	if err != nil {
		return nil, err
	}
	req, err := http.NewRequest("POST", uri, body)
	if err != nil {
		panic(err)
	}
	for k, v := range cookies {
		req.Header.Add("Cookie", fmt.Sprintf("%s=%s", k, v))
	}
	req.Header.Add("Content-Type", writer.FormDataContentType())
	return req, nil
}

func resumableUpload(fp string, parent Node) (uresp UpResp) {
	const MCS = int64(5 * 1024 * 1024) // max. chunk size
	mcs := fmt.Sprintf("%d", MCS)
	fn := ospath.Base(fp)
	uri, err := url.Parse(upload_url)
	uri.Path += fmt.Sprintf("/folder/resumable/%s", parent.Id)
	if err != nil {
		panic(err)
	}
	fh, err := os.Open(fp)
	if err != nil {
		panic(err)
	}
	defer fh.Close()
	stat, err := fh.Stat()
	if err != nil {
		panic(err)
	}
	fsize := stat.Size()
	filesize := fmt.Sprintf("%d", fsize)
	chunks := math.Ceil(float64(fsize) / float64(MCS))
	tchunks := fmt.Sprintf("%d", chunks)
	rts := fmt.Sprintf("%d", int64(math.Min(float64(fsize), float64(MCS))))
	rt, err := GetFileContentType(fh)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	uid, err := uuid.NewV4()
	if err != nil {
		panic(err)
	}
	rin := uid.String()
	chunk_number := 1
	data := map[string]string{
		"resumableChunkNumber":      "1",
		"resumableChunkSize":        mcs,
		"resumableCurrentChunkSize": rts,
		"resumableTotalSize":        filesize,
		"resumableType":             rt,
		"resumableIdentifier":       rin,
		"resumableFilename":         fn,
		"resumableRelativePath":     fn,
		"resumableTotalChunks":      tchunks,
	}
	offset := int64(0)
	for offset < fsize {
		chunk := makeChunk(fh, offset)
		data["resumableCurrentChunkSize"] = fmt.Sprintf("%d", len(chunk))
		req, err := uploadRequest(uri.String(), data, chunk)
		if err != nil {
			panic(err)
		}
		fmt.Printf("uploading chunk #%s (%d bytes)...\n", data["resumableChunkNumber"], len(chunk))
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			fmt.Println("upload request error:", err)
			return
		}
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("read body error:", err)
			return
		}
		fmt.Println("server status:", resp.Status)
		if resp.StatusCode != 200 {
			fmt.Println("error: bad server status:", resp.Status)
			fmt.Println("reponse body:", string(body))
			return
		}
		err = json.Unmarshal(body, &uresp)
		if err != nil {
			fmt.Println(err)
			fmt.Println("reponse body:", string(body))
			return
		}
		chunk_number += 1
		data["resumableChunkNumber"] = fmt.Sprintf("%d", chunk_number)
		offset += int64(len(chunk))
	}
	return
}

func rename(n Node, newname string) string {
	uri, _ := url.Parse(base_url)
	uri.Path += "/api/v1/hfsEdge/rename"
	data := ReqData{Id: n.Id, NewFilename: newname, Directory: n.IsDirectory}
	err, body := apiReq("POST", uri.String(), data)
	if err != nil {
		fmt.Println(err)
	}
	return body
}

func deleteNode(n Node) string {
	uri, _ := url.Parse(base_url)
	uri.Path += fmt.Sprintf("/api/v1/hfsEdge/delete")
	data := ReqData{Id: n.Id, Directory: n.IsDirectory}
	err, body := apiReq("POST", uri.String(), data)
	if err != nil {
		fmt.Println(err)
	}
	return body
}

func main() {
	var args struct {
		Path     string `arg:"positional,help:node path"`
		User     string `arg:"-u,help:account user name"`
		Tree     bool   `arg:"-t,help:recursive (tree-style) folder list"`
		Recurse  int    `arg:"-R,help:recursion depth"`
		Info     bool   `arg:"-i,help:compute total size for the specified folder"`
		Mkdir    string `arg:"-m,help:create a new folder in the specified parent folder path"`
		Upload   string `arg:"-p,help:upload a file to the specified parent folder path"`
		Download bool   `arg:"-d,help:download item(s) under the specified path"`
		Rename   string `arg:"-r,help:rename item at the specified path"`
		Remove   bool   `arg:"-x,help:delete item(s) under the specified path"`
		Verbose  bool   `arg:"-v,help:verbose mode"`
	}
	args.Path = "/"
	args.Recurse = 0
	args.User = "current_user"
	arg.MustParse(&args)
	p := args.Path
	email, pwd := userCfg(args.User)
	login(email, pwd)
	ok, node := pathToID(p, "0")
	if !ok {
		os.Exit(1)
	}
	fmt.Printf("user: %s; node Id: %s\n\n", email, node.Id)
	switch {
	case args.Download:
		if !node.IsDirectory {
			download(node, "")
		} else {
			dl := downsync(node, node.Name, 0)
			dls, _ := utils.NiceBytes(dl)
			fmt.Printf("total downloaded: %s\n", dls)
		}
	case args.Info:
		const quota = int64(2 * 1024 * 1024 * 1024)
		fmt.Printf("computing tree size for folder %q...\n", args.Path)
		tsize := treeSize(node.Id, 0)
		ts, _ := utils.NiceBytes(tsize)
		if args.Path == "/" {
			left := quota - tsize
			l, _ := utils.NiceBytes(left)
			fmt.Printf("used: %10s\nleft: %10s", ts, l)
		} else {
			fmt.Printf("path: %s\ntotal size: %s\n", args.Path, ts)
		}
	case args.Tree:
		treeList(node.Id, 0, args.Recurse)
	case args.Mkdir != "":
		if !node.IsDirectory {
			fmt.Println("error:", node.Name, "is not a folder")
		} else {
			newf := createFolder(args.Mkdir, node.Id)
			fmt.Printf("%+v\n", newf)
		}
	case args.Upload != "":
		resp := resumableUpload(args.Upload, node)
		fmt.Printf("%+v\n", resp)
	case args.Rename != "":
		resp := rename(node, args.Rename)
		fmt.Println(resp)
	case args.Remove:
		resp := deleteNode(node)
		fmt.Println(resp)
	default:
		lf := listFolder(node.Id)
		sort.Sort(ByName(lf.Children))
		for _, c := range lf.Children {
			if c.IsDirectory {
				fmt.Printf("%s/\n", c.Name)
			} else {
				pfs, _ := strconv.Atoi(c.PhysicalFileSize)
				fsize, _ := utils.NiceBytes(int64(pfs))
				fmt.Printf("%-67s %10s\n", c.Name, fsize)
			}
		}
	}
}

Emulates browser API requests. Cookies are returned in the login response body