upload.go

package main

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/Azure/azure-sdk-for-go/services/datalake/store/2016-11-01/filesystem"
	"github.com/Azure/go-autorest/autorest/azure/auth"
)

const (
	accountName = "<account name>"
	destination = "<destination>"
	inputFile   = `<big file for upload>`
	bufferSize  = 4 * 1024 * 1024
)

func main() {
	a, err := auth.NewAuthorizerFromEnvironment()
	if err != nil {
		panic(err)
	}
	client := filesystem.NewClient()
	client.Authorizer = a

	f, err := os.Open(inputFile)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	resp, err := client.Create(context.Background(), accountName, destination, nil, nil, filesystem.DATA, nil, nil)
	if err != nil {
		panic(err)
	}
	fmt.Println(resp.Status)

	buffer := make([]byte, bufferSize, bufferSize)
	for {
		n, err := f.Read(buffer)
		if err == io.EOF {
			break
		}
		flag := filesystem.DATA
		if n < bufferSize {
			// last chunk
			flag = filesystem.CLOSE
		}
		chunk := ioutil.NopCloser(bytes.NewReader(buffer))
		fmt.Printf("uploading chunk %d...", n)
		resp, err = client.Append(context.Background(), accountName, destination, chunk, nil, flag, nil, nil)
		if err != nil {
			panic(err)
		}
		fmt.Printf("%d\n", resp.StatusCode)
	}
	fmt.Println("done!")
}