main.go

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"

	"github.com/ahamilton55/zapier-test/mcchunker"
	"github.com/bradfitz/gomemcache/memcache"
)

func main() {
	dat, err := ioutil.ReadFile("bigoldfile.dat")
	if err != nil {
		log.Fatalf("unable to read data: %v\n", err)
	}

	mc := memcache.New("127.0.0.1:11211")

	key := "test_key"
	if err := mcchunker.Send(mc, key, dat); err != nil {
		log.Fatalf("unable to set data: %v\n", err)
	}

	chkData, err := mcchunker.Get(mc, key)
	if err != nil {
		log.Fatalf("unable to get data: %v\n", err)
	}

	fmt.Printf("Original length: %d\nStored length: %d\n", len(dat), len(chkData))
	if bytes.Compare(dat, chkData) != 0 {
		log.Fatalf("incorrect data returned")
	}

	fmt.Println("Success")

	mc.FlushAll()
}

mcchunker.go

package mcchunker

import (
	"fmt"
	"math"

	"github.com/bradfitz/gomemcache/memcache"
)

const (
	megabyte = 1.0 * 1000 * 1000

	keyFmt = "%s_%d"
)

// MCChunker is a helper type for storing the memcached client
type MCChunker struct {
	mc *memcache.Client
}

// New will produce a
func New(mc *memcache.Client) *MCChunker {
	return &MCChunker{mc: mc}
}

// Send a file to memcached and chunk up the file if neccessary
func Send(mc *memcache.Client, key string, data []byte) error {
	var err error

	lower := 0
	if len(data) > megabyte {
		maxChunk := int(math.Ceil(float64(len(data)) / megabyte))
		for i := 0; i < maxChunk; i++ {
			upper := ((i + 1) * int(megabyte)) - 1
			if upper >= len(data) {
				upper = len(data)
			}

			err = mc.Set(&memcache.Item{
				Key:   fmt.Sprintf(keyFmt, key, i),
				Value: data[lower:upper],
			})
			if err != nil {
				return err
			}
			lower = upper
		}

	} else {
		err = mc.Set(&memcache.Item{Key: key, Value: data})
		if err != nil {
			return err
		}
	}

	return nil
}

// Get a file from memcached, if needed, get the chunks of a file
func Get(mc *memcache.Client, key string) ([]byte, error) {
	item, err := mc.Get(key)
	if err == nil {
		return item.Value, nil
	}

	var data []byte
	for i := 0; true; i++ {
		item, err := mc.Get(fmt.Sprintf(keyFmt, key, i))
		if err != nil && i == 0 {
			return nil, err
		} else if err != nil {
			break
		}

		data = append(data, item.Value...)
	}

	return data, nil
}