copy-s3-prefix.go

package main

import (
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	aws_session "github.com/aws/aws-sdk-go/aws/session"
	aws_s3 "github.com/aws/aws-sdk-go/service/s3"
)

const (
        REGION = "us-east-1"
	SOURCE_BUCKET = "setme"
	DEST_BUCKET   = "setmetoo"
        CONCURRENCY   = 100
)

var PREFIXES = map[string]string{
	"source/bucket-prefix/": "dest/bucket-prefix/",

}

func main() {
	session := aws_session.Must(aws_session.NewSession(aws.NewConfig().WithRegion(REGION)))
	s3 := aws_s3.New(session)
	limiter := make(chan struct{}, CONCURRENCY)

	for sourcePrefix, destPrefix := range PREFIXES {
		input := &aws_s3.ListObjectsInput{
			Bucket: aws.String(SOURCE_BUCKET),
			Prefix: aws.String(sourcePrefix),
		}
		err := s3.ListObjectsPages(input, func(output *aws_s3.ListObjectsOutput, lastPage bool) bool {
			for _, obj := range output.Contents {
				destKey := strings.Split(*obj.Key, "/")[1]
				copyInput := &aws_s3.CopyObjectInput{
					Bucket:     aws.String(DEST_BUCKET),
					Key:        aws.String(destPrefix + destKey),
					CopySource: aws.String(SOURCE_BUCKET + "/" + *obj.Key),
				}

				limiter <- struct{}{}
				fmt.Printf("s3://%s -> s3://%s/%s\n", *copyInput.CopySource, *copyInput.Bucket, *copyInput.Key)
				go func(input *aws_s3.CopyObjectInput) {
					if _, err := s3.CopyObject(copyInput); err != nil {
						panic(err)
					}
					<-limiter
				}(copyInput)
			}
			return true
		})
		if err != nil {
			panic(err)
		}
	}
}