delete_s3.go

package main

import (
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

func exitErrorf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
}

func main() {
	bucket := "prod.assets.onquidd.com"
	sess, _ := session.NewSession(&aws.Config{
		Region: aws.String("us-east-1")},
	)
	svc := s3.New(sess)

	totalIterated := 0
	input := &s3.ListObjectsV2Input{
		Bucket: aws.String(bucket),
		Prefix: aws.String("shelfies/uploads"),
	}

	total := 0
	pageNum := 0
	objects := []s3manager.BatchDeleteObject{}
	err := svc.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
		pageNum++

		for _, item := range page.Contents {
			if strings.Contains(*item.Key, "-share.jpg") || strings.Contains(*item.Key, "-share.png") || strings.Contains(*item.Key, "@w") {
				objects = append(objects, s3manager.BatchDeleteObject{
					Object: &s3.DeleteObjectInput{
						Key:    item.Key,
						Bucket: aws.String(bucket),
					},
				})
			}
		}

		if len(objects) >= 1000 {
			start := time.Now()
			batcher := s3manager.NewBatchDelete(sess, func(bd *s3manager.BatchDelete) {
				bd.BatchSize = 1000
			})
			if err := batcher.Delete(aws.BackgroundContext(), &s3manager.DeleteObjectsIterator{
				Objects: objects,
			}); err != nil {
				exitErrorf("Unable to delete items in bucket %q, %v", bucket, err)
			}
			total += len(objects)
			elapsed := time.Since(start)
			fmt.Println("Total deleted ", total, "items from bucket", bucket, " took: ", elapsed)
			objects = []s3manager.BatchDeleteObject{}
		}

		totalIterated += len(page.Contents)
		fmt.Println("Total iterated", totalIterated, "items from bucket", bucket)


		return !lastPage
	})
	if err != nil {
		exitErrorf("Unable to list items in bucket %q, %v", bucket, err)
	}

}