Find any file sync failed folders under Tencent Cloud COS bucket 查找腾讯云COS有同步(移动)失败文件的文件夹

lost-in-cos.go

package main

import (
	"context"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"sync"

	"github.com/tencentyun/cos-go-sdk-v5"
)

var tencent_cos_bucket = "https://your-own-bucket.cos.ap-shanghai.myqcloud.com/"
var u, _ = url.Parse(tencent_cos_bucket)
var bucket = &cos.BaseURL{BucketURL: u}
var rules = regexp.MustCompile(`[.](jpg|png)$`)

func lost(src string) ([]string, error) {
	result := []string{}
	c := cos.NewClient(bucket, &http.Client{})

	marker := ""
	origin := map[string]bool{}
	basedir := "your-src-basedir"
	isTruncated := true
	for isTruncated {
		opt := &cos.BucketGetOptions{
			Prefix:       filepath.Join(basedir, src),
			Marker:       marker,
			EncodingType: "url",
		}
		v, _, err := c.Bucket.Get(context.Background(), opt)
		if err != nil {
			return result, err
		}

		for _, c := range v.Contents {
			key, _ := cos.DecodeURIComponent(c.Key)

			if strings.HasSuffix(key, "/") || !rules.MatchString(key) {
				continue
			}

			origin[key[len(basedir):]] = true
		}
		marker = v.NextMarker
		isTruncated = v.IsTruncated
	}

	basedir = "your-dst-basedir"
	isTruncated = true

	for isTruncated {
		opt := &cos.BucketGetOptions{
			Prefix:       filepath.Join(basedir, src),
			Marker:       marker,
			EncodingType: "url",
		}
		v, _, err := c.Bucket.Get(context.Background(), opt)
		if err != nil {
			return err
		}
		if len(v.Contents) == 0 {
			result = append(result, filepath.Join(src, "*"))
			break
		}

		for _, c := range v.Contents {
			key, _ := cos.DecodeURIComponent(c.Key)

			if strings.HasSuffix(key, "/") || !rules.MatchString(key) {
				continue
			}

			key = key[len(basedir):]
			if _, ok := origin[key]; ok {
				delete(origin, key)
			}
		}
		marker = v.NextMarker
		isTruncated = v.IsTruncated
	}
	if len(origin) > 0 {
    // only list folders, if want files, update code here
		dirs := map[string]bool{}
		for k, _ := range origin {
			key := filepath.Dir(k)
			if _, ok := dirs[key]; !ok {
				dirs[key] = true
				result = append(result, key)
			}
		}
	}
	return result, nil
}

func download(wg *sync.WaitGroup, c *cos.Client, keysCh <-chan []string) {
	defer wg.Done()
	for keys := range keysCh {
		key := keys[0]
		filename := keys[1]
		_, err := c.Object.GetToFile(context.Background(), key, filename, nil)
		if err != nil {
			log.Println(err)
		}
	}
}

func cosdl(src string) error {
	c := cos.NewClient(bucket, &http.Client{})
	ch := make(chan []string, 3)
	var wg sync.WaitGroup
	threadpool := 3
	for i := 0; i < threadpool; i++ {
		wg.Add(1)
		go download(&wg, c, ch)
	}

	isTruncated := true

	basedir := "your-src-basedir"
	prefix := filepath.Join(basedir, src)
	marker := ""
	for isTruncated {
		opt := &cos.BucketGetOptions{
			Prefix:       prefix,
			Marker:       marker,
			EncodingType: "url",
		}
		v, _, err := c.Bucket.Get(context.Background(), opt)
		if err != nil {
			return err
		}
		for _, c := range v.Contents {
			key, _ := cos.DecodeURIComponent(c.Key)
			if strings.HasSuffix(key, "/") || !rules.MatchString(key) {
				continue
			}
			localfile := filepath.Join("_todos", key[len(basedir):])
			if _, err := os.Stat(path.Dir(localfile)); err != nil && os.IsNotExist(err) {
				os.MkdirAll(path.Dir(localfile), os.ModePerm)
			}
			if strings.HasSuffix(localfile, "/") {
				continue
			}
			ch <- []string{key, localfile}
		}
		marker = v.NextMarker
		isTruncated = v.IsTruncated
	}
	close(ch)
	wg.Wait()
	return nil
}

func main() {
	subdir := "sub-folder"
	if len(os.Args) > 1 {
		subdir = os.Args[1]
	}
	//

	list, err := cosdir(subdir)
	if err != nil {
		log.Fatal(err)
	}
	// list := []string{}
	// ioutil.WriteFile("lost.todos", []byte(strings.Join(list, "\n")), 0644)
	// body, err := ioutil.ReadFile(os.Args[1])
	// if err != nil {
	// 	fmt.Println(err)
	// 	return
	// }
	// spaces := " \t\r\n"
	// lines := bytes.Split(body, []byte("\n"))
	// for _, line := range lines {
	// 	line = bytes.Trim(line, spaces)
	// 	if len(line) == 0 || bytes.HasSuffix(line, []byte("*")) {
	// 		continue
	// 	}
	//  list = append(list, line)
	// }

	for _, line := range list {
		err = cosdl(string(line))
		if err != nil {
			log.Fatal(err)
			return
		}
	}
}