检查大批量的gzip文件是否是正常的

go

package main

import (
	"compress/gzip"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
)

func checkGzip(path string) bool {
	handle, err := os.Open(path)
	if err != nil {
		log.Println(err)
		return false
	}
	defer handle.Close()

	gzReader, err := gzip.NewReader(handle)
	if err != nil {
		log.Println(err)
		return false
	}
	defer gzReader.Close()
	buf := make([]byte, 64*1024)
	for {
		_, err := gzReader.Read(buf)
		if err != nil {
			if err != io.EOF {
				log.Println(err)
				return false
			}
			return true
		}
	}
	return true
}

func main() {
	root := "my/gzipfile/dir"
	fileChan := func() chan string {
		fchan := make(chan string)
		go func() {
			defer close(fchan)
			err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
				if err != nil {
					log.Println(err)
				}
				if !info.IsDir() && strings.HasSuffix(path, "gz") {
					fchan <- path
				}
				return nil
			})
			if err != nil {
				log.Fatalln(err)
			}
		}()
		return fchan
	}()

	log.Println("begin")
	var wg sync.WaitGroup
	for i := 0; i < runtime.NumCPU(); i++ {
		wg.Add(1)
		go func(group *sync.WaitGroup, files chan string) {
			defer group.Done()
			for fpath := range files {
				if !checkGzip(fpath) {
					log.Println("remove path:", fpath)
					os.Remove(fpath)
				}
			}
		}(&wg, fileChan)
	}
	wg.Wait()
	log.Println("done")
}