Created
June 7, 2019 14:32
-
-
Save jovandeginste/520758ace97496e304e3e4309cab1d87 to your computer and use it in GitHub Desktop.
Code to merge striped files into the "actual" file
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"fmt" | |
"os" | |
"strconv" | |
"syscall" | |
"time" | |
"github.com/dustin/go-humanize" | |
) | |
var numread, numout, outSize uint64 | |
func main() { | |
var err error | |
if len(os.Args) < 4 { | |
fmt.Println("Parameters: stripesize output file1...") | |
os.Exit(0) | |
} | |
stripeSizeStr := os.Args[1] | |
outSizeStr := os.Args[2] | |
outfile := os.Args[3] | |
s := len(os.Args) - 4 | |
stripeSize, err := strconv.Atoi(stripeSizeStr) | |
if err != nil { | |
panic(err) | |
} | |
outSize, err = strconv.ParseUint(outSizeStr, 10, 64) | |
if err != nil { | |
panic(err) | |
} | |
fdOut, err := syscall.Open(outfile, syscall.O_CREAT|syscall.O_EXCL|syscall.O_RDWR, 0600) | |
if err != nil { | |
panic(err) | |
return | |
} | |
defer syscall.Close(fdOut) | |
fd := make([]int, s, s) | |
for i := 0; i < s; i++ { | |
fd[i], err = syscall.Open(os.Args[i+4], syscall.O_RDONLY, 0777) | |
if err != nil { | |
fmt.Print(err.Error(), "\n") | |
return | |
} | |
defer syscall.Close(fd[i]) | |
} | |
var bytesread, bytesout int | |
buffer := make([]byte, stripeSize) | |
loop := true | |
go func() { | |
for { | |
showProgress() | |
time.Sleep(5 * time.Second) | |
} | |
}() | |
for { | |
for _, file := range fd { | |
bytesread, err = syscall.Read(file, buffer) | |
if err != nil { | |
panic(err) | |
} | |
if bytesread < stripeSize { | |
loop = false | |
break | |
} | |
numread += uint64(bytesread) | |
bytesout, err = syscall.Write(fdOut, buffer) | |
if err != nil { | |
panic(err) | |
} | |
numout += uint64(bytesout) | |
} | |
if numout > outSize { | |
loop = false | |
} | |
if !loop { | |
break | |
} | |
} | |
showProgress() | |
} | |
func showProgress() { | |
fmt.Printf("Read: %s; written: %s/%s\n", humanize.Bytes(numread), humanize.Bytes(numout), humanize.Bytes(outSize)) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment