Skip to content

Instantly share code, notes, and snippets.

@jovandeginste
Created June 7, 2019 14:32
Show Gist options
  • Save jovandeginste/520758ace97496e304e3e4309cab1d87 to your computer and use it in GitHub Desktop.
Save jovandeginste/520758ace97496e304e3e4309cab1d87 to your computer and use it in GitHub Desktop.
Code to merge striped files into the "actual" file
package main
import (
"fmt"
"os"
"strconv"
"syscall"
"time"
"github.com/dustin/go-humanize"
)
var numread, numout, outSize uint64
func main() {
var err error
if len(os.Args) < 4 {
fmt.Println("Parameters: stripesize output file1...")
os.Exit(0)
}
stripeSizeStr := os.Args[1]
outSizeStr := os.Args[2]
outfile := os.Args[3]
s := len(os.Args) - 4
stripeSize, err := strconv.Atoi(stripeSizeStr)
if err != nil {
panic(err)
}
outSize, err = strconv.ParseUint(outSizeStr, 10, 64)
if err != nil {
panic(err)
}
fdOut, err := syscall.Open(outfile, syscall.O_CREAT|syscall.O_EXCL|syscall.O_RDWR, 0600)
if err != nil {
panic(err)
return
}
defer syscall.Close(fdOut)
fd := make([]int, s, s)
for i := 0; i < s; i++ {
fd[i], err = syscall.Open(os.Args[i+4], syscall.O_RDONLY, 0777)
if err != nil {
fmt.Print(err.Error(), "\n")
return
}
defer syscall.Close(fd[i])
}
var bytesread, bytesout int
buffer := make([]byte, stripeSize)
loop := true
go func() {
for {
showProgress()
time.Sleep(5 * time.Second)
}
}()
for {
for _, file := range fd {
bytesread, err = syscall.Read(file, buffer)
if err != nil {
panic(err)
}
if bytesread < stripeSize {
loop = false
break
}
numread += uint64(bytesread)
bytesout, err = syscall.Write(fdOut, buffer)
if err != nil {
panic(err)
}
numout += uint64(bytesout)
}
if numout > outSize {
loop = false
}
if !loop {
break
}
}
showProgress()
}
func showProgress() {
fmt.Printf("Read: %s; written: %s/%s\n", humanize.Bytes(numread), humanize.Bytes(numout), humanize.Bytes(outSize))
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment