Split a file into multiple GZIP files

GzipSplitter.scala

import java.io.InputStream

import better.files._
import squants.information._, InformationConversions._

object GzipSplitter {
  /** Splits the $inputstream into approximately equal chunks of $splitSize gzip files under $outputDirectory */
  def split(
    inputStream      : InputStream,
    outputDirectory  : File         = File.newTemporaryDirectory(),
    outputFilePrefix : String       = "part-",
    splitSize        : Information  = 50.mb,
    initialGuess     : Int          = 1e5.toInt
  ): Unit = {
    val lines = inputStream.lines
    Iterator.from(0)
      .takeWhile(_ => lines.nonEmpty)
      .foldLeft(initialGuess) { case (guess, part) =>
        val file = outputDirectory / s"${outputFilePrefix}${part}.gz"
        for {
          gzip   <- file.gzipOutputStream()
          writer <- gzip.printWriter().autoClosed
          line   <- lines.take(guess)
        } writer.println(line)
        println(s"Wrote $guess lines to ${file.name} (${file.size.mb.value.toInt} MB)")
        // Probe the file we just wrote to update our guess of how many lines it takes to make a $splitSize chunk
        // Note: file sizes do not scale linearly (esp. for compressed streams); but this is a good enough approximation
        (splitSize * guess / file.size.bytes).toInt
      }
  }
}