Chunking Array Serializer/Deserializer in Scala

gistfile1.scala

// We want a generic way to chunk large arrays into segments of byte arrays
// and to do so in a way that doesn't blow up memory for large objects, ie ability to chunk / page / stream

trait ChunkingArraySerDe[T] {
  def apply(data: Array[T], chunkSize: Int): Iterator[Array[Byte]]
  def unapply(serialized: Iterator[Array[Byte]]): Array[T]
}

// For arrays with fixed-size elements
trait PrimitiveChunkingSerDe[@specialized(Int, Long, Boolean) T] extends ChunkingArraySerDe[T] {
  def elementSize: Int
  def serializeChunk(dataSlice: Array[T]): Array[Byte]
  
  def apply(data: Array[T], chunkSize: Int): Iterator[Array[Byte]] = {
    val elementsPerChunk = chunkSize / elementSize
    data.grouped(elementsPerChunk).map(serializeChunk(_))
  }
}

// Example implementation for Array[Int]
object IntArrayChunkingSerDe extends PrimitiveChunkingSerDe[Int] {
  val elementSize = 4
  def serializeChunk(dataSlice: Array[Int]): Array[Byte] = {
    val bb = java.nio.ByteBuffer.allocate(dataSlice.length * 4)
    bb.asIntBuffer.put(dataSlice)
    bb.array
  }
}

// Example of using it:
for (chunk <- IntArrayChunkingSerDe(myIntArray)) { // write chunk to your favorite device? }
    

should be dataSlice.length * elementSize on line 26.