Created
January 26, 2017 18:42
-
-
Save defndaines/fbd17db3b55919e9eb2074b03c2e5f1a to your computer and use it in GitHub Desktop.
Scala Avro encoding and decoding to bytes without using additional libraries (From Confluent or Twitter)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.io.ByteArrayOutputStream | |
import org.apache.avro.Schema | |
import org.apache.avro.io.{BinaryDecoder, BinaryEncoder, DatumReader, DatumWriter, DecoderFactory, EncoderFactory} | |
import org.apache.avro.reflect.{ReflectDatumReader, ReflectDatumWriter} | |
import org.apache.avro.specific.SpecificRecordBase | |
class AvroCodec[A <: SpecificRecordBase](schema: Schema) { | |
val decoderFactory: DecoderFactory = DecoderFactory.get() | |
val avroReader: DatumReader[A] = new ReflectDatumReader[A](schema) | |
val encoderFactory: EncoderFactory = EncoderFactory.get() | |
val avroWriter: DatumWriter[A] = new ReflectDatumWriter[A](schema) | |
def decode(bytes: Array[Byte], reuseDecoder: Option[BinaryDecoder] = Option.empty): A = { | |
val decoder = decoderFactory.binaryDecoder(bytes, reuseDecoder.orNull) | |
avroReader.read(null.asInstanceOf[A], decoder) // scalastyle:ignore null | |
} | |
def encode(bytes: A, reuseEncoder: Option[BinaryEncoder] = Option.empty): Array[Byte] = { | |
val stream = new ByteArrayOutputStream | |
val binaryEncoder = encoderFactory.binaryEncoder(stream, reuseEncoder.orNull) | |
avroWriter.write(bytes, binaryEncoder) | |
binaryEncoder.flush() | |
stream.toByteArray | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I had trouble tracking down example code for directly converting to or from Avro when using Kafka. Many of the examples use either Confluent's library, which I believe depends upon using schema-registry already, or Twitter's Bijection library. I'm not saying to avoid those libraries, but I wanted raw encoding and decoding to prove some stuff out.