Created
October 29, 2013 23:59
-
-
Save massie/7224868 to your computer and use it in GitHub Desktop.
Here's an example of how to embed Avro objects into a Kryo stream. You only need to register each Avro Specific class in the KryoRegistrator using the AvroSerializer class below and you're ready to go.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
* Copyright (c) 2013. Regents of the University of California | |
* | |
* Licensed under the Apache License, Version 2.0 (the "License"); | |
* you may not use this file except in compliance with the License. | |
* You may obtain a copy of the License at | |
* | |
* http://www.apache.org/licenses/LICENSE-2.0 | |
* | |
* Unless required by applicable law or agreed to in writing, software | |
* distributed under the License is distributed on an "AS IS" BASIS, | |
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
* See the License for the specific language governing permissions and | |
* limitations under the License. | |
*/ | |
package edu.berkeley.cs.amplab.adam.serialization | |
import org.apache.avro.specific.{SpecificDatumWriter, SpecificDatumReader, SpecificRecord} | |
import com.esotericsoftware.kryo.{Kryo, Serializer} | |
import com.esotericsoftware.kryo.io.{Input, Output} | |
import org.apache.avro.io.{BinaryDecoder, DecoderFactory, BinaryEncoder, EncoderFactory} | |
import spark.KryoRegistrator | |
import edu.berkeley.cs.amplab.adam.avro.{ADAMGenotype, ADAMPileup, ADAMRecord} | |
import edu.berkeley.cs.amplab.adam.models.{MatedReferencePositionSerializer, MatedReferencePosition, ReferencePositionSerializer, ReferencePosition} | |
class AvroSerializer[T <: SpecificRecord : ClassManifest] extends Serializer[T] { | |
val reader = new SpecificDatumReader[T](classManifest[T].erasure.asInstanceOf[Class[T]]) | |
val writer = new SpecificDatumWriter[T](classManifest[T].erasure.asInstanceOf[Class[T]]) | |
var encoder = null.asInstanceOf[BinaryEncoder] | |
var decoder = null.asInstanceOf[BinaryDecoder] | |
setAcceptsNull(false) | |
def write(kryo: Kryo, output: Output, record: T) = { | |
encoder = EncoderFactory.get().directBinaryEncoder(output, encoder) | |
writer.write(record, encoder) | |
} | |
def read(kryo: Kryo, input: Input, klazz: Class[T]): T = this.synchronized { | |
decoder = DecoderFactory.get().directBinaryDecoder(input, decoder) | |
reader.read(null.asInstanceOf[T], decoder) | |
} | |
} | |
class AdamKryoRegistrator extends KryoRegistrator { | |
override def registerClasses(kryo: Kryo) { | |
kryo.register(classOf[ADAMRecord], new AvroSerializer[ADAMRecord]()) | |
kryo.register(classOf[ADAMPileup], new AvroSerializer[ADAMPileup]()) | |
kryo.register(classOf[ADAMGenotype], new AvroSerializer[ADAMGenotype]()) | |
kryo.register(classOf[ReferencePosition], new ReferencePositionSerializer) | |
kryo.register(classOf[MatedReferencePosition], new MatedReferencePositionSerializer) | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment