Skip to content

Instantly share code, notes, and snippets.

@0x1b-xyz
Created October 10, 2014 16:58
Show Gist options
  • Save 0x1b-xyz/95b42a399a4ee38a4c77 to your computer and use it in GitHub Desktop.
Save 0x1b-xyz/95b42a399a4ee38a4c77 to your computer and use it in GitHub Desktop.
Read/write avro dynamic
/**
* Writes the {@link Encounter}s into the specified avro file.
*/
void write(List<Encounter> encounters, Path output) {
AvroKeyValue<EncounterKey, Encounter> kvRecord =
new AvroKeyValue<EncounterKey, Encounter>(new GenericData.Record(schemaEncounterKv))
DataFileWriter<GenericRecord> fileWriter =
new DataFileWriter<GenericRecord>(dataModel.createDatumWriter(schemaEncounterKv))
FSDataOutputStream fsDataOutputStream = fileSystem.create(output, true)
fileWriter.create(schemaEncounterKv, fsDataOutputStream)
try {
encounters.each {
kvRecord.setKey(new EncounterKey(it.facilityCode.value, it.patNum.value))
kvRecord.setValue(it)
fileWriter.append(kvRecord.get())
}
} finally {
fileWriter.close()
}
}
/**
* Reads the {@link Encounter} instances from an avro file. Not efficient, only for testing.
*/
Map<EncounterKey, Encounter> read(Path input) {
SeekableInput fsInput = new FsInput(input, configuration)
GenericData.Record genericRecord = new GenericData.Record(schemaEncounterKv)
AvroKeyValue<EncounterKey, Encounter> kvRecord = new AvroKeyValue<EncounterKey, Encounter>(genericRecord)
DataFileReader<GenericRecord> fileReader = new DataFileReader<GenericRecord>(fsInput, datumReader)
Map<EncounterKey, Encounter> results = [:]
try {
while (fileReader.hasNext()) {
fileReader.next(genericRecord)
results[kvRecord.key] = kvRecord.value
}
} finally {
fileReader.close()
}
results
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment