Skip to content

Instantly share code, notes, and snippets.

@kkirsanov
Created June 1, 2017 16:04
Show Gist options
  • Save kkirsanov/8f685a4c3992bd8ff3065b092b3d7773 to your computer and use it in GitHub Desktop.
Save kkirsanov/8f685a4c3992bd8ff3065b092b3d7773 to your computer and use it in GitHub Desktop.
from fastavro import writer, reader
schema = {
'doc': 'A weather reading.',
'name': 'Weather',
'namespace': 'test',
'type': 'record',
'fields': [
{'name': 'station', 'type': 'string'},
{'name': 'time', 'type': 'long'},
{'name': 'temp', 'type': 'int'},
],
}
# 'records' can be any iterable (including a generator)
def datagen(cnt):
for x in range(cnt):
yield {u'station': u'011990-99999', u'temp': 0, u'time': 1433269388}
#
with open('weather.avro', 'wb') as out:
writer(out, schema, datagen(100000))
with open('weather.avro', 'rb') as fo:
reader = reader(fo)
schema = reader.schema
for record in reader:
pass
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment