LouisAmon

import time

import numpy
import redis

# a 2D array to serialize
A = 10 * numpy.random.randn(10000).reshape(1000, 10)

# flatten the 2D NumPy array and save it as a binary string

LouisAmon

(jd@XXX) /home/jd/shovel> time ./shovel.py jd_test
got 901 messages

real    0m1.617s
user    0m0.558s
sys     0m0.044s

LouisAmon

(jd@XXX) /home/jd/shovel> time ./shovel.py jd_test
got 901 messages

real    0m1.617s
user    0m0.558s
sys     0m0.044s

LouisAmon

def pandas_to_elasticsearch(df):
    """
    Generator that converts `pandas.DataFrame` rows into ElasticSearch actions
    """
    # ElasticSearch will raise an exception on `NaN` values
    df = df.dropna()
    for i, row in df.iterrows():
        action = {
            '_op_type': 'index',
            '_index': 'my_index',

LouisAmon

import pandas
import fastavro

def avro_df(filepath, encoding):
    # Open file stream
    with open(filepath, encoding) as fp:
        # Configure Avro reader
        reader = fastavro.reader(fp)
        # Load records in memory
        records = [r for r in reader]