Skip to content

Instantly share code, notes, and snippets.

View rmax's full-sized avatar
:octocat:
ヾ(⌐■_■)ノ♪

R Max Espinoza rmax

:octocat:
ヾ(⌐■_■)ノ♪
View GitHub Profile
@rmax
rmax / dask_avro.py
Last active September 17, 2018 19:28
An Avro reader for Dask (with fastavro)
"""A fastavro-based avro reader for Dask.
Disclaimer: This code was recovered from dask's distributed project.
"""
import io
import fastavro
import json
from dask import delayed
@rmax
rmax / dask_elasticsearch.py
Last active May 3, 2018 13:51
An Elasticsearch reader for Dask
from dask import delayed
from elasticsearch import Elasticsearch
from elasticsearch.helpers import scan
def read_elasticsearch(query=None, npartitions=8, client_cls=None,
client_kwargs=None, **kwargs):
"""Reads documents from Elasticsearch.
By default, documents are sorted by ``_doc``. For more information see the
@rmax
rmax / elasticsearch.yml
Created January 28, 2017 04:27 — forked from reyjrar/elasticsearch.yml
ElasticSearch config for a write-heavy cluster
##################################################################
# /etc/elasticsearch/elasticsearch.yml
#
# Base configuration for a write heavy cluster
#
# Cluster / Node Basics
cluster.name: logng
# Node can have abritrary attributes we can use for routing
from js2xml import parse, pretty_print as tostring
from js2xml.jsonlike import make_dict, getall as get_json_objs
MAKE_DICT_TYPES = (
# Types that can be handled by make_dict.
'array',
'object',
'property',
'string',
@rmax
rmax / entropy.py
Created January 10, 2020 14:26
Entropy experiment
"""Entropy experiment."""
from dataclasses import dataclass
from math import inf
from secrets import randbits
DEFAULT_STEP: int = 64 # shall we use bigger step?
@dataclass
class Entropy: