Skip to content

Instantly share code, notes, and snippets.

View hartym's full-sized avatar
👽
Crafting great software

Romain Dorgueil hartym

👽
Crafting great software
View GitHub Profile
@hartym
hartym / transform.py
Last active March 25, 2017 10:26
Bonobo - Step 2 - Transform the data
def my_transform(i: int) -> str:
yield str(i)
@hartym
hartym / load.py
Created March 25, 2017 10:27
Bonobo - Step 3 - Load the data
def my_load(s: str):
with open('/tmp/output.txt', 'a+') as f:
f.write(s+'\n')
@hartym
hartym / execute.py
Created March 25, 2017 10:29
Bonobo - Step 4 - Create your graph and execute it
import bonobo
graph = bonobo.Graph(
my_extract,
my_transform,
my_load,
)
if __name__ == '__main__'
bonobo.run(graph)
import argparse
import datetime
import os
import time
import bonobo
from atomicwrites import atomic_write
from bonobo.constants import NOT_MODIFIED
from fs.osfs import OSFS
@hartym
hartym / hello_mondrian.py
Last active November 6, 2017 14:48
Hello, Mondrian
import logging
import mondrian
# One line setup (excepthook=True tells mondrian to handle uncaught exceptions)
mondrian.setup(excepthook=True)
# Use logging, as usual.
logger = logging.getLogger()
logger.setLevel(logging.INFO)
@hartym
hartym / example_job.py
Created November 11, 2017 18:43 — forked from cw-andrews/example_job.py
CSV Sanitizer Bonobo ETL Job with Atomic Writes
import argparse
from time import sleep
import os
import re
import bonobo
from atomicwrites import atomic_write
from fs.osfs import OSFS
from texools.files import new_file_check
@hartym
hartym / default.py
Created November 29, 2017 10:16
bonobo 0.6 - default.py
import bonobo
def get_graph(**options):
graph = bonobo.Graph()
graph.add_chain(...)
return graph
def get_services(**options):
return {}
import bonobo
from bonobo.config import ContextProcessor, use_context
from bonobo.constants import NOT_MODIFIED
def extract():
yield 'hello'
yield 'world'
{
"name": "my-project",
"dependencies": {
"some-dep": "file:$(python -c 'import os, some_dep; print(os.path.dirname(some_dep.__file__))')/"
}
}
import bonobo
from bonobo.util.bags import BagType
from bonobo_quirksmode.config import set_output_type
def generate():
yield from ('a', 'b', 'c')
LetterBag = BagType('LetterBag', ['letter'])