mathiasbc

# Proposal: Medallion ETL pipeline for Beyond analytics

## Context

Our analytics queries run against tables fed by ad-hoc scripts. Every source
has its own loading logic, nothing is replayable, and when a dashboard number
looks off we basically do archaeology. Data volume is growing fast
and this approach doesn't survive it.

This proposal: one batch pipeline, three-layer medallion on S3, Spark for

mathiasbc

import json
from flask import Flask, make_response
import datetime
from bson.objectid import ObjectId
from bson.tz_util import utc
from bson import EPOCH_AWARE, SON
from bson.py3compat import iteritems
from bson.py3compat import text_type

mathiasbc

import re

l = [[1,2,[3]],4]

print([int(x) for x in re.findall('[0-9]+', str(l))])

mathiasbc

#!/usr/bin/python3

import requests

URL = "http://www.letsrevolutionizetesting.com/challenge.json"


def follow(url):
	response = requests.get(url).json()
	while 'follow' in response:

mathiasbc

from sqlalchemy import MetaData, create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base

import datetime

# development database credentials
DEV_DB_USER = 'some_user'
DEV_DB_PASSWORD = 'some_password'
DEV_DB_DATABASE = 'database_name'

mathiasbc

{% macro form_field(field) -%}
    {% set with_label = kwargs.pop('with_label', False) %}
    {% set placeholder = '' %}
    {% if not with_label %}
        {% set placeholder = field.label.text %}
    {% endif %}
    <div class="control-group {% if field.errors %}error{% endif %}">
        {% if with_label %}
            <label for="{{ field.id }}" class="control-label">
                {{ field.label.text }}{% if field.flags.required %} *{% endif %}: