Skip to content

Instantly share code, notes, and snippets.

View cicdw's full-sized avatar

Chris White cicdw

View GitHub Profile
@cicdw
cicdw / add_prefect.md
Last active February 22, 2019 20:20
Diff after adding in Prefect code
- import os
...
+ import prefect
+ from prefect import Flow, Parameter, task
+ from prefect.client import Secret
+ from prefect.schedules import CronSchedule

+ @task
def get_standup_date():
@cicdw
cicdw / post_standup.py
Last active July 30, 2019 04:26
Non-Prefect pieces of the post standup workflow
import datetime
from google.cloud.firestore import Client
import os
import random
import requests
def get_standup_date():
"""
Returns the current date, formatted, which maps

Test

$$ \langle X, X + Y \rangle \leq 2 $$

@cicdw
cicdw / dask_timing.py
Last active November 10, 2017 17:06
Code snippets for Medium blog post
%%timeit -n 100
_, _ = dask.compute(results['w_overlay'],
results['predict_another_thing'])
# 100 loops, best of 3: 1.61 ms per loop
@cicdw
cicdw / dask_task_graph_example.py
Created November 10, 2017 16:11
Code snippets for Medium blog post
@delayed
def predict_another_thing(x, y, z):
# model scoring code goes here
pass
@delayed
def complicated_feature_b(x, y):
# lets imagine this feature is *very* expensive to create
# and takes a full minute to process
@cicdw
cicdw / dask_delayed_example.py
Last active November 10, 2017 16:10
Code snippets for Medium blog post
from dask import delayed
# won't be evaluated until we call .compute()
fast_predict_over_time = delayed(fast_predict_over_time)
## using the same numpy arrays from above...
%%timeit -n 100
_ = fast_predict_over_time(x, y, z, False, res).compute()
@cicdw
cicdw / calling_guvectorize.py
Created November 10, 2017 16:07
Code snippets for Medium blog post
res = np.zeros((n, 15))
%%timeit -n 100
_ = fast_predict_over_time(x, y, z, False, res) # 100 loops, best of 3: 575 µs per loop
@cicdw
cicdw / numba_guvectorize_example.py
Created November 10, 2017 16:06
Code snippets for Medium blog post
from numba import guvectorize
@guvectorize('i8, f8, f8, b1, f8[:], f8[:]',
'(), (), (), (), (s) -> (s)')
def fast_predict_over_time(x, y, z, overlay, _, out):
adj = 1.5 if overlay else 1.0
for t in range(len(out)):
out[t] = adj * (t * x ** 2 + y - 2 * z - 2 * t)
@cicdw
cicdw / numba_jit_example.py
Last active November 10, 2017 16:37
Code Snippets for Medium blog post
from numba import jit
@jit
def jitted_func(x, y, z, overlay=False):
"Predicts a quantity at times = 0, 1, ... 14"
out = np.zeros((x.shape[0], 15))
for t in range(15):
out[:, t] = t * x ** 2 + y - 2 * z - 2 * t
adj = 1.5 if overlay else 1.0
@cicdw
cicdw / predict_over_time.py
Last active November 10, 2017 15:59
Code Snippets for Medium blog post
def predict_over_time(x, y, z, overlay=False):
"Predicts a quantity at times = 0, 1, ... 14"
out = np.zeros((x.shape[0], 15))
for t in range(15):
out[:, t] = t * x ** 2 + y - 2 * z - 2 * t
adj = 1.5 if overlay else 1.0
return adj * out