andrewm4894

cd handle-github-errors
npm i -D serverless-dotenv-plugin
npm install

andrewm4894

npm install serverless -g
serverless install -u https://github.com/serverless/examples/tree/master/google-python-simple-http-endpoint -n handle-github-events

andrewm4894

service: handle-github-events

frameworkVersion: ">=1.2.0 <2.0.0"

package:
  exclude:
    - node_modules/**
    - .gitignore
    - .git/**

andrewm4894

"""
GCP HTTP Cloud Function to handle github webhook events.

Some code stolen from here: https://github.com/carlos-jenkins/python-github-webhooks/blob/master/webhooks.py

"""
# -*- coding: utf-8 -*-
import hmac
import json
import datetime

andrewm4894

ARG BASE_CONTAINER=gcr.io/kubeflow-images-public/tensorflow-1.13.1-notebook-cpu:v0.5.0
FROM $BASE_CONTAINER

LABEL maintainer="[email protected]"
LABEL version="01"

RUN pip3 install git+https://github.com/andrewm4894/my_utils.git#egg=my_utils

RUN pip3 install kfp --upgrade

andrewm4894

service: serverless-learn-lambda

provider:
  name: aws
  runtime: python3.6
  region: us-west-2
  stage: dev  
  role: arn:aws:iam::XXX:role/serverless-lambda

functions:

andrewm4894

import json
from my_utils.os_utils import subprocess_execute
from my_dev.dev import hello_world
import pandas as pd


def run(event=dict(), context=dict()):
    ''' Function to be called by serverless lambda
    '''
    # make a dummy df to ensure pandas available to the lambda function

andrewm4894

ARG BASE_CONTAINER=jupyter/scipy-notebook
FROM $BASE_CONTAINER

LABEL maintainer="[email protected]"
LABEL version="01"

USER $NB_UID

# install specific package versions i want to use here
RUN conda install --quiet --yes \

andrewm4894

from multiprocessing import  Pool
from functools import partial
import numpy as np 
import pandas as pd


def parallelize_dataframe(df, func, n_pool=4, col_subsets=None, join_how='outer',**kwargs):
    ''' 
    Function to take a df, a function with args, and a list of column subsets to apply function to. 
    Resulting list of df's are then joined back together based on the df index.

andrewm4894

# this does not work
pipeline = {
    {"resample" : {"type" : "trans", "name" : "resample", "kwargs" : {"rule" : "1min"}}}
}
pipeline
'''
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-10-7305ba79e664> in <module>
      1 pipeline = {