JD-V

from __future__ import print_function
import sys
import ssl
import time
import datetime
import logging, traceback
import paho.mqtt.client as mqtt

IoT_protocol_name = "x-amzn-mqtt-ca"
aws_iot_endpoint = "iot.us-east-1.amazonaws.com" # <random>.iot.<region>.amazonaws.com

JD-V

create setup.py
user pip freeze to list all the depndencies and then add it to setup.py

activate virtual env
source ./bin/activate


to run dataflow 
python df.py --project <gcp_proj_name> --input_topic projects/cldcvr-dev/topics/<topic-name> --runner DataflowRunner --temp_location gs://<bucket location for dumpig backup>/ --setup_file /Users/jd-v/datflow/setup.py --job_name <name_that_will_show_up_in_gcp>

JD-V

import pandas as pd
import boto3

def splitandpushtos3(x):
    #store file locally
    x.to_csv(r'{}.csv'.format(x.name))
    #push to s3 (inorder to use boto you need to have aws-cli configured)
    s3 = boto3.resource('s3')
    s3.Bucket('my-bucket').upload_file(r'{}.csv'.format(x.name), r'{}.csv'.format(x.name))