jplsightm

# coding: utf-8

# In[12]:

import pymongo
from datetime import timedelta


# In[4]:

jplsightm

### Keybase proof

I hereby claim:

  * I am jplsightm on github.
  * I am sightmjpl (https://keybase.io/sightmjpl) on keybase.
  * I have a public key ASBIFMlk-Tq7bpp2utgHxTy8gqzu_igdfm242gCs-FwAdwo

To claim this, I am signing this object:

jplsightm

sudo apt install -y ssh
sudo apt install -y zsh
sudo apt install -y byobu
sudo apt-get -y update
sudo apt-get install -y \
    apt-transport-https \
    ca-certificates \
    curl \
    software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

jplsightm

import pandas as pd
from tabulate import tabulate

def pandas_df_to_markdown_table(df):
    # Dependent upon ipython
    # shamelessly stolen from https://stackoverflow.com/questions/33181846/programmatically-convert-pandas-dataframe-to-markdown-table
    from IPython.display import Markdown, display
    fmt = ['---' for i in range(len(df.columns))]
    df_fmt = pd.DataFrame([fmt], columns=df.columns)
    df_formatted = pd.concat([df_fmt, df])

jplsightm

from datetime import datetime, timedelta
import pandas as pd

def get_fw(date, fiscal_start=datetime(1970, 1,1), calendar_day=False):
    """
    Obtain fiscal week from a datetime object.
    
    :fiscal_start: Indicate the start of a fiscal year
    :calendar_day: If False the first full week is Week 1.
    """

jplsightm

def process_files(path, extention, func, *args, **kwargs):
    """
    Take a directory of files and apply a function to those files.
    
    The first parameter of the function (`func`) must be a file name. This is typically 
    the file to parse to df before apply some function
    """
    dfs = {}
    
    for fname in os.listdir(path):

jplsightm

import pandas as pd

def step_back_ts(frame, ts_col, shift):
    timestamps = pd.DataFrame(frame[ts_col].iloc[shift:], columns=['timestamp'])
    timestamps.reset_index(inplace=True, drop=True)
    for i in range(shift):
        timestamps.loc[len(timestamps)+1, 'timestamp'] = np.nan
    return timestamps

jplsightm

import pandas as pd

def sensor_csv(frame, sensor_name, sensor_column, prefix, keep_columns, timestamp):
    """
    frame = input frame
    sensor_name = sensor name to filter on
    sensor_column = column that contains the sensor name
    prefix = prefilx to add to column (make themn unique)
    keep_columns = what columns should be kept .... I am not doing any checks on data types, make this a list
    timestamp = timestamp column

jplsightm

import pymongo

# get mongo sslog ids and ORDERIDS
ORDERIDs = {log['_id']: log['data']['fieldvalues']['ORDERID']['value']
            for log in sslog.find({'data.fieldvalues.ORDERID.value': {'$exists': True}}, {'data.fieldvalues.ORDERID.value': 1})}

# do some parsing because there was all sorts of badness - floats cast as strings, integers, etc
def order_id_to_string(_id, orderid):
    try: 
        orderid = str(int(float(orderid)))

jplsightm

def mongo_objs(conn_str, database, tests=[lambda x: x.document_count({})]):
    client = pymongo.MongoClient(conn_str)
    db = client[database]
    sslog = db.sslog
    cycle = db.cycle
    sslog_results = {}
    cycle_results = {}
    for test in tests:
        try:
            sslog_results[test.__name__] = test(sslog)