A collection of useful commands for data processing nauticraft logs

nauti_note.md

Useful bits of python for Nauticraft Notebooks

from IPython.display import HTML
HTML('''<script>
code_show=true; 
function code_toggle() {
 if (code_show){
 $('div.input').hide();
 } else {
 $('div.input').show();
 }
 code_show = !code_show
} 
$( document ).ready(code_toggle);
</script>
The raw code for this IPython notebook is by default hidden for easier reading.
To toggle on/off the raw code, click <a href="javascript:code_toggle()">here</a>.''')

import math
import pandas as pd
from pandas import *
import matplotlib.pyplot as plt
from plotly import __version__
from plotly.offline import download_plotlyjs, init_notebook_mode, iplot
#from plotly.offline import *
from plotly.graph_objs import *

init_notebook_mode()

For Reading and plotting CSV files from VB

nautiData = pd.read_csv(r'./NC_2P8m_160404105822_single acting def damp.bin.tab', delimiter='\t')
print list(nautiData.columns.values)
# create a time array to plot against
logtime = nautiData['logtime']

#Calculate front pitch suspension travel
craft_speed = nautiData['craft_speed']
front_pitch_cyl_travel = (nautiData['flx']+nautiData['rlx'])/2.0
# Conver the FALSE or TRUE to an int so it can be ploted
statusrollpumprightlock = nautiData['statusrollpumprightlock'].astype(int)



# Make scatter trace object with data
trace1 = Scatter(x=logtime, y=craft_speed, name='craft_speed', mode='lines', yaxis='y')
trace2 = Scatter(x=logtime, y=front_pitch_cyl_travel, name='front_pitch_cyl_travel',mode='lines', yaxis='y2')
# trace3 = Scatter(x=logtime, y=roll_mode, name='roll_mode', mode='lines', yaxis='y3')
# trace4 = Scatter(x=logtime, y=roll_valve_flow, name='roll_db_flow_bottom', mode='lines', yaxis='y2')

data = Data([trace1, trace2]) #, trace2, trace3, trace4])

# Construct the Layout object
xaxis = XAxis(title='Seconds')
yaxis = YAxis(title='speed knots')
yaxis2=YAxis(title='avg front cyl travel mm', overlaying='y', side='right')
#yaxis3=YAxis(title='mm', overlaying='y', side='right',  position=0.08)

layout = Layout(title='cylinder movment during 2play single acting damping test against seas',
                xaxis1=xaxis,
                yaxis = yaxis,
                yaxis2 = yaxis2)
                #yaxis3 = yaxis3
                #)

fig = Figure(data=data, layout=layout)

iplot(fig)

from __future__ import division
from sympy import *

# For printing sympy objects
from IPython.display import display
# This changes the output of display to use mathjax which displays 
# nice math objects (looks like latex).
init_printing(use_latex='mathjax')
%matplotlib inline

from sympy import symbols
from sympy.plotting import plot

x = symbols('x')
 
# It limits the value range of y axis of the graph.
plot(1/(x - 2), (x, 0, 4), ylim=(-10, 10))

s,z,T = symbols('s z T')
#simplify(3*(x+3))
#s = 2*(1-1/z)/(T*(1/z+1))
s = 2*(1-1/z)/(0.04*(1/z+1))
display(s)

CSV CAN logging processing functions

nautiData = pd.read_csv(r'./NC_2P8m_170830013413_acc test 2play water test 1.bin_log.csv', delimiter=',')
print("Stored Variables are: \n", list(nautiData.columns.values))
print(nautiData[' Signal'].unique())

var_names = ['sen_rlx', 'sen_flx', 'cpu_usage']
plot_data = nautiData.loc[(nautiData[' Signal'].isin(var_names))]
#print(plot_data)

#pd_flx = plot_data.loc[(nautiData[' Signal'] == 'sen_flx')]
#print(pd_flx[' Value'].values)
#print(pd_flx['Logtime'].values)

# Plot each of the variables in the var_names list
data = [
    Scatter(
        x=plot_data.loc[(nautiData[' Signal'] == var_names[i])]['Logtime'].values, # assign x as the dataframe column 'x'
        y=plot_data.loc[(nautiData[' Signal'] == var_names[i])][' Value'].values,
        name=var_names[i] #,type='scattergl' # use for very large plots
        #yaxis="y{x}".format(x=(i) if i!=0 else "")
    ) for i in range(len(var_names))
]

# Set the layout
xaxis = XAxis(title='Seconds')
layout = Layout(title='2play Signals',
                xaxis1=xaxis
                )

# Set the number of yaxis
yaxis = [YAxis(title='mm'), 
         YAxis(title='mm2', overlaying='y', side='right',  position=0.94), 
         YAxis(title='%', overlaying='y', side='right',  position=0.9)]
layout.yaxis = yaxis[0]
layout.yaxis2 = yaxis[1]
# Set the plots to specfic yaxis numbers
data[0].yaxis = 'y'
data[1].yaxis = 'y'
data[2].yaxis = 'y2'

fig = Figure(data=data, layout=layout)
iplot(fig)

Old more verbose method of plotting

df_accz_row = nautiData[(nautiData[' Signal'] =='sen_accz')]
sen_accz = df_accz_row[' Value'].values
print sen_accz
#sen_accz = df_accz_row[5,:]
sen_accz_logtime = df_accz_row[' Logtime'].values
print len(sen_accz)

# Make scatter trace object with data
trace1 = Scatter(x=sen_accz_logtime, y=sen_accz, name='sen_accz', mode='lines', yaxis='y')
data = Data([trace1])
# # Construct the Layout object
xaxis = XAxis(title='Seconds')
yaxis = YAxis(title='bool')
yaxis2=YAxis(title='bool', overlaying='y', side='left', position=0.07)
yaxis3=YAxis(title='db travel mm', overlaying='y', side='right',  position=0.94)
yaxis4=YAxis(title='roll angle degs', overlaying='y', side='right',  position=0.88)

yaxis5=YAxis(title='knots', overlaying='y', side='right',  position=0.88)
yaxis6=YAxis(title='roll angle degs', overlaying='y', side='right',  position=0.88)

layout = Layout(title='Acceleration Z 2play 100Hz Gbox',
                xaxis1=xaxis,
                yaxis = yaxis                
                )

fig = Figure(data=data, layout=layout)

iplot(fig)

Select Data from dataframe, edit the data and save as a csv file

#nautiData.where()
trim_data = nautiData.loc[(nautiData['Logtime'].values >= 3378) & (nautiData['Logtime'].values <= 3435)]
trim_data
# Set the start of the log time back to zero.
#trim_data['Logtime'] = trim_data.loc[:,('Logtime')].apply(lambda x: x-3373)
print(trim_data.columns.values)
print(trim_data.loc[:,'Logtime'].values)
trim_data.loc[:,'Logtime'] = trim_data.loc[:,'Logtime'].apply(lambda x: x-3373)
trim_data.to_csv('trimmed_OperationalLog_01.csv', sep=',', index=False)

# Print the dataframe as a table
# Set the max number of rows and columns to display when printing.
with pd.option_context('display.max_rows', 1000, 'display.max_columns', 5):
    display(trim_data)
    
# Select certain signals to save
signals = ['sen_FL_pos', 'sen_FR_pos']
selected_df=trim_data.loc[nautiData[' Signal'].apply(lambda x: x in signals)]


un_data = nautiData.loc[(nautiData[' Message'].str.contains('C|196|0x80', regex=True)==False)]

with pd.option_context('display.max_rows', 2000, 'display.max_columns', 5):
    display(un_data)

For ploting very large data sets

Uses plotly's FigureWidget
Needs plotly > 3.0.0

import plotly.graph_objs as go

# Needs plotly > 3.0.0


var_names = ['sen_rlx', 'sen_flx', 'sen_speed', 'val_pin_pro', 'sen_pitch']

plot_data = nautiData.loc[(nautiData[' Signal'].isin(var_names))]

# Plot each of the variables in the var_names list
data = [
    Scatter(
        x=plot_data.loc[(nautiData[' Signal'] == var_names[i])]['Logtime'].values, # assign x as the dataframe column 'x'
        y=plot_data.loc[(nautiData[' Signal'] == var_names[i])][' Value'].values,
        name=var_names[i]#,
        #type='scattergl'
        #yaxis="y{x}".format(x=(i) if i!=0 else "")
    ) for i in range(len(var_names))
]


# Set the layout
xaxis = go.layout.XAxis(title='Seconds')
layout = Layout(title='Yanmar Signals'
                )

# Set the number of yaxis
yaxis = [go.layout.YAxis(title='mm'),
         go.layout.YAxis(title='Amps', overlaying='y1', side='left',  position=0.94),
         go.layout.YAxis(title='V', overlaying='y1', side='right',  position=0.9),
         go.layout.YAxis(title='mm', overlaying='y1', side='right',  position=0.8)
         
        ]

for ind in range(len(yaxis)):
    layout['yaxis'+str(ind+1)] = yaxis[ind]
 
# Set the plots to specfic yaxis numbers
data[0].yaxis = 'y1'
data[1].yaxis = 'y4'
data[2].yaxis = 'y3'
data[3].yaxis = 'y2'

fig_widget = go.FigureWidget(data, layout=layout)
fig_widget

Other Plotly plotting types such as Scattergl

import plotly.graph_objs as go

# Needs plotly > 3.0.0


var_names = ['sen_rlx', 'sen_flx', 'sen_speed', 'val_pin_pro', 'sen_pitch']

plot_data = nautiData.loc[(nautiData[' Signal'].isin(var_names))]

# Plot each of the variables in the var_names list
data = [
    Scattergl(
        x=plot_data.loc[(nautiData[' Signal'] == var_names[i])]['Logtime'].values, # assign x as the dataframe column 'x'
        y=plot_data.loc[(nautiData[' Signal'] == var_names[i])][' Value'].values,
        name=var_names[i]#,
        #mode = 'markers
    ) for i in range(len(var_names))
]


# Set the layout
xaxis = go.layout.XAxis(title='Seconds')
layout = Layout(title='Yanmar Signals'
                )

# Set the number of yaxis
yaxis = [go.layout.YAxis(title='mm'),
         go.layout.YAxis(title='Amps', overlaying='y1', side='left',  position=0.94),
         go.layout.YAxis(title='V', overlaying='y1', side='right',  position=0.9),
         go.layout.YAxis(title='mm', overlaying='y1', side='right',  position=0.8)
         
        ]

for ind in range(len(yaxis)):
    layout['yaxis'+str(ind+1)] = yaxis[ind]
 
# Set the plots to specfic yaxis numbers
data[0].yaxis = 'y1'
data[1].yaxis = 'y4'
data[2].yaxis = 'y3'
data[3].yaxis = 'y2'

fig_widget = go.FigureWidget(data, layout=layout)
fig_widget

# To edit dataframe values
E.g to convert the pitch raw can signal form radians to degrees use the following:

pitch_df = nautiData.loc[nautiData[' Signal'] == 'sen_pitch', ' Value']
pitch_df_num = pd.to_numeric(pitch_df)
pitch_df_deg = pitch_df_num.multiply(57.3)

plot_data_new = plot_data.copy()
plot_data_new.update(pitch_df_deg)

plot_data = plot_data_new

print(plot_data)

To Check the max difference in time between CAN messages

signal_name = 'ecu_heartbeat'

signal_val = nautiData.loc[nautiData[' Signal'] == signal_name, ' Value']
signal_val_num = pd.to_numeric(signal_val)
signal_val_time = nautiData.loc[nautiData[' Signal'] == signal_name, 'Logtime']

min_time = signal_val_time.iloc[1]
max_time = signal_val_time.iloc[-1]

num_sigs = len(signal_val_time)
print(min_time, max_time, num_sigs)

avg_rate = (max_time - min_time)/ num_sigs
print("avg rate =", avg_rate)

diff_signal_val_time = np.diff(signal_val_time)

max_diff = np.max(diff_signal_val_time)
print("max_diff time = ", max_diff)


#signal_val_deg = pitch_df_num.multiply(57.3)

#plot_data_new = plot_data.copy()
#plot_data_new.update(pitch_df_deg)

#plot_data = plot_data_new