Code to process durafet pH data

read_durafet.py

# -*- coding: utf-8 -*-
"""
:DESCRIPTION:

:REQUIRES:

:TODO:

:AUTHOR: John Franco Saraceno
:ORGANIZATION: U.S. Geological Survey, United States Department of Interior
:CONTACT: [email protected]
:VERSION: 0.1
Wed Aug 10 14:45:07 2016
"""
# =============================================================================
# IMPORT STATEMENTS
# =============================================================================
import glob
import os

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
# =============================================================================
# METHODS
# =============================================================================


def rd_durafet(filename):
    # df.convert_objects(convert_numeric=True)
    # filename = os.sep.join([directory, 'Durafet_Raw_Burst_2016-07-05.csv'])
    df = pd.read_csv(filename, parse_dates=[[0, 1]], index_col=0,
                     na_values=['NAN', np.nan])

    return df


def mVtopH(pHvoltage, Thermovoltage):
    #     ThermoVoltage_med=mean['Thermovoltage']
    #     pHVoltage_med=mean['pHvoltage']
    pHCalVoltage = 0
    pHCalpH = 7.01
    pHCalTemp = 295.24  # in K
    ThermoRes = ((10000*6018)/Thermovoltage) - 10000
    # converts resistance to temperature
    ThermoTemp = 62.297563875548043*np.exp(-0.000106588369686*ThermoRes)
    Eo = pHCalVoltage - pHCalpH * 8.31415 * pHCalTemp * np.log(10)/96487
    ST = 8.31415 * (ThermoTemp + 273.15)/96487 * np.log(10)
    EoT = Eo - 0.001 * (ThermoTemp - (pHCalTemp - 273.15))
    ph_Isfet_med = (pHvoltage/1000 - EoT)/ST
    return ph_Isfet_med


def build_burst(directory, fmatch):
    """Function to build a dataframe from all similair toa files in a directory
    that match fmatch using readtoa"""
    files = []
    for name in glob.glob(os.path.join(directory, fmatch)):
        if os.path.isfile(os.path.join(directory, name)):
            files.append(name)
    #    print "{} logger files appended".format(len(files))
    frame = pd.concat((rd_durafet(os.path.join(directory, f)) for f in files))
    # drop duplicate indices
    frame = frame[~frame.index.duplicated()]
    # sort by index
    frame.sort_index(axis=0, level=None, ascending=True, inplace=True,)
    # kind='quicksort', na_position='last', sort_remaining=True, by=None)
    frame.index = pd.DatetimeIndex(frame.index)
    # drop duplicate/stuck values
    #    frame.drop_duplicates(keep='first', inplace=True)
    return frame
# =============================================================================
# MAIN METHOD AND TESTING AREA
# =============================================================================

directory = r'C:\Campbellsci\LoggerNet\durafet'
fmatch = 'Durafet*.csv'
durafet_frame = build_burst(directory, fmatch)
durafet_frame['pH'] = mVtopH(durafet_frame['pHvoltage'],
                             durafet_frame['Thermovoltage'])
fig = plt.figure()
ax = fig.gca()
ax.set_ylabel('pH')
ax.set_ylim(0, 14)
durafet_frame['pH'].plot(ax=ax)