tpjfern03

# -*- coding: utf-8 -*-
"""
LICENSE: BSD (same as pandas)
example use of pandas with oracle mysql postgresql sqlite
    - updated 9/18/2012 with better column name handling; couple of bug fixes.
    - used ~20 times for various ETL jobs.  Mostly MySQL, but some Oracle. 

    to do:  
            save/restore index (how to check table existence? just do select count(*)?), 
            finish odbc, 

tpjfern03

#List unique values in a DataFrame column
pd.unique(df.column_name.ravel())

#Convert Series datatype to numeric, getting rid of any non-numeric values
df['col'] = df['col'].astype(str).convert_objects(convert_numeric=True)

#Grab DataFrame rows where column has certain values
valuelist = ['value1', 'value2', 'value3']
df = df[df.column.isin(value_list)]

tpjfern03

"""
tiny script to convert a pandas data frame into a JSON object
"""

import ujson as json
import pandas
import numpy as np

df = pandas.DataFrame({
    "time" : [1,2,3,4,5],

tpjfern03

# Program:  extractUDMdata.py
# Description: This is a program to extract the customer UDM from a formatted Excel file
#
# Author: Terence Fernandes
# Date: 03/16/2018
# Version:  1.0
# pyinstaller extractUDMdata.py --add-data=logging.yaml:. --add-data=extractUDMdata.ini:.  --add-data=udmFile.xlsx:.
# OR pyinstaller -F extractUDMdata.py --hidden-import=_cffi_backend
# --hidden-import=_cffi_backend
# pyi_rth_qt4plugins.py

tpjfern03

version: 1
disable_existing_loggers: False
formatters:
    simple:
        format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"

handlers:
    console:
        class: logging.StreamHandler
        level: DEBUG

tpjfern03

LOAD DATA
INFILE csvDevices.csv
append
INTO TABLE UTIL_DEVICE
fields terminated by '^' optionally enclosed by '"'
trailing nullcols
(device_name "trim(:device_name)",
device_type "trim(:device_type)",
sub_station_abbr "trim(:sub_station_abbr)",
device_id "DEVICE_ID_SEQ.NEXTVAL" )

tpjfern03

#!/usr/bin/env bash


export DATE_0=`date +'%m/%d/%Y'`
export TIMESTAMP=`date +'%Y%m%d%H%M%S'`


PROGNAME=$(basename $0)
error_exit()
{

tpjfern03

echo "Loading data....." 
sqlldr schema/pw@sid control=hdc_stage_points.ctl rows=10000 bad=hdc_stage_points.bad log=hdc_stage_points.log errors=100000

# Wait until the data is loaded & log file is available
cat hdc_stage_points.log | grep "Rows successfully loaded." > temp.log
IFS=" "
while read -r records b c d; do
    export RC=$records
    #echo "${RC}"
done < temp.log

tpjfern03

from acsprism import vde
from acsprism import rtdb_init, rtdb_station_list, rtdb_point_list, RtdbPoint

# Open the file for a status station
# status_file = vde.open_status_telem(10, 'R', 'S', read_only=False)
#records = status_record.record_count()
#logger.info("Reading %d records: station=%d, category=%s, type=%s from rtdb." % (records, stn, cat, pointType))

for station, station_name in rtdb_station_list(names=True):
   for cat in categoryList:

tpjfern03

rtdb_init()
# Open the file for a status station
# status_file = vde.open_status_telem(10, 'R', 'S', read_only=False)
for station, station_name in rtdb_station_list(names=True):
    if station in stationList:
        if station_name != '':
            for cat in categoryList:
                for pt in pointTypeList:
                    # print('Listing points for ' + str(station) + '/' + cat + '/' + pt)
                    # Open the vde file...