Created
February 21, 2017 22:18
-
-
Save OlyDLG/0a68d5776bab1b15c16fb61b9f5f396d to your computer and use it in GitHub Desktop.
Samples of Code--Python and MatLab--written for WA St. Dept. of Ecology, 2010-2013
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Telnet acquisition of moorings data | |
| # | |
| # Author: David Goldsmith, Formerly Wash. State Dept. of Ecology | |
| # Original release date: < 9/15/2011 | |
| import sys | |
| import array as A | |
| import datetime as DT | |
| from time import * | |
| from os import linesep | |
| from dateutil.parser import parse as dateparse | |
| from telnetlib import Telnet as T | |
| from socket import timeout | |
| class getdata(object): | |
| def __init__(self, ipport, out, logging, | |
| Max=20, writeOut=1, | |
| prompt=["S>", "<Executed/>"]): | |
| self.host, self.port = ipport | |
| self.out = out | |
| self.logging = logging | |
| self.max = Max | |
| self.prompt = prompt | |
| self.writeOut = writeOut | |
| def write_progress(self, progress_code):#, delay=5): | |
| self.out.write(progress_code + linesep) | |
| self.out.flush() | |
| def getdata(self, hours2sample, offset): | |
| NS = int(4 * hours2sample) # There are 4 samples per hour | |
| offset = int(4 * offset) | |
| attempts = 1 | |
| prompt = self.prompt; Max = self.max | |
| tn = T(self.host, self.port, Max) | |
| tn.write("\r\n") | |
| tries = 0 | |
| idx, ju, nk = tn.expect(prompt, Max) | |
| while (idx < 0) or ((prompt[idx] not in prompt) and (tries < Max)): | |
| # try sending Esc | |
| tn.write(chr(27)) | |
| sleep(5) | |
| tn.write("\r\n") | |
| idx, ju, nk = tn.expect(prompt, Max) | |
| tries += 1 | |
| if tries == Max: | |
| return ("Couldn't get a prompt", None, None, None) | |
| # Send stop, wait for prompt, send ds, wait for "logging" to be in reply | |
| # Note: presence of logging in reply does not guarantee the stop was successful, | |
| # thus the need for the while loop which follows | |
| prompt = prompt[idx] | |
| if self.writeOut: | |
| self.write_progress("Connection established, issuing stop command") | |
| tn.write("stop\r\n") | |
| tn.read_until(prompt, Max) | |
| tn.write("ds\r\n") | |
| ju, nk, up2match = tn.expect(["logging"], Max); | |
| # Get type (16 or 37) | |
| type_determiner = up2match.split('V')[0] | |
| if "16" in type_determiner: | |
| typ = "16" | |
| elif "37" in type_determiner: | |
| typ = "37" | |
| else: | |
| return ("Sorry! Instrument type not recognized!", None, None, None) | |
| if self.writeOut: | |
| self.write_progress(typ) | |
| # Now issue stop followed by ds until "not logging" is in reply, or there | |
| # have been Max unsuccessful attempts | |
| while ("not logging" not in up2match) and (attempts < Max): | |
| attempts += 1 | |
| tn.write("stop\r\n") | |
| tn.read_until(prompt, Max) | |
| tn.write("ds\r\n") | |
| ju, nk, up2match = tn.expect(["logging"], Max) | |
| if attempts >= Max: # if there were too many attempts | |
| return ("Sorry! Couldn't stop logging.", None, None, None) | |
| self.logging = False | |
| if self.writeOut: | |
| self.write_progress("ds returned a status of not logging, proceeding") | |
| tn.write("\r\n") | |
| tn.read_until(prompt, Max); | |
| if self.writeOut: | |
| self.write_progress("Computer (UTC) time: " + strftime('%m-%d-%Y %H:%M:%S', gmtime())) | |
| tn.write("ds\r\n") | |
| ju, nk, tmp = tn.expect(["\d\d\s\w\w\w\s\d\d\d\d", "\d\d-\d\d-\d\d\d\d"], Max) | |
| if self.writeOut: | |
| self.write_progress(tmp) | |
| date = dateparse(tmp[-11:]) | |
| if date is None: | |
| return ("Sorry! dateparse failed!", None, None, None) | |
| tmp = tn.read_until("\r\n", Max) | |
| if self.writeOut: | |
| self.write_progress("Time = " + tmp) | |
| time = dateparse(tmp[-10:-3]) | |
| if "16" in typ: | |
| lookfor = "samples = " | |
| else: | |
| lookfor = "samplenumber = " | |
| data = tn.read_until(lookfor, Max) | |
| if self.writeOut: | |
| self.write_progress(data) | |
| tmp = tn.read_until(',', Max) | |
| if self.writeOut: | |
| self.write_progress(tmp) | |
| ns = int(tmp.strip(','), 10) | |
| # have number of samples, get rest of header... | |
| data = tn.read_until(prompt, Max) | |
| if self.writeOut: | |
| self.write_progress(data) | |
| # ...make sure we're in decimal mode... | |
| if "16" in typ: | |
| fmtcmd = "outputformat=3\r\n" | |
| elif prompt=="S>": | |
| fmtcmd = "format=2\r\n" | |
| else: | |
| fmtcmd = "" | |
| if fmtcmd: | |
| tn.write(fmtcmd) | |
| tn.read_until(prompt, Max) | |
| # ...and get the decimal data | |
| if (ns - offset < 0) or (offset < 0): | |
| raise | |
| range = "dd0," + str(ns-offset) + "\r\n" # this is the default | |
| if ns-offset-NS >= 0: # this is the more likely event | |
| range = "dd" + str(ns-offset-NS+1) + "," + str(ns-offset) + "\r\n" | |
| tn.write(range) | |
| data = tn.read_until(prompt) | |
| if self.writeOut: | |
| self.write_progress(data) | |
| last_dt = data[-25:-7] | |
| # now get hex data | |
| if prompt != "S>": | |
| tn.write("QS\r\n") | |
| return (tn, date, time, last_dt) | |
| elif "16" in typ: | |
| fmtcmd = "outputformat=1\r\n" | |
| else: | |
| fmtcmd = "format=0\r\n" | |
| tn.write(fmtcmd) # change the output format to hex | |
| data = tn.read_until(prompt) | |
| if self.writeOut: | |
| self.write_progress(data) | |
| tn.write(range) | |
| data = tn.read_until(prompt) | |
| if self.writeOut: | |
| self.write_progress(data) | |
| if "16" in typ: | |
| fmtcmd = "outputformat=3\r\n" | |
| else: | |
| fmtcmd = "format=2\r\n" | |
| tn.write(fmtcmd) # change the output format back to dec | |
| data = tn.read_until(prompt) | |
| if self.writeOut: | |
| self.write_progress(data) | |
| return (tn, date, time, last_dt) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ''' | |
| Note: For "code-sample" puproses only; DOES NOT RUN AS IS; | |
| anything enclosed in <> represents redacted identifying information | |
| autoTelnetDriver: "Driver" for telnet acquisition of moorings data | |
| Inputs: | |
| * "site" (required): the eight-character site-code indicating the station | |
| from which data is to be acquired, e.g., MUK01BR | |
| * "hours2sample" (optional): number of hours to "grab"; default is 30 | |
| * "offset" (optional): end-time of the "grab," in hours before "now," i.e., | |
| runtime; default is 0 | |
| Example: if the program is run at 12 PM (local time) with an offset of 24 | |
| and an hours2sample of 12, then the program will (try to) grab all records | |
| from the previous day between 12 AM and 12 PM (local time). | |
| * "writeOut" (optional): whether or not to produce output; default is 1 (True) | |
| (Included for development/debugging purposes, i.e., so that the program | |
| can be run without actually generating a new result file.) | |
| Outputs: None | |
| Globals: None | |
| Local Classes: | |
| * kill_timeout: | |
| Local Subroutines: | |
| * figure_out_restart: | |
| * get_serial_ports: | |
| * kill_serial_ports: | |
| * report_failure: | |
| Purpose: "Drives" the daily acquisition, via telnet, of data from moored CTDs | |
| Workflow: | |
| Author: David Goldsmith, formerly Wash. State Dept. of Ecology | |
| Release date: 10/1/2013 | |
| See also: autogetdata.py | |
| ''' | |
| import os | |
| import os.path as op | |
| import sys | |
| from math import floor | |
| from telnetlib import Telnet as T | |
| from socket import timeout | |
| from smtplib import SMTP | |
| import autogetdata as GD | |
| from time import * | |
| import datetime as DT | |
| # Global constants & initializations | |
| logging = True | |
| prompt = "S>" | |
| prompt23 = "#>" | |
| Max = 10 | |
| progress_code = '' | |
| IPsPORTs = {"MUK01BR": (<IP Address>, 4001), | |
| "MUK01SR": (<IP Address>, 4002), | |
| "MCH01BR": (<IP Address>, 4001), | |
| "MCH01SR": (<IP Address>, 4002), | |
| "WPA13CR": (<IP Address>, 8001), | |
| "SQX01CF": (<IP Address>, 4001), | |
| "HQ": (<IP Address>, 4001)} | |
| server = "REDACTED" | |
| dest = "\\\\" + server + "\\<Path>\\rt\\Telnet\\Output\\" | |
| # Classes | |
| class kill_timeout(timeout): | |
| def __init__(self): | |
| timeout.__init__(self) | |
| # Functions | |
| def figure_out_restart(date, time): | |
| now = DT.datetime(date.year, date.month, date.day, | |
| time.hour, time.minute) | |
| dt = DT.timedelta(seconds=15*60) | |
| temp = now + dt | |
| minute = 15*floor(temp.minute/15.0) | |
| return DT.datetime(temp.year, temp.month, temp.day, temp.hour, int(minute)) | |
| def get_serial_ports(connection): | |
| connection.write("who\r\n") | |
| connection.read_until("\r\n", Max) | |
| tmp = connection.read_until("\r\n", Max) | |
| pids = [] | |
| while "#>" not in tmp: | |
| tmp = connection.read_until("\r\n", Max) | |
| if "serial" in tmp: | |
| pids.append(tmp[0]) | |
| return pids | |
| def kill_serial_ports(connection, pids): | |
| for pid in pids: | |
| connection.write("kill %s\r\n"%pid) | |
| connection.read_until("\r\n", Max) | |
| def report_failure(site, progress_code): | |
| msg = "Telnet data download from " + site + " failed with the following progress code:\n" | |
| msg += progress_code | |
| mailer = SMTP(<IP Address>) | |
| mailer.sendmail(<Sender Email>, | |
| [<Recipient Emails>], | |
| msg) | |
| mailer.quit() | |
| #def driver(site, hours2sample=36): | |
| site = sys.argv[1] | |
| if site[:2]=="HQ": | |
| kind = site[2:] | |
| site = site[:2] | |
| Path = op.join(site, kind) | |
| else: | |
| Path = site | |
| if site == "WPA13CR": | |
| Max = 20 | |
| if len(sys.argv) > 2: | |
| hours2sample=float(sys.argv[2]) | |
| else: | |
| hours2sample=30 | |
| if len(sys.argv) > 3: | |
| offset=max(float(sys.argv[3]), 0) | |
| else: | |
| offset = 0 | |
| if len(sys.argv) > 4: | |
| writeOut=int(sys.argv[4]) | |
| else: | |
| writeOut=1 | |
| ip, port = IPsPORTs[site] | |
| ##if "MCH01BR" in site: | |
| ## path = "ManchesterB" | |
| ##elif "MCH01SR" in site: | |
| ## path = "ManchesterS" | |
| ##elif "MUK01BR" in site: | |
| ## path = "MukilteoB" | |
| ##elif "MUK01SR" in site: | |
| ## path = "MukilteoS" | |
| ##elif "SQX01CF" in site: | |
| ## path = "SquaxinC" | |
| ##elif "WPA13CR" in site: | |
| ## path = "Wilapa" | |
| ##else: | |
| ## path = "Site code not recognized. Terminating" | |
| ##path = site | |
| if "Terminating" not in Path: | |
| fn = site + strftime('_Day%d%m%YTime%H%M', gmtime()) | |
| Path = op.join(dest, Path) | |
| path = op.join(Path, fn) | |
| try: | |
| if writeOut: | |
| out = open(path, "w") | |
| out.write(site + os.linesep) | |
| gd = GD.getdata((ip, port), out, logging, Max, 1) # Instantiate the getdata object | |
| else: | |
| gd = GD.getdata((ip, port), None, logging, Max, 0) # Instantiate the getdata object | |
| #print "getdata object created, attmpting to open telnet conn." | |
| for tries in range(5): | |
| try: | |
| #print 'Try ', tries | |
| tn23 = T(ip, 23, Max) | |
| break | |
| except timeout, to: | |
| if tries==4: | |
| raise to | |
| else: | |
| pass | |
| #print "Telnet connection opened, attempting mult. ser. port dect/kill" | |
| progress_code = "Detecting & killing multiple serial ports" | |
| if gd.writeOut: | |
| gd.write_progress(progress_code) | |
| tmp = tn23.expect(["#>", "login:"], Max)[2] | |
| if "login:" in tmp: | |
| tn23.write("admin\r\n") | |
| tmp = tn23.read_until("password:", Max) | |
| tn23.write("strix!12\r\n") | |
| tn23.read_until("#>", Max) | |
| progress_code = "Logged in or login not needed" | |
| if gd.writeOut: | |
| gd.write_progress(progress_code) | |
| pids = get_serial_ports(tn23) | |
| npids = len(pids) | |
| tries = 0 | |
| while npids and (tries < Max): | |
| tries += 1 | |
| kill_serial_ports(tn23, pids) | |
| pids = get_serial_ports(tn23) | |
| npids = len(pids) | |
| if npids==0: | |
| progress_code = "Multiple serial ports killed or never existed" | |
| if gd.writeOut: | |
| gd.write_progress(progress_code) | |
| else: | |
| progress_code = site + " timed out trying to kill multiple serial ports. " | |
| progress_code += "Giving up. Note: data logging should not have stopped." | |
| raise kill_timeout | |
| ### This is where the work gets done ### | |
| tn, date, time, last_dt = gd.getdata(hours2sample, offset) | |
| ######################################## | |
| if site != 'HQ': | |
| if isinstance(tn, T): | |
| # issue delayed logging restart | |
| progress_code = "Issuing restart commands" | |
| if gd.writeOut: | |
| gd.write_progress(progress_code) | |
| restart = figure_out_restart(date, time) | |
| cmnd = restart.date().strftime("startmmddyy=%m%d%y\r\n") | |
| tn.write(cmnd) | |
| progress_code = tn.read_until(prompt, Max) | |
| if gd.writeOut: | |
| gd.write_progress(progress_code) | |
| cmnd = restart.time().strftime("starthhmmss=%H%M%S\r\n") | |
| tn.write(cmnd) | |
| progress_code = tn.read_until(prompt, Max) | |
| if gd.writeOut: | |
| gd.write_progress(progress_code) | |
| cmnd = "startlater\r\n" | |
| tn.write(cmnd) | |
| progress_code = tn.read_until(prompt, Max) | |
| if gd.writeOut: | |
| gd.write_progress(progress_code) | |
| if "start time =" not in progress_code: | |
| progress_code = "Restart failed to read final 'start time ='" + os.linesep | |
| progress_code += "Please check to see if " + site + " restarted." | |
| report_failure(site, progress_code) | |
| if gd.writeOut: | |
| gd.write_progress(progress_code) | |
| elif type(tn) is str: | |
| progress_code = "The following error was returned trying to telnet download data " | |
| progress_code += "from " + site + ":" + os.linesep + tn + os.linesep | |
| progress_code += "This should have taken place after checking for/killing multiple " | |
| progress_code += "serial ports, but before logging restart commands were issued; " | |
| progress_code += "please check " + site + " to see if data logging needs to be " | |
| progress_code += "restarted." | |
| report_failure(site, progress_code) | |
| if gd.writeOut: | |
| gd.write_progress(progress_code) | |
| gd.out.close() | |
| else: | |
| raise | |
| except kill_timeout: | |
| report_failure(site, progress_code) | |
| if gd.writeOut: | |
| gd.write_progress(progress_code) | |
| gd.out.close() | |
| except: | |
| t, v, b = sys.exc_info() | |
| progress_code += os.linesep + str(t) + str(v) | |
| progress_code += os.linesep + "The above unhandled exception prematurely aborted " | |
| progress_code += "telnet data download from " + site + "." + os.linesep | |
| progress_code += "Tries to open telnet connection = " + str(tries) + os.linesep | |
| progress_code += "It is unknown if data logging was stopped at the time of failure-" | |
| progress_code += "please check manually." | |
| report_failure(site, progress_code) | |
| if "gd" in dir(): | |
| try: | |
| gd.write_progress(progress_code) | |
| gd.out.close() | |
| except IOError: | |
| pass | |
| finally: | |
| if "tn23" in dir(): tn23.close() | |
| if "tn" in dir(): tn.close() | |
| #print progress_code | |
| if "tn" in dir(): tn.close() | |
| if "tn23" in dir(): tn23.close() | |
| if "gd" in dir() and gd.writeOut: | |
| if not gd.out.closed: gd.out.close() | |
| progress_code = site + " Done!" | |
| print Path, fn |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| function success = PushData(files, EN_recip, DB, subfuncAccess) | |
| %{ | |
| Note: For "code-sample" purposes only; WILL NOT RUN AS IS; | |
| Anything in <> represents REDACTED identifying information | |
| PushData: Push telemetered moorings data into EAPMW | |
| Inputs: (all are optional, though if a latter one is supplied, then the | |
| previous ones must be also) | |
| * files: A string (or cell array of strings) indicating path(s) to ascii | |
| text file(s) containing data output from autoTelnetDriver.py (or | |
| similarly formatted data); if absent, PushData runs in "Interactive | |
| Mode" (see Workflow below) | |
| * EN_recip: A cell array of string(s) providing the email address(es) | |
| of those to whom error notification should be sent; if absent, | |
| value depends on the version being run | |
| * DB: A struct specifying the database to which data should be pushed; | |
| default is struct('EAPMW', {'EAPMW'}) | |
| * subfuncAccess: if not present or equivalent to logical True, signals | |
| that "external" access to sub-functions parseInput and pushEAPMW | |
| should be provided via globals hParseInput and hPushEAPMW (see | |
| Globals below) | |
| Outputs: | |
| * success: String indicating successful completion, or containing an | |
| error message | |
| Globals: | |
| * AC & AR: Logicals indicating whether or not the current input file was: | |
| A) Created by autoTelnetDriver.py; and | |
| B) if so, was the result of the scheduled Run thereof; | |
| respectively | |
| * eN: Custom object encapsulating Matlab email functionality, incl. | |
| recipients list | |
| * hParseInput & hPushEAPMW: Function handles providing access to their | |
| namesake functions outside of this m-file; or empty strings. | |
| * IM: Indicates whether or not PushData was run in "Interactive Mode" | |
| Local Subroutines: parseInput, distinguish, check4ParseError, pushEAPMW | |
| Purpose: Robustly push autoTelnetDriver-acquired ("telemetered") moorings | |
| data into EAPMW Table ResultTelemetered, while guarding against dupli- | |
| cates, logging data "origin," and notifying of any errors via email | |
| Workflow: | |
| * If no inputs, being run in Interactive Mode (IM), otherwise, "batch | |
| mode," which is the norm. The essential differences are how the | |
| user: | |
| A) specifies the input files: IM presents the user with a file | |
| chooser dialog, whereas batch mode requires command-line spec- | |
| ification; and | |
| B) receives error notification, i.e., via GUI message or email. | |
| Otherwise, the workflows are the same. | |
| * Assignment of inputs, or their respective default values if not supplied | |
| * For each input file: | |
| * Call parseInput (see its docstring for its detailed workflow), with | |
| inputs specifying the file to process and the target database (i.e., | |
| EAPMW); result is returned into variable "parseResult" | |
| * If parseResult is empty or a string (i.e., an error message) assign | |
| it to output variable "success," then procede to next file; | |
| otherwise: | |
| * Assign to variable "source" a value, determined from the values of | |
| AR and AC (see Globals above), indicating the 'origin' of the data | |
| * Call pushEAPMW (see its docstring for its detailed workflow), with | |
| inputs parseResult, DB, and source; result is returned into variable | |
| "success"; procede to next file | |
| * Report errors (via GUI dialog if in IM, via email otherwise) and exit | |
| Author: David Goldsmith, formerly Wash. State Dept. of Ecology | |
| Release date: 10/1/2013 | |
| See also: query, Insert, Depth | |
| %} | |
| global eN hParseInput hPushEAPMW IM AR AC | |
| IM = false; AR = true; | |
| if ~nargin % Called w/out args = interactive mode | |
| IM = true; | |
| [files, pth, ~] = uigetfile('*.*', 'Please choose a file to push.',... | |
| 'MultiSelect', 'on'); | |
| else | |
| pth = ''; | |
| end | |
| if nargin < 4 || subfuncAccess | |
| hParseInput = @parseInput; | |
| hPushEAPMW = @pushEAPMW; | |
| else | |
| hParseInput = ''; hPushEAPMW = ''; | |
| end | |
| if nargin < 3 | |
| DB = struct('EAPMW', {'EAPMW'}); | |
| end | |
| if nargin < 2 | |
| if regexp(mfilename, '_dev') | |
| EN_recip = {'<Redacted email address>'}; | |
| else | |
| EN_recip = {'<Redacted email address>',... | |
| '<Redacted email address>',... | |
| '<Redacted email address>',... | |
| '<Redacted email address>',... | |
| '<Redacted email address>'}; | |
| end | |
| end | |
| eN = email_notification(EN_recip); | |
| if isnumeric(files) | |
| success = ['Error: PushData called without an input file name, '... | |
| 'and user cancelled out of choosing one. ']; | |
| else | |
| if ~iscell(files), files = {files}; end | |
| for t=files, target = t{:}; | |
| parseResult = parseInput(fullfile(pth,target), DB.EAPMW); | |
| if isempty(parseResult) || ischar(parseResult) | |
| success = parseResult; | |
| else | |
| source = 4*AR + 2*AC; | |
| success = pushEAPMW(parseResult, DB, source); | |
| end | |
| end | |
| end | |
| if exist('pth', 'var') && ~AR% && ~isdeployed | |
| uiwait(msgbox(['Push to Result Exit Status: ' success],... | |
| 'pushEAPMW Exit Status')); | |
| else | |
| if ~strcmp(success, 'Success!'), eN.appendMsg(success); end | |
| if ~isempty(eN.msg), eN.notify('PushData error'); end | |
| end | |
| end | |
| function out = parseInput(target, DB) | |
| global IM AR AC | |
| site = ''; | |
| errPrefix = 'PushData.parseInput ERROR: '; | |
| errSuffix = [' Please check the input file manually (i.e., with a text '... | |
| 'editor).']; | |
| if isdir(target) | |
| files = dir(strcat(target, '\*Day*')); | |
| if isempty(files) | |
| out = [errPrefix(1:end-2) ' (occurred before site determination): No '... | |
| 'files meeting name requirements found in ' target '; nothing pushed.']; | |
| return | |
| end | |
| latest = files([files(:).datenum] == max([files(:).datenum])); | |
| target = strcat(target, '\', latest.name); | |
| end | |
| % Determine, from the target filename, if the target file was: | |
| % A) created by autoTelnetDriver; and | |
| % B) if so, by the scheduled run | |
| [AR, AC, conn] = distinguish(target, DB); | |
| % Get complete list of mooring sites against which to check site | |
| % determination below | |
| try | |
| curs = exec(conn, 'SELECT SiteCode FROM Site WHERE SiteTypeCode=''M'''); | |
| assert(isempty(curs.Message), [errPrefix 'acquisition of mooring site '... | |
| 'list failed at the "exec" stage with error "' curs.Message '," ']) | |
| curs = fetch(curs); | |
| assert(isempty(curs.Message), [errPrefix 'acquisition of mooring site '... | |
| 'list failed at the "fetch" stage with error "' curs.Message '," ']) | |
| catch ME | |
| out = [ME.message 'which is prior to determination of site being '... | |
| 'processed.']; | |
| return | |
| end | |
| allSites = curs.Data; | |
| if all(islogical([AR AC])) % If successful... | |
| % Open and process file | |
| fid = fopen(target, 'r'); | |
| try | |
| site = fgetl(fid); | |
| assert(ismember(site, allSites), [errPrefix 'the station ID obtained '... | |
| 'from the input file, ' site ', is unknown.' errSuffix(1:end-1) ... | |
| ', or if this is a new site, please add it to column SiteCode in '... | |
| 'EAPMW''s Site Table with SiteTypeCode ''M''.']) | |
| for i=1:9 % skip 9 lines | |
| fgetl(fid); | |
| end | |
| type = fgetl(fid); | |
| assert(ismember(type, {'16', '37', '37p'}), [errPrefix 'the CTD type '... | |
| 'obtained from the input file, ' type ', is unknown.' errSuffix]) | |
| type = str2double(type(1:2)); | |
| for i=1:3 | |
| fgetl(fid); % skip 3 lines | |
| end | |
| line = fgetl(fid); | |
| nc = length(line); | |
| assert(nc > 19, [errPrefix 'the line in the input file that is sup'... | |
| 'posed to contain the computer time is too short.' errSuffix]) | |
| comptime = datenum(line(nc-19:nc)); | |
| assert(datenum('1970-01-01') < comptime, [errPrefix 'conversion of '... | |
| 'the computer date-time resulted in a date before 1970.' errSuffix(... | |
| 1:end-1) '; if the date and time are valid, there''s either a '... | |
| 'problem with one or the other''s format, or a bug in PushData.']) | |
| line = fgetl(fid); | |
| while (isempty(line) || ~strcmp(line(end-1:end), 'ds')) && ~feof(fid) | |
| % skip forward until the ds command is found | |
| line = fgetl(fid); | |
| end | |
| assert(~feof(fid), [errPrefix 'end-of-file reached looking for the '... | |
| '"ds" command.' errSuffix]) | |
| fgetl(fid); | |
| line = fgetl(fid); | |
| assert(length(line) > 11, [errPrefix 'the line in the input file that '... | |
| 'is supposed to contain the CTD "ds" command date is too short.'... | |
| errSuffix]) | |
| date = line(end-11:end); | |
| fgetl(fid); | |
| line = fgetl(fid); | |
| assert(length(line) > 8, [errPrefix 'the line in the input file that '... | |
| 'is supposed to contain the CTD "ds" command time is too short.' ... | |
| errSuffix]) | |
| time = line(end-8:end); | |
| insttime = datenum(strcat(date, time)); | |
| assert(datenum('1970-01-01') < insttime, [errPrefix 'conversion of '... | |
| 'the CTD "ds" command date-time resulted in a date before 1970.'... | |
| errSuffix(1:end-1) '; if the "ds" command date and time appear '... | |
| 'valid, there''s either a problem with one or the other''s format, '... | |
| 'or there''s a bug in PushData.']) | |
| offset = comptime - insttime; | |
| if abs(offset) > 1/288; % 1/288 = 5 minutes in units of days | |
| out = ['ALERT: while processing ' site ', the difference between '... | |
| 'the CTD "ds" command date-time and the computer date-time is '... | |
| 'greater than five minutes']; | |
| if ~IM | |
| return | |
| else | |
| button = questdlg([out ', suggesting a possible typo in the input '... | |
| 'file''s header.' char(10) 'Do you want to continue anyway?'],... | |
| 'Large computer/instrument time difference alert', 'Yes', 'No',... | |
| 'No'); | |
| if isempty(button) || strcmp(button, 'No') | |
| return | |
| end | |
| end | |
| end | |
| line = fgetl(fid); | |
| while (isempty(line) || ~strcmp(line(1:2), 'dd')) && ~feof(fid) % skip forward until the dd command is found | |
| line = fgetl(fid); | |
| end | |
| assert(~feof(fid), [errPrefix 'end-of-file reached looking for the '... | |
| '"dd" command.' errSuffix]) | |
| fgetl(fid); % skip a line | |
| if type == 37 % skip 8 more. This used to be 6--did something change? | |
| for i=1:8 % If so, when? How far back before this breaks? | |
| fgetl(fid); | |
| end | |
| parsestr = '%n %n %n %s %s'; | |
| else | |
| parsestr = '%n %n %n %n %n %n %n %n %s %s'; | |
| end | |
| marker = ftell(fid); | |
| data = textscan(fid, parsestr, 'Delimiter', ','); | |
| N = size(data, 2); | |
| % Check for parse error | |
| nBad = check4ParseError(data); | |
| % Handle parse error | |
| if nBad % if nBad is non-zero | |
| if type == 37 % Parse err probably indicates recs incl. pressure | |
| type = '37p'; % note this for pushData, | |
| parsestr = '%n %n %n %n %s %s'; % change the parsing string | |
| fseek(fid, marker, 'bof'); % rewind, and | |
| data = textscan(fid, parsestr, 'Delimiter', ','); % rescan the data | |
| N = size(data, 2); | |
| % check again | |
| nBad = check4ParseError(data); | |
| end | |
| while nBad | |
| % At this point nBad ~= 0 indicates a different problem, | |
| % probably incomplete record(s) in the input file. If that | |
| % is the case, the first N-nBad columns (N = total # of cols) | |
| % will be one longer than last nBad columns and, most | |
| % importantly, the file pointer will be situated precisely | |
| % following the error, i.e., somewhere in the middle of a | |
| % line. Thus we can continue by: removing from data any suc- | |
| % cessful conversions from the beginning of the line and | |
| % throwing away the rest of the problem line; calling text- | |
| % scan again; vertically concatenating the result to data; | |
| % checking the result; and repeating this process 'til: nBad | |
| % is 0; we encounter a line w/ just "S>"; or we reach end of | |
| % file. | |
| % Remove incomplete data line | |
| for i=1:N-nBad | |
| data{i} = data{i}(1:end-1); | |
| end | |
| % Get rid of rest of bad line | |
| fgetl(fid); | |
| % Call textscan again | |
| newdata = textscan(fid, parsestr, 'Delimiter', ','); | |
| % Append newdata to data | |
| for i=1:N | |
| data{i} = [data{i}; newdata{i}]; | |
| end | |
| % Check new data | |
| nBad = check4ParseError(newdata); | |
| end | |
| end | |
| fclose(fid); | |
| out = {offset, site, data, type}; | |
| catch ME | |
| fclose(fid); | |
| if regexp(mfilename, '_dev') | |
| out = getReport(ME); | |
| elseif isempty(site) | |
| out = ['"' ME.message '" occurred prior to determination of site '... | |
| 'being processed.']; | |
| else | |
| out = ['"' ME.message '" occurred while processing site ' site '.']; | |
| end | |
| end | |
| elseif ischar(AR) % if AR contains a message | |
| out = ['While processing ' site ', determination of the "auto-run" indi'... | |
| 'cator returned the following error: ' AR '. Pushing skipped.']; | |
| else | |
| out = ['While processing ' site ', one or both of the autoTelnetDriver-'... | |
| 'created and scheduled indicators failed to return the required '... | |
| 'logical-type. Pushing skipped.']; | |
| end | |
| end | |
| function [AR, AC, conn] = distinguish(target, DB) | |
| conn = []; % In case there's an error before conn creation | |
| AR = false; % Assume not auto-run as default | |
| % ACformat = the "Regular Expression" of the generic format of an aTD-created | |
| % input filename. A positive match of this against the target filename | |
| % implies that said file was produced by aTD (assuming proper user observance | |
| % of the prohibition against using this format for manually-created files) | |
| ACformat = ['[A-Z]{3}[0-9]{2}[A-Z]{2}_Day[0-3][0-9][0-1]'... | |
| '[0-9][2][0-9]{3}Time[0-2][0-9][0-5][0-9]']; | |
| AC = ~isempty(regexp(target, ACformat, 'once')) && ... | |
| isempty(regexp(target, '((AD)|(SP))M', 'once')); | |
| if AC % If it was aTD-created... | |
| % ...determine if it was by a scheduled run of aTD | |
| % Extract the creation datetime from the filename | |
| Y = target(end-11:end-8); | |
| M = target(end-13:end-12); | |
| D = target(end-15:end-14); | |
| h = target(end-3:end-2); | |
| m = target(end-1:end); | |
| CD = datenum(str2double(Y), str2double(M), str2double(D)); | |
| % Get the current SiteID's scheduled run times (which are "local" time) | |
| % from the DB | |
| connectArray = {DB, '', ''}; | |
| if strcmp(computer, 'PCWIN64') | |
| connectArray = [connectArray,... | |
| 'Vendor', 'MICROSOFT SQL SERVER',... | |
| 'Server', 'ECYDBLCYADDD3\cots',... | |
| 'AuthType', 'Windows']; | |
| end | |
| conn = database(connectArray{:}); | |
| setdbprefs('DataReturnFormat', 'cellarray') | |
| if isempty(conn.Message) | |
| SiteID = regexp(target, '[A-Z]{3}[0-9]{2}[A-Z]{2}', 'match',... | |
| 'once'); | |
| getDGTq = ['SELECT DailyGrabTime, DGTstartDate FROM MooringSpec '... | |
| 'WHERE SiteCode=''' SiteID ''' ORDER BY DGTstartDate ASC']; | |
| curs = fetch(exec(conn, getDGTq)); | |
| if ~isempty(curs.Data) % If that was successful | |
| CDs = datenum(curs.Data(:,2)); | |
| DGT = strtrim(curs.Data(sum(CD > CDs),1)); | |
| if ~strcmpi(DGT, 'null') % If the result wasn't null | |
| % extract the hour and minute from it | |
| DGTh = str2double(DGT{:}(1:2)); | |
| DGTm = str2double(DGT{:}(4:5)); | |
| else % If the result was null | |
| % Assume the hour is nine am | |
| DGTh = 9; | |
| % And the minute doesn't matter | |
| DGTm = NaN; | |
| end | |
| % Create a myDotNetDateTime object from the datetime pieces | |
| fnDT = myDotNetDateTime([Y '-' M '-' D ' ' h ':' m]); | |
| if fnDT.isDST, dh = -7; else dh = -8; end | |
| % Now, the following gives a positive match for matching hours | |
| % and either exactly matching minutes or any minute within | |
| % :01-:14. The latter was included for backwards compatibility | |
| % because, by the time of this writing, there had already been | |
| % some changes, reflected in scheduled-run filenames, in the | |
| % various sites' DGT's (e.g., 12:09 and 12:05 for MCH01BR). | |
| % All prior DGT's, however, always lay in the interval 12:01- | |
| % 12:14. Thus, MCH01BR filenames ending in either 05 or 09 | |
| % will match the current MCH01BR DGT of 12:05. This comes, | |
| % however, at the expense of the possibility of "false posi- | |
| % tives," i.e., if aTD is used manually between 12:01 and | |
| % 12:14 (local time), PD (this program) will see it as having | |
| % been created by the scheduled run. It is assumed, however, | |
| % that the frequency of such false positives will be samll, | |
| % and should be totally avoidable in the future by prohibition | |
| % of manual aTD use during that time frame. (This is highly | |
| % advisable anyway, since this is the interval during which | |
| % the instruments are being called by the scheduled tasks.) | |
| AR = (fnDT.data{1}.Hour + dh == DGTh) && (... | |
| (fnDT.data{1}.Minute == DGTm) || (... | |
| (0 < fnDT.data{1}.Minute) && (15 > fnDT.data{1}.Minute)... | |
| ) ); | |
| end | |
| % close(conn) | |
| else | |
| AR = conn.Message; | |
| end | |
| end | |
| end | |
| function nBad = check4ParseError(data) | |
| check = size(data{1},1); | |
| nBad = sum(cellfun(@(x) size(x,1)~=check, data(2:end))); | |
| end | |
| function success = pushEAPMW(offset_site_data_type, db, source) | |
| site = ''; | |
| success = ''; | |
| try | |
| fatalErrMsg = '; sorry, this is a "fatal" error: bye!'; | |
| nonFatalMsg = ['; this message is advisory, '... | |
| 'processing was not aborted']; | |
| ADT = datestr(utcnow(), 'mm/dd/yyyy HH:MM:SS'); | |
| [offset, site, data, ~] = deal(offset_site_data_type{:}); | |
| nc = numel(data); | |
| nr = numel(data{1}); | |
| try | |
| firstDT = combineDateAndTime(data{end-1}(1), data{end}(1)); | |
| lastDT = combineDateAndTime(data{end-1}(end), data{end}(end)); | |
| catch ME | |
| success = ['While processing ' site ', "' ME.message '" error thrown '... | |
| 'trying to get datetimes of first and/or last scans, which '... | |
| 'are needed to remove duplicates and get current parameter '... | |
| 'list from DB' fatalErrMsg]; | |
| return | |
| end | |
| % Create wrapped DB object | |
| DB = DBwrapper(db.EAPMW); | |
| setdbprefs('NullStringWrite', 'null') | |
| if ~isempty(DB.err) | |
| success = ['While processing ' site ', "' DB.err{:} '" error trying '... | |
| 'to create database wrapper object' fatalErrMsg]; | |
| return | |
| end | |
| % Get site's SiteID, and latitude for depth calc. | |
| Q = query(DB, struct('tbls', {{'Site'}},... | |
| 'cols', {{'SiteID', 'LatitudeDecimal'}},... | |
| 'where', {{['SiteCode=''' site '''']}})); | |
| if isempty(Q.errCode) | |
| Q.callSubmitStatement | |
| end | |
| if ~isempty(Q.errCode) | |
| success = ['Error "' Q.errCode{:} '" trying to get ' site '''s '... | |
| 'EAPMW SiteID and latitude' fatalErrMsg]; | |
| return | |
| end | |
| EAPsiteID = Q.result{1}; | |
| lat = Q.result{2}; | |
| % Get site's parameter list for the date being processed | |
| Q = query(DB, struct('tbls', {{'MooringSpec'}},... | |
| 'cols', {{'DISTINCT', 'ParameterList',... | |
| 'PLstartDate'}},... | |
| 'where', {{['SiteCode=''' site '''']}},... | |
| 'orderBy', {{{'PLstartDate', 'asc'}}})); | |
| if isempty(Q.errCode) | |
| Q.callSubmitStatement | |
| end | |
| if ~isempty(Q.errCode) | |
| success = ['Error "' Q.errCode{:} '" trying to get ' site '''s '... | |
| 'current parameter list' fatalErrMsg]; | |
| return | |
| else | |
| PLDTs = datenum(Q.result(:,2)); | |
| PLstart = strtrim(Q.result{sum(PLDTs < firstDT),1}); | |
| PLend = strtrim(Q.result{sum(PLDTs < lastDT),1}); | |
| if ~strcmp(PLstart, PLend) | |
| success = ['Error: ' site '''s parameter list at the start of the '... | |
| 'interval, ' datestr(firstDT) ',' char(10) 'and at the '... | |
| 'end, ' datestr(lastDT) ', were different; please split '... | |
| 'the input file' char(10) 'into differnt files with dis'... | |
| 'tinct parameter lists' fatalErrMsg]; | |
| return | |
| else | |
| PL = {PLend}; | |
| end | |
| end | |
| % Confirm that the number of data columns = number of parameters + 2, | |
| % the plus 2 accounting for the date and time columns, then elim. dupes | |
| if nc ~= (numel(PL{:}) + 2) | |
| success = ['Error: number of parameters in ' site '''s current list '... | |
| 'didn''t match number of columns of data' fatalErrMsg]; | |
| return | |
| end | |
| % Extract reported date & time "descriptions" from data | |
| d = data{end-1}; t = data{end}; | |
| % Convert to matlab DSN's and "normalize" | |
| MatlabDTs = combineDateAndTime(d, t) + offset; | |
| % Determine datetimes within current range already in the DB... | |
| Q = query(DB, struct('tbls', {{'ResultTelemetered'}})); | |
| Q.setPropVals('cols', {'DISTINCT', 'MatlabDatetime'}) | |
| Q.setPropVals('where', {['SiteID = ' num2str(EAPsiteID) ' AND '... | |
| 'MatlabDatetime BETWEEN ' num2str(... | |
| firstDT + offset - 0.01) ' AND ' num2str(... | |
| lastDT + offset + 0.01)]}) % 0.01 ~ 15 min. | |
| if isempty(Q.errCode) | |
| Q.callSubmitStatement | |
| end | |
| % ...and translate this into scans to "keep" | |
| if strcmp(Q.errCode, 'Query returned no data') % No extant records in | |
| % current datetime interval | |
| % in ResultTelemetered | |
| keepers = true(nr,1); | |
| elseif isempty(Q.result) % Something went wrong | |
| success = ['Error "' Q.errCode{:} '" trying to check ResultTelemetered '... | |
| 'for extant records for ' site ' within the current data''s '... | |
| 'datetime range' fatalErrMsg]; | |
| % return | |
| else | |
| dupeDTs = [Q.result{:}]; | |
| keepers = arrayfun(@(x) min(abs(dupeDTs-x)) >= 1/288, MatlabDTs); | |
| % 1/288 = 5 min. in units of days | |
| if all(~keepers) | |
| success = ['For ' site ', no data not already in ResultTelemetered, '... | |
| 'nothing to push therein! Bye!']; | |
| % return | |
| end | |
| end | |
| if ~isempty(success), return; end | |
| % Remove duplicates | |
| MatlabDTs = MatlabDTs(keepers); | |
| UTCdts = datestr(MatlabDTs, 'mm/dd/yyyy HH:MM:SS'); | |
| nKept = sum(keepers); | |
| empties = NaN(nKept,1); | |
| nulls = constCellStrArr('null', [nKept 1]); | |
| % Unpack rest of data while PL is still properly ordered, discarding | |
| % voltages Aux1-3... | |
| extdata = struct(); | |
| for i=1:nc-2 | |
| PL{:}(i) = strtrim(PL{:}(i)); | |
| if ~strcmp(PL{:}(i), 'n') | |
| extdata.(PL{:}(i)) = data{i}(keepers); | |
| end | |
| end | |
| % ...remove n's from paramlist... | |
| PL = regexprep(PL{:}, 'n', ''); | |
| % ...and recast as a cellstr array of single chars | |
| PL = cellstr(PL(:)); | |
| % Now get the list of EAPMW's corresponding ParameterIDs | |
| % Prepare query object | |
| Q = query(DB, struct('tbls', {{'Parameter'}})); | |
| Q.setPropVals('cols', {'MatlabProcessingID', 'ParameterID'}) | |
| Q.setPropVals('where', {['MatlabProcessingID IN (''' PL{1} ''''... | |
| sprintf(',''%s''', PL{2:end}) ')']}) | |
| if isempty(Q.errCode) | |
| Q.callSubmitStatement | |
| end | |
| if ~isempty(Q.errCode) | |
| success = ['Error "' Q.errCode{:} '" trying to get ParameterID''s '... | |
| 'corresponding to the current parameter list for ' site ... | |
| '. ' fatalErrMsg]; | |
| return | |
| end | |
| % Unpack results | |
| paraMap = struct(); | |
| for i=1:numel(PL), | |
| paraMap.(Q.result{i,1}(1))=Q.result{i,2}; | |
| end | |
| % Prepare insertion object | |
| I = Insert(DB, struct('tbls', {{'ResultTelemetered'}})); | |
| if ~isempty(I.errCode) | |
| success = ['Error "' I.errCode{:} '" trying to create data '... | |
| 'insertion object for site ' site fatalErrMsg]; | |
| return | |
| end | |
| % Prepare insertion data | |
| Data = struct('SiteID', {repmat(EAPsiteID, nKept, 1)},... | |
| 'AnalyzedDatetime', {constCellStrArr(ADT, [nKept 1])},... | |
| 'BinSampleSize', {ones(nKept, 1)},... | |
| 'DataProcCode', {repmat(source, nKept, 1)},... | |
| 'UserCode', {constCellStrArr('MTLB', [nKept 1])},... | |
| 'MatlabDatetime',{MatlabDTs},... | |
| 'UTCDatetime',{UTCdts},... | |
| 'ActualDepthDecimal', {nans([nKept 1])},... | |
| 'ParameterID', {[]},... | |
| 'ResultDecimal', {[]},... | |
| 'ResultVoltage', {[]}); | |
| I.setPropVals('cols', fieldnames(Data)) | |
| params = fieldnames(extdata); | |
| if ismember('P', params) | |
| % Compute depth | |
| Data.ActualDepthDecimal = Depth(extdata.P, lat); | |
| end | |
| for p=params', p = p{:}; % for each parameter | |
| if ismember(p, {'T', 'C', 'P', 'S'}) | |
| Data.ResultDecimal = extdata.(p); | |
| Data.ResultVoltage = empties; | |
| Data.ParameterID = repmat(paraMap.(p), nKept, 1); | |
| else | |
| tmp = Data.AnalyzedDatetime; | |
| Data.AnalyzedDatetime = nulls; | |
| Data.ResultVoltage = extdata.(p); | |
| Data.ResultDecimal = empties; | |
| Data.ParameterID = repmat(paraMap.(p(1)), nKept, 1); | |
| end | |
| I.setPropVals('data', Data) | |
| if ~isempty(I.errCode) | |
| msg = ['Error "' I.errCode{:} '" trying to build insertion '... | |
| 'object for parameter ' p ', site ' site nonFatalMsg]; | |
| success = [success msg]; | |
| continue | |
| end | |
| if all(strcmp(Data.AnalyzedDatetime, 'null')) && exist('tmp', 'var') | |
| Data.AnalyzedDatetime = tmp; | |
| end | |
| I.callSubmitStatement | |
| if ~isempty(I.errCode) | |
| msg = ['Error "' I.errCode{:} '" trying to insert parameter '... | |
| p ' for site ' site nonFatalMsg]; | |
| success = [success msg]; | |
| continue | |
| end | |
| end | |
| if isempty(success) | |
| success = 'Success!'; | |
| end | |
| catch ME | |
| if exist('conn', 'var') | |
| close(conn); | |
| end | |
| success = ['While processing ' site ', error ' ME.message 'thrown.']; | |
| end | |
| end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment