liquidgenius

import datetime
import dateutil.parser
timestring1 = '2007-03-04T21:08:12.127'
timestring2 = '2012-03-04 12:08:12.354'
dt1 = datetime.datetime.strptime(timestring1, '%Y-%m-%dT%H:%M:%S.%f')
dt2 = datetime.datetime.strptime(timestring2, '%Y-%m-%d %H:%M:%S.%f')
dup1 = dateutil.parser.parse(timestring1)
dup2 = dateutil.parser.parse(timestring2)

liquidgenius

Amazon changed the install in Linux 2. One no-longer using 'yum' See: https://aws.amazon.com/amazon-linux-2/release-notes/

Docker CE Install

sudo amazon-linux-extras install docker
sudo service docker start
sudo usermod -a -G docker ec2-user

liquidgenius

def low_pass_filter_anomaly_detection(df: pd.DataFrame,
                                      column_name: str,
                                      rolling_window: int,
                                      number_of_stdevs_away_from_mean: float):
    """
    Implement a low-pass filter to detect anomalies in a time series, and save the filter outputs
    (True/False) to a new column in the dataframe.
    Arguments:
        df: Pandas dataframe
        column_name: string. Name of the column that we want to detect anomalies in

liquidgenius

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {},
    'formatters': {
        'syslog': {
            'format': '%(asctime)s ' + HOSTNAME_SHORT +
                      ' %(name)s: %(levelname)s: %(message)s',
            'datefmt': '%b %d %H:%M:%S',
        }

liquidgenius

import os
import yaml
import logging.config
import logging
import coloredlogs

def setup_logging(default_path='logging.yaml', default_level=logging.INFO, env_key='LOG_CFG'):
    """
    | **@author:** Prathyush SP
    | Logging Setup

liquidgenius

PSQL

Magic words:

psql -U postgres

Some interesting flags (to see all, use -h or --help depending on your psql version):

• -E: will describe the underlaying queries of the \ commands (cool for learning!)
• -l: psql will list all databases and then exit (useful if the user you connect with doesn't has a default database, like at AWS RDS)

liquidgenius

DROP FUNCTION IF EXISTS get_role_to_actor_and_actor_to_role( INOUT BIGINT, OUT JSONB, OUT JSONB );
CREATE OR REPLACE FUNCTION get_role_to_actor_and_actor_to_role(
    INOUT program_id    BIGINT,
    OUT   actor_to_role JSONB,
    OUT   role_to_actor JSONB)
    RETURNS RECORD IMMUTABLE
AS $plpython_function$
import json

liquidgenius

--- If you are missing the pypythonu extension, install it first
--- (Ubuntu): sudo apt-get install python-psycopg2

--- Activate plpython 
CREATE EXTENSION plpythonu;

--- Create the profiling function
CREATE OR REPLACE FUNCTION similarity(text, text) RETURNS numeric AS $$
import difflib
return difflib.SequenceMatcher(None,args[0], args[1]).ratio()

liquidgenius

CREATE OR REPLACE FUNCTION public.geocode_google(IN inaddress text, OUT address text, OUT longitude double precision, OUT latitude double precision)
  RETURNS record AS
$BODY$ 
from geopy.geocoders import GoogleV3
geolocator = GoogleV3()
try:
  address, (latitude, longitude) = geolocator.geocode(inaddress,timeout=1,exactly_one=True)
  return address, longitude, latitude
except:
  return None, None, None

liquidgenius

CREATE OR REPLACE FUNCTION crop(image bytea, rect box)
 RETURNS bytea
 LANGUAGE plpythonu
AS $function$
if ('io' in SD) and ('StringIO' in SD) and ('Image' in SD):
    io = SD['io']
    StringIO = SD['StringIO']
    Image = SD['Image']
else:
    import io, StringIO