liquidgenius

PSQL

Magic words:

psql -U postgres

Some interesting flags (to see all, use -h or --help depending on your psql version):

• -E: will describe the underlaying queries of the \ commands (cool for learning!)
• -l: psql will list all databases and then exit (useful if the user you connect with doesn't has a default database, like at AWS RDS)

liquidgenius

import os
import yaml
import logging.config
import logging
import coloredlogs

def setup_logging(default_path='logging.yaml', default_level=logging.INFO, env_key='LOG_CFG'):
    """
    | **@author:** Prathyush SP
    | Logging Setup

liquidgenius

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {},
    'formatters': {
        'syslog': {
            'format': '%(asctime)s ' + HOSTNAME_SHORT +
                      ' %(name)s: %(levelname)s: %(message)s',
            'datefmt': '%b %d %H:%M:%S',
        }

liquidgenius

#!/usr/bin/env python
# -*- coding: utf-8 -*-
__license__ = "MIT"
__version__ = "0.0.1"
__maintainer__ = "https://github.com/liquidgenius"
__status__ = "Development"


from pathlib import Path
from pyngrok import ngrok

liquidgenius

from pathlib import Path
from pyngrok import ngrok


class ExternalAccess:
    """ ExternalAccess provides convenience methods for using PyNgrok with FastAPI. It assumes 
    you have a custom NGROK config file at resources/ngrok.yml. Pyngrok install is required: 
    https://pyngrok.readthedocs.io/en/latest/
    
    USAGE:

liquidgenius

import pymysql
import pandas as pd
from snowflake.sqlalchemy import URL as SFURL
from sqlalchemy import create_engine
from sqlalchemy.engine.url import URL


class OmniDB:
    """ A minimal Python class for uniform API to both Snowflake and MySQL databases. 
    TODO: Error handling

liquidgenius

def low_pass_filter_anomaly_detection(df: pd.DataFrame,
                                      column_name: str,
                                      rolling_window: int,
                                      number_of_stdevs_away_from_mean: float):
    """
    Implement a low-pass filter to detect anomalies in a time series, and save the filter outputs
    (True/False) to a new column in the dataframe.
    Arguments:
        df: Pandas dataframe
        column_name: string. Name of the column that we want to detect anomalies in

liquidgenius

from datetime import datetime
from pathlib import Path


def remove_expired_files(file_path, expiry_days=60, use_UTC=False):
    """  Checks if the creation day of a file is greater than the file_expiry_days and deletes it if it is.
    :param file_path: Pathlib.Path: The path to a file
    :param expiry_days: int: Age in days after which the file should be deleted
    :param use_UTC: bool: Flag to determine if now should be in UTC time, defaults to False.
    :return: None

liquidgenius

Amazon changed the install in Linux 2. One no-longer using 'yum' See: https://aws.amazon.com/amazon-linux-2/release-notes/

Docker CE Install

sudo amazon-linux-extras install docker
sudo service docker start
sudo usermod -a -G docker ec2-user

liquidgenius

import datetime
import dateutil.parser
timestring1 = '2007-03-04T21:08:12.127'
timestring2 = '2012-03-04 12:08:12.354'
dt1 = datetime.datetime.strptime(timestring1, '%Y-%m-%dT%H:%M:%S.%f')
dt2 = datetime.datetime.strptime(timestring2, '%Y-%m-%d %H:%M:%S.%f')
dup1 = dateutil.parser.parse(timestring1)
dup2 = dateutil.parser.parse(timestring2)