dcragusa

import re
import os
import sys
import time
import shutil
import datetime as dt
from functools import wraps
from typing import Optional as O, Callable as C, List as L

from selenium import webdriver

dcragusa

import os
import stat

from paramiko import SSHClient, SFTPClient, AutoAddPolicy

from tools.file_tools import make_775_dir, file_exists

join = os.path.join

dcragusa

import requests
from requests.adapters import HTTPAdapter

PROXIES = {
    'http': 'http://proxy.example.com:3128',
    'https': 'https://proxy.example.com:3128'
}


class ProxyUAAdapter(HTTPAdapter):

dcragusa

# From https://stackoverflow.com/a/242531/2864129
# Put in pythonpath and 'import debug'

import sys

def info(type_, value, tb):
    if hasattr(sys, 'ps1') or not sys.stderr.isatty() or type_ is KeyboardInterrupt:
        # we are in interactive mode / we don't have a tty-like device / user-triggered, so we call the default hook
        sys.__excepthook__(type_, value, tb)
    else:

dcragusa

echo killing

# Get progress group id from pid
PGID=$(ps opgid `cat ${PID_PATH}` | grep -o '[0-9]*')
# Kill the process group
kill -- -$PGID
echo killed
# Now that it's killed, don't forget to remove the PID file
rm ${PID_PATH}

dcragusa

class AttrDict(dict):
    # access data by dot notation e.g. {'a': 1} -> d.a = 1
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.__dict__ = self

    def __getattr__(self, name):
        return self.get(name, None)

dcragusa

def error_handling(rollback_toggle=False):
    def wrap(func: C) -> C:
        @wraps(func)
        def func_wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except Exception as e:
                tb = sys.exc_info()[2]
                exception_locals = tb.tb_next.tb_frame.f_locals

dcragusa

import pandas as pd
from collections import OrderedDict
from pandas.core.indexes.multi import MultiIndex

def multiindex_to_nested_dict(df: pd.DataFrame, value_only = False) -> OrderedDict:
    if isinstance(df.index, MultiIndex):
        return OrderedDict((k, multiindex_to_nested_dict(df.loc[k])) for k in df.index.remove_unused_levels().levels[0])
    else:
        if value_only:
            return OrderedDict((k, df.loc[k].values[0]) for k in df.index)

dcragusa

function r() {
    clear && printf '\033[3J'
}

dcragusa

screen ionice -c3 rm -rv dir_to_delete/ | pv -l > /dev/null