seahrh

# kill all processes that match this command
pkill -u user -f "gunicorn myapp.app.main:app"

# kill sentence-transformers processes SIGKILL
pkill -9 -u user -f "myapp/venv/bin/python3 -c from multiprocessing"


# manual kill parent and child processes
# see uvicorn multiprocessing bug https://github.com/encode/uvicorn/issues/852
PARENT_PID=$(pgrep -u user -f "uvicorn myapp.app.main:app --port=1234")

seahrh

# -v, --verbose
# -a, --archive   want recursion and preserve almost everything
# -P                          same as --partial --progress
# -z, --compress              compress file data during the transfer


rsync -vaPz file.txt user@server:~/user

rsync -vaPz dir user@server:~/user

seahrh

import json
import multiprocessing

def dowork(qids, shared_dict, lock):
  lock.acquire()  # type: ignore
  for qid in qids:
    # update shared_dict
    pass
  lock.release()

seahrh

cat input/squad20/dev-v2.0.json | jq '.data[].paragraphs[].qas[] | select(.id=="5ad24ce8d7d075001a428c0e")' -C | less -r

cat input/squad20/dev-v2.0.json | jq ".data[].title" -C | less -r

cat input/squad20/dev-v2.0.json | jq ".data[].paragraphs[].qas[].id" -C | less -r

cat input/squad20/dev-v2.0.json | jq ".data[].paragraphs[].qas[].id" -C | grep -ni 'needle'

seahrh

index = df[df["title"].str.len() == 0].index
df.drop(index=index, inplace=True)

index = df[df["is_disamb"] == 1].index
df.drop(index=index, inplace=True)

seahrh

import logging
from logging.config import fileConfig


LOG_INI = CONF[ENVIRONMENT]["LOG_INI"]
LOG_DIR = CONF[ENVIRONMENT]["LOG_DIR"]


def get_logger(name: str = None):
    logging.config.fileConfig(LOG_INI, defaults={"logdir": LOG_DIR})

seahrh

2xu
3com
3d younique
3m
3m scotchlite
4head
5 star
55 soul
81 customs
81stgeneration

seahrh

def safe_func(default_value=None):
    def decorate(f):
        @functools.wraps(f)
        def wrapper(*args, **kwds):
            try:
                res = f(*args, **kwds)
            except:
                res = default_value
            return res
        return wrapper

seahrh

%%time
len1 = len(train)
train["col"] = train["col"].str.strip()
train.drop(train[train["col"].str.len() == 0].index, inplace=True)
len2 = len(train)
print(f"{len1 - len2} rows deleted")

seahrh

import pathlib
from datetime import datetime

ts = datetime.now().strftime('%Y%m%d_%H%M%S')
job_dir = f"models/mlp/{ts}"
pathlib.Path(job_dir).mkdir(parents=True, exist_ok=True)