ftfarias

import os
import json
import urllib3

url_delete = 'http://172.31.x.x:9200/my_index/_delete_by_query'
query_delete = {
  "query": { "match": { "fk_xxx": day }}
}

ftfarias

#! /usr/bin/env python
from argparse import ArgumentParser

import boto3
import pandas as pd

CSV_FILENAME = 'ec2_reserves.csv'

ec2 = boto3.client('ec2')

ftfarias

import boto3
from io import TextIOWrapper, BytesIO
from gzip import GzipFile
import csv
import logging
import collections
from tqdm import tqdm_notebook as tqdm

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

ftfarias

# !pip install watermark
%load_ext watermark
%watermark -udtz -v -m -iv -rg

ftfarias

--------------------------------- Process
SELECT * FROM pg_stat_activity WHERE state = 'active';
SELECT pg_cancel_backend(<pid of the process>)
SELECT pg_terminate_backend(<pid of the process>)

--------------------------------- LOCKS
Ref: https://wiki.postgresql.org/wiki/Lock_Monitoring

SELECT relation::regclass, * FROM pg_locks WHERE NOT GRANTED;

ftfarias

import sqlalchemy
import sys


def print_clear_dag(dag_id):
    """Clear all information for a DAG from airflow postgres database"""
    list_tables = ['xcom', 'task_instance', 'sla_miss', 'log', 'job', 'dag_run', 'dag',
                   'dag_stats', 'task_fail']
    for table in list_tables:
        queries = ["DELETE FROM {} WHERE dag_id='{}'".format(table, dag_id),

ftfarias

# parse iso time
datetime.strptime('2019-04-17T09:02:16.00428Z', '%Y-%m-%dT%H:%M:%S.%fZ')


import itertools

def grouper(iterable, n):
    it = iter(iterable)
    while True:
       chunk = tuple(itertools.islice(it, n))

ftfarias

import copy

def verify_partial_valid_solution(A,w):
    if len(w[0]) > 2:
        return False
    w1 = sum(w[1])
    w2 = sum(w[2])
    w3 = sum(w[3])
    a = sum(A)
    if a + w1 < w2: return False

ftfarias

# Purging All Unused or Dangling Images, Containers, Volumes, and Networks
docker system prune -a


# Remove all images
docker rmi $(docker images -a -q)

ftfarias

A Hacker's Guide to Python string and Natural Language Processing (NLP) packages

Preprocessing

• The Python Standard Library, especially str.methods and string module are powerful for text processing. Start there.
• regex - Extends Python's Standard Library re module while being backwards-compatible.
• chardet - Finds character encoding.
• ftfy - Take in bad Unicode and output good Unicode. Seriously automagical.
• ploygot - Helpful for multilingual preprocessing.