achinta

#Video conversion tools
xcode-select --install
#from https://gist.github.com/clayton/6196167#gistcomment-2777274
brew install ffmpeg --with-aom \
--with-chromaprint \
--with-fdk-aac \
--with-fontconfig \
--with-freetype \
--with-frei0r \
--with-game-music-emu \

achinta

"""
Groupby, apply aggregations and rename columns for a dataframe. When we apply multiple aggregations to a df, 
two level column names are created. Instead, this method returns a dataframe with format 'colname_agg'

example: 
    df = pd.DataFrame({'A': [1, 1, 1, 2, 2],
                   'B': range(5),
                   'C': range(5)})
    groupByCols = ['A']
    agg = {'B': ['sum','mean'], 'C': 'min'}

achinta

# export data from database as csv and copy to google storage
gsutil cp abhyasis.csv gs://my-bucket

#create dataset in bigquery (onetime, say 'aims')
bq --location=US mk --dataset [PROJECT_ID]:aims

#create table in bigquery (onetime, say 'abhyasis')
bq mk --table [PROJECT_ID]:aims.abhyasis

#load from google storage to bigquery

achinta

import pyspark.sql.functions as F
from pyspark.sql import Window

df = spark.createDataFrame([
    ('d1',None),
    ('d2',10),
    ('d3',None),
    ('d4',30),
    ('d5',None),
    ('d6',None),

achinta

from collections.abc import Iterable

class CategoryEncoder(object):
    """
    Once fit method is called, sklearn.preprocessing.LabelEncoder cannot encode new categories. 
    In this category encoder, fit can be called any number times. It encodes categories which it has not seen before,
    without changing the encoding of existing categories. 
    
    Usually the first category has encoded value of zero. We can override it with value 'start'
    """

achinta

"""
Run this in the root of the downloaded folder. It will copy the pdf files from sub folders to base folder
"""
import os
from glob import glob
import shutil

cwd = os.getcwd()
if not os.path.isdir('renamed'):
    os.mkdir('renamed')

achinta

from sklearn.base import TransformerMixin

class CuCategoryEncoder(TransformerMixin):
    """
    Runs on GPU using cudf
    Once fit method is called, sklearn.preprocessing.LabelEncoder cannot encode new categories. 
    In this category encoder, fit can be called any number times. It encodes categories which it has not seen before,
    without changing the encoding of existing categories. 
    """
    # categories as series

achinta

from pathlib import Path
import time
import json
from typing import Dict

def lock_file(func):
    '''
    Decorator which 
    '''
    def wrapper(path: Path, data):

achinta

achinta

// create random variable with timestamp as suffix
var ts = pm.variables.replaceIn("{{$timestamp}}"); 
pm.collectionVariables.set("name_ts", 'test ' + ts);

//create timestamp variables
var moment = require('moment');
pm.collectionVariables.set('arrival_date', moment().format(("YYYY-MM-DD HH:mm:ss")));
pm.collectionVariables.set('departure_date', moment().add(5, 'days').format(("YYYY-MM-DD HH:mm:ss")));