dardanxhymshiti

def highlight_columns(df, rows=20, color='lightgreen', columns_to_shadow=[], columns_to_show=[]):
    highlight = lambda slice_of_df: 'background-color: %s' % color
    sample_df = df.head(rows)
    if len(columns_to_show) != 0:
        sample_df = sample_df[columns_to_show]
    highlighted_df = sample_df.style.applymap(highlight, subset=pd.IndexSlice[:, columns_to_shadow])
    return highlighted_df

dardanxhymshiti

import boto3
from os.path import join


def lambda_handler(event, context):
    emr = boto3.client('emr')
    version = 'latest'
    main_path = join('s3://<artifacts-bucket-name>', version, 'main.py')
    modules_path = join('s3://<artifacts-bucket-name>', version, 'module_seed.zip')

dardanxhymshiti

from setuptools import setup, find_packages

__version__ = '1.0.0'

setup(
    name='pyspark_seed', 
    version=__version__, 
    packages=find_packages(), 
    install_requires=['pyspark', 'boto3']
)

dardanxhymshiti

python setup.py --version

dardanxhymshiti

python3 -m venv venv

. ./venv/bin/activate

pip install -e .

dardanxhymshiti

def process(spark, input_path, output_path, save_mode='append'):
    # read data
    df = spark.read.parquet(input_path)

    # processing
    pass

    # output
    df.write.parquet(output_path, save_mode=save_mode)

dardanxhymshiti

from module_seed.jobs import \
    job_one_seed,\
    job_two_seed
from module_seed.utils.logger_utils import get_logger
from module_seed.utils.spark_utils import create_spark_session


jobs = {
    'job_one_seed': job_one_seed.process,
    'job_two_seed': job_two_seed.process

dardanxhymshiti

import ast
import sys
from module_seed.run import run


if __name__ == '__main__':
    str_parameters = sys.argv[1]
    parameters = ast.literal_eval(str_parameters)
    run(parameters)

dardanxhymshiti

list_of_fruits = ['apple', 'orange', 'pear', 'cherry', 'banana']
list_of_scores = [80, 98, 50, 55, 100]

tuple_of_person_info = ('John', 'Doe', 20, 'Denver', 'Colorado')
tuple_of_car_info = ('BMW', 'X5', 5000, 'Black', True)

dardanxhymshiti

Hydrogen, Helium, Lithium, Beryllium, Boron, Carbon, Nitrogen, Oxygen, Fluorine, Neon, Sodium, Magnesium, 
Aluminum, Silicon, Phosphorus, Sulfur, Chlorine, Argon, Potassium, Calcium, Scandium, Titanium, Vanadium, 
Chromium, Manganese, Iron, Cobalt, Nickel, Copper, Zinc, Gallium, Germanium, Arsenic, Selenium, 
Bromine, Krypton, Rubidium, Strontium, Yttrium, Zirconium, Niobium, Molybdenum, Technetium, Ruthenium, Rhodium, 
Palladium, Silver, Cadmium, Indium, Tin, Antimony, Tellurium, Iodine, Xenon, Cesium, Barium, 
Lanthanum, Cerium, Praseodymium, Neodymium, Promethium, Samarium, Europium, Gadolinium, Terbium, Dysprosium, Holmium, 
Erbium, Thulium, Ytterbium, Lutetium, Hafnium, Tantalum, Tungsten, Rhenium, Osmium, Iridium, Platinum, 
Gold, Mercury, Thallium, Lead, Bismuth, Polonium, Astatine, Radon, Francium, Radium, Actinium, 
Thorium, Protactinium, Uranium, Neptunium, Plutonium, Americium, Curium, Berkelium, Californium, Einsteinium, Fermium, 
Mendelevium, Nobelium, Lawrencium, Rutherfordium, Dubnium, Seaborgi