Kevin Kho kvnkho

👋

Feel free to message me. Contact info in profile

kvnkho / fugue.py

Last active April 18, 2021 21:37

Fugue Example

	# Import statements
	from fugue import FugueWorkflow, SparkExecutionEngine
	from typing import List, Any, Dict, Iterable

	# Area code to state
	area_to_state = {"217": "IL", "312": "IL", "415": "CA", "352": "FL"}

	# schema: *, inferred_state:str
	def fill_location(df:Iterable[Dict[str,Any]]) -> Iterable[Dict[str,Any]]:
	for row in df:

kvnkho / comparison.py

Last active April 18, 2021 21:24

Comparing Pandas and Spark

	# Comparison of creating inferred_state column
	area_to_state = {"217": "IL", "312": "IL", "415": "CA", "352": "FL"}

	# Pandas implementation
	df['inferred_state'] = df['home_state']\
	.fillna(df['work_state'])\
	.fillna(df['phone'].str.slice(0,3).map(area_to_state))

	# Spark implementation
	from pyspark.sql.functions import coalesce, col, substring, create_map, lit

kvnkho / median.py

Last active March 17, 2021 18:56

Simple median

	#schema: user_id:int, measurement:int
	def get_median(df:pd.DataFrame) -> pd.DataFrame:
	return pd.DataFrame({'user_id': [df.iloc[0]['user_id']],
	'median' : [df[['measurement']].median()]})

kvnkho / pandera.py

Created May 8, 2021 14:57

	import pandera as pa

	price_check = pa.DataFrameSchema({
	"Price": pa.Column(pa.Int, pa.Check.in_range(min_value=5,max_value=20)),
	})

	# schema: *
	def price_validation(df:pd.DataFrame) -> pd.DataFrame:
	price_check.validate(df)
	return df

kvnkho / fugue-validation.py

Last active May 8, 2021 15:12

	import pandera as pa

	price_check = pa.DataFrameSchema({
	"Price": pa.Column(pa.Int, pa.Check.in_range(min_value=5,max_value=20)),
	})

	# schema: *
	def price_validation(df:pd.DataFrame) -> pd.DataFrame:
	price_check.validate(df)
	return df

kvnkho / validation_df.py

Created May 8, 2021 15:14

	import pandas as pd

	df = pd.DataFrame({'State': ['FL','FL','FL','CA','CA','CA'],
	'City': ['Tampa', 'Orlando', 'Miami', 'Oakland', 'San Francisco', 'San Jose'],
	'Price': [8, 12, 10, 16, 20, 16]})

kvnkho / validation_by_partition.py

Last active May 30, 2021 23:29

	import pandera as pa
	from pandera import Column, Check, DataFrameSchema
	from fugue import FugueWorkflow
	from fugue_spark import SparkExecutionEngine

	price_check_FL = pa.DataFrameSchema({
	"Price": Column(pa.Float, Check.in_range(min_value=7,max_value=13)),
	})

	price_check_CA = pa.DataFrameSchema({

kvnkho / dask-report-efficient.html

Created September 28, 2021 22:08

This file has been truncated, but you can view the full file.

kvnkho / importing_pycaret.py

Last active January 10, 2022 06:15

Importing PyCaret

	from pycaret.datasets import get_data
	df = get_data("titanic")

kvnkho / compare_models.py

Created January 6, 2022 18:46

	from pycaret.classification import *

	clf = setup(data = df,
	target = "Survived",
	session_id = 123,
	silent = True,
	verbose = False,
	html = False)

	models = compare_models(fold = 5,