kvnkho

from fugue import transform
from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()
results = transform(df, 
                    transform_img,
                    schema="*",
                    engine=spark)
          

kvnkho

import requests
from typing import Any, Dict, Iterable
from PIL import Image
from io import BytesIO

def transform_img(df: List[Dict[str, Any]]) -> Iterable[str, Any]:
    for row in df:
        try:
            response = requests.get(row["ImgUrl"], timeout=5)
            img = Image.open(BytesIO(response.content))

kvnkho

import pandas as pd

df = pd.DataFrame({"col1": [1,2,3], "col2": ["a", "b", "c"]})
df2 = pd.DataFrame({"col1": [1,2,3], "col2": ["d", "e", "f"]})

df.to_parquet("/tmp/test1.parquet")
df2.to_parquet("/tmp/test2.parquet")

from fugue_sql import fsql
from typing import Iterable, List, Any, Dict

kvnkho

fugue_profile(dask_df, partition={"by":["a","b"]}, engine=client)

kvnkho

import coiled

coiled.create_software_environment(
    name="profiling",
    pip=["fugue[dask]", "whylogs"],
)

kvnkho

from dask.distributed import Client
from coiled import Cluster
from whylogs.api.fugue import fugue_profile

cluster = Cluster(name="quickstart", software="profiling", n_workers=2)
client = Client(cluster)

fugue_profile(df, engine=client).to_pandas()

kvnkho

fugue_profile(spark_df, partition={"by":["a","b"]}, engine=spark_session)

kvnkho

from whylogs.api.fugue import fugue_profile
from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()
fugue_profile(spark_df, engine=spark)

kvnkho

from whylogs.api.fugue import fugue_profile

fugue_profile(pandas_df).to_pandas()

kvnkho

import pandas as pd
data = {
    "animal": ["cat", "hawk", "snake", "cat"],
    "legs": [4, 2, 0, 4],
    "weight": [4.3, 1.8, 1.3, 4.1],
}

df = pd.DataFrame(data)

import whylogs as why