Skip to content

Instantly share code, notes, and snippets.

@Jdoz
Jdoz / clean_columns.py
Last active July 28, 2021 20:40
[Clean Names] Clean Pandas column names #python #pandas #data-cleansing
import pandas as pd
import unicodedata
import re
from typing import Hashable, List, Collection, Union
_underscorer1 = re.compile(r"(.)([A-Z][a-z]+)")
_underscorer2 = re.compile("([a-z0-9])([A-Z])")
def _camel2snake(col_name: str) -> str:
@Jdoz
Jdoz / pandas_reduce_memory.py
Last active February 6, 2020 15:51
[Pandas Memory Reduce] Convert numeric columns in pandas dataframe to smaller byte sizes. #python #pandas
def reduce_mem_usage(df, verbose=True):
"""Converts numeric columns to properly sized bytes to reduce overall dataframe size in memory"""
numerics = ["int16", "int32", "int64", "float16", "float32", "float64"]
start_mem = df.memory_usage().sum() / 1024 ** 2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":