basic dataset exploration

dataset-exploration.py

import pandas as pd
import os
from matplotlib import pyplot as plt
import numpy as np

def read_csvs(dirPath):
    # Reads in the OpenFace output csv files

    # read input directory
    files = os.listdir(dirPath)

    # get the files that are .csv files and sort them numerically
    csvs = [csv for csv in files if csv.endswith('.csv')]
    csvs.sort(key = lambda x: int(x[1:-4]))
    
    # create a merged dataframe that will hold all the .csv files
    for inx,csv in enumerate(csvs):
        
        if inx == 0:
            merged_df = pd.read_csv(dirPath + csv)
            merged_df['video'] = 0 
        else :
            temp_df = pd.read_csv(dirPath + csv)
            temp_df['video'] = inx
            merged_df = merged_df.append(temp_df)
    # return the merged df 
    return merged_df.reset_index(drop=True)

# load happy data
happyPath = 'dataset/happy_frames_openface/'
happy_df = read_csvs(happyPath)

# load nervous data
nervousPath = 'dataset/nervous_frames_openface/'
nervous_df = read_csvs(nervousPath)

# Extract just the AUs from the .csv outputs
AUs = [' AU01_r', ' AU02_r', ' AU04_r', ' AU05_r', ' AU06_r', ' AU07_r',
       ' AU09_r', ' AU10_r', ' AU12_r', ' AU14_r', ' AU15_r', ' AU17_r',
       ' AU20_r', ' AU23_r', ' AU25_r', ' AU26_r', ' AU45_r', 'video']
happy_au_df = happy_df.loc[:,AUs]
nervous_au_df = nervous_df.loc[:, AUs]

# Calculate the maximum AU intensity per AU per video
happy_max_au = happy_au_df.groupby('video').max()
nervous_max_au = nervous_au_df.groupby('video').max() 

# Count the number of occurences that are above 2.5
happy_au_arr = (happy_max_au > 2.5).sum()
nervous_au_arr = (nervous_max_au > 2.5).sum()

# Create plotting vectors
x1 = [' AU01', ' AU02', ' AU04', ' AU05', ' AU06', ' AU07',
       ' AU09', ' AU10', ' AU12', ' AU14', ' AU15', ' AU17',
       ' AU20', ' AU23', ' AU25', ' AU26', ' AU45']

# Calculate proportional representation
y1 = happy_au_arr.values / happy_au_arr.sum()

y2 = nervous_au_arr.values / nervous_au_arr.sum()

# Set the stage for plotting and then plot the data
ind = np.arange(happy_au_arr.index.size)
width = 0.3

plt.figure(num=1, figsize=(10,7),dpi=75)
p1 = plt.bar(ind,y1, width = width, label = 'Happy')
plt.xticks(rotation = 90)
p2 = plt.bar(ind+width,y2, width = width, label = 'Nervous')
plt.ylabel('Proportion')
plt.title('Action Unit Occurence By Smile Type')
plt.xticks(ind + width / 2, x1)
plt.legend(loc='best')