violin plots of version activation time

violin_plots_for_upgrade_analysis.py

#!/usr/bin/env python3
"""
Violin Plot Distribution Analysis for ARO-HCP Upgrades

Generates version activation time distribution visualization using violin plots
to show the full distribution of data.

Outliers removed using IQR (Interquartile Range) method - standard statistical approach.
"""

import json
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from pathlib import Path
import warnings

warnings.filterwarnings('ignore', category=UserWarning)
sns.set_style("whitegrid")
plt.rcParams['font.size'] = 10


def load_data(data_file='data.json'):
    """Load upgrade data from JSON file."""
    with open(data_file) as f:
        data = [json.loads(line) for line in f]

    df = pd.DataFrame(data)
    df['csPolicyStartTime'] = pd.to_datetime(df['csPolicyStartTime'])
    df['upgradeDay'] = df['csPolicyStartTime'].dt.date
    df['upgradeDayLabel'] = df['csPolicyStartTime'].dt.strftime('%b %d')

    return df


def remove_outliers(series, iqr_multiplier=1.5):
    """
    Remove outliers using IQR (Interquartile Range) method.

    This is a robust statistical method that works well with skewed distributions.
    - Calculates Q1 (25th percentile) and Q3 (75th percentile)
    - IQR = Q3 - Q1
    - Removes values below Q1 - iqr_multiplier*IQR or above Q3 + iqr_multiplier*IQR

    Default iqr_multiplier=1.5 is the standard used in box plots.
    """
    if len(series) == 0 or len(series) < 2:
        return series

    q1 = series.quantile(0.25)
    q3 = series.quantile(0.75)
    iqr = q3 - q1

    # If IQR is 0 or very small, return all data
    if iqr < 0.001:
        return series

    # Calculate bounds
    lower_bound = q1 - (iqr_multiplier * iqr)
    upper_bound = q3 + (iqr_multiplier * iqr)

    return series[(series >= lower_bound) & (series <= upper_bound)]


def generate_version_activation_distribution_daily(df, output_dir='.'):
    """
    Generate violin plot showing distribution of version activation time by day.
    """
    print("\n" + "="*80)
    print("  GENERATING: Version Activation Time Distribution (Violin Plots)")
    print("="*80)

    version_activation_df = df[df['backendToPartialMinutes'].notna()].copy()

    # Calculate total time to version activation
    version_activation_df['totalToVersionActive'] = version_activation_df.apply(
        lambda row: (row['creationToBackendSelectionMinutes'] + row['backendToPartialMinutes'])
                    if pd.notna(row['creationToBackendSelectionMinutes']) and row['upgradeNumber'] == 1
                    else row['backendToPartialMinutes'],
        axis=1
    )

    # Remove outliers
    version_activation_df['clean_value'] = version_activation_df.groupby('upgradeDayLabel')['totalToVersionActive'].transform(
        lambda x: remove_outliers(x) if len(x) > 0 else x
    )
    clean_df = version_activation_df[version_activation_df['clean_value'].notna()]

    if len(clean_df) == 0:
        print("⚠ No data available after removing outliers")
        return None

    fig, ax = plt.subplots(figsize=(24, 10))

    days_order = sorted(clean_df['upgradeDayLabel'].unique())

    # Create violin plot
    sns.violinplot(data=clean_df, x='upgradeDayLabel', y='clean_value',
                  order=days_order, ax=ax, inner='box', cut=0,
                  palette='coolwarm', saturation=0.8)

    # Add annotations
    for i, day in enumerate(days_order):
        day_data = clean_df[clean_df['upgradeDayLabel'] == day]['clean_value']
        count = len(day_data)
        median = day_data.median()
        min_val = day_data.min()

        y_max = ax.get_ylim()[1]

        # Count at top
        ax.text(i, y_max * 0.95, f'n={count}',
               ha='center', va='top', fontsize=9, fontweight='bold', color='darkblue')

        # Median value just above the minimum of this violin
        ax.text(i, min_val * 0.85, f'{median:.1f}',
               ha='center', va='center', fontsize=8, fontweight='bold',
               color='darkred',
               bbox=dict(boxstyle='round,pad=0.2', facecolor='white', alpha=0.8, edgecolor='darkred'))

    # Overall median line
    overall_median = clean_df['clean_value'].median()
    ax.axhline(y=overall_median, color='red', linestyle='--', linewidth=2,
              alpha=0.6, label=f'Overall median: {overall_median:.1f} min')

    ax.set_title('Time for Selected Version to Become Active in System\n' +
                 'Distribution by day (outliers removed using IQR method)',
                 fontsize=16, fontweight='bold', pad=20)
    ax.set_xlabel('Day', fontsize=14, fontweight='bold')
    ax.set_ylabel('Time to Version Active (minutes)', fontsize=14, fontweight='bold')
    ax.tick_params(axis='x', rotation=45)
    ax.legend(loc='upper right', fontsize=12)
    ax.grid(axis='y', alpha=0.3, linestyle='--')

    plt.tight_layout()
    output_file = Path(output_dir) / "violin_version_activation_time_daily.png"
    plt.savefig(output_file, dpi=300, bbox_inches='tight', facecolor='white')
    plt.close()

    print(f"✓ Saved: {output_file.name}")
    return output_file


def main():
    print("\n" + "="*80)
    print("  ARO-HCP VIOLIN PLOT ANALYSIS")
    print("  Distribution Visualizations (Outliers Removed)")
    print("="*80)

    try:
        print("\nLoading data from data.json...")
        df = load_data()
        print(f"✓ Loaded {len(df)} upgrade records")

        output_dir = Path(__file__).parent

        activation_graph = generate_version_activation_distribution_daily(df, output_dir)

        print("\n" + "="*80)
        print("  GENERATION COMPLETE")
        print("="*80)
        print("\nGenerated visualization:")
        print(f"  {activation_graph.name}")
        print(f"     → Version activation time distribution by day")
        print("\n✓ All violin plot visualizations generated successfully!")
        print("  (Outliers removed using IQR statistical method)")
        print("="*80 + "\n")

    except Exception as e:
        print(f"\n❌ ERROR: {e}")
        import traceback
        traceback.print_exc()
        return 1

    return 0


if __name__ == '__main__':
    import sys
    sys.exit(main())