plot_upgrades_by_cluster.py

plot_upgrades_by_cluster.py

#!/usr/bin/env python3
"""
Create separate cluster upgrade analysis graphs for each cluster
"""
import json
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from datetime import datetime
from collections import defaultdict
import numpy as np
import os

# Read the JSON data
data = []
with open('data.json', 'r') as f:
    for line in f:
        data.append(json.loads(line))

print(f"Loaded {len(data)} upgrade records")

# Group data by cluster
clusters = defaultdict(list)
for record in data:
    cluster_name = record.get('cluster', 'unknown')
    clusters[cluster_name].append(record)

print(f"\nFound {len(clusters)} unique clusters")

# Create output directory
os.makedirs('cluster_analysis', exist_ok=True)

# Generate analysis for each cluster
for cluster_name, cluster_data in clusters.items():
    print(f"\nGenerating analysis for {cluster_name} ({len(cluster_data)} upgrades)...")

    # Sort by time
    cluster_data_sorted = sorted([d for d in cluster_data if d.get('backendTriggerTime')],
                                  key=lambda x: x['backendTriggerTime'])

    if not cluster_data_sorted:
        print(f"  Skipping {cluster_name} - no data with backendTriggerTime")
        continue

    # Create figure with same layout as main script
    fig = plt.figure(figsize=(20, 15))
    gs = fig.add_gridspec(3, 2, height_ratios=[1, 1, 1], hspace=0.3, wspace=0.3)

    ax_timeline1 = fig.add_subplot(gs[0, 0])
    ax_timeline2 = fig.add_subplot(gs[0, 1])
    ax_timeline3 = fig.add_subplot(gs[1, 0])
    ax_timeline4 = fig.add_subplot(gs[1, 1])
    ax_timeline5 = fig.add_subplot(gs[2, 0])
    ax_version = fig.add_subplot(gs[2, 1])

    cluster_display = cluster_name.replace('prod-', '').replace('-svc-1', '')
    fig.suptitle(f'Cluster Upgrade Analysis - {cluster_display.upper()}', fontsize=18, fontweight='bold')

    # Collect timeline data
    timeline_data = {
        'times': [],
        'backend_to_partial': [],
        'backend_to_completion': [],
        'hypershift_reaction': [],
        'selection_to_trigger': [],
        'backend_trigger_to_cs': []
    }

    for record in cluster_data_sorted:
        if record.get('backendTriggerTime'):
            time_val = datetime.fromisoformat(record['backendTriggerTime'].replace('Z', '+00:00'))
            timeline_data['times'].append(time_val)

            timeline_data['backend_to_partial'].append(
                record.get('backendToPartialMinutes') if record.get('backendToPartialMinutes') is not None else np.nan)
            timeline_data['backend_to_completion'].append(
                record.get('backendToCompletionMinutes') if record.get('backendToCompletionMinutes') is not None else np.nan)
            timeline_data['hypershift_reaction'].append(
                record.get('hypershiftReactionDurationMinutes') if record.get('hypershiftReactionDurationMinutes') is not None else np.nan)
            timeline_data['selection_to_trigger'].append(
                record.get('backendSelectionToTriggerMinutes') if record.get('backendSelectionToTriggerMinutes') is not None else np.nan)
            timeline_data['backend_trigger_to_cs'].append(
                record.get('backendTriggerToCsMinutes') if record.get('backendTriggerToCsMinutes') is not None else np.nan)

    # TIMELINE 1: Backend to Partial
    values = timeline_data['backend_to_partial']
    valid_indices = [i for i, v in enumerate(values) if not np.isnan(v)]
    valid_times = [timeline_data['times'][i] for i in valid_indices]
    valid_values = [values[i] for i in valid_indices]

    if valid_values:
        ax_timeline1.plot(valid_times, valid_values, marker='o', linestyle='-',
                         linewidth=1.5, markersize=5, alpha=0.7, color='#2E86AB')

        p50 = np.percentile(valid_values, 50)
        p75 = np.percentile(valid_values, 75)
        p95 = np.percentile(valid_values, 95)
        p99 = np.percentile(valid_values, 99)

        ax_timeline1.axhline(y=p50, color='green', linestyle='--', linewidth=2, label=f'P50: {p50:.1f}m', alpha=0.6)
        ax_timeline1.axhline(y=p75, color='orange', linestyle='--', linewidth=2, label=f'P75: {p75:.1f}m', alpha=0.6)
        ax_timeline1.axhline(y=p95, color='red', linestyle='--', linewidth=2, label=f'P95: {p95:.1f}m', alpha=0.6)
        ax_timeline1.axhline(y=p99, color='darkred', linestyle=':', linewidth=2, label=f'P99: {p99:.1f}m', alpha=0.6)
        ax_timeline1.set_ylim(0, p95 * 1.3)
        ax_timeline1.legend(loc='upper left', fontsize=8, framealpha=0.9)

    ax_timeline1.set_ylabel('Duration (minutes)', fontsize=11, fontweight='bold')
    ax_timeline1.set_xlabel('Backend Trigger Time', fontsize=10)
    ax_timeline1.set_title('Backend to Partial Duration Over Time', fontsize=12, fontweight='bold')
    ax_timeline1.grid(True, alpha=0.3, linestyle=':', linewidth=0.5)
    ax_timeline1.xaxis.set_major_formatter(mdates.DateFormatter('%m/%d'))
    ax_timeline1.xaxis.set_major_locator(mdates.DayLocator(interval=1))
    plt.setp(ax_timeline1.xaxis.get_majorticklabels(), rotation=45, ha='right', fontsize=9)

    # TIMELINE 2: Backend to Completion
    values = timeline_data['backend_to_completion']
    valid_indices = [i for i, v in enumerate(values) if not np.isnan(v)]
    valid_times = [timeline_data['times'][i] for i in valid_indices]
    valid_values = [values[i] for i in valid_indices]

    if valid_values:
        ax_timeline2.plot(valid_times, valid_values, marker='s', linestyle='-',
                         linewidth=1.5, markersize=5, alpha=0.7, color='#A23B72')

        p50 = np.percentile(valid_values, 50)
        p75 = np.percentile(valid_values, 75)
        p95 = np.percentile(valid_values, 95)
        p99 = np.percentile(valid_values, 99)

        ax_timeline2.axhline(y=p50, color='green', linestyle='--', linewidth=2, label=f'P50: {p50:.1f}m', alpha=0.6)
        ax_timeline2.axhline(y=p75, color='orange', linestyle='--', linewidth=2, label=f'P75: {p75:.1f}m', alpha=0.6)
        ax_timeline2.axhline(y=p95, color='red', linestyle='--', linewidth=2, label=f'P95: {p95:.1f}m', alpha=0.6)
        ax_timeline2.axhline(y=p99, color='darkred', linestyle=':', linewidth=2, label=f'P99: {p99:.1f}m', alpha=0.6)
        ax_timeline2.set_ylim(0, p95 * 1.3)
        ax_timeline2.legend(loc='upper left', fontsize=8, framealpha=0.9)

    ax_timeline2.set_ylabel('Duration (minutes)', fontsize=11, fontweight='bold')
    ax_timeline2.set_xlabel('Backend Trigger Time', fontsize=10)
    ax_timeline2.set_title('Backend to Completion Duration Over Time', fontsize=12, fontweight='bold')
    ax_timeline2.grid(True, alpha=0.3, linestyle=':', linewidth=0.5)
    ax_timeline2.xaxis.set_major_formatter(mdates.DateFormatter('%m/%d'))
    ax_timeline2.xaxis.set_major_locator(mdates.DayLocator(interval=1))
    plt.setp(ax_timeline2.xaxis.get_majorticklabels(), rotation=45, ha='right', fontsize=9)

    # TIMELINE 3: Hypershift Reaction
    values = timeline_data['hypershift_reaction']
    valid_indices = [i for i, v in enumerate(values) if not np.isnan(v)]
    valid_times = [timeline_data['times'][i] for i in valid_indices]
    valid_values = [values[i] for i in valid_indices]

    if valid_values:
        ax_timeline3.plot(valid_times, valid_values, marker='^', linestyle='-',
                         linewidth=1.5, markersize=5, alpha=0.7, color='#C73E1D')

        p50 = np.percentile(valid_values, 50)
        p75 = np.percentile(valid_values, 75)
        p95 = np.percentile(valid_values, 95)
        p99 = np.percentile(valid_values, 99)

        ax_timeline3.axhline(y=p50, color='green', linestyle='--', linewidth=2, label=f'P50: {p50:.1f}m', alpha=0.6)
        ax_timeline3.axhline(y=p75, color='orange', linestyle='--', linewidth=2, label=f'P75: {p75:.1f}m', alpha=0.6)
        ax_timeline3.axhline(y=p95, color='red', linestyle='--', linewidth=2, label=f'P95: {p95:.1f}m', alpha=0.6)
        ax_timeline3.axhline(y=p99, color='darkred', linestyle=':', linewidth=2, label=f'P99: {p99:.1f}m', alpha=0.6)
        ax_timeline3.set_ylim(0, p95 * 1.3)
        ax_timeline3.legend(loc='upper left', fontsize=8, framealpha=0.9)

    ax_timeline3.set_ylabel('Duration (minutes)', fontsize=11, fontweight='bold')
    ax_timeline3.set_xlabel('Backend Trigger Time', fontsize=10)
    ax_timeline3.set_title('Hypershift Reaction Duration Over Time', fontsize=12, fontweight='bold')
    ax_timeline3.grid(True, alpha=0.3, linestyle=':', linewidth=0.5)
    ax_timeline3.xaxis.set_major_formatter(mdates.DateFormatter('%m/%d'))
    ax_timeline3.xaxis.set_major_locator(mdates.DayLocator(interval=1))
    plt.setp(ax_timeline3.xaxis.get_majorticklabels(), rotation=45, ha='right', fontsize=9)

    # TIMELINE 4: Selection to Trigger
    values = timeline_data['selection_to_trigger']
    valid_indices = [i for i, v in enumerate(values) if not np.isnan(v)]
    valid_times = [timeline_data['times'][i] for i in valid_indices]
    valid_values = [values[i] for i in valid_indices]

    if valid_values:
        ax_timeline4.plot(valid_times, valid_values, marker='d', linestyle='-',
                         linewidth=1.5, markersize=5, alpha=0.7, color='#F18F01')

        p50 = np.percentile(valid_values, 50)
        p75 = np.percentile(valid_values, 75)
        p95 = np.percentile(valid_values, 95)
        p99 = np.percentile(valid_values, 99)

        ax_timeline4.axhline(y=p50, color='green', linestyle='--', linewidth=2, label=f'P50: {p50:.1f}m', alpha=0.6)
        ax_timeline4.axhline(y=p75, color='orange', linestyle='--', linewidth=2, label=f'P75: {p75:.1f}m', alpha=0.6)
        ax_timeline4.axhline(y=p95, color='red', linestyle='--', linewidth=2, label=f'P95: {p95:.1f}m', alpha=0.6)
        ax_timeline4.axhline(y=p99, color='darkred', linestyle=':', linewidth=2, label=f'P99: {p99:.1f}m', alpha=0.6)
        ax_timeline4.set_ylim(0, p95 * 1.3)
        ax_timeline4.legend(loc='upper left', fontsize=8, framealpha=0.9)

    ax_timeline4.set_ylabel('Duration (minutes)', fontsize=11, fontweight='bold')
    ax_timeline4.set_xlabel('Backend Trigger Time', fontsize=10)
    ax_timeline4.set_title('Selection to Trigger Duration Over Time', fontsize=12, fontweight='bold')
    ax_timeline4.grid(True, alpha=0.3, linestyle=':', linewidth=0.5)
    ax_timeline4.xaxis.set_major_formatter(mdates.DateFormatter('%m/%d'))
    ax_timeline4.xaxis.set_major_locator(mdates.DayLocator(interval=1))
    plt.setp(ax_timeline4.xaxis.get_majorticklabels(), rotation=45, ha='right', fontsize=9)

    # TIMELINE 5: Backend Trigger to CS
    values = timeline_data['backend_trigger_to_cs']
    valid_indices = [i for i, v in enumerate(values) if not np.isnan(v)]
    valid_times = [timeline_data['times'][i] for i in valid_indices]
    valid_values = [values[i] for i in valid_indices]

    if valid_values:
        ax_timeline5.plot(valid_times, valid_values, marker='p', linestyle='-',
                         linewidth=1.5, markersize=5, alpha=0.7, color='#6A994E')

        p50 = np.percentile(valid_values, 50)
        p75 = np.percentile(valid_values, 75)
        p95 = np.percentile(valid_values, 95)
        p99 = np.percentile(valid_values, 99)

        ax_timeline5.axhline(y=p50, color='green', linestyle='--', linewidth=2, label=f'P50: {p50:.1f}m', alpha=0.6)
        ax_timeline5.axhline(y=p75, color='orange', linestyle='--', linewidth=2, label=f'P75: {p75:.1f}m', alpha=0.6)
        ax_timeline5.axhline(y=p95, color='red', linestyle='--', linewidth=2, label=f'P95: {p95:.1f}m', alpha=0.6)
        ax_timeline5.axhline(y=p99, color='darkred', linestyle=':', linewidth=2, label=f'P99: {p99:.1f}m', alpha=0.6)
        ax_timeline5.set_ylim(0, p95 * 1.3)
        ax_timeline5.legend(loc='upper left', fontsize=8, framealpha=0.9)

    ax_timeline5.set_ylabel('Duration (minutes)', fontsize=11, fontweight='bold')
    ax_timeline5.set_xlabel('Backend Trigger Time', fontsize=10)
    ax_timeline5.set_title('Backend Trigger to CS Processing Time Over Time', fontsize=12, fontweight='bold')
    ax_timeline5.grid(True, alpha=0.3, linestyle=':', linewidth=0.5)
    ax_timeline5.xaxis.set_major_formatter(mdates.DateFormatter('%m/%d'))
    ax_timeline5.xaxis.set_major_locator(mdates.DayLocator(interval=1))
    plt.setp(ax_timeline5.xaxis.get_majorticklabels(), rotation=45, ha='right', fontsize=9)

    # Version Distribution
    version_counts = defaultdict(int)
    for record in cluster_data:
        version = record['version']
        version_counts[version] += 1

    versions = sorted(version_counts.keys(), key=lambda v: [int(x) for x in v.split('.')])
    counts = [version_counts[v] for v in versions]

    colors_map = {'4.19': '#2E86AB', '4.20': '#A23B72', '4.21': '#F18F01', '4.22': '#6A994E'}
    colors = [colors_map.get('.'.join(v.split('.')[:2]), '#808080') for v in versions]

    bars = ax_version.bar(range(len(versions)), counts, color=colors, alpha=0.7, edgecolor='black')
    ax_version.set_xticks(range(len(versions)))
    ax_version.set_xticklabels(versions, rotation=45, ha='right', fontsize=9)
    ax_version.set_ylabel('Number of Upgrades', fontsize=11)
    ax_version.set_title('Upgrades by Target Version', fontsize=12, fontweight='bold')
    ax_version.grid(True, alpha=0.3, axis='y')

    for bar in bars:
        height = bar.get_height()
        if height > 0:
            ax_version.text(bar.get_x() + bar.get_width()/2, height, f'{int(height)}',
                           ha='center', va='bottom', fontweight='bold', fontsize=8)

    # Collect data for statistics
    backend_partial_all = [v for v in timeline_data['backend_to_partial'] if not np.isnan(v)]
    backend_completion_all = [v for v in timeline_data['backend_to_completion'] if not np.isnan(v)]

    plt.tight_layout()

    safe_cluster_name = cluster_display.replace('/', '_').replace(' ', '_')
    output_file = f'cluster_analysis/{safe_cluster_name}_analysis.png'
    plt.savefig(output_file, dpi=300, bbox_inches='tight')
    plt.close()

    print(f"  ✓ Saved: {output_file}")

    # Generate text summary
    summary_file = f'cluster_analysis/{safe_cluster_name}_summary.txt'
    with open(summary_file, 'w') as f:
        f.write("="*70 + "\n")
        f.write(f"CLUSTER UPGRADE ANALYSIS - {cluster_display.upper()}\n")
        f.write("="*70 + "\n\n")

        f.write(f"Total Upgrades: {len(cluster_data)}\n")
        f.write(f"  Partial: {len([r for r in cluster_data if r['updateState'] == 'Partial'])}\n")
        f.write(f"  Completed: {len([r for r in cluster_data if r['updateState'] == 'Completed'])}\n\n")

        if backend_partial_all:
            f.write("Backend to Partial Duration (All Upgrades)\n")
            f.write(f"  Count: {len(backend_partial_all)}\n")
            f.write(f"  Mean: {np.mean(backend_partial_all):.2f} minutes\n")
            f.write(f"  Median (P50): {np.percentile(backend_partial_all, 50):.2f} minutes\n")
            f.write(f"  P75: {np.percentile(backend_partial_all, 75):.2f} minutes\n")
            f.write(f"  P95: {np.percentile(backend_partial_all, 95):.2f} minutes\n")
            f.write(f"  P99: {np.percentile(backend_partial_all, 99):.2f} minutes\n")
            f.write(f"  Min: {np.min(backend_partial_all):.2f} minutes\n")
            f.write(f"  Max: {np.max(backend_partial_all):.2f} minutes\n\n")

        if backend_completion_all:
            f.write("Backend to Completion Duration (Completed Upgrades Only)\n")
            f.write(f"  Count: {len(backend_completion_all)}\n")
            f.write(f"  Mean: {np.mean(backend_completion_all):.2f} minutes\n")
            f.write(f"  Median (P50): {np.percentile(backend_completion_all, 50):.2f} minutes\n")
            f.write(f"  P75: {np.percentile(backend_completion_all, 75):.2f} minutes\n")
            f.write(f"  P95: {np.percentile(backend_completion_all, 95):.2f} minutes\n")
            f.write(f"  P99: {np.percentile(backend_completion_all, 99):.2f} minutes\n")
            f.write(f"  Min: {np.min(backend_completion_all):.2f} minutes\n")
            f.write(f"  Max: {np.max(backend_completion_all):.2f} minutes\n\n")

        f.write("Version Distribution\n")
        for version in versions:
            count = version_counts[version]
            f.write(f"  {version}: {count} upgrades\n")

    print(f"  ✓ Saved: {summary_file}")

print(f"\n{'='*70}")
print(f"COMPLETE! Generated analysis for {len(clusters)} clusters")
print(f"Output directory: cluster_analysis/")
print(f"{'='*70}")