Skip to content

Instantly share code, notes, and snippets.

@machi1990
Created May 12, 2026 12:03
Show Gist options
  • Select an option

  • Save machi1990/f34442cd8c1102237264db72491d3c05 to your computer and use it in GitHub Desktop.

Select an option

Save machi1990/f34442cd8c1102237264db72491d3c05 to your computer and use it in GitHub Desktop.
upgrade pickup time after initial cluster creation analysis
#!/usr/bin/env python3
"""
Time from cluster creation to backend upgrade decision
Combines all findings into one complete view
"""
import json
import matplotlib.pyplot as plt
import numpy as np
from pathlib import Path
from collections import defaultdict, Counter
# Read the data
data_file = Path(__file__).parent / "data.json"
with open(data_file, 'r') as f:
records = [json.loads(line) for line in f]
# Categorize all clusters
all_clusters = []
zstream_by_version = defaultdict(list)
ystream_clusters = []
outliers = []
for record in records:
duration = record.get('creationToBackendSelectionMinutes')
if duration is None:
continue
cluster_name = record['clusterResourceID'].split('/')[-1]
initial_ver = record.get('initialVersion')
upgrade_ver = record.get('firstUpgradeVersion')
cluster_info = {
'name': cluster_name,
'duration': duration,
'initial': initial_ver,
'upgrade': upgrade_ver,
'env': record.get('cluster', 'unknown')
}
# Categorize
if duration >= 60:
outliers.append(cluster_info)
else:
all_clusters.append(cluster_info)
# Determine upgrade type
if initial_ver and upgrade_ver:
init_parts = initial_ver.split('.')
upgrade_parts = upgrade_ver.split('.')
if len(init_parts) >= 2 and len(upgrade_parts) >= 2:
if init_parts[0] == upgrade_parts[0] and init_parts[1] == upgrade_parts[1]:
# Z-stream
version = f"{init_parts[0]}.{init_parts[1]}"
zstream_by_version[version].append(cluster_info)
elif init_parts[0] == upgrade_parts[0]:
# Y-stream
ystream_clusters.append(cluster_info)
# Calculate statistics
all_durations = [c['duration'] for c in all_clusters]
print("="*90)
print("COMPREHENSIVE BACKEND DECISION TIME ANALYSIS")
print("="*90)
print()
# Overall stats
print(f"Total clusters: {len(all_clusters) + len(outliers)}")
print(f"Normal (<1hr): {len(all_clusters)} ({len(all_clusters)/(len(all_clusters)+len(outliers))*100:.1f}%)")
print(f"Outliers: {len(outliers)} ({len(outliers)/(len(all_clusters)+len(outliers))*100:.1f}%)")
print()
# Z-stream vs Y-stream
total_zstream = sum(len(clusters) for clusters in zstream_by_version.values())
print(f"Z-stream upgrades: {total_zstream} clusters")
print(f"Y-stream upgrades: {len(ystream_clusters)} clusters")
print()
# Version breakdown for z-stream
print("Z-STREAM BY VERSION:")
version_stats = {}
for version in sorted(zstream_by_version.keys()):
clusters = zstream_by_version[version]
durations = [c['duration'] for c in clusters]
median = np.median(durations)
version_stats[version] = {
'count': len(clusters),
'median': median,
'mean': np.mean(durations),
'min': np.min(durations),
'max': np.max(durations)
}
print(f" v{version}: {len(clusters):3} clusters, median {median:.1f} min")
print()
# Y-stream stats
if ystream_clusters:
y_durations = [c['duration'] for c in ystream_clusters]
print(f"Y-STREAM STATS:")
print(f" Count: {len(ystream_clusters)} clusters")
print(f" Median: {np.median(y_durations):.1f} min")
print(f" Range: {np.min(y_durations):.1f} - {np.max(y_durations):.1f} min")
print()
# Create comprehensive visualization
fig = plt.figure(figsize=(20, 14))
gs = fig.add_gridspec(3, 3, height_ratios=[1.2, 1.2, 1], hspace=0.35, wspace=0.3)
fig.suptitle('Comprehensive Analysis: Time from Cluster Creation to Backend Upgrade Decision',
fontsize=18, fontweight='bold', y=0.995)
# ========== Plot 1: Overall cumulative distribution ==========
ax1 = fig.add_subplot(gs[0, 0])
sorted_durations = sorted(all_durations)
cumulative_pct = np.arange(1, len(sorted_durations) + 1) / len(sorted_durations) * 100
ax1.plot(sorted_durations, cumulative_pct, linewidth=3, color='steelblue', marker='o',
markersize=2, alpha=0.7)
ax1.fill_between(sorted_durations, cumulative_pct, alpha=0.3, color='steelblue')
# Key percentiles
for pct in [50, 90, 95]:
idx = int(len(sorted_durations) * pct / 100) - 1
val = sorted_durations[idx]
ax1.axhline(y=pct, color='red', linestyle='--', alpha=0.3, linewidth=1)
ax1.axvline(x=val, color='red', linestyle='--', alpha=0.3, linewidth=1)
ax1.set_xlabel('Minutes', fontsize=11, fontweight='bold')
ax1.set_ylabel('Cumulative %', fontsize=11, fontweight='bold')
ax1.set_title('Overall: Cumulative Distribution', fontsize=12, fontweight='bold')
ax1.grid(True, alpha=0.3)
# ========== Plot 2: Z-stream by version scatter ==========
ax2 = fig.add_subplot(gs[0, 1])
colors = ['#1f77b4', '#ff7f0e', '#2ca02c']
sorted_versions = sorted(zstream_by_version.keys())
for idx, version in enumerate(sorted_versions):
durations = [c['duration'] for c in zstream_by_version[version]]
x_pos = [idx + np.random.uniform(-0.2, 0.2) for _ in durations]
ax2.scatter(x_pos, durations, alpha=0.6, s=40, color=colors[idx],
edgecolors='black', linewidth=0.5, label=f'v{version} (n={len(durations)})')
median_val = np.median(durations)
ax2.plot([idx - 0.3, idx + 0.3], [median_val, median_val],
color='red', linewidth=3, alpha=0.8, zorder=10)
ax2.text(idx, median_val + 0.3, f'{median_val:.1f}', ha='center',
fontsize=9, fontweight='bold', color='red')
ax2.set_xticks(range(len(sorted_versions)))
ax2.set_xticklabels(sorted_versions)
ax2.set_xlabel('Version', fontsize=11, fontweight='bold')
ax2.set_ylabel('Decision Time (minutes)', fontsize=11, fontweight='bold')
ax2.set_title('Z-Stream: Decision Time by Version', fontsize=12, fontweight='bold')
ax2.grid(True, alpha=0.3, axis='y')
ax2.legend(loc='upper right', fontsize=9)
# ========== Plot 3: Z-stream vs Y-stream comparison ==========
ax3 = fig.add_subplot(gs[0, 2])
z_durations = [c['duration'] for version in zstream_by_version.values() for c in version]
comparison_data = []
comparison_labels = []
comparison_counts = []
comparison_data.append(z_durations)
comparison_labels.append('Z-stream\n(patch)')
comparison_counts.append(len(z_durations))
if ystream_clusters:
comparison_data.append(y_durations)
comparison_labels.append('Y-stream\n(minor)')
comparison_counts.append(len(ystream_clusters))
positions = range(len(comparison_data))
bp = ax3.violinplot(comparison_data, positions=positions, widths=0.6,
showmedians=True, showextrema=True)
for pc in bp['bodies']:
pc.set_facecolor('lightblue')
pc.set_alpha(0.7)
bp['cmedians'].set_color('red')
bp['cmedians'].set_linewidth(2)
# Add median values
for pos, data, label, count in zip(positions, comparison_data, comparison_labels, comparison_counts):
median = np.median(data)
ax3.text(pos, median + 0.5, f'{median:.1f} min\n(n={count})',
ha='center', fontsize=10, fontweight='bold')
ax3.set_xticks(positions)
ax3.set_xticklabels(comparison_labels)
ax3.set_ylabel('Decision Time (minutes)', fontsize=11, fontweight='bold')
ax3.set_title('Z-Stream vs Y-Stream Comparison', fontsize=12, fontweight='bold')
ax3.grid(True, alpha=0.3, axis='y')
# ========== Plot 4: Version medians bar chart ==========
ax4 = fig.add_subplot(gs[1, 0])
versions = sorted(version_stats.keys())
medians = [version_stats[v]['median'] for v in versions]
counts = [version_stats[v]['count'] for v in versions]
bars = ax4.bar(range(len(versions)), medians, color=['#1f77b4', '#ff7f0e', '#2ca02c'],
alpha=0.7, edgecolor='black', linewidth=1.5)
for i, (bar, med, cnt) in enumerate(zip(bars, medians, counts)):
ax4.text(bar.get_x() + bar.get_width()/2, med + 0.2,
f'{med:.1f} min\n({cnt} clusters)',
ha='center', va='bottom', fontsize=10, fontweight='bold')
ax4.set_xticks(range(len(versions)))
ax4.set_xticklabels(versions)
ax4.set_xlabel('Version', fontsize=11, fontweight='bold')
ax4.set_ylabel('Median Decision Time (min)', fontsize=11, fontweight='bold')
ax4.set_title('Z-Stream: Median by Version', fontsize=12, fontweight='bold')
ax4.grid(True, alpha=0.3, axis='y')
# ========== Plot 5: Time buckets distribution ==========
ax5 = fig.add_subplot(gs[1, 1])
buckets = [
('0-3 min', 0, 3),
('3-5 min', 3, 5),
('5-8 min', 5, 8),
('8-10 min', 8, 10),
('10-15 min', 10, 15),
('15+ min', 15, 60),
]
bucket_labels = []
bucket_counts = []
for label, min_val, max_val in buckets:
count = sum(1 for d in all_durations if min_val <= d < max_val)
bucket_labels.append(label)
bucket_counts.append(count)
bars = ax5.bar(bucket_labels, bucket_counts, color='steelblue', alpha=0.7,
edgecolor='black', linewidth=1.5)
for bar, count in zip(bars, bucket_counts):
if count > 0:
pct = (count / len(all_durations)) * 100
ax5.text(bar.get_x() + bar.get_width()/2, bar.get_height(),
f'{count}\n({pct:.1f}%)',
ha='center', va='bottom', fontsize=9, fontweight='bold')
ax5.set_xlabel('Time Range', fontsize=11, fontweight='bold')
ax5.set_ylabel('Number of Clusters', fontsize=11, fontweight='bold')
ax5.set_title('Distribution by Time Bucket', fontsize=12, fontweight='bold')
ax5.grid(True, alpha=0.3, axis='y')
plt.setp(ax5.xaxis.get_majorticklabels(), rotation=45, ha='right')
# ========== Plot 6: Outliers ==========
ax6 = fig.add_subplot(gs[1, 2])
if outliers:
outlier_names = [c['name'][:25] for c in sorted(outliers, key=lambda x: x['duration'], reverse=True)]
outlier_hours = [c['duration']/60 for c in sorted(outliers, key=lambda x: x['duration'], reverse=True)]
y_pos = range(len(outliers))
bars = ax6.barh(y_pos, outlier_hours, color='darkred', alpha=0.7,
edgecolor='black', linewidth=1.5)
ax6.set_yticks(y_pos)
ax6.set_yticklabels(outlier_names, fontsize=9)
ax6.set_xlabel('Hours', fontsize=11, fontweight='bold')
ax6.set_title(f'Outliers (>= 1 hour): {len(outliers)} clusters', fontsize=12, fontweight='bold', color='darkred')
ax6.grid(True, alpha=0.3, axis='x')
for i, (bar, val) in enumerate(zip(bars, outlier_hours)):
days = val / 24
ax6.text(val, i, f' {days:.1f} days', va='center', fontsize=9, fontweight='bold')
else:
ax6.text(0.5, 0.5, 'No outliers', ha='center', va='center',
transform=ax6.transAxes, fontsize=12, style='italic')
ax6.set_xticks([])
ax6.set_yticks([])
# ========== Plot 7: COMPREHENSIVE SUMMARY ==========
ax7 = fig.add_subplot(gs[2, :])
ax7.axis('off')
summary = f"""
┌{'─'*115}┐
│{' '*45}COMPREHENSIVE SUMMARY{' '*48}│
├{'─'*115}┤
│ │
│ OVERALL STATISTICS │
│ {'─'*111}│
│ Total Clusters: {len(all_clusters) + len(outliers):>4} │
│ Normal (<1hr): {len(all_clusters):>4} ({len(all_clusters)/(len(all_clusters)+len(outliers))*100:>5.1f}%) → Median: {np.median(all_durations):.1f} min, 95th percentile: {np.percentile(all_durations, 95):.1f} min│
│ Outliers (>=1hr): {len(outliers):>4} ({len(outliers)/(len(all_clusters)+len(outliers))*100:>5.1f}%) │
│ │
├{'─'*115}┤
│ DECISION TIME BY CLUSTER TYPE │
│ {'─'*111}│
│ │
│ Z-STREAM UPGRADES (patch, e.g., 4.20.17 → 4.20.21): │
│ • Total clusters: {total_zstream:>4} ({total_zstream/len(all_clusters)*100:>5.1f}% of normal clusters) │
│ • Overall median: {np.median(z_durations):>6.2f} min │
"""
# Add version breakdown
for version in sorted(version_stats.keys()):
stats = version_stats[version]
summary += f"│ • Version {version}: {stats['median']:>6.2f} min (n={stats['count']:>3}) → Range: {stats['min']:.1f}-{stats['max']:.1f} min{' '*(30)}│\n"
if ystream_clusters:
summary += f"""│ │
│ Y-STREAM UPGRADES (cross-minor, e.g., 4.20.20 → 4.21.14): │
│ • Total clusters: {len(ystream_clusters):>4} ({len(ystream_clusters)/len(all_clusters)*100:>5.1f}% of normal clusters) │
│ • Median: {np.median(y_durations):>6.2f} min │
│ • Range: {np.min(y_durations):>6.2f} - {np.max(y_durations):.2f} min │
│ • Pattern: cp-ystream-upgrade-4-21-* │
"""
summary += f"""│ │
├{'─'*115}┤
│ KEY FINDINGS │
│ {'─'*111}│
│ │
│ 1. OVERALL: For newly created clusters, the RP backend typically decides an upgrade is necessary │
│ within 6-11 minutes of cluster creation. │
│ │
│ 2. VERSION IMPACT (Z-stream): Version 4.20 takes significantly longer than 4.19 and 4.21 │
│ • v4.20: {version_stats['4.20']['median']:.1f} min median (slowest) │
│ • v4.21: {version_stats['4.21']['median']:.1f} min median (fastest) │
│ • v4.19: {version_stats['4.19']['median']:.1f} min median (middle) │
│ → v4.20 is ~{(version_stats['4.20']['median']/version_stats['4.21']['median'] - 1)*100:.0f}% slower than v4.21 │
│ │
"""
if ystream_clusters:
summary += f"""│ 3. UPGRADE TYPE: Y-stream upgrades take ~2x longer than Z-stream │
│ • Z-stream (patch): ~{np.median(z_durations):.1f} min median │
│ • Y-stream (cross-minor): ~{np.median(y_durations):.1f} min median │
│ │
"""
summary += f"""│ 4. NAMING PATTERNS: │
│ • cluster-zstream-*: Fast decisions (2-11 min), {total_zstream} clusters │
│ • cp-ystream-upgrade-*: Slower decisions (12-13 min), {len(ystream_clusters)} clusters │
│ │
└{'─'*115}┘
"""
ax7.text(0.02, 0.98, summary, transform=ax7.transAxes,
fontsize=9.5, verticalalignment='top', family='monospace',
bbox=dict(boxstyle='round', facecolor='lightgreen', alpha=0.2, pad=1))
plt.savefig('analysis.png', dpi=300, bbox_inches='tight')
print("\n" + "="*90)
print("Analysis graph saved as 'analysis.png'")
print("="*90)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment