H200 GPU EC2 availability dashboard across US AWS regions

h200_ec2_availability.py

#!/usr/bin/env python3
"""
h200_ec2_availability.py

Read-only collector for H200 GPU EC2 availability signals across US AWS regions.

Signals collected:
  1. Instance type offerings by AZ ID (is the type offered in that AZ?)
  2. EC2 Spot placement score for an H200 fleet (1-10 likelihood signal, not a guarantee)
  3. EC2 Capacity Block offerings for future reservations (actual purchasable slots)

This script deliberately avoids RunInstances/CreateCapacityReservation probes because they
can create billable resources.
"""

from __future__ import annotations

import argparse
import concurrent.futures as futures
import csv
import json
import sys
from collections import defaultdict
from datetime import datetime, timezone, timedelta
from decimal import Decimal, InvalidOperation
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple

import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError, NoCredentialsError


DEFAULT_H200_INSTANCE_TYPES = ["p5e.48xlarge", "p5en.48xlarge"]

BOTO_CONFIG = Config(
    retries={"max_attempts": 10, "mode": "adaptive"},
    connect_timeout=5,
    read_timeout=60,
    user_agent_extra="h200-ec2-availability-dashboard/0.1",
)


def utc_now() -> datetime:
    return datetime.now(timezone.utc)


def iso_utc(value: Any) -> Optional[str]:
    if value is None:
        return None
    if isinstance(value, datetime):
        if value.tzinfo is None:
            value = value.replace(tzinfo=timezone.utc)
        return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
    return str(value)


def json_default(value: Any) -> Any:
    if isinstance(value, datetime):
        return iso_utc(value)
    if isinstance(value, Decimal):
        return float(value)
    return str(value)


def parse_csv_list(value: str) -> List[str]:
    return [item.strip() for item in value.split(",") if item.strip()]


def money_to_decimal(value: Any) -> Optional[Decimal]:
    if value is None:
        return None
    try:
        return Decimal(str(value))
    except (InvalidOperation, ValueError):
        return None


def make_session(profile: Optional[str]) -> boto3.Session:
    if profile:
        return boto3.Session(profile_name=profile)
    return boto3.Session()


def paginate(client: Any, operation_name: str, result_key: str, **kwargs: Any) -> Iterable[Dict[str, Any]]:
    """
    Generic paginator wrapper. Some EC2 APIs support Boto3 paginators; some only expose NextToken.
    """
    if client.can_paginate(operation_name):
        paginator = client.get_paginator(operation_name)
        for page in paginator.paginate(**kwargs):
            for item in page.get(result_key, []):
                yield item
        return

    method = getattr(client, operation_name)
    next_token = None
    while True:
        request = dict(kwargs)
        if next_token:
            request["NextToken"] = next_token
        page = method(**request)
        for item in page.get(result_key, []):
            yield item
        next_token = page.get("NextToken")
        if not next_token:
            break


def discover_us_regions(
    session: boto3.Session,
    include_govcloud: bool,
    include_not_opted_in: bool,
) -> List[str]:
    """
    Discovers US commercial regions from EC2 DescribeRegions.
    GovCloud generally requires GovCloud-partition credentials and is excluded by default.
    """
    ec2 = session.client("ec2", region_name="us-east-1", config=BOTO_CONFIG)
    response = ec2.describe_regions(AllRegions=True)

    regions: List[str] = []
    for region in response.get("Regions", []):
        name = region.get("RegionName", "")
        opt_in_status = region.get("OptInStatus", "opt-in-not-required")

        if not name.startswith("us-"):
            continue
        if not include_govcloud and name.startswith("us-gov-"):
            continue
        if not include_not_opted_in and opt_in_status not in (
            "opt-in-not-required",
            "opted-in",
            None,
        ):
            continue

        regions.append(name)

    return sorted(set(regions))


def get_availability_zones(ec2: Any) -> Tuple[Dict[str, Dict[str, Any]], Dict[str, str]]:
    response = ec2.describe_availability_zones(
        AllAvailabilityZones=False,
        Filters=[{"Name": "state", "Values": ["available"]}],
    )

    az_by_id: Dict[str, Dict[str, Any]] = {}
    az_name_to_id: Dict[str, str] = {}

    for az in response.get("AvailabilityZones", []):
        az_id = az.get("ZoneId")
        az_name = az.get("ZoneName")
        if not az_id or not az_name:
            continue
        az_by_id[az_id] = {
            "availability_zone_id": az_id,
            "availability_zone_name": az_name,
            "zone_type": az.get("ZoneType"),
            "state": az.get("State"),
        }
        az_name_to_id[az_name] = az_id

    return az_by_id, az_name_to_id


def describe_h200_instance_specs(ec2: Any, instance_types: Sequence[str]) -> Dict[str, Dict[str, Any]]:
    """
    Instance type metadata is regional, so query each target type in each region.
    """
    specs: Dict[str, Dict[str, Any]] = {}

    for instance_type in instance_types:
        specs[instance_type] = {
            "instance_type": instance_type,
            "valid_in_region": False,
            "vcpu": None,
            "memory_gib": None,
            "gpu_names": None,
            "gpu_count": None,
            "total_gpu_memory_gib": None,
            "supported_usage_classes": None,
            "efa_supported": None,
            "network_performance": None,
            "error": None,
        }

        try:
            response = ec2.describe_instance_types(InstanceTypes=[instance_type])
            instance_info = response.get("InstanceTypes", [])
            if not instance_info:
                continue

            info = instance_info[0]
            gpu_info = info.get("GpuInfo", {})
            gpus = gpu_info.get("Gpus", [])
            total_gpu_memory_mib = gpu_info.get("TotalGpuMemoryInMiB")
            if total_gpu_memory_mib is None:
                total_gpu_memory_mib = sum(
                    gpu.get("Count", 0) * gpu.get("MemoryInfo", {}).get("SizeInMiB", 0)
                    for gpu in gpus
                )

            specs[instance_type].update(
                {
                    "valid_in_region": True,
                    "vcpu": info.get("VCpuInfo", {}).get("DefaultVCpus"),
                    "memory_gib": round(info.get("MemoryInfo", {}).get("SizeInMiB", 0) / 1024, 2)
                    if info.get("MemoryInfo", {}).get("SizeInMiB") is not None
                    else None,
                    "gpu_names": ",".join(
                        sorted({gpu.get("Name", "") for gpu in gpus if gpu.get("Name")})
                    ),
                    "gpu_count": sum(gpu.get("Count", 0) for gpu in gpus) if gpus else None,
                    "total_gpu_memory_gib": round(total_gpu_memory_mib / 1024, 2)
                    if total_gpu_memory_mib
                    else None,
                    "supported_usage_classes": ",".join(info.get("SupportedUsageClasses", [])),
                    "efa_supported": info.get("NetworkInfo", {}).get("EfaSupported"),
                    "network_performance": info.get("NetworkInfo", {}).get("NetworkPerformance"),
                }
            )

        except ClientError as exc:
            specs[instance_type]["error"] = format_client_error(exc)

    return specs


def get_instance_type_offerings_by_az_id(
    ec2: Any,
    instance_types: Sequence[str],
) -> Dict[str, set]:
    offered_by_type: Dict[str, set] = defaultdict(set)

    for offering in paginate(
        ec2,
        "describe_instance_type_offerings",
        "InstanceTypeOfferings",
        LocationType="availability-zone-id",
        Filters=[{"Name": "instance-type", "Values": list(instance_types)}],
    ):
        instance_type = offering.get("InstanceType")
        location = offering.get("Location")
        if instance_type and location:
            offered_by_type[instance_type].add(location)

    return offered_by_type


def get_spot_scores_by_az_id(
    ec2: Any,
    region: str,
    instance_types: Sequence[str],
    target_instance_count: int,
) -> Tuple[Dict[str, int], Optional[str]]:
    """
    Scores the H200 fleet across AZs in this region. This is a Spot-only likelihood signal.
    """
    if not instance_types:
        return {}, None

    try:
        response_items = paginate(
            ec2,
            "get_spot_placement_scores",
            "SpotPlacementScores",
            InstanceTypes=list(instance_types),
            TargetCapacity=target_instance_count,
            TargetCapacityUnitType="units",
            SingleAvailabilityZone=True,
            RegionNames=[region],
            MaxResults=1000,
        )
        scores: Dict[str, int] = {}
        for score in response_items:
            az_id = score.get("AvailabilityZoneId")
            if az_id:
                scores[az_id] = score.get("Score")
        return scores, None
    except ClientError as exc:
        return {}, format_client_error(exc)
    except BotoCoreError as exc:
        return {}, str(exc)


def get_capacity_block_offerings(
    ec2: Any,
    instance_type: str,
    instance_count: int,
    duration_hours: int,
    start_date_range: datetime,
    end_date_range: datetime,
    max_offerings: int,
    all_availability_zones: bool,
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
    """
    Finds purchasable EC2 Capacity Block offerings matching the query.
    Does not purchase anything.
    """
    if not hasattr(ec2, "describe_capacity_block_offerings"):
        return [], "Boto3/botocore is too old for describe_capacity_block_offerings; upgrade boto3 and botocore."

    params = {
        "InstanceType": instance_type,
        "InstanceCount": instance_count,
        "CapacityDurationHours": duration_hours,
        "StartDateRange": start_date_range,
        "EndDateRange": end_date_range,
        "AllAvailabilityZones": all_availability_zones,
        "MaxResults": min(max(max_offerings, 1), 1000),
    }

    try:
        offerings: List[Dict[str, Any]] = []
        for offering in paginate(
            ec2,
            "describe_capacity_block_offerings",
            "CapacityBlockOfferings",
            **params,
        ):
            offerings.append(offering)
            if len(offerings) >= max_offerings:
                break
        return offerings, None
    except ClientError as exc:
        return [], format_client_error(exc)
    except BotoCoreError as exc:
        return [], str(exc)


def summarize_capacity_blocks_for_row(capacity_blocks: List[Dict[str, Any]]) -> Dict[str, Any]:
    if not capacity_blocks:
        return {
            "capacity_block_offering_count": 0,
            "capacity_block_next_start_utc": None,
            "capacity_block_next_end_utc": None,
            "capacity_block_lowest_upfront_fee": None,
            "capacity_block_currency": None,
        }

    sorted_by_start = sorted(
        capacity_blocks,
        key=lambda item: item.get("start_date_utc") or "9999-12-31T00:00:00Z",
    )
    next_block = sorted_by_start[0]

    lowest_fee: Optional[Decimal] = None
    currency = None
    for offering in capacity_blocks:
        fee = money_to_decimal(offering.get("upfront_fee"))
        if fee is not None and (lowest_fee is None or fee < lowest_fee):
            lowest_fee = fee
            currency = offering.get("currency")

    return {
        "capacity_block_offering_count": len(capacity_blocks),
        "capacity_block_next_start_utc": next_block.get("start_date_utc"),
        "capacity_block_next_end_utc": next_block.get("end_date_utc"),
        "capacity_block_lowest_upfront_fee": str(lowest_fee) if lowest_fee is not None else None,
        "capacity_block_currency": currency,
    }


def normalize_capacity_block_offering(
    region: str,
    offering: Dict[str, Any],
    az_name_to_id: Dict[str, str],
) -> Dict[str, Any]:
    az_name = offering.get("AvailabilityZone")
    return {
        "region": region,
        "availability_zone_name": az_name,
        "availability_zone_id": az_name_to_id.get(az_name),
        "zone_type": offering.get("ZoneType"),
        "instance_type": offering.get("InstanceType"),
        "instance_count": offering.get("InstanceCount"),
        "start_date_utc": iso_utc(offering.get("StartDate")),
        "end_date_utc": iso_utc(offering.get("EndDate")),
        "duration_hours": offering.get("CapacityBlockDurationHours"),
        "duration_minutes": offering.get("CapacityBlockDurationMinutes"),
        "upfront_fee": offering.get("UpfrontFee"),
        "currency": offering.get("CurrencyCode"),
        "tenancy": offering.get("Tenancy"),
        "capacity_block_offering_id": offering.get("CapacityBlockOfferingId"),
        "ultraserver_type": offering.get("UltraserverType"),
        "ultraserver_count": offering.get("UltraserverCount"),
    }


def format_client_error(exc: ClientError) -> str:
    error = exc.response.get("Error", {})
    code = error.get("Code", "ClientError")
    message = error.get("Message", str(exc))
    return f"{code}: {message}"


def collect_region(
    session: boto3.Session,
    region: str,
    instance_types: Sequence[str],
    target_instance_count: int,
    include_spot: bool,
    capacity_block_days: int,
    capacity_block_start_days: int,
    capacity_block_duration_hours: int,
    capacity_block_max_offerings: int,
    capacity_block_all_azs: bool,
) -> Dict[str, Any]:
    region_errors: List[str] = []
    ec2 = session.client("ec2", region_name=region, config=BOTO_CONFIG)

    try:
        az_by_id, az_name_to_id = get_availability_zones(ec2)
    except (ClientError, BotoCoreError) as exc:
        return {
            "region": region,
            "records": [],
            "capacity_block_offerings": [],
            "errors": [f"Failed to describe AZs: {format_client_error(exc) if isinstance(exc, ClientError) else str(exc)}"],
        }

    specs = describe_h200_instance_specs(ec2, instance_types)

    try:
        offered_by_type = get_instance_type_offerings_by_az_id(ec2, instance_types)
    except (ClientError, BotoCoreError) as exc:
        offered_by_type = defaultdict(set)
        region_errors.append(
            f"Failed to describe instance type offerings: {format_client_error(exc) if isinstance(exc, ClientError) else str(exc)}"
        )

    spot_scores_by_az_id: Dict[str, int] = {}
    if include_spot:
        spot_types = [t for t in instance_types if offered_by_type.get(t)]
        spot_scores_by_az_id, spot_error = get_spot_scores_by_az_id(
            ec2,
            region,
            spot_types,
            target_instance_count,
        )
        if spot_error:
            region_errors.append(f"Spot placement score unavailable: {spot_error}")

    capacity_blocks_by_type_az: Dict[Tuple[str, Optional[str]], List[Dict[str, Any]]] = defaultdict(list)
    normalized_capacity_blocks: List[Dict[str, Any]] = []

    if capacity_block_days > 0:
        now = utc_now()
        start_range = now + timedelta(days=capacity_block_start_days)
        end_range = now + timedelta(days=capacity_block_days)

        for instance_type in instance_types:
            offerings, cb_error = get_capacity_block_offerings(
                ec2=ec2,
                instance_type=instance_type,
                instance_count=target_instance_count,
                duration_hours=capacity_block_duration_hours,
                start_date_range=start_range,
                end_date_range=end_range,
                max_offerings=capacity_block_max_offerings,
                all_availability_zones=capacity_block_all_azs,
            )
            if cb_error:
                region_errors.append(f"Capacity Block offerings unavailable for {instance_type}: {cb_error}")

            for offering in offerings:
                normalized = normalize_capacity_block_offering(region, offering, az_name_to_id)
                normalized_capacity_blocks.append(normalized)
                capacity_blocks_by_type_az[
                    (normalized.get("instance_type"), normalized.get("availability_zone_id"))
                ].append(normalized)

    records: List[Dict[str, Any]] = []
    all_az_ids = sorted(set(az_by_id.keys()) | set(spot_scores_by_az_id.keys()))

    # Add AZ IDs seen only from capacity block offerings, if any.
    for _, az_id in capacity_blocks_by_type_az.keys():
        if az_id:
            all_az_ids.append(az_id)
    all_az_ids = sorted(set(all_az_ids))

    for instance_type in instance_types:
        spec = specs.get(instance_type, {})
        for az_id in all_az_ids:
            az_meta = az_by_id.get(az_id, {})
            capacity_blocks = capacity_blocks_by_type_az.get((instance_type, az_id), [])
            row = {
                "generated_at_utc": iso_utc(utc_now()),
                "region": region,
                "availability_zone_id": az_id,
                "availability_zone_name": az_meta.get("availability_zone_name"),
                "zone_type": az_meta.get("zone_type"),
                "instance_type": instance_type,
                "valid_instance_type_in_region": spec.get("valid_in_region"),
                "offered_in_az": az_id in offered_by_type.get(instance_type, set()),
                "spot_placement_score_h200_fleet": spot_scores_by_az_id.get(az_id),
                "target_instance_count": target_instance_count,
                "vcpu": spec.get("vcpu"),
                "memory_gib": spec.get("memory_gib"),
                "gpu_names": spec.get("gpu_names"),
                "gpu_count": spec.get("gpu_count"),
                "total_gpu_memory_gib": spec.get("total_gpu_memory_gib"),
                "supported_usage_classes": spec.get("supported_usage_classes"),
                "efa_supported": spec.get("efa_supported"),
                "network_performance": spec.get("network_performance"),
                "instance_spec_error": spec.get("error"),
            }
            row.update(summarize_capacity_blocks_for_row(capacity_blocks))
            records.append(row)

    return {
        "region": region,
        "records": records,
        "capacity_block_offerings": normalized_capacity_blocks,
        "errors": region_errors,
    }


HTML_TEMPLATE = """\
<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8" />
  <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
  <title>H200 EC2 Availability Dashboard</title>
  <style>
    :root {
      --bg: #0f1117; --surface: #1a1d27; --surface2: #22263a; --border: #2e3350;
      --accent: #6c7fff; --green: #22d17a; --red: #f25c5c; --yellow: #f5c542;
      --text: #e2e6f0; --muted: #7b839e; --card-radius: 12px;
    }
    * { box-sizing: border-box; margin: 0; padding: 0; }
    body { background: var(--bg); color: var(--text); font-family: 'Segoe UI', system-ui, -apple-system, sans-serif; font-size: 14px; min-height: 100vh; }
    header { background: linear-gradient(135deg, #1a1d27 0%, #0f1117 100%); border-bottom: 1px solid var(--border); padding: 24px 32px 20px; display: flex; align-items: center; justify-content: space-between; gap: 16px; flex-wrap: wrap; }
    .header-left h1 { font-size: 22px; font-weight: 700; letter-spacing: -0.3px; background: linear-gradient(135deg, #6c7fff, #a78bfa); -webkit-background-clip: text; -webkit-text-fill-color: transparent; background-clip: text; }
    .header-left p { color: var(--muted); font-size: 12px; margin-top: 4px; }
    .header-meta { display: flex; gap: 16px; align-items: center; flex-wrap: wrap; }
    .meta-chip { background: var(--surface2); border: 1px solid var(--border); border-radius: 20px; padding: 5px 12px; font-size: 12px; color: var(--muted); }
    .meta-chip span { color: var(--text); font-weight: 600; }
    main { padding: 24px 32px; max-width: 1400px; margin: 0 auto; }
    .summary-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 16px; margin-bottom: 28px; }
    .stat-card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--card-radius); padding: 20px; position: relative; overflow: hidden; }
    .stat-card::before { content: ''; position: absolute; top: 0; left: 0; right: 0; height: 3px; }
    .stat-card.blue::before  { background: linear-gradient(90deg, #6c7fff, #a78bfa); }
    .stat-card.green::before { background: linear-gradient(90deg, #22d17a, #4ade80); }
    .stat-card.orange::before{ background: linear-gradient(90deg, #ff8c42, #f5c542); }
    .stat-card.purple::before{ background: linear-gradient(90deg, #a78bfa, #e879f9); }
    .stat-card.red::before   { background: linear-gradient(90deg, #f25c5c, #fb923c); }
    .stat-label { font-size: 11px; text-transform: uppercase; letter-spacing: 0.8px; color: var(--muted); margin-bottom: 8px; }
    .stat-value { font-size: 32px; font-weight: 700; line-height: 1; }
    .stat-sub { font-size: 11px; color: var(--muted); margin-top: 6px; }
    .section-title { font-size: 15px; font-weight: 600; color: var(--text); margin-bottom: 14px; display: flex; align-items: center; gap: 8px; }
    .section-title::after { content: ''; flex: 1; height: 1px; background: var(--border); }
    .filters { display: flex; gap: 10px; flex-wrap: wrap; margin-bottom: 20px; align-items: center; }
    .filter-label { color: var(--muted); font-size: 12px; }
    .filter-btn { background: var(--surface2); border: 1px solid var(--border); border-radius: 20px; padding: 5px 14px; font-size: 12px; color: var(--muted); cursor: pointer; transition: all .15s; }
    .filter-btn:hover, .filter-btn.active { background: var(--accent); border-color: var(--accent); color: #fff; }
    .card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--card-radius); overflow: hidden; margin-bottom: 28px; }
    table { width: 100%; border-collapse: collapse; }
    thead tr { background: var(--surface2); }
    th { padding: 10px 14px; text-align: left; font-size: 11px; text-transform: uppercase; letter-spacing: 0.6px; color: var(--muted); white-space: nowrap; font-weight: 600; }
    td { padding: 11px 14px; border-top: 1px solid var(--border); font-size: 13px; vertical-align: middle; }
    tr:hover td { background: rgba(255,255,255,.02); }
    .badge { display: inline-flex; align-items: center; gap: 4px; border-radius: 6px; padding: 2px 8px; font-size: 11px; font-weight: 600; white-space: nowrap; }
    .badge-green  { background: rgba(34,209,122,.12); color: #22d17a; border: 1px solid rgba(34,209,122,.25); }
    .badge-red    { background: rgba(242,92,92,.12);  color: #f25c5c; border: 1px solid rgba(242,92,92,.25); }
    .badge-yellow { background: rgba(245,197,66,.12); color: #f5c542; border: 1px solid rgba(245,197,66,.25); }
    .badge-blue   { background: rgba(108,127,255,.12); color: #6c7fff; border: 1px solid rgba(108,127,255,.25); }
    .badge-muted  { background: rgba(123,131,158,.12); color: #7b839e; border: 1px solid rgba(123,131,158,.2); }
    .score-bar-wrap { display: flex; align-items: center; gap: 8px; }
    .score-bar { height: 6px; width: 60px; background: var(--border); border-radius: 99px; overflow: hidden; }
    .score-bar-fill { height: 100%; border-radius: 99px; }
    .score-num { font-size: 12px; font-weight: 600; min-width: 18px; }
    .cb-price { font-weight: 700; color: var(--green); }
    .cb-id { font-family: monospace; font-size: 11px; color: var(--muted); }
    .notes-list { list-style: none; display: flex; flex-direction: column; gap: 8px; }
    .notes-list li { background: var(--surface2); border: 1px solid var(--border); border-left: 3px solid var(--accent); border-radius: 6px; padding: 10px 14px; font-size: 12px; color: var(--muted); line-height: 1.6; }
    .error-card { background: rgba(242,92,92,.08); border: 1px solid rgba(242,92,92,.25); border-left: 3px solid #f25c5c; border-radius: 8px; padding: 12px 16px; margin-bottom: 28px; font-size: 12px; color: #f25c5c; }
    .error-card strong { display: block; margin-bottom: 4px; }
    .error-card p { color: var(--muted); line-height: 1.5; word-break: break-word; }
    .two-col { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; margin-bottom: 28px; }
    @media (max-width: 900px) { .two-col { grid-template-columns: 1fr; } }
    .spec-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; padding: 16px 20px; }
    .spec-item .spec-k { font-size: 11px; text-transform: uppercase; letter-spacing: 0.6px; color: var(--muted); margin-bottom: 3px; }
    .spec-item .spec-v { font-size: 15px; font-weight: 600; }
    tr.hidden { display: none; }
    footer { text-align: center; padding: 24px; color: var(--muted); font-size: 11px; border-top: 1px solid var(--border); margin-top: 12px; }
  </style>
</head>
<body>
<script>const DATA = __DATA_JSON__;</script>
<header>
  <div class="header-left">
    <h1>&#x26A1; H200 EC2 Availability Dashboard</h1>
    <p id="generated-at">Generated at &#8212;</p>
  </div>
  <div class="header-meta">
    <div class="meta-chip">Target Count: <span id="target-count">&#8212;</span></div>
    <div class="meta-chip">CB Window: <span id="cb-window">&#8212;</span></div>
    <div class="meta-chip">CB Duration: <span id="cb-duration">&#8212;</span></div>
  </div>
</header>
<main>
  <div class="summary-grid" id="summary-grid"></div>
  <div id="error-section"></div>
  <div class="two-col">
    <div>
      <div class="section-title">Instance Specifications</div>
      <div class="card"><div class="spec-grid" id="spec-grid"></div></div>
    </div>
    <div>
      <div class="section-title">Capacity Block Query Config</div>
      <div class="card"><div class="spec-grid" id="cb-config-grid"></div></div>
    </div>
  </div>
  <div class="section-title">AZ Availability by Region</div>
  <div class="filters">
    <span class="filter-label">Filter by region:</span>
    <button class="filter-btn active" onclick="filterRegion('all',this)">All</button>
    <span id="region-filters"></span>
    <span class="filter-label" style="margin-left:12px">Instance type:</span>
    <button class="filter-btn active" onclick="filterType('all',this)">All</button>
    <span id="type-filters"></span>
  </div>
  <div class="card" style="margin-bottom:28px">
    <table>
      <thead><tr><th>Region</th><th>AZ</th><th>AZ ID</th><th>Instance Type</th><th>Valid in Region</th><th>Offered in AZ</th><th>Spot Score</th><th>Usage Classes</th><th>EFA</th><th>CB Offerings</th><th>Next CB Start</th><th>Lowest CB Fee</th></tr></thead>
      <tbody id="avail-tbody"></tbody>
    </table>
  </div>
  <div class="section-title">Capacity Block Offerings</div>
  <div class="card" style="margin-bottom:28px">
    <table>
      <thead><tr><th>Region</th><th>AZ</th><th>Instance Type</th><th>Start (UTC)</th><th>End (UTC)</th><th>Duration</th><th>Upfront Fee</th><th>Offering ID</th></tr></thead>
      <tbody id="cb-tbody"></tbody>
    </table>
  </div>
  <div class="section-title">Notes &amp; Disclaimers</div>
  <ul class="notes-list" id="notes-list"></ul>
</main>
<footer id="footer">Dashboard generated from embedded JSON data.</footer>
<script>
  function fmtDate(iso) {
    if (!iso) return '\u2014';
    const d = new Date(iso);
    return d.toLocaleString('en-US', {month:'short',day:'numeric',year:'numeric',hour:'2-digit',minute:'2-digit',timeZone:'UTC',timeZoneName:'short'});
  }
  function fmtDateShort(iso) {
    if (!iso) return '\u2014';
    const d = new Date(iso);
    return d.toLocaleString('en-US', {month:'short',day:'numeric',hour:'2-digit',minute:'2-digit',timeZone:'UTC',timeZoneName:'short'});
  }
  function bool(v) {
    if (v===null||v===undefined) return '<span class="badge badge-muted">\u2014</span>';
    return v ? '<span class="badge badge-green">\u2713 Yes</span>' : '<span class="badge badge-red">\u2717 No</span>';
  }
  function scoreBar(s) {
    if (s===null||s===undefined) return '<span style="color:var(--muted)">\u2014</span>';
    const c = s>=7?'#22d17a':s>=4?'#f5c542':'#f25c5c';
    return '<div class="score-bar-wrap"><div class="score-bar"><div class="score-bar-fill" style="width:'+Math.min(s/10*100,100)+'%;background:'+c+'"></div></div><span class="score-num" style="color:'+c+'">'+s+'</span></div>';
  }
  document.getElementById('generated-at').textContent = 'Generated: ' + fmtDate(DATA.generated_at_utc);
  document.getElementById('target-count').textContent = DATA.target_instance_count;
  document.getElementById('cb-window').textContent = DATA.capacity_block_query.days_ahead + ' days';
  document.getElementById('cb-duration').textContent = DATA.capacity_block_query.duration_hours + 'h';
  document.getElementById('footer').textContent = 'Data generated ' + fmtDate(DATA.generated_at_utc) + ' \u00b7 Regions scanned: ' + DATA.regions_scanned.join(', ');
  const validRecords = DATA.records.filter(r => r.valid_instance_type_in_region);
  const offeredCount = DATA.records.filter(r => r.offered_in_az).length;
  const totalCB = DATA.capacity_block_offerings.length;
  const lowestFee = totalCB ? Math.min(...DATA.capacity_block_offerings.map(o => parseFloat(o.upfront_fee))) : null;
  const cbRegions = new Set(DATA.capacity_block_offerings.map(o => o.region)).size;
  const errCount = Object.keys(DATA.errors||{}).length;
  const stats = [
    {label:'Regions Scanned', value:DATA.regions_scanned.length, sub:DATA.regions_scanned.join(', '), cls:'blue'},
    {label:'AZs with H200 Offered', value:offeredCount, sub:'offered_in_az = true', cls:'green'},
    {label:'CB Offerings Available', value:totalCB, sub:cbRegions+' regions', cls:'orange'},
    {label:'Lowest CB Fee', value:lowestFee!==null?'$'+lowestFee.toFixed(2):'N/A', sub:'USD upfront', cls:'purple'},
    {label:'Scan Errors', value:errCount, sub:errCount?'See error section below':'Clean scan', cls:errCount?'red':'blue'},
  ];
  document.getElementById('summary-grid').innerHTML = stats.map(s =>
    '<div class="stat-card '+s.cls+'"><div class="stat-label">'+s.label+'</div><div class="stat-value">'+s.value+'</div><div class="stat-sub">'+s.sub+'</div></div>'
  ).join('');
  const sample = validRecords[0];
  if (sample) {
    document.getElementById('spec-grid').innerHTML = [
      {k:'GPU', v:sample.gpu_count+'\u00d7 '+sample.gpu_names},
      {k:'GPU Memory', v:sample.total_gpu_memory_gib+' GiB total'},
      {k:'vCPUs', v:sample.vcpu},
      {k:'System RAM', v:sample.memory_gib+' GiB'},
      {k:'Network', v:sample.network_performance},
      {k:'EFA Supported', v:sample.efa_supported?'Yes':'No'},
      {k:'Usage Classes', v:sample.supported_usage_classes},
      {k:'Instance Family', v:'p5e / p5en .48xlarge'},
    ].map(i => '<div class="spec-item"><div class="spec-k">'+i.k+'</div><div class="spec-v">'+i.v+'</div></div>').join('');
  }
  const cbq = DATA.capacity_block_query;
  document.getElementById('cb-config-grid').innerHTML = [
    {k:'Enabled', v:cbq.enabled?'Yes':'No'},
    {k:'Start From Now', v:cbq.start_days_from_now+' days'},
    {k:'Days Ahead', v:cbq.days_ahead+' days'},
    {k:'Duration', v:cbq.duration_hours+' hours'},
    {k:'Max Offerings/Region', v:cbq.max_offerings_per_region_type},
    {k:'Instance Types', v:DATA.instance_types.join(', ')},
  ].map(i => '<div class="spec-item"><div class="spec-k">'+i.k+'</div><div class="spec-v">'+i.v+'</div></div>').join('');
  const errors = DATA.errors||{};
  if (Object.keys(errors).length) {
    document.getElementById('error-section').innerHTML = Object.entries(errors).map(([r,msgs]) =>
      '<div class="error-card"><strong>\u26a0 Region Scan Error: '+r+'</strong>'+msgs.map(m=>'<p>'+m+'</p>').join('')+'</div>'
    ).join('');
  }
  const regions = [...new Set(DATA.records.map(r => r.region))];
  const types   = [...new Set(DATA.records.map(r => r.instance_type))];
  let activeRegion = 'all', activeType = 'all';
  document.getElementById('region-filters').innerHTML = regions.map(r =>
    `<button class="filter-btn" onclick="filterRegion('${r}',this)">${r}</button>`
  ).join('');
  document.getElementById('type-filters').innerHTML = types.map(t =>
    `<button class="filter-btn" onclick="filterType('${t}',this)">${t}</button>`
  ).join('');
  function renderTable() {
    document.getElementById('avail-tbody').innerHTML = DATA.records.map(r => {
      const hide = (activeRegion!=='all'&&r.region!==activeRegion)||(activeType!=='all'&&r.instance_type!==activeType);
      const cbBadge = r.capacity_block_offering_count>0 ? '<span class="badge badge-blue">'+r.capacity_block_offering_count+'</span>' : '<span class="badge badge-muted">0</span>';
      const fee = r.capacity_block_lowest_upfront_fee ? '<span class="cb-price">$'+parseFloat(r.capacity_block_lowest_upfront_fee).toFixed(2)+'</span>' : '\u2014';
      const uc = r.supported_usage_classes ? r.supported_usage_classes.split(',').map(c=>'<span class="badge badge-blue" style="margin:1px">'+c.trim()+'</span>').join(' ') : '<span class="badge badge-muted">\u2014</span>';
      return '<tr'+(hide?' class="hidden"':'')+'>'
        +'<td><strong>'+r.region+'</strong></td>'
        +'<td>'+r.availability_zone_name+'</td>'
        +'<td style="color:var(--muted);font-size:12px">'+r.availability_zone_id+'</td>'
        +'<td><span class="badge badge-blue">'+r.instance_type+'</span></td>'
        +bool(r.valid_instance_type_in_region).replace(/^/, '<td>').replace(/$/, '</td>')
        +bool(r.offered_in_az).replace(/^/, '<td>').replace(/$/, '</td>')
        +'<td>'+scoreBar(r.spot_placement_score_h200_fleet)+'</td>'
        +'<td>'+uc+'</td>'
        +bool(r.efa_supported).replace(/^/, '<td>').replace(/$/, '</td>')
        +'<td>'+cbBadge+'</td>'
        +'<td style="font-size:12px">'+fmtDateShort(r.capacity_block_next_start_utc)+'</td>'
        +'<td>'+fee+'</td></tr>';
    }).join('');
  }
  renderTable();
  function filterRegion(val, btn) {
    activeRegion = val;
    document.querySelectorAll('#region-filters .filter-btn').forEach(b => b.classList.remove('active'));
    document.querySelector('.filters .filter-btn').classList.remove('active');
    btn.classList.add('active');
    renderTable();
  }
  function filterType(val, btn) {
    activeType = val;
    document.querySelectorAll('#type-filters .filter-btn').forEach(b => b.classList.remove('active'));
    document.querySelectorAll('.filters .filter-btn')[1].classList.remove('active');
    btn.classList.add('active');
    renderTable();
  }
  document.getElementById('cb-tbody').innerHTML = DATA.capacity_block_offerings
    .slice().sort((a,b) => new Date(a.start_date_utc)-new Date(b.start_date_utc))
    .map(o => {
      const dur = o.duration_minutes>0 ? o.duration_hours+'h '+o.duration_minutes+'m' : o.duration_hours+'h';
      return '<tr>'
        +'<td><strong>'+o.region+'</strong></td>'
        +'<td>'+o.availability_zone_name+' <span style="color:var(--muted);font-size:11px">('+o.availability_zone_id+')</span></td>'
        +'<td><span class="badge badge-blue">'+o.instance_type+'</span></td>'
        +'<td>'+fmtDateShort(o.start_date_utc)+'</td>'
        +'<td>'+fmtDateShort(o.end_date_utc)+'</td>'
        +'<td><span class="badge badge-yellow">'+dur+'</span></td>'
        +'<td class="cb-price">$'+parseFloat(o.upfront_fee).toFixed(2)+' '+o.currency+'</td>'
        +'<td class="cb-id">'+o.capacity_block_offering_id+'</td></tr>';
    }).join('');
  document.getElementById('notes-list').innerHTML = DATA.notes.map(n => '<li>'+n+'</li>').join('');
</script>
</body>
</html>
"""


def generate_html(payload: Dict[str, Any], path: str) -> None:
    """Write a self-contained HTML dashboard with *payload* embedded as JSON."""
    json_str = json.dumps(payload, default=json_default, separators=(",", ":"))
    html = HTML_TEMPLATE.replace("__DATA_JSON__", json_str)
    with open(path, "w", encoding="utf-8") as handle:
        handle.write(html)


def write_csv(path: str, rows: Sequence[Dict[str, Any]], fieldnames: Sequence[str]) -> None:
    with open(path, "w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=fieldnames, extrasaction="ignore")
        writer.writeheader()
        for row in rows:
            writer.writerow(row)


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Collect read-only EC2 H200 GPU availability signals across US regions."
    )
    parser.add_argument("--profile", help="AWS CLI profile name. Defaults to normal Boto3 credential chain.")
    parser.add_argument(
        "--regions",
        help="Comma-separated region allow-list. Defaults to all opted-in US commercial regions.",
    )
    parser.add_argument(
        "--include-govcloud",
        action="store_true",
        help="Include us-gov-* if your credentials/partition support it.",
    )
    parser.add_argument(
        "--include-not-opted-in",
        action="store_true",
        help="Include regions even if the account is not opted in. Calls may fail.",
    )
    parser.add_argument(
        "--instance-types",
        default=",".join(DEFAULT_H200_INSTANCE_TYPES),
        help="Comma-separated instance types. Default: p5e.48xlarge,p5en.48xlarge",
    )
    parser.add_argument(
        "--target-instance-count",
        type=int,
        default=1,
        help="Number of instances researchers want. Used for Spot score and Capacity Block search.",
    )
    parser.add_argument(
        "--no-spot",
        action="store_true",
        help="Skip EC2 Spot placement score collection.",
    )
    parser.add_argument(
        "--capacity-block-days",
        type=int,
        default=56,
        help="Search Capacity Block offerings from now through this many days ahead. Use 0 to disable.",
    )
    parser.add_argument(
        "--capacity-block-start-days",
        type=int,
        default=0,
        help="Earliest Capacity Block start date, in days from now.",
    )
    parser.add_argument(
        "--capacity-block-duration-hours",
        type=int,
        default=24,
        help="Capacity Block duration in hours. Must match AWS Capacity Block duration rules.",
    )
    parser.add_argument(
        "--capacity-block-max-offerings",
        type=int,
        default=50,
        help="Max Capacity Block offerings to keep per region and instance type.",
    )
    parser.add_argument(
        "--capacity-block-all-azs",
        action="store_true",
        help="Include Capacity Block offerings from all AZs/Local Zones, regardless of opt-in status.",
    )
    parser.add_argument("--workers", type=int, default=8, help="Number of regions to scan concurrently.")
    parser.add_argument("--json-out", default="h200_ec2_availability.json")
    parser.add_argument("--csv-out", default="h200_ec2_availability.csv")
    parser.add_argument("--capacity-block-csv-out", default="h200_capacity_block_offerings.csv")
    parser.add_argument(
        "--html-out",
        default="h200_dashboard.html",
        help="Path for the self-contained HTML dashboard. Pass an empty string to skip.",
    )

    args = parser.parse_args()

    if args.target_instance_count < 1:
        raise SystemExit("--target-instance-count must be >= 1")
    if args.capacity_block_days < 0:
        raise SystemExit("--capacity-block-days must be >= 0")
    if args.capacity_block_duration_hours < 1:
        raise SystemExit("--capacity-block-duration-hours must be >= 1")
    if args.capacity_block_max_offerings < 1:
        raise SystemExit("--capacity-block-max-offerings must be >= 1")

    instance_types = parse_csv_list(args.instance_types)
    if not instance_types:
        raise SystemExit("--instance-types must contain at least one instance type")

    try:
        session = make_session(args.profile)
        if args.regions:
            regions = parse_csv_list(args.regions)
        else:
            regions = discover_us_regions(
                session,
                include_govcloud=args.include_govcloud,
                include_not_opted_in=args.include_not_opted_in,
            )
    except NoCredentialsError:
        print("No AWS credentials found. Configure AWS_PROFILE, aws sso login, or environment credentials.", file=sys.stderr)
        return 2
    except (ClientError, BotoCoreError) as exc:
        print(f"Failed to discover regions: {exc}", file=sys.stderr)
        return 2

    all_records: List[Dict[str, Any]] = []
    all_capacity_blocks: List[Dict[str, Any]] = []
    errors: Dict[str, List[str]] = {}
    generated_at = iso_utc(utc_now())

    with futures.ThreadPoolExecutor(max_workers=max(args.workers, 1)) as executor:
        future_to_region = {
            executor.submit(
                collect_region,
                session,
                region,
                instance_types,
                args.target_instance_count,
                not args.no_spot,
                args.capacity_block_days,
                args.capacity_block_start_days,
                args.capacity_block_duration_hours,
                args.capacity_block_max_offerings,
                args.capacity_block_all_azs,
            ): region
            for region in regions
        }

        for future in futures.as_completed(future_to_region):
            region = future_to_region[future]
            try:
                result = future.result()
                all_records.extend(result.get("records", []))
                all_capacity_blocks.extend(result.get("capacity_block_offerings", []))
                if result.get("errors"):
                    errors[region] = result["errors"]
            except Exception as exc:  # Defensive: keep other regions from failing.
                errors[region] = [f"Unhandled error: {exc!r}"]

    all_records.sort(
        key=lambda row: (
            row.get("region") or "",
            row.get("availability_zone_id") or "",
            row.get("instance_type") or "",
        )
    )
    all_capacity_blocks.sort(
        key=lambda row: (
            row.get("region") or "",
            row.get("instance_type") or "",
            row.get("start_date_utc") or "",
            row.get("availability_zone_id") or "",
        )
    )

    payload = {
        "generated_at_utc": generated_at,
        "regions_scanned": regions,
        "instance_types": instance_types,
        "target_instance_count": args.target_instance_count,
        "capacity_block_query": {
            "enabled": args.capacity_block_days > 0,
            "start_days_from_now": args.capacity_block_start_days,
            "days_ahead": args.capacity_block_days,
            "duration_hours": args.capacity_block_duration_hours,
            "max_offerings_per_region_type": args.capacity_block_max_offerings,
        },
        "notes": [
            "offered_in_az means EC2 lists this instance type as offered in that AZ; it is not a real-time On-Demand capacity guarantee.",
            "spot_placement_score_h200_fleet is a Spot-only likelihood score for the H200 instance-type set and target count; it is not a guarantee.",
            "capacity_block_offerings are future purchasable reservations returned by DescribeCapacityBlockOfferings; this script does not purchase them.",
        ],
        "records": all_records,
        "capacity_block_offerings": all_capacity_blocks,
        "errors": errors,
    }

    with open(args.json_out, "w", encoding="utf-8") as handle:
        json.dump(payload, handle, indent=2, default=json_default)

    summary_fields = [
        "generated_at_utc",
        "region",
        "availability_zone_id",
        "availability_zone_name",
        "zone_type",
        "instance_type",
        "valid_instance_type_in_region",
        "offered_in_az",
        "spot_placement_score_h200_fleet",
        "target_instance_count",
        "vcpu",
        "memory_gib",
        "gpu_names",
        "gpu_count",
        "total_gpu_memory_gib",
        "supported_usage_classes",
        "efa_supported",
        "network_performance",
        "capacity_block_offering_count",
        "capacity_block_next_start_utc",
        "capacity_block_next_end_utc",
        "capacity_block_lowest_upfront_fee",
        "capacity_block_currency",
        "instance_spec_error",
    ]

    capacity_block_fields = [
        "region",
        "availability_zone_id",
        "availability_zone_name",
        "zone_type",
        "instance_type",
        "instance_count",
        "start_date_utc",
        "end_date_utc",
        "duration_hours",
        "duration_minutes",
        "upfront_fee",
        "currency",
        "tenancy",
        "capacity_block_offering_id",
        "ultraserver_type",
        "ultraserver_count",
    ]

    write_csv(args.csv_out, all_records, summary_fields)
    write_csv(args.capacity_block_csv_out, all_capacity_blocks, capacity_block_fields)

    if args.html_out:
        generate_html(payload, args.html_out)

    print(f"Scanned {len(regions)} regions: {', '.join(regions)}")
    print(f"Wrote summary JSON: {args.json_out}")
    print(f"Wrote summary CSV: {args.csv_out}")
    print(f"Wrote Capacity Block CSV: {args.capacity_block_csv_out}")
    if args.html_out:
        print(f"Wrote HTML dashboard:  {args.html_out}")
    if errors:
        print(f"Completed with errors in {len(errors)} region(s). See JSON 'errors' field.", file=sys.stderr)
        return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(main())