gcp-antigravity-billing-analysis.sh

#!/bin/bash
# =============================================================================
# GCP Billing & Cloud Logging Cost Analysis Script
# =============================================================================
# This script analyzes GCP billing data from BigQuery and identifies
# the top cost drivers, with a focus on Cloud Logging costs.
#
# Usage: ./scripts/billing-analysis.sh [OPTIONS]
#
# Options:
#   --days N        Number of days to analyze (default: 30)
#   --project ID    GCP project ID (default: [my-gcp-project])
#   --logs-only     Only run the Cloud Logging analysis
#   --costs-only    Only run the billing costs analysis
#   --json          Output in JSON format
#   --help          Show this help message
# =============================================================================

set -e

# Default values
PROJECT_ID="[my-gcp-project]"
DAYS=30
BILLING_TABLE="[my-gcp-project].all_billing_data.gcp_billing_export_resource_v1_[my-billing-account]"
LOGS_ONLY=false
COSTS_ONLY=false
JSON_OUTPUT=false
TOP_SERVICES_FILE=$(mktemp)

# Cleanup temp file on exit
trap "rm -f $TOP_SERVICES_FILE" EXIT

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
BOLD='\033[1m'

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --days)
            DAYS="$2"
            shift 2
            ;;
        --project)
            PROJECT_ID="$2"
            shift 2
            ;;
        --logs-only)
            LOGS_ONLY=true
            shift
            ;;
        --costs-only)
            COSTS_ONLY=true
            shift
            ;;
        --json)
            JSON_OUTPUT=true
            shift
            ;;
        --help)
            head -24 "$0" | tail -20
            exit 0
            ;;
        *)
            echo "Unknown option: $1"
            exit 1
            ;;
    esac
done

# Helper functions
print_header() {
    echo ""
    echo -e "${BLUE}${BOLD}═══════════════════════════════════════════════════════════════${NC}"
    echo -e "${BLUE}${BOLD}  $1${NC}"
    echo -e "${BLUE}${BOLD}═══════════════════════════════════════════════════════════════${NC}"
    echo ""
}

print_section() {
    echo ""
    echo -e "${GREEN}▶ $1${NC}"
    echo -e "${GREEN}─────────────────────────────────────────────────────────────────${NC}"
}

# Get the date range for billing analysis
get_billing_date_range() {
    bq query --use_legacy_sql=false --format=prettyjson --quiet '
    SELECT 
      FORMAT_DATE("%Y-%m-%d", MIN(DATE(usage_start_time))) as earliest_date,
      FORMAT_DATE("%Y-%m-%d", MAX(DATE(usage_start_time))) as latest_date
    FROM `'"$BILLING_TABLE"'`
    WHERE service.description NOT IN ("Duet AI", "Looker Studio")' 2>/dev/null
}

# Print daily trend summary
print_daily_trend() {
    local FILTER_CLAUSE="$1"
    local LABEL="$2"
    
    # Calculate D-1 and D-2 relative to LATEST_DATE to avoid partial data
    local TREND_DATE_1=$(date -v-1d -j -f "%Y-%m-%d" "$LATEST_DATE" "+%Y-%m-%d" 2>/dev/null || date -d "$LATEST_DATE - 1 day" "+%Y-%m-%d")
    local TREND_DATE_2=$(date -v-2d -j -f "%Y-%m-%d" "$LATEST_DATE" "+%Y-%m-%d" 2>/dev/null || date -d "$LATEST_DATE - 2 days" "+%Y-%m-%d")
    
    # Query for daily costs
    local COST_DATA=$(bq query --use_legacy_sql=false --format=csv --quiet "
    SELECT 
      DATE(usage_start_time) as usage_date,
      SUM(cost) as total_cost
    FROM \`$BILLING_TABLE\`
    WHERE DATE(usage_start_time) IN (DATE('$TREND_DATE_1'), DATE('$TREND_DATE_2'))
      AND $FILTER_CLAUSE
    GROUP BY 1
    ORDER BY 1 DESC
    " 2>/dev/null)
    
    local COST_1=$(echo "$COST_DATA" | grep "$TREND_DATE_1" | cut -d',' -f2)
    local COST_2=$(echo "$COST_DATA" | grep "$TREND_DATE_2" | cut -d',' -f2)
    
    # Defaults in case of missing data
    COST_1=${COST_1:-0}
    COST_2=${COST_2:-0}
    
    if (( $(echo "$COST_2 > 0" | bc -l) )); then
        local DIFF=$(echo "$COST_1 - $COST_2" | bc -l)
        local PCT=$(echo "scale=2; ($DIFF / $COST_2) * 100" | bc -l)
        
        local COLOR=$GREEN
        local ARROW="▼"
        
        if (( $(echo "$DIFF > 0" | bc -l) )); then
            COLOR=$YELLOW # or RED
            ARROW="▲"
        fi
        
        printf "  ${BOLD}Daily Trend ($TREND_DATE_1 vs $TREND_DATE_2):${NC} ${COLOR}$ARROW $PCT%%${NC} ($(printf "%.2f" $COST_1) vs $(printf "%.2f" $COST_2))\n"
    else
        printf "  ${BOLD}Daily Trend ($TREND_DATE_1 vs $TREND_DATE_2):${NC} Insufficient data for comparison.\n"
    fi
    echo ""
}

# =============================================================================
# BILLING COSTS ANALYSIS
# =============================================================================
run_costs_analysis() {
    print_header "💰 GCP BILLING COSTS ANALYSIS (Last $DAYS Days of Available Data)"
    
    # Check billing data freshness
    print_section "📅 Billing Data Freshness"
    echo "Checking date range of billing export..."
    
    DATE_RANGE=$(get_billing_date_range)
    LATEST_DATE=$(echo "$DATE_RANGE" | grep -o '"latest_date": "[^"]*"' | cut -d'"' -f4)
    EARLIEST_DATE=$(echo "$DATE_RANGE" | grep -o '"earliest_date": "[^"]*"' | cut -d'"' -f4)
    
    echo -e "  Earliest data: ${YELLOW}$EARLIEST_DATE${NC}"
    echo -e "  Latest data:   ${YELLOW}$LATEST_DATE${NC}"
    echo ""
    echo -e "  ${RED}⚠️  Note: Billing data export has a delay. Recent days may be missing.${NC}"
    
    # Calculate date range for queries
    END_DATE="$LATEST_DATE"
    START_DATE=$(date -v-${DAYS}d -j -f "%Y-%m-%d" "$END_DATE" "+%Y-%m-%d" 2>/dev/null || date -d "$END_DATE - $DAYS days" "+%Y-%m-%d")
    
    echo ""
    echo -e "  Analyzing period: ${GREEN}$START_DATE${NC} to ${GREEN}$END_DATE${NC}"
    
    # Top services by cost
    print_section "🏆 Top Services by Cost"
    print_daily_trend "TRUE" "Overall"
    
    if [ "$JSON_OUTPUT" = true ]; then
        FORMAT="prettyjson"
    else
        FORMAT="pretty"
    fi
    
    bq query --use_legacy_sql=false --format=$FORMAT --quiet '
    SELECT 
      service.description as service_name,
      ROUND(SUM(cost), 2) as total_cost,
      COUNT(*) as billing_entries
    FROM `'"$BILLING_TABLE"'`
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
    GROUP BY service.description
    HAVING SUM(cost) > 10
    ORDER BY total_cost DESC
    LIMIT 15'
    
    # Top SKUs by cost
    print_section "📊 Top SKUs by Cost"
    
    bq query --use_legacy_sql=false --format=$FORMAT --quiet '
    SELECT 
      sku.description as sku_description,
      service.description as service_name,
      ROUND(SUM(cost), 2) as total_cost,
      usage.unit as usage_unit
    FROM `'"$BILLING_TABLE"'`
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
    GROUP BY sku.description, service.description, usage.unit
    HAVING SUM(cost) > 20
    ORDER BY total_cost DESC
    LIMIT 20'
    
    # Costs by Organization (org_id)
    print_section "🏢 Costs by Organization (org_id)"
    
    bq query --use_legacy_sql=false --format=$FORMAT --quiet '
    SELECT 
      l.value as org_id,
      ROUND(SUM(cost), 2) as total_cost
    FROM `'"$BILLING_TABLE"'`, UNNEST(labels) as l
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
      AND l.key = "org_id"
    GROUP BY 1
    HAVING SUM(cost) > 1
    ORDER BY total_cost DESC
    LIMIT 20'
    
    # Cloud Run costs by resource and billing mode
    print_section "🚀 Cloud Run Costs by Service & Billing Mode"
    print_daily_trend 'service.description = "Cloud Run" AND NOT (sku.description LIKE "%job%" OR sku.description LIKE "%Job%")' "Cloud Run Services"
    
    bq query --use_legacy_sql=false --format=$FORMAT --quiet '
    SELECT 
      resource.name as cloud_run_service,
      CASE 
        WHEN sku.description LIKE "%Instance-based billing%" THEN "Instance Based (Min Instances)"
        ELSE "Request Based"
      END as billing_mode,
      ROUND(SUM(cost), 2) as total_cost
    FROM `'"$BILLING_TABLE"'`
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
      AND service.description = "Cloud Run"
      AND NOT (sku.description LIKE "%job%" OR sku.description LIKE "%Job%")
    GROUP BY 1, 2
    HAVING SUM(cost) > 1
    ORDER BY total_cost DESC, cloud_run_service
    LIMIT 30'
    
    # Cloud Run Jobs costs
    print_section "📦 Cloud Run Costs by Job"
    print_daily_trend 'service.description = "Cloud Run" AND (sku.description LIKE "%job%" OR sku.description LIKE "%Job%")' "Cloud Run Jobs"
    
    bq query --use_legacy_sql=false --format=$FORMAT --quiet '
    SELECT 
      resource.name as cloud_run_job,
      ROUND(SUM(cost), 2) as total_cost
    FROM `'"$BILLING_TABLE"'`
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
      AND service.description = "Cloud Run"
      AND (sku.description LIKE "%job%" OR sku.description LIKE "%Job%")
    GROUP BY 1
    HAVING SUM(cost) > 1
    ORDER BY total_cost DESC
    LIMIT 30'
    
    # Capture top Cloud Run services for recommendation filtering
    bq query --use_legacy_sql=false --format=csv --quiet '
    SELECT DISTINCT resource.name
    FROM `'"$BILLING_TABLE"'`
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
      AND service.description = "Cloud Run"
    GROUP BY 1
    HAVING SUM(cost) > 1
    ORDER BY SUM(cost) DESC
    LIMIT 30' 2>/dev/null | tail -n +2 > "$TOP_SERVICES_FILE"
    
    # BigQuery Analysis costs by project
    print_section "📈 BigQuery Analysis Costs by Project"
    print_daily_trend 'sku.description LIKE "Analysis%"' "BigQuery Analysis"
    
    bq query --use_legacy_sql=false --format=$FORMAT --quiet '
    SELECT 
      project.id as project_id,
      ROUND(SUM(cost), 2) as total_cost,
      ROUND(SUM(usage.amount) / 1e12, 2) as total_usage_tb
    FROM `'"$BILLING_TABLE"'`
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
      AND sku.description LIKE "Analysis%"
    GROUP BY project.id
    HAVING SUM(cost) > 50
    ORDER BY total_cost DESC
    LIMIT 15'
    
    # BigQuery Storage costs
    print_section "💾 BigQuery Storage Costs Analysis"
    print_daily_trend 'service.description = "BigQuery" AND sku.description LIKE "%Storage%"' "BigQuery Storage"
    
    echo "  Top Storage Costs by Dataset:"
    bq query --use_legacy_sql=false --format=$FORMAT --quiet '
    SELECT 
      project.id as project_id,
      -- Handle cases where resource.name is just the dataset ID or a full path
      COALESCE(SPLIT(resource.name, "/")[SAFE_OFFSET(3)], resource.name) as dataset_name,
      ROUND(SUM(cost), 2) as total_cost,
      ROUND(SUM(usage.amount) / 2.6e18, 2) as storage_tb_months
    FROM `'"$BILLING_TABLE"'`
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
      AND service.description = "BigQuery"
      AND sku.description LIKE "%Storage%"
    GROUP BY 1, 2
    HAVING SUM(cost) > 1
    ORDER BY total_cost DESC
    LIMIT 20'

    # Capture top datasets for detailed table analysis
    TOP_DATASETS_FILE=$(mktemp)
    bq query --use_legacy_sql=false --format=csv --quiet '
    SELECT 
      project.id as project_id,
      COALESCE(SPLIT(resource.name, "/")[SAFE_OFFSET(3)], resource.name) as dataset_name
    FROM `'"$BILLING_TABLE"'`
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
      AND service.description = "BigQuery"
      AND sku.description LIKE "%Storage%"
    GROUP BY 1, 2
    HAVING SUM(cost) > 5
    ORDER BY SUM(cost) DESC
    LIMIT 10' 2>/dev/null | tail -n +2 > "$TOP_DATASETS_FILE"

    echo ""
    echo -e "${BOLD}  🔬 Detailed Table Analysis for Top Datasets:${NC}"
    
    while IFS=, read -r proj ds; do
        if [ -z "$proj" ] || [ -z "$ds" ]; then continue; fi
        
        echo ""
        echo -e "  ${YELLOW}Dataset: $proj:$ds${NC}"
        
        # Dynamically determine the dataset location
        # Try jq first, fallback to grep
        RAW_JSON=$(bq show --format=json "$proj:$ds" 2>/dev/null)
        LOCATION=$(echo "$RAW_JSON" | jq -r '.location // empty' 2>/dev/null)
        
        if [ -z "$LOCATION" ]; then
             # Fallback: parse from pretty output or grep json
             LOCATION=$(echo "$RAW_JSON" | grep -o '"location":"[^"]*"' | cut -d'"' -f4)
        fi

        if [ -z "$LOCATION" ]; then
             LOCATION="europe-west3" # Default fallback
        fi
        
        # Format region for INFORMATION_SCHEMA (e.g., "europe-west3" -> "region-europe-west3")
        # If location is "US" or "EU", it is just "region-us" / "region-eu"
        REGION_QUALIFIER="region-$LOCATION"

        # Query INFORMATION_SCHEMA.TABLE_STORAGE
        # Groups tables by prefix (e.g. "events_20230101" -> "events_")
        
        bq query --use_legacy_sql=false --location="$LOCATION" --format=pretty --quiet "
        WITH TableInfo AS (
          SELECT
            table_name,
            total_logical_bytes,
            -- Heuristic: patterns like name_YYYYMMDD or name_2024...
            REGEXP_REPLACE(table_name, r'(_\d{8}.*|_\d{4}.*)$', '') as table_prefix
          FROM \`$proj.$REGION_QUALIFIER.INFORMATION_SCHEMA.TABLE_STORAGE\`
          WHERE table_schema = '$ds'
        )
        SELECT
          table_prefix,
          COUNT(*) as table_count,
          ROUND(SUM(total_logical_bytes) / 1e12, 2) as total_tb
        FROM TableInfo
        GROUP BY 1
        ORDER BY total_tb DESC
        LIMIT 5
        " 2>/dev/null || echo "    (Unable to query table stats in location $LOCATION - verify permissions)"

    done < "$TOP_DATASETS_FILE"
    
    rm -f "$TOP_DATASETS_FILE"
    
    echo ""
    echo "  Storage Costs by Storage Class (SKU):"
    bq query --use_legacy_sql=false --format=$FORMAT --quiet '
    SELECT 
      sku.description as storage_class,
      ROUND(SUM(cost), 2) as total_cost,
      ROUND(SUM(usage.amount) / 2.6e18, 2) as storage_tb_months
    FROM `'"$BILLING_TABLE"'`
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
      AND service.description = "BigQuery"
      AND sku.description LIKE "%Storage%"
    GROUP BY 1
    HAVING SUM(cost) > 1
    ORDER BY total_cost DESC
    LIMIT 20'
    
    
    # Cloud Storage Costs (Buckets)
    print_section "🗄️ Cloud Storage Costs by Bucket"
    print_daily_trend 'service.description = "Cloud Storage"' "Cloud Storage"

    bq query --use_legacy_sql=false --format=$FORMAT --quiet '
    SELECT 
      -- Fallback to Project ID + SKU if bucket name is missing (e.g. Early Delete, Network costs)
      COALESCE(resource.name, CONCAT(project.id, " (", sku.description, ")")) as bucket_name,
      ROUND(SUM(cost), 2) as total_cost,
      ROUND(SUM(usage.amount) / 1e9, 2) as total_usage_gb
    FROM `'"$BILLING_TABLE"'`
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
      AND service.description = "Cloud Storage"
    GROUP BY 1
    HAVING SUM(cost) > 1
    ORDER BY total_cost DESC
    LIMIT 15'

    # BigQuery Reservation API Costs
    print_section "🎟️ BigQuery Reservation Costs"
    print_daily_trend 'service.description = "BigQuery Reservation API"' "BigQuery Reservation API"

    bq query --use_legacy_sql=false --format=$FORMAT --quiet '
    SELECT 
      -- Fallback to Project ID if reservation name is missing (e.g. Pay-as-you-go Editions)
      COALESCE(resource.name, project.id) as reservation_name,
      sku.description as sku_description,
      ROUND(SUM(cost), 2) as total_cost,
      ROUND(SUM(usage.amount) / 3600, 2) as usage_hours
    FROM `'"$BILLING_TABLE"'`
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
      AND service.description = "BigQuery Reservation API"
    GROUP BY 1, 2
    HAVING SUM(cost) > 1
    ORDER BY total_cost DESC
    LIMIT 15'

    # Cloud Logging costs breakdown
    print_section "📝 Cloud Logging Costs Breakdown"
    print_daily_trend 'service.description = "Cloud Logging"' "Cloud Logging"
    
    bq query --use_legacy_sql=false --format=$FORMAT --quiet '
    SELECT 
      labels.value as resource_type,
      ROUND(SUM(cost), 2) as total_cost,
      ROUND(SUM(usage.amount) / 1e9, 2) as total_usage_gb,
      ROUND(SUM(cost) * 100 / (
        SELECT SUM(cost) 
        FROM `'"$BILLING_TABLE"'` 
        WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'") 
          AND service.description = "Cloud Logging"
      ), 2) as pct_of_logging_costs
    FROM `'"$BILLING_TABLE"'`,
    UNNEST(labels) as labels
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
      AND service.description = "Cloud Logging"
      AND labels.key = "goog-resource-type"
    GROUP BY labels.value
    HAVING SUM(cost) > 1
    ORDER BY total_cost DESC'
    
    # Daily Cloud Logging trend
    print_section "📉 Daily Cloud Logging Cost Trend (Last 10 Days)"
    
    bq query --use_legacy_sql=false --format=$FORMAT --quiet '
    SELECT 
      DATE(usage_start_time) as usage_date,
      ROUND(SUM(cost), 2) as daily_cost,
      ROUND(SUM(usage.amount) / 1e9, 2) as daily_usage_gb
    FROM `'"$BILLING_TABLE"'`
    WHERE DATE(usage_start_time) BETWEEN DATE("'"$START_DATE"'") AND DATE("'"$END_DATE"'")
      AND service.description = "Cloud Logging"
      AND sku.description = "Log Storage cost"
    GROUP BY DATE(usage_start_time)
    ORDER BY usage_date DESC
    LIMIT 10'
}

# =============================================================================
# CLOUD LOGGING ANALYSIS
# =============================================================================
run_logs_analysis() {
    print_header "📋 CLOUD LOGGING ANALYSIS (Last 24 Hours)"
    
    # Cloud Run revisions - top log generators
    print_section "🔝 Top Cloud Run Services by Log Entry Count"
    
    echo "Fetching log entries from Cloud Logging (this may take a minute)..."
    echo ""
    
    YESTERDAY=$(date -v-1d "+%Y-%m-%dT00:00:00Z" 2>/dev/null || date -d "yesterday" "+%Y-%m-%dT00:00:00Z")
    
    echo -e "${BOLD}Cloud Run Services (by log entry count):${NC}"
    echo ""
    
    gcloud logging read "resource.type=\"cloud_run_revision\" timestamp>=\"$YESTERDAY\"" \
        --project="$PROJECT_ID" \
        --format="value(resource.labels.service_name)" \
        --limit=50000 2>/dev/null | sort | uniq -c | sort -rn | head -25 | \
    while read count service; do
        printf "  %8s  %s\n" "$count" "$service"
    done
    
    # Cloud Run services with WARNING+ severity
    print_section "⚠️ Cloud Run Services with Most Warnings/Errors"
    
    echo -e "${BOLD}Services with WARNING+ severity logs:${NC}"
    echo ""
    
    gcloud logging read "resource.type=\"cloud_run_revision\" severity>=WARNING timestamp>=\"$YESTERDAY\"" \
        --project="$PROJECT_ID" \
        --format="value(resource.labels.service_name)" \
        --limit=20000 2>/dev/null | sort | uniq -c | sort -rn | head -20 | \
    {
        i=0
        while read count service; do
            printf "  %8s  %s\n" "$count" "$service"
            
            # Show common error for top 10 services
            if [ $i -lt 10 ]; then
                COMMON_MSG=$(gcloud logging read "resource.type=\"cloud_run_revision\" resource.labels.service_name=\"$service\" severity>=WARNING timestamp>=\"$YESTERDAY\"" \
                    --project="$PROJECT_ID" \
                    --format="value(jsonPayload.message, textPayload)" \
                    --limit=100 2>/dev/null | sort | uniq -c | sort -rn | head -1 | awk '{$1=""; print $0}' | sed 's/^ *//')
                
                if [ -n "$COMMON_MSG" ]; then
                    # Truncate to 100 chars
                    echo -e "             ${RED}↳ Most common: ${COMMON_MSG:0:100}...${NC}"
                fi
            fi
            ((i++))
        done
    }
    
    # Cloud Run Jobs
    print_section "📦 Cloud Run Jobs by Log Entry Count"
    
    echo -e "${BOLD}Cloud Run Jobs:${NC}"
    echo ""
    
    gcloud logging read "resource.type=\"cloud_run_job\" timestamp>=\"$YESTERDAY\"" \
        --project="$PROJECT_ID" \
        --format="value(resource.labels.job_name)" \
        --limit=20000 2>/dev/null | sort | uniq -c | sort -rn | head -15 | \
    while read count job; do
        printf "  %8s  %s\n" "$count" "$job"
    done
    
    # HTTP Load Balancer logs
    print_section "🌐 HTTP Load Balancer Logs"
    
    echo -e "${BOLD}Load Balancer log volume:${NC}"
    echo ""
    
    LB_COUNT=$(gcloud logging read "resource.type=\"http_load_balancer\" timestamp>=\"$YESTERDAY\"" \
        --project="$PROJECT_ID" \
        --format="value(resource.labels.forwarding_rule_name)" \
        --limit=10000 2>/dev/null | wc -l | tr -d ' ')
    
    echo "  Total entries (sample): $LB_COUNT"
}

# =============================================================================
# RECOMMENDATIONS
# =============================================================================
# =============================================================================
# DYNAMIC RECOMMENDATIONS
# =============================================================================
fetch_active_recommendations() {
    print_header "💡 DYNAMIC COST RECOMMENDATIONS"

    # Check if gcloud is available and authenticated
    if ! command -v gcloud &> /dev/null; then
        echo "gcloud command not found. Skipping dynamic recommendations."
        return
    fi
    
    echo "Fetching active cost recommendations for top services..."
    echo ""

    # Get active regions for Cloud Run services
    REGIONS=$(gcloud run services list --project="$PROJECT_ID" --format="value(region)" 2>/dev/null | sort -u)
    
    if [ -z "$REGIONS" ]; then
        echo "No active Cloud Run regions found or permission denied."
    else
        REC_OUTPUT_FILE=$(mktemp)
        
        for region in $REGIONS; do
            # Fetch recommendations for the region
            RECOMMENDATIONS_JSON=$(gcloud recommender recommendations list \
                --project="$PROJECT_ID" \
                --location="$region" \
                --recommender=google.run.service.CostRecommender \
                --format="json" 2>/dev/null)
            
            # Process each recommendation
            echo "$RECOMMENDATIONS_JSON" | jq -c '.[]' 2>/dev/null | while read -r rec; do
                # Extract service name from targetResources
                # format: //run.googleapis.com/projects/PROJECT/locations/REGION/services/SERVICE_NAME
                SERVICE_NAME=$(echo "$rec" | jq -r '.targetResources[1] // ""' | awk -F'/' '{print $NF}')
                
                # Check if this service is in our top cost list
                if grep -q "^$SERVICE_NAME$" "$TOP_SERVICES_FILE"; then
                    SUBTYPE=$(echo "$rec" | jq -r '.recommenderSubtype')
                    SAVINGS=$(echo "$rec" | jq -r '.primaryImpact.costProjection.cost.units // "0"')
                    CURRENCY=$(echo "$rec" | jq -r '.primaryImpact.costProjection.cost.currencyCode // "USD"')
                    PRIORITY=$(echo "$rec" | jq -r '.priority')
                    
                    # Absolute value of savings (it usually comes as negative)
                    SAVINGS_ABS=${SAVINGS#-}
                    
                    echo -e "${YELLOW}➤ Service: ${BOLD}$SERVICE_NAME${NC}"
                    echo -e "  Region: $region"
                    echo -e "  Type: $SUBTYPE"
                    echo -e "  Priority: $PRIORITY"
                    echo -e "  Potential Savings: ${GREEN}$SAVINGS_ABS $CURRENCY / month${NC}"
                    echo ""
                fi
            done >> "$REC_OUTPUT_FILE"
        done
        
        if [ -s "$REC_OUTPUT_FILE" ]; then
            cat "$REC_OUTPUT_FILE"
        else
            echo "No high-priority cost recommendations found for the analyzed top services."
        fi
        
        rm -f "$REC_OUTPUT_FILE"
    fi
    echo ""
}

print_static_recommendations() {
    print_header "💡 COST OPTIMIZATION RECOMMENDATIONS"
    
    echo -e "${YELLOW}1. Cloud Logging Optimizations:${NC}"
    echo "   • Review Cloud Run log verbosity (reduce DEBUG logs in production)"
    echo "   • Create log exclusion filters for health checks:"
    echo '     resource.type="cloud_run_revision" AND httpRequest.requestUrl:"/health"'
    echo "   • Reduce _Default bucket retention (currently 30 days)"
    echo ""
    
    echo -e "${YELLOW}2. BigQuery Cost Optimizations:${NC}"
    echo "   • Add partitioning/clustering to frequently queried tables"
    echo "   • Consider slot-based pricing for heavy tenants"
    echo "   • Review scheduled queries for optimization"
    echo ""
    
    echo -e "${YELLOW}3. Cloud Run Optimizations:${NC}"
    echo "   • Review services with high CPU/Memory costs"
    echo "   • Consider instance-based billing for consistent traffic"
    echo "   • Optimize min/max instances configuration"
    echo ""
}

# =============================================================================
# MAIN
# =============================================================================
main() {
    echo ""
    echo -e "${BOLD}╔═══════════════════════════════════════════════════════════════╗${NC}"
    echo -e "${BOLD}║       GCP BILLING & LOGGING COST ANALYSIS                     ║${NC}"
    echo -e "${BOLD}║       Project: $PROJECT_ID                         ║${NC}"
    echo -e "${BOLD}║       Generated: $(date '+%Y-%m-%d %H:%M:%S')                       ║${NC}"
    echo -e "${BOLD}╚═══════════════════════════════════════════════════════════════╝${NC}"
    
    if [ "$LOGS_ONLY" = true ]; then
        run_logs_analysis
    elif [ "$COSTS_ONLY" = true ]; then
        run_costs_analysis
    else
        run_costs_analysis
        run_logs_analysis
        fetch_active_recommendations
        print_static_recommendations
    fi
    
    print_header "✅ ANALYSIS COMPLETE"
    echo "Report generated at: $(date '+%Y-%m-%d %H:%M:%S')"
    echo ""
}

main