Seed script for Nightshift significant events (local dev)

seed_nightshift.sh

#!/usr/bin/env bash
set -euo pipefail

ES_URL="${ES_URL:-http://localhost:9236}"
ES_AUTH="${ES_AUTH:-elastic:changeme}"
INDEX=".significant_events-events"

NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
HOUR_AGO=$(date -u -v-1H +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -d "1 hour ago" +%Y-%m-%dT%H:%M:%SZ)
TWO_HOURS_AGO=$(date -u -v-2H +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -d "2 hours ago" +%Y-%m-%dT%H:%M:%SZ)
THREE_HOURS_AGO=$(date -u -v-3H +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -d "3 hours ago" +%Y-%m-%dT%H:%M:%SZ)
SIX_HOURS_AGO=$(date -u -v-6H +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -d "6 hours ago" +%Y-%m-%dT%H:%M:%SZ)
YESTERDAY=$(date -u -v-24H +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -d "24 hours ago" +%Y-%m-%dT%H:%M:%SZ)

echo "Seeding significant events into $ES_URL / $INDEX ..."

BODY=$(cat <<NDJSON
{"create":{}}
{"@timestamp":"${HOUR_AGO}","created_at":"${HOUR_AGO}","event_id":"evt-001","discovery_slug":"disc-web-latency","status":"promoted","stream_names":["logs.web-frontend","logs.api-gateway"],"title":"Web latency spike across frontend and API gateway","summary":"P95 latency jumped from 120ms to 890ms on web-frontend and api-gateway services starting at 14:23 UTC. Correlated with a deployment of api-gateway v2.8.1 that introduced a new middleware layer.","root_cause":"Deployment of api-gateway v2.8.1 introduced synchronous database lookup in authentication middleware, blocking the event loop under load.","criticality":0.85,"confidence":0.92,"recommendations":["Roll back api-gateway to v2.7.9","Add connection pooling to the auth middleware","Set up latency alerting at P95 > 300ms"],"rule_names":["latency-p95-spike","error-rate-increase"],"kibana.space_ids":["default"]}
{"create":{}}
{"@timestamp":"${TWO_HOURS_AGO}","created_at":"${TWO_HOURS_AGO}","event_id":"evt-002","discovery_slug":"disc-memory-leak","status":"acknowledged","stream_names":["metrics.payment-service","logs.payment-service"],"title":"Memory leak in payment service causing OOM restarts","summary":"Payment service pods restarting every ~45 minutes due to OOM kills. Heap usage grows linearly from 512MB to 2GB between restarts. Started after enabling the new transaction batching feature.","root_cause":"Transaction batching feature accumulates uncommitted transaction references in an unbounded array, never releasing them after commit.","criticality":0.78,"confidence":0.88,"recommendations":["Disable transaction batching feature flag","Add bounded queue with configurable max size","Implement periodic GC sweep for completed transactions"],"rule_names":["memory-growth-linear","pod-restart-frequency"],"kibana.space_ids":["default"]}
{"create":{}}
{"@timestamp":"${THREE_HOURS_AGO}","created_at":"${THREE_HOURS_AGO}","event_id":"evt-003","discovery_slug":"disc-disk-pressure","status":"promoted","stream_names":["metrics.elasticsearch-cluster","logs.elasticsearch"],"title":"Elasticsearch disk pressure causing index throttling","summary":"Elasticsearch cluster disk usage crossed 85% watermark on 3 of 5 data nodes. Write throttling engaged, causing upstream log ingestion backpressure and data loss in the pipeline.","root_cause":"Retention policy misconfiguration after index lifecycle migration left 90-day indices active instead of the intended 30-day rollover.","criticality":0.91,"confidence":0.95,"recommendations":["Delete indices older than 30 days immediately","Fix ILM policy to enforce 30-day rollover","Add disk usage alerting at 75% threshold","Consider adding hot-warm-cold architecture"],"rule_names":["disk-watermark-breach","write-rejection-rate"],"kibana.space_ids":["default"]}
{"create":{}}
{"@timestamp":"${SIX_HOURS_AGO}","created_at":"${SIX_HOURS_AGO}","event_id":"evt-004","discovery_slug":"disc-dns-resolution","status":"resolved","stream_names":["logs.dns-resolver","metrics.network"],"title":"Intermittent DNS resolution failures in us-east-1","summary":"DNS resolution failures spiked to 12% of queries for internal service names in us-east-1 AZ-b. Caused cascading timeouts in service-to-service communication.","root_cause":"CoreDNS pods scaled down during a node maintenance window, reducing capacity below the threshold needed for peak traffic.","criticality":0.72,"confidence":0.90,"recommendations":["Set PodDisruptionBudget for CoreDNS","Increase minimum replica count to 5","Add DNS resolution monitoring per AZ"],"rule_names":["dns-failure-rate","service-timeout-cascade"],"kibana.space_ids":["default"]}
{"create":{}}
{"@timestamp":"${YESTERDAY}","created_at":"${YESTERDAY}","event_id":"evt-005","discovery_slug":"disc-cert-expiry","status":"closed","stream_names":["logs.ingress-controller"],"title":"TLS certificate near expiry on ingress controller","summary":"The wildcard certificate for *.internal.acme.co expires in 48 hours. Automated renewal failed silently due to a permissions issue with the cert-manager ClusterIssuer.","root_cause":"cert-manager service account lost access to the DNS01 challenge solver role after a recent RBAC audit tightened permissions.","criticality":0.65,"confidence":0.97,"recommendations":["Manually renew the certificate","Restore DNS01 solver permissions for cert-manager SA","Add certificate expiry alerting at 7-day threshold"],"rule_names":["cert-expiry-warning"],"kibana.space_ids":["default"]}
{"create":{}}
{"@timestamp":"${HOUR_AGO}","created_at":"${HOUR_AGO}","event_id":"evt-006","discovery_slug":"disc-kafka-lag","status":"promoted","stream_names":["metrics.kafka-cluster","logs.order-processing"],"title":"Kafka consumer lag growing on order-processing topic","summary":"Consumer group order-processors showing monotonically increasing lag on partitions 0-7. Current lag: 2.4M messages. Processing rate dropped from 15k/s to 3k/s.","root_cause":"Schema registry became unavailable briefly, causing deserialisation failures. Consumers entered retry loops with exponential backoff, never fully recovering throughput.","criticality":0.82,"confidence":0.86,"recommendations":["Restart consumer group to clear retry state","Add circuit breaker for schema registry calls","Implement dead-letter queue for poison messages","Alert on consumer lag > 500k messages"],"rule_names":["consumer-lag-growth","throughput-degradation"],"kibana.space_ids":["default"]}
NDJSON
)

RESULT=$(curl -s -u "$ES_AUTH" -X POST "$ES_URL/$INDEX/_bulk?refresh=true" \
  -H "Content-Type: application/x-ndjson" \
  --data-binary "$BODY
")

ERRORS=$(echo "$RESULT" | python3 -c "import sys,json; r=json.load(sys.stdin); print(r.get('errors', True))")
if [ "$ERRORS" = "True" ] || [ "$ERRORS" = "true" ]; then
  echo "ERROR: Bulk index had failures:"
  echo "$RESULT" | python3 -m json.tool
  exit 1
fi

echo "Successfully indexed 6 significant events."
echo ""
echo "Verifying..."
COUNT=$(curl -s -u "$ES_AUTH" "$ES_URL/$INDEX/_count" | python3 -c "import sys,json; print(json.load(sys.stdin).get('count', 0))")
echo "Total documents in $INDEX: $COUNT"
echo ""
echo "Done! Nightshift page should show these events."