Hardened AWS Org — Terraform: SCPs, EventBridge security alerts, weekly resource watchdog

00-README.md

Hardened AWS Org — Terraform

The important pieces from my weekend hardening pass. All values anonymized (account IDs, emails, usernames). Code is straight from the live setup, just stripped of identifiers.

Files

1. 01-scps.tf — DenyServices (13 statements) + EnforceMFA. Region pin, instance-class allowlist, audit-tamper denies, root-action denies, Bedrock gate.
2. 02-eventbridge-rules.tf — all 14 EventBridge rules + the Lambda + DLQ. Every rule targets the same handler.
3. 03-mgmt-events-notifier.js — the router. Single dispatcher picks a severity + formatter, fans out to SES + Slack.
4. 04-resource-watchdog.tf — weekly cron Lambda. Cron rule, function, IAM (assume OrganizationAccountAccessRole in every member, then describe/stop).
5. 05-resource-watchdog.js — walks each member account across 4 approved regions: stops EC2/ECS/RDS, releases unattached EIPs, flags empty ALBs and unattached EBS, auto-applies 30-day log retention.

Anonymization

• Account IDs → 111111111111
• Domain example.com (replace with yours)
• Usernames admin.one / admin.two (replace with your SSO usernames formatted as ${lower(firstName)}.${lower(lastName)})
• Slack webhook is stored in SSM Parameter Store and pulled at runtime

01-scps.tf

// ============================================================
// Service Control Policies — DenyServices + EnforceMFA
// Attached at the org root. Names/IDs anonymized.
// ============================================================

locals {
  // Replace with your privileged account ID(s) that should be exempt.
  privileged_account_id = "111111111111"
}

// ------------------------------------------------------------
// DenyServices — bundled deny rails (region pin, instance
// class allowlist, audit-tamper denies, root-action denies,
// Bedrock gate, etc.)
// ------------------------------------------------------------
data "aws_iam_policy_document" "deny_services" {
  statement {
    sid       = "DenyEC2Instances"
    effect    = "Deny"
    actions   = ["ec2:RunInstances", "ec2:StartInstances"]
    resources = ["*"]
    condition {
      test     = "StringNotLike"
      variable = "aws:userid"
      values   = ["*:admin.one", "*:admin.two"]
    }
  }

  statement {
    sid       = "DenyExpensiveInstanceTypes"
    effect    = "Deny"
    actions   = ["ec2:RunInstances", "ec2:StartInstances"]
    resources = ["arn:aws:ec2:*:*:instance/*"]
    condition {
      test     = "ForAnyValue:StringLike"
      variable = "ec2:InstanceType"
      values = [
        "u-*",
        "p4*",
        "p5*",
        "x1e*",
        "trn1*",
        "inf2*",
        "hpc*",
        "*.metal",
        "*.24xlarge",
        "*.48xlarge",
      ]
    }
  }

  statement {
    sid       = "DenyECSTasks"
    effect    = "Deny"
    actions   = ["ecs:RunTask"]
    resources = ["*"]
    condition {
      test     = "StringNotEquals"
      variable = "aws:username"
      values   = ["admin.one"]
    }
  }

  statement {
    sid       = "DenyLargeRDSInstances"
    effect    = "Deny"
    actions   = ["rds:CreateDBInstance", "rds:CreateDBCluster"]
    resources = ["*"]
    condition {
      test     = "ForAnyValue:StringNotLike"
      variable = "rds:DatabaseClass"
      values = [
        "db.t3.micro",
        "db.t3.small",
        "db.t4g.micro",
        "db.t4g.small",
      ]
    }
    condition {
      test     = "StringNotLike"
      variable = "aws:userid"
      values   = ["*:admin.one", "*:admin.two"]
    }
  }

  statement {
    sid    = "DenyCloudTrailTampering"
    effect = "Deny"
    actions = [
      "cloudtrail:DeleteTrail",
      "cloudtrail:StopLogging",
      "cloudtrail:UpdateTrail",
      "cloudtrail:PutEventSelectors",
    ]
    resources = ["*"]
  }

  statement {
    sid       = "DenyTrailBucketDeletion"
    effect    = "Deny"
    actions   = ["s3:DeleteBucket", "s3:DeleteBucketPolicy"]
    resources = ["arn:aws:s3:::my-org-cloudtrail-events"]
  }

  statement {
    sid       = "DenyLeaveOrganization"
    effect    = "Deny"
    actions   = ["organizations:LeaveOrganization"]
    resources = ["*"]
  }

  statement {
    sid       = "DenyRootUserActions"
    effect    = "Deny"
    actions   = ["*"]
    resources = ["*"]
    condition {
      test     = "StringLike"
      variable = "aws:PrincipalArn"
      values   = ["arn:aws:iam::*:root"]
    }
  }

  statement {
    sid       = "DenyS3PublicAccess"
    effect    = "Deny"
    actions   = ["s3:PutAccountPublicAccessBlock"]
    resources = ["*"]
  }

  statement {
    sid       = "DenyPublicEC2Sharing"
    effect    = "Deny"
    actions   = ["ec2:ModifySnapshotAttribute", "ec2:ModifyImageAttribute"]
    resources = ["*"]
    condition {
      test     = "StringEquals"
      variable = "ec2:Add/group"
      values   = ["all"]
    }
  }

  statement {
    sid       = "DenyIAMUserCreation"
    effect    = "Deny"
    actions   = ["iam:CreateUser"]
    resources = ["*"]
  }

  statement {
    sid    = "DenyNonApprovedRegions"
    effect = "Deny"
    not_actions = [
      "iam:*",
      "organizations:*",
      "account:*",
      "cloudfront:*",
      "route53:*",
      "route53domains:*",
      "sts:*",
      "support:*",
      "trustedadvisor:*",
      "health:*",
      "budgets:*",
      "ce:*",
      "waf:*",
      "globalaccelerator:*",
      "sso:*",
      "sso-directory:*",
      "identitystore:*",
      "bedrock:*", // global inference profiles route across regions
    ]
    resources = ["*"]
    condition {
      test     = "StringNotEquals"
      variable = "aws:RequestedRegion"
      values = [
        "us-east-1",
        "us-west-2",
        "eu-central-1",
        "eu-west-1",
      ]
    }
  }

  statement {
    sid       = "DenyBedrock"
    effect    = "Deny"
    actions   = ["bedrock:*"]
    resources = ["*"]
    condition {
      test     = "StringNotLike"
      variable = "aws:userid"
      values   = ["*:admin.one", "*:admin.two"]
    }
    condition {
      test     = "StringNotEquals"
      variable = "aws:PrincipalAccount"
      values   = [local.privileged_account_id]
    }
  }
}

resource "aws_organizations_policy" "scp_deny_services" {
  name        = "DenyServices"
  description = "Org-wide deny rails: regions, instance classes, audit tampering, etc."
  content     = data.aws_iam_policy_document.deny_services.json
}

resource "aws_organizations_policy_attachment" "scp_deny_services_root" {
  policy_id = aws_organizations_policy.scp_deny_services.id
  target_id = aws_organizations_organization.main.roots[0].id
}

// ------------------------------------------------------------
// EnforceMFA — deny all-but-MFA-setup when MFA absent.
// Only applies to IAM users (not roles, not SSO sessions).
// ------------------------------------------------------------
data "aws_iam_policy_document" "enforce_mfa" {
  statement {
    sid       = "AllowViewAccountInfo"
    effect    = "Allow"
    actions   = ["iam:GetAccountPasswordPolicy", "iam:GetAccountSummary", "iam:ListVirtualMFADevices"]
    resources = ["*"]
  }

  statement {
    sid    = "AllowManageOwnPasswords"
    effect = "Allow"
    actions = [
      "iam:ChangePassword",
      "iam:GetUser",
      "iam:CreateLoginProfile",
      "iam:DeleteLoginProfile",
      "iam:GetLoginProfile",
      "iam:UpdateLoginProfile",
    ]
    resources = ["*"]
  }

  statement {
    sid    = "AllowManageOwnVirtualMFADevice"
    effect = "Allow"
    actions = [
      "iam:CreateVirtualMFADevice",
      "iam:DeleteVirtualMFADevice",
    ]
    resources = ["*"]
  }

  statement {
    sid    = "AllowManageOwnUserMFA"
    effect = "Allow"
    actions = [
      "iam:DeactivateMFADevice",
      "iam:EnableMFADevice",
      "iam:ListMFADevices",
      "iam:ResyncMFADevice",
    ]
    resources = ["*"]
  }

  statement {
    sid    = "DenyAllExceptListedIfNoMFA"
    effect = "Deny"
    not_actions = [
      "iam:CreateVirtualMFADevice",
      "iam:EnableMFADevice",
      "iam:ChangePassword",
      "iam:GetUser",
      "iam:CreateLoginProfile",
      "iam:DeleteLoginProfile",
      "iam:GetLoginProfile",
      "iam:UpdateLoginProfile",
      "iam:ListMFADevices",
      "iam:ListVirtualMFADevices",
      "iam:ResyncMFADevice",
      "iam:DeleteVirtualMFADevice",
      "sts:GetSessionToken",
    ]
    resources = ["*"]
    condition {
      test     = "BoolIfExists"
      variable = "aws:MultiFactorAuthPresent"
      values   = ["false"]
    }
    condition {
      test     = "StringLike"
      variable = "aws:PrincipalArn"
      values   = ["arn:aws:iam::*:user/*"]
    }
  }
}

resource "aws_organizations_policy" "enforce_mfa" {
  name        = "EnforceMFA"
  description = "Deny everything except MFA setup when MFA is not present (IAM users only)."
  content     = data.aws_iam_policy_document.enforce_mfa.json
}

resource "aws_organizations_policy_attachment" "enforce_mfa_root" {
  policy_id = aws_organizations_policy.enforce_mfa.id
  target_id = aws_organizations_organization.main.roots[0].id
}

02-eventbridge-rules.tf

// ============================================================
// EventBridge rules — 14 patterns, all targeting one Lambda.
// Pattern: tight source + eventName match, fan-out happens
// inside the Lambda (see 03-mgmt-events-notifier.js).
// ============================================================

resource "aws_cloudwatch_event_rule" "root_login" {
  name        = "aws-root-login"
  description = "Root user console login"
  event_pattern = jsonencode({
    source = ["aws.signin"]
    detail = {
      userIdentity = { type = ["Root"] }
      eventName    = ["ConsoleLogin"]
    }
  })
}

resource "aws_cloudwatch_event_rule" "console_login_no_mfa" {
  name        = "aws-console-login-no-mfa"
  description = "Console login without MFA by non-root user"
  event_pattern = jsonencode({
    source = ["aws.signin"]
    detail = {
      eventName           = ["ConsoleLogin"]
      additionalEventData = { MFAUsed = ["No"] }
      userIdentity        = { type = [{ "anything-but" = "Root" }] }
    }
  })
}

resource "aws_cloudwatch_event_rule" "console_login_failure" {
  name        = "aws-console-login-failure"
  description = "Console login failure"
  event_pattern = jsonencode({
    source = ["aws.signin"]
    detail = {
      eventName        = ["ConsoleLogin"]
      responseElements = { ConsoleLogin = ["Failure"] }
    }
  })
}

resource "aws_cloudwatch_event_rule" "cloudtrail_tampering" {
  name        = "aws-cloudtrail-tampering"
  description = "CloudTrail logging stopped or event selectors modified"
  event_pattern = jsonencode({
    source = ["aws.cloudtrail"]
    detail = {
      eventName = ["StopLogging", "DeleteTrail", "UpdateTrail", "PutEventSelectors"]
    }
  })
}

resource "aws_cloudwatch_event_rule" "iam_user_created" {
  name        = "aws-iam-user-created"
  description = "IAM user created"
  event_pattern = jsonencode({
    source = ["aws.iam"]
    detail = { eventName = ["CreateUser"] }
  })
}

resource "aws_cloudwatch_event_rule" "iam_access_key_created" {
  name        = "aws-iam-access-key-created"
  description = "IAM access key created"
  event_pattern = jsonencode({
    source = ["aws.iam"]
    detail = { eventName = ["CreateAccessKey"] }
  })
}

resource "aws_cloudwatch_event_rule" "iam_login_profile_created" {
  name        = "aws-iam-login-profile-created"
  description = "Console access added to an existing IAM user"
  event_pattern = jsonencode({
    source = ["aws.iam"]
    detail = { eventName = ["CreateLoginProfile"] }
  })
}

resource "aws_cloudwatch_event_rule" "mfa_deactivated" {
  name        = "aws-iam-mfa-deactivated"
  description = "MFA device deactivated for an IAM user"
  event_pattern = jsonencode({
    source = ["aws.iam"]
    detail = { eventName = ["DeactivateMFADevice"] }
  })
}

resource "aws_cloudwatch_event_rule" "identity_center_user_created" {
  name        = "aws-identity-center-user-created"
  description = "Identity Center user created"
  event_pattern = jsonencode({
    source = ["aws.identitystore"]
    detail = { eventName = ["CreateUser"] }
  })
}

resource "aws_cloudwatch_event_rule" "security_group_internet" {
  name        = "aws-security-group-internet"
  description = "Security group ingress rule added (lambda filters for 0.0.0.0/0 or ::/0)"
  event_pattern = jsonencode({
    source = ["aws.ec2"]
    detail = { eventName = ["AuthorizeSecurityGroupIngress"] }
  })
}

resource "aws_cloudwatch_event_rule" "cost_anomaly" {
  name        = "aws-cost-anomaly"
  description = "Cost anomaly detected by AWS Cost Anomaly Detection"
  event_pattern = jsonencode({
    source      = ["aws.ce"]
    detail-type = ["AWS Cost Anomaly Detection Alert"]
  })
}

resource "aws_cloudwatch_event_rule" "budget_alert" {
  name        = "aws-budget-alert"
  description = "AWS Budget threshold reached or forecasted to be reached"
  event_pattern = jsonencode({
    source      = ["aws.budgets"]
    detail-type = ["AWS Budget Notification"]
  })
}

resource "aws_cloudwatch_event_rule" "access_analyzer_finding" {
  name        = "aws-access-analyzer-finding"
  description = "IAM Access Analyzer detected external access to a resource"
  event_pattern = jsonencode({
    source      = ["aws.access-analyzer"]
    detail-type = ["Access Analyzer Finding"]
    detail      = { status = ["ACTIVE"] }
  })
}

resource "aws_cloudwatch_event_rule" "org_events" {
  name        = "aws-org-mgmt-events"
  description = "Organizations management events"
  event_pattern = jsonencode({
    source = ["aws.organizations"]
  })
}

// ------------------------------------------------------------
// All rules target the same Lambda.
// ------------------------------------------------------------
locals {
  notifier_rules = {
    root_login                   = aws_cloudwatch_event_rule.root_login.name
    console_login_no_mfa         = aws_cloudwatch_event_rule.console_login_no_mfa.name
    console_login_failure        = aws_cloudwatch_event_rule.console_login_failure.name
    cloudtrail_tampering         = aws_cloudwatch_event_rule.cloudtrail_tampering.name
    iam_user_created             = aws_cloudwatch_event_rule.iam_user_created.name
    iam_access_key_created       = aws_cloudwatch_event_rule.iam_access_key_created.name
    iam_login_profile_created    = aws_cloudwatch_event_rule.iam_login_profile_created.name
    mfa_deactivated              = aws_cloudwatch_event_rule.mfa_deactivated.name
    identity_center_user_created = aws_cloudwatch_event_rule.identity_center_user_created.name
    security_group_internet      = aws_cloudwatch_event_rule.security_group_internet.name
    cost_anomaly                 = aws_cloudwatch_event_rule.cost_anomaly.name
    budget_alert                 = aws_cloudwatch_event_rule.budget_alert.name
    access_analyzer_finding      = aws_cloudwatch_event_rule.access_analyzer_finding.name
    org_events                   = aws_cloudwatch_event_rule.org_events.name
  }
}

resource "aws_cloudwatch_event_target" "notifier" {
  for_each  = local.notifier_rules
  rule      = each.value
  target_id = "SendToLambda"
  arn       = aws_lambda_function.mgmt_events_notifier.arn
}

resource "aws_sqs_queue" "lambda_dlq" {
  name                      = "mgmt-events-notifier-dlq"
  message_retention_seconds = 1209600
}

resource "aws_lambda_function" "mgmt_events_notifier" {
  function_name    = "mgmt-events-notifier"
  role             = aws_iam_role.lambda_exec.arn
  handler          = "mgmt-events-notifier.handler"
  runtime          = "nodejs18.x"
  filename         = data.archive_file.notifier.output_path
  source_code_hash = data.archive_file.notifier.output_base64sha256
  memory_size      = 1024
  timeout          = 10

  dead_letter_config {
    target_arn = aws_sqs_queue.lambda_dlq.arn
  }

  environment {
    variables = {
      SES_FROM_EMAIL         = "noreply@example.com"
      SES_TO_EMAIL           = "security@example.com"
      SLACK_WEBHOOK_SSM_PATH = "/mgmt-events-notifier/slack-webhook-url"
    }
  }
}

data "archive_file" "notifier" {
  type        = "zip"
  source_file = "${path.module}/lambda/mgmt-events-notifier.js"
  output_path = "${path.module}/lambda/mgmt-events-notifier.zip"
}

03-mgmt-events-notifier.js

// ============================================================
// One Lambda handles all 14 EventBridge rules. The router picks
// a severity + formatter, then fans out to SES + Slack.
// ============================================================

const { SESClient, SendEmailCommand } = require("@aws-sdk/client-ses");
const { SSMClient, GetParameterCommand } = require("@aws-sdk/client-ssm");

const sesClient = new SESClient({ region: process.env.AWS_REGION });
const ssmClient = new SSMClient({ region: process.env.AWS_REGION });
const FROM_EMAIL = process.env.SES_FROM_EMAIL;
const TO_EMAIL = process.env.SES_TO_EMAIL;

let slackWebhookUrl = null;

async function getSlackWebhookUrl() {
  if (slackWebhookUrl) return slackWebhookUrl;
  const resp = await ssmClient.send(
    new GetParameterCommand({ Name: process.env.SLACK_WEBHOOK_SSM_PATH, WithDecryption: true })
  );
  slackWebhookUrl = resp.Parameter.Value;
  return slackWebhookUrl;
}

exports.handler = async (event) => {
  try {
    const notification = buildNotification(event);
    if (!notification) {
      console.log("Event filtered out, no notification sent");
      return;
    }

    const { subject, html, text, slack } = notification;
    const webhookUrl = await getSlackWebhookUrl();

    await Promise.all([
      sendEmail(subject, html, text),
      sendSlack(webhookUrl, slack),
    ]);
  } catch (error) {
    console.error(`Error sending notification: ${error}`);
    throw error;
  }
};

async function sendEmail(subject, html, text) {
  const response = await sesClient.send(
    new SendEmailCommand({
      Source: FROM_EMAIL,
      Destination: { ToAddresses: [TO_EMAIL] },
      Message: {
        Subject: { Data: subject },
        Body: { Html: { Data: html }, Text: { Data: text } },
      },
    })
  );
  console.log(`Email sent: ${response.MessageId}`);
}

async function sendSlack(webhookUrl, payload) {
  const resp = await fetch(webhookUrl, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(payload),
  });
  if (!resp.ok) throw new Error(`Slack webhook returned ${resp.status}: ${await resp.text()}`);
  console.log("Slack notification sent");
}

// ---------------------------------------------------------------------------
// Routing — single dispatcher per event signature
// ---------------------------------------------------------------------------

function buildNotification(event) {
  const eventName = event.detail?.eventName;
  const source = event.source;
  const userType = event.detail?.userIdentity?.type;

  if (eventName === "ConsoleLogin" && userType === "Root") {
    return render("critical", "Root User Login", formatRootLogin(event));
  }
  if (eventName === "ConsoleLogin" && event.detail?.responseElements?.ConsoleLogin === "Failure") {
    return render("medium", "Console Login Failure", formatConsoleLoginFailure(event));
  }
  if (eventName === "ConsoleLogin") {
    return render("medium", "Console Login Without MFA", formatNoMFALogin(event));
  }
  if (eventName === "StopLogging") {
    return render("critical", "CloudTrail Logging Stopped", formatCloudTrailTampering(event));
  }
  if (eventName === "PutEventSelectors") {
    return render("critical", "CloudTrail Event Selectors Modified", formatCloudTrailTampering(event));
  }
  if (source === "aws.iam" && eventName === "CreateUser") {
    return render("info", "IAM User Created", formatIAMUserCreated(event));
  }
  if (source === "aws.iam" && eventName === "CreateAccessKey") {
    return render("info", "IAM Access Key Created", formatIAMAccessKeyCreated(event));
  }
  if (source === "aws.identitystore" && eventName === "CreateUser") {
    return render("info", "Identity Center User Created", formatIdentityCenterUserCreated(event));
  }
  if (eventName === "DeactivateMFADevice") {
    return render("critical", "MFA Device Deactivated", formatMFADeactivated(event));
  }
  if (eventName === "AuthorizeSecurityGroupIngress") {
    return formatSecurityGroupExposure(event); // returns null if no 0.0.0.0/0 or ::/0
  }
  if (event["detail-type"] === "Access Analyzer Finding") {
    return render("high", "Access Analyzer Finding", formatAccessAnalyzerFinding(event));
  }
  if (event["detail-type"] === "AWS Budget Notification") {
    return render("medium", "Budget Threshold Reached", formatBudgetAlert(event));
  }
  if (event["detail-type"] === "AWS Cost Anomaly Detection Alert") {
    return render("high", "Cost Anomaly Detected", formatCostAnomaly(event));
  }
  if (source === "aws.iam" && eventName === "CreateLoginProfile") {
    return render("high", "Console Access Added to IAM User", formatLoginProfileCreated(event));
  }

  return render("info", "AWS Management Event", formatMgmtEvent(event));
}

// ---------------------------------------------------------------------------
// Rendering
// ---------------------------------------------------------------------------

const SEVERITY = {
  critical: { bg: "#dc2626", label: "CRITICAL" },
  high:     { bg: "#ea580c", label: "HIGH" },
  medium:   { bg: "#d97706", label: "MEDIUM" },
  info:     { bg: "#2563eb", label: "INFO" },
};

function render(severity, title, { fields, extraHtml = "", extraSlackBlocks = [] }) {
  const { bg, label } = SEVERITY[severity] ?? SEVERITY.info;
  const subject = `[${label}] ${title}`;

  const html  = buildHtml(bg, label, title, fields, extraHtml);
  const text  = buildText(label, title, fields);
  const slack = buildSlack(bg, label, title, fields, extraSlackBlocks);

  return { subject, html, text, slack };
}

function buildHtml(bg, label, title, fields, extra) {
  const rowsHtml = fields
    .map(
      ([k, v]) =>
        `<tr>
          <td style="color:#6b7280;font-size:13px;padding:7px 16px 7px 0;white-space:nowrap;vertical-align:top">${k}</td>
          <td style="font-size:13px;color:#111827;font-family:'SF Mono',Menlo,monospace;word-break:break-all;padding:7px 0">${esc(v)}</td>
        </tr>`
    )
    .join("");

  return `<!DOCTYPE html>
<html>
<head><meta charset="utf-8"></head>
<body style="margin:0;padding:24px;background:#f3f4f6;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif">
  <div style="max-width:560px;margin:0 auto;background:#fff;border-radius:8px;overflow:hidden;box-shadow:0 1px 3px rgba(0,0,0,.1)">
    <div style="background:${bg};padding:18px 24px">
      <span style="color:rgba(255,255,255,.7);font-size:11px;font-weight:600;letter-spacing:.08em;text-transform:uppercase">${label}</span>
      <h1 style="margin:4px 0 0;color:#fff;font-size:17px;font-weight:600">${esc(title)}</h1>
    </div>
    <div style="padding:20px 24px">
      <table style="width:100%;border-collapse:collapse">${rowsHtml}</table>
      ${extra}
    </div>
    <div style="padding:14px 24px;background:#f9fafb;border-top:1px solid #e5e7eb;font-size:11px;color:#9ca3af">
      AWS Organization Management Alert &middot; example.com
    </div>
  </div>
</body>
</html>`;
}

function buildText(label, title, fields) {
  const rows = fields.map(([k, v]) => `${k.padEnd(18)} ${v ?? "N/A"}`).join("\n");
  return `[${label}] ${title}\n${"=".repeat(50)}\n${rows}`;
}

function buildSlack(color, label, title, fields, extraBlocks = []) {
  const lines = fields
    .filter(([, v]) => v != null)
    .map(([k, v]) => `*${k}:* ${v}`)
    .join("\n");

  const blocks = [
    { type: "header",  text: { type: "plain_text", text: `[${label}] ${title}`, emoji: false } },
    { type: "section", text: { type: "mrkdwn", text: lines } },
    ...extraBlocks,
    { type: "context", elements: [{ type: "mrkdwn", text: "AWS Organization · example.com" }] },
  ];

  return { attachments: [{ color, blocks }] };
}

function esc(v) {
  if (v == null) return '<span style="color:#9ca3af">N/A</span>';
  return String(v)
    .replace(/&/g, "&amp;")
    .replace(/</g, "&lt;")
    .replace(/>/g, "&gt;")
    .replace(/"/g, "&quot;");
}

// ---------------------------------------------------------------------------
// Field extractors (a sample — same pattern for the rest)
// ---------------------------------------------------------------------------

function formatRootLogin(event) {
  return {
    fields: [
      ["Account ID", event.detail?.userIdentity?.accountId],
      ["Source IP",  event.detail?.sourceIPAddress],
      ["Result",     event.detail?.responseElements?.ConsoleLogin],
      ["Time",       event.time],
    ],
  };
}

function formatConsoleLoginFailure(event) {
  const id = event.detail?.userIdentity;
  return {
    fields: [
      ["Account ID", id?.accountId],
      ["User",       id?.arn ?? id?.principalId],
      ["Source IP",  event.detail?.sourceIPAddress],
      ["Time",       event.time],
    ],
  };
}

function formatNoMFALogin(event) {
  const id = event.detail?.userIdentity;
  return {
    fields: [
      ["Account ID", id?.accountId],
      ["User",       id?.arn ?? id?.userName ?? id?.principalId],
      ["Source IP",  event.detail?.sourceIPAddress],
      ["Time",       event.time],
    ],
  };
}

function formatCloudTrailTampering(event) {
  const id = event.detail?.userIdentity;
  return {
    fields: [
      ["Event",      event.detail?.eventName],
      ["Account ID", id?.accountId],
      ["User",       id?.arn ?? id?.principalId],
      ["Source IP",  event.detail?.sourceIPAddress],
      ["Trail",      event.detail?.requestParameters?.trailName],
      ["Time",       event.time],
    ],
  };
}

function formatIAMUserCreated(event) {
  const id = event.detail?.userIdentity;
  const newUser = event.detail?.responseElements?.user;
  return {
    fields: [
      ["New User",   newUser?.userName],
      ["User ARN",   newUser?.arn],
      ["Account ID", id?.accountId],
      ["Created By", id?.arn ?? id?.principalId],
      ["Source IP",  event.detail?.sourceIPAddress],
      ["Time",       event.time],
    ],
  };
}

function formatIAMAccessKeyCreated(event) {
  const id = event.detail?.userIdentity;
  const key = event.detail?.responseElements?.accessKey;
  return {
    fields: [
      ["Key ID",     key?.accessKeyId],
      ["For User",   key?.userName ?? event.detail?.requestParameters?.userName],
      ["Account ID", id?.accountId],
      ["Created By", id?.arn ?? id?.principalId],
      ["Source IP",  event.detail?.sourceIPAddress],
      ["Time",       event.time],
    ],
  };
}

function formatIdentityCenterUserCreated(event) {
  const id = event.detail?.userIdentity;
  const req = event.detail?.requestParameters;
  const email = req?.emails?.find((e) => e.primary)?.value ?? req?.emails?.[0]?.value;
  return {
    fields: [
      ["User Name",      req?.userName],
      ["Email",          email],
      ["Identity Store", req?.identityStoreId],
      ["Created By",     id?.arn ?? id?.principalId],
      ["Time",           event.time],
    ],
  };
}

function formatSecurityGroupExposure(event) {
  const items = event.detail?.requestParameters?.ipPermissions?.items ?? [];
  const openRules = items.filter((item) => {
    const ipv4Open = item.ipRanges?.items?.some((r) => r.cidrIp === "0.0.0.0/0");
    const ipv6Open = item.ipv6Ranges?.items?.some((r) => r.cidrIpv6 === "::/0");
    return ipv4Open || ipv6Open;
  });

  if (openRules.length === 0) return null;

  const id = event.detail?.userIdentity;

  const ruleLines = openRules.map((rule) => {
    const proto = rule.ipProtocol === "-1" ? "all traffic" : rule.ipProtocol;
    const ports = rule.fromPort != null ? `port ${rule.fromPort}-${rule.toPort}` : "all ports";
    return { proto, ports };
  });

  const rulesHtml  = ruleLines.map(({ proto, ports }) => `<li>${esc(proto)} ${esc(ports)}</li>`).join("");
  const rulesSlack = ruleLines.map(({ proto, ports }) => `• \`${proto}\` ${ports}`).join("\n");
  const rulesText  = ruleLines.map(({ proto, ports }) => `  • ${proto} ${ports}`).join("\n");

  const base = render("high", "Security Group Opened to Internet", {
    fields: [
      ["Security Group", event.detail?.requestParameters?.groupId],
      ["Account ID",     id?.accountId],
      ["Changed By",     id?.arn ?? id?.principalId],
      ["Source IP",      event.detail?.sourceIPAddress],
      ["Time",           event.time],
    ],
    extraHtml: `<p style="margin:16px 0 6px;font-size:13px;color:#6b7280">Rules opened to internet:</p><ul style="margin:0;padding-left:20px">${rulesHtml}</ul>`,
    extraSlackBlocks: [
      { type: "section", text: { type: "mrkdwn", text: `*Rules opened to internet:*\n${rulesSlack}` } },
    ],
  });

  base.text += `\n\nRules opened to internet:\n${rulesText}`;
  return base;
}

function formatCostAnomaly(event) {
  const d = event.detail;
  return {
    fields: [
      ["Monitor",  d?.monitorName],
      ["Service",  d?.anomalyDetails?.serviceName ?? d?.rootCauses?.[0]?.service],
      ["Impact",   d?.impact?.totalActualSpend   != null ? `$${parseFloat(d.impact.totalActualSpend).toFixed(2)}`   : null],
      ["Expected", d?.impact?.totalExpectedSpend != null ? `$${parseFloat(d.impact.totalExpectedSpend).toFixed(2)}` : null],
      ["Account",  d?.accountId ?? event.account],
      ["Time",     event.time],
    ],
  };
}

function formatLoginProfileCreated(event) {
  const id = event.detail?.userIdentity;
  return {
    fields: [
      ["User",       event.detail?.requestParameters?.userName],
      ["Account",    id?.accountId],
      ["Created By", id?.arn ?? id?.principalId],
      ["IP",         event.detail?.sourceIPAddress],
      ["Time",       event.time],
    ],
  };
}

function formatMFADeactivated(event) {
  const id = event.detail?.userIdentity;
  return {
    fields: [
      ["User",    event.detail?.requestParameters?.userName],
      ["Account", id?.accountId],
      ["By",      id?.arn ?? id?.principalId],
      ["IP",      event.detail?.sourceIPAddress],
      ["Time",    event.time],
    ],
  };
}

function formatAccessAnalyzerFinding(event) {
  const d = event.detail;
  return {
    fields: [
      ["Resource",      d?.resource],
      ["Resource Type", d?.resourceType],
      ["Finding Type",  d?.findingType],
      ["Is Public",     d?.isPublic != null ? String(d.isPublic) : null],
      ["Account",       d?.accountId],
      ["Region",        d?.region],
      ["Time",          event.time],
    ],
  };
}

function formatBudgetAlert(event) {
  const d = event.detail;
  return {
    fields: [
      ["Budget",     d?.budgetName],
      ["Limit",      d?.budgetLimit?.amount    ? `$${d.budgetLimit.amount}`    : null],
      ["Actual",     d?.currentSpend?.amount   ? `$${d.currentSpend.amount}`   : null],
      ["Forecasted", d?.forecastedSpend?.amount ? `$${d.forecastedSpend.amount}` : null],
      ["Alert Type", d?.notificationType],
      ["Account",    d?.accountId],
      ["Time",       event.time],
    ],
  };
}

function formatMgmtEvent(event) {
  const id          = event.detail?.userIdentity;
  const principalId = id?.principalId ?? "";
  const user        = principalId.includes(":") ? principalId.split(":").pop() : (id?.userName ?? principalId);

  return {
    fields: [
      ["Event",   event.detail?.eventName],
      ["Account", id?.accountId],
      ["User",    user || null],
      ["IP",      event.detail?.sourceIPAddress],
      ["Time",    event.time],
    ],
  };
}

04-resource-watchdog.tf

// ============================================================
// Resource Watchdog — weekly cron Lambda that walks every
// member account and remediates / reports drift.
// ============================================================

variable "slack_webhook_ssm_path" {
  type    = string
  default = "/resource-watchdog/slack-webhook-url"
}

data "aws_caller_identity" "current" {}
data "aws_region" "current" {}

// ------------------------------------------------------------
// Cron — every Friday 06:00 UTC
// ------------------------------------------------------------
resource "aws_cloudwatch_event_rule" "weekly" {
  name                = "resource-watchdog-weekly"
  description         = "Walk every member account, stop running EC2/ECS/RDS, etc."
  schedule_expression = "cron(0 6 ? * FRI *)"
}

resource "aws_cloudwatch_event_target" "lambda" {
  rule      = aws_cloudwatch_event_rule.weekly.name
  target_id = "ResourceWatchdogLambda"
  arn       = aws_lambda_function.resource_watchdog.arn
}

resource "aws_lambda_permission" "eventbridge" {
  statement_id  = "AllowExecutionFromEventBridge"
  action        = "lambda:InvokeFunction"
  function_name = aws_lambda_function.resource_watchdog.function_name
  principal     = "events.amazonaws.com"
  source_arn    = aws_cloudwatch_event_rule.weekly.arn
}

// ------------------------------------------------------------
// Lambda
// ------------------------------------------------------------
resource "aws_sqs_queue" "dlq" {
  name                      = "resource-watchdog-dlq"
  message_retention_seconds = 1209600
}

data "archive_file" "package" {
  type        = "zip"
  source_file = "${path.module}/lambda/resource-watchdog.js"
  output_path = "${path.module}/lambda/resource-watchdog.zip"
}

resource "aws_lambda_function" "resource_watchdog" {
  function_name    = "resource-watchdog"
  role             = aws_iam_role.lambda_exec.arn
  handler          = "resource-watchdog.handler"
  runtime          = "nodejs18.x"
  filename         = data.archive_file.package.output_path
  source_code_hash = data.archive_file.package.output_base64sha256
  timeout          = 300

  dead_letter_config { target_arn = aws_sqs_queue.dlq.arn }

  environment {
    variables = {
      SLACK_WEBHOOK_SSM_PATH = var.slack_webhook_ssm_path
      MANAGEMENT_ACCOUNT_ID  = data.aws_caller_identity.current.account_id
    }
  }
}

resource "aws_cloudwatch_log_group" "lambda" {
  name              = "/aws/lambda/${aws_lambda_function.resource_watchdog.function_name}"
  retention_in_days = 14
}

// ------------------------------------------------------------
// IAM — assume OrganizationAccountAccessRole in every member,
// then describe + stop resources from there.
// ------------------------------------------------------------
resource "aws_iam_role" "lambda_exec" {
  name = "resource-watchdog"
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Action    = "sts:AssumeRole"
      Effect    = "Allow"
      Principal = { Service = "lambda.amazonaws.com" }
    }]
  })
}

resource "aws_iam_role_policy" "logs" {
  name = "logs"
  role = aws_iam_role.lambda_exec.id
  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Effect   = "Allow"
      Action   = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"]
      Resource = "*"
    }]
  })
}

resource "aws_iam_role_policy" "orgs_and_sts" {
  name = "orgs-and-sts"
  role = aws_iam_role.lambda_exec.id
  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect   = "Allow"
        Action   = ["organizations:ListAccounts"]
        Resource = "*"
      },
      {
        Effect   = "Allow"
        Action   = ["sts:AssumeRole"]
        Resource = "arn:aws:iam::*:role/OrganizationAccountAccessRole"
      },
      {
        Effect = "Allow"
        Action = [
          "ec2:DescribeInstances",
          "ec2:StopInstances",
          "ec2:DescribeVolumes",
          "ec2:DescribeAddresses",
          "ec2:ReleaseAddress",
          "ecs:ListClusters",
          "ecs:ListServices",
          "ecs:DescribeServices",
          "ecs:UpdateService",
          "rds:DescribeDBInstances",
          "rds:StopDBInstance",
          "rds:DescribeDBClusters",
          "rds:StopDBCluster",
          "elasticloadbalancing:DescribeLoadBalancers",
          "elasticloadbalancing:DescribeTargetGroups",
          "elasticloadbalancing:DescribeTargetHealth",
          "logs:DescribeLogGroups",
          "logs:PutRetentionPolicy",
        ]
        Resource = "*"
      },
    ]
  })
}

resource "aws_iam_role_policy" "ssm" {
  name = "ssm"
  role = aws_iam_role.lambda_exec.id
  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Effect   = "Allow"
      Action   = ["ssm:GetParameter"]
      Resource = "arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter${var.slack_webhook_ssm_path}"
    }]
  })
}

resource "aws_iam_role_policy" "dlq" {
  name = "dlq"
  role = aws_iam_role.lambda_exec.id
  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Effect   = "Allow"
      Action   = ["sqs:SendMessage"]
      Resource = aws_sqs_queue.dlq.arn
    }]
  })
}

05-resource-watchdog.js

// ============================================================
// Resource Watchdog — runs weekly, hops into each member
// account via OrganizationAccountAccessRole, then stops or
// flags drift across the 4 approved regions.
// ============================================================

const { OrganizationsClient, ListAccountsCommand } = require("@aws-sdk/client-organizations");
const { STSClient, AssumeRoleCommand } = require("@aws-sdk/client-sts");
const {
  EC2Client,
  DescribeInstancesCommand, StopInstancesCommand,
  DescribeVolumesCommand,
  DescribeAddressesCommand, ReleaseAddressCommand,
} = require("@aws-sdk/client-ec2");
const {
  ECSClient,
  ListClustersCommand, ListServicesCommand, DescribeServicesCommand, UpdateServiceCommand,
} = require("@aws-sdk/client-ecs");
const {
  RDSClient,
  DescribeDBInstancesCommand, StopDBInstanceCommand,
  DescribeDBClustersCommand, StopDBClusterCommand,
} = require("@aws-sdk/client-rds");
const {
  ElasticLoadBalancingV2Client,
  DescribeLoadBalancersCommand, DescribeTargetGroupsCommand, DescribeTargetHealthCommand,
} = require("@aws-sdk/client-elastic-load-balancing-v2");
const {
  CloudWatchLogsClient,
  DescribeLogGroupsCommand, PutRetentionPolicyCommand,
} = require("@aws-sdk/client-cloudwatch-logs");
const { SSMClient, GetParameterCommand } = require("@aws-sdk/client-ssm");

const REGIONS            = ["us-east-1", "us-west-2", "eu-central-1", "eu-west-1"];
const ROLE_NAME          = "OrganizationAccountAccessRole";
const LOG_RETENTION_DAYS = 30;

const orgsClient   = new OrganizationsClient({ region: "us-east-1" });
const stsClient    = new STSClient({ region: "us-east-1" });
const ssmClient    = new SSMClient({ region: process.env.AWS_REGION });
const MGMT_ACCOUNT = process.env.MANAGEMENT_ACCOUNT_ID;

let slackWebhookUrl = null;

async function getSlackWebhookUrl() {
  if (slackWebhookUrl) return slackWebhookUrl;
  const resp = await ssmClient.send(
    new GetParameterCommand({ Name: process.env.SLACK_WEBHOOK_SSM_PATH, WithDecryption: true })
  );
  slackWebhookUrl = resp.Parameter.Value;
  return slackWebhookUrl;
}

async function assumeRole(accountId) {
  const resp = await stsClient.send(new AssumeRoleCommand({
    RoleArn:         `arn:aws:iam::${accountId}:role/${ROLE_NAME}`,
    RoleSessionName: "resource-watchdog",
    DurationSeconds: 900,
  }));
  return {
    accessKeyId:     resp.Credentials.AccessKeyId,
    secretAccessKey: resp.Credentials.SecretAccessKey,
    sessionToken:    resp.Credentials.SessionToken,
  };
}

exports.handler = async () => {
  const accountsResp = await orgsClient.send(new ListAccountsCommand({}));
  const accounts     = accountsResp.Accounts.filter(a => a.Status === "ACTIVE" && a.Id !== MGMT_ACCOUNT);

  const remediated = [];
  const flagged    = [];

  await Promise.all(accounts.map(async (account) => {
    let credentials;
    try {
      credentials = await assumeRole(account.Id);
    } catch (err) {
      console.warn(`Cannot assume role in ${account.Name} (${account.Id}): ${err.message}`);
      return;
    }

    await Promise.all(REGIONS.map(async (region) => {
      const cc      = { credentials };
      const results = await Promise.allSettled([
        stopEC2(account.Name, region, cc),
        stopECS(account.Name, region, cc),
        stopRDS(account.Name, region, cc),
        releaseEIPs(account.Name, region, cc),
        reportEBS(account.Name, region, cc),
        reportEmptyLBs(account.Name, region, cc),
        fixLogGroupRetention(account.Name, region, cc),
      ]);

      for (const result of results) {
        if (result.status === "fulfilled") {
          for (const item of result.value) {
            (item.flagged ? flagged : remediated).push(item);
          }
        }
      }
    }));
  }));

  const webhookUrl = await getSlackWebhookUrl();
  await sendSlack(webhookUrl, remediated, flagged);
};

// ---------------------------------------------------------------------------
// Checks
// ---------------------------------------------------------------------------

async function stopEC2(account, region, cc) {
  const results = [];
  const ec2     = new EC2Client({ region, ...cc });
  const resp    = await ec2.send(new DescribeInstancesCommand({
    Filters: [{ Name: "instance-state-name", Values: ["running"] }],
  }));
  const instances = resp.Reservations.flatMap(r => r.Instances);
  if (instances.length === 0) return results;

  await ec2.send(new StopInstancesCommand({ InstanceIds: instances.map(i => i.InstanceId) }));
  for (const i of instances) {
    const name = i.Tags?.find(t => t.Key === "Name")?.Value ?? i.InstanceId;
    results.push({ type: "EC2", account, region, resource: name, action: "stopped" });
  }
  return results;
}

async function stopECS(account, region, cc) {
  const results      = [];
  const ecs          = new ECSClient({ region, ...cc });
  const clustersResp = await ecs.send(new ListClustersCommand({}));
  if (clustersResp.clusterArns.length === 0) return results;

  await Promise.all(clustersResp.clusterArns.map(async (clusterArn) => {
    const clusterName  = clusterArn.split("/").pop();
    const servicesResp = await ecs.send(new ListServicesCommand({ cluster: clusterArn }));
    if (servicesResp.serviceArns.length === 0) return;

    const descResp = await ecs.send(new DescribeServicesCommand({ cluster: clusterArn, services: servicesResp.serviceArns }));
    const active   = descResp.services.filter(s => s.desiredCount > 0 || s.runningCount > 0);

    await Promise.all(active.map(async (svc) => {
      await ecs.send(new UpdateServiceCommand({ cluster: clusterArn, service: svc.serviceArn, desiredCount: 0 }));
      results.push({ type: "ECS", account, region, resource: `${clusterName}/${svc.serviceName}`, action: "scaled to 0" });
    }));
  }));
  return results;
}

async function stopRDS(account, region, cc) {
  const results = [];
  const rds     = new RDSClient({ region, ...cc });

  const [instancesResp, clustersResp] = await Promise.all([
    rds.send(new DescribeDBInstancesCommand({})),
    rds.send(new DescribeDBClustersCommand({})),
  ]);

  const auroraClusterIds = new Set(
    (clustersResp.DBClusters ?? [])
      .filter(c => c.Status === "available")
      .map(c => c.DBClusterIdentifier)
  );

  for (const cluster of clustersResp.DBClusters ?? []) {
    if (cluster.Status !== "available") continue;
    await rds.send(new StopDBClusterCommand({ DBClusterIdentifier: cluster.DBClusterIdentifier }));
    results.push({ type: "RDS Cluster", account, region, resource: cluster.DBClusterIdentifier, action: "stopped" });
  }

  for (const db of instancesResp.DBInstances ?? []) {
    if (db.DBStatus !== "available") continue;
    if (db.DBClusterIdentifier && auroraClusterIds.has(db.DBClusterIdentifier)) continue;
    await rds.send(new StopDBInstanceCommand({ DBInstanceIdentifier: db.DBInstanceIdentifier }));
    results.push({ type: "RDS", account, region, resource: db.DBInstanceIdentifier, action: "stopped" });
  }

  return results;
}

async function releaseEIPs(account, region, cc) {
  const results    = [];
  const ec2        = new EC2Client({ region, ...cc });
  const resp       = await ec2.send(new DescribeAddressesCommand({}));
  const unattached = (resp.Addresses ?? []).filter(a => !a.AssociationId);

  for (const addr of unattached) {
    await ec2.send(new ReleaseAddressCommand({ AllocationId: addr.AllocationId }));
    results.push({ type: "EIP", account, region, resource: addr.PublicIp, action: "released" });
  }
  return results;
}

async function reportEBS(account, region, cc) {
  const results = [];
  const ec2     = new EC2Client({ region, ...cc });
  const resp    = await ec2.send(new DescribeVolumesCommand({
    Filters: [{ Name: "status", Values: ["available"] }],
  }));

  for (const vol of resp.Volumes ?? []) {
    const name = vol.Tags?.find(t => t.Key === "Name")?.Value ?? vol.VolumeId;
    results.push({ type: "EBS", account, region, resource: `${name} (${vol.Size}GB)`, action: "unattached — review", flagged: true });
  }
  return results;
}

async function reportEmptyLBs(account, region, cc) {
  const results = [];
  const elb     = new ElasticLoadBalancingV2Client({ region, ...cc });
  const lbsResp = await elb.send(new DescribeLoadBalancersCommand({}));

  await Promise.all((lbsResp.LoadBalancers ?? []).map(async (lb) => {
    const tgsResp = await elb.send(new DescribeTargetGroupsCommand({ LoadBalancerArn: lb.LoadBalancerArn }));
    const tgs     = tgsResp.TargetGroups ?? [];

    if (tgs.length === 0) {
      results.push({ type: "ALB/NLB", account, region, resource: lb.LoadBalancerName, action: "no target groups — review", flagged: true });
      return;
    }

    const healthResults = await Promise.all(tgs.map(tg =>
      elb.send(new DescribeTargetHealthCommand({ TargetGroupArn: tg.TargetGroupArn }))
    ));
    const totalTargets = healthResults.reduce((s, r) => s + (r.TargetHealthDescriptions ?? []).length, 0);
    if (totalTargets === 0) {
      results.push({ type: "ALB/NLB", account, region, resource: lb.LoadBalancerName, action: "no registered targets — review", flagged: true });
    }
  }));
  return results;
}

async function fixLogGroupRetention(account, region, cc) {
  const results = [];
  const logs    = new CloudWatchLogsClient({ region, ...cc });
  let nextToken;

  do {
    const resp        = await logs.send(new DescribeLogGroupsCommand({ nextToken }));
    const noRetention = (resp.logGroups ?? []).filter(g => !g.retentionInDays);

    await Promise.all(noRetention.map(async (g) => {
      await logs.send(new PutRetentionPolicyCommand({
        logGroupName:    g.logGroupName,
        retentionInDays: LOG_RETENTION_DAYS,
      }));
      results.push({ type: "Log Group", account, region, resource: g.logGroupName, action: `retention set to ${LOG_RETENTION_DAYS}d` });
    }));

    nextToken = resp.nextToken;
  } while (nextToken);

  return results;
}

// ---------------------------------------------------------------------------
// Slack
// ---------------------------------------------------------------------------

async function sendSlack(webhookUrl, remediated, flagged) {
  const now   = new Date();
  const total = remediated.length + flagged.length;
  let text;

  if (total === 0) {
    text = `All clear across all member accounts.`;
  } else {
    const parts = [];

    if (remediated.length > 0) {
      const lines = groupLogGroups(remediated)
        .map(r => `• *${r.type}* \`${r.resource}\` ${r.action} — ${r.region} (${r.account})`)
        .join("\n");
      parts.push(`*Remediated (${remediated.length}):*\n${lines}`);
    }

    if (flagged.length > 0) {
      const lines = flagged
        .map(r => `• *${r.type}* \`${r.resource}\` — ${r.region} (${r.account})`)
        .join("\n");
      parts.push(`*Flagged for review (${flagged.length}):*\n${lines}`);
    }

    text = parts.join("\n\n");
  }

  const blocks = [
    { type: "header",  text: { type: "plain_text", text: `Resource Watchdog — ${now.toDateString()}`, emoji: false } },
    { type: "section", text: { type: "mrkdwn", text } },
    { type: "context", elements: [{ type: "mrkdwn", text: "AWS Organization · example.com" }] },
  ];

  const resp = await fetch(webhookUrl, {
    method:  "POST",
    headers: { "Content-Type": "application/json" },
    body:    JSON.stringify({ blocks }),
  });
  if (!resp.ok) throw new Error(`Slack webhook returned ${resp.status}: ${await resp.text()}`);
  console.log(`Watchdog complete. Remediated: ${remediated.length}, Flagged: ${flagged.length}`);
}

function groupLogGroups(results) {
  const logGroups = results.filter(r => r.type === "Log Group");
  const rest      = results.filter(r => r.type !== "Log Group");

  const byAccountRegion = {};
  for (const lg of logGroups) {
    const key = `${lg.account}|${lg.region}`;
    byAccountRegion[key] = byAccountRegion[key] ?? { ...lg, count: 0 };
    byAccountRegion[key].count++;
  }

  const grouped = Object.values(byAccountRegion).map(g => ({
    ...g,
    resource: `${g.count} log group${g.count > 1 ? "s" : ""}`,
    action:   `retention set to ${LOG_RETENTION_DAYS}d`,
  }));

  return [...rest, ...grouped];
}