feat(monitoring): alert on ECS services that cannot start tasks

CLAUDE.md

@@ -221,6 +221,7 @@ EventBridge ──► javabin-security SNS ──► slack-alert Lambda:
   Security Hub findings (NEW only) ──► #platform-security-alerts
   GuardDuty findings              ──► #platform-security-alerts
   IAM / resource / login events   ──► #javabin-infra-alerts
+  ECS service can't start tasks   ──► #javabin-infra-alerts (once per service per day)
 Cost Anomaly ──► javabin-alerts SNS ──► slack-alert Lambda ──► #javabin-cost-alerts
 
 Scheduled:

docs/lambda-functions.md

@@ -17,6 +17,8 @@ Terraform in `terraform/platform/lambdas/`.
 
 Features: GitHub OIDC attribution (extracts actor/repo/SHA from session tags), cost estimation via shared pricing module, Security Hub finding formatting.
 
+ECS `SERVICE_TASK_START_IMPAIRED` events from the `javabin-platform` cluster only (a crash-looping service) are posted to #javabin-infra-alerts at most once per service per UTC day. Legacy clusters (vaultwarden, cakeredux, moresleep-test, dennis-test) are not covered. As with the other deduplicated alerts, the dedup key is written before the Slack post, so a failed post suppresses that service's alert for the rest of the day; if the dedup table is unreadable, dedup fails open.
+
 Uses the `javabin-alert-dedup` DynamoDB table for deduplication of alerts (prevents repeated notifications for the same finding).
 
 ## cost-report

terraform/lambda-src/slack_alert/handler.py

@@ -1172,6 +1172,57 @@ def format_budget_alert(parsed):
     return {"blocks": blocks, "text": fallback}
 
 
+def format_ecs_service_impaired(parsed):
+    """ECS service cannot start tasks (SERVICE_TASK_START_IMPAIRED) — Block Kit.
+
+    ECS keeps emitting this event while a service crash-loops, so alert at most
+    once per service per UTC day (persistent DynamoDB dedup). A crash loop
+    burns Fargate starts, image pulls and AWS Config items until someone acts.
+    """
+    detail = parsed.get("detail", {})
+    if detail.get("eventName") != "SERVICE_TASK_START_IMPAIRED":
+        return None
+
+    resources = parsed.get("resources") or []
+    service_arn = resources[0] if resources else "unknown"
+    service = service_arn.rsplit("/", 1)[-1]
+    cluster = detail.get("clusterArn", "").rsplit("/", 1)[-1] or "unknown"
+    region = parsed.get("region", "")
+
+    day = (parsed.get("time") or datetime.now(timezone.utc).isoformat())[:10]
+    dedup_key = f"ecs-start-impaired:{service_arn}:{day}"
+    if is_finding_already_alerted(dedup_key):
+        logger.info("Suppressed repeat ECS start-impaired alert: %s", dedup_key)
+        return None
+    record_finding_alert(dedup_key)
+
+    events_url = (f"https://{region}.console.aws.amazon.com/ecs/v2/clusters/{cluster}"
+                  f"/services/{service}/events?region={region}")
+
+    blocks = [
+        {"type": "header",
+         "text": {"type": "plain_text", "text": ":warning: ECS service cannot start tasks",
+                  "emoji": True}},
+        {"type": "section",
+         "text": {"type": "mrkdwn",
+                  "text": (f"*{service}* keeps failing to start tasks and ECS is backing off. "
+                           "Usual causes: missing image, failing health check, bad secret. "
+                           "Fix the deploy or scale the service to 0.")}},
+        {"type": "section", "fields": [
+            {"type": "mrkdwn", "text": f"*Service*\n`{service}`"},
+            {"type": "mrkdwn", "text": f"*Cluster*\n`{cluster}`"},
+        ]},
+        {"type": "section",
+         "text": {"type": "mrkdwn", "text": f"<{events_url}|Service events in the ECS console>"}},
+        {"type": "divider"},
+        {"type": "context", "elements": [
+            {"type": "mrkdwn", "text": "Source: ECS Service Action (alerts once per service per day)"}
+        ]},
+    ]
+
+    return {"blocks": blocks, "text": f"ECS service {service} cannot start tasks"}
+
+
 # ---------------------------------------------------------------------------
 # Weekly Security Hub summary
 # ---------------------------------------------------------------------------
@@ -1382,6 +1433,10 @@ def format_structured_alert(subject, parsed):
     if "anomalyId" in parsed or "rootCauses" in parsed:
         return format_cost_anomaly(parsed)
 
+    # ECS service crash loop (EventBridge "ECS Service Action")
+    if parsed.get("detail-type") == "ECS Service Action":
+        return format_ecs_service_impaired(parsed)
+
     # EventBridge / CloudTrail events
     if "detail-type" in parsed:
         detail = parsed.get("detail", {})

terraform/lambda-src/tests/test_slack_alert_ecs.py

@@ -0,0 +1,94 @@
+"""Tests for slack_alert ECS start-impaired alerts — run: python3 -m unittest discover -s terraform/lambda-src/tests"""
+
+import importlib.util
+import os
+import sys
+import types
+import unittest
+
+LAMBDA_SRC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
+
+
+class FakeTable:
+    def __init__(self):
+        self.items = {}
+
+    def get_item(self, Key):
+        item = self.items.get(Key["finding_key"])
+        return {"Item": item} if item else {}
+
+    def put_item(self, Item):
+        self.items[Item["finding_key"]] = Item
+
+
+def load(table):
+    boto3 = types.ModuleType("boto3")
+    boto3.client = lambda *a, **kw: object()
+    boto3.resource = lambda *a, **kw: types.SimpleNamespace(Table=lambda name: table)
+    botocore = types.ModuleType("botocore")
+    botocore_config = types.ModuleType("botocore.config")
+    botocore_config.Config = lambda **kw: None
+    sys.modules.update({"boto3": boto3, "botocore": botocore, "botocore.config": botocore_config})
+    os.environ.update({
+        "INFRA_WEBHOOK_PARAM": "/x/infra",
+        "COST_WEBHOOK_PARAM": "/x/cost",
+        "SECURITY_TOPIC_ARN": "arn:aws:sns:eu-central-1:111111111111:javabin-security",
+    })
+    if LAMBDA_SRC not in sys.path:
+        sys.path.insert(0, LAMBDA_SRC)
+    for name in [m for m in sys.modules if m == "shared" or m.startswith("shared.")]:
+        del sys.modules[name]
+    spec = importlib.util.spec_from_file_location(
+        "slack_alert_handler", os.path.join(LAMBDA_SRC, "slack_alert", "handler.py"))
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+
+def event(service="testteam-platform-test-app", name="SERVICE_TASK_START_IMPAIRED",
+          time="2026-10-24T10:00:00Z"):
+    return {
+        "version": "0",
+        "detail-type": "ECS Service Action",
+        "source": "aws.ecs",
+        "account": "111111111111",
+        "time": time,
+        "region": "eu-central-1",
+        "resources": [f"arn:aws:ecs:eu-central-1:111111111111:service/javabin-platform/{service}"],
+        "detail": {
+            "eventType": "WARN",
+            "eventName": name,
+            "clusterArn": "arn:aws:ecs:eu-central-1:111111111111:cluster/javabin-platform",
+        },
+    }
+
+
+class EcsStartImpairedTests(unittest.TestCase):
+    def setUp(self):
+        self.table = FakeTable()
+        self.mod = load(self.table)
+
+    def test_formats_service_and_console_link(self):
+        msg = self.mod.format_structured_alert("subject", event())
+        self.assertEqual(msg["text"], "ECS service testteam-platform-test-app cannot start tasks")
+        rendered = str(msg["blocks"])
+        self.assertIn("`javabin-platform`", rendered)
+        self.assertIn(
+            "https://eu-central-1.console.aws.amazon.com/ecs/v2/clusters/javabin-platform"
+            "/services/testteam-platform-test-app/events?region=eu-central-1",
+            rendered,
+        )
+
+    def test_alerts_once_per_service_per_day(self):
+        self.assertIsNotNone(self.mod.format_structured_alert("s", event()))
+        self.assertIsNone(self.mod.format_structured_alert("s", event(time="2026-10-24T18:00:00Z")))
+        self.assertIsNotNone(self.mod.format_structured_alert("s", event(service="other-app")))
+        self.assertIsNotNone(self.mod.format_structured_alert("s", event(time="2026-10-25T08:00:00Z")))
+
+    def test_ignores_other_ecs_service_actions(self):
+        self.assertIsNone(self.mod.format_structured_alert("s", event(name="SERVICE_STEADY_STATE")))
+        self.assertEqual(self.table.items, {})
+
+
+if __name__ == "__main__":
+    unittest.main()

terraform/platform/monitoring/main.tf

@@ -502,6 +502,35 @@ resource "aws_dynamodb_table" "alert_dedup" {
   }
 }
 
+################################################################################
+# ECS services that cannot start tasks → javabin-security (#javabin-infra-alerts)
+#
+# A service stuck failing task starts (missing image, failing health check)
+# crash-loops until someone notices: wasted Fargate starts, image pulls and
+# AWS Config items for every ENI. slack-alert posts once per service per day.
+# Scoped to the platform cluster (compute/: "${var.project}-platform").
+################################################################################
+
+resource "aws_cloudwatch_event_rule" "ecs_task_start_impaired" {
+  name        = "${var.project}-ecs-task-start-impaired"
+  description = "ECS services in the platform cluster that cannot start tasks"
+
+  event_pattern = jsonencode({
+    source      = ["aws.ecs"]
+    detail-type = ["ECS Service Action"]
+    detail = {
+      eventName  = ["SERVICE_TASK_START_IMPAIRED"]
+      clusterArn = ["arn:aws:ecs:${var.region}:${var.aws_account_id}:cluster/${var.project}-platform"]
+    }
+  })
+}
+
+resource "aws_cloudwatch_event_target" "ecs_task_start_impaired_sns" {
+  rule      = aws_cloudwatch_event_rule.ecs_task_start_impaired.name
+  target_id = "send-to-security-sns"
+  arn       = aws_sns_topic.security.arn
+}
+
 ################################################################################
 # Cost Allocation Tags
 ################################################################################