From 2c15cdb1ac8458145c6526b94fd59011775f6608 Mon Sep 17 00:00:00 2001 From: Alexanderamiri Date: Sat, 24 Oct 2026 15:10:49 +0200 Subject: [PATCH 1/2] feat(monitoring): alert on ECS services that cannot start tasks EventBridge rule on SERVICE_TASK_START_IMPAIRED for the platform cluster, routed through javabin-security to #javabin-infra-alerts. slack-alert formats it and dedups once per service per UTC day in the alert-dedup table. A crash-looping service otherwise burns Fargate starts, image pulls and AWS Config items unnoticed (testteam-platform-test-app, ~80 attempts/day). Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 1 + docs/lambda-functions.md | 2 + terraform/lambda-src/slack_alert/handler.py | 55 +++++++++++ .../lambda-src/tests/test_slack_alert_ecs.py | 94 +++++++++++++++++++ terraform/platform/monitoring/main.tf | 29 ++++++ 5 files changed, 181 insertions(+) create mode 100644 terraform/lambda-src/tests/test_slack_alert_ecs.py diff --git a/CLAUDE.md b/CLAUDE.md index ab9f499..0bc22d7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -221,6 +221,7 @@ EventBridge ──► javabin-security SNS ──► slack-alert Lambda: Security Hub findings (NEW only) ──► #platform-security-alerts GuardDuty findings ──► #platform-security-alerts IAM / resource / login events ──► #javabin-infra-alerts + ECS service can't start tasks ──► #javabin-infra-alerts (once per service per day) Cost Anomaly ──► javabin-alerts SNS ──► slack-alert Lambda ──► #javabin-cost-alerts Scheduled: diff --git a/docs/lambda-functions.md b/docs/lambda-functions.md index 3ee17e9..8a561f4 100644 --- a/docs/lambda-functions.md +++ b/docs/lambda-functions.md @@ -17,6 +17,8 @@ Terraform in `terraform/platform/lambdas/`. Features: GitHub OIDC attribution (extracts actor/repo/SHA from session tags), cost estimation via shared pricing module, Security Hub finding formatting. +ECS `SERVICE_TASK_START_IMPAIRED` events from the platform cluster (a crash-looping service) are posted to #javabin-infra-alerts at most once per service per UTC day. + Uses the `javabin-alert-dedup` DynamoDB table for deduplication of alerts (prevents repeated notifications for the same finding). ## cost-report diff --git a/terraform/lambda-src/slack_alert/handler.py b/terraform/lambda-src/slack_alert/handler.py index 87d3df8..53fb0c9 100644 --- a/terraform/lambda-src/slack_alert/handler.py +++ b/terraform/lambda-src/slack_alert/handler.py @@ -1172,6 +1172,57 @@ def format_budget_alert(parsed): return {"blocks": blocks, "text": fallback} +def format_ecs_service_impaired(parsed): + """ECS service cannot start tasks (SERVICE_TASK_START_IMPAIRED) — Block Kit. + + ECS keeps emitting this event while a service crash-loops, so alert at most + once per service per UTC day (persistent DynamoDB dedup). A crash loop + burns Fargate starts, image pulls and AWS Config items until someone acts. + """ + detail = parsed.get("detail", {}) + if detail.get("eventName") != "SERVICE_TASK_START_IMPAIRED": + return None + + resources = parsed.get("resources") or [] + service_arn = resources[0] if resources else "unknown" + service = service_arn.rsplit("/", 1)[-1] + cluster = detail.get("clusterArn", "").rsplit("/", 1)[-1] or "unknown" + region = parsed.get("region", "") + + day = (parsed.get("time") or datetime.now(timezone.utc).isoformat())[:10] + dedup_key = f"ecs-start-impaired:{service_arn}:{day}" + if is_finding_already_alerted(dedup_key): + logger.info("Suppressed repeat ECS start-impaired alert: %s", dedup_key) + return None + record_finding_alert(dedup_key) + + events_url = (f"https://{region}.console.aws.amazon.com/ecs/v2/clusters/{cluster}" + f"/services/{service}/events?region={region}") + + blocks = [ + {"type": "header", + "text": {"type": "plain_text", "text": ":warning: ECS service cannot start tasks", + "emoji": True}}, + {"type": "section", + "text": {"type": "mrkdwn", + "text": (f"*{service}* keeps failing to start tasks and ECS is backing off. " + "Usual causes: missing image, failing health check, bad secret. " + "Fix the deploy or scale the service to 0.")}}, + {"type": "section", "fields": [ + {"type": "mrkdwn", "text": f"*Service*\n`{service}`"}, + {"type": "mrkdwn", "text": f"*Cluster*\n`{cluster}`"}, + ]}, + {"type": "section", + "text": {"type": "mrkdwn", "text": f"<{events_url}|Service events in the ECS console>"}}, + {"type": "divider"}, + {"type": "context", "elements": [ + {"type": "mrkdwn", "text": "Source: ECS Service Action (alerts once per service per day)"} + ]}, + ] + + return {"blocks": blocks, "text": f"ECS service {service} cannot start tasks"} + + # --------------------------------------------------------------------------- # Weekly Security Hub summary # --------------------------------------------------------------------------- @@ -1382,6 +1433,10 @@ def format_structured_alert(subject, parsed): if "anomalyId" in parsed or "rootCauses" in parsed: return format_cost_anomaly(parsed) + # ECS service crash loop (EventBridge "ECS Service Action") + if parsed.get("detail-type") == "ECS Service Action": + return format_ecs_service_impaired(parsed) + # EventBridge / CloudTrail events if "detail-type" in parsed: detail = parsed.get("detail", {}) diff --git a/terraform/lambda-src/tests/test_slack_alert_ecs.py b/terraform/lambda-src/tests/test_slack_alert_ecs.py new file mode 100644 index 0000000..8f900b8 --- /dev/null +++ b/terraform/lambda-src/tests/test_slack_alert_ecs.py @@ -0,0 +1,94 @@ +"""Tests for slack_alert ECS start-impaired alerts — run: python3 -m unittest discover -s terraform/lambda-src/tests""" + +import importlib.util +import os +import sys +import types +import unittest + +LAMBDA_SRC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") + + +class FakeTable: + def __init__(self): + self.items = {} + + def get_item(self, Key): + item = self.items.get(Key["finding_key"]) + return {"Item": item} if item else {} + + def put_item(self, Item): + self.items[Item["finding_key"]] = Item + + +def load(table): + boto3 = types.ModuleType("boto3") + boto3.client = lambda *a, **kw: object() + boto3.resource = lambda *a, **kw: types.SimpleNamespace(Table=lambda name: table) + botocore = types.ModuleType("botocore") + botocore_config = types.ModuleType("botocore.config") + botocore_config.Config = lambda **kw: None + sys.modules.update({"boto3": boto3, "botocore": botocore, "botocore.config": botocore_config}) + os.environ.update({ + "INFRA_WEBHOOK_PARAM": "/x/infra", + "COST_WEBHOOK_PARAM": "/x/cost", + "SECURITY_TOPIC_ARN": "arn:aws:sns:eu-central-1:111111111111:javabin-security", + }) + if LAMBDA_SRC not in sys.path: + sys.path.insert(0, LAMBDA_SRC) + for name in [m for m in sys.modules if m == "shared" or m.startswith("shared.")]: + del sys.modules[name] + spec = importlib.util.spec_from_file_location( + "slack_alert_handler", os.path.join(LAMBDA_SRC, "slack_alert", "handler.py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def event(service="testteam-platform-test-app", name="SERVICE_TASK_START_IMPAIRED", + time="2026-10-24T10:00:00Z"): + return { + "version": "0", + "detail-type": "ECS Service Action", + "source": "aws.ecs", + "account": "111111111111", + "time": time, + "region": "eu-central-1", + "resources": [f"arn:aws:ecs:eu-central-1:111111111111:service/javabin-platform/{service}"], + "detail": { + "eventType": "WARN", + "eventName": name, + "clusterArn": "arn:aws:ecs:eu-central-1:111111111111:cluster/javabin-platform", + }, + } + + +class EcsStartImpairedTests(unittest.TestCase): + def setUp(self): + self.table = FakeTable() + self.mod = load(self.table) + + def test_formats_service_and_console_link(self): + msg = self.mod.format_structured_alert("subject", event()) + self.assertEqual(msg["text"], "ECS service testteam-platform-test-app cannot start tasks") + rendered = str(msg["blocks"]) + self.assertIn("`javabin-platform`", rendered) + self.assertIn( + "https://eu-central-1.console.aws.amazon.com/ecs/v2/clusters/javabin-platform" + "/services/testteam-platform-test-app/events?region=eu-central-1", + rendered, + ) + + def test_alerts_once_per_service_per_day(self): + self.assertIsNotNone(self.mod.format_structured_alert("s", event())) + self.assertIsNone(self.mod.format_structured_alert("s", event(time="2026-10-24T18:00:00Z"))) + self.assertIsNotNone(self.mod.format_structured_alert("s", event(service="other-app"))) + self.assertIsNotNone(self.mod.format_structured_alert("s", event(time="2026-10-25T08:00:00Z"))) + + def test_ignores_other_ecs_service_actions(self): + self.assertIsNone(self.mod.format_structured_alert("s", event(name="SERVICE_STEADY_STATE"))) + self.assertEqual(self.table.items, {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/terraform/platform/monitoring/main.tf b/terraform/platform/monitoring/main.tf index 4b527f8..c8efa7e 100644 --- a/terraform/platform/monitoring/main.tf +++ b/terraform/platform/monitoring/main.tf @@ -502,6 +502,35 @@ resource "aws_dynamodb_table" "alert_dedup" { } } +################################################################################ +# ECS services that cannot start tasks → javabin-security (#javabin-infra-alerts) +# +# A service stuck failing task starts (missing image, failing health check) +# crash-loops until someone notices: wasted Fargate starts, image pulls and +# AWS Config items for every ENI. slack-alert posts once per service per day. +# Scoped to the platform cluster (compute/: "${var.project}-platform"). +################################################################################ + +resource "aws_cloudwatch_event_rule" "ecs_task_start_impaired" { + name = "${var.project}-ecs-task-start-impaired" + description = "ECS services in the platform cluster that cannot start tasks" + + event_pattern = jsonencode({ + source = ["aws.ecs"] + detail-type = ["ECS Service Action"] + detail = { + eventName = ["SERVICE_TASK_START_IMPAIRED"] + clusterArn = ["arn:aws:ecs:${var.region}:${var.aws_account_id}:cluster/${var.project}-platform"] + } + }) +} + +resource "aws_cloudwatch_event_target" "ecs_task_start_impaired_sns" { + rule = aws_cloudwatch_event_rule.ecs_task_start_impaired.name + target_id = "send-to-security-sns" + arn = aws_sns_topic.security.arn +} + ################################################################################ # Cost Allocation Tags ################################################################################ From 02d15072c1d817132a46c0abf366b1d883491016 Mon Sep 17 00:00:00 2001 From: Alexanderamiri Date: Sat, 24 Oct 2026 15:41:37 +0200 Subject: [PATCH 2/2] docs(monitoring): state ECS impaired-alert scope and dedup behaviour Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/lambda-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/lambda-functions.md b/docs/lambda-functions.md index 8a561f4..a6e20fb 100644 --- a/docs/lambda-functions.md +++ b/docs/lambda-functions.md @@ -17,7 +17,7 @@ Terraform in `terraform/platform/lambdas/`. Features: GitHub OIDC attribution (extracts actor/repo/SHA from session tags), cost estimation via shared pricing module, Security Hub finding formatting. -ECS `SERVICE_TASK_START_IMPAIRED` events from the platform cluster (a crash-looping service) are posted to #javabin-infra-alerts at most once per service per UTC day. +ECS `SERVICE_TASK_START_IMPAIRED` events from the `javabin-platform` cluster only (a crash-looping service) are posted to #javabin-infra-alerts at most once per service per UTC day. Legacy clusters (vaultwarden, cakeredux, moresleep-test, dennis-test) are not covered. As with the other deduplicated alerts, the dedup key is written before the Slack post, so a failed post suppresses that service's alert for the rest of the day; if the dedup table is unreadable, dedup fails open. Uses the `javabin-alert-dedup` DynamoDB table for deduplication of alerts (prevents repeated notifications for the same finding).