From f2c6127015e6540f253ad64e6cb3cf6a56a4f7ef Mon Sep 17 00:00:00 2001 From: Alexanderamiri Date: Mon, 26 Oct 2026 15:02:50 +0100 Subject: [PATCH] feat(platform): app-to-app calls with private networking and scopes app.yaml calls/provides declare private app-to-app edges, optionally with OAuth scopes. Every app gets its own task security group and a Cloud Map name in the new javabin.internal namespace; app-peers-sync opens ingress only where both sides agree and manages the M2M resource servers and clients. The developer boundary protects the shared namespace and the private zone's records (6,067 of 6,144 bytes). Rebased onto routing.login (#158), RDS/DNS adoption (#155), Lambda log retention (#159, adds the app-peers-sync log group) and the app-auth-sync restart rules (#167). Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 11 +- docs/app-yaml-reference.md | 116 +++ docs/lambda-functions.md | 19 +- docs/platform-modules.md | 5 +- docs/reusable-modules.md | 18 + scripts/generate-modules.py | 184 ++++- scripts/registry.py | 1 + scripts/tests/test_generate_modules.py | 112 ++- .../lambda-src/app_peers_sync/handler.py | 775 ++++++++++++++++++ .../lambda-src/app_secret_rollout/handler.py | 7 +- .../lambda-src/tests/test_app_peers_sync.py | 513 ++++++++++++ .../tests/test_app_secret_rollout.py | 10 +- terraform/modules/app-network/main.tf | 140 ++++ terraform/modules/app-network/outputs.tf | 64 ++ terraform/modules/app-network/variables.tf | 62 ++ terraform/modules/ecs-service/main.tf | 8 + terraform/modules/ecs-service/variables.tf | 11 + terraform/platform/iam/boundary.tf | 32 +- terraform/platform/lambdas/log-groups.tf | 1 + terraform/platform/lambdas/main.tf | 241 ++++++ terraform/platform/lambdas/variables.tf | 22 + terraform/platform/main.tf | 6 +- terraform/platform/networking/main.tf | 22 + terraform/platform/networking/outputs.tf | 25 + 24 files changed, 2384 insertions(+), 21 deletions(-) create mode 100644 terraform/lambda-src/app_peers_sync/handler.py create mode 100644 terraform/lambda-src/tests/test_app_peers_sync.py create mode 100644 terraform/modules/app-network/main.tf create mode 100644 terraform/modules/app-network/outputs.tf create mode 100644 terraform/modules/app-network/variables.tf diff --git a/CLAUDE.md b/CLAUDE.md index 411ea5b..405a1e1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -92,12 +92,12 @@ terraform/platform/ variables.tf Root variables outputs.tf Exported values backend.tf S3 + DynamoDB state backend - networking/ VPC, subnets, NAT, SGs + networking/ VPC, subnets, NAT, SGs, javabin.internal Cloud Map namespace ingress/ ALB, ACM, Route53 iam/ GitHub OIDC, CI roles, permission boundary compute/ ECS cluster, ECR base config monitoring/ SNS, EventBridge, Config, GuardDuty, Security Hub - lambdas/ slack-alert, cost-report, daily-cost-check, compliance-reporter, resource-tagger, budget-enforcer, override-cleanup, team-provisioner, apply-gate, securityhub-summary, password-set, ci-broker, app-auth-sync, plan-review, app-secret-rollout + lambdas/ slack-alert, cost-report, daily-cost-check, compliance-reporter, resource-tagger, budget-enforcer, override-cleanup, team-provisioner, apply-gate, securityhub-summary, password-set, ci-broker, app-auth-sync, plan-review, app-secret-rollout, app-peers-sync identity/ Cognito user pools (internal + external), auth-claims trigger, auth memberships table. Identity Center is in terraform/org/ ``` @@ -137,6 +137,7 @@ terraform/state/ | `terraform/modules/service-queue/` | SQS queue + DLQ with IAM policy output | | `terraform/modules/service-alarm/` | CloudWatch alarms for ECS service | | `terraform/modules/cognito-app-client/` | Requests a Cognito app client via SSM; the `app-auth-sync` Lambda creates it | +| `terraform/modules/app-network/` | Per-app task SG, Cloud Map name `{team}-{app}.javabin.internal`, app-to-app request via SSM; the `app-peers-sync` Lambda wires the edges | ### GitHub Actions Workflows ``` @@ -175,6 +176,7 @@ terraform/state/ | `app-auth-sync` | Creates/deletes Cognito app clients requested by app.yaml `auth:` and `routing.login` (SSM request → client + credentials + allowed groups) | | `auth-claims` | Pre-token trigger: registry groups → `cognito:groups` (internal pool only), enforces per-client allowed groups; runs as `auth-claims-external` on the external pool (in `identity/`) | | `plan-review` | LLM review of app Terraform plans with platform credentials; writes `risk.json` (invoked by app tf-plan) | +| `app-peers-sync` | App-to-app calls from app.yaml `calls:`/`provides:` (SSM `network` request → SG rules, Cognito M2M clients + scopes, caller's peer URL/credential params; daily ownership audits) | | `app-secret-rollout` | Restarts `{team}-{app}` when one of its secrets in `/javabin/apps/` changes (ECS reads secrets only at task start) | ### Scripts @@ -265,6 +267,9 @@ All parameters are in `eu-central-1`. Use `--profile javabin --region eu-central | `/javabin/apps/{team}/{app}/cognito-client-{id,secret,identity-provider}` | String / SecureString / String | Written by app-auth-sync (only on change; a change restarts the app), read by ECS execution role | | `/javabin/apps/{team}/{app}/auth-status` | String (JSON) | Written by app-auth-sync, read by the team and `scripts/auth-doctor.py` | | `/javabin/platform/google-oauth/internal/{client-id,client-secret}` | String / SecureString | Internal pool's Google OIDC IdP (identity TF; IdP is created only when both exist) | +| `/javabin/apps/{team}/{app}/network` | String | app.yaml `calls:`/`provides:` request (written by app TF, read by app-peers-sync) | +| `/javabin/apps/{team}/{app}/peer-{t}-{a}-{url,scopes}`, `m2m-client-{id,secret}` | String / SecureString | Written by app-peers-sync, read by ECS execution role | +| `/javabin/apps/{team}/{app}/peers-status` | String | Written by app-peers-sync: state of the app's calls and consumers | ## GCP Connection @@ -319,7 +324,7 @@ Team group OWNERs are admin-managed and never removed by the sync; non-@java.no - **ECS deploy stabilization**: platform-test-app task registers but service fails health check - **Cognito**: pools deployed; internal pool's `GoogleSAML` IdP was created outside Terraform. It is replaced by the Terraform-managed Google OIDC IdP at cutover (docs/cognito-google-setup.md), then deleted - **SSO sessions carry no `team` tag**: the Identity Center ABAC attribute (`${path:enterprise.team}`) is never set on users, so `${aws:PrincipalTag/team}` is empty for humans. Per-team permission sets (`javabin-team-{team}`) therefore use `javabin-team-{team}-guardrails` (the developer boundary rendered with the team name) instead. The ABAC-based `javabin-developer` permission set in `terraform/org` cannot reach team resources -- **App databases share one network path**: service-rds allows ingress from the shared ECS tasks SG, so every team's tasks can reach every team's DB port (password still required). Fix: per-app security groups +- **App databases share one network path** (fixed by `app-network`): service-rds now allows ingress only from the app's own `{team}-{app}-tasks` SG; each app moves off the shared ECS tasks SG on its next pipeline run - **Team provisioner Lambda**: All sync functions working (Google/GitHub/Budget/Cognito/Identity Center). Password-set flow deployed. - **`registered_app_repos` manually managed**: Being replaced with team-scoped IAM roles (repo→team resolved via GitHub API at runtime) - **Cost allocation tags**: `team`, `service`, `environment`, `managed-by`, `repo` are active (Terraform). `created-by` and `commit` stay inactive on purpose (no cost signal) diff --git a/docs/app-yaml-reference.md b/docs/app-yaml-reference.md index 3b2b010..c76b684 100644 --- a/docs/app-yaml-reference.md +++ b/docs/app-yaml-reference.md @@ -469,6 +469,118 @@ alarms: Alarms are created only for services with `routing:`. +### calls / provides (app-to-app) + +Apps can call each other privately inside the platform VPC, without going through the internet or +the public ALB. Both sides must agree: the **caller** lists the app it calls under `calls:`, and the +**callee** lists the caller under `provides.consumers:`. Either side removing its entry closes the +connection again. + +```yaml +# Caller: cupcake-backend (team cupcake) +calls: + - app: moresleep # required + team: moresleep # optional, default: this app's team + scopes: [talks.read] # optional; omit for network-only + env: SP_BASE_URL # optional URL variable name, default MORESLEEP_URL +``` + +```yaml +# Callee: moresleep (team moresleep) +provides: + scopes: # optional; scope: description (a list of names also works) + talks.read: Read talks and speakers + talks.write: Create and update talks + consumers: # who may call this app + - app: cupcake-backend + team: cupcake # optional, default: this app's team + scopes: [talks.read] # optional; must be listed in provides.scopes +``` + +**What the platform does** + +- Every app (with or without `calls`/`provides`) gets its own security group `{team}-{name}-tasks` + and a private DNS name `{team}-{name}.javabin.internal` (Cloud Map). The ALB can reach only the + app's `compute.port`; nothing else can, until a consumer is allowed. +- When both sides agree, the platform Lambda `javabin-app-peers-sync` opens the callee's port to the + caller's security group, usually within a minute of the second apply. +- **Scopes** (optional) are OAuth2 client-credentials scopes in the internal Cognito pool. The callee + gets a resource server `{team}-{name}`; each caller that requests scopes gets one machine-to-machine + client. Granted scopes are what the caller requested **and** the callee allows for that caller. +- Routing is not needed on either side. An app with `provides:` and no `routing:` is reachable only + from its consumers inside the VPC. + +**Injected into the caller** + +| Variable | Kind | Value | +|---|---|---| +| `` (default `{APP}_URL`) | secret (SSM) | `http://{team}-{app}.javabin.internal:{port}` | +| `` with `_URL` → `_SCOPES` (only with `scopes`) | secret (SSM) | Scopes actually **granted** for that callee, space-separated, e.g. `moresleep-moresleep/talks.read`; `none` until something is granted | +| `PEER_AUTH_TOKEN_URL` (any call with `scopes`) | env | Token endpoint of the internal pool | +| `PEER_AUTH_CLIENT_ID`, `PEER_AUTH_CLIENT_SECRET` (any call with `scopes`) | secrets | The caller's M2M client | + +**Injected into the callee** (when `provides.scopes` is set): `PEER_AUTH_ISSUER`, +`PEER_AUTH_JWKS_URL`, `PEER_AUTH_SCOPE_PREFIX` (`{team}-{name}/`). + +**Calling with scopes.** Request one token per callee, with that callee's scopes variable, and cache it +until it expires (60 minutes). Skip the request while the variable is `none`: + +```bash +curl -s -u "$PEER_AUTH_CLIENT_ID:$PEER_AUTH_CLIENT_SECRET" \ + -d grant_type=client_credentials --data-urlencode "scope=$SP_BASE_SCOPES" "$PEER_AUTH_TOKEN_URL" +# -> {"access_token": "...", "expires_in": 3600}; send it as Authorization: Bearer +``` + +Cognito refuses the whole token request (`invalid_scope`) if any requested scope is not granted, so +always request exactly the scopes variable, never a hard-coded list: it holds what the client really has. +When the callee grants fewer scopes (or more), the variable changes and the platform restarts the caller +so it picks up the new value. The callee +verifies the token's signature against `PEER_AUTH_JWKS_URL`, its `iss` against `PEER_AUTH_ISSUER`, and +checks that `scope` contains `${PEER_AUTH_SCOPE_PREFIX}` (Spring Security +`oauth2ResourceServer().jwt()`, `jose` in Node). One token per callee means a token can never be +replayed against another app. + +**Status.** Each side can see the state of its edges: + +```bash +aws ssm get-parameter --name /javabin/apps///peers-status --query Parameter.Value --output text +# calls: active | pending-consent | callee-missing | callee-invalid, with granted/denied scopes +# consumers: active | pending | caller-invalid +``` + +If an app's request becomes invalid (its `peers-status` shows `{"error": ...}`), the app is frozen rather +than removed: its existing rules, scopes and parameters stay, so a bad apply does not cut off its peers. +The other side can still withdraw while it is frozen. Removing the app (destroying its Terraform) removes +its edges for real. + +**Security notes** + +- A network-only call (no `scopes`) is **not caller authentication**. It only means "tasks with the + caller's security group can reach the port". ECS attaches security groups and registers Cloud Map + instances through its own service role, so a team that deliberately runs a task with another app's + group ID would get that app's network identity. The daily audit reports this, but does not block it. + Endpoints that change data or return anything sensitive should require scopes. +- The platform denies teams changing the private `javabin.internal` DNS zone, and the daily audit + compares its records with Cloud Map. Cloud Map service names are not enforced at creation: a squatted + `{your-team}-{app}` name makes your apply fail and is reported by the daily audit; ask the platform + team to remove it. + +**First deploy.** The caller's task definition references `peer-*-url`, `peer-*-scopes` and +`m2m-client-*`, which `javabin-app-peers-sync` writes shortly after the apply that adds the call. If the +Lambda has not written them (or failed), tasks fail to start with `ResourceInitializationError` and the +deployment rolls back. Check `peers-status` and the `#javabin-infra-alerts` channel, then re-run the +deploy. + +**Rules and limits** + +- Names: `app` as in `name:`, `team` as in `team:`. Scope names: lowercase letters, digits, `.`, `_`, + `-`, max 40 characters. At most 20 calls, 50 consumers, 25 scopes. +- The injected names must not also be set in `environment:`, a resource `env` or `auth.env`. Two calls + to apps with the same name in different teams need an explicit `env:`. +- Traffic is plain HTTP inside the VPC. Already-issued tokens stay valid for up to 60 minutes after a + scope is withdrawn, and open connections can outlive a removed rule until they close. +- At most about 25 apps can provide scopes (Cognito resource servers per pool). + ## Not supported These fields were documented earlier but are not implemented. Using them fails `expand` with an explanation. @@ -507,3 +619,7 @@ All app resources are prefixed with the team name for ABAC enforcement. | DNS record | `{routing.host}` | | SSM (Cognito) | `/javabin/apps/{team}/{name}/cognito-client-{id,secret}` | | Cognito client (`routing.login`) | `{team}-{name}.login-{N}`, credentials under `/javabin/apps/{team}/{name}/login-{N}/` | +| Task security group | `{team}-{name}-tasks` | +| Private DNS name | `{team}-{name}.javabin.internal` | +| SSM (app-to-app) | `/javabin/apps/{team}/{name}/network` (request), `peer-{t}-{a}-url`, `peer-{t}-{a}-scopes`, `m2m-client-{id,secret}`, `peers-status` | +| Cognito M2M | resource server `{team}-{name}`, client `m2m.{team}-{name}` (internal pool) | diff --git a/docs/lambda-functions.md b/docs/lambda-functions.md index de22850..d9bf4fc 100644 --- a/docs/lambda-functions.md +++ b/docs/lambda-functions.md @@ -187,6 +187,23 @@ Idempotent: skips resources that already have a `created-by` tag (preserves orig **Manual run:** `aws lambda invoke --function-name javabin-app-auth-sync --payload '{"action":"reconcile"}' --cli-binary-format raw-in-base64-out /dev/stdout` +## app-peers-sync + +**Trigger:** EventBridge "Parameter Store Change" for `/javabin/apps/*/*/network` (Create/Update/Delete), plus a daily `{"action": "reconcile"}` at 03:45 UTC +**Purpose:** App-to-app calls declared in `app.yaml` (`calls:` / `provides:`). The request's path identifies the app (only its team can write there); the security group in the request must be that app's own (`{team}-{app}-tasks`, tag `team={team}`), so a team cannot make the Lambda open another team's group. + +**Flow:** Every run recomputes everything from all requests (event payloads are ignored) → an edge exists when the caller `calls` the callee and the callee lists the caller in `consumers` → ingress rule on the callee's SG from the caller's SG on the callee port, tagged `managed-by=javabin-app-peers-sync` (rules the team added itself are never touched) → Cognito resource server `{t}-{a}` per callee with scopes and client-credentials client `m2m.{t}-{a}` per caller requesting scopes (internal pool, 60 min tokens, granted = requested ∩ allowed; no grants = OAuth disabled) → writes `peer-{t}-{a}-url`, `peer-{t}-{a}-scopes` (the scopes the caller's client really holds for that callee, or `none`), `m2m-client-id`, `m2m-client-secret` (SecureString) and `peers-status` into the caller's path, only when a value changed. Parameters, clients and servers nobody needs are deleted. An app whose request is invalid is frozen, not removed: its parameters, managed rules, resource server and client scopes stay, as long as every valid side of an edge still declares it (the valid side can still withdraw); it gets `{"error": ...}` in `peers-status`, and its peers see `callee-invalid` / `caller-invalid`. + +**Audits (reconcile only, Slack `#javabin-infra-alerts`, never fixed automatically):** Cloud Map services whose name does not match their `team` tag or that have instances registered by another ECS service; records in the namespace's Route53 private zone that are not A records Cloud Map wrote for a registered instance (teams are also denied changing the zone in the developer boundary); task ENIs carrying `{t}-{a}-tasks` that belong to a different ECS service. + +**Concurrency:** reserved concurrency 1. + +**Manual run:** `aws lambda invoke --function-name javabin-app-peers-sync --payload '{"action":"reconcile"}' --cli-binary-format raw-in-base64-out /dev/stdout` + +| SSM Parameter | Use | +|---------------|-----| +| `/javabin/slack/platform-resource-alerts-webhook` | Audit findings and errors | + ## auth-claims **Trigger:** Cognito pre-token-generation (V2) on the internal pool (`javabin-internal`), every sign-in and token refresh @@ -218,7 +235,7 @@ Team roles are denied writing `risk.json`, `review.json` and `override.json` in ## app-secret-rollout **Trigger:** EventBridge "Parameter Store Change" (Update) under `/javabin/apps/` -**Purpose:** ECS resolves task secrets only at start, so after a team sets a secret value (`javabin secret set`) this forces a new deployment of `{team}-{app}` in `javabin-platform`. Teams have no `ecs:UpdateService` themselves (it would also allow changing networking and task definitions). Cognito client parameters (`cognito-client-id`, `-secret`, `-identity-provider`) also restart the app: app-auth-sync writes them only when the value changes (replaced client, IdP cutover), so running tasks never keep a deleted client ID and a no-op reconcile restarts nothing. Ignored: `auth`, `auth-status` (rewritten every sync), `db-master-password`. A missing service is logged and skipped. +**Purpose:** ECS resolves task secrets only at start, so after a team sets a secret value (`javabin secret set`) this forces a new deployment of `{team}-{app}` in `javabin-platform`. Teams have no `ecs:UpdateService` themselves (it would also allow changing networking and task definitions). Cognito client parameters (`cognito-client-id`, `-secret`, `-identity-provider`) also restart the app: app-auth-sync writes them only when the value changes (replaced client, IdP cutover), so running tasks never keep a deleted client ID and a no-op reconcile restarts nothing. Ignored: `auth`, `auth-status` (rewritten every sync), `db-master-password`, `network`, `peers-status`. Peer URLs and scopes and `m2m-client-*` (app-peers-sync) do restart the caller. A missing service is logged and skipped. ## Shared Module: pricing diff --git a/docs/platform-modules.md b/docs/platform-modules.md index b0e4999..4b958ea 100644 --- a/docs/platform-modules.md +++ b/docs/platform-modules.md @@ -23,9 +23,10 @@ VPC, subnets, NAT gateway, security groups. | Public subnets | 3 AZs, tagged `tier=public` | | Private subnets | 3 AZs, tagged `tier=private` | | NAT Gateway | Single NAT in first public subnet | -| SGs | `javabin-alb-sg` (80/443 inbound), `javabin-ecs-tasks-sg` (ALB-only inbound) | +| SGs | `javabin-alb-sg` (80/443 inbound), `javabin-ecs-tasks-sg` (ALB-only inbound; legacy, apps now get their own `{team}-{app}-tasks` group from `app-network`) | +| Cloud Map namespace | `javabin.internal` (private DNS, tagged `team=shared`) — app-to-app names `{team}-{app}.javabin.internal` | -**Outputs:** `vpc_id`, `public_subnet_ids`, `private_subnet_ids`, `alb_sg_id`, `ecs_tasks_sg_id` +**Outputs:** `vpc_id`, `vpc_arn`, `public_subnet_ids`, `private_subnet_ids`, `alb_sg_id`, `ecs_tasks_sg_id`, `service_discovery_namespace_{id,arn,name}` ## ingress diff --git a/docs/reusable-modules.md b/docs/reusable-modules.md index cd30dd0..9c36146 100644 --- a/docs/reusable-modules.md +++ b/docs/reusable-modules.md @@ -68,6 +68,8 @@ Supports `environment` (map) and `secrets` (map of name => ARN) for container co **Tag propagation:** `enable_ecs_managed_tags = true` and `propagate_tags = "SERVICE"` ensure Fargate task-level compute costs are attributed to the team via Cost Explorer. +`service_registry_arns` (list, default `[]`) registers the tasks in Cloud Map (from `app-network`). + **Outputs:** `service_name`, `task_definition_arn`, `log_group_name` ## service-bucket @@ -176,6 +178,22 @@ Requests a Cognito OAuth client for the app. It does **not** create the client: **Inputs:** `app_name`, `team`, `pool` (internal/external), `client_type` (confidential/public/none), `callback_urls`, `logout_urls`, `allowed_oauth_scopes`, `allowed_groups`, `external_pool_custom_domain`, `slot` (optional `login-N`: an extra client `{team}-{app_name}.login-N` with everything under `/{project}/apps/{team}/{app_name}/login-N/`) **Outputs:** `user_pool_id`, `user_pool_domain`, `issuer_url`, `well_known_url`, `client_id_arn`, `client_secret_arn`, `identity_provider_arn` (SSM ARNs for ECS `secrets`), `auth_status_parameter_name`, `client_id_parameter_name`, `client_secret_parameter_name`, `request_parameter_name`, `request_value` (the request JSON; depends on the parameter existing) +## app-network + +Emitted for every app. The app's own task security group `{team}-{name}-tasks` (ALB ingress on the +app port when routed, all egress, tag `javabin-network=app`), its Cloud Map service `{team}-{name}` +in the shared `{project}.internal` namespace, and the app-to-app request +`/{project}/apps/{team}/{name}/network` (security group, port, `calls`, `provides`). The platform +`app-peers-sync` Lambda turns the requests into security group rules, Cognito M2M clients and the +caller's `peer-*-url` / `peer-*-scopes` / `m2m-client-*` parameters. See +[app-yaml-reference.md#calls--provides-app-to-app](app-yaml-reference.md#calls--provides-app-to-app). + +**Inputs:** `name`, `team`, `project`, `vpc_id`, `port`, `alb_ingress`, `calls` (`"team/app"` => scopes), +`provides_scopes` (scope => description), `consumers` (`"team/app"` => scopes), `internal_pool_name`, +`internal_pool_domain` (Cognito domain prefix for `token_url`, default `javabin-internal`) +**Outputs:** `security_group_id`, `registry_arn`, `dns_name`, `request_parameter_name`, `peer_url_arns`, `peer_scopes_arns`, +`m2m_client_id_arn`, `m2m_client_secret_arn`, `token_url`, `issuer_url`, `jwks_url`, `scope_prefix` + ## Policy Composition Pattern Resource modules export `access_policy_json`. The `service-role` module accepts a map of these. `scripts/generate-modules.py` composes all policies from declared resources. diff --git a/scripts/generate-modules.py b/scripts/generate-modules.py index b37a572..e956adb 100755 --- a/scripts/generate-modules.py +++ b/scripts/generate-modules.py @@ -67,8 +67,15 @@ class AppYamlError(ValueError): "auth": {"pool", "client", "callback_urls", "logout_urls", "scopes", "groups", "env", "extra_hosts"}, "environment": None, "alarms": {"enabled", "cpu_threshold", "memory_threshold", "error_5xx_threshold"}, + "calls": None, + "provides": {"scopes", "consumers"}, } +# App-to-app calls (see docs/app-yaml-reference.md#app-to-app-calls) +CALL_SCHEMA = {"app", "team", "scopes", "env"} +CONSUMER_SCHEMA = {"app", "team", "scopes"} +MAX_CALLS, MAX_CONSUMERS, MAX_SCOPES = 20, 50, 25 + RESOURCE_SCHEMA = { "buckets": {"name", "env", "versioning", "expire_days"}, "databases": {"name", "env", "engine", "hash_key", "hash_key_type", "range_key", @@ -111,6 +118,13 @@ class AppYamlError(ValueError): # An adopted database keeps its live shape; these would suggest otherwise. ADOPT_REJECTED = ("instance_class", "allocated_storage", "engine_version", "multi_az") +# Cognito scope names may not contain "/", quotes, backslashes or spaces +SCOPE_RE = re.compile(r"^[a-z][a-z0-9._-]{0,39}$") + +# Injected when the app calls a peer with scopes / provides scopes +PEER_CALLER_AUTH_ENV = ("PEER_AUTH_TOKEN_URL", "PEER_AUTH_CLIENT_ID", "PEER_AUTH_CLIENT_SECRET") +PEER_CALLEE_AUTH_ENV = ("PEER_AUTH_ISSUER", "PEER_AUTH_JWKS_URL", "PEER_AUTH_SCOPE_PREFIX") + def host_zone(host): """The app domain a host lives under, or None if it is not one label under one.""" @@ -316,6 +330,116 @@ def validate(app): if isinstance(v, (dict, list)): raise AppYamlError(f"environment.{k}: values must be strings, numbers or booleans") + parse_peers(app) + + +def _scope_list(section, value, allowed=None): + """A list of unique scope names; each must be in `allowed` when given.""" + if value is None: + return [] + if not isinstance(value, list) or not all(isinstance(s, str) for s in value): + raise AppYamlError(f"{section}: expected a list of scope names") + for s in value: + if not SCOPE_RE.match(s): + raise AppYamlError( + f"{section}: {s!r} is not a valid scope name " + "(lowercase letters, digits, '.', '_' and '-', max 40 chars)" + ) + if allowed is not None and s not in allowed: + raise AppYamlError(f"{section}: {s!r} is not listed in provides.scopes") + if len(set(value)) != len(value): + raise AppYamlError(f"{section}: duplicate scope names") + return list(value) + + +def _peer_ref(section, item, own_team): + """Resolve {app, team?} to (team, app).""" + peer_app, peer_team = item.get("app"), item.get("team", own_team) + if not NAME_RE.match(str(peer_app or "")): + raise AppYamlError(f"{section}.app: {peer_app!r} must be an app name (lowercase letters, digits, hyphens)") + if not TEAM_RE.match(str(peer_team or "")): + raise AppYamlError(f"{section}.team: {peer_team!r} must be a team name (lowercase letters)") + return peer_team, peer_app + + +def peer_env_names(peer_app, env=None): + """(URL variable, scopes variable) for a call.""" + url_env = env or f"{peer_app.upper().replace('-', '_')}_URL" + scopes_env = url_env[:-len("_URL")] + "_SCOPES" if url_env.endswith("_URL") else f"{url_env}_SCOPES" + return url_env, scopes_env + + +def parse_peers(app): + """Validate and normalise `calls:` and `provides:`. + + Returns (calls, provides_scopes, consumers): + calls: [{team, app, key: "team/app", scopes, url_env, scopes_env}] + provides_scopes: {scope: description} + consumers: {"team/app": [scope, ...]} + """ + own = (app.get("team"), app.get("name")) + + calls, seen = [], set() + raw_calls = app.get("calls") + if raw_calls is not None: + if not isinstance(raw_calls, list): + raise AppYamlError("calls: expected a list of {app, team, scopes, env}") + if len(raw_calls) > MAX_CALLS: + raise AppYamlError(f"calls: at most {MAX_CALLS} entries") + for i, item in enumerate(raw_calls): + section = f"calls[{i}]" + _check_keys(section, item, CALL_SCHEMA) + ref = _peer_ref(section, item, own[0]) + if ref == own: + raise AppYamlError(f"{section}: an app cannot call itself") + if ref in seen: + raise AppYamlError(f"{section}: {ref[0]}/{ref[1]} is listed twice") + seen.add(ref) + if item.get("env") is not None and not ENV_RE.match(str(item["env"])): + raise AppYamlError(f"{section}.env: {item['env']!r} is not a valid variable name") + url_env, scopes_env = peer_env_names(ref[1], item.get("env")) + calls.append({ + "team": ref[0], "app": ref[1], "key": f"{ref[0]}/{ref[1]}", + "scopes": _scope_list(f"{section}.scopes", item.get("scopes")), + "url_env": url_env, "scopes_env": scopes_env, + }) + + provides = app.get("provides") + provides_scopes, consumers = {}, {} + if provides is not None: + _check_keys("provides", provides, SCHEMA["provides"]) + raw_scopes = provides.get("scopes") + if isinstance(raw_scopes, list): + raw_scopes = {s: s for s in _scope_list("provides.scopes", raw_scopes)} + if raw_scopes is not None: + if not isinstance(raw_scopes, dict): + raise AppYamlError("provides.scopes: expected a mapping of scope: description (or a list of names)") + if len(raw_scopes) > MAX_SCOPES: + raise AppYamlError(f"provides.scopes: at most {MAX_SCOPES} scopes") + for s, desc in raw_scopes.items(): + _scope_list("provides.scopes", [s]) + if not isinstance(desc, str) or not 1 <= len(desc) <= 256: + raise AppYamlError(f"provides.scopes.{s}: description must be a string of 1-256 characters") + provides_scopes[s] = desc + + raw_consumers = provides.get("consumers") or [] + if not isinstance(raw_consumers, list): + raise AppYamlError("provides.consumers: expected a list of {app, team, scopes}") + if len(raw_consumers) > MAX_CONSUMERS: + raise AppYamlError(f"provides.consumers: at most {MAX_CONSUMERS} entries") + for i, item in enumerate(raw_consumers): + section = f"provides.consumers[{i}]" + _check_keys(section, item, CONSUMER_SCHEMA) + ref = _peer_ref(section, item, own[0]) + if ref == own: + raise AppYamlError(f"{section}: an app cannot consume itself") + key = f"{ref[0]}/{ref[1]}" + if key in consumers: + raise AppYamlError(f"{section}: {key} is listed twice") + consumers[key] = _scope_list(f"{section}.scopes", item.get("scopes"), allowed=provides_scopes) + + return calls, provides_scopes, consumers + # --------------------------------------------------------------------------- # HCL generation helpers @@ -595,6 +719,25 @@ def src(mod): print(f"::warning file=app.yaml::{warning}") blocks.append(module_block("routing", src("service-routing"), routing_attrs)) + # Per-app network: own security group, Cloud Map registration and the + # app-to-app request read by the platform app-peers-sync Lambda (always). + calls, provides_scopes, consumers = parse_peers(app) + network_attrs = { + "name": name, + "team": team, + "project": PROJECT, + "vpc_id": Ref("module.platform.vpc_id"), + "port": compute.get("port", 8000), + "alb_ingress": bool(routing.get("host")), + } + if calls: + network_attrs["calls"] = {c["key"]: c["scopes"] for c in calls} + if provides_scopes: + network_attrs["provides_scopes"] = provides_scopes + if consumers: + network_attrs["consumers"] = consumers + blocks.append(module_block("network", src("app-network"), network_attrs)) + # Collection modules (buckets, databases, secrets, queues) policy_map = {} # name -> module output ref for access_policy_json env_map = {} # env var name -> module output ref @@ -642,7 +785,7 @@ def src(mod): "allocated_storage": db.get("allocated_storage", 20), "subnet_ids": Ref("module.platform.private_subnet_ids"), "vpc_id": Ref("module.platform.vpc_id"), - "allowed_security_group_ids": Ref("[module.platform.ecs_tasks_security_group_id]"), + "allowed_security_group_ids": Ref("[module.network.security_group_id]"), "backup_retention_period": db.get("backup_retention_period", 7), "multi_az": db.get("multi_az", False), "deletion_protection": db.get("deletion_protection", True), @@ -751,6 +894,42 @@ def src(mod): env_map.update(auth_env) secret_map.update(auth_secrets) + # App-to-app env: the peer URL (a parameter, since the callee's port lives + # in another team's state), the scopes actually granted (a parameter too: + # Cognito refuses the whole token if one requested scope is not granted, + # and a change restarts the caller through app-secret-rollout), and M2M creds. + peer_env, peer_secrets = {}, {} + peer_names = [n for c in calls for n in (c["url_env"], c["scopes_env"] if c["scopes"] else None) if n] + for c in calls: + peer_secrets[c["url_env"]] = Ref(f'module.network.peer_url_arns["{c["key"]}"]') + if c["scopes"]: + peer_secrets[c["scopes_env"]] = Ref(f'module.network.peer_scopes_arns["{c["key"]}"]') + if any(c["scopes"] for c in calls): + peer_env["PEER_AUTH_TOKEN_URL"] = Ref("module.network.token_url") + peer_secrets["PEER_AUTH_CLIENT_ID"] = Ref("module.network.m2m_client_id_arn") + peer_secrets["PEER_AUTH_CLIENT_SECRET"] = Ref("module.network.m2m_client_secret_arn") + if provides_scopes: + peer_env["PEER_AUTH_ISSUER"] = Ref("module.network.issuer_url") + peer_env["PEER_AUTH_JWKS_URL"] = Ref("module.network.jwks_url") + peer_env["PEER_AUTH_SCOPE_PREFIX"] = f"{team}-{name}/" + + if any(c["scopes"] for c in calls): + peer_names += PEER_CALLER_AUTH_ENV + if provides_scopes: + peer_names += PEER_CALLEE_AUTH_ENV + if len(set(peer_names)) != len(peer_names): + dup = sorted({n for n in peer_names if peer_names.count(n) > 1}) + raise AppYamlError(f"calls: variable name(s) {', '.join(dup)} used twice — set env: on the calls") + taken = {*(app.get("environment") or {}), *env_map, *secret_map} + clash = sorted(set(peer_names) & taken) + if clash: + raise AppYamlError( + f"calls/provides: {', '.join(clash)} is injected by the platform but also set by " + "environment:, resources or auth — remove it there or set env: on the call" + ) + env_map.update(peer_env) + secret_map.update(peer_secrets) + # Task role (always) blocks.append(module_block("task_role", src("service-role"), { "name": name, @@ -781,7 +960,8 @@ def src(mod): "execution_role_arn": Ref("module.platform.execution_role_arn"), "task_role_arn": Ref("module.task_role.role_arn"), "subnet_ids": Ref("module.platform.private_subnet_ids"), - "security_group_ids": Ref("[module.platform.ecs_tasks_security_group_id]"), + "security_group_ids": Ref("[module.network.security_group_id]"), + "service_registry_arns": Ref("[module.network.registry_arn]"), "region": region, "container_user": str(compute.get("user", "1000")), "readonly_root_filesystem": compute.get("readonly_root_filesystem", True), diff --git a/scripts/registry.py b/scripts/registry.py index 72e620d..5fff334 100644 --- a/scripts/registry.py +++ b/scripts/registry.py @@ -47,6 +47,7 @@ "service-secret": "terraform/modules/service-secret", "service-queue": "terraform/modules/service-queue", "cognito-app-client": "terraform/modules/cognito-app-client", + "app-network": "terraform/modules/app-network", } # --------------------------------------------------------------------------- diff --git a/scripts/tests/test_generate_modules.py b/scripts/tests/test_generate_modules.py index 6a9b3e6..7d63005 100644 --- a/scripts/tests/test_generate_modules.py +++ b/scripts/tests/test_generate_modules.py @@ -474,6 +474,116 @@ def test_adopt_dns_must_be_a_bool(self): self.assertRejected(app, "routing.adopt_dns must be true or false") +CALLER = { # cupcake-backend calling moresleep privately instead of over the internet + "name": "cupcake-backend", + "team": "cupcake", + "compute": {"port": 8080}, + "routing": {"host": "cupcake-backend.javazone.no", "priority": 101}, + "calls": [{"app": "moresleep", "team": "moresleep", "scopes": ["talks.read"], "env": "SP_BASE_URL"}, + {"app": "cupcake"}], +} + +CALLEE = { # private-only app (no routing) offering scopes + "name": "moresleep", + "team": "moresleep", + "compute": {"port": 8082}, + "resources": {"databases": [{"name": "main", "engine": "postgres", "env": "DATABASE_URL"}]}, + "provides": { + "scopes": {"talks.read": "Read talks", "talks.write": "Write talks"}, + "consumers": [{"app": "cupcake-backend", "team": "cupcake", "scopes": ["talks.read"]}], + }, +} + + +class PeersValidationTest(unittest.TestCase): + def assertRejected(self, app, fragment): + with self.assertRaises(gen.AppYamlError) as cm: + generate(app) + self.assertIn(fragment, str(cm.exception)) + + def test_self_reference_is_rejected(self): + self.assertRejected({**CALLER, "calls": [{"app": "cupcake-backend"}]}, "cannot call itself") + self.assertRejected({**CALLEE, "provides": {"consumers": [{"app": "moresleep"}]}}, "cannot consume itself") + + def test_duplicates_are_rejected(self): + self.assertRejected({**CALLER, "calls": [{"app": "x"}, {"app": "x", "team": "cupcake"}]}, "listed twice") + self.assertRejected({**CALLEE, "provides": {"consumers": [{"app": "a", "team": "b"}, {"app": "a", "team": "b"}]}}, + "listed twice") + + def test_bad_names_and_fields_are_rejected(self): + self.assertRejected({**CALLER, "calls": [{"app": "Moresleep"}]}, "calls[0].app") + self.assertRejected({**CALLER, "calls": [{"app": "x", "team": "more-sleep"}]}, "calls[0].team") + self.assertRejected({**CALLER, "calls": [{"app": "x", "scope": ["a"]}]}, "unknown field(s) scope") + self.assertRejected({**CALLER, "calls": [{"app": "x", "scopes": ["has space"]}]}, "not a valid scope name") + self.assertRejected({**CALLER, "calls": [{"app": "x", "env": "1BAD"}]}, "calls[0].env") + self.assertRejected({**CALLER, "calls": {"app": "x"}}, "calls: expected a list") + self.assertRejected({**CALLEE, "provides": {"scopes": {"a": ""}}}, "description") + + def test_consumer_scopes_must_be_provided(self): + app = {**CALLEE, "provides": {"scopes": ["talks.read"], + "consumers": [{"app": "x", "scopes": ["talks.write"]}]}} + self.assertRejected(app, "not listed in provides.scopes") + + def test_injected_names_must_not_collide(self): + # cupcake today sets SP_BASE_URL in environment:; moving to calls means removing it there + self.assertRejected({**CUPCAKE, "calls": [{"app": "moresleep", "team": "moresleep", "env": "SP_BASE_URL"}]}, + "SP_BASE_URL is injected") + self.assertRejected({**CALLER, "calls": [{"app": "moresleep", "team": "a"}, {"app": "moresleep", "team": "b"}]}, + "MORESLEEP_URL used twice") + self.assertRejected({**CALLER, "calls": [{"app": "x", "env": "PEER_AUTH_CLIENT_ID"}, {"app": "y", "scopes": ["a"]}]}, + "PEER_AUTH_CLIENT_ID used twice") + + def test_limits(self): + self.assertRejected({**CALLER, "calls": [{"app": f"a{i}"} for i in range(21)]}, "at most 20") + + +class PeersGenerationTest(unittest.TestCase): + def test_every_app_gets_its_own_security_group_and_registry(self): + for app in (CUPCAKE, FROSTING, FULL, {"name": "worker", "team": "core"}): + main = generate(app)["main.tf"] + self.assertIn('module "network"', main) + self.assertIn("security_group_ids = [module.network.security_group_id]", main) + self.assertIn("[module.network.registry_arn]", main) + self.assertNotIn("ecs_tasks_security_group_id", main) + + def test_rds_accepts_only_the_apps_own_group(self): + main = generate(FULL)["main.tf"] + self.assertIn("allowed_security_group_ids = [module.network.security_group_id]", main) + + def test_alb_ingress_follows_routing(self): + self.assertIn("alb_ingress = true", generate(CALLER)["main.tf"]) + self.assertIn("alb_ingress = false", generate(CALLEE)["main.tf"]) + + def test_caller_gets_url_scopes_and_m2m_credentials(self): + main = generate(CALLER)["main.tf"] + self.assertIn('"moresleep/moresleep" = ["talks.read"]', main) + self.assertIn('"cupcake/cupcake" = []', main) + self.assertRegex(main, r'SP_BASE_URL\s+= module.network.peer_url_arns\["moresleep/moresleep"\]') + self.assertRegex(main, r'CUPCAKE_URL\s+= module.network.peer_url_arns\["cupcake/cupcake"\]') + self.assertRegex(main, r'SP_BASE_SCOPES\s+= module\.network\.peer_scopes_arns\["moresleep/moresleep"\]') + self.assertNotIn("CUPCAKE_SCOPES", main) + self.assertRegex(main, r"PEER_AUTH_TOKEN_URL\s+= module.network.token_url") + self.assertRegex(main, r"PEER_AUTH_CLIENT_SECRET\s+= module.network.m2m_client_secret_arn") + self.assertNotIn("PEER_AUTH_ISSUER", main) + + def test_network_only_caller_gets_no_m2m_credentials(self): + main = generate({**CALLER, "calls": [{"app": "cupcake"}]})["main.tf"] + self.assertIn("CUPCAKE_URL", main) + self.assertNotIn("PEER_AUTH", main) + + def test_callee_gets_verification_settings(self): + main = generate(CALLEE)["main.tf"] + self.assertIn('"talks.read" = "Read talks"', main) + self.assertIn('"cupcake/cupcake-backend" = ["talks.read"]', main) + self.assertRegex(main, r"PEER_AUTH_ISSUER\s+= module.network.issuer_url") + self.assertRegex(main, r'PEER_AUTH_SCOPE_PREFIX\s+= "moresleep-moresleep/"') + self.assertNotIn("target_group_arn", main) + + def test_scope_list_shorthand(self): + main = generate({**CALLEE, "provides": {"scopes": ["talks.read"]}})["main.tf"] + self.assertIn('"talks.read" = "talks.read"', main) + + @unittest.skipUnless(shutil.which("terraform"), "terraform not installed") class FmtTest(unittest.TestCase): """tf-plan runs `terraform fmt -check`; generated files must pass it.""" @@ -482,7 +592,7 @@ def test_generated_files_are_fmt_clean(self): public = {**FROSTING, "auth": {"pool": "internal", "client": "public", "groups": ["helter"], "env": {"COGNITO_CLIENT_ID": "NUXT_PUBLIC_OIDC_CLIENT_ID"}}} for app in (CUPCAKE, FROSTING, FULL, MORESLEEP, public, REAL_FROSTING, REAL_CUPCAKE, WIKI, - {"name": "worker", "team": "core"}): + {"name": "worker", "team": "core"}, CALLER, CALLEE): with tempfile.TemporaryDirectory() as d: for name, content in generate(app).items(): with open(os.path.join(d, name), "w") as f: diff --git a/terraform/lambda-src/app_peers_sync/handler.py b/terraform/lambda-src/app_peers_sync/handler.py new file mode 100644 index 0000000..7e508df --- /dev/null +++ b/terraform/lambda-src/app_peers_sync/handler.py @@ -0,0 +1,775 @@ +"""App peers sync — app-to-app calls declared in app.yaml (`calls:` / `provides:`). + +App Terraform (app-network module) writes one request per app: + + /{project}/apps/{team}/{app}/network (String JSON) + {"version": 1, "security_group_id": "sg-…", "port": 8080, + "calls": {"moresleep/moresleep": ["talks.read"]}, + "provides": {"scopes": {"talks.read": "Read talks"}, + "consumers": {"cupcake/cupcake-backend": ["talks.read"]}}} + +Only team {team} can write under that path (developer boundary DenyBadName), so +the path is the app's identity; nothing in the payload is trusted for identity. +The security group in the payload must be the app's own (name +{team}-{app}-tasks, tag team={team}), or the request is ignored. + +An edge caller -> callee exists when the caller lists the callee in `calls` +AND the callee lists the caller in `provides.consumers`. For each edge: + + - ingress rule on the callee's SG from the caller's SG, tcp callee port, + tagged managed-by=javabin-app-peers-sync (rules teams add themselves are + never touched); + - granted scopes = requested ∩ allowed for that consumer ∩ provides.scopes, + via a Cognito resource server "{t}-{a}" per callee and one + client-credentials client "m2m.{t}-{a}" per caller (internal pool). + +Parameters written into the caller's own path, only when the value changes +(so a reconcile never restarts apps through app-secret-rollout): + + peer-{t}-{a}-url String http://{t}-{a}.{project}.internal:{port} + peer-{t}-{a}-scopes String scopes the caller's client actually holds for + that callee, space-separated ("none" if none); + only for calls that request scopes + m2m-client-id String + m2m-client-secret SecureString + peers-status String JSON per call: active | pending-consent | + callee-missing | callee-invalid + +An app whose `network` request is invalid (bad payload, wrong security +group, ...) is frozen, not removed: its parameters, its managed rules and +its resource server stay as they are, so one bad apply does not cut off its +peers. The valid side of an edge can still withdraw: a frozen edge is kept +only while every valid side still declares it. Deleting the `network` +parameter removes the app for real. + +Every invocation recomputes everything from all requests, so event order +and missed events do not matter. + +Triggers: + - EventBridge "Parameter Store Change" on */network (Create/Update/Delete) + - Daily {"action": "reconcile"}: same sync plus ownership audits + (Cloud Map names, Route53 records vs Cloud Map instances, borrowed + security groups) reported to Slack. +""" + +import json +import logging +import os +import re + +import boto3 + +try: + from shared.slack import get_webhook_url, post_to_slack +except ImportError: # unit tests load the handler without shared/ + get_webhook_url = post_to_slack = None + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +PROJECT = os.environ.get("PROJECT", "javabin") +VPC_ID = os.environ.get("VPC_ID", "") +POOL_ID = os.environ.get("INTERNAL_POOL_ID", "") +NAMESPACE_ID = os.environ.get("NAMESPACE_ID", "") +HOSTED_ZONE_ID = os.environ.get("HOSTED_ZONE_ID", "") +NAMESPACE_NAME = f"{PROJECT}.internal" +CLUSTER = os.environ.get("CLUSTER", f"{PROJECT}-platform") +WEBHOOK_PARAM = os.environ.get("INFRA_WEBHOOK_PARAM", "") + +MANAGED_BY = f"{PROJECT}-app-peers-sync" +APPS_PATH = f"/{PROJECT}/apps" +DEFAULT_PORT = 8000 +RS_MARKER = " (app.yaml)" +CLIENT_PREFIX = "m2m." +TOKEN_MINUTES = 60 + +TEAM = r"[a-z]{1,12}" +APP = r"[a-z][a-z0-9-]{0,19}" +PARAM_RE = re.compile(rf"^{re.escape(APPS_PATH)}/(?P{TEAM})/(?P{APP})/(?P[^/]+)$") +KEY_RE = re.compile(rf"^(?P{TEAM})/(?P{APP})$") +SCOPE_RE = re.compile(r"^[a-z][a-z0-9._-]{0,39}$") +SG_RE = re.compile(r"^sg-[0-9a-f]{8,17}$") +PEER_PARAM_RE = re.compile(rf"^peer-(?P{TEAM})-(?P{APP})-(?:url|scopes)$") +NO_SCOPES = "none" +MAX_CALLS, MAX_CONSUMERS, MAX_SCOPES = 20, 50, 25 + +ssm = boto3.client("ssm") +ec2 = boto3.client("ec2") +cognito = boto3.client("cognito-idp") +servicediscovery = boto3.client("servicediscovery") +ecs = boto3.client("ecs") +route53 = boto3.client("route53") + + +class InvalidRequest(ValueError): + pass + + +def _ident(key): + """'team/app' -> 'team-app' (unambiguous: team names have no hyphen).""" + return key.replace("/", "-") + + +def _key(ident): + """'team-app' -> 'team/app'.""" + return ident.replace("-", "/", 1) + + +# --------------------------------------------------------------------------- +# Request parsing +# --------------------------------------------------------------------------- + +def _scopes(value, where, allowed=None): + if not isinstance(value, list) or not all(isinstance(s, str) and SCOPE_RE.match(s) for s in value): + raise InvalidRequest(f"{where}: expected a list of scope names") + if allowed is not None and not set(value) <= set(allowed): + raise InvalidRequest(f"{where}: scopes not in provides.scopes") + return sorted(set(value)) + + +def _refs(value, where, own, limit): + if not isinstance(value, dict): + raise InvalidRequest(f"{where}: expected an object") + if len(value) > limit: + raise InvalidRequest(f"{where}: more than {limit} entries") + for key in value: + if not KEY_RE.match(str(key)): + raise InvalidRequest(f"{where}: {key!r} is not team/app") + if key == own: + raise InvalidRequest(f"{where}: an app cannot reference itself") + return value + + +def parse_request(raw, team, app): + """Validate a `network` parameter value written by the app-network module.""" + try: + req = json.loads(raw) + except (TypeError, ValueError) as e: + raise InvalidRequest(f"not JSON: {e}") + if not isinstance(req, dict) or req.get("version") != 1: + raise InvalidRequest("expected a JSON object with version 1") + + sg = req.get("security_group_id") + if not isinstance(sg, str) or not SG_RE.match(sg): + raise InvalidRequest("security_group_id is not a security group ID") + port = req.get("port") + if not isinstance(port, int) or isinstance(port, bool) or not 1 <= port <= 65535: + raise InvalidRequest("port must be 1-65535") + + own = f"{team}/{app}" + calls = {k: _scopes(v, f"calls.{k}") for k, v in _refs(req.get("calls") or {}, "calls", own, MAX_CALLS).items()} + + provides = req.get("provides") or {} + if not isinstance(provides, dict): + raise InvalidRequest("provides: expected an object") + scopes = provides.get("scopes") or {} + if not isinstance(scopes, dict) or len(scopes) > MAX_SCOPES: + raise InvalidRequest(f"provides.scopes: expected an object of at most {MAX_SCOPES} scopes") + for s, desc in scopes.items(): + if not SCOPE_RE.match(str(s)) or not isinstance(desc, str) or not 1 <= len(desc) <= 256: + raise InvalidRequest(f"provides.scopes: invalid scope {s!r}") + consumers = { + k: _scopes(v, f"provides.consumers.{k}", allowed=scopes) + for k, v in _refs(provides.get("consumers") or {}, "provides.consumers", own, MAX_CONSUMERS).items() + } + + return {"team": team, "app": app, "key": own, "sg": sg, "port": port, + "calls": calls, "scopes": dict(scopes), "consumers": consumers} + + +# --------------------------------------------------------------------------- +# State loading +# --------------------------------------------------------------------------- + +def load_parameters(): + """All parameters under /{project}/apps: {name: value} (SecureStrings encrypted).""" + params = {} + for page in ssm.get_paginator("get_parameters_by_path").paginate(Path=APPS_PATH, Recursive=True): + for p in page["Parameters"]: + params[p["Name"]] = p["Value"] + return params + + +def app_security_groups(): + """App task groups in the platform VPC: {sg_id: (name, team tag)}.""" + groups = {} + filters = [{"Name": "tag:javabin-network", "Values": ["app"]}] + if VPC_ID: + filters.append({"Name": "vpc-id", "Values": [VPC_ID]}) + for page in ec2.get_paginator("describe_security_groups").paginate(Filters=filters): + for g in page["SecurityGroups"]: + tags = {t["Key"]: t["Value"] for t in g.get("Tags", [])} + groups[g["GroupId"]] = (g["GroupName"], tags.get("team")) + return groups + + +def load_requests(params, groups): + """Return (requests {key: spec}, invalid {key: reason}).""" + requests, invalid = {}, {} + for name, value in params.items(): + m = PARAM_RE.match(name) + if not m or m["name"] != "network": + continue + team, app = m["team"], m["app"] + key = f"{team}/{app}" + try: + spec = parse_request(value, team, app) + # The payload's SG must be this app's own group, or a team could + # make the Lambda open another team's group. + if groups.get(spec["sg"]) != (f"{team}-{app}-tasks", team): + raise InvalidRequest(f"{spec['sg']} is not the security group {team}-{app}-tasks of team {team}") + except InvalidRequest as e: + logger.warning("ignoring %s: %s", name, e) + invalid[key] = str(e) + continue + requests[key] = spec + return requests, invalid + + +# --------------------------------------------------------------------------- +# Desired state +# --------------------------------------------------------------------------- + +def compute(requests, frozen=()): + """Edges, grants and per-app status from all valid requests. + + `frozen` are apps whose request is invalid: a call to one is + "callee-invalid" (its existing rule and scopes are kept elsewhere). + + Returns { + "rules": {(callee_sg, caller_sg, port): (caller_key, callee_team)}, + "grants": {caller_key: {callee_key: [scope, ...]}}, # only callers requesting scopes + "status": {app_key: {...}}, + "servers": {callee_key: {scope: description}}, + } + """ + rules, grants, status, servers = {}, {}, {}, {} + for key, spec in requests.items(): + if spec["scopes"]: + servers[key] = spec["scopes"] + + for caller_key, caller in requests.items(): + calls_status = {} + for callee_key, requested in caller["calls"].items(): + callee = requests.get(callee_key) + granted = [] + if callee is None: + state = "callee-invalid" if callee_key in frozen else "callee-missing" + elif caller_key not in callee["consumers"]: + state = "pending-consent" + else: + state = "active" + allowed = set(callee["consumers"][caller_key]) & set(callee["scopes"]) + granted = sorted(set(requested) & allowed) + rules[(callee["sg"], caller["sg"], callee["port"])] = (caller_key, callee["team"]) + calls_status[callee_key] = { + "state": state, + "url": peer_url(callee_key, callee), + "granted": granted, + "denied": sorted(set(requested) - set(granted)), + } + if requested: + grants.setdefault(caller_key, {})[callee_key] = granted + + consumers_status = {} + for consumer_key in caller["consumers"]: + consumer = requests.get(consumer_key) + active = consumer is not None and caller_key in consumer["calls"] + consumers_status[consumer_key] = ( + "active" if active else "caller-invalid" if consumer_key in frozen else "pending") + + if calls_status or consumers_status: + status[caller_key] = {"calls": calls_status, "consumers": consumers_status} + return {"rules": rules, "grants": grants, "status": status, "servers": servers} + + +def frozen_edge(caller_key, callee_key, requests, invalid, scope=None, port=None): + """Whether an existing edge (or one scope of it) touching a frozen app is kept: + at least one side is frozen and every valid side still declares it.""" + caller_frozen, callee_frozen = caller_key in invalid, callee_key in invalid + if not (caller_frozen or callee_frozen): + return False + if not caller_frozen: + calls = requests.get(caller_key, {}).get("calls", {}) + if callee_key not in calls or (scope is not None and scope not in calls[callee_key]): + return False + if not callee_frozen: + callee = requests.get(callee_key) + if callee is None or caller_key not in callee["consumers"]: + return False + if port is not None and port != callee["port"]: + return False + if scope is not None and (scope not in callee["consumers"][caller_key] or scope not in callee["scopes"]): + return False + return True + + +def peer_url(callee_key, callee): + port = callee["port"] if callee else DEFAULT_PORT + return f"http://{_ident(callee_key)}.{PROJECT}.internal:{port}" + + +# --------------------------------------------------------------------------- +# Security group rules +# --------------------------------------------------------------------------- + +def managed_rules(): + """Rules this Lambda created: {(sg, referenced_sg, port): (rule_id, caller key)}.""" + rules = {} + pages = ec2.get_paginator("describe_security_group_rules").paginate( + Filters=[{"Name": "tag:managed-by", "Values": [MANAGED_BY]}] + ) + for page in pages: + for r in page["SecurityGroupRules"]: + ref = (r.get("ReferencedGroupInfo") or {}).get("GroupId") + if r.get("IsEgress") or not ref: + continue + tags = {t["Key"]: t["Value"] for t in r.get("Tags", [])} + rules[(r["GroupId"], ref, r.get("FromPort"))] = (r["SecurityGroupRuleId"], tags.get("javabin-peer", "")) + return rules + + +def sync_rules(desired, errors, requests=None, invalid=None, sg_owner=None): + """Make managed rules match `desired`. Rules on edges touching a frozen + (invalid) app are kept while the valid side still declares the edge; + sg_owner maps every known app security group to its app key.""" + requests, invalid, sg_owner = requests or {}, invalid or {}, sg_owner or {} + existing = managed_rules() + for (sg, ref, port), (rule_id, peer) in existing.items(): + if (sg, ref, port) in desired: + continue + callee_key = sg_owner.get(sg) + if callee_key and peer and frozen_edge(peer, callee_key, requests, invalid, port=port): + logger.info("keeping %s on %s from %s (frozen edge %s -> %s)", rule_id, sg, ref, peer, callee_key) + continue + try: + ec2.revoke_security_group_ingress(GroupId=sg, SecurityGroupRuleIds=[rule_id]) + logger.info("revoked %s on %s from %s port %s", rule_id, sg, ref, port) + except Exception as e: # group or rule already gone + logger.info("revoke %s on %s: %s", rule_id, sg, e) + + for (sg, ref, port), (caller_key, callee_team) in desired.items(): + if (sg, ref, port) in existing: + continue + try: + ec2.authorize_security_group_ingress( + GroupId=sg, + IpPermissions=[{ + "IpProtocol": "tcp", "FromPort": port, "ToPort": port, + "UserIdGroupPairs": [{"GroupId": ref, "Description": f"app.yaml peer {caller_key}"}], + }], + TagSpecifications=[{"ResourceType": "security-group-rule", "Tags": [ + {"Key": "managed-by", "Value": MANAGED_BY}, + {"Key": "javabin-peer", "Value": caller_key}, + {"Key": "team", "Value": callee_team}, + ]}], + ) + logger.info("authorized %s -> %s port %s", ref, sg, port) + except Exception as e: + if "InvalidPermission.Duplicate" in str(e): + # The callee team already opened this itself; nothing to own. + logger.info("rule %s -> %s port %s already exists (team-managed)", ref, sg, port) + else: + logger.error("authorize %s -> %s port %s: %s", ref, sg, port, e) + errors.append(f"security group rule {ref} -> {sg}:{port}: {e}") + + +# --------------------------------------------------------------------------- +# Cognito: resource servers and M2M clients +# --------------------------------------------------------------------------- + +def _resource_servers(): + servers, token = {}, None + while True: + kw = {"UserPoolId": POOL_ID, "MaxResults": 50} + if token: + kw["NextToken"] = token + resp = cognito.list_resource_servers(**kw) + for rs in resp.get("ResourceServers", []): + servers[rs["Identifier"]] = rs + token = resp.get("NextToken") + if not token: + return servers + + +def _m2m_clients(): + """{client name: client id} for clients this Lambda owns.""" + clients, token = {}, None + while True: + kw = {"UserPoolId": POOL_ID, "MaxResults": 60} + if token: + kw["NextToken"] = token + resp = cognito.list_user_pool_clients(**kw) + for c in resp.get("UserPoolClients", []): + if c["ClientName"].startswith(CLIENT_PREFIX): + clients[c["ClientName"]] = c["ClientId"] + token = resp.get("NextToken") + if not token: + return clients + + +def _scope_set(rs): + return {s["ScopeName"]: s["ScopeDescription"] for s in rs.get("Scopes", [])} + + +def _put_server(identifier, scopes): + cognito.update_resource_server( + UserPoolId=POOL_ID, Identifier=identifier, Name=f"{identifier}{RS_MARKER}", + Scopes=[{"ScopeName": s, "ScopeDescription": d} for s, d in sorted(scopes.items())], + ) + + +def _client_settings(name, scopes): + settings = { + "UserPoolId": POOL_ID, + "ClientName": name, + "ExplicitAuthFlows": ["ALLOW_REFRESH_TOKEN_AUTH"], + "AccessTokenValidity": TOKEN_MINUTES, + "TokenValidityUnits": {"AccessToken": "minutes"}, + } + if scopes: + settings.update(AllowedOAuthFlows=["client_credentials"], AllowedOAuthScopes=sorted(scopes), + AllowedOAuthFlowsUserPoolClient=True) + else: + # Nothing granted yet: the client exists (so the caller's secrets + # resolve and its tasks start) but cannot get tokens. + settings["AllowedOAuthFlowsUserPoolClient"] = False + return settings + + +def sync_cognito(plan, errors, requests=None, invalid=None): + """Keep resource servers and M2M clients in line. + + Returns (creds {caller_key: (id, secret|None)}, clients, ours, + held {caller_key: set of scopes its client now holds}, wanted client names). + Scopes on frozen edges are kept (see frozen_edge); frozen callers' clients + are only narrowed when a valid callee withdraws. + """ + requests, invalid = requests or {}, invalid or {} + servers = _resource_servers() + ours = {i: rs for i, rs in servers.items() if rs.get("Name", "").endswith(RS_MARKER)} + + # 1. Create servers; widen changed ones to old ∪ new so no client loses a + # scope that still exists while it is being updated. + available = set() + for callee_key, scopes in plan["servers"].items(): + identifier = _ident(callee_key) + try: + if identifier not in servers: + cognito.create_resource_server( + UserPoolId=POOL_ID, Identifier=identifier, Name=f"{identifier}{RS_MARKER}", + Scopes=[{"ScopeName": s, "ScopeDescription": d} for s, d in sorted(scopes.items())], + ) + logger.info("created resource server %s", identifier) + elif identifier not in ours: + raise RuntimeError("an identifier with that name exists and is not managed by app.yaml") + elif _scope_set(ours[identifier]) != scopes: + _put_server(identifier, {**_scope_set(ours[identifier]), **scopes}) + available |= {f"{identifier}/{s}" for s in scopes} + except Exception as e: + logger.error("resource server %s: %s", identifier, e) + errors.append(f"resource server {identifier}: {e}") + + def kept(caller_key, current): + out = set() + for full in current: + ident, _, scope = full.partition("/") + if frozen_edge(caller_key, _key(ident), requests, invalid, scope=scope): + out.add(full) + return out + + # 2. Clients: one per caller that requests any scope, plus frozen callers'. + clients = _m2m_clients() + creds, held, wanted = {}, {}, set() + frozen_callers = [k for k in invalid if f"{CLIENT_PREFIX}{_ident(k)}" in clients] + for caller_key in (*plan["grants"], *frozen_callers): + per_callee = plan["grants"].get(caller_key, {}) + name = f"{CLIENT_PREFIX}{_ident(caller_key)}" + wanted.add(name) + scopes = {f"{_ident(k)}/{s}" for k, granted in per_callee.items() for s in granted} & available + try: + if name not in clients: + resp = cognito.create_user_pool_client(GenerateSecret=True, **_client_settings(name, scopes)) + client = resp["UserPoolClient"] + logger.info("created client %s", name) + else: + client = cognito.describe_user_pool_client(UserPoolId=POOL_ID, ClientId=clients[name])["UserPoolClient"] + current = set(client.get("AllowedOAuthScopes") or []) if client.get("AllowedOAuthFlowsUserPoolClient") else set() + scopes |= kept(caller_key, current) + if current != scopes or client.get("AccessTokenValidity") != TOKEN_MINUTES: + cognito.update_user_pool_client(ClientId=client["ClientId"], **_client_settings(name, scopes)) + logger.info("updated client %s scopes %s", name, sorted(scopes)) + held[caller_key] = scopes + if caller_key not in invalid: # frozen apps' parameters are not touched + creds[caller_key] = (client["ClientId"], client.get("ClientSecret")) + except Exception as e: + logger.error("client %s: %s", name, e) + errors.append(f"client {name}: {e}") + + return creds, clients, ours, held, wanted + + +def cleanup_cognito(plan, clients, ours, wanted_clients, errors, invalid=None): + """Delete clients and servers no longer requested (after clients dropped + their scopes). Frozen (invalid) callees keep their server as it is.""" + frozen_servers = {_ident(k) for k in (invalid or {})} + for name, client_id in clients.items(): + if name not in wanted_clients: + try: + cognito.delete_user_pool_client(UserPoolId=POOL_ID, ClientId=client_id) + logger.info("deleted client %s", name) + except Exception as e: + errors.append(f"delete client {name}: {e}") + + wanted_servers = {_ident(k): v for k, v in plan["servers"].items()} + for identifier, rs in ours.items(): + if identifier in frozen_servers: + continue + try: + if identifier not in wanted_servers: + cognito.delete_resource_server(UserPoolId=POOL_ID, Identifier=identifier) + logger.info("deleted resource server %s", identifier) + elif _scope_set(rs) != wanted_servers[identifier]: + _put_server(identifier, wanted_servers[identifier]) + except Exception as e: + errors.append(f"resource server {identifier}: {e}") + + +# --------------------------------------------------------------------------- +# Parameters in the caller's path +# --------------------------------------------------------------------------- + +def _put(name, value, team, app, secure=False, exists=False): + kw = {"Name": name, "Value": value, "Type": "SecureString" if secure else "String", + "Description": f"Managed by {MANAGED_BY} for {team}-{app} — do not edit"} + if exists: + ssm.put_parameter(Overwrite=True, **kw) + else: + ssm.put_parameter(Tags=[ + {"Key": "team", "Value": team}, + {"Key": "service", "Value": app}, + {"Key": "managed-by", "Value": MANAGED_BY}, + ], **kw) + logger.info("wrote %s", name) + + +def sync_parameters(params, requests, invalid, plan, creds, errors, held=None): + """Write what changed, delete what is no longer needed.""" + held = held or {} + desired = {} + for key, spec in requests.items(): + base = f"{APPS_PATH}/{key}" + for callee_key, requested in spec["calls"].items(): + ident = _ident(callee_key) + url_name, scopes_name = f"{base}/peer-{ident}-url", f"{base}/peer-{ident}-scopes" + call = plan["status"][key]["calls"][callee_key] + if call["state"] == "callee-invalid" and url_name in params: + desired[url_name] = params[url_name] # the frozen callee's port is unknown + else: + desired[url_name] = call["url"] + if requested: + # What the client really holds, so the caller never requests a + # scope it was not granted (Cognito would refuse the whole token) + if key in held: + have = sorted(s for s in held[key] if s.startswith(f"{ident}/")) + desired[scopes_name] = " ".join(have) or NO_SCOPES + else: # client sync failed this run + desired[scopes_name] = params.get(scopes_name, NO_SCOPES) + if key in plan["status"]: + desired[f"{base}/peers-status"] = json.dumps(plan["status"][key], sort_keys=True) + for key, reason in invalid.items(): + desired[f"{APPS_PATH}/{key}/peers-status"] = json.dumps({"error": reason}, sort_keys=True) + + for name, value in desired.items(): + if params.get(name) == value: + continue + team, app = PARAM_RE.match(name).group("team", "app") + try: + _put(name, value, team, app, exists=name in params) + except Exception as e: + errors.append(f"write {name}: {e}") + + # Client credentials: the secret is rewritten whenever the client ID changes + # (client recreated) or the secret parameter is missing. SecureString values + # are not decrypted, so they are never compared. + managed_creds = set() + for key, (client_id, secret) in creds.items(): + team, app = key.split("/") + id_name, secret_name = f"{APPS_PATH}/{key}/m2m-client-id", f"{APPS_PATH}/{key}/m2m-client-secret" + managed_creds |= {id_name, secret_name} + try: + changed = params.get(id_name) != client_id + if changed: + _put(id_name, client_id, team, app, exists=id_name in params) + if secret and (changed or secret_name not in params): + _put(secret_name, secret, team, app, secure=True, exists=secret_name in params) + except Exception as e: + errors.append(f"write credentials of {key}: {e}") + + # Delete managed parameters nobody needs. Invalid requests keep theirs, so + # a broken request does not restart or break the app. + frozen = tuple(f"{APPS_PATH}/{k}/" for k in invalid) + stale = [ + name for name in params + if (m := PARAM_RE.match(name)) and _managed_name(m["name"]) + and name not in desired and name not in managed_creds + and not (frozen and name.startswith(frozen)) + ] + for i in range(0, len(stale), 10): + try: + ssm.delete_parameters(Names=stale[i:i + 10]) + logger.info("deleted %s", stale[i:i + 10]) + except Exception as e: + errors.append(f"delete {stale[i:i + 10]}: {e}") + + +def _managed_name(name): + return bool(PEER_PARAM_RE.match(name)) or name in ("m2m-client-id", "m2m-client-secret", "peers-status") + + +# --------------------------------------------------------------------------- +# Audits (reconcile only): report, never fix +# --------------------------------------------------------------------------- + +def audit_cloud_map(): + """Every service in the namespace is {team}-… with a matching team tag and + is registered only by the ECS service of the same name.""" + findings = [] + if not NAMESPACE_ID: + return findings + pages = servicediscovery.get_paginator("list_services").paginate( + Filters=[{"Name": "NAMESPACE_ID", "Values": [NAMESPACE_ID], "Condition": "EQ"}] + ) + registered = {} # service name -> instance IPv4 addresses + for page in pages: + for svc in page["Services"]: + name = svc["Name"] + registered.setdefault(name, set()) + tags = {t["Key"]: t["Value"] for t in + servicediscovery.list_tags_for_resource(ResourceARN=svc["Arn"]).get("Tags", [])} + team = tags.get("team", "") + if not team or not name.startswith(f"{team}-"): + findings.append(f"Cloud Map service {name} has team tag {team or '(none)'}") + for ipage in servicediscovery.get_paginator("list_instances").paginate(ServiceId=svc["Id"]): + for inst in ipage["Instances"]: + attrs = inst.get("Attributes") or {} + if attrs.get("AWS_INSTANCE_IPV4"): + registered[name].add(attrs["AWS_INSTANCE_IPV4"]) + owner = attrs.get("ECS_SERVICE_NAME") + if owner != name: + findings.append(f"Cloud Map service {name} has an instance registered by {owner or 'a non-ECS caller'}") + findings.extend(audit_route53(registered)) + return findings + + +def audit_route53(registered): + """Every record in the namespace's private zone is an A record Cloud Map + wrote for one of its instances (teams are denied changing the zone; this + catches anything that slips through).""" + findings = [] + if not HOSTED_ZONE_ID: + return findings + suffix = f".{NAMESPACE_NAME}" + for page in route53.get_paginator("list_resource_record_sets").paginate(HostedZoneId=HOSTED_ZONE_ID): + for rr in page["ResourceRecordSets"]: + name, rtype = rr["Name"].rstrip("."), rr["Type"] + if name == NAMESPACE_NAME and rtype in ("SOA", "NS"): + continue + if rtype != "A" or not name.endswith(suffix) or "AliasTarget" in rr: + findings.append(f"unexpected {rtype} record {name} in the private zone") + continue + service = name[:-len(suffix)] + values = {v["Value"] for v in rr.get("ResourceRecords", [])} + if service not in registered: + findings.append(f"A record {name} has no Cloud Map service") + elif values - registered[service]: + findings.append(f"A record {name} points at {', '.join(sorted(values - registered[service]))}, not a registered instance") + return findings + + +def audit_borrowed_groups(groups): + """A task carrying {t}-{a}-tasks must belong to ECS service {t}-{a}.""" + findings = [] + task_arns = [] + for page in ecs.get_paginator("list_tasks").paginate(cluster=CLUSTER): + task_arns.extend(page["taskArns"]) + eni_owner = {} + for i in range(0, len(task_arns), 100): + for task in ecs.describe_tasks(cluster=CLUSTER, tasks=task_arns[i:i + 100])["tasks"]: + service = (task.get("group") or "").removeprefix("service:") + for att in task.get("attachments", []): + for d in att.get("details", []): + if d.get("name") == "networkInterfaceId": + eni_owner[d["value"]] = service + enis = list(eni_owner) + for i in range(0, len(enis), 100): + resp = ec2.describe_network_interfaces(NetworkInterfaceIds=enis[i:i + 100]) + for eni in resp["NetworkInterfaces"]: + owner = eni_owner.get(eni["NetworkInterfaceId"], "") + for g in eni.get("Groups", []): + if g["GroupId"] not in groups: + continue + expected = groups[g["GroupId"]][0].removesuffix("-tasks") + if owner != expected: + findings.append(f"security group {groups[g['GroupId']][0]} is attached to a task of {owner or 'an unknown service'}") + return findings + + +def report(lines): + if not lines or not WEBHOOK_PARAM or post_to_slack is None: + return + text = "*app-peers-sync* found problems with app-to-app calls:\n" + "\n".join(f"• {l}" for l in lines[:30]) + try: + post_to_slack(get_webhook_url(ssm, WEBHOOK_PARAM), + {"text": text, "blocks": [{"type": "section", "text": {"type": "mrkdwn", "text": text}}]}) + except Exception as e: + logger.error("Slack report failed: %s", e) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def handler(event, context): + reconcile = (event or {}).get("action") == "reconcile" + errors = [] + + params = load_parameters() + groups = app_security_groups() + requests, invalid = load_requests(params, groups) + plan = compute(requests, frozen=invalid) + + # Which app owns each security group: valid apps by their request, frozen + # apps by the group named after them + sg_owner = {spec["sg"]: k for k, spec in requests.items()} + names = {f"{k.replace('/', '-')}-tasks": k for k in invalid} + for sg, (gname, team) in groups.items(): + if gname in names and names[gname].split("/")[0] == team: + sg_owner[sg] = names[gname] + + sync_rules(plan["rules"], errors, requests, invalid, sg_owner) + creds, clients, ours, held, wanted = sync_cognito(plan, errors, requests, invalid) + sync_parameters(params, requests, invalid, plan, creds, errors, held) + cleanup_cognito(plan, clients, ours, wanted, errors, invalid) + + findings = [] + if reconcile: + for audit in (audit_cloud_map, lambda: audit_borrowed_groups(groups)): + try: + findings.extend(audit()) + except Exception as e: + logger.error("audit failed: %s", e) + errors.append(f"audit: {e}") + findings.extend(f"invalid request {k}: {r}" for k, r in sorted(invalid.items())) + report(findings + errors) + + result = { + "status": "error" if errors else "ok", + "apps": len(requests), + "edges": len(plan["rules"]), + "invalid": invalid, + "errors": errors, + "findings": findings, + } + logger.info("result %s", json.dumps(result)) + return result diff --git a/terraform/lambda-src/app_secret_rollout/handler.py b/terraform/lambda-src/app_secret_rollout/handler.py index 87e1205..3694bd6 100644 --- a/terraform/lambda-src/app_secret_rollout/handler.py +++ b/terraform/lambda-src/app_secret_rollout/handler.py @@ -18,7 +18,10 @@ (rewritten on every sync), cognito-client-secret (a new secret only comes with a new client, and app-auth-sync writes the ID right after it, so restarting on the secret would deploy once with the old ID) and the RDS -master password (the database keeps the original). +master password (the database keeps the original), the app-to-app request +(network) and its status (peers-status). Peer URLs and M2M client +credentials written by app-peers-sync do restart the caller, so it picks +them up. """ import logging @@ -33,7 +36,7 @@ PROJECT = os.environ.get("PROJECT", "javabin") CLUSTER = os.environ.get("CLUSTER", f"{PROJECT}-platform") SECRET_RE = re.compile(rf"^/{re.escape(PROJECT)}/apps/(?P[a-z]{{1,12}})/(?P[a-z][a-z0-9-]{{0,19}})/(?P[^/]+)$") -PLATFORM_MANAGED = re.compile(r"^(auth|auth-status|cognito-client-secret|db-master-password)$") +PLATFORM_MANAGED = re.compile(r"^(auth|auth-status|cognito-client-secret|db-master-password|network|peers-status)$") ecs = boto3.client("ecs") diff --git a/terraform/lambda-src/tests/test_app_peers_sync.py b/terraform/lambda-src/tests/test_app_peers_sync.py new file mode 100644 index 0000000..a32173c --- /dev/null +++ b/terraform/lambda-src/tests/test_app_peers_sync.py @@ -0,0 +1,513 @@ +"""Tests for app_peers_sync — run: python3 -m unittest discover terraform/lambda-src/tests""" + +import importlib.util +import json +import os +import sys +import types +import unittest + +PREFIX = "/javabin/apps" + + +class Pager: + def __init__(self, fn): + self.fn = fn + + def paginate(self, **kw): + return [self.fn(**kw)] + + +class FakeSSM: + def __init__(self, params=None): + self.params = dict(params or {}) + self.puts, self.deleted = [], [] + + def get_paginator(self, name): + assert name == "get_parameters_by_path" + return Pager(lambda **kw: {"Parameters": [{"Name": k, "Value": v} for k, v in self.params.items()]}) + + def put_parameter(self, Name, Value, Type, Description, Overwrite=False, Tags=None): + if Name in self.params and not Overwrite: + raise AssertionError(f"{Name} exists, Overwrite missing") + if Overwrite and Tags: + raise AssertionError("Tags cannot be combined with Overwrite") + self.params[Name] = Value + self.puts.append((Name, Type)) + + def delete_parameters(self, Names): + for n in Names: + self.params.pop(n, None) + self.deleted.extend(Names) + + +class FakeEC2: + def __init__(self, groups): + self.groups = groups # sg -> (name, team) + self.rules = {} # rule_id -> (sg, ref, port, managed, peer) + self.authorized, self.revoked = [], [] + self.n = 0 + + def get_paginator(self, name): + if name == "describe_security_groups": + return Pager(lambda **kw: {"SecurityGroups": [ + {"GroupId": g, "GroupName": n, "Tags": [{"Key": "team", "Value": t}]} + for g, (n, t) in self.groups.items()]}) + assert name == "describe_security_group_rules" + return Pager(lambda **kw: {"SecurityGroupRules": [ + {"SecurityGroupRuleId": rid, "GroupId": sg, "IsEgress": False, "FromPort": port, "ToPort": port, + "ReferencedGroupInfo": {"GroupId": ref}, "Tags": [{"Key": "javabin-peer", "Value": peer}]} + for rid, (sg, ref, port, managed, peer) in self.rules.items() if managed]}) + + def add_rule(self, sg, ref, port, managed=True, peer=""): + self.n += 1 + self.rules[f"sgr-{self.n}"] = (sg, ref, port, managed, peer) + + def authorize_security_group_ingress(self, GroupId, IpPermissions, TagSpecifications): + perm = IpPermissions[0] + ref = perm["UserIdGroupPairs"][0]["GroupId"] + if any(r[:3] == (GroupId, ref, perm["FromPort"]) for r in self.rules.values()): + raise Exception("InvalidPermission.Duplicate") + tags = {t["Key"]: t["Value"] for t in TagSpecifications[0]["Tags"]} + assert tags["managed-by"] == "javabin-app-peers-sync" + self.add_rule(GroupId, ref, perm["FromPort"], peer=tags["javabin-peer"]) + self.authorized.append((GroupId, ref, perm["FromPort"])) + + def revoke_security_group_ingress(self, GroupId, SecurityGroupRuleIds): + for rid in SecurityGroupRuleIds: + sg, ref, port, *_ = self.rules.pop(rid) + assert sg == GroupId + self.revoked.append((sg, ref, port)) + + +class FakeCognito: + def __init__(self): + self.servers, self.clients = {}, {} # identifier -> rs ; id -> client + self.log = [] + self.n = 0 + + def _check_scopes(self, scopes): + for s in scopes or []: + ident, name = s.split("/") + assert name in {x["ScopeName"] for x in self.servers[ident]["Scopes"]}, f"unknown scope {s}" + + def list_resource_servers(self, UserPoolId, MaxResults, NextToken=None): + return {"ResourceServers": list(self.servers.values())} + + def create_resource_server(self, UserPoolId, Identifier, Name, Scopes): + assert Identifier not in self.servers + self.servers[Identifier] = {"Identifier": Identifier, "Name": Name, "Scopes": Scopes} + self.log.append(("create-rs", Identifier)) + + def update_resource_server(self, UserPoolId, Identifier, Name, Scopes): + names = {s["ScopeName"] for s in Scopes} + for c in self.clients.values(): # Cognito refuses dropping a scope a client uses + for s in c.get("AllowedOAuthScopes", []): + ident, name = s.split("/") + assert ident != Identifier or name in names, f"{s} still in use" + self.servers[Identifier] = {"Identifier": Identifier, "Name": Name, "Scopes": Scopes} + self.log.append(("update-rs", Identifier, sorted(names))) + + def delete_resource_server(self, UserPoolId, Identifier): + for c in self.clients.values(): + assert not any(s.startswith(f"{Identifier}/") for s in c.get("AllowedOAuthScopes", [])) + del self.servers[Identifier] + self.log.append(("delete-rs", Identifier)) + + def list_user_pool_clients(self, UserPoolId, MaxResults, NextToken=None): + return {"UserPoolClients": [{"ClientId": i, "ClientName": c["ClientName"]} for i, c in self.clients.items()]} + + def create_user_pool_client(self, GenerateSecret, UserPoolId, ClientName, **kw): + self._check_scopes(kw.get("AllowedOAuthScopes")) + self.n += 1 + cid = f"client{self.n}" + self.clients[cid] = {"ClientId": cid, "ClientName": ClientName, "ClientSecret": f"secret{self.n}", **kw} + self.log.append(("create-client", ClientName)) + return {"UserPoolClient": self.clients[cid]} + + def describe_user_pool_client(self, UserPoolId, ClientId): + return {"UserPoolClient": dict(self.clients[ClientId])} + + def update_user_pool_client(self, ClientId, UserPoolId, ClientName, **kw): + self._check_scopes(kw.get("AllowedOAuthScopes")) + c = self.clients[ClientId] + for k in ("AllowedOAuthScopes", "AllowedOAuthFlows"): + c.pop(k, None) + c.update(kw) + self.log.append(("update-client", ClientName, sorted(kw.get("AllowedOAuthScopes", [])))) + + def delete_user_pool_client(self, UserPoolId, ClientId): + assert self.clients.pop(ClientId) + self.log.append(("delete-client", ClientId)) + + +class FakeServiceDiscovery: + def __init__(self, services=()): + self.services = list(services) # (name, team, [ecs owners]) + + def get_paginator(self, name): + if name == "list_services": + return Pager(lambda **kw: {"Services": [{"Id": n, "Name": n, "Arn": f"arn:{n}"} for n, _, _ in self.services]}) + def inst(o): + owner, _, ip = o.partition("@") + return {"Id": "i", "Attributes": {"ECS_SERVICE_NAME": owner, **({"AWS_INSTANCE_IPV4": ip} if ip else {})}} + return Pager(lambda ServiceId: {"Instances": [ + inst(o) for n, _, owners in self.services if n == ServiceId for o in owners]}) + + def list_tags_for_resource(self, ResourceARN): + team = next(t for n, t, _ in self.services if f"arn:{n}" == ResourceARN) + return {"Tags": [{"Key": "team", "Value": team}]} + + +class FakeRoute53: + def __init__(self, records=()): + self.records = list(records) # (name, type, [values]) + + def get_paginator(self, name): + return Pager(lambda HostedZoneId: {"ResourceRecordSets": [ + {"Name": f"{n}.", "Type": t, "ResourceRecords": [{"Value": v} for v in vals]} + for n, t, vals in self.records]}) + + +class FakeECS: + def __init__(self, tasks=()): + self.tasks = list(tasks) # (service, eni) + + def get_paginator(self, name): + return Pager(lambda **kw: {"taskArns": [f"t{i}" for i in range(len(self.tasks))]}) + + def describe_tasks(self, cluster, tasks): + return {"tasks": [{"group": f"service:{s}", "attachments": [ + {"details": [{"name": "networkInterfaceId", "value": e}]}]} for s, e in self.tasks]} + + +def load(ssm, ec2, cognito, sd=None, ecs=None, env=None, r53=None): + clients = {"ssm": ssm, "ec2": ec2, "cognito-idp": cognito, + "servicediscovery": sd or FakeServiceDiscovery(), "ecs": ecs or FakeECS(), + "route53": r53 or FakeRoute53()} + boto3 = types.ModuleType("boto3") + boto3.client = lambda svc, **kw: clients[svc] + sys.modules["boto3"] = boto3 + os.environ.update({"INTERNAL_POOL_ID": "pool", "NAMESPACE_ID": "ns-1", "HOSTED_ZONE_ID": "Z1", + "INFRA_WEBHOOK_PARAM": "", **(env or {})}) + path = os.path.join(os.path.dirname(__file__), "..", "app_peers_sync", "handler.py") + spec = importlib.util.spec_from_file_location("app_peers_sync_handler", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def request(sg, port=8080, calls=None, scopes=None, consumers=None): + return json.dumps({"version": 1, "security_group_id": sg, "port": port, "calls": calls or {}, + "provides": {"scopes": scopes or {}, "consumers": consumers or {}}}) + + +GROUPS = { + "sg-0000000a": ("cupcake-cupcake-backend-tasks", "cupcake"), + "sg-0000000b": ("moresleep-moresleep-tasks", "moresleep"), + "sg-0000000c": ("cupcake-cupcake-tasks", "cupcake"), +} +CALLER = f"{PREFIX}/cupcake/cupcake-backend" +CALLEE = f"{PREFIX}/moresleep/moresleep" +SCOPES = {"talks.read": "Read talks", "talks.write": "Write talks"} + + +def world(params, groups=GROUPS, **kw): + ssm, ec2, cognito = FakeSSM(params), FakeEC2(dict(groups)), FakeCognito() + return load(ssm, ec2, cognito, **kw), ssm, ec2, cognito + + +def both_sides(caller_scopes=("talks.read", "talks.write"), consumer_scopes=("talks.read",), callee_port=8082): + return { + f"{CALLER}/network": request("sg-0000000a", calls={"moresleep/moresleep": list(caller_scopes)}), + f"{CALLEE}/network": request("sg-0000000b", port=callee_port, scopes=SCOPES, + consumers={"cupcake/cupcake-backend": list(consumer_scopes)}), + } + + +class ConsentTest(unittest.TestCase): + def test_edge_needs_both_sides_and_grants_the_intersection(self): + h, ssm, ec2, cognito = world(both_sides()) + result = h.handler({}, None) + self.assertEqual(result["status"], "ok", result) + self.assertEqual(ec2.authorized, [("sg-0000000b", "sg-0000000a", 8082)]) + + status = json.loads(ssm.params[f"{CALLER}/peers-status"]) + call = status["calls"]["moresleep/moresleep"] + self.assertEqual((call["state"], call["granted"], call["denied"]), ("active", ["talks.read"], ["talks.write"])) + self.assertEqual(json.loads(ssm.params[f"{CALLEE}/peers-status"])["consumers"], + {"cupcake/cupcake-backend": "active"}) + self.assertEqual(ssm.params[f"{CALLER}/peer-moresleep-moresleep-url"], + "http://moresleep-moresleep.javabin.internal:8082") + # Only what was granted, not what was requested (talks.write is denied) + self.assertEqual(ssm.params[f"{CALLER}/peer-moresleep-moresleep-scopes"], "moresleep-moresleep/talks.read") + + (client,) = cognito.clients.values() + self.assertEqual(client["ClientName"], "m2m.cupcake-cupcake-backend") + self.assertEqual(client["AllowedOAuthScopes"], ["moresleep-moresleep/talks.read"]) + self.assertEqual(client["AllowedOAuthFlows"], ["client_credentials"]) + self.assertEqual(ssm.params[f"{CALLER}/m2m-client-id"], client["ClientId"]) + self.assertIn((f"{CALLER}/m2m-client-secret", "SecureString"), ssm.puts) + + def test_caller_without_consent_gets_url_and_disabled_client_but_no_rule(self): + params = both_sides() + params[f"{CALLEE}/network"] = request("sg-0000000b", port=8082, scopes=SCOPES) + h, ssm, ec2, cognito = world(params) + h.handler({}, None) + self.assertEqual(ec2.authorized, []) + self.assertEqual(json.loads(ssm.params[f"{CALLER}/peers-status"])["calls"]["moresleep/moresleep"]["state"], + "pending-consent") + (client,) = cognito.clients.values() + self.assertFalse(client["AllowedOAuthFlowsUserPoolClient"]) + self.assertIn(f"{CALLER}/m2m-client-id", ssm.params) + self.assertEqual(ssm.params[f"{CALLER}/peer-moresleep-moresleep-scopes"], "none") + + def test_unknown_callee_gets_default_port_url(self): + params = {f"{CALLER}/network": request("sg-0000000a", calls={"moresleep/moresleep": []})} + h, ssm, ec2, cognito = world(params) + h.handler({}, None) + self.assertEqual(ssm.params[f"{CALLER}/peer-moresleep-moresleep-url"], + "http://moresleep-moresleep.javabin.internal:8000") + self.assertEqual(json.loads(ssm.params[f"{CALLER}/peers-status"])["calls"]["moresleep/moresleep"]["state"], + "callee-missing") + self.assertEqual(cognito.clients, {}) # network-only call: no client + + def test_consumer_waiting_for_caller_is_pending(self): + params = {f"{CALLEE}/network": request("sg-0000000b", consumers={"cupcake/cupcake-backend": []})} + h, ssm, *_ = world(params) + h.handler({}, None) + self.assertEqual(json.loads(ssm.params[f"{CALLEE}/peers-status"])["consumers"], + {"cupcake/cupcake-backend": "pending"}) + + +class RulesTest(unittest.TestCase): + def test_withdrawn_consent_revokes_only_managed_rules(self): + h, ssm, ec2, cognito = world(both_sides()) + ec2.add_rule("sg-0000000b", "sg-0000000a", 8082) # managed, now stale + ec2.add_rule("sg-0000000b", "sg-0000000c", 9999, managed=False) # the team's own rule + ssm.params[f"{CALLEE}/network"] = request("sg-0000000b", port=8082, scopes=SCOPES) + h.handler({}, None) + self.assertEqual(ec2.revoked, [("sg-0000000b", "sg-0000000a", 8082)]) + self.assertEqual(len(ec2.rules), 1) + + def test_port_change_replaces_the_rule(self): + h, ssm, ec2, cognito = world(both_sides(callee_port=9000)) + ec2.add_rule("sg-0000000b", "sg-0000000a", 8082) + h.handler({}, None) + self.assertEqual(ec2.revoked, [("sg-0000000b", "sg-0000000a", 8082)]) + self.assertEqual(ec2.authorized, [("sg-0000000b", "sg-0000000a", 9000)]) + + def test_rule_the_team_already_added_is_not_an_error(self): + h, ssm, ec2, cognito = world(both_sides()) + ec2.add_rule("sg-0000000b", "sg-0000000a", 8082, managed=False) + self.assertEqual(h.handler({}, None)["status"], "ok") + + def test_security_group_of_another_team_is_refused(self): + params = both_sides() + # moresleep claims cupcake's SG as its own to get it opened + params[f"{CALLEE}/network"] = request("sg-0000000c", port=8082, scopes=SCOPES, + consumers={"cupcake/cupcake-backend": []}) + h, ssm, ec2, cognito = world(params) + result = h.handler({}, None) + self.assertIn("moresleep/moresleep", result["invalid"]) + self.assertEqual(ec2.authorized, []) + self.assertIn("error", json.loads(ssm.params[f"{CALLEE}/peers-status"])) + + +class CognitoTest(unittest.TestCase): + def test_scope_removed_from_callee_is_dropped_from_client_first(self): + h, ssm, ec2, cognito = world(both_sides(consumer_scopes=("talks.read", "talks.write"))) + h.handler({}, None) + (client,) = cognito.clients.values() + self.assertEqual(len(client["AllowedOAuthScopes"]), 2) + + ssm.params[f"{CALLEE}/network"] = request("sg-0000000b", port=8082, scopes={"talks.read": "Read"}, + consumers={"cupcake/cupcake-backend": ["talks.read"]}) + cognito.log.clear() + self.assertEqual(h.handler({}, None)["status"], "ok") + self.assertEqual(client["AllowedOAuthScopes"], ["moresleep-moresleep/talks.read"]) + self.assertEqual(cognito.log[-1], ("update-rs", "moresleep-moresleep", ["talks.read"])) + + def test_callee_gone_deletes_server_after_client_lost_scopes(self): + h, ssm, ec2, cognito = world(both_sides()) + h.handler({}, None) + del ssm.params[f"{CALLEE}/network"] + self.assertEqual(h.handler({}, None)["status"], "ok") + self.assertEqual(cognito.servers, {}) + (client,) = cognito.clients.values() + self.assertFalse(client["AllowedOAuthFlowsUserPoolClient"]) + + def test_foreign_resource_server_is_left_alone(self): + h, ssm, ec2, cognito = world(both_sides()) + cognito.servers["moresleep-moresleep"] = {"Identifier": "moresleep-moresleep", "Name": "hand-made", "Scopes": []} + cognito.servers["other"] = {"Identifier": "other", "Name": "other", "Scopes": []} + result = h.handler({}, None) + self.assertEqual(result["status"], "error") + self.assertIn("other", cognito.servers) + self.assertEqual(cognito.servers["moresleep-moresleep"]["Name"], "hand-made") + + def test_caller_removed_deletes_client_and_parameters(self): + h, ssm, ec2, cognito = world(both_sides()) + h.handler({}, None) + del ssm.params[f"{CALLER}/network"] + h.handler({}, None) + self.assertEqual(cognito.clients, {}) + self.assertFalse([n for n in ssm.params if n.startswith(f"{CALLER}/")]) + self.assertEqual(ec2.rules, {}) + + +class ParametersTest(unittest.TestCase): + def test_second_run_writes_nothing(self): + h, ssm, ec2, cognito = world(both_sides()) + h.handler({}, None) + ssm.puts.clear() + cognito.log.clear() + h.handler({"action": "reconcile"}, None) + self.assertEqual(ssm.puts, []) + self.assertEqual(cognito.log, []) + + def test_invalid_request_keeps_existing_credentials(self): + h, ssm, ec2, cognito = world(both_sides()) + h.handler({}, None) + ssm.params[f"{CALLER}/network"] = "not json" + h.handler({}, None) + self.assertIn(f"{CALLER}/m2m-client-id", ssm.params) + self.assertIn(f"{CALLER}/peer-moresleep-moresleep-url", ssm.params) + self.assertEqual(len(cognito.clients), 1) + + def test_callee_narrowing_scopes_updates_the_scopes_parameter(self): + h, ssm, ec2, cognito = world(both_sides(consumer_scopes=("talks.read", "talks.write"))) + h.handler({}, None) + self.assertEqual(ssm.params[f"{CALLER}/peer-moresleep-moresleep-scopes"], + "moresleep-moresleep/talks.read moresleep-moresleep/talks.write") + ssm.params[f"{CALLEE}/network"] = request("sg-0000000b", port=8082, scopes=SCOPES, + consumers={"cupcake/cupcake-backend": ["talks.read"]}) + h.handler({}, None) + self.assertEqual(ssm.params[f"{CALLER}/peer-moresleep-moresleep-scopes"], "moresleep-moresleep/talks.read") + + def test_parse_rejects_bad_requests(self): + h, *_ = world({}) + bad = [ + "[]", + json.dumps({"version": 2}), + request("sg-zzz"), + request("sg-0000000a", port=0), + request("sg-0000000a", calls={"cupcake/cupcake-backend": []}), # self + request("sg-0000000a", calls={"Cupcake/x": []}), + request("sg-0000000a", calls={"a/b": ["bad scope"]}), + request("sg-0000000a", scopes={"a": "A"}, consumers={"x/y": ["b"]}), # not provided + ] + for raw in bad: + with self.assertRaises(h.InvalidRequest, msg=raw): + h.parse_request(raw, "cupcake", "cupcake-backend") + + +class FrozenTest(unittest.TestCase): + """An invalid request freezes the app; it does not cut off its peers.""" + + def test_invalid_callee_keeps_rule_server_and_caller_scopes(self): + h, ssm, ec2, cognito = world(both_sides()) + h.handler({}, None) + ssm.puts.clear() + ssm.params[f"{CALLEE}/network"] = "not json" + result = h.handler({}, None) + self.assertIn("moresleep/moresleep", result["invalid"]) + self.assertEqual(ec2.revoked, []) + self.assertEqual(len(ec2.rules), 1) + self.assertIn("moresleep-moresleep", cognito.servers) + (client,) = cognito.clients.values() + self.assertEqual(client["AllowedOAuthScopes"], ["moresleep-moresleep/talks.read"]) + self.assertEqual(ssm.params[f"{CALLER}/peer-moresleep-moresleep-url"], + "http://moresleep-moresleep.javabin.internal:8082") + self.assertEqual(ssm.params[f"{CALLER}/peer-moresleep-moresleep-scopes"], "moresleep-moresleep/talks.read") + self.assertEqual(json.loads(ssm.params[f"{CALLER}/peers-status"])["calls"]["moresleep/moresleep"]["state"], + "callee-invalid") + # Only the status parameters changed + self.assertEqual({n for n, _ in ssm.puts}, {f"{CALLER}/peers-status", f"{CALLEE}/peers-status"}) + + # Fixed again: nothing to redo + ssm.params[f"{CALLEE}/network"] = both_sides()[f"{CALLEE}/network"] + ec2.authorized.clear() + self.assertEqual(h.handler({}, None)["status"], "ok") + self.assertEqual(ec2.authorized, []) + + def test_valid_caller_can_still_withdraw_from_invalid_callee(self): + h, ssm, ec2, cognito = world(both_sides()) + h.handler({}, None) + ssm.params[f"{CALLEE}/network"] = "not json" + ssm.params[f"{CALLER}/network"] = request("sg-0000000a") + h.handler({}, None) + self.assertEqual(ec2.rules, {}) + self.assertEqual(cognito.clients, {}) + self.assertIn("moresleep-moresleep", cognito.servers) # frozen callee's own server stays + + def test_valid_callee_can_still_withdraw_from_invalid_caller(self): + h, ssm, ec2, cognito = world(both_sides()) + h.handler({}, None) + ssm.params[f"{CALLER}/network"] = "not json" + ssm.params[f"{CALLEE}/network"] = request("sg-0000000b", port=8082, scopes=SCOPES) + self.assertEqual(h.handler({}, None)["status"], "ok") + self.assertEqual(ec2.rules, {}) + (client,) = cognito.clients.values() # kept, but without the withdrawn scope + self.assertFalse(client["AllowedOAuthFlowsUserPoolClient"]) + + def test_invalid_caller_keeps_its_edge(self): + h, ssm, ec2, cognito = world(both_sides()) + h.handler({}, None) + ssm.params[f"{CALLER}/network"] = "not json" + h.handler({}, None) + self.assertEqual(ec2.revoked, []) + (client,) = cognito.clients.values() + self.assertEqual(client["AllowedOAuthScopes"], ["moresleep-moresleep/talks.read"]) + self.assertEqual(json.loads(ssm.params[f"{CALLEE}/peers-status"])["consumers"], + {"cupcake/cupcake-backend": "caller-invalid"}) + + def test_deleted_request_is_removed_for_real(self): + h, ssm, ec2, cognito = world(both_sides()) + h.handler({}, None) + del ssm.params[f"{CALLEE}/network"] + h.handler({}, None) + self.assertEqual(ec2.rules, {}) + self.assertEqual(cognito.servers, {}) + self.assertEqual(ssm.params[f"{CALLER}/peer-moresleep-moresleep-scopes"], "none") + + +class AuditTest(unittest.TestCase): + def test_reconcile_reports_squatting_and_borrowed_groups(self): + sd = FakeServiceDiscovery([ + ("moresleep-moresleep", "cupcake", ["moresleep-moresleep"]), # wrong team tag + ("cupcake-cupcake", "cupcake", ["evil-thing"]), # foreign registration + ("cupcake-cupcake-backend", "cupcake", ["cupcake-cupcake-backend"]), + ]) + ecs = FakeECS([("evil-thing", "eni-1"), ("cupcake-cupcake-backend", "eni-2")]) + h, ssm, ec2, cognito = world({}, sd=sd, ecs=ecs) + ec2.describe_network_interfaces = lambda NetworkInterfaceIds: {"NetworkInterfaces": [ + {"NetworkInterfaceId": "eni-1", "Groups": [{"GroupId": "sg-0000000a"}]}, + {"NetworkInterfaceId": "eni-2", "Groups": [{"GroupId": "sg-0000000a"}]}, + ]} + findings = h.handler({"action": "reconcile"}, None)["findings"] + self.assertEqual(len(findings), 3, findings) + self.assertTrue(any("evil-thing" in f and "security group" in f for f in findings)) + + def test_reconcile_reports_records_cloud_map_did_not_write(self): + sd = FakeServiceDiscovery([("cupcake-cupcake", "cupcake", ["cupcake-cupcake@10.0.1.5"])]) + r53 = FakeRoute53([ + ("javabin.internal", "SOA", ["x"]), ("javabin.internal", "NS", ["x"]), + ("cupcake-cupcake.javabin.internal", "A", ["10.0.1.5"]), # fine + ("cupcake-cupcake.javabin.internal", "A", ["10.0.9.9"]), # hijacked + ("moresleep-moresleep.javabin.internal", "A", ["10.0.9.9"]), # no service + ("x.javabin.internal", "CNAME", ["evil.example.com"]), + ]) + h, *_ = world({}, sd=sd, r53=r53) + findings = h.handler({"action": "reconcile"}, None)["findings"] + self.assertEqual(len(findings), 3, findings) + self.assertTrue(any("10.0.9.9" in f and "cupcake-cupcake" in f for f in findings)) + + def test_audits_only_run_on_reconcile(self): + sd = FakeServiceDiscovery([("moresleep-moresleep", "cupcake", [])]) + h, *_ = world({}, sd=sd) + self.assertEqual(h.handler({}, None)["findings"], []) + + +if __name__ == "__main__": + unittest.main() diff --git a/terraform/lambda-src/tests/test_app_secret_rollout.py b/terraform/lambda-src/tests/test_app_secret_rollout.py index 906b334..010eb8c 100644 --- a/terraform/lambda-src/tests/test_app_secret_rollout.py +++ b/terraform/lambda-src/tests/test_app_secret_rollout.py @@ -55,7 +55,8 @@ def test_platform_managed_and_non_update_events_are_ignored(self): # A new secret always comes with a new ID; the ID's Update restarts "/javabin/apps/cupcake/frosting/cognito-client-secret", "/javabin/apps/core/api/db-master-password", "/javabin/platform/github-app-key", - "/javabin/apps/cupcake/frosting/login-1/cognito-client-id"): + "/javabin/apps/cupcake/frosting/login-1/cognito-client-id", + "/javabin/apps/cupcake/frosting/network", "/javabin/apps/cupcake/frosting/peers-status"): self.assertEqual(h.handler(event(name), None)["status"], "ignored") self.assertEqual(h.handler(event("/javabin/apps/cupcake/frosting/x", "Create"), None)["status"], "ignored") self.assertEqual(ecs.calls, []) @@ -68,6 +69,13 @@ def test_cognito_client_changes_restart_the_app(self): self.assertEqual(result["status"], "restarted", param) self.assertEqual(ecs.calls, [("javabin-platform", "cupcake-cupcake", True)]) + def test_peer_url_and_m2m_credentials_restart_the_caller(self): + ecs = FakeECS() + h = load(ecs) + for name in ("/javabin/apps/cupcake/frosting/peer-moresleep-moresleep-url", + "/javabin/apps/cupcake/frosting/m2m-client-secret"): + self.assertEqual(h.handler(event(name), None)["status"], "restarted") + def test_missing_service_is_not_an_error(self): ecs = FakeECS(missing={"cupcake-frosting"}) self.assertEqual(load(ecs).handler(event("/javabin/apps/cupcake/frosting/key"), None)["status"], "no-service") diff --git a/terraform/modules/app-network/main.tf b/terraform/modules/app-network/main.tf new file mode 100644 index 0000000..9d0c2f1 --- /dev/null +++ b/terraform/modules/app-network/main.tf @@ -0,0 +1,140 @@ +################################################################################ +# App Network — the app's own security group, private DNS name and its +# app-to-app request (app.yaml `calls:` / `provides:`). +# +# Emitted for every app. Resources here are owned by the app's team: +# +# - Security group {team}-{name}-tasks for the app's ECS tasks. The ALB may +# reach the app port (when routed); nothing else may, until a peer rule +# is added. No inline rules, so Terraform never strips peer rules. +# - Cloud Map service {team}-{name} in the shared private namespace +# {project}.internal, so peers reach http://{team}-{name}.{project}.internal:{port} +# from inside the VPC. +# - SSM parameter /{project}/apps/{team}/{name}/network with the app's SG, +# port, calls and provides. Only this team can write under its path +# (developer boundary DenyBadName), so the path is the app's identity. +# +# The platform app-peers-sync Lambda reads every app's `network` parameter +# and, for each edge both sides declared (caller `calls` the callee and the +# callee lists the caller in `consumers`): +# +# - adds an ingress rule on the callee's SG from the caller's SG, on the +# callee's port; +# - for scopes, keeps a Cognito resource server {team}-{name} for the callee +# and a client-credentials client m2m.{team}-{name} for the caller in the +# internal pool; +# - writes into the caller's own path: +# peer-{t}-{a}-url (String) http://{t}-{a}.{project}.internal:{port} +# peer-{t}-{a}-scopes (String) granted scopes ("none" if none), calls with scopes +# m2m-client-id (String) +# m2m-client-secret (SecureString) +# peers-status (String JSON) active / pending-consent / callee-missing / callee-invalid +# +# Those parameter ARNs are deterministic, so the task definition references +# them up front (outputs below); deriving them from the request parameter's +# ARN makes the task definition wait for the request. +################################################################################ + +data "aws_region" "current" {} + +data "aws_security_group" "alb" { + count = var.alb_ingress ? 1 : 0 + + filter { + name = "group-name" + values = ["${var.project}-alb-sg"] + } + filter { + name = "vpc-id" + values = [var.vpc_id] + } +} + +data "aws_service_discovery_dns_namespace" "internal" { + name = "${var.project}.internal" + type = "DNS_PRIVATE" +} + +data "aws_cognito_user_pools" "internal" { + count = local.uses_scopes ? 1 : 0 + name = var.internal_pool_name +} + +locals { + region = data.aws_region.current.name + uses_scopes = length(var.provides_scopes) > 0 || anytrue([for s in values(var.calls) : length(s) > 0]) + pool_id = local.uses_scopes ? tolist(data.aws_cognito_user_pools.internal[0].ids)[0] : "" + issuer_url = local.uses_scopes ? "https://cognito-idp.${local.region}.amazonaws.com/${local.pool_id}" : "" + + # //apps//, as an ARN prefix + param_arn_prefix = trimsuffix(aws_ssm_parameter.network.arn, "/network") +} + +resource "aws_security_group" "tasks" { + name = "${var.team}-${var.name}-tasks" + description = "ECS tasks of ${var.team}-${var.name}; peer rules are managed by ${var.project}-app-peers-sync" + vpc_id = var.vpc_id + revoke_rules_on_delete = true + + tags = { + Name = "${var.team}-${var.name}-tasks" + # Lets app-peers-sync add peer rules to this group (its IAM condition) + "javabin-network" = "app" + } +} + +resource "aws_vpc_security_group_ingress_rule" "alb" { + count = var.alb_ingress ? 1 : 0 + + security_group_id = aws_security_group.tasks.id + referenced_security_group_id = data.aws_security_group.alb[0].id + from_port = var.port + to_port = var.port + ip_protocol = "tcp" + description = "ALB to the app port" +} + +resource "aws_vpc_security_group_egress_rule" "all" { + security_group_id = aws_security_group.tasks.id + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = "-1" + description = "All outbound" +} + +resource "aws_service_discovery_service" "this" { + name = "${var.team}-${var.name}" + + dns_config { + namespace_id = data.aws_service_discovery_dns_namespace.internal.id + routing_policy = "MULTIVALUE" + + dns_records { + ttl = 10 + type = "A" + } + } + + # ECS reports task health, so only healthy tasks are returned + health_check_custom_config { + failure_threshold = 1 + } + + # Deregister leftover instances instead of failing a destroy + force_destroy = true +} + +resource "aws_ssm_parameter" "network" { + name = "/${var.project}/apps/${var.team}/${var.name}/network" + description = "App-to-app request for ${var.team}-${var.name} — read by the platform app-peers-sync Lambda" + type = "String" + value = jsonencode({ + version = 1 + security_group_id = aws_security_group.tasks.id + port = var.port + calls = var.calls + provides = { + scopes = var.provides_scopes + consumers = var.consumers + } + }) +} diff --git a/terraform/modules/app-network/outputs.tf b/terraform/modules/app-network/outputs.tf new file mode 100644 index 0000000..34ce46a --- /dev/null +++ b/terraform/modules/app-network/outputs.tf @@ -0,0 +1,64 @@ +output "security_group_id" { + description = "The app's task security group" + value = aws_security_group.tasks.id + # The ECS service switches to this group only once the ALB can reach it + depends_on = [aws_vpc_security_group_ingress_rule.alb, aws_vpc_security_group_egress_rule.all] +} + +output "registry_arn" { + description = "Cloud Map service ARN for the ECS service's service_registries" + value = aws_service_discovery_service.this.arn +} + +output "dns_name" { + description = "Private DNS name peers use to reach this app" + value = "${var.team}-${var.name}.${var.project}.internal" +} + +output "request_parameter_name" { + description = "SSM parameter holding the app-to-app request" + value = aws_ssm_parameter.network.name +} + +output "peer_url_arns" { + description = "\"team/app\" => ARN of the parameter app-peers-sync writes the peer's URL to" + value = { for k in keys(var.calls) : k => "${local.param_arn_prefix}/peer-${replace(k, "/", "-")}-url" } +} + +output "peer_scopes_arns" { + description = "\"team/app\" => ARN of the parameter app-peers-sync writes the granted scopes to (calls with scopes only)" + value = { + for k, scopes in var.calls : k => "${local.param_arn_prefix}/peer-${replace(k, "/", "-")}-scopes" + if length(scopes) > 0 + } +} + +output "m2m_client_id_arn" { + description = "ARN of the parameter app-peers-sync writes the M2M client ID to" + value = "${local.param_arn_prefix}/m2m-client-id" +} + +output "m2m_client_secret_arn" { + description = "ARN of the parameter app-peers-sync writes the M2M client secret to" + value = "${local.param_arn_prefix}/m2m-client-secret" +} + +output "token_url" { + description = "OAuth2 token endpoint of the internal pool (client_credentials)" + value = "https://${var.internal_pool_domain}.auth.${local.region}.amazoncognito.com/oauth2/token" +} + +output "issuer_url" { + description = "Issuer of M2M access tokens (empty when the app uses no scopes)" + value = local.issuer_url +} + +output "jwks_url" { + description = "JWKS URL for verifying M2M access tokens (empty when the app uses no scopes)" + value = local.uses_scopes ? "${local.issuer_url}/.well-known/jwks.json" : "" +} + +output "scope_prefix" { + description = "Prefix of this app's scopes in access tokens" + value = "${var.team}-${var.name}/" +} diff --git a/terraform/modules/app-network/variables.tf b/terraform/modules/app-network/variables.tf new file mode 100644 index 0000000..ddc4201 --- /dev/null +++ b/terraform/modules/app-network/variables.tf @@ -0,0 +1,62 @@ +variable "name" { + description = "App name" + type = string +} + +variable "team" { + description = "Owning team" + type = string +} + +variable "project" { + description = "Platform project name (resource and parameter prefix)" + type = string + default = "javabin" +} + +variable "vpc_id" { + description = "Platform VPC ID" + type = string +} + +variable "port" { + description = "Container port; the ALB and peers reach the app on it" + type = number + default = 8000 +} + +variable "alb_ingress" { + description = "Allow the platform ALB to reach the app port (the app has routing)" + type = bool + default = false +} + +variable "calls" { + description = "Apps this app calls: \"team/app\" => scopes requested (empty list for network-only)" + type = map(list(string)) + default = {} +} + +variable "provides_scopes" { + description = "Scopes this app offers to callers: scope => description" + type = map(string) + default = {} +} + +variable "consumers" { + description = "Apps allowed to call this app: \"team/app\" => scopes allowed (empty list for network-only)" + type = map(list(string)) + default = {} +} + +variable "internal_pool_name" { + description = "Name of the internal Cognito user pool (M2M tokens are issued there)" + type = string + default = "javabin-internal" +} + +variable "internal_pool_domain" { + description = "Cognito hosted-domain prefix of the internal pool (its OAuth2 token endpoint). Today the same as the pool name; change both here if the domain ever changes" + type = string + default = "javabin-internal" +} diff --git a/terraform/modules/ecs-service/main.tf b/terraform/modules/ecs-service/main.tf index f8e89f7..e5b4416 100644 --- a/terraform/modules/ecs-service/main.tf +++ b/terraform/modules/ecs-service/main.tf @@ -100,6 +100,14 @@ resource "aws_ecs_service" "this" { } } + # Cloud Map registration (app-network), so peers reach the tasks privately + dynamic "service_registries" { + for_each = var.service_registry_arns + content { + registry_arn = service_registries.value + } + } + # Roll back a deployment whose tasks never become healthy, instead of # leaving ecs-deploy waiting for services-stable to time out. deployment_circuit_breaker { diff --git a/terraform/modules/ecs-service/variables.tf b/terraform/modules/ecs-service/variables.tf index 7a67ee6..bc7d82e 100644 --- a/terraform/modules/ecs-service/variables.tf +++ b/terraform/modules/ecs-service/variables.tf @@ -80,6 +80,17 @@ variable "target_group_arn" { default = "" } +variable "service_registry_arns" { + description = "Cloud Map service ARNs to register the tasks in (at most one; from app-network)" + type = list(string) + default = [] + + validation { + condition = length(var.service_registry_arns) <= 1 + error_message = "ECS Fargate services support at most one service registry." + } +} + variable "desired_count" { description = "Number of tasks to run" type = number diff --git a/terraform/platform/iam/boundary.tf b/terraform/platform/iam/boundary.tf index 8ef55ec..f9ccf9b 100644 --- a/terraform/platform/iam/boundary.tf +++ b/terraform/platform/iam/boundary.tf @@ -86,10 +86,11 @@ locals { Action = [ "ec2:CreateVpc", "ec2:DeleteVpc", "ec2:ModifyVpcAttribute", "ec2:CreateSubnet", "ec2:DeleteSubnet", - "ec2:CreateInternetGateway", "ec2:DeleteInternetGateway", - "ec2:AttachInternetGateway", "ec2:DetachInternetGateway", - "ec2:CreateNatGateway", "ec2:DeleteNatGateway", - "ec2:CreateRouteTable", "ec2:DeleteRouteTable" + # Create/Delete/Attach/Detach (also egress-only IGWs); saves bytes + "ec2:*InternetGateway", "ec2:*NatGateway", + "ec2:CreateRouteTable", "ec2:DeleteRouteTable", + # Teams only write records; zones are platform-managed + "route53:DeleteHostedZone", ] Resource = "*" }, @@ -106,12 +107,13 @@ locals { "s3:PutBucketVersioning", "s3:PutEncryptionConfiguration", "dynamodb:DeleteTable", "dynamodb:UpdateTable" ] + # Single-region account: *:* region/account saves bytes Resource = [ - "arn:aws:ecs:${var.region}:${var.aws_account_id}:cluster/${var.project}-platform", - "arn:aws:elasticloadbalancing:${var.region}:${var.aws_account_id}:loadbalancer/app/${var.project}-*", + "arn:aws:ecs:*:*:cluster/${var.project}-platform", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/${var.project}-*", "arn:aws:s3:::${var.project}-terraform-*", "arn:aws:s3:::${var.project}-ci-*", - "arn:aws:dynamodb:${var.region}:${var.aws_account_id}:table/${var.project}-terraform-*", + "arn:aws:dynamodb:*:*:table/${var.project}-terraform-*", ] }, { @@ -149,9 +151,11 @@ locals { "ecs:DeleteCluster", "ecs:UpdateCluster", "ecs:UpdateClusterSettings", "iam:DeleteRole", "iam:UpdateRole", "iam:DeleteRolePolicy", "iam:UpdateAssumeRolePolicy", "iam:PutRolePolicy", "iam:AttachRolePolicy", "iam:DetachRolePolicy", - "route53:DeleteHostedZone", "sns:DeleteTopic", "sns:SetTopicAttributes", + "sns:DeleteTopic", "sns:SetTopicAttributes", # Shared SGs (ECS tasks, ALB): a team rule here opens or cuts every app "ec2:AuthorizeSecurityGroup*", "ec2:RevokeSecurityGroup*", + # Shared Cloud Map namespace for app-to-app DNS + "servicediscovery:Delete*", ] Resource = "*" Condition = { StringEquals = { "aws:ResourceTag/team" = "shared" } } @@ -167,6 +171,7 @@ locals { "elasticloadbalancing:AddTags", "elasticloadbalancing:RemoveTags", "iam:TagRole", "iam:UntagRole", "sns:TagResource", "sns:UntagResource", + "servicediscovery:TagResource", "servicediscovery:UntagResource", ] Resource = "*" Condition = { StringEquals = { "aws:ResourceTag/team" = ["shared", "platform"] } } @@ -253,6 +258,17 @@ locals { "arn:aws:states:*:*:stateMachine:$${aws:PrincipalTag/team}-*", ] }, + # -- APP-TO-APP DNS (last, so plan diffs show an append) -- + { + # App-to-app private zone: only Cloud Map (its service role) writes it. + # Route53 ignores tag conditions, hence the record-name condition; the + # last VPC of a private zone cannot be disassociated. + Sid = "DenyAppDNS" + Effect = "Deny" + Action = "route53:ChangeResourceRecordSets" + Resource = "*" + Condition = { "ForAnyValue:StringLike" = { "route53:ChangeResourceRecordSetsNormalizedRecordNames" = "*${var.project}.internal" } } + }, ] }) } diff --git a/terraform/platform/lambdas/log-groups.tf b/terraform/platform/lambdas/log-groups.tf index c5bdb3e..51a4937 100644 --- a/terraform/platform/lambdas/log-groups.tf +++ b/terraform/platform/lambdas/log-groups.tf @@ -13,6 +13,7 @@ locals { platform_lambda_names = toset([ "app-auth-sync", + "app-peers-sync", "app-secret-rollout", "apply-gate", "budget-enforcer", diff --git a/terraform/platform/lambdas/main.tf b/terraform/platform/lambdas/main.tf index 7c3e2ba..3a76360 100644 --- a/terraform/platform/lambdas/main.tf +++ b/terraform/platform/lambdas/main.tf @@ -2006,3 +2006,244 @@ resource "aws_lambda_permission" "app_secret_rollout_eventbridge" { principal = "events.amazonaws.com" source_arn = aws_cloudwatch_event_rule.app_secret_rollout_trigger.arn } + +################################################################################ +# App Peers Sync — app-to-app calls declared in app.yaml (calls / provides) +# +# App Terraform (app-network module) writes //apps///network +# with the app's security group, port, calls and consumers. The parameter path +# is the app's identity (only that team can write it). For every edge both +# sides declared, this Lambda adds the security group rule, keeps the Cognito +# resource servers and M2M clients in the internal pool, and writes the peer +# URLs and client credentials into the caller's own path. +# See terraform/lambda-src/app_peers_sync/handler.py. +################################################################################ + +data "archive_file" "app_peers_sync" { + type = "zip" + output_path = "${path.module}/builds/app_peers_sync.zip" + output_file_mode = "0644" + + source { + content = file("${local.lambda_src_path}/app_peers_sync/handler.py") + filename = "handler.py" + } + source { + content = file("${local.lambda_src_path}/shared/__init__.py") + filename = "shared/__init__.py" + } + source { + content = file("${local.lambda_src_path}/shared/slack.py") + filename = "shared/slack.py" + } +} + +resource "aws_iam_role" "app_peers_sync" { + name = "${var.project}-app-peers-sync" + # The org boundary denies creating any role without one + permissions_boundary = var.org_boundary_arn + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "lambda.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_role_policy" "app_peers_sync" { + name = "${var.project}-app-peers-sync" + role = aws_iam_role.app_peers_sync.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "ReadAppRequests" + Effect = "Allow" + Action = ["ssm:GetParameter", "ssm:GetParametersByPath"] + Resource = [ + "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/${var.project}/apps", + "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/${var.project}/apps/*", + ] + }, + { + Sid = "WritePeerParameters" + Effect = "Allow" + Action = [ + "ssm:PutParameter", + "ssm:DeleteParameter", + "ssm:DeleteParameters", + "ssm:AddTagsToResource", + ] + Resource = [ + "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/${var.project}/apps/*/*/peer-*", + "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/${var.project}/apps/*/*/m2m-client-*", + "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/${var.project}/apps/*/*/peers-status", + ] + }, + { + Sid = "ReadSlackWebhook" + Effect = "Allow" + Action = "ssm:GetParameter" + Resource = "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter/${var.project}/slack/platform-resource-alerts-webhook" + }, + { + # Only app task groups (app-network tags them) in the platform VPC + Sid = "EditAppSecurityGroups" + Effect = "Allow" + Action = ["ec2:AuthorizeSecurityGroupIngress", "ec2:RevokeSecurityGroupIngress"] + Resource = "arn:aws:ec2:${var.region}:${var.aws_account_id}:security-group/*" + Condition = { + StringEquals = { "aws:ResourceTag/javabin-network" = "app" } + ArnEquals = { "ec2:Vpc" = var.vpc_arn } + } + }, + { + Sid = "EditPeerRules" + Effect = "Allow" + Action = ["ec2:AuthorizeSecurityGroupIngress", "ec2:RevokeSecurityGroupIngress"] + Resource = "arn:aws:ec2:${var.region}:${var.aws_account_id}:security-group-rule/*" + }, + { + Sid = "TagPeerRulesOnCreate" + Effect = "Allow" + Action = "ec2:CreateTags" + Resource = "arn:aws:ec2:${var.region}:${var.aws_account_id}:security-group-rule/*" + Condition = { + StringEquals = { "ec2:CreateAction" = "AuthorizeSecurityGroupIngress" } + } + }, + { + # EC2 Describe*, Cloud Map List* and ecs:ListTasks have no resource-level permissions + Sid = "Describe" + Effect = "Allow" + Action = [ + "ec2:DescribeSecurityGroups", + "ec2:DescribeSecurityGroupRules", + "ec2:DescribeNetworkInterfaces", + "servicediscovery:ListServices", + "servicediscovery:ListInstances", + "servicediscovery:ListTagsForResource", + "ecs:ListTasks", + ] + Resource = "*" + }, + { + # Read-only; the zone ID is only known after apply, so keep this + # statement readable in the plan + Sid = "AuditPrivateZone" + Effect = "Allow" + Action = "route53:ListResourceRecordSets" + Resource = "arn:aws:route53:::hostedzone/*" + }, + { + Sid = "DescribeAppTasks" + Effect = "Allow" + Action = "ecs:DescribeTasks" + Resource = "arn:aws:ecs:${var.region}:${var.aws_account_id}:task/${var.project}-platform/*" + }, + { + Sid = "ManageM2M" + Effect = "Allow" + Action = [ + "cognito-idp:CreateResourceServer", + "cognito-idp:UpdateResourceServer", + "cognito-idp:DeleteResourceServer", + "cognito-idp:DescribeResourceServer", + "cognito-idp:ListResourceServers", + "cognito-idp:CreateUserPoolClient", + "cognito-idp:UpdateUserPoolClient", + "cognito-idp:DeleteUserPoolClient", + "cognito-idp:DescribeUserPoolClient", + "cognito-idp:ListUserPoolClients", + ] + Resource = var.internal_user_pool_arn + }, + ] + }) +} + +resource "aws_iam_role_policy_attachment" "app_peers_sync_logs" { + role = aws_iam_role.app_peers_sync.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +resource "aws_lambda_function" "app_peers_sync" { + function_name = "${var.project}-app-peers-sync" + role = aws_iam_role.app_peers_sync.arn + handler = "handler.handler" + runtime = "python3.12" + timeout = 120 + memory_size = 128 + filename = data.archive_file.app_peers_sync.output_path + source_code_hash = data.archive_file.app_peers_sync.output_base64sha256 + + # One sync at a time: every run recomputes everything, so runs must not race + reserved_concurrent_executions = 1 + + environment { + variables = { + PROJECT = var.project + VPC_ID = var.vpc_id + INTERNAL_POOL_ID = var.internal_user_pool_id + NAMESPACE_ID = var.service_discovery_namespace_id + HOSTED_ZONE_ID = var.service_discovery_zone_id + CLUSTER = "${var.project}-platform" + INFRA_WEBHOOK_PARAM = "/${var.project}/slack/platform-resource-alerts-webhook" + } + } +} + +resource "aws_cloudwatch_event_rule" "app_peers_sync_trigger" { + name = "${var.project}-app-peers-sync-trigger" + description = "app.yaml calls/provides request created, changed or deleted" + + event_pattern = jsonencode({ + source = ["aws.ssm"] + detail-type = ["Parameter Store Change"] + detail = { + name = [{ wildcard = "/${var.project}/apps/*/*/network" }] + operation = ["Create", "Update", "Delete"] + } + }) +} + +resource "aws_cloudwatch_event_target" "app_peers_sync" { + rule = aws_cloudwatch_event_rule.app_peers_sync_trigger.name + target_id = "invoke-app-peers-sync" + arn = aws_lambda_function.app_peers_sync.arn +} + +resource "aws_lambda_permission" "app_peers_sync_eventbridge" { + statement_id = "AllowEventBridge" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.app_peers_sync.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.app_peers_sync_trigger.arn +} + +# Daily safety net plus the ownership audits (Cloud Map names, private zone +# records, borrowed SGs) +resource "aws_cloudwatch_event_rule" "app_peers_sync_reconcile" { + name = "${var.project}-app-peers-sync-reconcile" + description = "Daily reconcile and audit of app-to-app calls" + schedule_expression = "cron(45 3 * * ? *)" +} + +resource "aws_cloudwatch_event_target" "app_peers_sync_reconcile" { + rule = aws_cloudwatch_event_rule.app_peers_sync_reconcile.name + target_id = "reconcile-app-peers-sync" + arn = aws_lambda_function.app_peers_sync.arn + input = jsonencode({ action = "reconcile" }) +} + +resource "aws_lambda_permission" "app_peers_sync_reconcile" { + statement_id = "AllowEventBridgeReconcile" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.app_peers_sync.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.app_peers_sync_reconcile.arn +} diff --git a/terraform/platform/lambdas/variables.tf b/terraform/platform/lambdas/variables.tf index 974b4d0..b7ca998 100644 --- a/terraform/platform/lambdas/variables.tf +++ b/terraform/platform/lambdas/variables.tf @@ -164,3 +164,25 @@ variable "atlassian_sync_apply" { type = bool default = false } + +# --- App-to-app (app-peers-sync) --- + +variable "vpc_id" { + description = "Platform VPC ID (app-peers-sync only edits app security groups in it)" + type = string +} + +variable "vpc_arn" { + description = "Platform VPC ARN (IAM condition for app-peers-sync security group edits)" + type = string +} + +variable "service_discovery_namespace_id" { + description = "Private app-to-app Cloud Map namespace ID (audited by app-peers-sync)" + type = string +} + +variable "service_discovery_zone_id" { + description = "Route53 private zone of the app-to-app namespace (app-peers-sync audits its records)" + type = string +} diff --git a/terraform/platform/main.tf b/terraform/platform/main.tf index 5e860b6..7775ea9 100644 --- a/terraform/platform/main.tf +++ b/terraform/platform/main.tf @@ -5,7 +5,7 @@ # GitHub Actions → plan → LLM review → apply # # Sub-modules: -# networking VPC, subnets, NAT, security groups +# networking VPC, subnets, NAT, security groups, app-to-app DNS namespace # ingress ALB, ACM wildcard certs (*.javazone.no, *.java.no, *.javabin.no) # iam GitHub OIDC, CI roles, permission boundary, ECS execution role # compute ECS cluster, public ECR images @@ -110,6 +110,10 @@ module "lambdas" { athena_workgroup = module.cost_analytics.athena_workgroup_name athena_results_bucket_arn = module.cost_analytics.athena_results_bucket_arn cur_data_bucket_arn = module.cost_analytics.cur_data_bucket_arn + vpc_id = module.networking.vpc_id + vpc_arn = module.networking.vpc_arn + service_discovery_namespace_id = module.networking.service_discovery_namespace_id + service_discovery_zone_id = module.networking.service_discovery_hosted_zone_id } module "identity" { diff --git a/terraform/platform/networking/main.tf b/terraform/platform/networking/main.tf index 9924e62..c5f9144 100644 --- a/terraform/platform/networking/main.tf +++ b/terraform/platform/networking/main.tf @@ -239,3 +239,25 @@ resource "aws_vpc_security_group_egress_rule" "ecs_all" { cidr_ipv4 = "0.0.0.0/0" ip_protocol = "-1" } + +################################################################################ +# Private DNS for app-to-app calls (app.yaml calls/provides) +# +# Every app registers {team}-{app}..internal here (app-network +# module). team=shared lets every team create its Cloud Map service in it; +# the developer boundary denies deleting or retagging the namespace. The +# namespace also creates a Route53 private hosted zone; Route53 ignores tag +# conditions, so the boundary denies record changes under .internal +# by record name (DenyAppDNS) and hosted-zone deletion outright. +################################################################################ + +resource "aws_service_discovery_private_dns_namespace" "internal" { + name = "${var.project}.internal" + description = "Private app-to-app DNS (app.yaml calls/provides)" + vpc = aws_vpc.main.id + + tags = { + Name = "${var.project}.internal" + team = "shared" + } +} diff --git a/terraform/platform/networking/outputs.tf b/terraform/platform/networking/outputs.tf index 24c2048..af72b2c 100644 --- a/terraform/platform/networking/outputs.tf +++ b/terraform/platform/networking/outputs.tf @@ -22,3 +22,28 @@ output "ecs_tasks_sg_id" { description = "Security group ID for ECS tasks" value = aws_security_group.ecs_tasks.id } + +output "vpc_arn" { + description = "ARN of the VPC" + value = aws_vpc.main.arn +} + +output "service_discovery_namespace_id" { + description = "ID of the private app-to-app Cloud Map namespace" + value = aws_service_discovery_private_dns_namespace.internal.id +} + +output "service_discovery_namespace_arn" { + description = "ARN of the private app-to-app Cloud Map namespace" + value = aws_service_discovery_private_dns_namespace.internal.arn +} + +output "service_discovery_namespace_name" { + description = "Name of the private app-to-app Cloud Map namespace" + value = aws_service_discovery_private_dns_namespace.internal.name +} + +output "service_discovery_hosted_zone_id" { + description = "Route53 private hosted zone Cloud Map created for the namespace (protected in the developer boundary)" + value = aws_service_discovery_private_dns_namespace.internal.hosted_zone +}