test(otel): cover replay and chained execution

.github/scripts/tests/test_opentelemetry_conformance_workflow.py

@@ -1,8 +1,12 @@
+import re
 from pathlib import Path
 
 import yaml
 
 
+SHARED_WORKFLOW_REF = "main"
+CONFORMANCE_TEST_REF = "main"
+
 WORKFLOW_PATH = (
     Path(__file__).parents[2] / "workflows" / "opentelemetry-conformance-tests.yml"
 )
@@ -18,7 +22,10 @@ def test_opentelemetry_conformance_caller_uses_current_workflow_contract() -> No
         "uses: aws/aws-durable-execution-conformance-tests/.github/workflows/"
         "opentelemetry-orchestrator.yml@"
     )
-    assert orchestrator in workflow
+    shared_ref = re.search(re.escape(orchestrator) + r"(\S+)", workflow)
+    assert shared_ref is not None
+    assert shared_ref.group(1) == SHARED_WORKFLOW_REF
+    assert f"default: {CONFORMANCE_TEST_REF}" in workflow
     assert "python-opentelemetry.yml@" not in workflow
     assert "\n      otlp_endpoint:" not in workflow
 
@@ -28,7 +35,7 @@ def test_opentelemetry_conformance_caller_uses_current_workflow_contract() -> No
         "resource_prefix: p",
         "sdk_repository: aws/aws-durable-execution-sdk-python",
         "sdk_ref: ${{ github.event.pull_request.head.sha || github.sha }}",
-        "conformance_test_ref: ${{ inputs.conformance_test_ref || 'main' }}",
+        f"conformance_test_ref: ${{{{ inputs.conformance_test_ref || '{CONFORMANCE_TEST_REF}' }}}}",
         "checkout_sdk: true",
         f"examples_dir: {EXAMPLES_DIR}",
         "adot_release_repository: aws-observability/aws-otel-python-instrumentation",

.github/workflows/opentelemetry-conformance-tests.yml

@@ -66,7 +66,7 @@ jobs:
       actions: write
       contents: read
       id-token: write
-    uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@d4b7df927ea7eb7022bcca339d2cddd387d3d650
+    uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@main
     with:
       language: python
       runs_on: codebuild-github-actions-runner-${{ github.run_id }}-${{ github.run_attempt }}

packages/aws-durable-execution-sdk-python-conformance-tests-otel/README.md

@@ -12,7 +12,7 @@ language-agnostic OTel conformance runner in
 which invokes each function, collects its spans from the configured backend, and
 asserts they match the shared requirement specification. That repository owns the
 runner, the requirement specifications, and the orchestration (backend matrix,
-ADOT layer resolution, collector build, long-running cycle); this package owns
+ADOT layer resolution, collector build); this package owns
 the Python handlers and SAM templates.
 
 This mirrors the non-OTel
@@ -33,7 +33,7 @@ template-long-running.yaml  # otel-long-running suite
 tests/                      # contract tests for the templates and handlers
 ```
 
-The 20 invocation and 20 execution requirements reuse the same scenario
+The invocation and execution requirements reuse the same scenario
 handlers; the view is selected per function through the `OTEL_PLUGIN_MODE`
 environment variable, which `common.otel_plugin_factory()` reads to pick
 `InvocationOtelPluginFactory` or `ExecutionOtelPluginFactory`. `template.yaml`
@@ -63,6 +63,13 @@ deploys only the view named by its `OtelSuite` parameter.
 | `otel-invocation-18` | `otel_18_chained_invoke_failure.handler` | Verifies failed chained-invoke telemetry. |
 | `otel-invocation-19` | `otel_19_execution_failure.handler` | Verifies telemetry for a direct handler failure. |
 | `otel-invocation-20` | `otel_20_virtual_context.handler` | Verifies a virtual child-context span without context checkpoints. |
+| `otel-invocation-21` | `otel_21_completed_step_replay.handler` | Local coverage verifies completed steps are skipped after wait/resume; cloud coverage is deferred. |
+| `otel-invocation-22` | `otel_22_user_function_context.handler` | Creates ordinary user spans under the active handler, attempt, child, branch, and iteration contexts. |
+| `otel-invocation-23` | `otel_23_callback_function_context.handler` | Verifies retry/check attempts, callback submitter, wrapped retry helper, and virtual-child callback parents. |
+| `otel-invocation-24` | `otel_24_invocation_retry_status.handler` | Raises a retryable invocation error after a completed step, then resumes with its saved result. |
+| `otel-invocation-25` | `otel_25_caught_callback_failure_replay.handler` | Catches a callback failure without diagnostic details, then replays it through two callback barriers. |
+| `otel-invocation-26` | `otel_26_external_callback_completion_replay.handler` | Exports an external callback completion before the observed step and preserves its export count across two later barriers. |
+| `otel-invocation-27` | `otel_27_chained_invoke_suspend.handler` | Verifies caller and child invocation spans when the child suspends for a durable wait and resumes. |
 | `otel-execution-1` | `otel_1_success.handler` | Verifies the execution-view workflow, step, and attempt hierarchy. |
 | `otel-execution-2` | `otel_2_wait_resume.handler` | Verifies the execution view across a resumed invocation. |
 | `otel-execution-3` | `otel_3_retry.handler` | Verifies the execution view across retry attempts. |
@@ -83,13 +90,91 @@ deploys only the view named by its `OtelSuite` parameter.
 | `otel-execution-18` | `otel_18_chained_invoke_failure.handler` | Verifies source and target failed workflow roots. |
 | `otel-execution-19` | `otel_19_execution_failure.handler` | Verifies a failed invocation without a completed workflow. |
 | `otel-execution-20` | `otel_20_virtual_context.handler` | Verifies a virtual child-context span under the workflow root. |
+| `otel-execution-21` | `otel_21_completed_step_replay.handler` | Local coverage verifies completed-operation spans are exported once across replay; cloud coverage is deferred. |
+| `otel-execution-22` | `otel_22_user_function_context.handler` | Observes active execution-view callback contexts without supplying or repairing parents. |
+| `otel-execution-23` | `otel_23_callback_function_context.handler` | Verifies the same SDK-owned callback lifecycle parents in the execution view. |
+| `otel-execution-24` | `otel_24_invocation_retry_status.handler` | Verifies invocation retry status independently of step retry and recovery re-exports. |
+| `otel-execution-25` | `otel_25_caught_callback_failure_replay.handler` | Verifies one failed callback export with ERROR and no exception details while the caught execution succeeds. |
+| `otel-execution-26` | `otel_26_external_callback_completion_replay.handler` | Verifies first-completion export timing and no duplicate export across later callback resumes. |
+| `otel-execution-27` | `otel_27_chained_invoke_suspend.handler` | Verifies caller/child workflow parentage and continuation links across child suspension. |
 | `otel-long-running-1` | `otel_long_running_1_wait.handler` | Verifies wait and resume telemetry across a long durable suspension. |
 | `otel-long-running-2` | `otel_long_running_2_retry.handler` | Verifies retry telemetry across a long durable backoff. |
 | `otel-long-running-3` | `otel_long_running_3_callback.handler` | Verifies callback telemetry when completion arrives after a long delay. |
 | `otel-long-running-4` | `otel_long_running_4_chained_invoke.handler` | Verifies chained-invoke telemetry while the target stays suspended. |
 
-The runner discovers each mapping from `TestingMetadata.TestDescription` on the
-functions in the templates.
+The runner discovers each active cloud mapping from
+`TestingMetadata.TestDescription` on the functions in the templates.
+
+## Deferred cloud coverage
+
+Case 21 is declared `NOT_IMPLEMENTED` in both views until the shared history
+requirement accepts `WaitSucceeded` either before or after
+`InvocationCompleted`. Both catalogs currently require invocation completion
+first, but the wait timer runs independently of invocation cleanup. The
+[execution-view cloud run](https://github.com/aws/aws-durable-execution-sdk-python/actions/runs/40588786563/job/122487525577)
+observed the reverse order while the execution succeeded with `before-after`.
+See the [review finding](https://github.com/aws/aws-durable-execution-sdk-python/pull/796#discussion_r4394770596).
+The handler and local tests remain: they verify one execution of each step
+body, `PENDING`/`SUCCEEDED` invocations, and one export per completed operation
+in the execution view. Re-enable the SAM functions and build targets after the
+shared requirement accepts both history orderings without relaxing those replay
+assertions.
+
+## Callback regressions awaiting fixes
+
+Cases 25 and 26 are active in both views, including their SAM functions, build
+targets, and local integration tests. These tests intentionally keep CI failing
+until the required SDK and testing-library fixes merge. They are ordinary
+failing tests, without skips, expected-failure markers, or failure suppression.
+
+- Case 25 needs terminal failure status without diagnostic details
+  ([#853](https://github.com/aws/aws-durable-execution-sdk-python/issues/853))
+  and consistent empty callback error history
+  ([#862](https://github.com/aws/aws-durable-execution-sdk-python/issues/862)).
+- Case 26 needs completion discovery and export
+  ([#854](https://github.com/aws/aws-durable-execution-sdk-python/issues/854)),
+  callback update production
+  ([#860](https://github.com/aws/aws-durable-execution-sdk-python/issues/860)),
+  and invocation-input acknowledgment
+  ([#861](https://github.com/aws/aws-durable-execution-sdk-python/issues/861)).
+- The additional case 17 callback-error regression also needs snapshot isolation
+  ([#852](https://github.com/aws/aws-durable-execution-sdk-python/issues/852)).
+  The existing case 17 handler and supplied-error coverage remain active.
+
+Case 25 asserts the local-runner contract proposed in #862: detailed failure
+history includes an empty `Payload` with `Truncated: false`, while metadata-only
+history omits the payload and sets `Truncated: true`. The exact empty history
+`Payload` shape remains **unverified** against deployed AWS Lambda and requires
+service evidence before claiming service parity. The test inspects raw exported
+spans and ordinary pytest diagnostics; it has no configurable proof-file output.
+
+Case 26 asserts inside the observed step that the terminal callback has already
+reached the exporter, then checks that its raw export count remains one at both
+later barriers and after completion. Recorded span timestamps alone are not
+evidence of export order.
+
+The additional case 17 test remains available for follow-up in
+[commit `6484e577`](https://github.com/aws/aws-durable-execution-sdk-python/commit/6484e577b478aeb6345e346515b40bd516cacb1f).
+Before restoring it, correct its data-inclusion expectations: detailed errors
+are untruncated; metadata-only history omits payloads and cannot reconstruct
+diagnostic details.
+
+Case 24 remains active and covers invocation `RETRY` becoming `RETRYING`/`UNSET`.
+
+## Callback coverage boundary
+
+Cases 22 and 23 create normal user spans from the active context, without an
+explicit parent or a copied execution ARN. They cover handler context, step and
+condition attempts, child and branch bodies, callback submission, wrapped
+`with_retry` body/strategy callbacks, and a virtual child. Case 23 places the
+retry helper and virtual child after callback completion so ordinary successful
+replay does not repeat their probes.
+
+This does not promise an operation/attempt parent for every arbitrary callback.
+General retry/wait policies, serializers, summary generators, and item naming
+have phase-specific caller scopes outside the wrapped user-function lifecycle;
+these cases do not impose a new ownership policy on them. Instrumentation
+extensions and user-created threads are outside this business-callback contract.
 
 ## How a handler maps to a requirement
 
@@ -197,6 +282,16 @@ write access; the runner identity needs list, read, and cleanup access.
 `.github/workflows/opentelemetry-conformance-tests.yml` calls the shared
 orchestrator in the conformance repository and points it at this package with
 `examples_dir`, so orchestration stays centralized while the handlers run from
-the commit under test. Pull requests run the invocation and execution suites plus
-a short (60-second) long-running cycle; the full multi-hour long-running cycle is
-driven by `workflow_dispatch` with `phase: launch` and `phase: check`.
+the commit under test. The shared workflow tracks the conformance repository's
+`main` branch, and the test catalog defaults to `main`. Manual runs can override
+the catalog with the `conformance_test_ref` input. Pull requests run the
+invocation and execution suites, with only case 21 reported as
+`NOT_IMPLEMENTED`. Cases 25 and 26 remain enabled while their prerequisite fixes
+are pending. The orchestrator no longer runs the legacy long-running
+suite; its templates and local contract tests remain in this package.
+
+Case 27 (`chained-invoke-suspend`) invokes a child that performs a one-second
+durable wait and returns the original payload after resuming. Both views
+check two caller invocations, two child invocations and the child wait's
+parentage and continuation links. It replaces the suspended-child scenario
+formerly exercised by the short mode of `otel-long-running-4`.

packages/aws-durable-execution-sdk-python-conformance-tests-otel/src/Makefile

@@ -1,45 +1,55 @@
 .PHONY: build-Otel1Success build-Otel2WaitResume build-Otel3Retry build-Otel4TerminalFailure \
 	build-Otel5ChildContext build-Otel6Parallel build-Otel7Map build-Otel8HandledFailure \
 	build-Otel9WaitForCondition build-Otel10WaitForCallback build-Otel11ChainedInvoke \
-	build-Otel11InvokeTarget build-Otel12ChildContextFailure build-Otel13ParallelFailure \
+	build-Otel27ChainedInvokeSuspend build-Otel27InvokeTarget build-Otel11InvokeTarget build-Otel12ChildContextFailure build-Otel13ParallelFailure \
 	build-Otel14MapFailure build-Otel15WaitInterrupted build-Otel16WaitForConditionFailure \
 	build-Otel17WaitForCallbackFailure build-Otel18ChainedInvokeFailure \
 	build-Otel18InvokeTarget build-Otel19ExecutionFailure build-Otel20VirtualContext \
+	build-Otel22UserFunctionContext build-Otel23CallbackFunctionContext build-Otel24InvocationRetryStatus \
+	build-Otel25CallbackFailureWithoutError build-Otel26ExternalCallbackReplay \
 	build-OtelExecution1Success \
 	build-OtelExecution2WaitResume build-OtelExecution3Retry \
 	build-OtelExecution4TerminalFailure build-OtelExecution5ChildContext \
 	build-OtelExecution6Parallel build-OtelExecution7Map build-OtelExecution8HandledFailure \
 	build-OtelExecution9WaitForCondition build-OtelExecution10WaitForCallback \
-	build-OtelExecution11ChainedInvoke build-OtelExecution11InvokeTarget \
+	build-OtelExecution11ChainedInvoke build-OtelExecution27ChainedInvokeSuspend build-OtelExecution27InvokeTarget build-OtelExecution11InvokeTarget \
 	build-OtelExecution12ChildContextFailure build-OtelExecution13ParallelFailure \
 	build-OtelExecution14MapFailure build-OtelExecution15WaitInterrupted \
 	build-OtelExecution16WaitForConditionFailure \
 	build-OtelExecution17WaitForCallbackFailure build-OtelExecution18ChainedInvokeFailure \
 	build-OtelExecution18InvokeTarget build-OtelExecution19ExecutionFailure \
 	build-OtelExecution20VirtualContext \
+	build-OtelExecution22UserFunctionContext build-OtelExecution23CallbackFunctionContext \
+	build-OtelExecution24InvocationRetryStatus \
+	build-OtelExecution25CallbackFailureWithoutError build-OtelExecution26ExternalCallbackReplay \
 	build-OtelLongRunning1Wait \
 	build-OtelLongRunning2Retry build-OtelLongRunning3Callback \
 	build-OtelLongRunning4ChainedInvoke build-OtelLongRunning4InvokeTarget
 
 build-Otel1Success build-Otel2WaitResume build-Otel3Retry build-Otel4TerminalFailure \
 build-Otel5ChildContext build-Otel6Parallel build-Otel7Map build-Otel8HandledFailure \
 build-Otel9WaitForCondition build-Otel10WaitForCallback build-Otel11ChainedInvoke \
-build-Otel11InvokeTarget build-Otel12ChildContextFailure build-Otel13ParallelFailure \
+build-Otel27ChainedInvokeSuspend build-Otel27InvokeTarget build-Otel11InvokeTarget build-Otel12ChildContextFailure build-Otel13ParallelFailure \
 build-Otel14MapFailure build-Otel15WaitInterrupted build-Otel16WaitForConditionFailure \
 build-Otel17WaitForCallbackFailure build-Otel18ChainedInvokeFailure \
 build-Otel18InvokeTarget build-Otel19ExecutionFailure build-Otel20VirtualContext \
+build-Otel22UserFunctionContext build-Otel23CallbackFunctionContext build-Otel24InvocationRetryStatus \
+build-Otel25CallbackFailureWithoutError build-Otel26ExternalCallbackReplay \
 build-OtelExecution1Success \
 build-OtelExecution2WaitResume build-OtelExecution3Retry \
 build-OtelExecution4TerminalFailure build-OtelExecution5ChildContext \
 build-OtelExecution6Parallel build-OtelExecution7Map build-OtelExecution8HandledFailure \
 build-OtelExecution9WaitForCondition build-OtelExecution10WaitForCallback \
-build-OtelExecution11ChainedInvoke build-OtelExecution11InvokeTarget \
+build-OtelExecution11ChainedInvoke build-OtelExecution27ChainedInvokeSuspend build-OtelExecution27InvokeTarget build-OtelExecution11InvokeTarget \
 build-OtelExecution12ChildContextFailure build-OtelExecution13ParallelFailure \
 build-OtelExecution14MapFailure build-OtelExecution15WaitInterrupted \
 build-OtelExecution16WaitForConditionFailure \
 build-OtelExecution17WaitForCallbackFailure build-OtelExecution18ChainedInvokeFailure \
 build-OtelExecution18InvokeTarget build-OtelExecution19ExecutionFailure \
 build-OtelExecution20VirtualContext \
+build-OtelExecution22UserFunctionContext build-OtelExecution23CallbackFunctionContext \
+build-OtelExecution24InvocationRetryStatus \
+build-OtelExecution25CallbackFailureWithoutError build-OtelExecution26ExternalCallbackReplay \
 build-OtelLongRunning1Wait \
 build-OtelLongRunning2Retry build-OtelLongRunning3Callback \
 build-OtelLongRunning4ChainedInvoke build-OtelLongRunning4InvokeTarget:

packages/aws-durable-execution-sdk-python-conformance-tests-otel/src/otel_21_completed_step_replay.py

@@ -0,0 +1,36 @@
+# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates.
+#
+# SPDX-License-Identifier: Apache-2.0
+"""Normal successful replay of a completed step for OTel case 21."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from aws_durable_execution_sdk_python import (
+    DurableContext,
+    StepContext,
+    durable_execution,
+    durable_step,
+)
+from aws_durable_execution_sdk_python.config import Duration
+from common import otel_plugin_factory, require_scenario
+
+
+@durable_step
+def before_wait(_step_context: StepContext) -> str:
+    return "before"
+
+
+@durable_step
+def after_wait(_step_context: StepContext) -> str:
+    return "after"
+
+
+@durable_execution(plugins=[otel_plugin_factory()])
+def handler(event: dict[str, Any], context: DurableContext) -> str:
+    require_scenario(event, "completed-step-replay")
+    before = context.step(before_wait(), name="otel-before-wait")
+    context.wait(Duration.from_seconds(1), name="otel-replay-wait")
+    after = context.step(after_wait(), name="otel-after-wait")
+    return f"{before}-{after}"