diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index 9910802c..69b7f88b 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -496,6 +496,25 @@ roboflow workflow-eval binding suggest --spec --dataset - roboflow workflow-eval run start --eval --body @run.json --wait # --wait gives up after 3600 s; Runs on a local inference server wait without limit. --timeout 0 = no limit. roboflow workflow-eval run get --eval --wait --timeout 0 + +# Runs on a local inference server (executionMode "browser") run without a browser tab. +# Start and run one in this process, then wait for the result: +roboflow workflow-eval run start --eval --body @run.json --wait --inference-url http://localhost:9001 +# Or run an existing local Run in the background (default --concurrency 4, max 16): +nohup roboflow workflow-eval run local -e --concurrency 4 > run-local.log 2>&1 & +# Interrupted (Ctrl-C, exit 130)? Run the same command again: captured outputs are resubmitted from the +# journal in ~/.cache/roboflow/workflow-evals/// (--state-dir), not re-run. +# Inference server unreachable (cannot connect, retried after 1, 2, 4, 8, 16 s)? Those Cases are not captured +# (they stay pending; PENDING in the summary) and the command exits 1, at once after 10 such Cases in a row; +# start the server and run it again. A read timeout (--case-timeout: per connection attempt and per read, not +# per Case), or a 408/429/5xx or dropped connection that persists for 3 attempts, is captured as a retryable +# failure. Worst case per Case for a server that stops responding: about 34 s of backoff plus 11 x --case-timeout +# (--case-timeout limits each connect and each read, and a Case makes up to 6 connection attempts and 3 connected +# ones); a response that keeps sending data is not cut off (no bound with --case-timeout 0). With --json, +# warnings are in "warnings" on success, or in "error.warnings" on failure. +# If `run start --wait --inference-url` fails while running the Cases locally, the error has the Run id +# (error.details.runId) and, unless the Run was cancelled, a `run local` command (with the --workspace, +# --concurrency and --case-timeout you passed) to resume it; running `run start` again would start another Run. roboflow workflow-eval execution overview --eval --run --json roboflow workflow-eval execution results --eval --run --failed-check roboflow workflow-eval compare --eval -x -x --json diff --git a/roboflow/cli/handlers/workflow_eval.py b/roboflow/cli/handlers/workflow_eval.py index 1a270743..21619ca8 100644 --- a/roboflow/cli/handlers/workflow_eval.py +++ b/roboflow/cli/handlers/workflow_eval.py @@ -18,8 +18,9 @@ import json import os import sys +import threading import time -from typing import Annotated, Any, Callable, Dict, List, Optional +from typing import Annotated, Any, Callable, Dict, List, Optional, Tuple import typer @@ -100,6 +101,12 @@ def _client(args: Any) -> Any: def _error_hint(exc: Any) -> Optional[str]: + from roboflow.core.workflow_eval_local_runner import LocalRunnerError + + if isinstance(exc, LocalRunnerError): + # The local runner words its own hints: workflow-evals:read/run scopes on 403, + # "start the inference server or pass --inference-url" on preflight, resume steps. + return exc.hint status, code = exc.status_code, (exc.code or "") if code == "FEATURE_DISABLED": return "Workflow Evals is not enabled for this workspace." @@ -129,14 +136,27 @@ def _exit_code(exc: Any) -> int: return 1 -def _fail(args: Any, exc: Exception) -> None: +def _fail(args: Any, exc: Exception, warnings: Optional[List[str]] = None) -> None: from roboflow.adapters.workflowevalsapi import WorkflowEvalError from roboflow.cli._output import output_error + from roboflow.core.workflow_eval_local_runner import LocalRunnerError if isinstance(exc, WorkflowEvalError): payload = exc.to_dict() payload.pop("hint", None) + if warnings: + payload["warnings"] = warnings output_error(args, json.dumps({"error": payload}), hint=_error_hint(exc), exit_code=_exit_code(exc)) + elif isinstance(exc, LocalRunnerError): + payload = exc.to_dict() + payload.pop("hint", None) + if exc.result is not None: + payload["result"] = exc.result.to_dict() + if warnings: + payload["warnings"] = warnings + output_error(args, json.dumps({"error": payload}, default=str), hint=_error_hint(exc), exit_code=exc.exit_code) + elif warnings and getattr(args, "json", False): + output_error(args, json.dumps({"error": {"message": str(exc), "warnings": warnings}})) else: output_error(args, str(exc)) @@ -147,10 +167,12 @@ def _invoke( text: Optional[Callable[[Any], Optional[str]]] = None, *, interactive: bool = False, + warnings: Optional[List[str]] = None, ) -> None: """Resolve the client, run *call*, and print the result (or a structured error). ``interactive`` keeps stdout attached so confirmation prompts stay visible. + ``warnings`` (collected with --json) go in the JSON error when *call* fails. """ import contextlib @@ -163,7 +185,7 @@ def _invoke( with contextlib.nullcontext() if interactive else suppress_sdk_output(args): result = call(client) except Exception as exc: # noqa: BLE001 — every failure becomes a structured CLI error - _fail(args, exc) + _fail(args, exc, warnings) return output(args, result, text=text(result) if text and not getattr(args, "json", False) else None) @@ -909,6 +931,23 @@ def preparation_cases( _PROGRESS_EVERY = 10.0 +class _Throttle: + """``ready()`` is true at most once every ``every`` seconds of ``time.monotonic`` (thread-safe).""" + + def __init__(self, every: float = _PROGRESS_EVERY) -> None: + self._every = every + self._last: Optional[float] = None + self._lock = threading.Lock() + + def ready(self) -> bool: + with self._lock: + now = time.monotonic() + if self._last is not None and now - self._last < self._every: + return False + self._last = now + return True + + def _run_id(admission: Dict[str, Any]) -> Optional[str]: run = admission.get("run") return run.get("id") if isinstance(run, dict) else admission.get("id") @@ -951,21 +990,19 @@ def _wait_for_run( show_progress = not args.json and not args.quiet started = time.monotonic() limit = timeout - last_progress: Optional[float] = None + throttle = _Throttle() def on_poll(run: Dict[str, Any]) -> None: - nonlocal limit, last_progress + nonlocal limit if limit is None: mode = execution_mode or _run_field(run, "executionMode") limit = 0 if mode == "browser" else _SERVER_RUN_WAIT_TIMEOUT state = _run_field(run, "state") if state in RUN_TERMINAL_STATES: return - now = time.monotonic() - if show_progress and (last_progress is None or now - last_progress >= _PROGRESS_EVERY): + if show_progress and throttle.ready(): print(_progress_line(run_id, run), file=sys.stderr) - last_progress = now - if limit and now - started >= limit: + if limit and time.monotonic() - started >= limit: raise TimeoutError(f"Run {run_id} is still {state} after {limit:g}s") return client.wait_for_run(eval_id, run_id, timeout=0, interval=interval, on_poll=on_poll) @@ -986,6 +1023,181 @@ def wait(client: Any, admission: Dict[str, Any]) -> Any: return wait +_STOPPING_NOTICE = "Stopping after in-flight cases; press Ctrl-C again to exit now." + + +def _check_local_options(args: Any, concurrency: Optional[int], case_timeout: Optional[float]) -> None: + from roboflow.cli._output import output_error + from roboflow.core.workflow_eval_local_runner import MAX_CONCURRENCY + + if concurrency is not None and not 1 <= concurrency <= MAX_CONCURRENCY: + output_error( + args, + f"--concurrency must be between 1 and {MAX_CONCURRENCY} (got {concurrency}).", + hint="Use the default 4 (the browser's limit); raise it only if the inference server has spare capacity.", + ) + if case_timeout is not None and case_timeout < 0: + output_error(args, f"--case-timeout must be 0 or positive (got {case_timeout:g}).", hint="Pass 0 for no limit.") + + +def _run_local_command( + args: Any, + run_id: str, + eval_id: str, + inference_url: Optional[str], + concurrency: Optional[int], + case_timeout: Optional[float], +) -> str: + """The ``run local`` command that resumes a Run, with the options the user passed (never the API key).""" + import shlex + + parts = ["roboflow"] + if args.workspace: + parts += ["--workspace", shlex.quote(str(args.workspace))] + parts += ["workflow-eval", "run", "local", shlex.quote(run_id), "-e", shlex.quote(eval_id)] + parts += ["--inference-url", shlex.quote(str(inference_url))] + if concurrency is not None: + parts += ["--concurrency", str(concurrency)] + if case_timeout is not None: + # 300.0 -> 300, 2.5 -> 2.5 (repr keeps every digit, unlike :g). + seconds = float(case_timeout) + parts += ["--case-timeout", str(int(seconds)) if seconds.is_integer() else repr(seconds)] + return " ".join(parts) + + +def _local_progress_line(progress: Dict[str, Any]) -> str: + remaining = progress.get("remaining") + left = "?" if remaining is None else remaining + cases = f"cases {progress['ok']} ok / {progress['failed']} failed / {left} remaining" + parts = [f"Execution {progress['execution']}/{progress['executions']}", cases] + if progress.get("alreadyDone"): + parts.append(f"{progress['alreadyDone']} already done") + if progress.get("skipped"): + parts.append(f"{progress['skipped']} skipped") + if progress.get("pending"): + parts.append(f"{progress['pending']} pending (inference server unreachable)") + parts.append(f"in flight {progress['inFlight']}") + latency = progress.get("lastLatencyMs") + parts.append("last latency " + ("-" if latency is None else f"{latency:.0f} ms")) + return " · ".join(parts) + + +def _warnings(args: Any) -> Tuple[List[str], Callable[[str], None]]: + """Return ``(warnings, warn)``: ``warn`` prints "Warning: ..." to stderr, or with --json collects into ``warnings``. + + With --json the warnings go in a ``"warnings"`` field of the final JSON object, + so stdout stays a single JSON document and stderr stays free of text. + """ + collected: List[str] = [] + + def warn(message: str) -> None: + if args.json: + collected.append(message) + else: + print(f"Warning: {message}", file=sys.stderr) + + return collected, warn + + +def _run_locally( + args: Any, + client: Any, + eval_id: str, + run_id: str, + resume_hint: str, + warn: Callable[[str], None], + *, + created_run: bool = False, + **options: Any, +) -> Any: + """Run a local Run's Cases with :class:`LocalRunner` and return its ``LocalRunResult``. + + Progress goes to stderr (throttled, not with --json/--quiet); warnings go to + ``warn``. SIGINT/SIGTERM stop gracefully while the runner works and the + previous handlers are restored afterwards. An interruption, or Cases left + pending because the inference server was unreachable, is re-raised with + ``resume_hint`` (formatted with ``state_dir``). With ``created_run`` (``run + start`` just created the Run, so running that command again would start + another one), every other resumable :class:`LocalRunnerError` gets ``resume_hint`` + in place of the runner's "run the command again", any other exception is + wrapped in a :class:`LocalRunnerError`, and every one gets ``runId`` in its ``details``. + """ + import contextlib + + from roboflow.core.workflow_eval_local_runner import ( + RESUME_HINT, + InferenceServerUnreachable, + LocalRunner, + LocalRunnerError, + LocalRunnerInterrupted, + install_signal_handlers, + ) + + show_progress = not args.json and not args.quiet + throttle = _Throttle() + + def on_progress(progress: Dict[str, Any]) -> None: + if show_progress and throttle.ready(): + print(_local_progress_line(progress), file=sys.stderr) + + signals_seen = 0 + + def on_signal(signum: int) -> None: + nonlocal signals_seen + signals_seen += 1 + if signals_seen == 1 and not args.json: + print(_STOPPING_NOTICE, file=sys.stderr) + + runner = LocalRunner(client, eval_id, run_id, on_progress=on_progress, warn=warn, **options) + # signal.signal only works on the main thread. Off it no handlers are installed, and Python + # delivers KeyboardInterrupt only to the main thread, so Ctrl-C does not reach the runner. + main_thread = threading.current_thread() is threading.main_thread() + handlers = install_signal_handlers(runner, on_signal=on_signal) if main_thread else contextlib.nullcontext() + try: + with handlers: + return runner.run() + except Exception as exc: # noqa: BLE001 — after run start, any failure must keep the Run id + if isinstance(exc, LocalRunnerError): + error = exc + elif created_run: + error = LocalRunnerError(str(exc)) # e.g. OSError creating the journal + else: + raise + resume = resume_hint.format(state_dir=runner.state_dir) + if isinstance(error, LocalRunnerInterrupted): + error.hint = resume + elif isinstance(error, InferenceServerUnreachable): + error.hint = "Start the inference server. " + resume + elif created_run and error.resumable: + # RESUME_HINT says "run the command again", which here would start another Run. + hint = (error.hint or "").replace(RESUME_HINT, "").strip() + error.hint = f"{hint} {resume}" if hint else resume + if created_run: + error.details = {**(error.details or {}), "runId": run_id} + if error is exc: + raise + raise error from exc + + +def _local_result_text(result: Dict[str, Any]) -> str: + from roboflow.cli._table import format_table + + header = ( + f"Run {result['runId']} · {result['state']} · {result['wallClockSeconds']:g}s" + f" · concurrency {result['concurrency']}" + ) + columns = { + "EXECUTION": "id", + "SUBMITTED": "submitted", + "FAILED": "failed", + "ALREADY DONE": "alreadyDone", + "SKIPPED": "skipped", + "PENDING": "pending", + } + rows = [{column: item[key] for column, key in columns.items()} for item in result["executions"]] + return f"{header}\n\n{format_table(rows, list(columns))}" if rows else header + + @run_app.command("list") def list_runs( ctx: typer.Context, @@ -1034,17 +1246,157 @@ def start_run( timeout: _TimeoutOpt = None, interval: _IntervalOpt = 5, idempotency_key: IdempotencyOpt = None, + inference_url: Annotated[ + Optional[str], + typer.Option( + "--inference-url", + help="With --wait, run a local Run's Cases against this inference server, then wait " + "(no effect, with a warning, when the Run executes on Roboflow)", + ), + ] = None, + concurrency: Annotated[ + Optional[int], + typer.Option("--concurrency", help="With --inference-url: inferences in flight, 1-16 (default 4)"), + ] = None, + case_timeout: Annotated[ + Optional[float], + typer.Option( + "--case-timeout", + help="With --inference-url: seconds per connection attempt and per read of each inference request, " + "not a total per Case; 0 = no limit (default 300). Worst case per Case for a server that stops " + "responding: about 34 s of backoff plus 11 x --case-timeout (--case-timeout limits each connect and " + "each read, and a Case makes up to 6 connection attempts and 3 connected ones); a response that keeps " + "sending data is not cut off (no bound with --case-timeout 0).", + ), + ] = None, ) -> None: """Start a grouped Run of one or more Workflows over the Eval's Dataset (may consume credits).""" + from roboflow.cli._output import output_error + args = ctx_to_args(ctx) + if inference_url and not wait: + output_error( + args, + "--inference-url requires --wait.", + hint="Add --wait, or start the Run and then use `roboflow workflow-eval run local`.", + ) + if not inference_url and (concurrency is not None or case_timeout is not None): + output_error(args, "--concurrency and --case-timeout require --inference-url.") + _check_local_options(args, concurrency, case_timeout) payload = _require(args, _load_object(args, body)) then = _then_wait(args, eval_id, timeout, interval) + local_options = _merge({"inference_url": inference_url}, concurrency=concurrency, case_timeout=case_timeout) + warnings, warn = _warnings(args) def call(c: Any) -> Any: admission = c.start_run(eval_id, payload, idempotency_key=idempotency_key) - return then(c, admission) if wait else admission + if not wait: + return admission + run_id = _run_id(admission) + mode = None + if inference_url and run_id: + mode = _run_field(admission, "executionMode") or _run_field(c.get_run(eval_id, run_id), "executionMode") + if mode != "browser": + warn(f"--inference-url has no effect: Run {run_id} executes on Roboflow; waiting for it.") + if mode == "browser" and run_id: + command = _run_local_command(args, run_id, eval_id, inference_url, concurrency, case_timeout) + # Escape braces: the hint is formatted with ``state_dir`` later. + resume = f"Resume with `{command.replace('{', '{{').replace('}', '}}')}`; the journal is in {{state_dir}}." + result = _run_locally(args, c, eval_id, run_id, resume, warn, created_run=True, **local_options) + if not args.json and not args.quiet: + done = sum(item.already_done for item in result.executions) + submitted = sum(item.submitted for item in result.executions) + failed = sum(item.failed for item in result.executions) + print( + f"Local run finished in {result.wall_clock_seconds:g}s: {submitted} submitted " + f"({failed} failed), {done} already done.", + file=sys.stderr, + ) + final = then(c, admission) + if warnings and isinstance(final, dict): + final = {**final, "warnings": warnings} + return final + + _invoke(args, call, warnings=warnings) + + +@run_app.command("local") +def run_local( + ctx: typer.Context, + run_id: Annotated[str, typer.Argument(help="Run id")], + eval_id: EvalOpt, + inference_url: Annotated[ + Optional[str], typer.Option("--inference-url", help="Inference server URL (default: the URL frozen in the Run)") + ] = None, + inference_api_key: Annotated[ + Optional[str], + typer.Option("--inference-api-key", help="API key sent to the inference server (default: the CLI's key)"), + ] = None, + execution: Annotated[ + Optional[List[str]], + typer.Option("--execution", "-x", help="Only these Execution ids (repeatable; default: all, in order)"), + ] = None, + concurrency: Annotated[int, typer.Option("--concurrency", help="Inferences in flight at once, 1-16")] = 4, + case_timeout: Annotated[ + float, + typer.Option( + "--case-timeout", + help="Seconds per connection attempt and per read of each inference request, not a total per Case; " + "0 = no limit. Worst case per Case for a server that stops responding: about 34 s of backoff plus " + "11 x --case-timeout (--case-timeout limits each connect and each read, and a Case makes up to 6 " + "connection attempts and 3 connected ones); a response that keeps sending data is not cut off (no " + "bound with --case-timeout 0).", + ), + ] = 300, + state_dir: Annotated[ + Optional[str], + typer.Option("--state-dir", help="Journal directory (default: ~/.cache/roboflow/workflow-evals//)"), + ] = None, + retry: Annotated[ + bool, typer.Option("--retry", help="Retry a failed or partially completed Run (when allowed) first") + ] = False, +) -> None: + """Run a local Run's Cases against your inference server, without a browser tab. - _invoke(args, call) + Calls the inference server like the web app's browser run and submits each + output for server-side scoring. Interrupt with Ctrl-C and run the same command + again to resume: captured outputs are journaled and resubmitted, not re-run. + """ + from roboflow.core.workflow_eval_local_runner import LocalRunnerError, _cancelled, retry_api + from roboflow.core.workflow_evals import RUN_TERMINAL_STATES + + args = ctx_to_args(ctx) + _check_local_options(args, concurrency, case_timeout) + warnings, warn = _warnings(args) + + def call(c: Any) -> Dict[str, Any]: + result = _run_locally( + args, + c, + eval_id, + run_id, + "Run the same command again to resume; the journal is in {state_dir}.", + warn, + inference_url=inference_url, + inference_api_key=inference_api_key, + execution_ids=execution, + concurrency=concurrency, + case_timeout=case_timeout, + state_dir=state_dir, + retry=retry, + ) + if result.state not in RUN_TERMINAL_STATES: + # The server finalizes the Run inside the last capture; read it once more in case + # that raced the runner's final read (or other Executions are still running). + run = retry_api(lambda: c.get_run(eval_id, run_id)) + result.state = _run_field(run, "state") + if _cancelled(run): + raise LocalRunnerError( + f"Run {run_id} was cancelled.", hint="Start a new Run to evaluate again.", result=result + ) + return {**result.to_dict(), "warnings": warnings} + + _invoke(args, call, _local_result_text, warnings=warnings) @run_app.command("cancel") diff --git a/roboflow/core/workflow_eval_local_runner.py b/roboflow/core/workflow_eval_local_runner.py new file mode 100644 index 00000000..149052ec --- /dev/null +++ b/roboflow/core/workflow_eval_local_runner.py @@ -0,0 +1,1666 @@ +"""Run a local Workflow Eval Run (``executionMode: "browser"``) without a browser tab. + +The runner does what the web app's browser run does: it reads each Execution's +frozen Cases page by page, calls the user's Inference server +(``POST {url}/workflows/run``) with the same request body as the browser worker, +and submits the raw output through the existing ``capture`` route. Scoring stays +on the server. A Case whose inputs failed to resolve (``workflowInputFailures``) +is not sent to the Inference server; it is captured as failed with the first +failure's message, as the engine does. + +Differences from the browser, all deliberate: + +* A sliding window of ``concurrency`` inferences (no per-page lockstep), and + capture submission outside the inference slot. +* Up to 3 attempts for HTTP 408/429/5xx from the Inference server and for a + connection dropped after it was made (reset, aborted, closed mid-response), + before anything is submitted. When they persist, the Case is captured as a + failure with ``retryable: true`` (``"Failed to fetch"`` for a dropped + connection). A read timeout (inference slower than ``case_timeout``) is not + retried, as in the browser, and is captured the same way. +* A request that could not connect (connection refused, DNS failure, connect + timeout, TLS or proxy error) is infrastructure, not a Case failure: it is + retried after 1, 2, 4, 8 and 16 s, and if it still cannot connect nothing is + captured and the Case stays pending on the server, so the next run picks it + up. A run that leaves Cases pending ends with :class:`InferenceServerUnreachable` + (exit 1); 10 Cases in a row that give up this way stop the run at once (any + attempt that reaches the server resets the count). +* Before every retry, a Case whose input-cases page is older than 8 minutes + re-reads that page, so the retry gets fresh signed media URLs. +* A local journal (``/journal.jsonl`` plus ``captures/.json``) + so that a restarted runner resubmits the stored capture instead of running + the inference again. + +Capture responses and what the runner does: + +| Response | Action | +| ------------------------------------------------- | -------------------------------------------------- | +| 200 | Submitted. | +| 409 "Different capture already submitted" | ``done_by_other`` (counted as already done). | +| 409 "Capture is being processed" (retryable) | Wait (see below), resend the same capture. | +| 409 "Case failure is not retryable" | ``skipped``. | +| 409 "Run no longer accepts captures" | Stop dispatching; the Run is closed. | +| 412 "Run attempt changed" | Re-read the Run, restart the Execution. | +| 409 "No retained capture" (on a ``resume``) | Re-read the Run, restart the Execution. | +| input-cases page with a different attemptCount | Re-read the Run, restart the Execution. | +| 413 | Submit a failure capture instead (too large). | +| 404 | Exit 3. | +| 400 | Exit 1, with the server's error. | + +A capture larger than 10 MiB as sent (the JSON that ``requests`` encodes) is +not sent at all: the failure capture of a 413 replaces it. + +A Case waits for the lease another runner holds on it (after that 409, or before +a ``resume`` of a capture the server retained) and re-reads its page every 5 s +meanwhile: it resends at once when the Case's checkpoint is ``completed`` or +``failed``, follows a changed ``leaseExpiresAt``, and gives up the Case when the +page closes the Run, is at another attempt, or no longer has the Case. + +A restarted Execution reads its Cases again: Cases the server already has are +not run again, and journaled captures of the same attempt are resubmitted. The +stored bytes are parsed and re-encoded by ``requests``, so the bytes on the wire +may differ, but the JSON value (what the server fingerprints) is identical, so +no Case gets a second, different capture in one attempt. + +Usage:: + + from roboflow.core.workflow_eval_local_runner import LocalRunner, install_signal_handlers + + runner = LocalRunner(evals, eval_id, run_id, concurrency=4) + with install_signal_handlers(runner): + result = runner.run() + print(result.to_dict()) + +Errors raise :class:`LocalRunnerError`, which carries the CLI exit code +(1 error, 2 auth, 3 not found); an interruption raises +:class:`LocalRunnerInterrupted` (exit code 130). +""" + +from __future__ import annotations + +import contextlib +import hashlib +import json +import logging +import os +import signal +import threading +import time +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple + +import requests +import urllib3 +from requests.adapters import HTTPAdapter + +from roboflow.adapters import workflowevalsapi +from roboflow.adapters.workflowevalsapi import WorkflowEvalError +from roboflow.core.workflow_evals import RUN_TERMINAL_STATES, WorkflowEvals +from roboflow.util import http + +logger = logging.getLogger(__name__) + +DEFAULT_CONCURRENCY = 4 +MAX_CONCURRENCY = 16 +DEFAULT_CASE_TIMEOUT = 300 +CAPTURE_TIMEOUT = 300 +PREFLIGHT_TIMEOUT = 15 +INFERENCE_ATTEMPTS = 3 +INFERENCE_BACKOFF_CAP = 30 +# Waits before each new attempt of a request that could not connect (6 attempts, 31 s). +CONNECT_BACKOFF = (1, 2, 4, 8, 16) +# Consecutive Cases that gave up without connecting before the run stops. +MAX_CONNECTION_FAILURES = 10 +API_BACKOFF_CAP = 60 +API_RETRY_BUDGET = 600 +# Signed media URLs live 10 minutes; re-read a page older than that minus a 2 minute margin. +PAGE_MAX_AGE = 8 * 60 +PROCESSING_RESENDS = 10 +# A Case waiting for a lease re-reads its page this often (seconds). +LEASE_POLL_INTERVAL = 5.0 +# The server's JSON body limit: bodyLimit "10mb" (10 MiB) in roboflow/roboflow +# app/functions/backend/setupExpress.ts:77, enforced by body-parser before any route. +CAPTURE_LIMIT_BYTES = 10 * 1024 * 1024 +MAX_RELOADS = 5 +ERROR_MAX_LENGTH = 2000 + +INFERENCE_HEADERS = {"Content-Type": "application/json", "ngrok-skip-browser-warning": "true"} +# The messages Chrome gives the browser worker for the same failures. +TIMEOUT_MESSAGE = "The operation was aborted due to timeout" +FETCH_FAILED_MESSAGE = "Failed to fetch" +CAPTURE_TOO_LARGE_MESSAGE = "Workflow output exceeds the 10 MB capture limit" +PREFLIGHT_HINT = "Start the inference server or pass --inference-url." +RESUME_HINT = "The journal is saved; run the command again to resume." + +_RETRYABLE_HTTP = (408, 429) + + +class LocalRunnerError(Exception): + """A local run failed. ``exit_code`` is the CLI exit code (1 error, 2 auth, 3 not found). + + ``result`` holds the run summary when one is available (for example a + cancelled Run), so the CLI can still print it. ``resumable`` is False when + running again cannot finish the Run (it was cancelled). + """ + + def __init__( + self, + message: str, + *, + hint: Optional[str] = None, + exit_code: int = 1, + details: Optional[Dict[str, Any]] = None, + result: Optional[LocalRunResult] = None, + resumable: bool = True, + ) -> None: + super().__init__(message) + self.message = message + self.hint = hint + self.exit_code = exit_code + self.details = details + self.result = result + self.resumable = resumable + + def to_dict(self) -> Dict[str, Any]: + """Return ``{"message", "hint"?, "details"?}`` for the CLI's structured error.""" + payload: Dict[str, Any] = {"message": self.message} + if self.hint: + payload["hint"] = self.hint + if self.details: + payload["details"] = self.details + return payload + + +class LocalRunnerInterrupted(LocalRunnerError): + """The run was stopped on request (SIGINT/SIGTERM). Exit code 130; the journal is kept.""" + + def __init__(self, message: str = "Interrupted.", *, result: Optional[LocalRunResult] = None) -> None: + super().__init__(message, hint=RESUME_HINT, exit_code=130, result=result) + + +class _Abandoned(LocalRunnerInterrupted): + """A wait or retry was abandoned because the runner is stopping.""" + + +class InferenceServerUnreachable(LocalRunnerError): + """The run finished, but some Cases were not run because the inference server was unreachable. + + Nothing was captured for them, so they are still pending on the server and + running again (with the inference server up) picks them up. Exit code 1; + ``result`` holds the run summary. + """ + + +@dataclass +class ExecutionTally: + """Per-Execution counts. + + ``failed`` counts submitted captures whose runtime has an error; ``pending`` + counts Cases left uncaptured because the inference server was unreachable. + """ + + id: str + submitted: int = 0 + already_done: int = 0 + skipped: int = 0 + failed: int = 0 + pending: int = 0 + + def to_dict(self) -> Dict[str, Any]: + return { + "id": self.id, + "submitted": self.submitted, + "alreadyDone": self.already_done, + "skipped": self.skipped, + "failed": self.failed, + "pending": self.pending, + } + + +@dataclass +class LocalRunResult: + """Outcome of :meth:`LocalRunner.run`. ``to_dict()`` is the CLI's ``--json`` output.""" + + run_id: str + state: Optional[str] + executions: List[ExecutionTally] + wall_clock_seconds: float + concurrency: int + + def to_dict(self) -> Dict[str, Any]: + return { + "runId": self.run_id, + "state": self.state, + "executions": [execution.to_dict() for execution in self.executions], + "wallClockSeconds": self.wall_clock_seconds, + "concurrency": self.concurrency, + } + + +# -- inference ----------------------------------------------------------------- + + +def materialize_media(value: Any, urls: Dict[str, Any]) -> Any: + """Replace every ``{artifactId}`` with ``{..., "type": "url", "value": }``. + + Mirrors the web app's ``materializeResultMedia``: records whose + ``artifactId`` has no signed URL are kept and searched recursively. + """ + if isinstance(value, list): + return [materialize_media(item, urls) for item in value] + if not isinstance(value, dict): + return value + artifact_id = value.get("artifactId") + capability = urls.get(artifact_id) if isinstance(artifact_id, str) else None + if isinstance(capability, dict): + return {**value, "type": "url", "value": capability.get("url")} + return {key: materialize_media(item, urls) for key, item in value.items()} + + +def case_inputs(item: Dict[str, Any], urls: Dict[str, Any]) -> Dict[str, Any]: + """Return the Workflow inputs for one ``input-cases`` item, with media materialized.""" + return materialize_media(item.get("workflowInputs") or {}, urls) + + +def input_failure_message(item: Dict[str, Any]) -> Optional[str]: + """Return the first ``workflowInputFailures`` message, or ``None`` when every input resolved. + + A Case with an input failure is not sent to the Inference server; it is + captured as failed, like the engine does (``output: null, runtime: {error}``). + """ + failures = item.get("workflowInputFailures") or [] + if not failures: + return None + first = failures[0] + message = first.get("message") if isinstance(first, dict) else None + return _truncate(message if isinstance(message, str) else "Workflow input could not be resolved") + + +def _dumps(value: Any) -> bytes: + # JSON.stringify's output: no whitespace, non-ASCII kept, no NaN. + return json.dumps(value, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode("utf-8") + + +def build_inference_body(api_key: str, specification: Any, inputs: Any, workflow_id: Any) -> bytes: + """Serialize the ``/workflows/run`` body exactly as the browser worker does.""" + return _dumps( + { + "api_key": api_key, + "specification": specification, + "inputs": inputs, + "workflow_id": workflow_id, + "is_preview": True, + "disable_sinks": True, + "debug": False, + } + ) + + +def _reject_constant(name: str) -> Any: + raise ValueError(f"Invalid JSON constant {name}") + + +def _parse_json(content: bytes) -> Any: + """Parse like ``response.json()`` in a browser: invalid JSON (including NaN) is ``None``.""" + try: + return json.loads(content, parse_constant=_reject_constant) + except ValueError: + return None + + +def _truncate(message: str) -> str: + # The server limits ``error`` to 2000 UTF-16 code units, as JavaScript counts them. + encoded = message.encode("utf-16-le") + if len(encoded) <= ERROR_MAX_LENGTH * 2: + return message + return encoded[: ERROR_MAX_LENGTH * 2].decode("utf-16-le", "ignore") + + +def _body_message(body: Optional[Dict[str, Any]], status: int) -> str: + if body is not None and isinstance(body.get("message"), str): + return body["message"] + if body is not None and isinstance(body.get("errorMsg"), str): + return body["errorMsg"] + return f"Workflow returned HTTP {status}" + + +@dataclass +class InferenceOutcome: + """One ``/workflows/run`` call, already shaped as the capture's ``output`` and ``runtime``.""" + + output: Any + runtime: Dict[str, Any] + # The request never reached the server (refused, DNS, connect timeout, TLS, proxy). + connection_failure: bool = False + responded: bool = False + # Connected, then the connection was dropped (reset, aborted, closed mid-response). + dropped: bool = False + + @property + def error(self) -> bool: + return "error" in self.runtime + + @property + def retryable(self) -> bool: + return self.runtime.get("retryable") is True + + +_NEVER_CONNECTED = tuple( + cls + for cls in ( + urllib3.exceptions.NewConnectionError, + getattr(urllib3.exceptions, "NameResolutionError", None), + urllib3.exceptions.ConnectTimeoutError, + ) + if cls is not None +) + + +def _never_connected(exc: requests.ConnectionError) -> bool: + """Whether a ``ConnectionError`` happened before the connection was made.""" + if isinstance(exc, (requests.exceptions.SSLError, requests.exceptions.ProxyError)): + return True + cause = exc.args[0] if exc.args else None + if isinstance(cause, urllib3.exceptions.MaxRetryError): + cause = cause.reason + return isinstance(cause, _NEVER_CONNECTED) + + +def run_workflow_inference( + session: requests.Session, + url: str, + *, + api_key: str, + specification: Any, + workflow_id: Any, + inputs: Any, + timeout: Optional[float], + monotonic: Callable[[], float] = time.monotonic, +) -> InferenceOutcome: + """Call ``{url}/workflows/run`` once and classify the result like the browser worker. + + Never raises for transport or HTTP failures; they become an error runtime + ``{latencyMs, error, timedOut, retryable}``. ``latencyMs`` covers the request + and the parsing of its body. + + The response is streamed so that sending the request and reading its body + fail separately. ``connection_failure`` marks a request that never reached + the server: while sending, ``ConnectTimeout``, ``SSLError``, ``ProxyError``, + and a ``ConnectionError`` caused by urllib3's ``NewConnectionError`` + (refused), ``NameResolutionError`` (DNS) or ``ConnectTimeoutError``. The + runner does not capture those. ``dropped`` marks a connection lost after it + was made: any other ``ConnectionError`` (reset, aborted, remote + disconnected), any ``ConnectionError`` while reading the body (``SSLError`` + and ``ProxyError`` included, e.g. TLS dropped mid-read) and + ``ChunkedEncodingError`` (closed mid-response); the runner retries and then + captures those like HTTP 5xx. A read timeout (``ReadTimeout``, also when it + hits while reading the body) is a ``timedOut`` failure, neither of the two. + """ + body = build_inference_body(api_key, specification, inputs, workflow_id) + started = monotonic() + state = {"retryable": False, "timedOut": False, "connection": False, "responded": False, "dropped": False} + + def failure(message: str) -> InferenceOutcome: + runtime = { + "latencyMs": (monotonic() - started) * 1000, + "error": _truncate(message), + "timedOut": state["timedOut"], + "retryable": state["retryable"], + } + return InferenceOutcome(None, runtime, state["connection"], state["responded"], state["dropped"]) + + def transport_failure(exc: requests.RequestException, *, sending: bool) -> InferenceOutcome: + if isinstance(exc, requests.Timeout): + # ConnectTimeout is also a ConnectionError: the server was never reached. + state.update( + retryable=True, timedOut=True, connection=sending and isinstance(exc, requests.ConnectionError) + ) + return failure(TIMEOUT_MESSAGE) + if isinstance(exc, requests.ConnectionError): + if exc.args and isinstance(exc.args[0], urllib3.exceptions.ReadTimeoutError): + # requests reports a read timeout while reading the body as a ConnectionError. + state.update(retryable=True, timedOut=True) + return failure(TIMEOUT_MESSAGE) + if sending and _never_connected(exc): + state.update(retryable=True, connection=True) + else: + state.update(retryable=True, dropped=True) + return failure(FETCH_FAILED_MESSAGE) + if isinstance(exc, requests.exceptions.ChunkedEncodingError): + state.update(retryable=True, dropped=True) + return failure(FETCH_FAILED_MESSAGE) + if isinstance(exc, requests.exceptions.ContentDecodingError): + state["retryable"] = True + return failure(FETCH_FAILED_MESSAGE) + return failure(str(exc) or "Local workflow request failed") + + try: + response = session.post( + f"{url.rstrip('/')}/workflows/run", + data=body, + headers=dict(INFERENCE_HEADERS), + timeout=timeout, + stream=True, + ) + except requests.RequestException as exc: + return transport_failure(exc, sending=True) + with contextlib.closing(response): + try: + content = response.content + except requests.RequestException as exc: + return transport_failure(exc, sending=False) + state["responded"] = True + parsed = _parse_json(content) + record = parsed if isinstance(parsed, dict) else None + status = response.status_code + if not 200 <= status < 300: + state["retryable"] = status in _RETRYABLE_HTTP or status >= 500 + return failure(_body_message(record, status)) + if record is None: + return failure("Workflow returned an invalid response") + if record.get("success") is False: + return failure(_body_message(record, status)) + if "outputs" not in record: + return failure("Workflow returned no outputs") + outputs = record["outputs"] + return InferenceOutcome( + outputs if isinstance(outputs, list) else [outputs], + {"latencyMs": (monotonic() - started) * 1000}, + responded=True, + ) + + +def check_inference_server( + session: requests.Session, url: str, api_key: Optional[str], *, timeout: float = PREFLIGHT_TIMEOUT +) -> None: + """Preflight: ``POST {url}/workflows/blocks/describe``, the web app's local-server check. + + Raises :class:`LocalRunnerError` (exit code 1) when the server is unreachable + or answers with a non-2xx status. + """ + headers = dict(INFERENCE_HEADERS) + if api_key: + headers["Api-Key"] = api_key + try: + response = session.post( + f"{url}/workflows/blocks/describe", + params={"api_key": api_key} if api_key else None, + data=_dumps({"dynamic_blocks_definitions": []}), + headers=headers, + timeout=timeout, + ) + except requests.Timeout: + raise LocalRunnerError( + f"The inference server at {url} did not respond within {timeout:g} seconds.", hint=PREFLIGHT_HINT + ) from None + except requests.RequestException: + raise LocalRunnerError(f"Could not reach the inference server at {url}.", hint=PREFLIGHT_HINT) from None + if not 200 <= response.status_code < 300: + parsed = _parse_json(response.content) + body = parsed if isinstance(parsed, dict) else {} + message = next( + (body[key] for key in ("message", "detail") if isinstance(body.get(key), str)), + f"Server returned {response.status_code}.", + ) + raise LocalRunnerError( + f"The inference server at {url} failed the preflight check: {message}", hint=PREFLIGHT_HINT + ) + + +# -- journal ------------------------------------------------------------------- + + +def default_state_dir(workspace: str, run_id: str) -> str: + """``$XDG_CACHE_HOME`` (or ``~/.cache``) ``/roboflow/workflow-evals//``.""" + base = os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache") + return os.path.join(base, "roboflow", "workflow-evals", _path_segment(workspace), _path_segment(run_id)) + + +def _path_segment(value: str) -> str: + return str(value).replace("/", "_").replace(os.sep, "_") or "_" + + +def _write_atomic(path: str, data: bytes) -> None: + temporary = f"{path}.{os.getpid()}.{threading.get_ident()}.tmp" + with open(temporary, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + + +class Journal: + """Append-only record of captures, so a restarted run resubmits them instead of re-running inference. + + ``journal.jsonl`` holds ``{"t": "captured", attempt, executionId, caseId, + shardIndex, sha256, path}`` and ``{"t": "submitted", attempt, executionId, + caseId, status, outcome}`` lines, each fsynced. ``path`` is relative to the + state directory. A later ``captured`` line for the same Case attempt + supersedes earlier ones. Unreadable lines (a torn last write) are ignored. + """ + + def __init__(self, state_dir: str) -> None: + self.state_dir = state_dir + self.path = os.path.join(state_dir, "journal.jsonl") + os.makedirs(os.path.join(state_dir, "captures"), exist_ok=True) + self._lock = threading.Lock() + self._captured: Dict[Tuple[int, str, str], Dict[str, Any]] = {} + self._load() + self._handle: Optional[Any] = open(self.path, "a", encoding="utf-8") # noqa: SIM115 + if self._torn_tail(): + # Terminate the torn line so the next record is not glued to it (and lost on reload). + self._handle.write("\n") + self._handle.flush() + os.fsync(self._handle.fileno()) + + def _torn_tail(self) -> bool: + with open(self.path, "rb") as handle: + handle.seek(0, os.SEEK_END) + if handle.tell() == 0: + return False + handle.seek(-1, os.SEEK_END) + return handle.read(1) != b"\n" + + def _load(self) -> None: + if not os.path.exists(self.path): + return + with open(self.path, encoding="utf-8") as handle: + for line in handle: + try: + entry = json.loads(line) + except ValueError: + continue + if isinstance(entry, dict) and entry.get("t") == "captured": + with contextlib.suppress(KeyError, TypeError, ValueError): + self._captured[(int(entry["attempt"]), str(entry["executionId"]), str(entry["caseId"]))] = entry + + def captured(self, attempt: int, execution_id: str, case_id: str) -> Optional[bytes]: + """Return the stored capture bytes for one Case attempt, or ``None`` (missing or corrupt file).""" + with self._lock: + entry = self._captured.get((attempt, execution_id, case_id)) + if entry is None: + return None + try: + with open(os.path.join(self.state_dir, str(entry["path"])), "rb") as handle: + data = handle.read() + except (OSError, KeyError): + return None + return data if hashlib.sha256(data).hexdigest() == entry.get("sha256") else None + + def record_capture(self, attempt: int, execution_id: str, case_id: str, shard_index: int, data: bytes) -> None: + """Write the capture file (tmp + fsync + rename), then append its ``captured`` line.""" + digest = hashlib.sha256(data).hexdigest() + relative = os.path.join("captures", f"{digest}.json") + target = os.path.join(self.state_dir, relative) + if not os.path.exists(target): + _write_atomic(target, data) + entry = { + "t": "captured", + "attempt": attempt, + "executionId": execution_id, + "caseId": case_id, + "shardIndex": shard_index, + "sha256": digest, + "path": relative, + } + self._append(entry) + with self._lock: + self._captured[(attempt, execution_id, case_id)] = entry + + def record_submission(self, attempt: int, execution_id: str, case_id: str, status: int, outcome: str) -> None: + self._append( + { + "t": "submitted", + "attempt": attempt, + "executionId": execution_id, + "caseId": case_id, + "status": status, + "outcome": outcome, + } + ) + + def _append(self, entry: Dict[str, Any]) -> None: + line = json.dumps(entry, separators=(",", ":")) + "\n" + with self._lock: + if self._handle is None: + return + self._handle.write(line) + self._handle.flush() + os.fsync(self._handle.fileno()) + + def close(self) -> None: + with self._lock: + if self._handle is not None: + self._handle.close() + self._handle = None + + +# -- runner -------------------------------------------------------------------- + + +def _field(resource: Any, key: str) -> Any: + """Read a Run field from ``{...}`` or ``{"run": {...}}``.""" + if not isinstance(resource, dict): + return None + nested = resource.get("run") + value = resource.get(key) + if value is None and isinstance(nested, dict): + value = nested.get(key) + return value + + +def _cancelled(run: Any) -> bool: + """``True`` for a cancelled Run, including one still finalizing its cancellation. + + Cancelling keeps the Run ``running`` (with ``cancellationRequestedAt`` set) + while the server marks the remaining Cases skipped, then finalizes it. + """ + state = _field(run, "state") + return state == "cancelled" or ( + state not in RUN_TERMINAL_STATES and _field(run, "cancellationRequestedAt") is not None + ) + + +def _parse_timestamp(value: Any) -> Optional[float]: + if not isinstance(value, str) or not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + except ValueError: + return None + + +def _already_done(checkpoint: Dict[str, Any], attempt: int) -> bool: + # browserRunQueue.ts: completed, or failed and not retryable / already this attempt. + state = checkpoint.get("state") + return state == "completed" or ( + state == "failed" and (checkpoint.get("retryable") is not True or checkpoint.get("attemptCount") == attempt) + ) + + +def _api_error(exc: WorkflowEvalError) -> LocalRunnerError: + status = exc.status_code + hint = exc.hint + if status == 401: + hint = "Check your API key, or run 'roboflow auth login'." + elif status == 403: + hint = "The API key needs the workflow-evals:read and workflow-evals:run scopes and cannot be folder-scoped." + elif status == 400: + hint = "The server rejected the local runner's request; please report this with the error details." + exit_code = 2 if status in (401, 403) else 3 if status == 404 else 1 + return LocalRunnerError(exc.message, hint=hint, exit_code=exit_code, details=exc.to_dict()) + + +def _transient(exc: BaseException) -> bool: + if isinstance(exc, WorkflowEvalError): + status = exc.status_code + return status is not None and (status in _RETRYABLE_HTTP or status >= 500) + return isinstance(exc, requests.RequestException) + + +def retry_api( + fn: Callable[[], Any], + *, + clock: Optional[Callable[[], float]] = None, + sleep: Optional[Callable[[float], Any]] = None, +) -> Any: + """Call ``fn()``, retrying Roboflow API timeouts, network errors, 408, 429 and 5xx. + + Backoff 1, 2, 4… up to 60 s; raises :class:`LocalRunnerError` (exit 1) after + 10 minutes of consecutive failures. Other errors propagate unchanged. + ``clock`` and ``sleep`` default to ``time.time`` and ``time.sleep``. + """ + clock = clock or time.time + sleep = sleep or time.sleep + first_failure: Optional[float] = None + delay = 1.0 + while True: + try: + return fn() + except (WorkflowEvalError, requests.RequestException) as exc: + if not _transient(exc): + raise + now = clock() + first_failure = now if first_failure is None else first_failure + if now - first_failure + delay > API_RETRY_BUDGET: + raise LocalRunnerError( + f"The Roboflow API kept failing for {API_RETRY_BUDGET // 60} minutes: {exc}", + hint=f"Check your network connection. {RESUME_HINT}", + ) from exc + sleep(delay) + delay = min(delay * 2, API_BACKOFF_CAP) + + +@dataclass +class _Pass: + """One pass over an Execution's Cases at one Run attempt.""" + + execution_id: str + index: int + total: int + specification: Any + workflow_id: Any + selected: Optional[int] + tally: ExecutionTally + attempt: int = 0 + # Set once the pass stops dispatching, so its leftover prefetch stops retrying. + ended: bool = False + + +@dataclass +class _Page: + cursor: Optional[str] + data: Dict[str, Any] + fetched_at: float + items: Dict[str, Dict[str, Any]] = field(default_factory=dict) + + def __post_init__(self) -> None: + self.items = {str(item["case"]["id"]): item for item in self.data.get("items") or []} + + @property + def attempt(self) -> int: + return int(self.data.get("attemptCount") or 0) + + @property + def urls(self) -> Dict[str, Any]: + return self.data.get("artifactReadUrls") or {} + + @property + def next_cursor(self) -> Optional[str]: + return self.data.get("nextCursor") or None + + @property + def closes_run(self) -> bool: + return bool(self.data.get("cancellationRequested")) or self.data.get("runState") in RUN_TERMINAL_STATES + + +@dataclass +class _Work: + ctx: _Pass + case_id: str + shard_index: int + attempt: int + cursor: Optional[str] + # When the page the Case's inputs came from was read; a retry re-reads an older page. + fetched_at: float = 0.0 + + +class LocalRunner: + """Runs a local Workflow Eval Run against a local Inference server. + + Args: + client: The workspace's :class:`WorkflowEvals` client. + eval_id: The Eval id. + run_id: The Run id; it must have ``executionMode: "browser"``. + inference_url: Inference server URL. Defaults to the URL frozen in the Run. + inference_api_key: API key sent to the Inference server. Defaults to the client's key. + execution_ids: Only these Executions (default: all, in Run order). + concurrency: Inferences in flight at once, 1 to 16. + case_timeout: Seconds allowed to connect and between bytes read for each inference attempt + (``requests`` timeout semantics), not a total deadline per Case; 0 means no limit. + state_dir: Journal directory. Defaults to :func:`default_state_dir`. + workspace: Workspace for the default ``state_dir``. Defaults to the client's. + retry: Retry a terminal Run (when the server allows it) before running. + on_progress: Receives a progress dict after every Case. + warn: Receives warnings meant for stderr. Defaults to the module logger. + clock: Wall clock in epoch seconds (leases, page age, API retry budget). + monotonic: Monotonic clock in seconds (latency, wall-clock time). + sleep: Sleeps for the given seconds. Defaults to a wait that a stop request interrupts. + session: ``requests.Session`` for the Inference server. + """ + + def __init__( + self, + client: WorkflowEvals, + eval_id: str, + run_id: str, + *, + inference_url: Optional[str] = None, + inference_api_key: Optional[str] = None, + execution_ids: Optional[Sequence[str]] = None, + concurrency: int = DEFAULT_CONCURRENCY, + case_timeout: float = DEFAULT_CASE_TIMEOUT, + state_dir: Optional[str] = None, + workspace: Optional[str] = None, + retry: bool = False, + on_progress: Optional[Callable[[Dict[str, Any]], None]] = None, + warn: Optional[Callable[[str], None]] = None, + clock: Callable[[], float] = time.time, + monotonic: Callable[[], float] = time.monotonic, + sleep: Optional[Callable[[float], Any]] = None, + session: Optional[requests.Session] = None, + ) -> None: + if not 1 <= concurrency <= MAX_CONCURRENCY: + raise ValueError(f"concurrency must be between 1 and {MAX_CONCURRENCY}") + if case_timeout < 0: + raise ValueError("case_timeout must be 0 (no limit) or positive") + self._client = client + self._eval_id = eval_id + self._run_id = run_id + self._inference_url = inference_url + self._inference_api_key = inference_api_key or client._api_key + self._execution_ids = list(dict.fromkeys(execution_ids)) if execution_ids else None + self._concurrency = concurrency + self._case_timeout = case_timeout or None + self._state_dir = state_dir or default_state_dir(workspace or client.workspace_url, run_id) + self._retry = retry + self._on_progress = on_progress + self._warn = warn or logger.warning + self._clock = clock + self._monotonic = monotonic + self._halt = threading.Event() + self._sleep: Callable[[float], Any] = sleep or self._halt.wait + self._owns_session = session is None + self._session = session or self._new_session() + + self._cond = threading.Condition(threading.RLock()) + self._stop_requested = False + self._abort = False + self._fatal: Optional[BaseException] = None + self._run_closed = False + self._reload = False + self._in_flight = 0 + self._outstanding = 0 + self._submitting = 0 + self._connection_failures = 0 + self._last_latency_ms: Optional[float] = None + self._threads: List[threading.Thread] = [] + self._tallies: List[ExecutionTally] = [] + self._journal: Optional[Journal] = None + self._url = "" + self._started = 0.0 + + def _new_session(self) -> requests.Session: + session = http.Session() + adapter = HTTPAdapter(pool_connections=1, pool_maxsize=max(10, self._concurrency)) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session + + # -- control --------------------------------------------------------------- + + def request_stop(self) -> None: + """Stop gracefully; a second call aborts at once. + + First call: no new inference is dispatched, in-flight inferences finish + and are journaled, already queued captures are submitted, retries + waiting in a backoff (inference or Roboflow API) are abandoned, then + :meth:`run` raises :class:`LocalRunnerInterrupted`. Second call: + :meth:`run` raises immediately without waiting for in-flight requests. + Safe to call from a signal handler or another thread. + """ + with self._cond: + if self._stop_requested: + self._abort = True + self._stop_requested = True + self._halt.set() + self._cond.notify_all() + + @property + def state_dir(self) -> str: + return self._state_dir + + @property + def aborting(self) -> bool: + """``True`` once :meth:`request_stop` was called twice.""" + return self._abort + + def _halting(self) -> bool: + return self._stop_requested or self._abort or self._fatal is not None + + def _dispatch_stopped(self) -> bool: + return self._halting() or self._run_closed or self._reload + + def _fail(self, exc: BaseException) -> None: + with self._cond: + if self._fatal is None: + self._fatal = exc + self._halt.set() + self._cond.notify_all() + + def _wait(self, predicate: Callable[[], bool], then: Optional[Callable[[], None]] = None) -> bool: + """Wait until ``predicate()`` (under the lock) or an abort; return ``False`` on abort. + + ``then`` runs under the same lock acquisition once the predicate holds. + """ + with self._cond: + while not predicate(): + if self._abort: + return False + # A bounded wait keeps the main thread responsive to signals on every platform. + self._cond.wait(0.5) + if self._abort: + return False + if then is not None: + then() + return True + + def _sleep_until(self, deadline: Optional[float]) -> None: + """Sleep until the wall-clock ``deadline``; raise :class:`_Abandoned` if the runner halts.""" + while deadline is not None and self._clock() < deadline: + if self._dispatch_stopped(): + raise _Abandoned + self._sleep(min(deadline - self._clock(), 5.0)) + + def _backoff(self, seconds: float, abandon: Optional[Callable[[], bool]] = None) -> None: + """Sleep before a retry; raise :class:`_Abandoned` if the runner halts (or ``abandon()``).""" + + def abandoned() -> bool: + return self._halting() or (abandon is not None and abandon()) + + if abandoned(): + raise _Abandoned + self._sleep(seconds) + if abandoned(): + raise _Abandoned + + # -- Roboflow API ---------------------------------------------------------- + + def _api( + self, fn: Callable[..., Any], *args: Any, abandon: Optional[Callable[[], bool]] = None, **kwargs: Any + ) -> Any: + """Call the Roboflow API, retrying timeouts, network errors, 408, 429 and 5xx. + + Backoff 1, 2, 4… up to 60 s; gives up (exit 1) after 10 minutes of + consecutive failures. Other errors propagate unchanged. Retries stop with + :class:`_Abandoned` when the runner halts or ``abandon()`` is true. + """ + return retry_api( + lambda: fn(*args, **kwargs), clock=self._clock, sleep=lambda seconds: self._backoff(seconds, abandon) + ) + + def _call(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: + """:meth:`_api`, mapping API errors to :class:`LocalRunnerError` exit codes.""" + try: + return self._api(fn, *args, **kwargs) + except WorkflowEvalError as exc: + raise _api_error(exc) from exc + + # -- run ------------------------------------------------------------------- + + def run(self) -> LocalRunResult: + """Run every selected Execution to the end and return the summary. + + Returns when the work is done (or the Run was already terminal). Raises + :class:`LocalRunnerError` on failure, including a cancelled Run (exit 1, + with ``result`` set), and :class:`LocalRunnerInterrupted` after + :meth:`request_stop`. + """ + self._started = self._monotonic() + try: + result = self._run() + if self._abort: + # An abort never reports success, even if the work happened to finish. + raise LocalRunnerInterrupted + return result + except KeyboardInterrupt: + with self._cond: + self._stop_requested = self._abort = True + self._halt.set() + self._cond.notify_all() + raise LocalRunnerInterrupted(result=self._result(None)) from None + except LocalRunnerInterrupted: + self._raise_if_halted() + raise + finally: + self._shutdown() + + def _run(self) -> LocalRunResult: + client, eval_id, run_id = self._client, self._eval_id, self._run_id + run = self._call(client.get_run, eval_id, run_id) + if _field(run, "executionMode") != "browser": + raise LocalRunnerError( + f"Run {run_id} does not run on a local inference server.", + hint="This Run executes on Roboflow; use `run get --wait`.", + ) + if _field(run, "state") in RUN_TERMINAL_STATES: + retry = _field(run, "retry") + if not (self._retry and isinstance(retry, dict) and retry.get("allowed")): + if self._retry: + self._warn(f"Run {run_id} is {_field(run, 'state')} and cannot be retried.") + return self._finish(run) + self._call(client.retry_run, eval_id, run_id, idempotency_key=workflowevalsapi.new_idempotency_key()) + + configuration = self._call(client.get_run_configuration, eval_id, run_id) + server = ((configuration or {}).get("sharedConfiguration") or {}).get("inferenceServer") or {} + frozen_url = str(server.get("url") or "").rstrip("/") + self._url = (self._inference_url or "").rstrip("/") or frozen_url + if not self._url: + raise LocalRunnerError(f"Run {run_id} has no inference server URL.", hint="Pass --inference-url.") + if self._inference_url and frozen_url and self._url != frozen_url: + self._warn( + f"Using the inference server {self._url} instead of {frozen_url}; the Run still records {frozen_url}." + ) + + executions = self._select_executions((self._call(client.list_executions, eval_id, run_id) or {}).get("items")) + check_inference_server(self._session, self._url, self._inference_api_key) + self._journal = Journal(self._state_dir) + for index, execution in enumerate(executions, 1): + if self._run_closed: + break + self._run_execution(index, len(executions), execution) + result = self._finish(self._call(client.get_run, eval_id, run_id)) + pending = sum(tally.pending for tally in self._tallies) + if pending: + cases = "1 Case was" if pending == 1 else f"{pending} Cases were" + raise InferenceServerUnreachable( + f"{cases} not run because the inference server at {self._url} was unreachable.", + hint="Start the inference server and run the same command again; captured Cases are not re-run.", + result=result, + ) + return result + + def _select_executions(self, items: Any) -> List[Dict[str, Any]]: + executions = [item for item in items or [] if isinstance(item, dict) and item.get("id") is not None] + if self._execution_ids: + by_id = {str(item["id"]): item for item in executions} + unknown = [execution_id for execution_id in self._execution_ids if execution_id not in by_id] + if unknown: + raise LocalRunnerError( + f"Execution {', '.join(unknown)} is not in Run {self._run_id}.", + hint="List the Run's Executions with `roboflow workflow-eval execution list`.", + exit_code=3, + ) + wanted = set(self._execution_ids) + executions = [item for item in executions if str(item["id"]) in wanted] + self._tallies = [ExecutionTally(str(item["id"])) for item in executions] + return executions + + def _result(self, state: Optional[str]) -> LocalRunResult: + return LocalRunResult( + run_id=self._run_id, + state=state, + executions=list(self._tallies), + wall_clock_seconds=round(self._monotonic() - self._started, 3), + concurrency=self._concurrency, + ) + + def _finish(self, run: Dict[str, Any]) -> LocalRunResult: + result = self._result(_field(run, "state")) + if _cancelled(run): + raise LocalRunnerError( + f"Run {self._run_id} was cancelled.", + hint="Start a new Run to evaluate again.", + result=result, + resumable=False, + ) + return result + + def _raise_if_halted(self) -> None: + if self._abort: + raise LocalRunnerInterrupted(result=self._result(None)) + if self._fatal is not None: + raise self._fatal + if self._stop_requested: + raise LocalRunnerInterrupted(result=self._result(None)) + + def _shutdown(self) -> None: + if not self._abort: + for thread in self._threads: + thread.join() + if self._journal is not None: + self._journal.close() + if self._owns_session: + self._session.close() + + # -- executions ------------------------------------------------------------ + + def _run_execution(self, index: int, total: int, execution: Dict[str, Any]) -> None: + execution_id = str(execution["id"]) + configuration = self._call(self._client.get_execution_configuration, self._eval_id, self._run_id, execution_id) + subject = ((configuration or {}).get("linkedPlan") or {}).get("workflowSubject") or {} + if subject.get("manifest") is None: + raise LocalRunnerError(f"Execution {execution_id} has no frozen Workflow specification.") + tally = self._tallies[index - 1] + selected = execution.get("selectedCaseCount") + reloads = 0 + while True: + ctx = _Pass( + execution_id=execution_id, + index=index, + total=total, + specification=subject["manifest"], + workflow_id=subject.get("id"), + selected=int(selected) if isinstance(selected, (int, float)) else None, + tally=tally, + ) + if not self._run_pass(ctx): + return + reloads += 1 + if reloads > MAX_RELOADS: + raise LocalRunnerError(f"The attempt of Run {self._run_id} kept changing; giving up.", hint=RESUME_HINT) + # 412 "Run attempt changed", 409 "No retained capture" on a resume, or a page at + # another attempt: reload the Run and restart this Execution at its current attempt. + # Every capture of the old pass has been answered (the pass drained), so the new pass + # sees them in the checkpoints; journal entries of an old attempt are ignored. + run = self._call(self._client.get_run, self._eval_id, self._run_id) + if _field(run, "state") in RUN_TERMINAL_STATES: + self._run_closed = True + return + tally.submitted = tally.already_done = tally.skipped = tally.failed = tally.pending = 0 + + def _run_pass(self, ctx: _Pass) -> bool: + """Dispatch one pass and drain it. Returns ``True`` when the Execution must restart.""" + with self._cond: + self._reload = False + try: + self._dispatch(ctx) + except BaseException as exc: + if not isinstance(exc, LocalRunnerInterrupted): + self._fail(exc) + raise + finally: + ctx.ended = True + self._wait(lambda: self._outstanding == 0) + self._raise_if_halted() + return self._reload and not self._run_closed + + def _fetch_page(self, ctx: _Pass, cursor: Optional[str], abandon: Optional[Callable[[], bool]] = None) -> _Page: + # Taken before the request: the server signs the URLs after this instant. + fetched_at = self._clock() + data = self._call( + self._client.list_execution_input_cases, + self._eval_id, + self._run_id, + ctx.execution_id, + limit=self._concurrency, + cursor=cursor, + abandon=abandon, + ) + if not isinstance(data, dict): + raise LocalRunnerError("The input-cases response is not an object.") + if any("workflowInputs" not in item for item in data.get("items") or []): + raise LocalRunnerError( + "The Roboflow server does not support local runner yet (input-cases has no workflowInputs).", + hint="Run this Eval from the browser until the server is updated.", + ) + return _Page(cursor, data, fetched_at) + + def _check_page(self, ctx: _Pass, page: _Page) -> bool: + """Return ``False`` (and flag why) when dispatching from ``page`` must stop.""" + with self._cond: + if page.closes_run: + self._run_closed = True + elif page.attempt != ctx.attempt: + self._reload = True + else: + return True + self._cond.notify_all() + return False + + def _prefetch(self, ctx: _Pass, cursor: str) -> Callable[[], _Page]: + """Fetch the next page in the background; the returned function waits for it.""" + box: Dict[str, Any] = {} + + def fetch() -> None: + try: + # Nobody waits for this page once dispatching stops, nor once its pass ends + # (the next pass resets ``_reload``). + box["page"] = self._fetch_page(ctx, cursor, abandon=lambda: ctx.ended or self._dispatch_stopped()) + except BaseException as exc: + box["error"] = exc + finally: + with self._cond: + box["done"] = True + self._cond.notify_all() + + self._spawn(fetch, holds_outstanding=False) + + def result() -> _Page: + if not self._wait(lambda: "done" in box): + raise _Abandoned + if "error" in box: + raise box["error"] + return box["page"] + + return result + + def _dispatch(self, ctx: _Pass) -> None: + page = self._fetch_page(ctx, None) + ctx.attempt = page.attempt + while self._check_page(ctx, page): + upcoming = self._prefetch(ctx, page.next_cursor) if page.next_cursor else None + for case_id in list(page.items): + if self._dispatch_stopped() or not self._dispatch_case(ctx, page, case_id): + return + if upcoming is None or self._dispatch_stopped(): + return + try: + page = upcoming() + except _Abandoned: + if self._halting(): + raise + # The prefetch gave up because of a 412 or a closed Run: end the pass. + return + + def _refresh(self, ctx: _Pass, page: _Page) -> bool: + """Re-read ``page`` (same cursor) if its signed URLs are about to expire.""" + if self._clock() - page.fetched_at <= PAGE_MAX_AGE: + return True + fresh = self._fetch_page(ctx, page.cursor) + page.data, page.fetched_at, page.items = fresh.data, fresh.fetched_at, fresh.items + return self._check_page(ctx, page) + + def _dispatch_case(self, ctx: _Pass, page: _Page, case_id: str) -> bool: + """Decide and dispatch one Case. Returns ``False`` when dispatching must stop.""" + while True: + if not self._refresh(ctx, page): + return False + item = page.items.get(case_id) + if item is None: + return True + checkpoint = item.get("checkpoint") or {} + if _already_done(checkpoint, page.attempt): + self._count(ctx, "already_done") + return True + assert self._journal is not None + stored = self._journal.captured(page.attempt, ctx.execution_id, case_id) + resume = ( + stored is None and bool(checkpoint.get("hasCapture")) and checkpoint.get("attemptCount") == page.attempt + ) + input_failure = input_failure_message(item) if stored is None and not resume else None + needs_slot = stored is None and not resume and input_failure is None + # Wait for room in the window (and, for a capture, in the submission backlog). + if not self._wait(lambda: self._dispatch_stopped() or self._has_room(needs_slot)): + return False + if self._dispatch_stopped(): + return False + if self._clock() - page.fetched_at > PAGE_MAX_AGE: + continue + work = _Work(ctx, case_id, int(item.get("shardIndex") or 0), page.attempt, page.cursor, page.fetched_at) + with self._cond: + self._outstanding += 1 + if needs_slot: + self._in_flight += 1 + if stored is not None: + self._spawn(self._submit, work, stored, None) + elif resume: + self._spawn(self._submit, work, None, _parse_timestamp(checkpoint.get("leaseExpiresAt"))) + elif input_failure is not None: + self._spawn(self._submit_input_failure, work, input_failure) + else: + self._spawn(self._infer_and_submit, work, case_inputs(item, page.urls)) + self._emit_progress(ctx) + return True + + def _has_room(self, needs_slot: bool) -> bool: + # In-flight inferences stay under N; captures waiting for or in submission + # (N submitting plus a queue of N) bound the backlog, so dispatch blocks when + # the Roboflow API is slow. + if needs_slot and self._in_flight >= self._concurrency: + return False + return self._outstanding < 2 * self._concurrency + + def _spawn(self, target: Callable[..., None], *args: Any, holds_outstanding: bool = True) -> None: + def body() -> None: + try: + target(*args) + except LocalRunnerInterrupted: + pass + except BaseException as exc: + self._fail(exc) + finally: + if holds_outstanding: + with self._cond: + self._outstanding -= 1 + self._cond.notify_all() + + thread = threading.Thread(target=body, name="roboflow-workflow-eval-local", daemon=True) + self._threads = [alive for alive in self._threads if alive.is_alive()] + self._threads.append(thread) + thread.start() + + # -- per Case -------------------------------------------------------------- + + def _infer_and_submit(self, work: _Work, inputs: Dict[str, Any]) -> None: + outcome: Optional[InferenceOutcome] = None + data = b"" + try: + outcome = self._infer(work, inputs) + if outcome is None or outcome.connection_failure: + # Unreachable inference server: infrastructure, not a Case failure. Nothing is + # captured or journaled, so the Case stays pending on the server. + return + capture = { + "attemptCount": work.attempt, + "shardIndex": work.shard_index, + "output": outcome.output, + "runtime": outcome.runtime, + } + try: + data = _dumps(capture) + except ValueError as exc: + # A lone surrogate or a non-finite number (1e400) has no JSON encoding. + message = _truncate(f"Workflow output could not be serialized: {exc}") + self._warn(f"Case {work.case_id}: {message}; submitting it as a failed Case.") + runtime = {"latencyMs": outcome.runtime.get("latencyMs"), "error": message} + capture.update(output=None, runtime={**runtime, "timedOut": False, "retryable": False}) + data = _dumps(capture) + assert self._journal is not None + self._journal.record_capture(work.attempt, work.ctx.execution_id, work.case_id, work.shard_index, data) + finally: + # The slot frees as soon as the inference result is journaled. + with self._cond: + self._in_flight -= 1 + if outcome is not None: + self._last_latency_ms = outcome.runtime.get("latencyMs") + self._cond.notify_all() + if outcome is not None and outcome.connection_failure: + self._count(work.ctx, "pending") + self._submit(work, data, None) + + def _submit_input_failure(self, work: _Work, message: str) -> None: + # The engine's input failure capture: no inference, and a runtime with only ``error``. + capture = { + "attemptCount": work.attempt, + "shardIndex": work.shard_index, + "output": None, + "runtime": {"error": message}, + } + data = _dumps(capture) + assert self._journal is not None + self._journal.record_capture(work.attempt, work.ctx.execution_id, work.case_id, work.shard_index, data) + self._submit(work, data, None) + + def _infer(self, work: _Work, inputs: Dict[str, Any]) -> Optional[InferenceOutcome]: + """Run the inference with retries; ``None`` means abandoned (nothing to capture). + + Two budgets per Case. An attempt that reached the server and failed with + HTTP 408/429/5xx or a dropped connection is retried, up to 3 attempts + (backoff 1, 2 s); the last outcome is returned and captured. An attempt + that could not connect is retried after 1, 2, 4, 8 and 16 s; when the + last one cannot connect either, the outcome (``connection_failure`` set, + not captured) is returned and the Case counts once toward the limit of + consecutive Cases that could not connect. Any attempt that reaches the + server resets that count. A read timeout (inference slower than + ``case_timeout``) is returned at once, like the browser's single attempt. + + Before each retry, inputs from a page older than ``PAGE_MAX_AGE`` are + rebuilt from a re-read of that page, so the signed URLs are fresh. + """ + ctx = work.ctx + connected_attempts = 0 + connect_failures = 0 + while True: + outcome = run_workflow_inference( + self._session, + self._url, + api_key=self._inference_api_key, + specification=ctx.specification, + workflow_id=ctx.workflow_id, + inputs=inputs, + timeout=self._case_timeout, + monotonic=self._monotonic, + ) + if outcome.connection_failure: + connect_failures += 1 + if connect_failures > len(CONNECT_BACKOFF): + self._count_unreachable_case() + return outcome + delay = float(CONNECT_BACKOFF[connect_failures - 1]) + else: + with self._cond: + self._connection_failures = 0 + connected_attempts += 1 + if outcome.runtime.get("timedOut"): + return outcome + if not outcome.error or not outcome.retryable or connected_attempts >= INFERENCE_ATTEMPTS: + return outcome + delay = min(2.0 ** (connected_attempts - 1), INFERENCE_BACKOFF_CAP) + try: + self._backoff(delay) + except _Abandoned: + return None + if self._clock() - work.fetched_at > PAGE_MAX_AGE: + fresh = self._reread_inputs(work) + # A stop or a fatal error during the re-read abandons the Case too. + if fresh is None or self._halting(): + return None + inputs = fresh + + def _count_unreachable_case(self) -> None: + """Count a Case that gave up without connecting; stop the run after ``MAX_CONNECTION_FAILURES`` in a row.""" + with self._cond: + self._connection_failures += 1 + if self._connection_failures >= MAX_CONNECTION_FAILURES: + self._fail( + LocalRunnerError( + f"The inference server at {self._url} failed to connect for {MAX_CONNECTION_FAILURES} Cases " + "in a row.", + hint=f"{PREFLIGHT_HINT} {RESUME_HINT}", + ) + ) + + def _reread_inputs(self, work: _Work) -> Optional[Dict[str, Any]]: + """Re-read the Case's page (same cursor) and rebuild its inputs with fresh signed URLs. + + Returns ``None`` when the Case must be abandoned: it left the page + (flagged ``_reload`` so the Execution restarts and finds the Case + wherever it is now), the page is at another attempt or closes the Run + (both flagged like :meth:`_check_page`, since ``work.attempt`` is the + pass's attempt), or dispatching stopped while the re-read was retrying. + The dispatcher's page is not touched. + """ + try: + page = self._fetch_page(work.ctx, work.cursor, abandon=self._dispatch_stopped) + except _Abandoned: + return None + if not self._check_page(work.ctx, page): + return None + item = page.items.get(work.case_id) + if item is None: + self._flag("_reload") + return None + work.fetched_at = page.fetched_at + return case_inputs(item, page.urls) + + def _submit(self, work: _Work, data: Optional[bytes], lease: Optional[float]) -> None: + """Submit a capture (``data``) or a ``resume`` (``data is None``) and act on the answer. + + Every resend sends the same JSON value (parsed from the same stored bytes). + A ``resume`` first waits for the ``lease`` another runner holds on the Case + (:meth:`_wait_for_lease`). + """ + ctx = work.ctx + resends = 0 + replaced = False + if data is not None: + body = json.loads(data) + # Measured as sent: ``requests`` re-encodes ``json=body`` with its default separators + # and ``ensure_ascii``, so the bytes on the wire are not ``data``. + if len(json.dumps(body, allow_nan=False).encode()) > CAPTURE_LIMIT_BYTES: + data = self._too_large(work, body) + replaced = True + if lease is not None: + self._wait_for_lease(work, lease) + while True: + if data is not None: + body = json.loads(data) + else: + body = {"attemptCount": work.attempt, "shardIndex": work.shard_index, "resume": True} + error: Optional[WorkflowEvalError] = None + if not self._wait(lambda: self._submitting < self._concurrency, self._take_submitter): + return + try: + self._api( + self._client.submit_capture, + self._eval_id, + self._run_id, + ctx.execution_id, + work.case_id, + body, + timeout=CAPTURE_TIMEOUT, + ) + except WorkflowEvalError as exc: + error = exc + finally: + with self._cond: + self._submitting -= 1 + self._cond.notify_all() + if error is None: + self._record(work, 200, "submitted") + failed = data is not None and "error" in (body.get("runtime") or {}) + self._count(ctx, "submitted", failed=failed) + return + status, message = error.status_code, error.message or "" + if status == 409 and message.startswith("Different capture already submitted"): + # Another runner (a tab or another CLI) won this Case. + self._record(work, 409, "done_by_other") + self._count(ctx, "already_done") + return + if status == 409 and ( + error.details.get("reason") == "non_retryable_failure" + or message.startswith("Case failure is not retryable") + ): + self._record(work, 409, "skipped") + self._count(ctx, "skipped") + return + if status == 409 and message.startswith("Run no longer accepts captures"): + self._record(work, 409, "run_closed") + self._flag("_run_closed") + return + if status == 409 and (error.retryable is True or message.startswith("Capture is being processed")): + resends += 1 + if resends > PROCESSING_RESENDS: + raise LocalRunnerError( + f"Case {work.case_id} is still being processed after {PROCESSING_RESENDS} resends.", + hint=RESUME_HINT, + details=error.to_dict(), + ) + # Without a leaseExpiresAt on the page, back off 2, 4, 8… s. + self._wait_for_lease(work, self._clock() + min(2.0**resends, API_BACKOFF_CAP), reread=True) + continue + if status == 412 or (status == 409 and message.startswith("No retained capture")): + # The Run attempt changed (or the retained capture is gone): restart the Execution. + self._record(work, status, "reload") + self._flag("_reload") + return + if status == 413 and data is not None and not replaced: + data = self._too_large(work, body) + replaced = True + continue + raise _api_error(error) from error + + def _take_submitter(self) -> None: + self._submitting += 1 + + def _too_large(self, work: _Work, body: Dict[str, Any]) -> bytes: + latency = (body.get("runtime") or {}).get("latencyMs") + self._warn(f"Case {work.case_id}: {CAPTURE_TOO_LARGE_MESSAGE}; submitting it as a failed Case.") + runtime: Dict[str, Any] = {"latencyMs": latency, "error": CAPTURE_TOO_LARGE_MESSAGE, "retryable": False} + if latency is None: + del runtime["latencyMs"] + data = _dumps( + {"attemptCount": work.attempt, "shardIndex": work.shard_index, "output": None, "runtime": runtime} + ) + assert self._journal is not None + self._journal.record_capture(work.attempt, work.ctx.execution_id, work.case_id, work.shard_index, data) + return data + + def _wait_for_lease(self, work: _Work, deadline: float, *, reread: bool = False) -> None: + """Wait until ``deadline``, the end of the lease another runner holds on the Case. + + Meanwhile the Case's page (same cursor) is re-read every + ``LEASE_POLL_INTERVAL`` seconds, and at once when ``reread``. Its + ``leaseExpiresAt`` replaces ``deadline``; a ``completed`` or ``failed`` + checkpoint ends the wait, and the resend gets the server's answer. + Raises :class:`_Abandoned` when dispatching stops, when the page closes + the Run or is at another attempt (flagged by :meth:`_check_page`), or + when the Case left the page (flagged ``_reload``, as in + :meth:`_reread_inputs`). + """ + next_read = self._clock() if reread else self._clock() + LEASE_POLL_INTERVAL + while self._clock() < deadline: + self._sleep_until(min(deadline, next_read)) + if self._clock() >= deadline: + return + if self._dispatch_stopped(): + raise _Abandoned + page = self._fetch_page(work.ctx, work.cursor, abandon=self._dispatch_stopped) + if not self._check_page(work.ctx, page): + raise _Abandoned + item = page.items.get(work.case_id) + if item is None: + self._flag("_reload") + raise _Abandoned + checkpoint = item.get("checkpoint") or {} + if checkpoint.get("state") in ("completed", "failed"): + return + deadline = _parse_timestamp(checkpoint.get("leaseExpiresAt")) or deadline + next_read = self._clock() + LEASE_POLL_INTERVAL + + def _record(self, work: _Work, status: int, outcome: str) -> None: + assert self._journal is not None + self._journal.record_submission(work.attempt, work.ctx.execution_id, work.case_id, status, outcome) + + def _flag(self, name: str) -> None: + with self._cond: + setattr(self, name, True) + self._cond.notify_all() + + # -- progress -------------------------------------------------------------- + + def _count(self, ctx: _Pass, name: str, *, failed: bool = False) -> None: + with self._cond: + setattr(ctx.tally, name, getattr(ctx.tally, name) + 1) + if failed: + ctx.tally.failed += 1 + self._emit_progress(ctx) + + def _emit_progress(self, ctx: _Pass) -> None: + if self._on_progress is None: + return + with self._cond: + tally = ctx.tally + done = tally.submitted + tally.already_done + tally.skipped + tally.pending + progress = { + "executionId": ctx.execution_id, + "execution": ctx.index, + "executions": ctx.total, + "ok": tally.submitted - tally.failed, + "failed": tally.failed, + "alreadyDone": tally.already_done, + "skipped": tally.skipped, + "pending": tally.pending, + "remaining": max(ctx.selected - done, 0) if ctx.selected is not None else None, + "inFlight": self._in_flight, + "lastLatencyMs": self._last_latency_ms, + } + self._on_progress(progress) + + +def run_local(client: WorkflowEvals, eval_id: str, run_id: str, **options: Any) -> LocalRunResult: + """Shorthand for ``LocalRunner(client, eval_id, run_id, **options).run()``.""" + return LocalRunner(client, eval_id, run_id, **options).run() + + +@contextlib.contextmanager +def install_signal_handlers( + runner: LocalRunner, + signals: Sequence[int] = (signal.SIGINT, signal.SIGTERM), + on_signal: Optional[Callable[[int], None]] = None, +) -> Iterator[None]: + """Route SIGINT/SIGTERM to ``runner.request_stop()`` and restore the previous handlers afterwards. + + Must be used from the main thread. The first signal stops gracefully, the second aborts: + it raises :class:`LocalRunnerInterrupted` from the handler, so a blocking API call on the + main thread is interrupted at once (:meth:`LocalRunner.run` attaches the result). + ``on_signal`` receives each signal number first (for example to tell the user what happens next). + """ + previous: Dict[int, Any] = {} + + def handler(signum: int, frame: Any) -> None: + if on_signal is not None: + on_signal(signum) + runner.request_stop() + if runner.aborting: + raise LocalRunnerInterrupted + + for signum in signals: + previous[signum] = signal.signal(signum, handler) + try: + yield + finally: + for signum, original in previous.items(): + signal.signal(signum, original) diff --git a/tests/cli/test_workflow_eval_handler.py b/tests/cli/test_workflow_eval_handler.py index db8c23e2..c5d709bb 100644 --- a/tests/cli/test_workflow_eval_handler.py +++ b/tests/cli/test_workflow_eval_handler.py @@ -4,14 +4,27 @@ import json import os +import signal import tempfile import unittest -from unittest.mock import ANY, patch +from typing import Any, Callable, Dict, List, Tuple +from unittest.mock import ANY, MagicMock, patch +import responses from typer.testing import CliRunner from roboflow.adapters.workflowevalsapi import WorkflowEvalError from roboflow.cli import app +from roboflow.cli.handlers import workflow_eval +from roboflow.core.workflow_eval_local_runner import ( + PREFLIGHT_HINT, + RESUME_HINT, + ExecutionTally, + InferenceServerUnreachable, + LocalRunnerError, + LocalRunnerInterrupted, + LocalRunResult, +) runner = CliRunner() @@ -32,7 +45,7 @@ "case": ["list", "get", "add", "replace", "delete", "import", "import-status", "upload"], "binding": ["suggest", "validate"], "preparation": ["start", "current", "get", "resume", "cases"], - "run": ["list", "get", "start", "cancel", "retry", "replay", "rescore", "delete", "config"], + "run": ["list", "get", "start", "local", "cancel", "retry", "replay", "rescore", "delete", "config"], "execution": ["list", "get", "config", "input-cases", "capture", "overview", "results", "result"], "export": ["start", "status", "download-url"], "embedding": ["start", "get", "cancel"], @@ -418,5 +431,563 @@ def test_progress_is_suppressed_with_json_and_quiet(self) -> None: self.assertEqual(json.loads(result.stdout)["state"], "completed") +RUNNER = "roboflow.core.workflow_eval_local_runner.LocalRunner" +LOCAL = ["workflow-eval", "run", "local", "r1", "-e", "e1"] + + +def _result(state: str = "completed") -> LocalRunResult: + tally = ExecutionTally("x1", submitted=9, already_done=2, skipped=0, failed=1) + return LocalRunResult(run_id="r1", state=state, executions=[tally], wall_clock_seconds=12.5, concurrency=4) + + +class _FakeRunner: + """Stands in for ``LocalRunner``: ``behaviour(self)`` is what ``run()`` does.""" + + instances: List["_FakeRunner"] = [] + behaviour: Callable[["_FakeRunner"], LocalRunResult] = staticmethod(lambda runner: _result()) + + def __init__(self, client: Any, eval_id: str, run_id: str, **options: Any) -> None: + self.eval_id, self.run_id, self.options = eval_id, run_id, options + self.state_dir = "/tmp/rf-state/r1" + self.stop_requests = 0 + _FakeRunner.instances.append(self) + + def request_stop(self) -> None: + self.stop_requests += 1 + + @property + def aborting(self) -> bool: + return self.stop_requests > 1 + + def run(self) -> LocalRunResult: + return type(self).behaviour(self) + + +def _fake(behaviour: Callable[[_FakeRunner], Any]) -> type: + _FakeRunner.instances = [] + return type("FakeRunner", (_FakeRunner,), {"behaviour": staticmethod(behaviour)}) + + +def _raise(exc: Exception) -> Callable[[_FakeRunner], Any]: + def behaviour(runner: _FakeRunner) -> Any: + raise exc + + return behaviour + + +class TestWorkflowEvalRunLocal(unittest.TestCase): + def _run(self, globals_: tuple = (), extra: tuple = (), behaviour=lambda runner: _result()): + with patch(RUNNER, _fake(behaviour)): + return _invoke(*globals_, *LOCAL, *extra) + + def test_json_output_shape_and_runner_options(self) -> None: + result = self._run( + ("--json",), + ("--inference-url", "http://gpu:9001", "-x", "x1", "-x", "x2", "--concurrency", "8", "--case-timeout", "0"), + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual( + json.loads(result.stdout), + { + "runId": "r1", + "state": "completed", + "executions": [{"id": "x1", "submitted": 9, "alreadyDone": 2, "skipped": 0, "failed": 1, "pending": 0}], + "wallClockSeconds": 12.5, + "concurrency": 4, + "warnings": [], + }, + ) + runner = _FakeRunner.instances[0] + self.assertEqual((runner.eval_id, runner.run_id), ("e1", "r1")) + self.assertEqual(runner.options["inference_url"], "http://gpu:9001") + self.assertEqual(runner.options["execution_ids"], ["x1", "x2"]) + self.assertEqual(runner.options["concurrency"], 8) + self.assertEqual(runner.options["case_timeout"], 0) + self.assertFalse(runner.options["retry"]) + + def test_text_output_is_a_short_table(self) -> None: + result = self._run() + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("Run r1 · completed · 12.5s · concurrency 4", result.stdout) + self.assertIn("ALREADY DONE", result.stdout) + self.assertIn("PENDING", result.stdout) + + def test_defaults(self) -> None: + result = self._run(("--json",)) + self.assertEqual(result.exit_code, 0, result.output) + options = _FakeRunner.instances[0].options + self.assertEqual((options["concurrency"], options["case_timeout"]), (4, 300)) + self.assertIsNone(options["inference_url"]) + self.assertIsNone(options["state_dir"]) + + def test_run_option_is_not_accepted(self) -> None: + with patch(RUNNER, _fake(lambda runner: _result())): + result = _invoke("workflow-eval", "run", "local", "-e", "e1", "-r", "r1") + self.assertEqual(result.exit_code, 2) + self.assertIn("No such option '-r'", result.output) + self.assertEqual(_FakeRunner.instances, []) + + def test_concurrency_out_of_range_is_rejected(self) -> None: + for value in ("0", "17"): + with self.subTest(concurrency=value): + result = self._run(("--json",), ("--concurrency", value)) + self.assertEqual(result.exit_code, 1) + error = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertIn("--concurrency must be between 1 and 16", error["message"]) + self.assertEqual(_FakeRunner.instances, []) + + def test_negative_case_timeout_is_rejected(self) -> None: + result = self._run((), ("--case-timeout", "-1")) + self.assertEqual(result.exit_code, 1) + self.assertIn("--case-timeout", result.stderr) + + def test_runner_error_exit_codes(self) -> None: + cases = { + 1: LocalRunnerError("Run r1 was cancelled.", hint="Start a new Run.", result=_result("cancelled")), + 2: LocalRunnerError("Missing scope", hint="needs workflow-evals:read and run", exit_code=2), + 3: LocalRunnerError("Run not found", exit_code=3), + } + for code, exc in cases.items(): + with self.subTest(exit_code=code): + result = self._run(("--json",), behaviour=_raise(exc)) + self.assertEqual(result.exit_code, code) + self.assertEqual(result.stdout, "") + error = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertEqual(error["message"], exc.message) + if exc.hint: + self.assertEqual(error["hint"], exc.hint) + cancelled = self._run(("--json",), behaviour=_raise(cases[1])) + error = json.loads(cancelled.stderr.strip().splitlines()[-1])["error"] + self.assertEqual(error["result"]["state"], "cancelled") + + def test_interrupted_exits_130_with_resume_hint(self) -> None: + result = self._run(behaviour=_raise(LocalRunnerInterrupted())) + self.assertEqual(result.exit_code, 130) + self.assertIn("Run the same command again to resume", result.stderr) + self.assertIn("/tmp/rf-state/r1", result.stderr) + + def test_signal_handlers_are_installed_and_restored(self) -> None: + before = {signum: signal.getsignal(signum) for signum in (signal.SIGINT, signal.SIGTERM)} + seen: Dict[str, Any] = {} + + def behaviour(runner: _FakeRunner) -> Any: + handler = signal.getsignal(signal.SIGTERM) + seen["installed"] = handler is not before[signal.SIGTERM] and callable(handler) + handler(signal.SIGINT, None) # first signal: notice + graceful stop + seen["stops"] = runner.stop_requests + handler(signal.SIGINT, None) # second: abort, raised from the handler + seen["returned"] = True + + result = self._run(behaviour=behaviour) + self.assertEqual(result.exit_code, 130) + self.assertTrue(seen["installed"]) + self.assertEqual(seen["stops"], 1) + self.assertNotIn("returned", seen) + self.assertEqual(_FakeRunner.instances[0].stop_requests, 2) + self.assertIn("/tmp/rf-state/r1", result.stderr) # resume hint + self.assertEqual(result.stderr.count("Stopping after in-flight cases"), 1) + self.assertEqual({signum: signal.getsignal(signum) for signum in before}, before) + + def test_progress_is_throttled_on_stderr(self) -> None: + clock = [0.0] + + def behaviour(runner: _FakeRunner) -> Any: + for done in range(25): + runner.options["on_progress"]( + { + "executionId": "x1", + "execution": 1, + "executions": 2, + "ok": done, + "failed": 1, + "alreadyDone": 0, + "skipped": 0, + "remaining": 30 - done, + "inFlight": 4, + "lastLatencyMs": 1234.4, + } + ) + clock[0] += 1 + return _result() + + with patch("time.monotonic", side_effect=lambda: clock[0]): + result = self._run(behaviour=behaviour) + self.assertEqual(result.exit_code, 0, result.output) + lines = [line for line in result.stderr.splitlines() if line.startswith("Execution")] + self.assertEqual(len(lines), 3) # t = 0, 10, 20 + self.assertEqual( + lines[0], "Execution 1/2 · cases 0 ok / 1 failed / 30 remaining · in flight 4 · last latency 1234 ms" + ) + + def test_progress_and_warnings_with_json_and_quiet(self) -> None: + def behaviour(runner: _FakeRunner) -> Any: + runner.options["warn"]("Using the inference server A instead of B") + progress = {"execution": 1, "executions": 1, "ok": 1, "failed": 0, "remaining": 0, "inFlight": 0} + runner.options["on_progress"](progress) + return _result() + + for flag in ("--json", "--quiet"): + with self.subTest(flag=flag): + result = self._run((flag,), behaviour=behaviour) + self.assertEqual(result.exit_code, 0, result.output) + self.assertNotIn("Execution 1/1", result.stderr) + if flag == "--json": + self.assertEqual(result.stderr, "") + summary = json.loads(result.stdout) + self.assertEqual(summary["state"], "completed") + self.assertEqual(summary["warnings"], ["Using the inference server A instead of B"]) + else: + self.assertIn("Warning: Using the inference server A instead of B", result.stderr) + + def test_unreachable_inference_server_exits_1_with_resume_hint(self) -> None: + error = InferenceServerUnreachable( + "2 Cases were not run because the inference server at http://localhost:9001 was unreachable.", + hint="Start the inference server and run the same command again.", + result=_result(), + ) + result = self._run(("--json",), behaviour=_raise(error)) + self.assertEqual(result.exit_code, 1) + self.assertEqual(result.stdout, "") + payload = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertIn("were not run", payload["message"]) + self.assertEqual( + payload["hint"], + "Start the inference server. Run the same command again to resume; the journal is in /tmp/rf-state/r1.", + ) + self.assertIn("pending", payload["result"]["executions"][0]) + + @patch(f"{CLIENT}.get_run") + def test_non_terminal_state_is_read_once_more(self, mock_get_run) -> None: + mock_get_run.return_value = {"id": "r1", "state": "completed"} + result = self._run(("--json",), behaviour=lambda runner: _result("running")) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(json.loads(result.stdout)["state"], "completed") + mock_get_run.assert_called_once_with("e1", "r1") + + @patch(f"{CLIENT}.get_run") + def test_run_still_finalizing_its_cancellation_exits_1(self, mock_get_run) -> None: + mock_get_run.return_value = {"run": {"id": "r1", "state": "running", "cancellationRequestedAt": "2026-10-22"}} + result = self._run(("--json",), behaviour=lambda runner: _result("running")) + self.assertEqual(result.exit_code, 1) + error = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertEqual(error["message"], "Run r1 was cancelled.") + + def _run_with_fake_clock(self) -> Tuple[Any, List[float]]: + clock = [1000.0] + sleeps: List[float] = [] + + def sleep(seconds: float) -> None: + sleeps.append(seconds) + clock[0] += seconds + + with patch("time.time", side_effect=lambda: clock[0]), patch("time.sleep", side_effect=sleep): + result = self._run(("--json",), behaviour=lambda runner: _result("running")) + return result, sleeps + + @patch(f"{CLIENT}.get_run") + def test_transient_error_on_the_final_read_is_retried(self, mock_get_run) -> None: + unavailable = WorkflowEvalError("Service Unavailable", status_code=503) + mock_get_run.side_effect = [unavailable, {"id": "r1", "state": "completed"}] + result, sleeps = self._run_with_fake_clock() + self.assertEqual(result.exit_code, 0, result.output) + summary = json.loads(result.stdout) + self.assertEqual(summary["state"], "completed") + self.assertEqual(summary["warnings"], []) + self.assertEqual(sleeps, [1.0]) + + @patch(f"{CLIENT}.get_run") + def test_final_read_failing_past_the_retry_budget_exits_1(self, mock_get_run) -> None: + mock_get_run.side_effect = WorkflowEvalError("Service Unavailable", status_code=503) + result, sleeps = self._run_with_fake_clock() + self.assertEqual(result.exit_code, 1) + self.assertEqual(result.stdout, "") + error = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertIn("kept failing for 10 minutes", error["message"]) + self.assertEqual(max(sleeps), 60) + self.assertLessEqual(sum(sleeps), 600) + + def test_warnings_go_to_the_json_error(self) -> None: + def behaviour(runner: _FakeRunner) -> Any: + runner.options["warn"]("Using the inference server A instead of B") + raise LocalRunnerError("Boom", hint="Fix it.") + + result = self._run(("--json",), behaviour=behaviour) + self.assertEqual(result.exit_code, 1) + error = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertEqual(error["message"], "Boom") + self.assertEqual(error["warnings"], ["Using the inference server A instead of B"]) + + def test_warnings_go_to_the_json_error_of_other_exceptions(self) -> None: + def behaviour(runner: _FakeRunner) -> Any: + runner.options["warn"]("Using the inference server A instead of B") + raise RuntimeError("Boom") + + result = self._run(("--json",), behaviour=behaviour) + self.assertEqual(result.exit_code, 1) + error = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertEqual(error["message"], "Boom") + self.assertEqual(error["warnings"], ["Using the inference server A instead of B"]) + + def test_other_errors_are_not_wrapped(self) -> None: + # Unlike `run start --inference-url`, `run local` did not create the Run: no runId to keep. + error = PermissionError(13, "Permission denied", "/tmp/rf-state/r1") + fail = MagicMock(wraps=workflow_eval._fail) + with patch.object(workflow_eval, "_fail", fail): + result = self._run(("--json",), behaviour=_raise(error)) + self.assertEqual(result.exit_code, 1) + self.assertIs(fail.call_args.args[1], error) + payload = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertIn("Permission denied", payload["message"]) + self.assertNotIn("details", payload) + + @patch(f"{CLIENT}.get_run") + def test_server_run_is_rejected_with_hint(self, mock_get_run) -> None: + mock_get_run.return_value = {"id": "r1", "state": "running", "executionMode": "server"} + with tempfile.TemporaryDirectory() as state_dir: + result = _invoke("--json", *LOCAL, "--state-dir", state_dir) + self.assertEqual(result.exit_code, 1) + error = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertEqual(error["hint"], "This Run executes on Roboflow; use `run get --wait`.") + + @responses.activate + @patch(f"{CLIENT}.list_executions", return_value={"items": [{"id": "x1"}]}) + @patch(f"{CLIENT}.get_run_configuration") + @patch(f"{CLIENT}.get_run", return_value={"id": "r1", "state": "running", "executionMode": "browser"}) + def test_preflight_failure_exits_1_with_hint(self, _get_run, mock_config, _executions) -> None: + mock_config.return_value = {"sharedConfiguration": {"inferenceServer": {"url": "http://localhost:9001"}}} + # No inference server is registered with ``responses``: the preflight POST fails to connect. + with tempfile.TemporaryDirectory() as state_dir: + result = _invoke("--json", *LOCAL, "--state-dir", state_dir) + self.assertEqual(result.exit_code, 1) + error = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertIn("http://localhost:9001", error["message"]) + self.assertEqual(error["hint"], PREFLIGHT_HINT) + + +class TestWorkflowEvalRunStartLocal(unittest.TestCase): + BODY = json.dumps({"executions": [{"subject": {}, "bindingSet": {}}]}) + + def _start( + self, + *extra: str, + behaviour=lambda runner: _result(), + json_flag: bool = True, + admission: Any = None, + get_run: Any = None, + workspace: bool = True, + ): + calls: List[str] = [] + + def tracked(runner: _FakeRunner) -> Any: + calls.append("runner") + return behaviour(runner) + + def wait(*args: Any, **kwargs: Any) -> Dict[str, Any]: + calls.append("wait") + return {"id": "r1", "state": "completed"} + + if admission is None: + admission = {"run": {"id": "r1", "state": "queued"}, "executionMode": "browser"} + start = MagicMock(return_value=admission) + with ( + patch(RUNNER, _fake(tracked)), + patch(f"{CLIENT}.start_run", start), + patch(f"{CLIENT}.wait_for_run", side_effect=wait) as mock_wait, + patch(f"{CLIENT}.get_run", get_run or MagicMock(side_effect=AssertionError("unexpected get_run"))), + ): + argv = ["--json"] if json_flag else [] + argv += ["workflow-eval", "run", "start", "-e", "e1", "--body", self.BODY, *extra] + if workspace: + result = _invoke(*argv) + else: + # No global --workspace: the configured default workspace is used. + with patch("roboflow.cli._resolver.resolve_default_workspace", return_value="ws"): + result = runner.invoke(app, ["--api-key", "key", *argv]) + return result, calls, start, mock_wait + + def test_wait_with_inference_url_runs_locally_then_waits(self) -> None: + result, calls, start, mock_wait = self._start( + "--wait", "--inference-url", "http://localhost:9001", "--concurrency", "2" + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(calls, ["runner", "wait"]) + self.assertEqual(json.loads(result.stdout), {"id": "r1", "state": "completed"}) + options = _FakeRunner.instances[0].options + self.assertEqual(options["inference_url"], "http://localhost:9001") + self.assertEqual(options["concurrency"], 2) + self.assertNotIn("case_timeout", options) # the runner default (300) applies + mock_wait.assert_called_once_with("e1", "r1", timeout=0, interval=5, on_poll=ANY) + + def test_wait_without_inference_url_only_waits(self) -> None: + result, calls, _, _ = self._start("--wait") + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(calls, ["wait"]) + + def test_runner_summary_goes_to_stderr(self) -> None: + result, _, _, _ = self._start("--wait", "--inference-url", "http://localhost:9001", json_flag=False) + self.assertEqual(result.exit_code, 0, result.output) + self.assertIn("Local run finished in 12.5s: 9 submitted (1 failed), 2 already done.", result.stderr) + self.assertEqual(json.loads(result.stdout)["state"], "completed") + + def test_inference_url_without_wait_is_a_usage_error(self) -> None: + result, calls, start, _ = self._start("--inference-url", "http://localhost:9001") + self.assertEqual(result.exit_code, 1) + self.assertIn("--inference-url requires --wait", result.stderr) + start.assert_not_called() + self.assertEqual(calls, []) + + def test_local_options_require_inference_url(self) -> None: + result, _, start, _ = self._start("--wait", "--concurrency", "2") + self.assertEqual(result.exit_code, 1) + self.assertIn("require --inference-url", result.stderr) + start.assert_not_called() + + def test_concurrency_out_of_range_is_rejected(self) -> None: + result, _, start, _ = self._start("--wait", "--inference-url", "http://x", "--concurrency", "17") + self.assertEqual(result.exit_code, 1) + start.assert_not_called() + + def test_server_run_with_inference_url_warns_and_only_waits(self) -> None: + admission = {"run": {"id": "r1", "state": "queued"}, "executionMode": "server"} + warning = "--inference-url has no effect: Run r1 executes on Roboflow; waiting for it." + for json_flag in (True, False): + with self.subTest(json=json_flag): + result, calls, _, mock_wait = self._start( + "--wait", "--inference-url", "http://x", admission=admission, json_flag=json_flag + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(calls, ["wait"]) # no runner: no preflight, no inference + self.assertEqual(_FakeRunner.instances, []) + final = json.loads(result.stdout) + self.assertEqual(final["state"], "completed") + if json_flag: + self.assertEqual(final["warnings"], [warning]) + self.assertEqual(result.stderr, "") + else: + self.assertNotIn("warnings", final) + self.assertIn(f"Warning: {warning}", result.stderr) + mock_wait.assert_called_once_with("e1", "r1", timeout=0, interval=5, on_poll=ANY) + + def test_execution_mode_missing_from_admission_is_read_once(self) -> None: + get_run = MagicMock(return_value={"id": "r1", "state": "queued", "executionMode": "server"}) + result, calls, _, _ = self._start( + "--wait", "--inference-url", "http://x", admission={"run": {"id": "r1"}}, get_run=get_run + ) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(calls, ["wait"]) + get_run.assert_called_once_with("e1", "r1") + self.assertEqual(len(json.loads(result.stdout)["warnings"]), 1) + + def test_runner_warnings_go_to_the_final_json(self) -> None: + def behaviour(runner: _FakeRunner) -> Any: + runner.options["warn"]("Using the inference server http://x instead of http://y") + return _result() + + result, calls, _, _ = self._start("--wait", "--inference-url", "http://x", behaviour=behaviour) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(calls, ["runner", "wait"]) + self.assertEqual(result.stderr, "") + self.assertEqual( + json.loads(result.stdout), + {"id": "r1", "state": "completed", "warnings": ["Using the inference server http://x instead of http://y"]}, + ) + + def test_without_warnings_the_final_json_has_no_warnings_field(self) -> None: + result, _, _, _ = self._start("--wait", "--inference-url", "http://x") + self.assertEqual(json.loads(result.stdout), {"id": "r1", "state": "completed"}) + + def test_unreachable_inference_server_hints_run_local(self) -> None: + error = InferenceServerUnreachable( + "1 Case was not run because the inference server at http://x was unreachable." + ) + result, calls, _, _ = self._start("--wait", "--inference-url", "http://x", behaviour=_raise(error)) + self.assertEqual(result.exit_code, 1) + self.assertEqual(calls, ["runner"]) + hint = json.loads(result.stderr.strip().splitlines()[-1])["error"]["hint"] + self.assertTrue(hint.startswith("Start the inference server. ")) + self.assertIn("run local r1 -e e1 --inference-url http://x", hint) + + def test_any_runner_error_hints_run_local_and_carries_the_run_id(self) -> None: + error = LocalRunnerError("Could not reach the inference server at http://x.", hint=PREFLIGHT_HINT) + result, calls, _, _ = self._start("--wait", "--inference-url", "http://x", behaviour=_raise(error)) + self.assertEqual(result.exit_code, 1) + self.assertEqual(calls, ["runner"]) + payload = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertTrue(payload["hint"].startswith(PREFLIGHT_HINT + " ")) + self.assertIn("run local r1 -e e1 --inference-url http://x", payload["hint"]) + self.assertEqual(payload["details"]["runId"], "r1") + + def test_interrupt_hints_run_local_to_resume(self) -> None: + result, calls, _, _ = self._start( + "--wait", "--inference-url", "http://x", behaviour=_raise(LocalRunnerInterrupted()) + ) + self.assertEqual(result.exit_code, 130) + self.assertEqual(calls, ["runner"]) + hint = json.loads(result.stderr.strip().splitlines()[-1])["error"]["hint"] + self.assertIn("run local r1 -e e1 --inference-url http://x", hint) + + def test_runner_resume_hint_is_replaced_by_run_local(self) -> None: + error = LocalRunnerError( + "Stopped after 10 connection failures in a row to the inference server at http://x.", + hint=f"{PREFLIGHT_HINT} {RESUME_HINT}", + ) + result, _, _, _ = self._start("--wait", "--inference-url", "http://x", behaviour=_raise(error), workspace=False) + self.assertEqual(result.exit_code, 1) + payload = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertNotIn("run the command again", payload["hint"]) + self.assertEqual( + payload["hint"], + f"{PREFLIGHT_HINT} Resume with `roboflow workflow-eval run local r1 -e e1 --inference-url http://x`; " + "the journal is in /tmp/rf-state/r1.", + ) + self.assertEqual(payload["details"]["runId"], "r1") + + def test_resume_hint_repeats_the_workspace_and_local_options(self) -> None: + error = LocalRunnerInterrupted() + for case_timeout, shown in (("0", "0"), ("2.5", "2.5"), ("300.0", "300")): + with self.subTest(case_timeout=case_timeout): + result, _, _, _ = self._start( + "--wait", + "--inference-url", + "http://x", + "--case-timeout", + case_timeout, + "--concurrency", + "8", + behaviour=_raise(error), + ) + self.assertEqual(result.exit_code, 130) + hint = json.loads(result.stderr.strip().splitlines()[-1])["error"]["hint"] + self.assertIn( + "Resume with `roboflow --workspace ws workflow-eval run local r1 -e e1 --inference-url http://x " + f"--concurrency 8 --case-timeout {shown}`; the journal is in /tmp/rf-state/r1.", + hint, + ) + self.assertNotIn("key", hint) + + def test_cancelled_run_has_no_resume_hint(self) -> None: + error = LocalRunnerError( + "Run r1 was cancelled.", + hint="Start a new Run to evaluate again.", + result=_result("cancelled"), + resumable=False, + ) + result, _, _, _ = self._start("--wait", "--inference-url", "http://x", behaviour=_raise(error)) + self.assertEqual(result.exit_code, 1) + payload = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertEqual(payload["hint"], "Start a new Run to evaluate again.") + self.assertEqual(payload["details"]["runId"], "r1") + + def test_other_errors_keep_the_run_id_and_resume_hint(self) -> None: + error = PermissionError(13, "Permission denied", "/tmp/rf-state/r1") + result, calls, _, _ = self._start("--wait", "--inference-url", "http://x", behaviour=_raise(error)) + self.assertEqual(result.exit_code, 1) + self.assertEqual(calls, ["runner"]) + payload = json.loads(result.stderr.strip().splitlines()[-1])["error"] + self.assertIn("Permission denied", payload["message"]) + self.assertIn("run local r1 -e e1 --inference-url http://x", payload["hint"]) + self.assertEqual(payload["details"]["runId"], "r1") + + if __name__ == "__main__": unittest.main() diff --git a/tests/fixtures/workflow_eval_local_inference_request.json b/tests/fixtures/workflow_eval_local_inference_request.json new file mode 100644 index 00000000..b0283a49 --- /dev/null +++ b/tests/fixtures/workflow_eval_local_inference_request.json @@ -0,0 +1,54 @@ +{ + "_source": "roboflow app/src/scripts/react/components/Workflows/Evals/results/localInference.worker.ts:29-43 (fetch to `${url}/workflows/run` with JSON.stringify of this body)", + "url": "http://localhost:9001/workflows/run", + "headers": { + "Content-Type": "application/json", + "ngrok-skip-browser-warning": "true" + }, + "body": { + "api_key": "sample_api_key", + "specification": { + "version": "1.0", + "inputs": [ + { + "type": "WorkflowImage", + "name": "image" + }, + { + "type": "WorkflowParameter", + "name": "confidence", + "default_value": 0.5 + } + ], + "steps": [ + { + "type": "roboflow_core/roboflow_object_detection_model@v2", + "name": "model", + "images": "$inputs.image", + "model_id": "coco/3", + "confidence": "$inputs.confidence" + } + ], + "outputs": [ + { + "type": "JsonField", + "name": "predictions", + "selector": "$steps.model.predictions" + } + ] + }, + "inputs": { + "image": { + "artifactId": "art_1", + "type": "url", + "value": "https://storage.example/art_1?sig=1" + }, + "confidence": 0.4, + "label": "\u00f1and\u00fa" + }, + "workflow_id": "wf_1", + "is_preview": true, + "disable_sinks": true, + "debug": false + } +} diff --git a/tests/test_slim_compat.py b/tests/test_slim_compat.py index 972eeacf..9e83f4c9 100644 --- a/tests/test_slim_compat.py +++ b/tests/test_slim_compat.py @@ -38,6 +38,16 @@ def test_import_rfapi(self): self.assertTrue(issubclass(RoboflowError, Exception)) + def test_import_workflow_eval_local_runner(self): + from roboflow.core.workflow_eval_local_runner import LocalRunner + + self.assertTrue(callable(LocalRunner)) + + def test_import_workflow_eval_run_local_cli(self): + from roboflow.cli.handlers.workflow_eval import run_local + + self.assertTrue(callable(run_local)) + def test_import_cli(self): from roboflow.cli import app diff --git a/tests/test_workflow_eval_local_runner.py b/tests/test_workflow_eval_local_runner.py new file mode 100644 index 00000000..3a48e5c7 --- /dev/null +++ b/tests/test_workflow_eval_local_runner.py @@ -0,0 +1,1773 @@ +"""Tests for the local Workflow Eval runner (``roboflow.core.workflow_eval_local_runner``). + +The Roboflow API and the Inference server are both faked with ``responses`` +callbacks; time is a fake clock whose ``sleep`` only advances it, so nothing +here sleeps for real. +""" + +from __future__ import annotations + +import copy +import io +import json +import os +import re +import signal +import socket +import ssl +import threading +from datetime import datetime, timezone +from http.client import RemoteDisconnected +from typing import Any, Callable, Dict, List, Optional +from unittest.mock import patch +from urllib.parse import parse_qs, unquote, urlparse + +import pytest +import requests +import responses +import urllib3 +from typer.testing import CliRunner + +from roboflow.cli import app +from roboflow.config import API_URL +from roboflow.core import workflow_eval_local_runner as local +from roboflow.core.workflow_eval_local_runner import ( + CAPTURE_TOO_LARGE_MESSAGE, + FETCH_FAILED_MESSAGE, + PREFLIGHT_HINT, + TIMEOUT_MESSAGE, + InferenceServerUnreachable, + Journal, + LocalRunner, + LocalRunnerError, + LocalRunnerInterrupted, + case_inputs, + install_signal_handlers, + materialize_media, + run_workflow_inference, +) +from roboflow.core.workflow_evals import WorkflowEvals + +WS, KEY = "test-ws", "test_key" +E, R, X = "eval-1", "run-1", "exec-1" +RUN = f"{API_URL}/workspaces/{WS}/workflow-evals/{E}/runs/{R}" +INF = "http://localhost:9001" +SPEC = {"version": "1.0", "inputs": [{"type": "WorkflowImage", "name": "image"}], "steps": [], "outputs": []} +FIXTURE = os.path.join(os.path.dirname(__file__), "fixtures", "workflow_eval_local_inference_request.json") +START = 1_800_000_000.0 + + +def iso(timestamp: float) -> str: + return datetime.fromtimestamp(timestamp, timezone.utc).isoformat() + + +def err(message: str, *, retryable: bool = False, details: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + error: Dict[str, Any] = {"code": "CONFLICT", "category": "conflict", "retryable": retryable, "message": message} + if details: + error["details"] = details + return {"error": error} + + +def case(case_id: str, *, checkpoint: Optional[Dict[str, Any]] = None, **fields: Any) -> Dict[str, Any]: + item = { + "case": {"id": case_id, "name": case_id}, + "shardIndex": 0, + "workflowInputs": {"image": {"artifactId": f"art-{case_id}"}, "name": case_id}, + "workflowInputFailures": [], + "checkpoint": checkpoint, + } + item.update(fields) + return item + + +def ok(name: Optional[str] = None) -> Any: + return (200, {}, json.dumps({"outputs": [{"case": name}]})) + + +def refused_error() -> requests.exceptions.ConnectionError: + # What requests raises when nothing listens on the port: the server was never reached. + reason = urllib3.exceptions.NewConnectionError(None, "Failed to establish a new connection: Connection refused") + return requests.exceptions.ConnectionError(urllib3.exceptions.MaxRetryError(None, "/workflows/run", reason)) + + +def refused(request: Any) -> Any: + return refused_error() + + +def reset_error() -> requests.exceptions.ConnectionError: + # Connected, then the server dropped the connection. + return requests.exceptions.ConnectionError( + urllib3.exceptions.ProtocolError("Connection aborted.", ConnectionResetError(104, "Connection reset by peer")) + ) + + +class BrokenBody(io.BufferedReader): + """A response body whose socket fails on the first read, after the status line arrived.""" + + def __init__(self, error: BaseException) -> None: + super().__init__(io.BytesIO(b"{}")) # type: ignore[arg-type] + self.error = error + + def read(self, *args: Any) -> bytes: + raise self.error + + read1 = read + + def readinto(self, *args: Any) -> int: + raise self.error + + +def tls_dropped(request: Any) -> Any: + # Connected and answered, then TLS failed mid-body (e.g. behind ngrok); requests raises SSLError. + return 200, {}, BrokenBody(ssl.SSLError("TLS connection dropped")) + + +# Waits of one Case whose inference server cannot be reached at all. +CONNECT_SLEEPS = [1.0, 2.0, 4.0, 8.0, 16.0] + + +class FakeClock: + def __init__(self) -> None: + self.now = START + self.sleeps: List[float] = [] + self._lock = threading.Lock() + + def time(self) -> float: + with self._lock: + return self.now + + def sleep(self, seconds: float) -> None: + with self._lock: + self.sleeps.append(seconds) + self.now += max(seconds, 0) + + def advance(self, seconds: float) -> None: + with self._lock: + self.now += seconds + + def set(self, value: float) -> None: + with self._lock: + self.now = value + + +class FakeInference: + """``POST /workflows/run``. Behaviours are keyed by the Case name sent in ``inputs.name``.""" + + def __init__(self) -> None: + self.requests: List[Any] = [] + self.behaviors: Dict[Any, Any] = {} + self.default: Callable[[Any], Any] = lambda request: ok(json.loads(request.body)["inputs"].get("name")) + self.active = 0 + self.max_active = 0 + self._lock = threading.Lock() + + @property + def names(self) -> List[Any]: + return [json.loads(request.body)["inputs"].get("name") for request in self.requests] + + def __call__(self, request: Any) -> Any: + name = json.loads(request.body)["inputs"].get("name") + with self._lock: + self.requests.append(request) + self.active += 1 + self.max_active = max(self.max_active, self.active) + behavior = self.behaviors.get(name, self.default) + if isinstance(behavior, list): + behavior = behavior.pop(0) if len(behavior) > 1 else behavior[0] + try: + return behavior(request) if callable(behavior) else behavior + finally: + with self._lock: + self.active -= 1 + + +class FakeServer: + """The Workflow Evals routes the runner calls, with a mutable Run and per-Case checkpoints.""" + + def __init__(self, rsps: responses.RequestsMock, items: List[Dict[str, Any]], *, mode: str = "browser") -> None: + self.lock = threading.Lock() + self.items = [dict(item, checkpoint=None) for item in items] + self.checkpoints = {item["case"]["id"]: item.get("checkpoint") for item in items} + self.run: Dict[str, Any] = {"id": R, "state": "running", "executionMode": mode, "retry": {"allowed": False}} + self.attempt = 1 + self.version = 0 + self.spec: Any = SPEC + self.workflow_id = "wf-1" + self.frozen_url = INF + self.capture_script: Dict[str, List[Any]] = {} + self.capture_default: Any = None + self.captures: List[Any] = [] + self.page_requests: List[Any] = [] + self.page_seen: Dict[Any, threading.Event] = {} + # Called (without the lock) before the page at this cursor is served. + self.before_page: Dict[Any, Callable[[], None]] = {} + # Mark a Case completed at the current attempt when its capture is accepted, like the server. + self.complete_on_capture = False + self.retried = 0 + rsps.add_callback(responses.GET, RUN, callback=lambda r: (200, {}, json.dumps(self.run))) + rsps.add_callback(responses.GET, f"{RUN}/configuration", callback=self._configuration) + rsps.add_callback(responses.POST, f"{RUN}/retry-failed", callback=self._retry) + rsps.add_callback( + responses.GET, + f"{RUN}/executions", + callback=lambda r: (200, {}, json.dumps({"items": [{"id": X, "selectedCaseCount": len(self.items)}]})), + ) + rsps.add_callback(responses.GET, f"{RUN}/executions/{X}/configuration", callback=self._execution) + rsps.add_callback( + responses.GET, re.compile(re.escape(f"{RUN}/executions/") + r"[^/]+/input-cases.*"), callback=self._page + ) + rsps.add_callback( + responses.POST, + re.compile(re.escape(f"{RUN}/executions/") + r"[^/]+/cases/([^/?]+)/capture"), + callback=self._capture, + ) + + def checkpoint(self, case_id: str, value: Optional[Dict[str, Any]]) -> None: + with self.lock: + self.checkpoints[case_id] = value + + def seen(self, cursor: Any) -> threading.Event: + with self.lock: + return self.page_seen.setdefault(cursor, threading.Event()) + + def capture_bodies(self, case_id: Optional[str] = None) -> List[Dict[str, Any]]: + return [json.loads(body) for cid, body in self.captures if case_id in (None, cid)] + + def _configuration(self, request: Any) -> Any: + server = {"kind": "local", "url": self.frozen_url} + return ( + 200, + {}, + json.dumps({"runId": R, "executionMode": "browser", "sharedConfiguration": {"inferenceServer": server}}), + ) + + def _execution(self, request: Any) -> Any: + plan = {"workflowSubject": {"id": self.workflow_id, "manifest": self.spec}} + return 200, {}, json.dumps({"execution": {"id": X}, "linkedPlan": plan}) + + def _retry(self, request: Any) -> Any: + with self.lock: + self.retried += 1 + self.attempt += 1 + self.run["state"] = "running" + return 202, {}, json.dumps({"runId": R}) + + def _page(self, request: Any) -> Any: + query = parse_qs(urlparse(request.url).query) + cursor = query.get("cursor", [None])[0] + limit = int(query.get("limit", ["50"])[0]) + hook = self.before_page.get(cursor) + if hook is not None: + hook() + with self.lock: + self.page_requests.append((cursor, limit)) + self.version += 1 + offset = int(cursor or 0) + items = copy.deepcopy(self.items[offset : offset + limit]) + urls = {} + for item in items: + item["checkpoint"] = copy.deepcopy(self.checkpoints.get(item["case"]["id"])) + for value in (item.get("workflowInputs") or {}).values(): + if isinstance(value, dict) and "artifactId" in value: + urls[value["artifactId"]] = { + "url": f"https://storage.example/{value['artifactId']}?sig={self.version}", + "expiresInSeconds": 600, + } + body = { + "items": items, + "nextCursor": str(offset + limit) if offset + limit < len(self.items) else None, + "attemptCount": self.attempt, + "runState": self.run["state"], + "cancellationRequested": False, + "artifactReadUrls": urls, + } + self.seen(cursor).set() + return 200, {}, json.dumps(body) + + def _capture(self, request: Any) -> Any: + case_id = unquote(re.search(r"/cases/([^/?]+)/capture", request.url).group(1)) + with self.lock: + self.captures.append((case_id, request.body)) + script = self.capture_script.get(case_id) + entry = script.pop(0) if script else self.capture_default + if callable(entry): + entry = entry() + if entry is None: + if self.complete_on_capture: + self.checkpoint( + case_id, {"state": "completed", "attemptCount": json.loads(request.body)["attemptCount"]} + ) + return 200, {}, json.dumps({"id": case_id, "state": "completed"}) + status, body = entry + return status, {}, body if isinstance(body, str) else json.dumps(body) + + +class Env: + def __init__(self, rsps: responses.RequestsMock, tmp_path: Any) -> None: + self.rsps = rsps + self.state_dir = str(tmp_path / "state") + self.clock = FakeClock() + self.inference = FakeInference() + self.warnings: List[str] = [] + self.progress: List[Dict[str, Any]] = [] + self.describe: List[Any] = [] + self.describe_response: Any = (200, {}, json.dumps({"blocks": []})) + self.server: Optional[FakeServer] = None + rsps.add_callback(responses.POST, f"{INF}/workflows/run", callback=self.inference) + rsps.add_callback(responses.POST, f"{INF}/workflows/blocks/describe", callback=self._describe) + + def _describe(self, request: Any) -> Any: + self.describe.append(request) + return self.describe_response + + def serve(self, items: List[Dict[str, Any]], **kwargs: Any) -> FakeServer: + self.server = FakeServer(self.rsps, items, **kwargs) + return self.server + + def runner(self, **kwargs: Any) -> LocalRunner: + options: Dict[str, Any] = { + "state_dir": self.state_dir, + "clock": self.clock.time, + "monotonic": self.clock.time, + "sleep": self.clock.sleep, + "warn": self.warnings.append, + "on_progress": self.progress.append, + } + options.update(kwargs) + return LocalRunner(WorkflowEvals(KEY, WS), E, R, **options) + + def journal(self) -> List[Dict[str, Any]]: + with open(os.path.join(self.state_dir, "journal.jsonl"), encoding="utf-8") as handle: + return [json.loads(line) for line in handle] + + def stored(self, entry: Dict[str, Any]) -> bytes: + with open(os.path.join(self.state_dir, entry["path"]), "rb") as handle: + return handle.read() + + +@pytest.fixture +def env(tmp_path: Any) -> Any: + with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: + yield Env(rsps, tmp_path) + + +def captured(env: Env) -> List[Dict[str, Any]]: + return [entry for entry in env.journal() if entry["t"] == "captured"] + + +def outcomes(env: Env) -> List[Any]: + return [ + (entry["caseId"], entry["status"], entry["outcome"]) for entry in env.journal() if entry["t"] == "submitted" + ] + + +# -- 1. request contract ----------------------------------------------------------- + + +def test_inference_request_matches_browser_worker_fixture(env: Env) -> None: + with open(FIXTURE, encoding="utf-8") as handle: + fixture = json.load(handle) + expected = fixture["body"] + server = env.serve( + [case("c1", workflowInputs={"image": {"artifactId": "art_1"}, "confidence": 0.4, "label": "ñandú"})] + ) + server.spec, server.workflow_id = expected["specification"], expected["workflow_id"] + server.version = 0 # the first page signs ?sig=1 + + result = env.runner(inference_api_key="sample_api_key").run() + + assert result.to_dict()["executions"] == [ + {"id": X, "submitted": 1, "alreadyDone": 0, "skipped": 0, "failed": 0, "pending": 0} + ] + (request,) = env.inference.requests + assert request.url == fixture["url"] + # Byte-for-byte what JSON.stringify produces for the worker's object literal. + assert request.body == json.dumps(expected, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + body = json.loads(request.body) + assert body == expected and list(body) == list(expected) + assert body["is_preview"] is True and body["disable_sinks"] is True and body["debug"] is False + for name, value in fixture["headers"].items(): + assert request.headers[name] == value + + # Preflight mirrors the web app's local server check. + (describe,) = env.describe + assert parse_qs(urlparse(describe.url).query) == {"api_key": ["sample_api_key"]} + assert describe.headers["Api-Key"] == "sample_api_key" + assert describe.headers["ngrok-skip-browser-warning"] == "true" + assert json.loads(describe.body) == {"dynamic_blocks_definitions": []} + + +def test_capture_is_the_worker_message_with_attempt_and_shard(env: Env) -> None: + server = env.serve([case("c1")]) + env.inference.default = lambda request: (200, {}, json.dumps({"outputs": {"single": 1}})) + env.runner().run() + (body,) = server.capture_bodies() + assert list(body) == ["attemptCount", "shardIndex", "output", "runtime"] + assert body["attemptCount"] == 1 and body["shardIndex"] == 0 + assert body["output"] == [{"single": 1}] + assert set(body["runtime"]) == {"latencyMs"} + + +# -- 2. media ---------------------------------------------------------------------- + + +def test_materialize_media_nested_and_in_lists() -> None: + urls = {"a1": {"url": "https://u/a1", "expiresInSeconds": 600}, "a2": {"url": "https://u/a2"}} + value = { + "image": {"artifactId": "a1", "role": "primary"}, + "nested": {"frames": [{"artifactId": "a2"}, {"artifactId": "unsigned"}, 3]}, + "plain": "x", + } + assert materialize_media(value, urls) == { + "image": {"artifactId": "a1", "role": "primary", "type": "url", "value": "https://u/a1"}, + "nested": { + "frames": [{"artifactId": "a2", "type": "url", "value": "https://u/a2"}, {"artifactId": "unsigned"}, 3] + }, + "plain": "x", + } + + +def test_case_inputs_materializes_media() -> None: + item = {"workflowInputs": {"image": {"artifactId": "a1"}, "confidence": 0.4}, "workflowInputFailures": []} + inputs = case_inputs(item, {"a1": {"url": "https://u/a1"}}) + assert inputs == {"image": {"artifactId": "a1", "type": "url", "value": "https://u/a1"}, "confidence": 0.4} + + +def test_runner_sends_materialized_inputs(env: Env) -> None: + env.serve([case("c1", workflowInputs={"image": {"artifactId": "art-c1"}})]) + env.runner().run() + inputs = json.loads(env.inference.requests[0].body)["inputs"] + assert inputs == {"image": {"artifactId": "art-c1", "type": "url", "value": "https://storage.example/art-c1?sig=1"}} + + +def test_input_failure_is_captured_as_failed_without_inference(env: Env) -> None: + message = 'Workflow input "mask" did not resolve' + server = env.serve( + [ + case("c1"), + case( + "c2", + shardIndex=3, + workflowInputFailures=[{"input": "mask", "message": message}, {"input": "other", "message": "x"}], + ), + case("c3"), + ] + ) + result = env.runner().run() + + assert sorted(env.inference.names) == ["c1", "c3"] + (body,) = server.capture_bodies("c2") + assert body == {"attemptCount": 1, "shardIndex": 3, "output": None, "runtime": {"error": message}} + assert set(body["runtime"]) == {"error"} + assert all(set(b["runtime"]) == {"latencyMs"} for cid in ("c1", "c3") for b in server.capture_bodies(cid)) + (entry,) = [entry for entry in captured(env) if entry["caseId"] == "c2"] + assert json.loads(env.stored(entry)) == body and entry["shardIndex"] == 3 + assert ("c2", 200, "submitted") in outcomes(env) + assert result.executions[0].to_dict() == { + "id": X, + "submitted": 3, + "alreadyDone": 0, + "skipped": 0, + "failed": 1, + "pending": 0, + } + + +def test_input_failure_message_is_truncated(env: Env) -> None: + server = env.serve([case("c1", workflowInputFailures=[{"input": "mask", "message": "x" * 3000}])]) + env.runner().run() + assert env.inference.requests == [] + assert server.capture_bodies()[0]["runtime"] == {"error": "x" * 2000} + + +# -- 3. sliding window ---------------------------------------------------------------- + + +class VirtualTime: + """Holds every inference until the test releases the one that ends first, then jumps the clock there.""" + + def __init__(self, clock: FakeClock, durations: Dict[str, float]) -> None: + self.clock = clock + self.durations = durations + self.cond = threading.Condition() + self.waiting: Dict[str, Any] = {} + + def __call__(self, request: Any) -> Any: + name = json.loads(request.body)["inputs"]["name"] + release = threading.Event() + with self.cond: + self.waiting[name] = (self.clock.time() + self.durations[name], release) + self.cond.notify_all() + assert release.wait(5) + return ok(name) + + def drive(self, window: int) -> None: + total = len(self.durations) + for done in range(total): + with self.cond: + assert self.cond.wait_for(lambda: len(self.waiting) >= min(window, total - done), timeout=5) + name = min(self.waiting, key=lambda key: self.waiting[key][0]) + end, release = self.waiting.pop(name) + self.clock.set(end) + release.set() + + +def test_sliding_window_beats_lockstep_and_respects_concurrency(env: Env) -> None: + durations = {"c1": 5.0, "c2": 1.0, "c3": 1.0, "c4": 1.0, "c5": 1.0} + server = env.serve([case(name) for name in durations]) + virtual = VirtualTime(env.clock, durations) + env.inference.default = virtual + runner = env.runner(concurrency=2) + outcome: Dict[str, Any] = {} + thread = threading.Thread(target=lambda: outcome.update(result=runner.run())) + thread.start() + virtual.drive(window=2) + thread.join(5) + + assert outcome["result"].executions[0].submitted == 5 + # Lockstep pages of 2 would take max(5, 1) + max(1, 1) + 1 = 7 s; the window finishes with c1. + assert env.clock.now - START == 5.0 + assert env.inference.max_active == 2 + assert {cid for cid, _ in server.captures} == set(durations) + latency = {cid: json.loads(body)["runtime"]["latencyMs"] for cid, body in server.captures} + assert latency["c1"] == 5000.0 and latency["c2"] == 1000.0 + assert all(cursor_limit[1] == 2 for cursor_limit in server.page_requests) + + +# -- 4. refresh --------------------------------------------------------------------- + + +def test_stale_page_is_reread_with_the_same_cursor_before_dispatch(env: Env) -> None: + server = env.serve([case("c1"), case("c2")]) + + def slow(request: Any) -> Any: + assert server.seen("1").wait(5) # the next page was prefetched at t=0 + env.clock.advance(9 * 60) + return ok("c1") + + env.inference.behaviors["c1"] = slow + env.runner(concurrency=1).run() + + assert server.page_requests == [(None, 1), ("1", 1), ("1", 1)] + image = json.loads(env.inference.requests[1].body)["inputs"]["image"] + assert image["value"] == "https://storage.example/art-c2?sig=3" + + +def test_retry_after_a_slow_attempt_rereads_the_page_for_fresh_urls(env: Env) -> None: + server = env.serve([case("c1"), case("c2")]) + + def slow_503(request: Any) -> Any: + env.clock.advance(9 * 60) # the signed URLs of c2's page are about to expire + return 503, {}, "Service Unavailable" + + env.inference.behaviors["c2"] = [slow_503, ok("c2")] + result = env.runner(concurrency=1).run() + + first, second = [json.loads(r.body)["inputs"]["image"]["value"] for r in env.inference.requests[1:]] + assert first == "https://storage.example/art-c2?sig=2" + assert second == "https://storage.example/art-c2?sig=3" + # c2's page (cursor "1") is read again before the retry, with the same cursor. + assert server.page_requests == [(None, 1), ("1", 1), ("1", 1)] + assert server.capture_bodies("c2")[0]["output"] == [{"case": "c2"}] + assert result.executions[0].submitted == 2 and result.executions[0].failed == 0 + + +def test_retry_within_the_page_max_age_reuses_the_inputs(env: Env) -> None: + server = env.serve([case("c1")]) + env.inference.behaviors["c1"] = [(503, {}, "Service Unavailable"), ok("c1")] + env.runner().run() + + assert len(env.inference.requests) == 2 + assert env.inference.requests[0].body == env.inference.requests[1].body + assert server.page_requests == [(None, 4)] + + +def test_case_that_left_the_reread_page_restarts_the_execution(env: Env) -> None: + server = env.serve([case("c1")]) + + def slow_503_and_move(request: Any) -> Any: + env.clock.advance(9 * 60) + with server.lock: + server.items = [case("c0"), case("c1")] # c1 moves to the second page + return 503, {}, "Service Unavailable" + + env.inference.behaviors["c1"] = [slow_503_and_move, ok("c1")] + result = env.runner(concurrency=1).run() + + # The re-read page no longer has c1: the attempt is abandoned uncaptured and the + # Execution restarts from the first page, where the new pass finds c1 on page "1". + assert server.page_requests == [(None, 1), (None, 1), (None, 1), ("1", 1)] + assert env.inference.names == ["c1", "c0", "c1"] + assert sorted(cid for cid, _ in server.captures) == ["c0", "c1"] + assert server.capture_bodies("c1")[0]["output"] == [{"case": "c1"}] + assert result.executions[0].to_dict()["pending"] == 0 + assert result.executions[0].submitted == 2 and result.executions[0].failed == 0 + + +def test_reread_page_at_another_attempt_restarts_the_execution(env: Env) -> None: + server = env.serve([case("c1")]) + + def slow_503(request: Any) -> Any: + env.clock.advance(9 * 60) + with server.lock: + server.attempt = 2 + return 503, {}, "Service Unavailable" + + env.inference.behaviors["c1"] = [slow_503, ok("c1")] + result = env.runner().run() + + # The attempt-1 inference is not retried nor captured; the restarted pass runs c1 at attempt 2. + assert len(env.inference.requests) == 2 + assert [body["attemptCount"] for body in server.capture_bodies()] == [2] + assert result.executions[0].submitted == 1 + + +def test_stop_requested_during_the_reread_abandons_the_retry(env: Env) -> None: + server = env.serve([case("c1")]) + runner = env.runner() + page_calls: List[int] = [] + + def slow_503(request: Any) -> Any: + env.clock.advance(9 * 60) + return 503, {}, "Service Unavailable" + + def stop_during_reread() -> None: + page_calls.append(1) + if len(page_calls) == 2: + runner.request_stop() # the re-read itself still returns a valid page + + env.inference.behaviors["c1"] = [slow_503, ok("c1")] + server.before_page[None] = stop_during_reread + with pytest.raises(LocalRunnerInterrupted) as raised: + runner.run() + + assert raised.value.exit_code == 130 + assert len(env.inference.requests) == 1 # no inference after the stop + assert server.page_requests == [(None, 4), (None, 4)] + assert server.captures == [] and captured(env) == [] + + +def test_reload_while_the_reread_is_in_backoff_abandons_the_case(env: Env) -> None: + server = env.serve([case("c1"), case("c2")]) + runner = env.runner(concurrency=2) + rereading = threading.Event() + page_calls: List[int] = [] + + def slow_503(request: Any) -> Any: + env.clock.advance(9 * 60) + return 503, {}, "Service Unavailable" + + def after_reread_started(request: Any) -> Any: + assert rereading.wait(5) + return ok("c1") + + def attempt_changed() -> Any: + server.attempt = 2 + return 412, {"error": {"code": "PRECONDITION_FAILED", "message": "Run attempt changed"}} + + def flaky_reread() -> None: + page_calls.append(1) + if len(page_calls) == 2: + rereading.set() + # Fails only after the 412, so its backoff is abandoned because of the reload. + with runner._cond: + assert runner._cond.wait_for(lambda: runner._reload, timeout=5) + raise requests.exceptions.ConnectionError("reset") + + env.inference.behaviors.update(c1=[after_reread_started, ok("c1")], c2=[slow_503, ok("c2")]) + server.capture_script["c1"] = [attempt_changed] + server.before_page[None] = flaky_reread + result = runner.run() + + # The failed re-read is not retried: the next page read is the restarted pass. + assert server.page_requests == [(None, 2), (None, 2)] + assert [body["attemptCount"] for body in server.capture_bodies("c2")] == [2] + assert [body["attemptCount"] for body in server.capture_bodies("c1")] == [1, 2] + assert result.executions[0].submitted == 2 + + +# -- 5. journal --------------------------------------------------------------------- + + +def test_journal_resubmits_stored_bytes_without_running_inference(env: Env) -> None: + server = env.serve([case("c1", checkpoint={"state": "completed", "attemptCount": 1}), case("c2")]) + original = Journal.record_capture + + def crash_after_capture(self: Journal, *args: Any) -> None: + original(self, *args) + raise RuntimeError("killed after captured") + + with patch.object(Journal, "record_capture", crash_after_capture), pytest.raises(RuntimeError): + env.runner().run() + assert env.inference.names == ["c2"] + assert server.captures == [] + (entry,) = captured(env) + stored = env.stored(entry) + + result = env.runner().run() + + assert env.inference.names == ["c2"] # no inference for c2 (journaled) nor c1 (completed) + assert [cid for cid, _ in server.captures] == ["c2"] + assert json.loads(server.captures[0][1]) == json.loads(stored) + assert env.stored(entry) == stored + assert result.executions[0].to_dict() == { + "id": X, + "submitted": 1, + "alreadyDone": 1, + "skipped": 0, + "failed": 0, + "pending": 0, + } + assert outcomes(env) == [("c2", 200, "submitted")] + + +def test_journal_entries_of_another_attempt_are_ignored(env: Env) -> None: + server = env.serve([case("c1")]) + Journal(env.state_dir).record_capture(7, X, "c1", 0, b'{"attemptCount":7}') + env.runner().run() + assert env.inference.names == ["c1"] + assert server.capture_bodies()[0]["attemptCount"] == 1 + + +def test_journal_torn_last_line_does_not_swallow_the_next_record(env: Env) -> None: + os.makedirs(env.state_dir) + with open(os.path.join(env.state_dir, "journal.jsonl"), "w", encoding="utf-8") as handle: + handle.write('{"t":"captured"') + journal = Journal(env.state_dir) + journal.record_capture(1, X, "c1", 0, b'{"attemptCount":1}') + journal.close() + reopened = Journal(env.state_dir) + reopened.close() + assert reopened.captured(1, X, "c1") == b'{"attemptCount":1}' + + +def test_case_with_retained_capture_waits_for_lease_and_resumes(env: Env) -> None: + lease = START + 42 + checkpoint = {"state": "running", "attemptCount": 1, "hasCapture": True, "leaseExpiresAt": iso(lease)} + server = env.serve([case("c1", checkpoint=checkpoint)]) + reads: List[float] = [] + server.before_page[None] = lambda: reads.append(env.clock.now - START) + env.runner().run() + assert env.inference.requests == [] + assert server.capture_bodies() == [{"attemptCount": 1, "shardIndex": 0, "resume": True}] + assert env.clock.now == lease + # The Case stays running: its page is re-read every 5 s and the resume waits for the lease. + assert reads == [0, 5, 10, 15, 20, 25, 30, 35, 40] + assert env.clock.sleeps == [5.0] * 8 + [2.0] + + +def test_retained_capture_resumes_before_the_lease_once_the_case_is_final(env: Env) -> None: + lease = START + 600 + checkpoint = {"state": "running", "attemptCount": 1, "hasCapture": True, "leaseExpiresAt": iso(lease)} + server = env.serve([case("c1", checkpoint=checkpoint)]) + + def other_runner_finishes() -> None: + if env.clock.now >= START + 10: + server.checkpoint("c1", {"state": "completed", "attemptCount": 1, "hasCapture": True}) + + server.before_page[None] = other_runner_finishes + env.runner().run() + assert env.inference.requests == [] + assert server.capture_bodies() == [{"attemptCount": 1, "shardIndex": 0, "resume": True}] + assert env.clock.now == START + 10 + + +# -- 6. capture responses --------------------------------------------------------------- + + +def test_capture_200_is_submitted(env: Env) -> None: + env.serve([case("c1")]) + result = env.runner().run() + assert result.to_dict() == { + "runId": R, + "state": "running", + "executions": [{"id": X, "submitted": 1, "alreadyDone": 0, "skipped": 0, "failed": 0, "pending": 0}], + "wallClockSeconds": 0.0, + "concurrency": 4, + } + assert [entry["t"] for entry in env.journal()] == ["captured", "submitted"] + assert env.progress[-1]["ok"] == 1 and env.progress[-1]["remaining"] == 0 + + +def test_capture_409_different_capture_counts_as_done_by_other(env: Env) -> None: + server = env.serve([case("c1")]) + server.capture_script["c1"] = [(409, err("Different capture already submitted for this Case attempt"))] + result = env.runner().run() + assert len(server.captures) == 1 + assert result.executions[0].already_done == 1 and result.executions[0].submitted == 0 + assert outcomes(env) == [("c1", 409, "done_by_other")] + + +PROCESSING_MESSAGE = "Capture is being processed or the attempt changed; retry the same payload" + + +def processing_until(server: FakeServer, lease: float) -> Callable[[], Any]: + """A capture answer: another runner holds c1 until ``lease``.""" + + def processing() -> Any: + server.checkpoint( + "c1", {"state": "running", "attemptCount": 1, "hasCapture": True, "leaseExpiresAt": iso(lease)} + ) + return 409, err(PROCESSING_MESSAGE, retryable=True) + + return processing + + +def test_capture_409_processing_waits_for_lease_and_resends_same_bytes(env: Env) -> None: + server = env.serve([case("c1")]) + lease = START + 30 + reads: List[float] = [] + sent: List[float] = [] + server.before_page[None] = lambda: reads.append(env.clock.now - START) + processing = processing_until(server, lease) + + def answer(response: Any = None) -> Callable[[], Any]: + def respond() -> Any: + sent.append(env.clock.now - START) + return response() if callable(response) else response + + return respond + + server.capture_script["c1"] = [answer(processing), answer()] + result = env.runner().run() + + assert len(server.captures) == 2 + assert server.captures[0][1] == server.captures[1][1] + # The lease comes from re-reading the same page at once; the Case stays running, so the + # page is re-read every 5 s and the capture is resent only when the lease ends. + assert all(request == (None, 4) for request in server.page_requests) + assert reads == [0, 0, 5, 10, 15, 20, 25] + assert sent == [0, 30] + assert len(env.inference.requests) == 1 + assert result.executions[0].submitted == 1 + + +def test_capture_409_processing_resends_before_the_lease_once_the_case_is_final(env: Env) -> None: + server = env.serve([case("c1")]) + lease = START + 600 + + def other_runner_finishes() -> None: + if env.clock.now >= START + 7: + server.checkpoint("c1", {"state": "completed", "attemptCount": 1, "hasCapture": False}) + + server.before_page[None] = other_runner_finishes + server.capture_script["c1"] = [ + processing_until(server, lease), + (409, err("Different capture already submitted for this Case attempt")), + ] + result = env.runner().run() + + assert len(server.captures) == 2 and server.captures[0][1] == server.captures[1][1] + assert env.clock.now == START + 10 # the first re-read after the Case completed + assert outcomes(env) == [("c1", 409, "done_by_other")] + assert result.executions[0].already_done == 1 and result.executions[0].submitted == 0 + + +def test_capture_409_processing_follows_a_lease_that_moves(env: Env) -> None: + server = env.serve([case("c1")]) + sent: List[float] = [] + + def lease_extended() -> None: + if env.clock.now >= START + 10: + lease = iso(START + 50) + server.checkpoint( + "c1", {"state": "running", "attemptCount": 1, "hasCapture": True, "leaseExpiresAt": lease} + ) + + def accepted() -> None: + sent.append(env.clock.now - START) + + server.before_page[None] = lease_extended + server.capture_script["c1"] = [processing_until(server, START + 30), accepted] + env.runner().run() + + assert sent == [50] + assert env.clock.now == START + 50 + + +@pytest.mark.parametrize("change", ["case_left_the_page", "attempt_changed"]) +def test_capture_409_processing_restarts_the_execution_when_the_wait_reread_drops_the_case( + env: Env, change: str +) -> None: + server = env.serve([case("c1")]) + lease = START + 600 + + def page_changes() -> None: + if env.clock.now >= START + 5 and server.attempt == 1 and len(server.items) == 1: + if change == "case_left_the_page": + # The other runner finished c1, which moved to the second page. + server.checkpoint("c1", {"state": "completed", "attemptCount": 1, "hasCapture": False}) + with server.lock: + server.items = [case("c0"), case("c1")] + else: + with server.lock: + server.attempt = 2 + + server.before_page[None] = page_changes + server.capture_script["c1"] = [processing_until(server, lease)] + result = env.runner(concurrency=1).run() + + # The first periodic re-read (t=5) abandons the wait: no resend at attempt 1, and the + # Execution restarts from the first page without waiting for the lease. + # Page reads: the pass, the immediate re-read, the t=5 re-read, then the restarted pass. + assert env.clock.now == START + 5 + counts = {"id": X, "submitted": 1, "skipped": 0, "failed": 0, "pending": 0} + if change == "case_left_the_page": + # The restarted pass runs c0 and finds c1 done on page "1". + assert server.page_requests == [(None, 1)] * 4 + [("1", 1)] + assert [body["attemptCount"] for body in server.capture_bodies("c1")] == [1] + assert env.inference.names == ["c1", "c0"] + assert outcomes(env) == [("c0", 200, "submitted")] + assert result.executions[0].to_dict() == dict(counts, alreadyDone=1) + else: + # The restarted pass runs c1 again and captures it once, at attempt 2. + assert server.page_requests == [(None, 1)] * 4 + assert [body["attemptCount"] for body in server.capture_bodies("c1")] == [1, 2] + assert env.inference.names == ["c1", "c1"] + assert outcomes(env) == [("c1", 200, "submitted")] + assert result.executions[0].to_dict() == dict(counts, alreadyDone=0) + + +def test_run_completed_while_waiting_for_a_lease_exits_0_without_waiting(env: Env) -> None: + server = env.serve([case("c1")]) + + def run_completes() -> None: + if env.clock.now > START: + server.run["state"] = "completed" + + server.before_page[None] = run_completes + server.capture_script["c1"] = [processing_until(server, START + 600)] + fake_time = {"clock": env.clock.time, "monotonic": env.clock.time, "sleep": env.clock.sleep} + argv = ["--workspace", WS, "--api-key", KEY, "--json", "workflow-eval", "run", "local", R, "-e", E] + with patch.object(local, "LocalRunner", lambda *args, **kwargs: LocalRunner(*args, **kwargs, **fake_time)): + result = CliRunner().invoke(app, argv + ["--state-dir", env.state_dir]) + + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["state"] == "completed" + assert env.clock.now == START + 5 # the first re-read saw the completed Run + assert len(server.captures) == 1 + + +def test_capture_409_non_retryable_failure_is_skipped(env: Env) -> None: + server = env.serve([case("c1")]) + server.capture_script["c1"] = [ + (409, err("Case failure is not retryable", details={"reason": "non_retryable_failure"})) + ] + result = env.runner().run() + assert result.executions[0].skipped == 1 and result.executions[0].submitted == 0 + assert outcomes(env) == [("c1", 409, "skipped")] + + +def test_capture_409_run_closed_stops_dispatch_and_cancelled_exits_1(env: Env) -> None: + server = env.serve([case(f"c{i}") for i in range(1, 6)]) + closed = (409, err("Run no longer accepts captures")) + + c2_in_flight = threading.Event() + + def cancel() -> Any: + # c1's slot frees before its capture is answered, so c2 is already running. + assert c2_in_flight.wait(5) + server.run["state"] = "cancelled" + return closed + + server.capture_script["c1"] = [cancel] + server.capture_default = closed + runner = env.runner(concurrency=1) + + def wait_until_closed(request: Any) -> Any: + c2_in_flight.set() + # Finish only after the 409, so nothing else can be dispatched before it. + with runner._cond: + assert runner._cond.wait_for(lambda: runner._run_closed, timeout=5) + return ok("c2") + + env.inference.behaviors["c2"] = wait_until_closed + with pytest.raises(LocalRunnerError) as raised: + runner.run() + assert raised.value.exit_code == 1 + assert raised.value.result is not None and raised.value.result.state == "cancelled" + assert env.inference.names == ["c1", "c2"] + assert ("c1", 409, "run_closed") in outcomes(env) + + +def test_run_closed_while_cancellation_is_finalizing_exits_1(env: Env) -> None: + server = env.serve([case("c1")]) + + def cancel() -> Any: + # The server keeps the Run running while it skips the remaining Cases. + server.run["cancellationRequestedAt"] = iso(START) + return 409, err("Run no longer accepts captures") + + server.capture_script["c1"] = [cancel] + with pytest.raises(LocalRunnerError, match="was cancelled") as raised: + env.runner().run() + assert raised.value.exit_code == 1 and not isinstance(raised.value, LocalRunnerInterrupted) + assert raised.value.resumable is False + assert ("c1", 409, "run_closed") in outcomes(env) + + +def test_capture_412_restarts_the_execution_at_the_new_attempt(env: Env) -> None: + server = env.serve([case("c1")]) + + def attempt_changed() -> Any: + server.attempt = 2 + return 412, { + "error": {"code": "PRECONDITION_FAILED", "message": "Run attempt changed; reload before submitting"} + } + + server.capture_script["c1"] = [attempt_changed] + result = env.runner().run() + + assert env.inference.names == ["c1", "c1"] # no output reused across attempts + assert [body["attemptCount"] for body in server.capture_bodies()] == [1, 2] + assert [entry["attempt"] for entry in captured(env)] == [1, 2] + assert result.executions[0].submitted == 1 + + +def test_412_while_the_next_page_prefetch_is_in_backoff_restarts_the_execution(env: Env) -> None: + server = env.serve([case("c1"), case("c2")]) + runner = env.runner(concurrency=1) + waiting_for_page = threading.Event() + original_prefetch = runner._prefetch + + def prefetch(ctx: Any, cursor: str) -> Any: + result = original_prefetch(ctx, cursor) + + def wait() -> Any: + waiting_for_page.set() + return result() + + return wait + + runner._prefetch = prefetch # type: ignore[method-assign] + + def attempt_changed() -> Any: + assert waiting_for_page.wait(5) # the dispatcher is waiting for the prefetched page + server.attempt = 2 + return 412, {"error": {"code": "PRECONDITION_FAILED", "message": "Run attempt changed"}} + + page_calls: List[int] = [] + + def flaky_page() -> None: + page_calls.append(1) + if len(page_calls) == 1: + # Fails only after the 412, so its backoff is abandoned because of the reload. + with runner._cond: + assert runner._cond.wait_for(lambda: runner._reload, timeout=5) + raise requests.exceptions.ConnectionError("reset") + + server.capture_script["c1"] = [attempt_changed] + server.before_page["1"] = flaky_page + result = runner.run() + + assert [body["attemptCount"] for body in server.capture_bodies("c1")] == [1, 2] + assert [body["attemptCount"] for body in server.capture_bodies("c2")] == [2] + assert result.executions[0].submitted == 2 + + +def output_captures(server: FakeServer, case_id: str) -> List[Dict[str, Any]]: + return [body for body in server.capture_bodies(case_id) if "output" in body] + + +def test_capture_409_no_retained_capture_on_resume_restarts_the_execution(env: Env) -> None: + checkpoint = {"state": "running", "attemptCount": 1, "hasCapture": True} + server = env.serve([case("c1", checkpoint=checkpoint), case("c2")]) + server.complete_on_capture = True + + def capture_lost() -> Any: + server.checkpoint("c1", {"state": "running", "attemptCount": 1, "hasCapture": False}) + return 409, err("No retained capture for this Case attempt") + + server.capture_script["c1"] = [capture_lost] + result = env.runner(concurrency=1).run() + + resume, capture = server.capture_bodies("c1") + assert resume == {"attemptCount": 1, "shardIndex": 0, "resume": True} + assert len(output_captures(server, "c1")) == 1 and capture["output"] == [{"case": "c1"}] + assert len(output_captures(server, "c2")) == 1 # c2 is not captured again by the restarted pass + assert sorted(env.inference.names) == ["c1", "c2"] + assert ("c1", 409, "reload") in outcomes(env) + tally = result.executions[0] + assert tally.submitted + tally.already_done == 2 + + +def test_page_at_another_attempt_restarts_the_execution_without_double_captures(env: Env) -> None: + server = env.serve([case(f"c{i}") for i in range(1, 5)]) + server.complete_on_capture = True + + def retried() -> None: + with server.lock: + server.attempt = 2 + + server.before_page["2"] = retried # the prefetched second page is at attempt 2 + result = env.runner(concurrency=2).run() + + attempts = [(cid, json.loads(body)["attemptCount"]) for cid, body in server.captures] + assert len(attempts) == len(set(attempts)) # no Case captured twice within one attempt + assert sorted(attempts) == [("c1", 1), ("c2", 1), ("c3", 2), ("c4", 2)] + assert sorted(env.inference.names) == ["c1", "c2", "c3", "c4"] + tally = result.executions[0] + assert (tally.already_done, tally.submitted) == (2, 2) + + +def test_capture_413_is_replaced_by_a_failure_capture(env: Env) -> None: + server = env.serve([case("c1")]) + server.capture_script["c1"] = [(413, "request entity too large")] + result = env.runner().run() + + first, second = server.capture_bodies() + assert second == { + "attemptCount": 1, + "shardIndex": 0, + "output": None, + "runtime": {"latencyMs": first["runtime"]["latencyMs"], "error": CAPTURE_TOO_LARGE_MESSAGE, "retryable": False}, + } + entries = captured(env) + assert len(entries) == 2 and json.loads(env.stored(entries[1])) == second + assert any(CAPTURE_TOO_LARGE_MESSAGE in warning for warning in env.warnings) + assert result.executions[0].submitted == 1 and result.executions[0].failed == 1 + + +def assert_replaced_without_sending(env: Env, server: FakeServer, result: Any) -> bytes: + """The capture was journaled, then replaced by the failure capture, the only one sent.""" + (body,) = server.capture_bodies() + assert body["output"] is None + assert {key: body["runtime"][key] for key in ("error", "retryable")} == { + "error": CAPTURE_TOO_LARGE_MESSAGE, + "retryable": False, + } + big, failure = captured(env) + assert json.loads(env.stored(failure)) == body # the journal's last capture for c1 wins + assert outcomes(env) == [("c1", 200, "submitted")] + assert result.executions[0].submitted == 1 and result.executions[0].failed == 1 + return env.stored(big) + + +def test_capture_over_the_limit_on_the_wire_is_replaced_without_sending_it(env: Env) -> None: + server = env.serve([case("c1")]) + output = [{"mask": "x" * local.CAPTURE_LIMIT_BYTES}] + env.inference.default = lambda request: (200, {}, json.dumps({"outputs": output})) + result = env.runner().run() + big = assert_replaced_without_sending(env, server, result) + assert json.loads(big)["output"] == output + + +def test_capture_size_is_measured_as_sent_with_ascii_escapes(env: Env) -> None: + server = env.serve([case("c1")]) + env.inference.default = lambda request: (200, {}, json.dumps({"outputs": [{"label": "ñ" * 1000}]})) + with patch.object(local, "CAPTURE_LIMIT_BYTES", 4000): + result = env.runner().run() + big = assert_replaced_without_sending(env, server, result) + # Under the limit as compact UTF-8, over it as requests sends it ("ñ"). + assert len(big) < 4000 < len(json.dumps(json.loads(big)).encode()) + + +def test_capture_at_the_limit_on_the_wire_is_sent_as_is(env: Env) -> None: + server = env.serve([case("c1")]) + output = [{"label": "ñ" * 1000}] + env.inference.default = lambda request: (200, {}, json.dumps({"outputs": output})) + expected = {"attemptCount": 1, "shardIndex": 0, "output": output, "runtime": {"latencyMs": 0.0}} + limit = len(json.dumps(expected).encode()) + with patch.object(local, "CAPTURE_LIMIT_BYTES", limit): + result = env.runner().run() + ((_, sent),) = server.captures + assert json.loads(sent) == expected and len(sent) == limit + assert len(captured(env)) == 1 and env.warnings == [] + assert result.executions[0].submitted == 1 and result.executions[0].failed == 0 + + +def test_capture_404_exits_3(env: Env) -> None: + server = env.serve([case("c1")]) + server.capture_script["c1"] = [ + (404, {"error": {"code": "NOT_FOUND", "message": "Case is not in this frozen snapshot shard"}}) + ] + with pytest.raises(LocalRunnerError) as raised: + env.runner().run() + assert raised.value.exit_code == 3 + + +def test_capture_400_exits_1_with_the_server_error(env: Env) -> None: + server = env.serve([case("c1")]) + server.capture_script["c1"] = [ + (400, {"error": {"code": "INVALID_REQUEST", "message": "Invalid browser Case capture"}}) + ] + with pytest.raises(LocalRunnerError) as raised: + env.runner().run() + assert raised.value.exit_code == 1 + assert raised.value.details["message"] == "Invalid browser Case capture" + assert raised.value.details["status"] == 400 + + +# -- 7. inference errors ---------------------------------------------------------------- + + +def test_inference_timeout_is_a_retryable_timed_out_failure_without_retries(env: Env) -> None: + server = env.serve([case("c1")]) + env.inference.default = lambda request: requests.exceptions.ReadTimeout("read timed out") + result = env.runner().run() + assert result.executions[0].pending == 0 and result.executions[0].failed == 1 + assert len(env.inference.requests) == 1 # one attempt, like the browser + assert env.clock.sleeps == [] + (body,) = server.capture_bodies() + assert body["output"] is None + assert body["runtime"]["error"] == TIMEOUT_MESSAGE + assert body["runtime"]["timedOut"] is True and body["runtime"]["retryable"] is True + + +@pytest.mark.parametrize( + ("response", "message"), + [ + ((422, {}, json.dumps({"message": "Invalid input image"})), "Invalid input image"), + ((400, {}, "bad"), "Workflow returned HTTP 400"), + ((200, {}, json.dumps({"success": False, "errorMsg": "Step failed"})), "Step failed"), + ((200, {}, json.dumps(["not", "an", "object"])), "Workflow returned an invalid response"), + ((200, {}, json.dumps({"success": True})), "Workflow returned no outputs"), + ], +) +def test_non_retryable_inference_failures_are_not_retried(env: Env, response: Any, message: str) -> None: + server = env.serve([case("c1")]) + env.inference.default = lambda request: response + result = env.runner().run() + assert len(env.inference.requests) == 1 + (body,) = server.capture_bodies() + assert body["runtime"]["error"] == message + assert body["runtime"]["retryable"] is False and body["runtime"]["timedOut"] is False + assert result.executions[0].failed == 1 + + +def test_inference_503_is_retried_3_times_then_captured_as_retryable(env: Env) -> None: + server = env.serve([case("c1")]) + env.inference.default = lambda request: (503, {}, "Service Unavailable") + result = env.runner().run() + assert len(env.inference.requests) == 3 + assert result.executions[0].pending == 0 + (body,) = server.capture_bodies() # exactly one capture + assert body["runtime"]["error"] == "Workflow returned HTTP 503" + assert body["runtime"]["retryable"] is True + + +def test_inference_retry_succeeds_after_a_429(env: Env) -> None: + server = env.serve([case("c1")]) + env.inference.behaviors["c1"] = [(429, {}, json.dumps({"message": "slow down"})), ok("c1")] + env.runner().run() + assert len(env.inference.requests) == 2 + assert server.capture_bodies()[0]["output"] == [{"case": "c1"}] + + +@pytest.mark.parametrize("outputs", ['[{"v":1e400}]', '[{"v":"\\ud800"}]'], ids=["non-finite-number", "lone-surrogate"]) +def test_unserializable_output_is_captured_as_a_non_retryable_failure(env: Env, outputs: str) -> None: + server = env.serve([case("c1")]) + env.inference.default = lambda request: (200, {}, '{"outputs":' + outputs + "}") + result = env.runner().run() + (body,) = server.capture_bodies() + assert body["output"] is None + runtime = body["runtime"] + assert runtime["error"].startswith("Workflow output could not be serialized: ") + assert runtime["timedOut"] is False and runtime["retryable"] is False and "latencyMs" in runtime + (entry,) = captured(env) + assert json.loads(env.stored(entry)) == body + assert any("could not be serialized" in warning for warning in env.warnings) + assert result.executions[0].submitted == 1 and result.executions[0].failed == 1 + + +def test_run_workflow_inference_wraps_outputs_and_truncates_errors() -> None: + session = requests.Session() + with responses.RequestsMock() as rsps: + rsps.add(responses.POST, f"{INF}/workflows/run", json={"outputs": None}) + rsps.add(responses.POST, f"{INF}/workflows/run", json={"message": "x" * 3000}, status=500) + options = {"api_key": "k", "specification": {}, "workflow_id": "w", "inputs": {}, "timeout": 1} + success = run_workflow_inference(session, INF + "/", **options) + failure = run_workflow_inference(session, INF, **options) + assert success.output == [None] and not success.error + assert len(failure.runtime["error"]) == 2000 and failure.retryable + + +def _name_resolution_error() -> requests.exceptions.ConnectionError: + reason = urllib3.exceptions.NameResolutionError("nohost", None, socket.gaierror(-2, "Name or service not known")) + return requests.exceptions.ConnectionError(urllib3.exceptions.MaxRetryError(None, "/", reason)) + + +# urllib3 1.26 has no NameResolutionError; there a DNS failure is a plain NewConnectionError. +HAS_NAME_RESOLUTION_ERROR = hasattr(urllib3.exceptions, "NameResolutionError") + + +@pytest.mark.parametrize( + ("error", "kind", "timed_out"), + [ + (refused_error(), "connect", False), + (requests.exceptions.ConnectionError(urllib3.exceptions.NewConnectionError(None, "refused")), "connect", False), + *([(_name_resolution_error(), "connect", False)] if HAS_NAME_RESOLUTION_ERROR else []), + (requests.exceptions.ConnectTimeout("connect timed out"), "connect", True), + (requests.exceptions.SSLError("certificate verify failed"), "connect", False), + (requests.exceptions.ProxyError("proxy refused"), "connect", False), + (reset_error(), "dropped", False), + ( + requests.exceptions.ConnectionError( + urllib3.exceptions.ProtocolError("Connection aborted.", RemoteDisconnected("closed")) + ), + "dropped", + False, + ), + (requests.exceptions.ConnectionError(BrokenPipeError(32, "Broken pipe")), "dropped", False), + (requests.exceptions.ChunkedEncodingError("Connection broken: IncompleteRead"), "dropped", False), + (requests.exceptions.ReadTimeout("read timed out"), None, True), + # requests reports a timeout while reading the body as a ConnectionError. + (requests.exceptions.ConnectionError(urllib3.exceptions.ReadTimeoutError(None, "/", "timed out")), None, True), + ], +) +def test_run_workflow_inference_classifies_connection_failures( + error: Exception, kind: Optional[str], timed_out: bool +) -> None: + with responses.RequestsMock() as rsps: + rsps.add(responses.POST, f"{INF}/workflows/run", body=error) + outcome = run_workflow_inference( + requests.Session(), INF, api_key="k", specification={}, workflow_id="w", inputs={}, timeout=1 + ) + assert outcome.connection_failure is (kind == "connect") + assert outcome.dropped is (kind == "dropped") + assert outcome.runtime["timedOut"] is timed_out and outcome.retryable and not outcome.responded + + +def test_tls_dropped_while_reading_the_body_is_a_dropped_connection() -> None: + with responses.RequestsMock() as rsps: + rsps.add_callback(responses.POST, f"{INF}/workflows/run", callback=tls_dropped) + outcome = run_workflow_inference( + requests.Session(), INF, api_key="k", specification={}, workflow_id="w", inputs={}, timeout=1 + ) + assert outcome.dropped and not outcome.connection_failure + assert outcome.runtime["error"] == FETCH_FAILED_MESSAGE + assert outcome.runtime["timedOut"] is False and outcome.retryable and not outcome.responded + + +class _StreamedResponse: + def __init__(self, error: Exception) -> None: + self.error = error + self.closed = False + + @property + def content(self) -> bytes: + raise self.error + + def close(self) -> None: + self.closed = True + + +class _StreamingSession: + def __init__(self, response: _StreamedResponse) -> None: + self.response = response + self.kwargs: Dict[str, Any] = {} + + def post(self, url: str, **kwargs: Any) -> _StreamedResponse: + self.kwargs = kwargs + return self.response + + +@pytest.mark.parametrize( + ("error", "kind", "timed_out"), + [ + # While reading the body the connection was made: these are drops, not connection failures. + (requests.exceptions.SSLError("EOF occurred in violation of protocol"), "dropped", False), + (requests.exceptions.ProxyError("proxy closed the tunnel"), "dropped", False), + (reset_error(), "dropped", False), + (requests.exceptions.ChunkedEncodingError("Connection broken: IncompleteRead"), "dropped", False), + (requests.exceptions.ConnectionError(urllib3.exceptions.ReadTimeoutError(None, "/", "timed out")), None, True), + (requests.exceptions.ReadTimeout("read timed out"), None, True), + (requests.exceptions.ContentDecodingError("bad gzip"), None, False), + ], +) +def test_run_workflow_inference_classifies_failures_while_reading_the_body( + error: Exception, kind: Optional[str], timed_out: bool +) -> None: + response = _StreamedResponse(error) + session = _StreamingSession(response) + outcome = run_workflow_inference( + session, # type: ignore[arg-type] + INF, + api_key="k", + specification={}, + workflow_id="w", + inputs={}, + timeout=1, + ) + assert session.kwargs["stream"] is True and response.closed + assert outcome.connection_failure is False + assert outcome.dropped is (kind == "dropped") + assert outcome.runtime["timedOut"] is timed_out and outcome.retryable and not outcome.responded + + +# -- 8. connection failures --------------------------------------------------------------- + + +def test_stops_after_10_consecutive_cases_that_cannot_connect(env: Env) -> None: + server = env.serve([case(f"c{i}") for i in range(1, 13)]) + env.inference.default = refused + with pytest.raises(LocalRunnerError) as raised: + env.runner(concurrency=1).run() + assert not isinstance(raised.value, InferenceServerUnreachable) + assert raised.value.exit_code == 1 + assert raised.value.message == f"The inference server at {INF} failed to connect for 10 Cases in a row." + assert PREFLIGHT_HINT in (raised.value.hint or "") + # Each of c1..c10 used its whole connect schedule (6 attempts); c11 was never dispatched. + assert env.inference.names == [f"c{i}" for i in range(1, 11) for _ in range(6)] + assert env.clock.sleeps == CONNECT_SLEEPS * 10 + # Connection failures are not Case failures: nothing is captured, the Cases stay pending on the server. + assert server.captures == [] + assert captured(env) == [] and outcomes(env) == [] + + +@pytest.mark.parametrize("tenth", ["ok", "dropped"]) +def test_a_case_that_reaches_the_server_resets_the_consecutive_count(env: Env, tenth: str) -> None: + names = [f"c{i}" for i in range(1, 20)] + server = env.serve([case(name) for name in names]) + env.inference.default = refused + # c10 reaches the server (an answer, or a connection dropped after it was made) between 9 + 9 Cases. + env.inference.behaviors["c10"] = ok("c10") if tenth == "ok" else (lambda request: reset_error()) + with pytest.raises(InferenceServerUnreachable) as raised: + env.runner(concurrency=1).run() + + assert raised.value.message == f"18 Cases were not run because the inference server at {INF} was unreachable." + assert [cid for cid, _ in server.captures] == ["c10"] + assert raised.value.result is not None + (tally,) = raised.value.result.executions + assert (tally.pending, tally.submitted, tally.failed) == (18, 1, 0 if tenth == "ok" else 1) + + +@pytest.mark.parametrize( + "error", + [ + pytest.param(refused_error(), id="refused"), + *([pytest.param(_name_resolution_error(), id="dns")] if HAS_NAME_RESOLUTION_ERROR else []), + pytest.param(requests.exceptions.ConnectTimeout("connect timed out"), id="connect-timeout"), + pytest.param(requests.exceptions.SSLError("certificate verify failed"), id="tls"), + ], +) +def test_unreachable_case_backs_off_1_2_4_8_16_then_is_left_pending(env: Env, error: Exception) -> None: + server = env.serve([case("c1"), case("c2")]) + env.inference.behaviors["c1"] = lambda request: error + with pytest.raises(LocalRunnerError) as raised: + env.runner(concurrency=1).run() + + assert isinstance(raised.value, InferenceServerUnreachable) + assert raised.value.exit_code == 1 + assert raised.value.message == f"1 Case was not run because the inference server at {INF} was unreachable." + assert "run the same command again" in (raised.value.hint or "") + assert env.inference.names == ["c1"] * 6 + ["c2"] + assert env.clock.sleeps == CONNECT_SLEEPS + assert [cid for cid, _ in server.captures] == ["c2"] # no capture POST for c1 + assert [entry["caseId"] for entry in captured(env)] == ["c2"] + assert raised.value.result is not None + (tally,) = raised.value.result.executions + assert (tally.pending, tally.submitted, tally.failed) == (1, 1, 0) + assert tally.to_dict()["pending"] == 1 + assert env.progress[-1]["pending"] == 1 and env.progress[-1]["remaining"] == 0 + + +@pytest.mark.parametrize( + "error", + [reset_error(), requests.exceptions.ChunkedEncodingError("Connection broken: IncompleteRead"), None], + ids=["reset", "dropped-mid-response", "tls-dropped-mid-body"], +) +def test_dropped_connection_is_retried_3_times_then_captured_as_retryable(env: Env, error: Optional[Exception]) -> None: + server = env.serve([case("c1"), case("c2")]) + env.inference.behaviors["c1"] = tls_dropped if error is None else (lambda request: error) + result = env.runner(concurrency=1).run() + + assert env.inference.names == ["c1"] * 3 + ["c2"] + assert env.clock.sleeps == [1.0, 2.0] + (body,) = server.capture_bodies("c1") + assert body["output"] is None + assert set(body["runtime"]) == {"latencyMs", "error", "timedOut", "retryable"} + assert body["runtime"]["error"] == FETCH_FAILED_MESSAGE + assert body["runtime"]["timedOut"] is False and body["runtime"]["retryable"] is True + assert [entry["caseId"] for entry in captured(env)] == ["c1", "c2"] + assert ("c1", 200, "submitted") in outcomes(env) + assert result.executions[0].to_dict() == { + "id": X, + "submitted": 2, + "alreadyDone": 0, + "skipped": 0, + "failed": 1, + "pending": 0, + } + + +def test_connected_and_connect_budgets_are_separate(env: Env) -> None: + server = env.serve([case("c1")]) + unavailable = (503, {}, "Service Unavailable") + # Two 503s use 2 of the 3 connected attempts; the refusals in between do not. + env.inference.behaviors["c1"] = [unavailable, refused, refused, unavailable, refused, ok("c1")] + result = env.runner().run() + assert len(env.inference.requests) == 6 + assert env.clock.sleeps == [1.0, 1.0, 2.0, 2.0, 4.0] + assert server.capture_bodies()[0]["output"] == [{"case": "c1"}] + assert result.executions[0].failed == 0 + + +def test_a_short_outage_does_not_fail_the_run(env: Env) -> None: + names = [f"c{i}" for i in range(1, 17)] + server = env.serve([case(name) for name in names]) + + def down_for_3_seconds(request: Any) -> Any: + if env.clock.time() < START + 3: + return refused_error() + return ok(json.loads(request.body)["inputs"]["name"]) + + env.inference.default = down_for_3_seconds + result = env.runner(concurrency=4).run() + + assert sorted(cid for cid, _ in server.captures) == sorted(names) + assert all(body["output"] is not None for body in server.capture_bodies()) + assert any(sleep == 1.0 for sleep in env.clock.sleeps) # the outage was hit and waited out + assert result.executions[0].to_dict() == { + "id": X, + "submitted": 16, + "alreadyDone": 0, + "skipped": 0, + "failed": 0, + "pending": 0, + } + + +def test_http_response_resets_the_connection_failure_count(env: Env) -> None: + env.serve([case(f"c{i}") for i in range(1, 6)]) + for name in ("c1", "c2", "c3", "c4", "c5"): + env.inference.behaviors[name] = [refused, refused, ok(name)] + result = env.runner(concurrency=1).run() + assert result.executions[0].submitted == 5 and result.executions[0].failed == 0 + + +# -- 9. stop requests --------------------------------------------------------------------- + + +def test_stop_request_journals_in_flight_results_and_raises_130(env: Env) -> None: + server = env.serve([case(f"c{i}") for i in range(1, 5)]) + runner = env.runner(concurrency=2) + both_in_flight = threading.Barrier(2, timeout=5) + stopped = threading.Event() + + def first(request: Any) -> Any: + both_in_flight.wait() + runner.request_stop() + stopped.set() + return ok("c1") + + def second(request: Any) -> Any: + both_in_flight.wait() + assert stopped.wait(5) + return ok("c2") + + env.inference.behaviors.update(c1=first, c2=second) + with pytest.raises(LocalRunnerInterrupted) as raised: + runner.run() + assert raised.value.exit_code == 130 + assert sorted(env.inference.names) == ["c1", "c2"] + assert sorted(entry["caseId"] for entry in captured(env)) == ["c1", "c2"] + assert sorted(cid for cid, _ in server.captures) == ["c1", "c2"] # queued captures still submitted + + +def test_second_stop_request_aborts_without_waiting(env: Env) -> None: + env.serve([case("c1")]) + runner = env.runner() + release = threading.Event() + + def stuck(request: Any) -> Any: + runner.request_stop() + runner.request_stop() + release.wait(5) + return ok("c1") + + env.inference.behaviors["c1"] = stuck + try: + with pytest.raises(LocalRunnerInterrupted): + runner.run() + assert not release.is_set() + finally: + release.set() + for thread in runner._threads: + thread.join(5) + + +@pytest.mark.skipif(not hasattr(signal, "pthread_kill"), reason="needs pthread_kill") +def test_second_signal_interrupts_a_blocking_api_call_on_the_main_thread(env: Env) -> None: + env.serve([case("c1")]) + runner = env.runner() + entered, release = threading.Event(), threading.Event() + + def blocking_get_run(*args: Any, **kwargs: Any) -> Any: + entered.set() + release.wait(2) + return {"id": R, "state": "completed", "executionMode": "browser"} + + def send_two_signals() -> None: + assert entered.wait(5) + main = threading.main_thread().ident + signal.pthread_kill(main, signal.SIGINT) + while not runner._stop_requested: + release.wait(0.01) + signal.pthread_kill(main, signal.SIGINT) + + sender = threading.Thread(target=send_two_signals) + try: + with patch.object(WorkflowEvals, "get_run", blocking_get_run), install_signal_handlers(runner): + sender.start() + with pytest.raises(LocalRunnerInterrupted) as raised: + runner.run() + assert not release.is_set() # interrupted inside get_run, not after it returned + assert raised.value.exit_code == 130 + assert raised.value.result is not None + finally: + release.set() + sender.join(5) + assert runner._threads == [] + + +def test_install_signal_handlers_routes_to_request_stop_and_restores(env: Env) -> None: + runner = env.runner() + before = signal.getsignal(signal.SIGINT) + with install_signal_handlers(runner): + handler = signal.getsignal(signal.SIGINT) + assert callable(handler) + handler(signal.SIGINT, None) + assert runner._stop_requested + assert signal.getsignal(signal.SIGINT) is before + + +# -- 10. startup checks --------------------------------------------------------------------- + + +def test_missing_workflow_inputs_means_the_server_is_too_old(env: Env) -> None: + item = case("c1") + del item["workflowInputs"] + env.serve([item]) + with pytest.raises(LocalRunnerError, match="server does not support local runner yet") as raised: + env.runner().run() + assert raised.value.exit_code == 1 + assert env.inference.requests == [] + + +def test_server_run_is_rejected_with_a_hint(env: Env) -> None: + env.serve([case("c1")], mode="server") + with pytest.raises(LocalRunnerError) as raised: + env.runner().run() + assert raised.value.exit_code == 1 + assert raised.value.hint == "This Run executes on Roboflow; use `run get --wait`." + + +@pytest.mark.parametrize( + "describe", + [(503, {}, json.dumps({"message": "warming up"})), requests.exceptions.ConnectionError("refused")], +) +def test_preflight_failure_exits_1_before_any_inference(env: Env, describe: Any) -> None: + env.serve([case("c1")]) + env.describe_response = describe + with pytest.raises(LocalRunnerError) as raised: + env.runner().run() + assert raised.value.exit_code == 1 and raised.value.hint == PREFLIGHT_HINT + assert env.inference.requests == [] + + +@pytest.mark.parametrize(("status", "exit_code"), [(401, 2), (403, 2), (404, 3)]) +def test_api_errors_map_to_exit_codes(env: Env, status: int, exit_code: int) -> None: + env.rsps.add(responses.GET, RUN, json={"error": {"code": "X", "message": "nope"}}, status=status) + with pytest.raises(LocalRunnerError) as raised: + env.runner().run() + assert raised.value.exit_code == exit_code + + +def test_unknown_execution_exits_3(env: Env) -> None: + env.serve([case("c1")]) + with pytest.raises(LocalRunnerError) as raised: + env.runner(execution_ids=["exec-404"]).run() + assert raised.value.exit_code == 3 + + +def test_terminal_run_returns_without_work_unless_retry_is_allowed(env: Env) -> None: + server = env.serve([case("c1")]) + server.run["state"] = "partially_completed" + assert env.runner().run().state == "partially_completed" + assert env.inference.requests == [] and env.describe == [] + + server.run["retry"] = {"allowed": True} + result = env.runner(retry=True).run() + assert server.retried == 1 + assert env.inference.names == ["c1"] + assert server.capture_bodies()[0]["attemptCount"] == 2 + assert result.executions[0].submitted == 1 + + +def test_inference_url_override_warns(env: Env) -> None: + server = env.serve([case("c1")]) + server.frozen_url = "http://gpu-box:9001/" + env.runner(inference_url=INF + "/").run() + assert len(env.inference.requests) == 1 + assert any("http://gpu-box:9001" in warning for warning in env.warnings) + + +def test_default_state_dir_respects_xdg_cache_home(tmp_path: Any) -> None: + with patch.dict(os.environ, {"XDG_CACHE_HOME": str(tmp_path)}): + assert local.default_state_dir("ws", "run-9") == os.path.join( + str(tmp_path), "roboflow", "workflow-evals", "ws", "run-9" + ) + + +# -- 11. retry happens only before submission ------------------------------------------------ + + +def test_capture_post_failure_never_reruns_inference(env: Env) -> None: + server = env.serve([case("c1")]) + server.capture_script["c1"] = [(503, "unavailable")] + result = env.runner().run() + assert len(env.inference.requests) == 1 + assert len(server.captures) == 2 and server.captures[0][1] == server.captures[1][1] + assert env.clock.sleeps == [1.0] + assert result.executions[0].submitted == 1 + + +def test_no_threads_are_left_behind(env: Env) -> None: + env.serve([case(f"c{i}") for i in range(1, 7)]) + before = threading.active_count() + env.runner(concurrency=3).run() + assert threading.active_count() == before + + +# -- 12. inference server down, then back (end to end through the CLI) ------------------------- + + +def test_inference_down_stops_then_the_same_command_completes_every_case_once(env: Env) -> None: + names = [f"c{i}" for i in range(1, 14)] + server = env.serve([case(name) for name in names]) + server.complete_on_capture = True + env.inference.default = refused + env.inference.behaviors.update(c1=ok("c1"), c2=ok("c2")) # the server goes down after c2 + fake_time = {"clock": env.clock.time, "monotonic": env.clock.time, "sleep": env.clock.sleep} + argv = ["--workspace", WS, "--api-key", KEY, "--json", "workflow-eval", "run", "local", R, "-e", E] + argv += ["--concurrency", "1", "--state-dir", env.state_dir] + + def invoke() -> Any: + before = threading.active_count() + with patch.object(local, "LocalRunner", lambda *args, **kwargs: LocalRunner(*args, **kwargs, **fake_time)): + result = CliRunner().invoke(app, argv) + assert threading.active_count() == before + return result + + first = invoke() + assert first.exit_code == 1, first.output + error = json.loads(first.stderr.strip().splitlines()[-1])["error"] + assert "failed to connect for 10 Cases in a row" in error["message"] and PREFLIGHT_HINT in error["hint"] + assert first.stdout == "" + # c3..c12 each used their 6 connect attempts: 10 Cases in a row. Only c1 and c2 were captured. + assert env.inference.names == ["c1", "c2"] + [f"c{i}" for i in range(3, 13) for _ in range(6)] + assert [cid for cid, _ in server.captures] == ["c1", "c2"] + assert [entry["caseId"] for entry in captured(env)] == ["c1", "c2"] # the journal is kept + + env.inference.requests.clear() + env.inference.behaviors.clear() + env.inference.default = lambda request: ok(json.loads(request.body)["inputs"]["name"]) + second = invoke() + + assert second.exit_code == 0, second.output + summary = json.loads(second.stdout) + assert summary["executions"][0]["alreadyDone"] == 2 and summary["executions"][0]["submitted"] == 11 + assert summary["executions"][0]["pending"] == 0 and summary["warnings"] == [] + assert env.inference.names == names[2:] # c1 and c2 are not inferred again + assert sorted(cid for cid, _ in server.captures) == sorted(names) # every Case captured exactly once