From fe7de5df81e0256cea4d9a8212e9dd215d4b4ba2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 22 Oct 2026 18:49:59 -0700 Subject: [PATCH] feat(agent): add run limits for spend, steps, and duration Adds a "Limit run" switch to the Agent block's advanced settings with max spend (USD), max steps, and max duration, and a choice of what happens when the agent still wants tools past a limit: return a final answer without tools (default) or fail the block. A new stopReason output reports whether a limit stopped the run. A step is one model response whose tool calls run. Limits are enforced at the two seams every provider already shares, so no provider tool loop changes: prepareConversationGeneration turns the agent's tools off in each protocol's own final-answer shape once a limit is reached, and executeProviderTool refuses tools past a limit. Calls that offer no agent tools (structured-output passes, Bedrock's forced structured_output tool) are never limited, so a run that answers within its limits is unaffected. One budget spans the selected model and its fallbacks; spend counts each attempt's own billed cost, excluding what it carried in from earlier attempts. Refusals are not journaled, so a retry with a fresh budget runs the call. Also renames isConversationContextError to isPermanentGenerationError, since it now covers the run-limit and usage-limit stops, and shares the billed user-stop construction between both stops. --- .../content/docs/workflows/blocks/agent.mdx | 4 +- apps/sim/blocks/blocks/agent.ts | 81 ++++- apps/sim/executor/execution/block-executor.ts | 3 + apps/sim/executor/handlers/agent/handler.ts | 32 +- apps/sim/executor/handlers/agent/types.ts | 6 + apps/sim/providers/anthropic/core.ts | 4 +- apps/sim/providers/azure-openai/index.ts | 4 +- apps/sim/providers/bedrock/index.ts | 4 +- apps/sim/providers/cerebras/index.ts | 4 +- apps/sim/providers/conversation-generation.ts | 12 +- apps/sim/providers/deepseek/index.ts | 4 +- apps/sim/providers/groq/index.ts | 4 +- apps/sim/providers/kimi/index.ts | 4 +- apps/sim/providers/litellm/index.ts | 4 +- apps/sim/providers/meta/index.ts | 4 +- apps/sim/providers/mistral/index.ts | 4 +- apps/sim/providers/nvidia/index.ts | 4 +- apps/sim/providers/ollama/core.ts | 4 +- .../openai-compat/chat-completions.ts | 4 +- apps/sim/providers/openai/core.ts | 4 +- apps/sim/providers/run-limits.test.ts | 289 ++++++++++++++++++ apps/sim/providers/run-limits.ts | 202 ++++++++++++ apps/sim/providers/run-usage-limit.test.ts | 4 +- apps/sim/providers/run-usage-limit.ts | 19 +- apps/sim/providers/runtime-context.ts | 8 + apps/sim/providers/sakana/index.ts | 4 +- apps/sim/providers/vllm/index.ts | 4 +- apps/sim/providers/xai/index.ts | 4 +- apps/sim/providers/zai/index.ts | 4 +- 29 files changed, 681 insertions(+), 51 deletions(-) create mode 100644 apps/sim/providers/run-limits.test.ts create mode 100644 apps/sim/providers/run-limits.ts diff --git a/apps/docs/content/docs/workflows/blocks/agent.mdx b/apps/docs/content/docs/workflows/blocks/agent.mdx index 154f29e2492..7b08a6d7739 100644 --- a/apps/docs/content/docs/workflows/blocks/agent.mdx +++ b/apps/docs/content/docs/workflows/blocks/agent.mdx @@ -121,7 +121,7 @@ Set **Response Format** to a JSON Schema when later blocks need named fields. Na Malformed JSON can fall back to plain text with `_responseFormatWarning`. Sim does not perform a general post-parse schema check on every provider result, so validate fields downstream before consequential actions. -Avoid schema fields named `model`, `tokens`, `toolCalls`, `providerTiming`, or `cost`. Sim's standard metadata overwrites those fields after parsing, even when a metadata value is absent. +Avoid schema fields named `model`, `tokens`, `toolCalls`, `providerTiming`, or `cost`. Sim's standard metadata overwrites those fields after parsing, even when a metadata value is absent. With **Limit run** on, `stopReason` is overwritten too. ```json { @@ -144,6 +144,7 @@ Some settings live under advanced, or appear only for models that support them: - **Temperature.** Controls variation on models that support it. Lower values tend to reduce variation; test accuracy on representative inputs rather than treating temperature as a factuality guarantee. - **Max output tokens.** Caps the response length. Defaults to the model's full limit. +- **Limit run.** Stops a long tool-using run at a **max spend** in USD, a **max number of steps**, or a **max duration** in seconds. A step is one model response whose tool calls run. Leave a field empty for no limit. A limit acts only when the agent wants to keep using tools, so a call or tool that is already running finishes first, and a run that finishes before reaching a limit is unaffected. Spend is measured the same way as ``, including tool costs, so a model Sim does not bill for, such as one on your own API key, adds only its tool costs. One budget covers the selected model and its fallbacks, and each try of **Retry on fail** starts a new one. **When a limit is reached** chooses what happens: **Return best answer** (the default) asks the model for a final answer without tools, which can spend slightly past the limit, and fails the block only if the model keeps calling tools anyway; **Fail block** fails the block if the agent asks for more tools. `` reports whether a limit stopped the run. - **Reasoning effort / Thinking level.** Controls supported models' reasoning budget. Higher settings can increase latency and token use; evaluate the result on your task. - **Prompt caching.** Reuses a stable system prompt and tool definitions on supported Claude models. Sim uses a five-minute cache. Cache-write pricing, read discounts, and minimum prefix length depend on the model; check [Anthropic's current requirements](https://platform.claude.com/docs/en/build-with-claude/prompt-caching#cache-limitations). Enable it for repeated runs sharing a prefix; a one-off run can pay a write premium without a later cache hit. - **API key.** Your key for the chosen provider. Hidden when hosted Sim supplies a key for the selected model, including Jev. @@ -163,6 +164,7 @@ After the agent runs, later blocks read its result by name: | `` | The tools the agent called, with their inputs and results | | `` | The model that ran the step | | `` | Estimated cost in USD, as an object with `input`, `output`, and `total`; read `` for the total | +| `` | With **Limit run** on: `completed`, or the limit that stopped the run: `spend_limit`, `step_limit`, or `time_limit` | The cost object can include additional fields such as tool costs or pricing. The Logs UI and run-resource API use credits. See [Costs](/platform/costs) for units and billing behavior. diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index fb03f56714a..8bd4291eabc 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -1,7 +1,7 @@ import { createLogger } from '@sim/logger' import { AgentIcon } from '@/components/icons' import { getModelFallbackSubBlock, MODEL_FALLBACK_INPUTS } from '@/blocks/model-fallbacks' -import type { BlockConfig } from '@/blocks/types' +import type { BlockConfig, SubBlockConfig } from '@/blocks/types' import { AuthMode, IntegrationType } from '@/blocks/types' import { getAgentModelOptions, @@ -42,6 +42,12 @@ const MODELS_WITHOUT_MEMORY = getModelsWithoutMemory() const EVALUATION_MODELS = getEvaluationModels() const MODELS_WITHOUT_CHAT_CONTROLS = [...MODELS_WITH_DEEP_RESEARCH, ...EVALUATION_MODELS] +const RUN_LIMIT_FIELD_CONDITION: SubBlockConfig['condition'] = { + field: 'runLimits', + value: true, + and: { field: 'model', value: MODELS_WITHOUT_CHAT_CONTROLS, not: true }, +} + // Helper function to get the tool ID from a block type const getToolIdFromBlock = (blockType: string): string | undefined => { try { @@ -422,6 +428,56 @@ Return ONLY the JSON array.`, not: true, }, }, + { + id: 'runLimits', + title: 'Limit run', + type: 'switch', + description: + 'A step is one model response whose tool calls run. Limits stop the agent from starting new steps; a call or tool already running finishes first. Spend is measured like the cost output, tools included.', + defaultValue: false, + mode: 'advanced', + condition: { + field: 'model', + value: MODELS_WITHOUT_CHAT_CONTROLS, + not: true, + }, + }, + { + id: 'maxRunCost', + title: 'Max spend (USD)', + type: 'short-input', + placeholder: 'No limit', + mode: 'advanced', + condition: RUN_LIMIT_FIELD_CONDITION, + }, + { + id: 'maxSteps', + title: 'Max steps', + type: 'short-input', + placeholder: 'No limit', + mode: 'advanced', + condition: RUN_LIMIT_FIELD_CONDITION, + }, + { + id: 'maxRunDuration', + title: 'Max duration (seconds)', + type: 'short-input', + placeholder: 'No limit', + mode: 'advanced', + condition: RUN_LIMIT_FIELD_CONDITION, + }, + { + id: 'onRunLimit', + title: 'When a limit is reached', + type: 'dropdown', + options: [ + { label: 'Return best answer', id: 'finalize' }, + { label: 'Fail block', id: 'fail' }, + ], + defaultValue: 'finalize', + mode: 'advanced', + condition: RUN_LIMIT_FIELD_CONDITION, + }, { id: 'responseFormat', title: 'Response Format', @@ -614,6 +670,24 @@ Return ONLY the JSON array.`, type: 'boolean', description: 'Cache the system prompt and tool definitions on models that support it', }, + runLimits: { + type: 'boolean', + description: 'Stop the agent at the spend, step, and duration limits below', + }, + maxRunCost: { type: 'number', description: 'Most this block may be billed, in USD' }, + maxSteps: { + type: 'number', + description: 'Most model responses whose tool calls may run', + }, + maxRunDuration: { + type: 'number', + description: 'Seconds after which the agent starts no new steps', + }, + onRunLimit: { + type: 'string', + description: + 'When the agent still wants tools past a limit: finalize to answer without them, or fail to fail the block', + }, ...MODEL_FALLBACK_INPUTS, tools: { type: 'json', description: 'Available tools configuration' }, skills: { type: 'json', description: 'Selected skills configuration' }, @@ -634,6 +708,11 @@ Return ONLY the JSON array.`, description: 'Provider timing information', }, cost: { type: 'json', description: 'Cost of the API call' }, + stopReason: { + type: 'string', + description: 'Why the agent stopped: completed, spend_limit, step_limit, or time_limit', + condition: { field: 'runLimits', value: true }, + }, interactionId: { type: 'string', description: 'Interaction ID for multi-turn deep research follow-ups', diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index a859f3004f1..a3b7cc706e7 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -1423,6 +1423,9 @@ export class BlockExecutor { providerTiming: executionOutput.providerTiming, cost: executionOutput.cost, model: executionOutput.model, + ...(executionOutput.stopReason !== undefined && { + stopReason: executionOutput.stopReason, + }), } parsedForFormat = true } catch (error) { diff --git a/apps/sim/executor/handlers/agent/handler.ts b/apps/sim/executor/handlers/agent/handler.ts index 0de3b244c88..1e3c70bf9a6 100644 --- a/apps/sim/executor/handlers/agent/handler.ts +++ b/apps/sim/executor/handlers/agent/handler.ts @@ -57,7 +57,7 @@ import { import { getAllBlocks, getBlock } from '@/blocks' import { assembleCustomBlockInputMapping, isCustomBlockType } from '@/blocks/custom/build-config' import type { BlockOutput } from '@/blocks/types' -import { normalizeFileInput } from '@/blocks/utils' +import { normalizeFileInput, parseOptionalNumberInput } from '@/blocks/utils' import { assertPermissionsAllowed, validateBlockType, @@ -126,6 +126,7 @@ import { getInlineHydrationMaxBytes, } from '@/providers/file-attachments.server' import { isAutoModel, isEvaluationModel, SIM_AUTO_MODEL_ID } from '@/providers/models' +import { AgentRunLimits } from '@/providers/run-limits' import { isRunAbortError } from '@/providers/streaming-tool-loop-shared' import { type ProviderToolInputProvenance, @@ -231,6 +232,7 @@ interface ExecuteAcrossModelsConfig { resultRegistry: ResolvedSecretTraceRegistry | undefined providerErrorRegistry: ResolvedSecretTraceRegistry | undefined agentConversation?: AgentTurnSession + runLimits?: AgentRunLimits } interface FormattedAgentTools { @@ -519,6 +521,7 @@ export class AgentBlockHandler implements BlockHandler { } const streamingConfig = this.getStreamingConfig(ctx, block) + const runLimits = this.createRunLimits(filteredInputs) const agentConversation = filteredInputs.memoryType && filteredInputs.memoryType !== 'none' && @@ -649,6 +652,7 @@ export class AgentBlockHandler implements BlockHandler { resultRegistry, providerErrorRegistry, agentConversation, + runLimits, }) if (servedRegistry) ctx.resolvedSecretTraceRegistry = servedRegistry @@ -667,6 +671,7 @@ export class AgentBlockHandler implements BlockHandler { if (this.isStreamingExecution(result)) { const streamingResult = result as StreamingExecution streamingResult.diagnosticResolvedSecretTraceRegistry = providerErrorRegistry + runLimits?.reportOn(streamingResult.execution.output) if (filteredInputs.memoryType && filteredInputs.memoryType !== 'none') { return this.wrapStreamForMemoryPersistence( ctx, @@ -689,12 +694,30 @@ export class AgentBlockHandler implements BlockHandler { ) } + if (runLimits && isRecordLike(result)) runLimits.reportOn(result) return result } finally { settlePrivateAgentSelectors() } } + /** The limits a block set while Limit run is on; parsed once and shared by every model it tries. */ + private createRunLimits(inputs: AgentInputs): AgentRunLimits | undefined { + if (inputs.runLimits !== true) return undefined + const maxRunDuration = parseOptionalNumberInput(inputs.maxRunDuration, 'Max duration', { + min: 1, + }) + return new AgentRunLimits({ + maxCost: parseOptionalNumberInput(inputs.maxRunCost, 'Max spend', { min: 0.01 }), + maxSteps: parseOptionalNumberInput(inputs.maxSteps, 'Max steps', { + integer: true, + min: 1, + }), + maxDurationMs: maxRunDuration === undefined ? undefined : maxRunDuration * 1000, + onLimit: inputs.onRunLimit === 'fail' ? 'fail' : 'finalize', + }) + } + private resolveToolUsageControls( tools: ToolInput[], canonicalModes?: Record @@ -2755,7 +2778,8 @@ export class AgentBlockHandler implements BlockHandler { config.responseFormat, resultRegistry, config.providerErrorRegistry, - config.agentConversation + config.agentConversation, + config.runLimits ) if ((hasNext || config.retryPrimaryOnStreamStart) && this.isStreamingExecution(result)) { result = await this.primeStreamingExecution(result as StreamingExecution) @@ -2974,7 +2998,8 @@ export class AgentBlockHandler implements BlockHandler { responseFormat: any, modelRuntimeRegistry: ResolvedSecretTraceRegistry | undefined, providerErrorRegistry: ResolvedSecretTraceRegistry | undefined, - agentConversation?: AgentTurnSession + agentConversation?: AgentTurnSession, + runLimits?: AgentRunLimits ): Promise { const providerId = providerRequest.provider const model = providerRequest.model @@ -3056,6 +3081,7 @@ export class AgentBlockHandler implements BlockHandler { agentMemoryContext: agentConversation ? { historyTokens: providerRequest.memoryHistoryTokens } : undefined, + runLimits, } ) diff --git a/apps/sim/executor/handlers/agent/types.ts b/apps/sim/executor/handlers/agent/types.ts index a092fee6959..b8e66b9b5be 100644 --- a/apps/sim/executor/handlers/agent/types.ts +++ b/apps/sim/executor/handlers/agent/types.ts @@ -50,6 +50,12 @@ export interface AgentInputs { verbosity?: string thinkingLevel?: string promptCaching?: boolean + // Run limits, applied only while `runLimits` is on + runLimits?: boolean + maxRunCost?: string | number + maxSteps?: string | number + maxRunDuration?: string | number + onRunLimit?: 'finalize' | 'fail' files?: unknown /** Ordered models tried when the request to `model` fails; see `normalizeFallbackModels`. */ fallbackModels?: Array & { model: string }> diff --git a/apps/sim/providers/anthropic/core.ts b/apps/sim/providers/anthropic/core.ts index 6cf80c3d820..66b31b61d72 100644 --- a/apps/sim/providers/anthropic/core.ts +++ b/apps/sim/providers/anthropic/core.ts @@ -21,7 +21,7 @@ import { createReadableStreamFromAnthropicStream, } from '@/providers/anthropic/utils' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -1006,7 +1006,7 @@ export async function executeAnthropicProviderRequest( } ) - if (isAbortError(error) || request.abortSignal?.aborted || isConversationContextError(error)) { + if (isAbortError(error) || request.abortSignal?.aborted || isPermanentGenerationError(error)) { throw error } diff --git a/apps/sim/providers/azure-openai/index.ts b/apps/sim/providers/azure-openai/index.ts index 37db6e20761..ad9227bf448 100644 --- a/apps/sim/providers/azure-openai/index.ts +++ b/apps/sim/providers/azure-openai/index.ts @@ -26,7 +26,7 @@ import { isResponsesEndpoint, } from '@/providers/azure-openai/utils' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -716,7 +716,7 @@ async function executeChatCompletionsRequest( } ) - if (isAbortError(error) || request.abortSignal?.aborted || isConversationContextError(error)) { + if (isAbortError(error) || request.abortSignal?.aborted || isPermanentGenerationError(error)) { throw error } diff --git a/apps/sim/providers/bedrock/index.ts b/apps/sim/providers/bedrock/index.ts index 7b7f19585a6..d0f77255cea 100644 --- a/apps/sim/providers/bedrock/index.ts +++ b/apps/sim/providers/bedrock/index.ts @@ -31,7 +31,7 @@ import { } from '@/providers/bedrock/utils' import { getCachedProviderClient } from '@/providers/client-cache' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -1060,7 +1060,7 @@ export const bedrockProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error } diff --git a/apps/sim/providers/cerebras/index.ts b/apps/sim/providers/cerebras/index.ts index 1a6d94f0db8..a341bc255f3 100644 --- a/apps/sim/providers/cerebras/index.ts +++ b/apps/sim/providers/cerebras/index.ts @@ -7,7 +7,7 @@ import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import type { CerebrasResponse } from '@/providers/cerebras/types' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -676,7 +676,7 @@ export const cerebrasProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error } diff --git a/apps/sim/providers/conversation-generation.ts b/apps/sim/providers/conversation-generation.ts index ac56db20abe..b104c587f99 100644 --- a/apps/sim/providers/conversation-generation.ts +++ b/apps/sim/providers/conversation-generation.ts @@ -21,6 +21,7 @@ import { isConversationHistoryNotice, } from '@/providers/conversation-metadata' import { getConversationModelLimits } from '@/providers/conversation-model' +import { applyRunLimits, RunLimitError } from '@/providers/run-limits' import { assertRunWithinUsageLimit, RunUsageLimitError } from '@/providers/run-usage-limit' import { isAbortError } from '@/providers/streaming-tool-loop-shared' import type { Message, ProviderRequest } from '@/providers/types' @@ -117,13 +118,15 @@ function contextError(message: string): Error { /** * Provider wrappers must preserve the permanent failures this module raises, context faults and - * the run's usage-limit stop, so fallback cannot restart them and the stop keeps its billed cost. + * the run's usage-limit and run-limit stops, so fallback cannot restart them and a stop keeps its + * billed cost. */ -export function isConversationContextError(error: unknown): boolean { +export function isPermanentGenerationError(error: unknown): boolean { return ( error instanceof AgentContextLimitError || error instanceof AgentContextProtocolError || - error instanceof RunUsageLimitError + error instanceof RunUsageLimitError || + error instanceof RunLimitError ) } @@ -288,10 +291,11 @@ function nativeSummaryMessage(summary: Message, protocol: ConversationProtocol): export async function prepareConversationGeneration( request: ProviderRequest, protocol: ConversationProtocol, - payload: T + requestedPayload: T ): Promise { const runtime = getConversationRequestContext(request) await assertRunWithinUsageLimit(runtime) + const payload = applyRunLimits(runtime, request, protocol, requestedPayload) if (!runtime?.agentConversation) return payload request.abortSignal?.throwIfAborted() if (!isRecordLike(payload)) diff --git a/apps/sim/providers/deepseek/index.ts b/apps/sim/providers/deepseek/index.ts index b2750a8090c..5a68714cf8f 100644 --- a/apps/sim/providers/deepseek/index.ts +++ b/apps/sim/providers/deepseek/index.ts @@ -6,7 +6,7 @@ import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -686,7 +686,7 @@ export const deepseekProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error } diff --git a/apps/sim/providers/groq/index.ts b/apps/sim/providers/groq/index.ts index 1357a763d18..c2eb6333e1e 100644 --- a/apps/sim/providers/groq/index.ts +++ b/apps/sim/providers/groq/index.ts @@ -11,7 +11,7 @@ import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -630,7 +630,7 @@ export const groqProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error } diff --git a/apps/sim/providers/kimi/index.ts b/apps/sim/providers/kimi/index.ts index 0434dc047d5..a5a0d798540 100644 --- a/apps/sim/providers/kimi/index.ts +++ b/apps/sim/providers/kimi/index.ts @@ -7,7 +7,7 @@ import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -754,7 +754,7 @@ export const kimiProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error } diff --git a/apps/sim/providers/litellm/index.ts b/apps/sim/providers/litellm/index.ts index 913816b5e60..04e60c97be3 100644 --- a/apps/sim/providers/litellm/index.ts +++ b/apps/sim/providers/litellm/index.ts @@ -8,7 +8,7 @@ import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -816,7 +816,7 @@ export const litellmProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error } diff --git a/apps/sim/providers/meta/index.ts b/apps/sim/providers/meta/index.ts index 25e33098d4f..df9772d5b8c 100644 --- a/apps/sim/providers/meta/index.ts +++ b/apps/sim/providers/meta/index.ts @@ -7,7 +7,7 @@ import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -696,7 +696,7 @@ export const metaProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error } diff --git a/apps/sim/providers/mistral/index.ts b/apps/sim/providers/mistral/index.ts index 03d225be391..cea27f102ad 100644 --- a/apps/sim/providers/mistral/index.ts +++ b/apps/sim/providers/mistral/index.ts @@ -7,7 +7,7 @@ import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -659,7 +659,7 @@ export const mistralProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error } diff --git a/apps/sim/providers/nvidia/index.ts b/apps/sim/providers/nvidia/index.ts index 956433308b9..387a067665f 100644 --- a/apps/sim/providers/nvidia/index.ts +++ b/apps/sim/providers/nvidia/index.ts @@ -7,7 +7,7 @@ import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -746,7 +746,7 @@ export const nvidiaProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error } diff --git a/apps/sim/providers/ollama/core.ts b/apps/sim/providers/ollama/core.ts index 1b2d0a621da..856d4790c4d 100644 --- a/apps/sim/providers/ollama/core.ts +++ b/apps/sim/providers/ollama/core.ts @@ -7,7 +7,7 @@ import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -721,7 +721,7 @@ export async function executeOllamaProviderRequest( } ) - if (isAbortError(error) || request.abortSignal?.aborted || isConversationContextError(error)) { + if (isAbortError(error) || request.abortSignal?.aborted || isPermanentGenerationError(error)) { throw error } diff --git a/apps/sim/providers/openai-compat/chat-completions.ts b/apps/sim/providers/openai-compat/chat-completions.ts index 5e20ba45b75..78e0f225d1f 100644 --- a/apps/sim/providers/openai-compat/chat-completions.ts +++ b/apps/sim/providers/openai-compat/chat-completions.ts @@ -13,7 +13,7 @@ import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -711,7 +711,7 @@ export async function executeChatCompletionRequest( `Error in ${providerName} request:`, errorDetails ) - if (isAbortError(error) || request.abortSignal?.aborted || isConversationContextError(error)) { + if (isAbortError(error) || request.abortSignal?.aborted || isPermanentGenerationError(error)) { throw error } diff --git a/apps/sim/providers/openai/core.ts b/apps/sim/providers/openai/core.ts index 85d53ec8efe..e8dc35f37c8 100644 --- a/apps/sim/providers/openai/core.ts +++ b/apps/sim/providers/openai/core.ts @@ -7,7 +7,7 @@ import type OpenAI from 'openai' import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -993,7 +993,7 @@ export async function executeResponsesProviderRequest( } ) - if (isAbortError(error) || request.abortSignal?.aborted || isConversationContextError(error)) { + if (isAbortError(error) || request.abortSignal?.aborted || isPermanentGenerationError(error)) { throw error } diff --git a/apps/sim/providers/run-limits.test.ts b/apps/sim/providers/run-limits.test.ts new file mode 100644 index 00000000000..210efd46ccc --- /dev/null +++ b/apps/sim/providers/run-limits.test.ts @@ -0,0 +1,289 @@ +import { billingUsageGateCacheMock } from '@sim/testing/mocks/billing-usage-gate-cache.mock' +import { + providersConversationHistoryMock, + providersConversationHistoryMockFns, +} from '@sim/testing/mocks/providers-conversation-history.mock' +import { toolsMock, toolsMockFns } from '@sim/testing/mocks/tools.mock' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { ConversationProtocol } from '@/lib/memory/conversation-types' +import { AgentTurnStateMachine } from '@/lib/memory/turn-state' +import { readTrustedExecutionCost } from '@/executor/utils/errors' +import { prepareConversationGeneration } from '@/providers/conversation-generation' +import type { TokenUsageCost } from '@/providers/pricing' +import { AgentRunLimits, RunLimitError, type RunLimitsConfig } from '@/providers/run-limits' +import { + executeProviderTool, + type ProviderRuntimeContext, + runWithProviderRuntimeContext, +} from '@/providers/runtime-context' +import type { ProviderRequest } from '@/providers/types' + +vi.mock('@/tools', () => toolsMock) +vi.mock('@/providers/conversation-history', () => providersConversationHistoryMock) +vi.mock('@/lib/billing/core/usage-gate-cache', () => billingUsageGateCacheMock) + +toolsMockFns.mockExecuteTool.mockResolvedValue({ success: true, output: {} }) + +const request = { model: 'test-model', tools: [{ id: 'lookup' }] } as ProviderRequest +const toolPayload = { + messages: [{ role: 'user', content: 'Do the task' }], + tools: [{ type: 'function', function: { name: 'lookup' } }], +} +const answerPayload = { messages: toolPayload.messages } + +function cost(total = 0): TokenUsageCost { + return { input: 0, output: 0, total } +} + +/** Binds a runtime to the request the way `executeProviderRequest` does for each model attempt. */ +function bindRuntime(runLimits: AgentRunLimits, settledCost = cost()): ProviderRuntimeContext { + const runtime: ProviderRuntimeContext = { runLimits, settledCost } + providersConversationHistoryMockFns.mockGetConversationRequestContext.mockReturnValue(runtime) + return runtime +} + +function limits(config: Partial): AgentRunLimits { + return new AgentRunLimits({ onLimit: 'finalize', ...config }) +} + +async function modelCall( + protocol: ConversationProtocol = 'chat-completions', + payload: object = toolPayload +) { + return prepareConversationGeneration(request, protocol, structuredClone(payload)) +} + +/** Runs the tool calls of one model response; returns whether each one ran. */ +async function toolCalls(runLimits: AgentRunLimits, count = 1): Promise { + toolsMockFns.mockExecuteTool.mockClear() + const results = await runWithProviderRuntimeContext({ runLimits }, () => + Promise.all(Array.from({ length: count }, () => executeProviderTool('lookup', {}))) + ) + return results.map(({ rawResponse }) => rawResponse.success) +} + +afterEach(() => { + vi.useRealTimers() + providersConversationHistoryMockFns.mockGetConversationRequestContext.mockReset() +}) + +describe('Agent run limits', () => { + it('leaves every call and tool untouched when no limit is set', async () => { + const runLimits = limits({}) + bindRuntime(runLimits) + + for (let step = 0; step < 3; step++) { + expect(await modelCall()).toEqual(toolPayload) + expect(await toolCalls(runLimits)).toEqual([true]) + } + expect(runLimits.stopReason).toBe('completed') + }) + + it('answers without tools once the allowed steps have run', async () => { + const runLimits = limits({ maxSteps: 2 }) + bindRuntime(runLimits) + + for (let step = 0; step < 2; step++) { + expect(await modelCall()).toEqual(toolPayload) + expect(await toolCalls(runLimits)).toEqual([true]) + } + expect(await modelCall()).toEqual(answerPayload) + expect(runLimits.stopReason).toBe('step_limit') + }) + + it('runs every parallel tool call of the last allowed step', async () => { + const runLimits = limits({ maxSteps: 1 }) + bindRuntime(runLimits) + + await modelCall() + expect(await toolCalls(runLimits, 3)).toEqual([true, true, true]) + }) + + it('fails a run that asks for tools past its limit, keeping the settled cost for billing', async () => { + const runLimits = limits({ maxSteps: 1, onLimit: 'fail' }) + bindRuntime(runLimits, cost(0.25)) + + await modelCall() + await toolCalls(runLimits) + expect(await modelCall()).toEqual(toolPayload) + expect(await toolCalls(runLimits)).toEqual([false]) + const error = await modelCall().catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(RunLimitError) + expect(readTrustedExecutionCost(error)?.total).toBe(0.25) + }) + + it('completes a failing-mode run whose model answers on the call after its last step', async () => { + const runLimits = limits({ maxSteps: 1, onLimit: 'fail' }) + bindRuntime(runLimits) + + await modelCall() + await toolCalls(runLimits) + expect(await modelCall()).toEqual(toolPayload) + expect(runLimits.stopReason).toBe('completed') + }) + + it('counts each attempt’s own spend once, not the cost it carried in from earlier attempts', async () => { + const runLimits = limits({ maxCost: 1 }) + const primary = bindRuntime(runLimits) + await modelCall() + primary.settledCost!.total = 0.6 + + const fallback = bindRuntime(runLimits, cost(0.6)) + expect(await modelCall()).toEqual(toolPayload) + fallback.settledCost!.total = 0.9 + expect(await modelCall()).toEqual(toolPayload) + fallback.settledCost!.total = 1 + + expect(await modelCall()).toEqual(answerPayload) + expect(runLimits.stopReason).toBe('spend_limit') + }) + + it('answers without tools once the time limit has passed', async () => { + vi.useFakeTimers() + const runLimits = limits({ maxDurationMs: 60_000 }) + bindRuntime(runLimits) + + expect(await modelCall()).toEqual(toolPayload) + vi.advanceTimersByTime(60_000) + expect(await toolCalls(runLimits)).toEqual([false]) + expect(await modelCall()).toEqual(answerPayload) + expect(runLimits.stopReason).toBe('time_limit') + }) + + it('does not stop a run that answered within its limits, whatever follow-up calls cost', async () => { + const runLimits = limits({ maxCost: 1 }) + const runtime = bindRuntime(runLimits) + await modelCall() + await toolCalls(runLimits) + await modelCall() + runtime.settledCost!.total = 1.2 + const structuredPass = { messages: toolPayload.messages, response_format: { type: 'json' } } + + for (let call = 0; call < 3; call++) { + expect(await modelCall('chat-completions', structuredPass)).toEqual(structuredPass) + } + expect(runLimits.stopReason).toBe('completed') + }) + + it('stops a provider that keeps calling tools after it was told to answer', async () => { + const runLimits = limits({ maxSteps: 1 }) + bindRuntime(runLimits) + await modelCall() + await toolCalls(runLimits) + + await modelCall() + expect(await toolCalls(runLimits)).toEqual([false]) + expect(toolsMockFns.mockExecuteTool).not.toHaveBeenCalled() + await modelCall() + await toolCalls(runLimits) + await expect(modelCall()).rejects.toBeInstanceOf(RunLimitError) + }) + + it('leaves a refused call pending, so a retry with a fresh budget runs it', async () => { + const runLimits = limits({ maxSteps: 1 }) + bindRuntime(runLimits) + await modelCall() + await toolCalls(runLimits) + await modelCall() + const journal = new AgentTurnStateMachine({ save: vi.fn() }) + await journal.captureStep({ + assistant: { role: 'assistant', content: '' }, + calls: [{ toolId: 'lookup', providerCallId: 'call-1', arguments: '{}' }], + native: { + protocol: 'chat-completions', + providerId: 'openai', + model: 'm', + binding: 'b', + value: {}, + }, + }) + const invocationId = journal.resolveInvocationId('call-1', 'lookup') + + await runWithProviderRuntimeContext({ runLimits, agentConversation: journal }, () => + executeProviderTool('lookup', { _context: { invocationId } }) + ) + + expect(journal.getPendingCalls().map((call) => call.invocationId)).toEqual([invocationId]) + }) + + it('gives a provider that cannot refuse tools its retry even when the limit stopped a step', async () => { + vi.useFakeTimers() + const runLimits = limits({ maxDurationMs: 1000 }) + bindRuntime(runLimits) + await modelCall('bedrock', { messages: [], toolConfig: { tools: [{}] } }) + vi.advanceTimersByTime(1000) + await toolCalls(runLimits) + + await modelCall('bedrock', { messages: [], toolConfig: { tools: [{}] } }) + await toolCalls(runLimits) + await expect( + modelCall('bedrock', { messages: [], toolConfig: { tools: [{}] } }) + ).resolves.toBeDefined() + }) + + it.each([ + { + protocol: 'chat-completions', + payload: { ...toolPayload, tool_choice: 'auto', parallel_tool_calls: false }, + expected: answerPayload, + }, + { + protocol: 'responses', + payload: { input: [], tools: [{ type: 'function', name: 'lookup' }], tool_choice: 'auto' }, + expected: { input: [], tool_choice: 'none' }, + }, + { + protocol: 'anthropic', + payload: { ...toolPayload, tool_choice: { type: 'any' } }, + expected: { ...toolPayload, tool_choice: { type: 'none' } }, + }, + { + protocol: 'gemini', + payload: { + contents: [], + config: { temperature: 0, tools: [{}], toolConfig: { functionCallingConfig: {} } }, + }, + expected: { contents: [], config: { temperature: 0 } }, + }, + { + protocol: 'bedrock', + payload: { messages: [], toolConfig: { tools: [{}], toolChoice: { any: {} } } }, + expected: { messages: [], toolConfig: { tools: [{}], toolChoice: { auto: {} } } }, + }, + { + protocol: 'bedrock', + payload: { + messages: [], + toolConfig: { tools: [{}], toolChoice: { tool: { name: 'structured_output' } } }, + }, + expected: { + messages: [], + toolConfig: { tools: [{}], toolChoice: { tool: { name: 'structured_output' } } }, + }, + }, + ] as const)( + 'answers in the $protocol wire format: $expected', + async ({ protocol, payload, expected }) => { + const runLimits = limits({ maxSteps: 1 }) + bindRuntime(runLimits) + await modelCall() + await toolCalls(runLimits) + + expect(await modelCall(protocol, payload)).toEqual(expected) + } + ) + + it('reports a stop that happens after the output was handed back, as a streamed run does', async () => { + const runLimits = limits({ maxSteps: 1 }) + bindRuntime(runLimits) + const output: Record = {} + + runLimits.reportOn(output) + expect(output.stopReason).toBe('completed') + await modelCall() + await toolCalls(runLimits) + await modelCall() + expect({ ...output }).toEqual({ stopReason: 'step_limit' }) + }) +}) diff --git a/apps/sim/providers/run-limits.ts b/apps/sim/providers/run-limits.ts new file mode 100644 index 00000000000..baa5d87e445 --- /dev/null +++ b/apps/sim/providers/run-limits.ts @@ -0,0 +1,202 @@ +import { isRecordLike, omit, toRecordOrNull } from '@sim/utils/object' +import { NonRetryableExecutionError } from '@/lib/execution/non-retryable-error' +import type { ConversationProtocol } from '@/lib/memory/conversation-types' +import type { TokenUsageCost } from '@/providers/pricing' +import { billedUserStop } from '@/providers/run-usage-limit' +import type { ProviderRuntimeContext } from '@/providers/runtime-context' +import type { ProviderRequest } from '@/providers/types' + +type RunLimitReason = 'spend_limit' | 'step_limit' | 'time_limit' + +export interface RunLimitsConfig { + /** Billed cost of the run's model calls and tools, in USD. */ + maxCost?: number + /** Model responses whose tool calls may run. */ + maxSteps?: number + maxDurationMs?: number + /** Once the agent still wants tools past a limit: answer without them, or fail the block. */ + onLimit: 'finalize' | 'fail' +} + +/** A run stopped at a limit its Agent block set. Never retried and never handed to a fallback. */ +export class RunLimitError extends NonRetryableExecutionError { + constructor(message: string) { + super(message) + this.name = 'RunLimitError' + } +} + +/** + * Tool rounds a provider may still request after being told to answer: Bedrock and Ollama cannot + * refuse tools, so their calls are refused instead, and a model that asks again fails the run. + */ +const MAX_IGNORED_ANSWER_REQUESTS = 1 + +/** What the model reads in place of a tool result once its run has stopped. */ +export const RUN_LIMIT_TOOL_REFUSAL = + 'Not run: this agent reached its run limit. Answer with the information you already have.' + +/** + * The spend, step, and time limits of one Agent block run, shared by every model it tries. + * + * A limit acts only when the agent wants to keep using tools: the call after the last allowed step + * answers without them, and a tool the model calls past a limit is refused. Calls that offer no + * agent tools, such as a provider's structured-output pass, are never limited. + */ +export class AgentRunLimits { + private readonly startedAt = Date.now() + /** Each model attempt's settled cost, with what it carried in from earlier attempts. */ + private readonly settledCosts = new Map() + private steps = 0 + private stepOpen = false + private reached: RunLimitReason | undefined + private output: Record | undefined + private toldToAnswer = false + private toolRefused = false + private ignoredAnswerRequests = 0 + + constructor(private readonly config: RunLimitsConfig) {} + + get stopReason(): RunLimitReason | 'completed' { + return this.reached ?? 'completed' + } + + /** Keeps `output.stopReason` current: a streamed run stops after its output is handed back. */ + reportOn(output: Record): void { + this.output = output + output.stopReason = this.stopReason + } + + /** + * Starts a model call. Returns true when a call that offers tools must answer without them, and + * throws when the run must fail because the model asked for tools past its limit. + */ + beginModelCall(settledCost: TokenUsageCost | undefined, offersTools: boolean): boolean { + if (settledCost && !this.settledCosts.has(settledCost)) { + this.settledCosts.set(settledCost, settledCost.total) + } + this.stepOpen = false + if (this.toolRefused) { + this.toolRefused = false + if ( + this.config.onLimit === 'fail' || + (this.toldToAnswer && ++this.ignoredAnswerRequests > MAX_IGNORED_ANSWER_REQUESTS) + ) { + throw this.failure(settledCost) + } + } + if (!offersTools || this.config.onLimit === 'fail') return false + const reached = this.reached ?? this.reachedLimit() + if (!reached) return false + this.stop(reached) + this.toldToAnswer = true + return true + } + + /** Admits a tool call; the first one of a model response starts a step. False refuses it. */ + admitTool(): boolean { + if (this.stepOpen) return true + const reached = this.reached ?? this.reachedLimit() + if (reached) { + this.stop(reached) + this.toolRefused = true + return false + } + this.stepOpen = true + this.steps++ + return true + } + + private stop(reason: RunLimitReason): void { + this.reached = reason + if (this.output) this.output.stopReason = reason + } + + private reachedLimit(): RunLimitReason | undefined { + const { maxCost, maxSteps, maxDurationMs } = this.config + if (maxCost !== undefined && this.spend() >= maxCost) return 'spend_limit' + if (maxSteps !== undefined && this.steps >= maxSteps) return 'step_limit' + if (maxDurationMs !== undefined && Date.now() - this.startedAt >= maxDurationMs) { + return 'time_limit' + } + return undefined + } + + /** Spend since each attempt began; a fallback or retry carries in journaled cost already counted. */ + private spend(): number { + let total = 0 + for (const [cost, carriedIn] of this.settledCosts) total += cost.total - carriedIn + return total + } + + private failure(settledCost: TokenUsageCost | undefined): RunLimitError { + return billedUserStop(new RunLimitError(this.describe()), settledCost) + } + + private describe(): string { + switch (this.reached) { + case 'spend_limit': + return `Agent stopped at its spend limit of $${this.config.maxCost}.` + case 'step_limit': + return `Agent stopped at its limit of ${this.config.maxSteps} steps.` + default: + return `Agent stopped at its time limit of ${(this.config.maxDurationMs ?? 0) / 1000} seconds.` + } + } +} + +/** + * The same request with the agent's tool calls turned off in the wire format of `protocol`, or + * undefined when it offers none to turn off. Each shape is the one that protocol's tool loops + * already send for their final answer: several chat-completions providers (Z.ai, Meta, Kimi) + * reject `tool_choice: 'none'`, so their tools are dropped; Anthropic and Bedrock reject tool + * history without tool definitions, so theirs stay, and Bedrock, which cannot refuse, relaxes a + * forced choice to `auto`. + */ +function withoutAgentToolCalls( + protocol: ConversationProtocol, + payload: T, + request: ProviderRequest +): T | undefined { + if (!isRecordLike(payload)) return undefined + switch (protocol) { + case 'chat-completions': + if (!payload.tools) return undefined + return omit(payload, ['tools', 'tool_choice', 'parallel_tool_calls']) as T + case 'responses': + if (!payload.tools) return undefined + return { ...omit(payload, ['tools']), tool_choice: 'none' } as T + case 'anthropic': + if (!payload.tools || toRecordOrNull(payload.tool_choice)?.type === 'none') return undefined + return { ...payload, tool_choice: { type: 'none' } } as T + case 'gemini': { + const config = toRecordOrNull(payload.config) + if (!config?.tools) return undefined + return { ...payload, config: omit(config, ['tools', 'toolConfig']) } as T + } + case 'bedrock': { + const toolConfig = toRecordOrNull(payload.toolConfig) + if (!toolConfig) return undefined + // Bedrock's structured output is a forced tool of its own, which must still run. + const forced = toRecordOrNull(toRecordOrNull(toolConfig.toolChoice)?.tool)?.name + if (typeof forced === 'string' && !request.tools?.some((tool) => tool.id === forced)) { + return undefined + } + return { ...payload, toolConfig: { ...toolConfig, toolChoice: { auto: {} } } } as T + } + } +} + +/** Applies the Agent's run limits to the model call about to send `payload`. */ +export function applyRunLimits( + runtime: ProviderRuntimeContext | undefined, + request: ProviderRequest, + protocol: ConversationProtocol, + payload: T +): T { + const runLimits = runtime?.runLimits + if (!runLimits) return payload + const answerPayload = withoutAgentToolCalls(protocol, payload, request) + const mustAnswer = runLimits.beginModelCall(runtime.settledCost, answerPayload !== undefined) + return mustAnswer && answerPayload ? answerPayload : payload +} diff --git a/apps/sim/providers/run-usage-limit.test.ts b/apps/sim/providers/run-usage-limit.test.ts index 3fd7e5f2a29..936a3a7da74 100644 --- a/apps/sim/providers/run-usage-limit.test.ts +++ b/apps/sim/providers/run-usage-limit.test.ts @@ -46,7 +46,7 @@ import type { ExecutionContext } from '@/executor/types' import { readTrustedExecutionCost } from '@/executor/utils/errors' import { executeProviderRequest } from '@/providers' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -136,7 +136,7 @@ describe('run usage limit', () => { await modelStep(request, 1, 1) return { content: '', model: request.model } } catch (error) { - if (isConversationContextError(error)) throw error + if (isPermanentGenerationError(error)) throw error throw new ProviderError(getErrorMessage(error), { startTime: '', endTime: '', diff --git a/apps/sim/providers/run-usage-limit.ts b/apps/sim/providers/run-usage-limit.ts index b971248d29c..773c853c682 100644 --- a/apps/sim/providers/run-usage-limit.ts +++ b/apps/sim/providers/run-usage-limit.ts @@ -12,7 +12,7 @@ import type { ProviderRuntimeContext } from '@/providers/runtime-context' /** * A run stopped at its payer's usage limit. Providers pass it through unwrapped (see - * `isConversationContextError`), so it keeps its billed cost and is never retried. + * `isPermanentGenerationError`), so it keeps its billed cost and is never retried. */ export class RunUsageLimitError extends NonRetryableExecutionError { constructor(message: string) { @@ -21,6 +21,19 @@ export class RunUsageLimitError extends NonRetryableExecutionError { } } +/** + * Attributes a run stop to the user, so it logs at info, and attaches the request's settled cost, + * so the stopped block is billed for the steps and tool calls it ran. + */ +export function billedUserStop( + error: T, + settledCost: TokenUsageCost | undefined +): T { + markFailureKind(error, 'user') + if (settledCost?.total) attachTrustedExecutionCost(error, settledCost) + return error +} + export function addSettledCost(settledCost: TokenUsageCost, cost: TokenUsageCost): void { settledCost.input += cost.input settledCost.output += cost.output @@ -104,7 +117,5 @@ export async function assertRunWithinUsageLimit( verdict.payer ?? attribution, verdict.scope ) - const error = markFailureKind(new RunUsageLimitError(message), 'user') - if (runtime.settledCost?.total) attachTrustedExecutionCost(error, runtime.settledCost) - throw error + throw billedUserStop(new RunUsageLimitError(message), runtime.settledCost) } diff --git a/apps/sim/providers/runtime-context.ts b/apps/sim/providers/runtime-context.ts index 4aa6bf9cc50..f924cac9e9f 100644 --- a/apps/sim/providers/runtime-context.ts +++ b/apps/sim/providers/runtime-context.ts @@ -23,6 +23,7 @@ import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-sec import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { stripCloneSuffixes } from '@/executor/utils/subflow-node-id-codec' import type { TokenUsageCost } from '@/providers/pricing' +import { type AgentRunLimits, RUN_LIMIT_TOOL_REFUSAL } from '@/providers/run-limits' import { isAbortError } from '@/providers/streaming-tool-loop-shared' import { getPreparedProviderToolInputProvenance } from '@/providers/tool-input-provenance' import type { ProviderId } from '@/providers/types' @@ -44,6 +45,8 @@ export interface ProviderRuntimeContext { failedFunctionToolCost?: { total: number } /** Billed cost of this request's settled model steps and tool calls, charged if the run stops it. */ settledCost?: TokenUsageCost + /** Spend, model-call, and time limits shared by every model the Agent block tries. */ + runLimits?: AgentRunLimits } export type ExecuteProviderToolOptions = ExecuteToolOptions @@ -256,6 +259,11 @@ export async function executeProviderTool( } return result } + // Left unrecorded: a refusal belongs to this try's budget, so a retry with a fresh one runs the call. + if (runtimeContext?.runLimits && !runtimeContext.runLimits.admitTool()) { + const refusal: ToolResponse = { success: false, output: {}, error: RUN_LIMIT_TOOL_REFUSAL } + return { rawResponse: refusal, modelResponse: refusal } + } const registry = options.resolvedSecretTraceRegistry ?? runtimeContext?.resolvedSecretTraceRegistry diff --git a/apps/sim/providers/sakana/index.ts b/apps/sim/providers/sakana/index.ts index 5ab637f4fde..d987a0bca45 100644 --- a/apps/sim/providers/sakana/index.ts +++ b/apps/sim/providers/sakana/index.ts @@ -7,7 +7,7 @@ import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -731,7 +731,7 @@ export const sakanaProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error } diff --git a/apps/sim/providers/vllm/index.ts b/apps/sim/providers/vllm/index.ts index 0fac82e9eaa..9df39011672 100644 --- a/apps/sim/providers/vllm/index.ts +++ b/apps/sim/providers/vllm/index.ts @@ -10,7 +10,7 @@ import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { getCachedProviderClient } from '@/providers/client-cache' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -782,7 +782,7 @@ export const vllmProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error } diff --git a/apps/sim/providers/xai/index.ts b/apps/sim/providers/xai/index.ts index bf708f25058..270591ea608 100644 --- a/apps/sim/providers/xai/index.ts +++ b/apps/sim/providers/xai/index.ts @@ -8,7 +8,7 @@ import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -738,7 +738,7 @@ export const xAIProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error } diff --git a/apps/sim/providers/zai/index.ts b/apps/sim/providers/zai/index.ts index 10ff0aec053..28fb693fb16 100644 --- a/apps/sim/providers/zai/index.ts +++ b/apps/sim/providers/zai/index.ts @@ -7,7 +7,7 @@ import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' import { - isConversationContextError, + isPermanentGenerationError, prepareConversationGeneration, } from '@/providers/conversation-generation' import { @@ -717,7 +717,7 @@ export const zaiProvider: ProviderConfig = { if ( isAbortError(error) || request.abortSignal?.aborted || - isConversationContextError(error) + isPermanentGenerationError(error) ) { throw error }