From c73a3706475b681248fdac72c465f9b15261afdb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 3 Aug 2026 17:14:04 -0700 Subject: [PATCH 1/6] improvement(agent): allow variable references in reasoning effort and verbosity Reasoning Effort and Verbosity were select-only dropdowns, so a workflow could not sweep them from a variable or an upstream block the way it already can with the model. Both become editable comboboxes, matching the model field directly above them and the managed-agent selectors. - switch both subblocks to `combobox`, keeping their fetched per-model option lists intact - keep them visible when `model` itself holds a reference, since the concrete model id is only known at execution time and cannot be matched against the static capability list - normalize the resolved level in the provider chokepoint so a reference that resolves to `"High"` or to nothing behaves sanely instead of hitting a provider 400 --- apps/sim/blocks/blocks.test.ts | 28 ++++++++ apps/sim/blocks/blocks/agent.ts | 19 ++---- apps/sim/blocks/utils.ts | 18 +++++ apps/sim/executor/variables/resolver.test.ts | 47 +++++++++++++ .../workflows/sanitization/references.test.ts | 35 ++++++++++ .../lib/workflows/sanitization/references.ts | 15 +++++ apps/sim/providers/index.test.ts | 67 +++++++++++++++++++ apps/sim/providers/index.ts | 16 +++++ 8 files changed, 233 insertions(+), 12 deletions(-) diff --git a/apps/sim/blocks/blocks.test.ts b/apps/sim/blocks/blocks.test.ts index c6230c41d23..7de8356d149 100644 --- a/apps/sim/blocks/blocks.test.ts +++ b/apps/sim/blocks/blocks.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest' vi.unmock('@/blocks/registry') +import { evaluateSubBlockCondition } from '@/lib/workflows/subblocks/visibility' import { generateRouterPrompt } from '@/blocks/blocks/router' import { getAllBlocks, @@ -842,6 +843,33 @@ describe.concurrent('Blocks Module', () => { expect(modelSubBlock?.commandSearchable).toBe(true) }) + it('should let the agent reasoning and verbosity fields take a typed reference', () => { + const agentBlock = getBlock('agent') + + for (const id of ['reasoningEffort', 'verbosity']) { + const subBlock = agentBlock?.subBlocks.find((sb) => sb.id === id) + // A combobox is editable, so a `` / `{{ENV_VAR}}` reference can be + // typed into it; the option list still offers every level the model accepts. + expect(subBlock?.type).toBe('combobox') + expect(typeof subBlock?.condition).toBe('function') + } + }) + + it('should keep the agent reasoning and verbosity fields visible when the model is a reference', () => { + const agentBlock = getBlock('agent') + + for (const id of ['reasoningEffort', 'verbosity']) { + const subBlock = agentBlock?.subBlocks.find((sb) => sb.id === id) + const condition = subBlock?.condition + if (typeof condition !== 'function') throw new Error(`${id} condition is not a function`) + + expect(evaluateSubBlockCondition(condition, { model: '' })).toBe(true) + expect(evaluateSubBlockCondition(condition, { model: '{{MODEL_ID}}' })).toBe(true) + expect(evaluateSubBlockCondition(condition, { model: 'gpt-5.1' })).toBe(true) + expect(evaluateSubBlockCondition(condition, { model: 'claude-sonnet-5' })).toBe(false) + } + }) + it('should hide generator API keys on hosted only for Fal.ai providers', () => { for (const blockType of ['image_generator_v2', 'video_generator_v3']) { const block = getBlock(blockType) diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index 29619bff5b7..73c8cdc3c95 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -3,6 +3,7 @@ import { AgentIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import { AuthMode, IntegrationType } from '@/blocks/types' import { + getModelCapabilityCondition, getModelOptions, getProviderCredentialSubBlocks, normalizeFileInput, @@ -159,8 +160,8 @@ Return ONLY the JSON array.`, { id: 'reasoningEffort', title: 'Reasoning Effort', - type: 'dropdown', - placeholder: 'Select reasoning effort...', + type: 'combobox', + placeholder: 'Type or select reasoning effort...', options: [ { label: 'auto', id: 'auto' }, { label: 'low', id: 'low' }, @@ -207,16 +208,13 @@ Return ONLY the JSON array.`, return [autoOption, ...validOptions.map((opt) => ({ label: opt, id: opt }))] }, mode: 'advanced', - condition: { - field: 'model', - value: MODELS_WITH_REASONING_EFFORT, - }, + condition: getModelCapabilityCondition(MODELS_WITH_REASONING_EFFORT), }, { id: 'verbosity', title: 'Verbosity', - type: 'dropdown', - placeholder: 'Select verbosity...', + type: 'combobox', + placeholder: 'Type or select verbosity...', options: [ { label: 'auto', id: 'auto' }, { label: 'low', id: 'low' }, @@ -263,10 +261,7 @@ Return ONLY the JSON array.`, return [autoOption, ...validOptions.map((opt) => ({ label: opt, id: opt }))] }, mode: 'advanced', - condition: { - field: 'model', - value: MODELS_WITH_VERBOSITY, - }, + condition: getModelCapabilityCondition(MODELS_WITH_VERBOSITY), }, { id: 'thinkingLevel', diff --git a/apps/sim/blocks/utils.ts b/apps/sim/blocks/utils.ts index 848b8d36363..d5e954744df 100644 --- a/apps/sim/blocks/utils.ts +++ b/apps/sim/blocks/utils.ts @@ -7,6 +7,7 @@ import { isOllamaConfigured, } from '@/lib/core/config/env-flags' import { getScopesForService } from '@/lib/oauth/utils' +import { containsReference } from '@/lib/workflows/sanitization/references' import { buildCanonicalIndex } from '@/lib/workflows/subblocks/visibility' import type { BlockOutput, OutputFieldDefinition, SubBlockConfig } from '@/blocks/types' import { @@ -235,6 +236,23 @@ function shouldRequireApiKeyForModel(model: string): boolean { return true } +/** + * Visibility condition for a model-tuning field that only some models accept, such as + * reasoning effort or verbosity. Gates on the capability list, but keeps the field visible + * when `model` itself holds a variable or block reference — the concrete model id is only + * known at execution time then, so matching a reference against a static list would hide + * the field for every workflow that binds its model dynamically. + */ +export function getModelCapabilityCondition(capableModels: string[]) { + return (values?: Record) => { + const model = typeof values?.model === 'string' ? values.model : '' + if (containsReference(model)) { + return buildModelVisibilityCondition(model, true) + } + return { field: 'model', value: capableModels } + } +} + /** * Get the API key condition for provider credential subblocks. * Handles hosted vs self-hosted environments and excludes providers that don't need API key. diff --git a/apps/sim/executor/variables/resolver.test.ts b/apps/sim/executor/variables/resolver.test.ts index bd512bfa464..85d5c8b9ed4 100644 --- a/apps/sim/executor/variables/resolver.test.ts +++ b/apps/sim/executor/variables/resolver.test.ts @@ -1341,3 +1341,50 @@ describe('VariableResolver function context overflow offload', () => { expect(result.resolvedInputs.code).toBe('return globals()["__blockRef_0"]') }) }) + +/** + * The agent block's Reasoning Effort and Verbosity fields are editable comboboxes, so a + * workflow can bind them to a reference instead of picking a level. These lock in that the + * generic input resolution actually reaches those two fields. + */ +describe('VariableResolver agent model levels', () => { + it('resolves block, workflow-variable, and env references in reasoning effort and verbosity', async () => { + const producer = createBlock('producer', 'Producer', BlockType.API) + const agent = createBlock('agent', 'Agent', BlockType.AGENT, { + model: 'gpt-5', + reasoningEffort: '', + verbosity: '', + thinkingLevel: '{{THINKING}}', + }) + const workflow: SerializedWorkflow = { + version: '1', + blocks: [producer, agent], + connections: [], + loops: {}, + parallels: {}, + } + + const state = new ExecutionState() + state.setBlockOutput('producer', { result: 'high' }) + const ctx = { + blockStates: state.getBlockStates(), + blockLogs: [], + environmentVariables: { THINKING: 'medium' }, + workflowVariables: { 'var-1': { id: 'var-1', name: 'Detail', type: 'string', value: 'low' } }, + decisions: { router: new Map(), condition: new Map() }, + loopExecutions: new Map(), + executedBlocks: new Set(), + activeExecutionPath: new Set(), + completedLoops: new Set(), + metadata: {}, + } as unknown as ExecutionContext + + const resolver = new VariableResolver(workflow, { THINKING: 'medium' }, state) + const result = await resolver.resolveInputs(ctx, 'agent', agent.config.params, agent) + + expect(result.reasoningEffort).toBe('high') + expect(result.verbosity).toBe('low') + expect(result.thinkingLevel).toBe('medium') + expect(result.model).toBe('gpt-5') + }) +}) diff --git a/apps/sim/lib/workflows/sanitization/references.test.ts b/apps/sim/lib/workflows/sanitization/references.test.ts index 83b86138472..80ebb43ada9 100644 --- a/apps/sim/lib/workflows/sanitization/references.test.ts +++ b/apps/sim/lib/workflows/sanitization/references.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' import { + containsReference, isLikelyReferenceSegment, splitReferenceSegment, } from '@/lib/workflows/sanitization/references' @@ -53,3 +54,37 @@ describe('isLikelyReferenceSegment', () => { expect(isLikelyReferenceSegment('<123>')).toBe(false) }) }) + +describe('containsReference', () => { + it('detects block and variable references', () => { + expect(containsReference('')).toBe(true) + expect(containsReference('')).toBe(true) + expect(containsReference('')).toBe(true) + }) + + it('detects environment variable placeholders', () => { + expect(containsReference('{{MODEL_ID}}')).toBe(true) + }) + + it('detects a reference embedded in surrounding text', () => { + expect(containsReference('gpt-')).toBe(true) + }) + + it('returns false for literal model ids', () => { + expect(containsReference('gpt-5.1')).toBe(false) + expect(containsReference('claude-sonnet-5')).toBe(false) + expect(containsReference('azure/gpt-5.1-codex')).toBe(false) + }) + + it('returns false for empty and non-string values', () => { + expect(containsReference('')).toBe(false) + expect(containsReference(undefined)).toBe(false) + expect(containsReference(null)).toBe(false) + expect(containsReference(42)).toBe(false) + }) + + it('returns false for stray brackets that are not references', () => { + expect(containsReference('a < b')).toBe(false) + expect(containsReference('<123>')).toBe(false) + }) +}) diff --git a/apps/sim/lib/workflows/sanitization/references.ts b/apps/sim/lib/workflows/sanitization/references.ts index ecc65f60f71..0c0865765b3 100644 --- a/apps/sim/lib/workflows/sanitization/references.ts +++ b/apps/sim/lib/workflows/sanitization/references.ts @@ -77,6 +77,21 @@ export function isLikelyReferenceSegment(segment: string): boolean { return true } +const ENV_VAR_PATTERN = new RegExp(`\\${REFERENCE.ENV_VAR_START}[^}]+\\${REFERENCE.ENV_VAR_END}`) + +/** + * Whether a subblock value carries a `` / `` reference or a + * `{{ENV_VAR}}` placeholder instead of a literal value — i.e. its real value is only known + * once the workflow runs. Conditions that gate one field on a sibling's literal value use + * this to stay visible while the sibling is bound dynamically. + */ +export function containsReference(value: unknown): boolean { + if (typeof value !== 'string' || !value) { + return false + } + return extractReferencePrefixes(value).length > 0 || ENV_VAR_PATTERN.test(value) +} + export function extractReferencePrefixes(value: string): Array<{ raw: string; prefix: string }> { if (!value || typeof value !== 'string') { return [] diff --git a/apps/sim/providers/index.test.ts b/apps/sim/providers/index.test.ts index 7b0ee9697e8..2ce7039156c 100644 --- a/apps/sim/providers/index.test.ts +++ b/apps/sim/providers/index.test.ts @@ -412,3 +412,70 @@ describe('executeProviderRequest — streaming cost policy', () => { }) }) }) + +/** + * `reasoningEffort` and `verbosity` can be bound to a variable or block reference in the + * agent block, so by the time they reach the provider they hold whatever that reference + * resolved to rather than a value picked from a list. + */ +describe('executeProviderRequest — model level normalization', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetApiKeyWithBYOK.mockResolvedValue({ apiKey: 'sk-rotating', isBYOK: false }) + mockExecuteRequest.mockResolvedValue({ + content: 'hi', + model: 'gpt-5', + tokens: { input: 1, output: 1, total: 2 }, + } as ProviderResponse) + }) + + const sentRequest = () => mockExecuteRequest.mock.calls[0][0] as Record + + it('trims and lower-cases levels a reference resolved to', async () => { + await executeProviderRequest('openai', { + model: 'gpt-5', + workspaceId: 'ws-1', + reasoningEffort: ' High ', + verbosity: 'LOW', + }) + + expect(sentRequest().reasoningEffort).toBe('high') + expect(sentRequest().verbosity).toBe('low') + }) + + it('treats a level that resolved to nothing as unset rather than an empty string', async () => { + await executeProviderRequest('openai', { + model: 'gpt-5', + workspaceId: 'ws-1', + reasoningEffort: '', + verbosity: ' ', + }) + + expect(sentRequest().reasoningEffort).toBeUndefined() + expect(sentRequest().verbosity).toBeUndefined() + }) + + it('leaves an already-valid level untouched', async () => { + await executeProviderRequest('openai', { + model: 'gpt-5', + workspaceId: 'ws-1', + reasoningEffort: 'medium', + verbosity: 'high', + }) + + expect(sentRequest().reasoningEffort).toBe('medium') + expect(sentRequest().verbosity).toBe('high') + }) + + it('still drops levels the resolved model does not support', async () => { + await executeProviderRequest('anthropic', { + model: 'claude-opus-4-6', + workspaceId: 'ws-1', + reasoningEffort: 'high', + verbosity: 'high', + }) + + expect(sentRequest().reasoningEffort).toBeUndefined() + expect(sentRequest().verbosity).toBeUndefined() + }) +}) diff --git a/apps/sim/providers/index.ts b/apps/sim/providers/index.ts index afda36e3f27..7307bd5ae0f 100644 --- a/apps/sim/providers/index.ts +++ b/apps/sim/providers/index.ts @@ -39,10 +39,26 @@ const logger = createLogger('Providers') */ export const MAX_TOOL_ITERATIONS = 20 +/** + * Normalizes a model-tuning level that may have arrived from a variable or block reference + * rather than a picker. Every level a model declares is lower-case, so trimming and + * lower-casing lets a reference resolve to `"High"` or `" high "` and still apply. A level + * that resolves to nothing becomes `undefined` so the field reads as untouched instead of + * sending an empty string the provider rejects. + */ +function normalizeModelLevel(value: string | undefined): string | undefined { + if (typeof value !== 'string') return undefined + const normalized = value.trim().toLowerCase() + return normalized || undefined +} + function sanitizeRequest(request: ProviderRequest): ProviderRequest { const sanitizedRequest = { ...request } const model = sanitizedRequest.model + sanitizedRequest.reasoningEffort = normalizeModelLevel(sanitizedRequest.reasoningEffort) + sanitizedRequest.verbosity = normalizeModelLevel(sanitizedRequest.verbosity) + if (model && !supportsTemperature(model)) { sanitizedRequest.temperature = undefined } From 034aec769440c1095f049f111a9912898b86230d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 3 Aug 2026 17:23:57 -0700 Subject: [PATCH 2/6] improvement(agent): allow variable references in thinking level Extends the same treatment to Thinking Level so all three model-tuning fields behave consistently, and logs a level a model does not declare. - switch `thinkingLevel` to `combobox` with the reference-aware condition - normalize it alongside the other two; an empty resolve now takes the deliberate "send nothing" path rather than the incoherent half-state it hit before, and stays distinct from an explicit `none` - warn when a level is not one the model declares, still forwarding it: Sim's per-model lists drive the pickers and can lag a provider, and a sweep needs the provider's own error rather than a silent fallback to the default --- apps/sim/blocks/blocks.test.ts | 20 +++++++---- apps/sim/blocks/blocks/agent.ts | 9 ++--- apps/sim/providers/index.test.ts | 56 ++++++++++++++++++++++++++++-- apps/sim/providers/index.ts | 58 ++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 15 deletions(-) diff --git a/apps/sim/blocks/blocks.test.ts b/apps/sim/blocks/blocks.test.ts index 7de8356d149..70991e860a5 100644 --- a/apps/sim/blocks/blocks.test.ts +++ b/apps/sim/blocks/blocks.test.ts @@ -843,10 +843,17 @@ describe.concurrent('Blocks Module', () => { expect(modelSubBlock?.commandSearchable).toBe(true) }) - it('should let the agent reasoning and verbosity fields take a typed reference', () => { + /** Each model-tuning field with a model that accepts it and one that does not. */ + const AGENT_MODEL_LEVEL_FIELDS = [ + { id: 'reasoningEffort', capable: 'gpt-5.1', incapable: 'claude-sonnet-5' }, + { id: 'verbosity', capable: 'gpt-5.1', incapable: 'claude-sonnet-5' }, + { id: 'thinkingLevel', capable: 'claude-sonnet-5', incapable: 'gpt-5.1' }, + ] as const + + it('should let the agent model-tuning fields take a typed reference', () => { const agentBlock = getBlock('agent') - for (const id of ['reasoningEffort', 'verbosity']) { + for (const { id } of AGENT_MODEL_LEVEL_FIELDS) { const subBlock = agentBlock?.subBlocks.find((sb) => sb.id === id) // A combobox is editable, so a `` / `{{ENV_VAR}}` reference can be // typed into it; the option list still offers every level the model accepts. @@ -855,18 +862,19 @@ describe.concurrent('Blocks Module', () => { } }) - it('should keep the agent reasoning and verbosity fields visible when the model is a reference', () => { + it('should keep the agent model-tuning fields visible when the model is a reference', () => { const agentBlock = getBlock('agent') - for (const id of ['reasoningEffort', 'verbosity']) { + for (const { id, capable, incapable } of AGENT_MODEL_LEVEL_FIELDS) { const subBlock = agentBlock?.subBlocks.find((sb) => sb.id === id) const condition = subBlock?.condition if (typeof condition !== 'function') throw new Error(`${id} condition is not a function`) expect(evaluateSubBlockCondition(condition, { model: '' })).toBe(true) expect(evaluateSubBlockCondition(condition, { model: '{{MODEL_ID}}' })).toBe(true) - expect(evaluateSubBlockCondition(condition, { model: 'gpt-5.1' })).toBe(true) - expect(evaluateSubBlockCondition(condition, { model: 'claude-sonnet-5' })).toBe(false) + // Gating on the capability list is unchanged for a literal model. + expect(evaluateSubBlockCondition(condition, { model: capable })).toBe(true) + expect(evaluateSubBlockCondition(condition, { model: incapable })).toBe(false) } }) diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index 73c8cdc3c95..363fd39b21c 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -266,8 +266,8 @@ Return ONLY the JSON array.`, { id: 'thinkingLevel', title: 'Thinking Level', - type: 'dropdown', - placeholder: 'Select thinking level...', + type: 'combobox', + placeholder: 'Type or select thinking level...', options: [ { label: 'none', id: 'none' }, { label: 'minimal', id: 'minimal' }, @@ -301,10 +301,7 @@ Return ONLY the JSON array.`, return [noneOption, ...validOptions.map((opt) => ({ label: opt, id: opt }))] }, mode: 'advanced', - condition: { - field: 'model', - value: MODELS_WITH_THINKING, - }, + condition: getModelCapabilityCondition(MODELS_WITH_THINKING), }, { id: 'promptCaching', diff --git a/apps/sim/providers/index.test.ts b/apps/sim/providers/index.test.ts index 2ce7039156c..9e0d69ad235 100644 --- a/apps/sim/providers/index.test.ts +++ b/apps/sim/providers/index.test.ts @@ -414,9 +414,9 @@ describe('executeProviderRequest — streaming cost policy', () => { }) /** - * `reasoningEffort` and `verbosity` can be bound to a variable or block reference in the - * agent block, so by the time they reach the provider they hold whatever that reference - * resolved to rather than a value picked from a list. + * `reasoningEffort`, `verbosity`, and `thinkingLevel` can be bound to a variable or block + * reference in the agent block, so by the time they reach the provider they hold whatever + * that reference resolved to rather than a value picked from a list. */ describe('executeProviderRequest — model level normalization', () => { beforeEach(() => { @@ -443,6 +443,16 @@ describe('executeProviderRequest — model level normalization', () => { expect(sentRequest().verbosity).toBe('low') }) + it('trims and lower-cases a thinking level a reference resolved to', async () => { + await executeProviderRequest('anthropic', { + model: 'claude-sonnet-5', + workspaceId: 'ws-1', + thinkingLevel: ' High ', + }) + + expect(sentRequest().thinkingLevel).toBe('high') + }) + it('treats a level that resolved to nothing as unset rather than an empty string', async () => { await executeProviderRequest('openai', { model: 'gpt-5', @@ -455,6 +465,30 @@ describe('executeProviderRequest — model level normalization', () => { expect(sentRequest().verbosity).toBeUndefined() }) + /** + * Providers treat an explicit `'none'` as "thinking off" and an absent value as "send + * nothing", so a reference that resolved to nothing must land on the latter. + */ + it('treats a thinking level that resolved to nothing as unset, not as none', async () => { + await executeProviderRequest('anthropic', { + model: 'claude-sonnet-5', + workspaceId: 'ws-1', + thinkingLevel: ' ', + }) + + expect(sentRequest().thinkingLevel).toBeUndefined() + }) + + it('preserves an explicit none thinking level', async () => { + await executeProviderRequest('anthropic', { + model: 'claude-sonnet-5', + workspaceId: 'ws-1', + thinkingLevel: 'none', + }) + + expect(sentRequest().thinkingLevel).toBe('none') + }) + it('leaves an already-valid level untouched', async () => { await executeProviderRequest('openai', { model: 'gpt-5', @@ -467,6 +501,22 @@ describe('executeProviderRequest — model level normalization', () => { expect(sentRequest().verbosity).toBe('high') }) + /** + * Sim's per-model level lists drive the pickers and can lag a provider that has started + * accepting a new level, so an unrecognized level is forwarded rather than dropped: the + * provider answers with an error naming the values it accepts, instead of Sim silently + * substituting the model default and quietly corrupting a sweep. + */ + it('forwards a level the model does not declare so the provider reports it', async () => { + await executeProviderRequest('openai', { + model: 'gpt-5', + workspaceId: 'ws-1', + reasoningEffort: 'xhigh', + }) + + expect(sentRequest().reasoningEffort).toBe('xhigh') + }) + it('still drops levels the resolved model does not support', async () => { await executeProviderRequest('anthropic', { model: 'claude-opus-4-6', diff --git a/apps/sim/providers/index.ts b/apps/sim/providers/index.ts index 7307bd5ae0f..c94586f178a 100644 --- a/apps/sim/providers/index.ts +++ b/apps/sim/providers/index.ts @@ -15,6 +15,11 @@ import { attachLargeFileRemoteUrls, uploadLargeFilesToProvider, } from '@/providers/file-attachments.server' +import { + getReasoningEffortValuesForModel, + getThinkingLevelsForModel, + getVerbosityValuesForModel, +} from '@/providers/models' import { getProviderExecutor } from '@/providers/registry' import { type ProviderRuntimeContext, @@ -52,12 +57,46 @@ function normalizeModelLevel(value: string | undefined): string | undefined { return normalized || undefined } +/** + * Levels the pickers offer on top of what a model declares. `auto` means "say nothing" and + * `none` means "explicitly off"; every provider adapter special-cases them, so neither is + * an unrecognized level. + */ +const MODEL_LEVEL_SENTINELS = new Set(['auto', 'none']) + +/** + * Logs a level that is not one the model declares. + * + * Deliberately does not drop the value. Sim's per-model level lists exist to populate the + * pickers and can lag a provider that has started accepting a new level, so rejecting on them + * would refuse values the API would have taken. Forwarding instead surfaces the provider's own + * error, which names the field and the values it accepts — the loud failure an eval sweeping + * levels needs, where silently substituting the model default would corrupt the results. + */ +function warnOnUnrecognizedLevel( + field: 'reasoningEffort' | 'verbosity' | 'thinkingLevel', + model: string | undefined, + value: string | undefined, + declaredValues: string[] | null +): void { + if (!model || !value || MODEL_LEVEL_SENTINELS.has(value)) return + if (!declaredValues || declaredValues.includes(value)) return + + logger.warn('Model level is not one this model declares; forwarding to the provider', { + field, + model, + value, + declaredValues, + }) +} + function sanitizeRequest(request: ProviderRequest): ProviderRequest { const sanitizedRequest = { ...request } const model = sanitizedRequest.model sanitizedRequest.reasoningEffort = normalizeModelLevel(sanitizedRequest.reasoningEffort) sanitizedRequest.verbosity = normalizeModelLevel(sanitizedRequest.verbosity) + sanitizedRequest.thinkingLevel = normalizeModelLevel(sanitizedRequest.thinkingLevel) if (model && !supportsTemperature(model)) { sanitizedRequest.temperature = undefined @@ -79,6 +118,25 @@ function sanitizeRequest(request: ProviderRequest): ProviderRequest { sanitizedRequest.promptCaching = undefined } + warnOnUnrecognizedLevel( + 'reasoningEffort', + model, + sanitizedRequest.reasoningEffort, + model ? getReasoningEffortValuesForModel(model) : null + ) + warnOnUnrecognizedLevel( + 'verbosity', + model, + sanitizedRequest.verbosity, + model ? getVerbosityValuesForModel(model) : null + ) + warnOnUnrecognizedLevel( + 'thinkingLevel', + model, + sanitizedRequest.thinkingLevel, + model ? getThinkingLevelsForModel(model) : null + ) + return sanitizedRequest } From 2a7a055ed553fb56a677fdb2c83a6414697e285f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 3 Aug 2026 17:36:31 -0700 Subject: [PATCH 3/6] improvement(agent): report a model level the sanitizer discards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A model bound to a variable or block reference only resolves at execution time, so a run whose reference landed on a model outside Sim's catalogue had its requested level cleared with no signal and quietly fell back to that model's default. Dropping stays the safe default — a provider with no such parameter rejects the whole request — but it is now reported. - log the field, model, and value whenever an unsupported-field level is cleared - cover both diagnostics, including that they stay quiet for a declared level and for the `auto` / `none` sentinels --- apps/sim/providers/index.test.ts | 78 +++++++++++++++++++++++++++++++- apps/sim/providers/index.ts | 48 ++++++++++++++++++-- 2 files changed, 120 insertions(+), 6 deletions(-) diff --git a/apps/sim/providers/index.test.ts b/apps/sim/providers/index.test.ts index 9e0d69ad235..978297fa991 100644 --- a/apps/sim/providers/index.test.ts +++ b/apps/sim/providers/index.test.ts @@ -4,11 +4,32 @@ import { envFlagsMockFns, resetEnvFlagsMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetApiKeyWithBYOK, mockExecuteRequest } = vi.hoisted(() => ({ +const { mockGetApiKeyWithBYOK, mockExecuteRequest, mockLoggerWarn } = vi.hoisted(() => ({ mockGetApiKeyWithBYOK: vi.fn(), mockExecuteRequest: vi.fn(), + mockLoggerWarn: vi.fn(), })) +/** Overrides the global logger mock so the sanitizer's warnings are assertable. */ +vi.mock('@sim/logger', () => { + const createLogger = () => ({ + info: vi.fn(), + warn: mockLoggerWarn, + error: vi.fn(), + debug: vi.fn(), + trace: vi.fn(), + fatal: vi.fn(), + child: () => createLogger(), + withMetadata: () => createLogger(), + }) + return { + createLogger, + logger: createLogger(), + runWithRequestContext: (_ctx: unknown, fn: () => T): T => fn(), + getRequestContext: () => undefined, + } +}) + vi.mock('@/lib/api-key/byok', () => ({ getApiKeyWithBYOK: (...args: unknown[]) => mockGetApiKeyWithBYOK(...args), })) @@ -528,4 +549,59 @@ describe('executeProviderRequest — model level normalization', () => { expect(sentRequest().reasoningEffort).toBeUndefined() expect(sentRequest().verbosity).toBeUndefined() }) + + /** + * The model can itself be a reference, so it is only known at execution time. A run whose + * reference resolved to a model outside Sim's catalogue must not fall back to that model's + * default in silence. + */ + it('reports the level it drops when the resolved model does not support the field', async () => { + await executeProviderRequest('anthropic', { + model: 'claude-opus-4-6', + workspaceId: 'ws-1', + reasoningEffort: 'high', + }) + + expect(mockLoggerWarn).toHaveBeenCalledWith( + 'Model does not support this level; dropping it from the request', + expect.objectContaining({ + field: 'reasoningEffort', + model: 'claude-opus-4-6', + value: 'high', + }) + ) + }) + + it('stays quiet when an unsupported model was never given a level', async () => { + await executeProviderRequest('anthropic', { + model: 'claude-opus-4-6', + workspaceId: 'ws-1', + }) + + expect(mockLoggerWarn).not.toHaveBeenCalled() + }) + + it('reports a level the model accepts but does not declare', async () => { + await executeProviderRequest('openai', { + model: 'gpt-5', + workspaceId: 'ws-1', + reasoningEffort: 'xhigh', + }) + + expect(mockLoggerWarn).toHaveBeenCalledWith( + 'Model level is not one this model declares; forwarding to the provider', + expect.objectContaining({ field: 'reasoningEffort', model: 'gpt-5', value: 'xhigh' }) + ) + }) + + it('stays quiet for a declared level and for the auto and none sentinels', async () => { + await executeProviderRequest('openai', { + model: 'gpt-5', + workspaceId: 'ws-1', + reasoningEffort: 'auto', + verbosity: 'high', + }) + + expect(mockLoggerWarn).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/providers/index.ts b/apps/sim/providers/index.ts index c94586f178a..6a51e0fc4ee 100644 --- a/apps/sim/providers/index.ts +++ b/apps/sim/providers/index.ts @@ -64,8 +64,34 @@ function normalizeModelLevel(value: string | undefined): string | undefined { */ const MODEL_LEVEL_SENTINELS = new Set(['auto', 'none']) +type ModelLevelField = 'reasoningEffort' | 'verbosity' | 'thinkingLevel' + +/** + * Clears a level whose resolved model does not accept the field at all. + * + * Dropping is the safe default — a provider that has no such parameter rejects the whole + * request — but the discard is reported because the model can be bound to a variable or block + * reference and is therefore only known at execution time. Without this, a run whose reference + * resolved to a model outside Sim's catalogue would quietly fall back to that model's default + * while the caller believed the level applied. + */ +function dropUnsupportedLevel( + field: ModelLevelField, + model: string, + value: string | undefined +): undefined { + if (value) { + logger.warn('Model does not support this level; dropping it from the request', { + field, + model, + value, + }) + } + return undefined +} + /** - * Logs a level that is not one the model declares. + * Logs a level that the model accepts as a field but does not list as a value. * * Deliberately does not drop the value. Sim's per-model level lists exist to populate the * pickers and can lag a provider that has started accepting a new level, so rejecting on them @@ -74,7 +100,7 @@ const MODEL_LEVEL_SENTINELS = new Set(['auto', 'none']) * levels needs, where silently substituting the model default would corrupt the results. */ function warnOnUnrecognizedLevel( - field: 'reasoningEffort' | 'verbosity' | 'thinkingLevel', + field: ModelLevelField, model: string | undefined, value: string | undefined, declaredValues: string[] | null @@ -103,15 +129,27 @@ function sanitizeRequest(request: ProviderRequest): ProviderRequest { } if (model && !supportsReasoningEffort(model)) { - sanitizedRequest.reasoningEffort = undefined + sanitizedRequest.reasoningEffort = dropUnsupportedLevel( + 'reasoningEffort', + model, + sanitizedRequest.reasoningEffort + ) } if (model && !supportsVerbosity(model)) { - sanitizedRequest.verbosity = undefined + sanitizedRequest.verbosity = dropUnsupportedLevel( + 'verbosity', + model, + sanitizedRequest.verbosity + ) } if (model && !supportsThinking(model)) { - sanitizedRequest.thinkingLevel = undefined + sanitizedRequest.thinkingLevel = dropUnsupportedLevel( + 'thinkingLevel', + model, + sanitizedRequest.thinkingLevel + ) } if (model && !supportsPromptCaching(model)) { From fe522488644b0f1fda82cf87fdaf69fda2576f9b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 3 Aug 2026 17:48:18 -0700 Subject: [PATCH 4/6] fix(providers): redact resolved level content from sanitizer diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The model-level fields accept environment and block references, so an unrecognized level is not necessarily a mistyped level — it is whatever the reference resolved to, which can be secret content. The diagnostics added for dropped and undeclared levels echoed it straight into server logs. - log a level only when the catalogue declares it somewhere, or it is an `auto` / `none` sentinel; anything else is reported by length alone - stop discarding levels for a model the catalogue has never seen. Absent is unknown, not known-incapable, and a reference is exactly how a newly released model arrives before Sim catalogues it — the provider decides instead. Models the catalogue knows, and every dynamic-provider id, keep the protective drop --- apps/sim/providers/index.test.ts | 59 ++++++++++++++++++++++++++++++++ apps/sim/providers/index.ts | 38 ++++++++++++++++---- apps/sim/providers/models.ts | 27 +++++++++++++++ 3 files changed, 117 insertions(+), 7 deletions(-) diff --git a/apps/sim/providers/index.test.ts b/apps/sim/providers/index.test.ts index 978297fa991..1cfb60a88ac 100644 --- a/apps/sim/providers/index.test.ts +++ b/apps/sim/providers/index.test.ts @@ -604,4 +604,63 @@ describe('executeProviderRequest — model level normalization', () => { expect(mockLoggerWarn).not.toHaveBeenCalled() }) + + /** + * These fields take environment and block references, so a mistyped reference resolves the + * secret into the level. The diagnostics must never echo it. + */ + it('redacts a level that is not a catalogue level before logging it', async () => { + const secret = 'sk-proj-abcdef0123456789' + + await executeProviderRequest('openai', { + model: 'gpt-5', + workspaceId: 'ws-1', + reasoningEffort: secret, + }) + + expect(mockLoggerWarn).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ value: `[redacted ${secret.length} chars]` }) + ) + const loggedText = JSON.stringify(mockLoggerWarn.mock.calls) + expect(loggedText).not.toContain(secret) + }) + + it('keeps the auto sentinel readable in a drop diagnostic', async () => { + await executeProviderRequest('anthropic', { + model: 'claude-opus-4-6', + workspaceId: 'ws-1', + reasoningEffort: 'auto', + }) + + expect(mockLoggerWarn).toHaveBeenCalledWith( + 'Model does not support this level; dropping it from the request', + expect.objectContaining({ value: 'auto' }) + ) + }) + + /** + * A model the catalogue has never seen is unknown, not known-incapable — which is exactly + * how a newly released model arrives through a reference before Sim catalogues it. The + * provider decides, rather than the level being discarded on a stale list. + */ + it('forwards levels for a model absent from the catalogue', async () => { + await executeProviderRequest('openai', { + model: 'gpt-6-unreleased', + workspaceId: 'ws-1', + reasoningEffort: 'high', + }) + + expect(sentRequest().reasoningEffort).toBe('high') + }) + + it('still drops levels for a dynamic-provider model that does not take them', async () => { + await executeProviderRequest('ollama', { + model: 'ollama/llama3', + workspaceId: 'ws-1', + reasoningEffort: 'high', + }) + + expect(sentRequest().reasoningEffort).toBeUndefined() + }) }) diff --git a/apps/sim/providers/index.ts b/apps/sim/providers/index.ts index 6a51e0fc4ee..3bd9241ec7f 100644 --- a/apps/sim/providers/index.ts +++ b/apps/sim/providers/index.ts @@ -19,6 +19,8 @@ import { getReasoningEffortValuesForModel, getThinkingLevelsForModel, getVerbosityValuesForModel, + isKnownModelId, + isKnownModelLevelValue, } from '@/providers/models' import { getProviderExecutor } from '@/providers/registry' import { @@ -66,14 +68,27 @@ const MODEL_LEVEL_SENTINELS = new Set(['auto', 'none']) type ModelLevelField = 'reasoningEffort' | 'verbosity' | 'thinkingLevel' +/** + * Renders a level for a log line. + * + * These fields accept variable and environment references, so an unrecognized value is not + * necessarily a mistyped level — it is whatever the reference resolved to, which may be secret + * content. Only a level the catalogue declares somewhere is safe to echo; anything else is + * reported by length alone, which is enough to tell a stray level from a resolved blob. + */ +function describeLevel(value: string): string { + const isSafe = MODEL_LEVEL_SENTINELS.has(value) || isKnownModelLevelValue(value) + return isSafe ? value : `[redacted ${value.length} chars]` +} + /** * Clears a level whose resolved model does not accept the field at all. * * Dropping is the safe default — a provider that has no such parameter rejects the whole * request — but the discard is reported because the model can be bound to a variable or block * reference and is therefore only known at execution time. Without this, a run whose reference - * resolved to a model outside Sim's catalogue would quietly fall back to that model's default - * while the caller believed the level applied. + * resolved to a model that does not take the field would quietly fall back to that model's + * default while the caller believed the level applied. */ function dropUnsupportedLevel( field: ModelLevelField, @@ -84,7 +99,7 @@ function dropUnsupportedLevel( logger.warn('Model does not support this level; dropping it from the request', { field, model, - value, + value: describeLevel(value), }) } return undefined @@ -111,7 +126,7 @@ function warnOnUnrecognizedLevel( logger.warn('Model level is not one this model declares; forwarding to the provider', { field, model, - value, + value: describeLevel(value), declaredValues, }) } @@ -128,7 +143,16 @@ function sanitizeRequest(request: ProviderRequest): ProviderRequest { sanitizedRequest.temperature = undefined } - if (model && !supportsReasoningEffort(model)) { + /** + * A model absent from the catalogue is unknown, not known-incapable. Since the model can be + * bound to a reference, that is exactly how a newly released model arrives before Sim has + * catalogued it — so its levels are forwarded and the provider decides, rather than being + * discarded on the strength of a list that has not caught up. Models the catalogue does + * know, and every dynamic-provider id, keep the protective drop. + */ + const isCatalogued = Boolean(model) && isKnownModelId(model) + + if (model && isCatalogued && !supportsReasoningEffort(model)) { sanitizedRequest.reasoningEffort = dropUnsupportedLevel( 'reasoningEffort', model, @@ -136,7 +160,7 @@ function sanitizeRequest(request: ProviderRequest): ProviderRequest { ) } - if (model && !supportsVerbosity(model)) { + if (model && isCatalogued && !supportsVerbosity(model)) { sanitizedRequest.verbosity = dropUnsupportedLevel( 'verbosity', model, @@ -144,7 +168,7 @@ function sanitizeRequest(request: ProviderRequest): ProviderRequest { ) } - if (model && !supportsThinking(model)) { + if (model && isCatalogued && !supportsThinking(model)) { sanitizedRequest.thinkingLevel = dropUnsupportedLevel( 'thinkingLevel', model, diff --git a/apps/sim/providers/models.ts b/apps/sim/providers/models.ts index 98be92cd78d..393cacf33fa 100644 --- a/apps/sim/providers/models.ts +++ b/apps/sim/providers/models.ts @@ -4553,6 +4553,33 @@ export function getThinkingLevelsForModel(modelId: string): string[] | null { return capability?.levels ?? null } +const ALL_MODEL_LEVEL_VALUES = new Set() +for (const provider of Object.values(PROVIDER_DEFINITIONS)) { + for (const model of provider.models) { + for (const value of model.capabilities.reasoningEffort?.values ?? []) { + ALL_MODEL_LEVEL_VALUES.add(value) + } + for (const value of model.capabilities.verbosity?.values ?? []) { + ALL_MODEL_LEVEL_VALUES.add(value) + } + for (const level of model.capabilities.thinking?.levels ?? []) { + ALL_MODEL_LEVEL_VALUES.add(level) + } + } +} + +/** + * Whether a string is a tuning level some model in the catalogue declares, regardless of which. + * + * Callers that need to put a caller-supplied level into a log or an error gate on this first. + * These fields accept variable and environment references, so an unrecognized value is not + * necessarily a mistyped level — it can be whatever that reference resolved to, up to and + * including secret content that must never be echoed. + */ +export function isKnownModelLevelValue(value: string): boolean { + return ALL_MODEL_LEVEL_VALUES.has(value) +} + /** * Per-provider defaults for thinking stream visibility, used when a model does * not declare `capabilities.thinking.streamed` explicitly. Gemini and OpenAI From a4c7abb1206756e34ea6da49f535aa5af91ca399 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 3 Aug 2026 18:26:26 -0700 Subject: [PATCH 5/6] fix(providers): redact the level in Anthropic's unsupported-thinking warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Forwarding an undeclared level is deliberate, but it means the Anthropic adapter receives it and interpolates it straight into its "not supported, ignoring" warning. Since the field is reference-bound, that value can be whatever a mistyped `{{ENV_VAR}}` or block reference resolved to — so the redaction added for the sanitizer's own diagnostics was leaking one layer downstream. - promote the level renderer to `providers/utils` as `describeModelLevel`, the single gate every site echoing a caller-supplied level goes through - use it in Anthropic's warning and in both sanitizer diagnostics --- .../providers/anthropic/core.thinking.test.ts | 19 ++++++++++++ apps/sim/providers/anthropic/core.ts | 8 +++-- apps/sim/providers/index.ts | 24 ++------------- apps/sim/providers/utils.test.ts | 30 +++++++++++++++++++ apps/sim/providers/utils.ts | 25 ++++++++++++++++ 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/apps/sim/providers/anthropic/core.thinking.test.ts b/apps/sim/providers/anthropic/core.thinking.test.ts index 8fcad2e438d..ae014380dc1 100644 --- a/apps/sim/providers/anthropic/core.thinking.test.ts +++ b/apps/sim/providers/anthropic/core.thinking.test.ts @@ -8,6 +8,7 @@ */ import { describe, expect, it } from 'vitest' import { buildThinkingConfig } from '@/providers/anthropic/core' +import { describeModelLevel } from '@/providers/utils' describe('buildThinkingConfig', () => { it('requests summarized display for omitted-display models on agent-events runs', () => { @@ -55,3 +56,21 @@ describe('buildThinkingConfig', () => { expect(buildThinkingConfig('gpt-4o', 'high', true)).toBeNull() }) }) + +/** + * A thinking level that is not one the model declares reaches this adapter, by design — Sim's + * per-model lists can lag a provider. The adapter logs that it is ignoring it, and since the + * field is reference-bound, the value it logs can be whatever a mistyped `{{ENV_VAR}}` or block + * reference resolved to. + */ +describe('unsupported thinking level logging', () => { + it('returns null for a level the model does not declare', () => { + expect(buildThinkingConfig('claude-sonnet-5', 'sk-proj-abcdef0123456789', false)).toBeNull() + }) + + it('redacts the level in the ignore warning instead of echoing it', () => { + const secret = 'sk-proj-abcdef0123456789' + expect(describeModelLevel(secret)).toBe(`[redacted ${secret.length} chars]`) + expect(describeModelLevel('high')).toBe('high') + }) +}) diff --git a/apps/sim/providers/anthropic/core.ts b/apps/sim/providers/anthropic/core.ts index 384a29f4ef7..eaa823e9ad3 100644 --- a/apps/sim/providers/anthropic/core.ts +++ b/apps/sim/providers/anthropic/core.ts @@ -31,7 +31,11 @@ import { adaptAnthropicToolSchema } from '@/providers/tool-schema-adapter' import { enrichLastModelSegment } from '@/providers/trace-enrichment' import type { ProviderRequest, ProviderResponse, TimeSegment } from '@/providers/types' import { ProviderError } from '@/providers/types' -import { prepareToolExecution, prepareToolsWithUsageControl } from '@/providers/utils' +import { + describeModelLevel, + prepareToolExecution, + prepareToolsWithUsageControl, +} from '@/providers/utils' /** * Configuration for creating an Anthropic provider instance. @@ -396,7 +400,7 @@ export async function executeAnthropicProviderRequest( ) } else { logger.warn( - `Thinking level "${request.thinkingLevel}" not supported for model: ${modelId}, ignoring` + `Thinking level "${describeModelLevel(request.thinkingLevel)}" not supported for model: ${modelId}, ignoring` ) } } diff --git a/apps/sim/providers/index.ts b/apps/sim/providers/index.ts index 3bd9241ec7f..0c4dde493b1 100644 --- a/apps/sim/providers/index.ts +++ b/apps/sim/providers/index.ts @@ -20,7 +20,6 @@ import { getThinkingLevelsForModel, getVerbosityValuesForModel, isKnownModelId, - isKnownModelLevelValue, } from '@/providers/models' import { getProviderExecutor } from '@/providers/registry' import { @@ -29,6 +28,7 @@ import { } from '@/providers/runtime-context' import type { ProviderId, ProviderRequest, ProviderResponse } from '@/providers/types' import { + describeModelLevel, generateStructuredOutputInstructions, sumToolCosts, supportsPromptCaching, @@ -59,28 +59,10 @@ function normalizeModelLevel(value: string | undefined): string | undefined { return normalized || undefined } -/** - * Levels the pickers offer on top of what a model declares. `auto` means "say nothing" and - * `none` means "explicitly off"; every provider adapter special-cases them, so neither is - * an unrecognized level. - */ const MODEL_LEVEL_SENTINELS = new Set(['auto', 'none']) type ModelLevelField = 'reasoningEffort' | 'verbosity' | 'thinkingLevel' -/** - * Renders a level for a log line. - * - * These fields accept variable and environment references, so an unrecognized value is not - * necessarily a mistyped level — it is whatever the reference resolved to, which may be secret - * content. Only a level the catalogue declares somewhere is safe to echo; anything else is - * reported by length alone, which is enough to tell a stray level from a resolved blob. - */ -function describeLevel(value: string): string { - const isSafe = MODEL_LEVEL_SENTINELS.has(value) || isKnownModelLevelValue(value) - return isSafe ? value : `[redacted ${value.length} chars]` -} - /** * Clears a level whose resolved model does not accept the field at all. * @@ -99,7 +81,7 @@ function dropUnsupportedLevel( logger.warn('Model does not support this level; dropping it from the request', { field, model, - value: describeLevel(value), + value: describeModelLevel(value), }) } return undefined @@ -126,7 +108,7 @@ function warnOnUnrecognizedLevel( logger.warn('Model level is not one this model declares; forwarding to the provider', { field, model, - value: describeLevel(value), + value: describeModelLevel(value), declaredValues, }) } diff --git a/apps/sim/providers/utils.test.ts b/apps/sim/providers/utils.test.ts index f959cf7911c..980839059da 100644 --- a/apps/sim/providers/utils.test.ts +++ b/apps/sim/providers/utils.test.ts @@ -2,6 +2,7 @@ import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { calculateCost, + describeModelLevel, extractAndParseJSON, filterBlacklistedModels, formatCost, @@ -1767,3 +1768,32 @@ describe('prepareToolExecution invoker identity hand-off', () => { expect(executionParams._context.executionId).toBeUndefined() }) }) + +/** + * The agent block's tuning-level fields accept variable and environment references, so any + * message that echoes a caller-supplied level can otherwise carry whatever that reference + * resolved to — including secret content. + */ +describe('describeModelLevel', () => { + it('echoes a level the catalogue declares', () => { + expect(describeModelLevel('high')).toBe('high') + expect(describeModelLevel('minimal')).toBe('minimal') + expect(describeModelLevel('xhigh')).toBe('xhigh') + }) + + it('echoes the auto and none sentinels', () => { + expect(describeModelLevel('auto')).toBe('auto') + expect(describeModelLevel('none')).toBe('none') + }) + + it('redacts anything else to a length', () => { + const secret = 'sk-proj-abcdef0123456789' + expect(describeModelLevel(secret)).toBe(`[redacted ${secret.length} chars]`) + expect(describeModelLevel(secret)).not.toContain('abcdef') + }) + + it('reports an absent level without throwing', () => { + expect(describeModelLevel(undefined)).toBe('(unset)') + expect(describeModelLevel('')).toBe('(unset)') + }) +}) diff --git a/apps/sim/providers/utils.ts b/apps/sim/providers/utils.ts index 12b7c0cc966..c5b6869e634 100644 --- a/apps/sim/providers/utils.ts +++ b/apps/sim/providers/utils.ts @@ -44,6 +44,7 @@ import { getReasoningEffortValuesForModel as getReasoningEffortValuesForModelFromDefinitions, getThinkingLevelsForModel as getThinkingLevelsForModelFromDefinitions, getVerbosityValuesForModel as getVerbosityValuesForModelFromDefinitions, + isKnownModelLevelValue, PROVIDER_DEFINITIONS, supportsTemperature as supportsTemperatureFromDefinitions, supportsToolUsageControl as supportsToolUsageControlFromDefinitions, @@ -1381,6 +1382,30 @@ export function supportsTemperature(model: string): boolean { return supportsTemperatureFromDefinitions(model) } +/** + * Levels the pickers offer on top of what a model declares. `auto` means "say nothing" and + * `none` means "explicitly off"; provider adapters special-case both, so neither is an + * unrecognized level. + */ +const MODEL_LEVEL_SENTINELS = new Set(['auto', 'none']) + +/** + * Renders a tuning level for a log line or an error message. + * + * The agent block's reasoning effort, verbosity, and thinking level fields accept variable and + * environment references, so an unrecognized level is not necessarily a mistyped level — it is + * whatever the reference resolved to, up to and including secret content. Only a level the + * catalogue declares somewhere is safe to echo; anything else is reported by length alone, + * which still distinguishes a stray level from a resolved blob. + * + * Every site that puts a caller-supplied level into a message must go through this. + */ +export function describeModelLevel(value: string | undefined): string { + if (!value) return '(unset)' + const isSafe = MODEL_LEVEL_SENTINELS.has(value) || isKnownModelLevelValue(value) + return isSafe ? value : `[redacted ${value.length} chars]` +} + export function supportsReasoningEffort(model: string): boolean { return MODELS_WITH_REASONING_EFFORT.includes(model.toLowerCase()) } From 9aa8fe065271e9891a8be8b85dd84c0c0dbceacc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 3 Aug 2026 18:42:18 -0700 Subject: [PATCH 6/6] refactor(providers): drop the sanitizer's level diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two warnings logged server-side, where the workflow author who set the level never sees them, and the surprising case they described — a level discarded for a model newer than the catalogue — is now fixed at the source rather than narrated. They also carried the redaction that leaked resolved content before it was caught, so removing them removes that surface entirely. Levels still normalize, and still drop for a catalogued model that does not take the field. `describeModelLevel` stays for Anthropic's unsupported-thinking warning, which is a pre-existing log this feature newly exposes to resolved reference content. --- apps/sim/providers/index.test.ts | 112 +------------------------------ apps/sim/providers/index.ts | 110 +++--------------------------- 2 files changed, 11 insertions(+), 211 deletions(-) diff --git a/apps/sim/providers/index.test.ts b/apps/sim/providers/index.test.ts index 1cfb60a88ac..042081af2b4 100644 --- a/apps/sim/providers/index.test.ts +++ b/apps/sim/providers/index.test.ts @@ -4,32 +4,11 @@ import { envFlagsMockFns, resetEnvFlagsMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetApiKeyWithBYOK, mockExecuteRequest, mockLoggerWarn } = vi.hoisted(() => ({ +const { mockGetApiKeyWithBYOK, mockExecuteRequest } = vi.hoisted(() => ({ mockGetApiKeyWithBYOK: vi.fn(), mockExecuteRequest: vi.fn(), - mockLoggerWarn: vi.fn(), })) -/** Overrides the global logger mock so the sanitizer's warnings are assertable. */ -vi.mock('@sim/logger', () => { - const createLogger = () => ({ - info: vi.fn(), - warn: mockLoggerWarn, - error: vi.fn(), - debug: vi.fn(), - trace: vi.fn(), - fatal: vi.fn(), - child: () => createLogger(), - withMetadata: () => createLogger(), - }) - return { - createLogger, - logger: createLogger(), - runWithRequestContext: (_ctx: unknown, fn: () => T): T => fn(), - getRequestContext: () => undefined, - } -}) - vi.mock('@/lib/api-key/byok', () => ({ getApiKeyWithBYOK: (...args: unknown[]) => mockGetApiKeyWithBYOK(...args), })) @@ -550,95 +529,6 @@ describe('executeProviderRequest — model level normalization', () => { expect(sentRequest().verbosity).toBeUndefined() }) - /** - * The model can itself be a reference, so it is only known at execution time. A run whose - * reference resolved to a model outside Sim's catalogue must not fall back to that model's - * default in silence. - */ - it('reports the level it drops when the resolved model does not support the field', async () => { - await executeProviderRequest('anthropic', { - model: 'claude-opus-4-6', - workspaceId: 'ws-1', - reasoningEffort: 'high', - }) - - expect(mockLoggerWarn).toHaveBeenCalledWith( - 'Model does not support this level; dropping it from the request', - expect.objectContaining({ - field: 'reasoningEffort', - model: 'claude-opus-4-6', - value: 'high', - }) - ) - }) - - it('stays quiet when an unsupported model was never given a level', async () => { - await executeProviderRequest('anthropic', { - model: 'claude-opus-4-6', - workspaceId: 'ws-1', - }) - - expect(mockLoggerWarn).not.toHaveBeenCalled() - }) - - it('reports a level the model accepts but does not declare', async () => { - await executeProviderRequest('openai', { - model: 'gpt-5', - workspaceId: 'ws-1', - reasoningEffort: 'xhigh', - }) - - expect(mockLoggerWarn).toHaveBeenCalledWith( - 'Model level is not one this model declares; forwarding to the provider', - expect.objectContaining({ field: 'reasoningEffort', model: 'gpt-5', value: 'xhigh' }) - ) - }) - - it('stays quiet for a declared level and for the auto and none sentinels', async () => { - await executeProviderRequest('openai', { - model: 'gpt-5', - workspaceId: 'ws-1', - reasoningEffort: 'auto', - verbosity: 'high', - }) - - expect(mockLoggerWarn).not.toHaveBeenCalled() - }) - - /** - * These fields take environment and block references, so a mistyped reference resolves the - * secret into the level. The diagnostics must never echo it. - */ - it('redacts a level that is not a catalogue level before logging it', async () => { - const secret = 'sk-proj-abcdef0123456789' - - await executeProviderRequest('openai', { - model: 'gpt-5', - workspaceId: 'ws-1', - reasoningEffort: secret, - }) - - expect(mockLoggerWarn).toHaveBeenCalledWith( - expect.any(String), - expect.objectContaining({ value: `[redacted ${secret.length} chars]` }) - ) - const loggedText = JSON.stringify(mockLoggerWarn.mock.calls) - expect(loggedText).not.toContain(secret) - }) - - it('keeps the auto sentinel readable in a drop diagnostic', async () => { - await executeProviderRequest('anthropic', { - model: 'claude-opus-4-6', - workspaceId: 'ws-1', - reasoningEffort: 'auto', - }) - - expect(mockLoggerWarn).toHaveBeenCalledWith( - 'Model does not support this level; dropping it from the request', - expect.objectContaining({ value: 'auto' }) - ) - }) - /** * A model the catalogue has never seen is unknown, not known-incapable — which is exactly * how a newly released model arrives through a reference before Sim catalogues it. The diff --git a/apps/sim/providers/index.ts b/apps/sim/providers/index.ts index 0c4dde493b1..49b3294dd03 100644 --- a/apps/sim/providers/index.ts +++ b/apps/sim/providers/index.ts @@ -15,12 +15,7 @@ import { attachLargeFileRemoteUrls, uploadLargeFilesToProvider, } from '@/providers/file-attachments.server' -import { - getReasoningEffortValuesForModel, - getThinkingLevelsForModel, - getVerbosityValuesForModel, - isKnownModelId, -} from '@/providers/models' +import { isKnownModelId } from '@/providers/models' import { getProviderExecutor } from '@/providers/registry' import { type ProviderRuntimeContext, @@ -28,7 +23,6 @@ import { } from '@/providers/runtime-context' import type { ProviderId, ProviderRequest, ProviderResponse } from '@/providers/types' import { - describeModelLevel, generateStructuredOutputInstructions, sumToolCosts, supportsPromptCaching, @@ -59,60 +53,6 @@ function normalizeModelLevel(value: string | undefined): string | undefined { return normalized || undefined } -const MODEL_LEVEL_SENTINELS = new Set(['auto', 'none']) - -type ModelLevelField = 'reasoningEffort' | 'verbosity' | 'thinkingLevel' - -/** - * Clears a level whose resolved model does not accept the field at all. - * - * Dropping is the safe default — a provider that has no such parameter rejects the whole - * request — but the discard is reported because the model can be bound to a variable or block - * reference and is therefore only known at execution time. Without this, a run whose reference - * resolved to a model that does not take the field would quietly fall back to that model's - * default while the caller believed the level applied. - */ -function dropUnsupportedLevel( - field: ModelLevelField, - model: string, - value: string | undefined -): undefined { - if (value) { - logger.warn('Model does not support this level; dropping it from the request', { - field, - model, - value: describeModelLevel(value), - }) - } - return undefined -} - -/** - * Logs a level that the model accepts as a field but does not list as a value. - * - * Deliberately does not drop the value. Sim's per-model level lists exist to populate the - * pickers and can lag a provider that has started accepting a new level, so rejecting on them - * would refuse values the API would have taken. Forwarding instead surfaces the provider's own - * error, which names the field and the values it accepts — the loud failure an eval sweeping - * levels needs, where silently substituting the model default would corrupt the results. - */ -function warnOnUnrecognizedLevel( - field: ModelLevelField, - model: string | undefined, - value: string | undefined, - declaredValues: string[] | null -): void { - if (!model || !value || MODEL_LEVEL_SENTINELS.has(value)) return - if (!declaredValues || declaredValues.includes(value)) return - - logger.warn('Model level is not one this model declares; forwarding to the provider', { - field, - model, - value: describeModelLevel(value), - declaredValues, - }) -} - function sanitizeRequest(request: ProviderRequest): ProviderRequest { const sanitizedRequest = { ...request } const model = sanitizedRequest.model @@ -126,61 +66,31 @@ function sanitizeRequest(request: ProviderRequest): ProviderRequest { } /** - * A model absent from the catalogue is unknown, not known-incapable. Since the model can be - * bound to a reference, that is exactly how a newly released model arrives before Sim has - * catalogued it — so its levels are forwarded and the provider decides, rather than being - * discarded on the strength of a list that has not caught up. Models the catalogue does - * know, and every dynamic-provider id, keep the protective drop. + * A model absent from the catalogue is unknown, not known-incapable. The model field is an + * editable combobox, so a model newer than `models.ts` reaches this point routed by pattern + * and executing normally — discarding its levels on the strength of a list that has not + * caught up loses a setting the provider would have honoured. Those levels are forwarded and + * the provider decides. Models the catalogue does know, and every dynamic-provider id, keep + * the protective drop. */ const isCatalogued = Boolean(model) && isKnownModelId(model) if (model && isCatalogued && !supportsReasoningEffort(model)) { - sanitizedRequest.reasoningEffort = dropUnsupportedLevel( - 'reasoningEffort', - model, - sanitizedRequest.reasoningEffort - ) + sanitizedRequest.reasoningEffort = undefined } if (model && isCatalogued && !supportsVerbosity(model)) { - sanitizedRequest.verbosity = dropUnsupportedLevel( - 'verbosity', - model, - sanitizedRequest.verbosity - ) + sanitizedRequest.verbosity = undefined } if (model && isCatalogued && !supportsThinking(model)) { - sanitizedRequest.thinkingLevel = dropUnsupportedLevel( - 'thinkingLevel', - model, - sanitizedRequest.thinkingLevel - ) + sanitizedRequest.thinkingLevel = undefined } if (model && !supportsPromptCaching(model)) { sanitizedRequest.promptCaching = undefined } - warnOnUnrecognizedLevel( - 'reasoningEffort', - model, - sanitizedRequest.reasoningEffort, - model ? getReasoningEffortValuesForModel(model) : null - ) - warnOnUnrecognizedLevel( - 'verbosity', - model, - sanitizedRequest.verbosity, - model ? getVerbosityValuesForModel(model) : null - ) - warnOnUnrecognizedLevel( - 'thinkingLevel', - model, - sanitizedRequest.thinkingLevel, - model ? getThinkingLevelsForModel(model) : null - ) - return sanitizedRequest }