diff --git a/apps/sim/app/api/knowledge/search/route.test.ts b/apps/sim/app/api/knowledge/search/route.test.ts index d12fd48e0aa..6fdf4cdfdc6 100644 --- a/apps/sim/app/api/knowledge/search/route.test.ts +++ b/apps/sim/app/api/knowledge/search/route.test.ts @@ -44,6 +44,11 @@ vi.mock('@/lib/tokenization/estimators', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn().mockReturnValue({ input: 0.00001042, output: 0, diff --git a/apps/sim/app/api/providers/baseten/models/route.test.ts b/apps/sim/app/api/providers/baseten/models/route.test.ts index fe53568ed51..e3c2cf4150a 100644 --- a/apps/sim/app/api/providers/baseten/models/route.test.ts +++ b/apps/sim/app/api/providers/baseten/models/route.test.ts @@ -17,6 +17,11 @@ const { })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, filterBlacklistedModels: mockFilterBlacklistedModels, isProviderBlacklisted: mockIsProviderBlacklisted, })) diff --git a/apps/sim/app/api/providers/ollama-cloud/models/route.test.ts b/apps/sim/app/api/providers/ollama-cloud/models/route.test.ts index 3b563a8f7cb..a4b4cbfedda 100644 --- a/apps/sim/app/api/providers/ollama-cloud/models/route.test.ts +++ b/apps/sim/app/api/providers/ollama-cloud/models/route.test.ts @@ -19,6 +19,11 @@ const { })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, filterBlacklistedModels: mockFilterBlacklistedModels, isProviderBlacklisted: mockIsProviderBlacklisted, })) diff --git a/apps/sim/app/api/providers/together/models/route.test.ts b/apps/sim/app/api/providers/together/models/route.test.ts index b7516070a0e..d5b9912dda1 100644 --- a/apps/sim/app/api/providers/together/models/route.test.ts +++ b/apps/sim/app/api/providers/together/models/route.test.ts @@ -19,6 +19,11 @@ const { })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, filterBlacklistedModels: mockFilterBlacklistedModels, isProviderBlacklisted: mockIsProviderBlacklisted, })) diff --git a/apps/sim/blocks/utils.test.ts b/apps/sim/blocks/utils.test.ts index 9d5a3adf24f..9cdb1d1df23 100644 --- a/apps/sim/blocks/utils.test.ts +++ b/apps/sim/blocks/utils.test.ts @@ -45,6 +45,11 @@ vi.mock('@/providers/models', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, getProviderFromModel: vi.fn(() => 'openai'), })) diff --git a/apps/sim/ee/access-control/utils/permission-check.test.ts b/apps/sim/ee/access-control/utils/permission-check.test.ts index c449d3d9608..8a9d502c80c 100644 --- a/apps/sim/ee/access-control/utils/permission-check.test.ts +++ b/apps/sim/ee/access-control/utils/permission-check.test.ts @@ -66,6 +66,11 @@ vi.mock('@/lib/permission-groups/types', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, getProviderFromModel: mockGetProviderFromModel, })) diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index 981352a69d5..0da21a6a431 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -30,6 +30,11 @@ import { executeTool } from '@/tools' process.env.NEXT_PUBLIC_APP_URL = 'http://localhost:3000' vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, getProviderFromModel: vi.fn().mockReturnValue('mock-provider'), transformBlockTool: vi.fn(), getBaseModelProviders: vi.fn().mockReturnValue({ openai: {}, anthropic: {} }), diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index 31a2268a404..8d510d1696e 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -53,10 +53,15 @@ import { stringifyJSON } from '@/executor/utils/json' import { resolveVertexCredential } from '@/executor/utils/vertex-credential' import { executeProviderRequest } from '@/providers' import { - INLINE_ATTACHMENT_THRESHOLD_BYTES, + formatAttachmentSizes, + getProviderFileStrategy, shouldUseLargeFilePath, supportsFileAttachments, } from '@/providers/attachments' +import { + canUseProviderLargeFilePath, + getInlineHydrationMaxBytes, +} from '@/providers/file-attachments.server' import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models' import { getProviderFromModel, transformBlockTool } from '@/providers/utils' import type { SerializedBlock } from '@/serializer/types' @@ -946,6 +951,8 @@ export class AgentBlockHandler implements BlockHandler { const requestId = ctx.executionId || ctx.workflowId || 'agent-files' const nextMessages = [...messages] + const inlineMaxBytes = getInlineHydrationMaxBytes(providerId) + for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) { const message = messages[messageIndex] if (!message.files?.length) { @@ -963,15 +970,37 @@ export class AgentBlockHandler implements BlockHandler { allowLargeValueWorkflowScope: ctx.allowLargeValueWorkflowScope, userId: ctx.userId, logger, - maxBytes: INLINE_ATTACHMENT_THRESHOLD_BYTES, + maxBytes: inlineMaxBytes, }) const missingFile = hydratedFiles.find( - (file) => !file.base64 && !shouldUseLargeFilePath(file, providerId) + (file) => + !file.base64 && + !(canUseProviderLargeFilePath(providerId) && shouldUseLargeFilePath(file, providerId)) ) if (missingFile) { + const { size: sizeMB, limit: inlineMB } = formatAttachmentSizes( + missingFile.size, + inlineMaxBytes + ) + const oversized = Number.isFinite(missingFile.size) && missingFile.size > inlineMaxBytes + /** + * Ordered by how general the cause is. A provider with no upload path at all cannot be + * helped by changing the file, and a deployment with no object storage cannot reach any + * upload path whatever the file is — so both outrank the format-specific case. Leading + * with the generated-document arm blamed the document on providers that have no upload + * path for anything, and on hosts whose only real problem was unconfigured storage. + */ + const reason = + getProviderFileStrategy(providerId) === 'inline' + ? `provider "${providerId}" has no large-file upload path` + : !canUseProviderLargeFilePath(providerId) + ? 'this deployment has no cloud file storage for the large-file upload path' + : `a generated document cannot use the large-file path for provider "${providerId}", because a signed URL points at the generation source rather than the rendered file` throw new Error( - `File "${missingFile.name}" could not be read for provider "${providerId}". The file may exceed the attachment size limit or may no longer be accessible.` + oversized + ? `File "${missingFile.name}" (${sizeMB}MB) exceeds the ${inlineMB}MB inline attachment limit, and ${reason}.` + : `File "${missingFile.name}" could not be read for provider "${providerId}". The file may no longer be accessible.` ) } diff --git a/apps/sim/executor/handlers/pi/keys.test.ts b/apps/sim/executor/handlers/pi/keys.test.ts index 2d33c2a30fa..5c0181d85b4 100644 --- a/apps/sim/executor/handlers/pi/keys.test.ts +++ b/apps/sim/executor/handlers/pi/keys.test.ts @@ -18,6 +18,11 @@ vi.mock('@/lib/api-key/byok', () => ({ getBYOKKey: mockGetBYOKKey, })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: mockCalculateCost, shouldBillModelUsage: mockShouldBill, })) diff --git a/apps/sim/executor/handlers/pi/pi-handler.test.ts b/apps/sim/executor/handlers/pi/pi-handler.test.ts index 61110401c4e..7a97fea5e45 100644 --- a/apps/sim/executor/handlers/pi/pi-handler.test.ts +++ b/apps/sim/executor/handlers/pi/pi-handler.test.ts @@ -77,6 +77,11 @@ vi.mock('@/providers/pi-providers', () => ({ resolvePiModelId: mockResolvePiModelId, })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, getProviderFromModel: mockGetProviderFromModel, })) vi.mock('@/blocks/utils', () => ({ diff --git a/apps/sim/lib/api-key/byok.test.ts b/apps/sim/lib/api-key/byok.test.ts index bcc8d81bfcb..c5da3daabf3 100644 --- a/apps/sim/lib/api-key/byok.test.ts +++ b/apps/sim/lib/api-key/byok.test.ts @@ -42,6 +42,11 @@ vi.mock('@/providers/models', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, PROVIDER_PLACEHOLDER_KEY: 'placeholder', })) diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts index 8ae05a41465..651c816f811 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts @@ -243,6 +243,11 @@ vi.mock('@/lib/workflows/skills/operations', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, getHostedModels: mockGetHostedModels, })) diff --git a/apps/sim/lib/model-router/resolve.test.ts b/apps/sim/lib/model-router/resolve.test.ts index a0cbbccd3a1..fcb68aded77 100644 --- a/apps/sim/lib/model-router/resolve.test.ts +++ b/apps/sim/lib/model-router/resolve.test.ts @@ -37,6 +37,11 @@ vi.mock('@/ee/access-control/utils/permission-check', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, getProviderFromModel: mockGetProviderFromModel, })) diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts index 6e62f378da9..38338975be0 100644 --- a/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts @@ -400,6 +400,40 @@ describe('hydrateUserFilesWithBase64', () => { expect(mockRedis.eval).toHaveBeenCalledOnce() }) + /** + * Reproduces the agent-attachment failure: a file under the inline limit whose base64 exceeds + * the 8 MiB single-Redis-write cap. The bytes are already read by the time the cache is + * written, so a refused cache write must degrade to "not cached", not fail the execution. + */ + it('still returns base64 when the value is too large to cache', async () => { + mockGetRedisClient.mockReturnValue(mockRedis) + const buffer = Buffer.alloc(9 * 1024 * 1024, 0x61) + mockDownloadFile.mockResolvedValueOnce(buffer) + const file: UserFile = { + id: 'file-1', + name: 'data_10mb.csv', + key: 'execution/workspace/workflow/exec-1/data_10mb.csv', + url: 'https://example.com/data_10mb.csv', + size: buffer.length, + type: 'text/csv', + context: 'execution', + } + + const hydrated = await hydrateUserFilesWithBase64( + { file }, + { + workspaceId: 'workspace', + workflowId: 'workflow', + executionId: 'exec-1', + userId: 'user-1', + maxBytes: 10 * 1024 * 1024, + } + ) + + expect(hydrated.file.base64).toBe(buffer.toString('base64')) + expect(mockRedis.eval).not.toHaveBeenCalled() + }) + it('releases indexed budget entries even when cache keys already expired', async () => { mockGetRedisClient.mockReturnValue(mockRedis) mockRedis.hgetall.mockResolvedValueOnce({ diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.ts index d9195d47c2e..9ef2c2243ba 100644 --- a/apps/sim/lib/uploads/utils/user-file-base64.server.ts +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.ts @@ -23,10 +23,7 @@ import { getExecutionRedisBudgetKeys, getExecutionRedisBudgetLimits, } from '@/lib/execution/redis-budget.server' -import { - ExecutionResourceLimitError, - isExecutionResourceLimitError, -} from '@/lib/execution/resource-errors' +import { ExecutionResourceLimitError } from '@/lib/execution/resource-errors' import { isGeneratedDocumentSourceType } from '@/lib/uploads/utils/file-utils' import type { UserFile } from '@/executor/types' @@ -196,6 +193,24 @@ class InMemoryBase64Cache implements Base64Cache { } } +/** + * The base64 cache only saves a repeat read from storage — the bytes it would have stored are + * already in hand by the time it is written. Exceeding a Redis budget therefore means "do not + * cache", never "fail the run": throwing here turned an oversized attachment into an opaque + * "Execution memory limit exceeded" on a request that had already read the file successfully. + */ +function logSkippedCacheWrite( + logger: Logger, + requestId: string | undefined, + file: UserFile, + error: ExecutionResourceLimitError +): void { + logger.warn( + `[${requestId ?? 'unknown'}] Skipping base64 cache write for ${file.name}: ${error.message}`, + { resource: error.resource, attemptedBytes: error.attemptedBytes } + ) +} + function createBase64Cache(options: Base64HydrationOptions, logger: Logger): Base64Cache { const redis = getRedisClient() const { executionId } = options @@ -228,11 +243,17 @@ function createBase64Cache(options: Base64HydrationOptions, logger: Logger): Bas const limits = getExecutionRedisBudgetLimits() if (valueBytes > limits.maxSingleWriteBytes) { - throw new ExecutionResourceLimitError({ - resource: 'redis_key_bytes', - attemptedBytes: valueBytes, - limitBytes: limits.maxSingleWriteBytes, - }) + logSkippedCacheWrite( + logger, + options.requestId, + file, + new ExecutionResourceLimitError({ + resource: 'redis_key_bytes', + attemptedBytes: valueBytes, + limitBytes: limits.maxSingleWriteBytes, + }) + ) + return } const cacheTtlSeconds = Math.max(ttlSeconds, limits.ttlSeconds) const budgetReservation: ExecutionRedisBudgetReservation = { @@ -261,19 +282,21 @@ function createBase64Cache(options: Base64HydrationOptions, logger: Logger): Bas )) as [number, string, number | string | null] const [allowed, resource, current] = result if (allowed !== 1) { - throw new ExecutionResourceLimitError({ - resource: - resource === 'user_redis_bytes' ? 'user_redis_bytes' : 'execution_redis_bytes', - attemptedBytes: valueBytes, - currentBytes: Number(current ?? 0), - limitBytes: - resource === 'user_redis_bytes' ? limits.maxUserBytes : limits.maxExecutionBytes, - }) + logSkippedCacheWrite( + logger, + options.requestId, + file, + new ExecutionResourceLimitError({ + resource: + resource === 'user_redis_bytes' ? 'user_redis_bytes' : 'execution_redis_bytes', + attemptedBytes: valueBytes, + currentBytes: Number(current ?? 0), + limitBytes: + resource === 'user_redis_bytes' ? limits.maxUserBytes : limits.maxExecutionBytes, + }) + ) } } catch (error) { - if (isExecutionResourceLimitError(error)) { - throw error - } logger.warn(`[${options.requestId}] Redis set failed, skipping cache`, error) } }, diff --git a/apps/sim/package.json b/apps/sim/package.json index 1f1bbb5e812..9e518dc2c33 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -37,7 +37,7 @@ "dependencies": { "@1password/sdk": "0.3.1", "@a2a-js/sdk": "1.0.0-alpha.0", - "@anthropic-ai/sdk": "0.114.0", + "@anthropic-ai/sdk": "0.115.0", "@aws-sdk/client-appconfig": "3.1032.0", "@aws-sdk/client-appconfigdata": "3.1032.0", "@aws-sdk/client-athena": "3.1032.0", @@ -73,7 +73,7 @@ "@earendil-works/pi-coding-agent": "0.80.10", "@floating-ui/dom": "1.7.6", "@google-cloud/storage": "7.21.0", - "@google/genai": "1.34.0", + "@google/genai": "2.13.0", "@hookform/resolvers": "5.2.2", "@linear/sdk": "40.0.0", "@marsidev/react-turnstile": "1.4.2", @@ -200,7 +200,7 @@ "nodemailer": "9.0.1", "nuqs": "2.8.9", "officeparser": "^5.2.0", - "openai": "^4.91.1", + "openai": "7.0.0", "pdf-lib": "1.17.1", "pdfjs-dist": "5.4.296", "postgres": "^3.4.5", diff --git a/apps/sim/providers/anthropic/streaming-tool-loop.test.ts b/apps/sim/providers/anthropic/streaming-tool-loop.test.ts index c0f730af6c1..558402119c5 100644 --- a/apps/sim/providers/anthropic/streaming-tool-loop.test.ts +++ b/apps/sim/providers/anthropic/streaming-tool-loop.test.ts @@ -24,6 +24,11 @@ vi.mock('@/tools', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, prepareToolExecution: mockPrepareToolExecution, calculateCost: () => ({ input: 0.01, output: 0.02, total: 0.03 }), sumToolCosts: () => 0, diff --git a/apps/sim/providers/attachments.test.ts b/apps/sim/providers/attachments.test.ts index 6ac38dc2b76..b9da7f426d2 100644 --- a/apps/sim/providers/attachments.test.ts +++ b/apps/sim/providers/attachments.test.ts @@ -2,6 +2,7 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' +import { LARGE_VALUE_THRESHOLD_BYTES } from '@/lib/execution/payloads/large-value-ref' import type { UserFile } from '@/executor/types' import { buildAnthropicMessageContent, @@ -10,11 +11,13 @@ import { buildOpenAICompatibleChatContent, buildOpenAIMessageContent, buildOpenRouterMessageContent, + formatAttachmentSizes, formatMessagesForProvider, getProviderAttachmentMaxBytes, getProviderFileStrategy, INLINE_ATTACHMENT_THRESHOLD_BYTES, inferAttachmentMimeType, + LARGE_FILE_PATH_THRESHOLD_BYTES, prepareProviderAttachments, shouldUseLargeFilePath, } from '@/providers/attachments' @@ -285,7 +288,88 @@ describe('provider attachments', () => { }) }) +describe('attachment limit formatting', () => { + /** + * Guards both directions of the unit bug: dividing every ceiling by 1024² reported OpenAI's + * decimal 50 MB as "48MB", and dividing every ceiling by 10⁶ reported Anthropic's 50 MiB as + * "52MB". Each vendor's number must come back as that vendor publishes it. + */ + it('reports each ceiling in the unit its vendor publishes', () => { + expect(formatAttachmentSizes(0, 50_000_000).limit).toBe('50') + expect(formatAttachmentSizes(0, 50 * 1024 * 1024).limit).toBe('50') + expect(formatAttachmentSizes(0, 20 * 1024 * 1024).limit).toBe('20') + expect(formatAttachmentSizes(0, 25 * 1024 * 1024).limit).toBe('25') + expect(formatAttachmentSizes(0, 10 * 1024 * 1024).limit).toBe('10') + }) + + /** + * Exercises `limit + 1` for every ceiling in the registry, which is the only input that can + * expose this: rounding both figures to the nearest hundredth rendered a file one byte over a + * 20 MiB cap as "20.00MB exceeds the 20MB limit". The previous version of this test asserted + * a file 0.03MB over and an openai file *under* the limit, so it passed while that was live. + */ + it('never renders an over-limit file as equal to the limit', () => { + const ceilings = [ + 50 * 1024 * 1024, + 25 * 1024 * 1024, + 20 * 1024 * 1024, + 10 * 1024 * 1024, + 6 * 1024 * 1024, + 50_000_000, + ] + for (const limit of ceilings) { + const justOver = formatAttachmentSizes(limit + 1, limit) + expect(justOver.size).not.toBe(justOver.limit) + } + + /** + * Every ceiling above divides to an exact integer, so floor/round/ceil are indistinguishable + * on them — only a ceiling with a fractional remainder pins the limit-side rounding. + */ + const fractional = formatAttachmentSizes(12_345_679, 12_345_678) + expect(fractional.size).not.toBe(fractional.limit) + }) + + it('keeps a comfortably over-limit size readable', () => { + const groq = formatAttachmentSizes(21_000_000, 20 * 1024 * 1024) + expect(groq.limit).toBe('20') + expect(groq.size).toBe('20.03') + }) +}) + describe('provider large-file capability', () => { + /** + * Guards the regression where every 6-10 MB attachment died with "Execution memory limit + * exceeded": past this size the base64 copy no longer fits the payload store, so an upload + * has to take over wherever one is reachable. + */ + it('starts preferring an upload before base64 outgrows the payload store', () => { + const encodedBytes = Math.ceil(LARGE_FILE_PATH_THRESHOLD_BYTES / 3) * 4 + expect(encodedBytes).toBeLessThanOrEqual(LARGE_VALUE_THRESHOLD_BYTES) + expect(LARGE_FILE_PATH_THRESHOLD_BYTES).toBeLessThan(INLINE_ATTACHMENT_THRESHOLD_BYTES) + }) + + /** + * A `remote-url` provider only fetches images and PDFs, so it must not take over from base64 + * early — text documents in the 6-10 MB band inline fine today and would start failing. + */ + /** A size we cannot read must still reach the uploader, which enforces the ceiling itself. */ + it('routes an unknown-size file to a files-api upload rather than stranding it', () => { + const unknown = { size: 0, type: 'text/csv' } + expect(shouldUseLargeFilePath(unknown, 'openai')).toBe(true) + expect(shouldUseLargeFilePath(unknown, 'anthropic')).toBe(false) + expect(shouldUseLargeFilePath({ size: Number.NaN, type: 'text/csv' }, 'openai')).toBe(true) + }) + + it('crosses over to an upload at different sizes for files-api and remote-url', () => { + const midBand = { size: LARGE_FILE_PATH_THRESHOLD_BYTES + 1, type: 'text/plain' } + expect(shouldUseLargeFilePath(midBand, 'openai')).toBe(true) + expect(shouldUseLargeFilePath(midBand, 'anthropic')).toBe(false) + + const aboveInline = { size: INLINE_ATTACHMENT_THRESHOLD_BYTES + 1, type: 'application/pdf' } + expect(shouldUseLargeFilePath(aboveInline, 'anthropic')).toBe(true) + }) + it('reports per-provider strategy and ceiling, defaulting others to inline', () => { expect(getProviderFileStrategy('openai')).toBe('files-api') expect(getProviderFileStrategy('google')).toBe('files-api') @@ -304,7 +388,7 @@ describe('provider large-file capability', () => { it('routes only oversized files on capable providers to the large-file path', () => { const small = { ...imageFile, size: 1024 } - const large = { ...imageFile, size: INLINE_ATTACHMENT_THRESHOLD_BYTES + 1 } + const large = { ...imageFile, size: LARGE_FILE_PATH_THRESHOLD_BYTES + 1 } expect(shouldUseLargeFilePath(small, 'openai')).toBe(false) expect(shouldUseLargeFilePath(large, 'openai')).toBe(true) expect(shouldUseLargeFilePath(large, 'bedrock')).toBe(false) @@ -313,7 +397,7 @@ describe('provider large-file capability', () => { it('does not expose generated source through a remote-url large-file path', () => { const generated = { ...pdfFile, - size: INLINE_ATTACHMENT_THRESHOLD_BYTES + 1, + size: LARGE_FILE_PATH_THRESHOLD_BYTES + 1, type: 'text/x-python-pdf', } expect(shouldUseLargeFilePath(generated, 'openai')).toBe(true) diff --git a/apps/sim/providers/attachments.ts b/apps/sim/providers/attachments.ts index 0cc4ebc5d6b..74b1437c676 100644 --- a/apps/sim/providers/attachments.ts +++ b/apps/sim/providers/attachments.ts @@ -15,6 +15,7 @@ import type { UserFile } from '@/executor/types' import { getProviderFileAttachment, INLINE_ATTACHMENT_MAX_BYTES, + LARGE_FILE_PATH_THRESHOLD_BYTES, type ProviderFileAttachmentStrategy, } from '@/providers/models' import type { ProviderId } from '@/providers/types' @@ -75,13 +76,12 @@ type ProviderFormattedMessage = { [key: string]: unknown } -/** - * Files at or below this size are inlined as base64, exactly as before. Larger files take - * the provider's large-file path. Keeping the threshold at the legacy 10 MB cap guarantees - * identical behaviour for existing attachments. - */ +/** Largest file that can be carried as inline base64 when no upload path is available. */ export const INLINE_ATTACHMENT_THRESHOLD_BYTES = INLINE_ATTACHMENT_MAX_BYTES +/** Re-exported so callers choosing a hydration cap do not reach into `models.ts` directly. */ +export { LARGE_FILE_PATH_THRESHOLD_BYTES } + export type ProviderFileStrategy = ProviderFileAttachmentStrategy /** Large-file delivery strategy for a provider, sourced from its `models.ts` definition. */ @@ -90,9 +90,17 @@ export function getProviderFileStrategy(providerId: ProviderId | string): Provid } /** - * True when an oversized file has a safe provider path. Remote URLs point at the - * primary storage object, so source-backed documents can only use artifact-aware - * Files API uploads. + * True when a file should be delivered through the provider's large-file path rather than as + * inline base64. + * + * The two strategies cross over at different sizes on purpose. `files-api` carries every type + * this provider already accepts, so it takes over as soon as base64 stops being cacheable. A + * `remote-url` provider only fetches images and PDFs, so switching early would start rejecting + * text documents that inline fine today; it therefore only takes over once inlining is no longer + * possible at all. + * + * Remote URLs point at the primary storage object, so source-backed generated documents can only + * use artifact-aware Files API uploads. */ export function shouldUseLargeFilePath( file: Pick, @@ -101,7 +109,16 @@ export function shouldUseLargeFilePath( const strategy = getProviderFileAttachment(providerId).strategy if (strategy === 'inline') return false if (strategy === 'remote-url' && isGeneratedDocumentSourceType(file.type)) return false - return Number.isFinite(file.size) && file.size > INLINE_ATTACHMENT_THRESHOLD_BYTES + const threshold = + strategy === 'files-api' ? LARGE_FILE_PATH_THRESHOLD_BYTES : INLINE_ATTACHMENT_THRESHOLD_BYTES + /** + * A file whose declared size is missing or zero cannot be routed by size. `files-api` uploads + * read the real bytes from storage and enforce the ceiling there, so routing one is always + * safe — and refusing to would strand it with neither base64 (hydration bails on the real + * length) nor a handle. + */ + if (!Number.isFinite(file.size) || file.size <= 0) return strategy === 'files-api' + return file.size > threshold } const PDF_MIME_TYPE = 'application/pdf' @@ -195,13 +212,44 @@ export function supportsFileAttachments(providerId: ProviderId | string): boolea /** * Real maximum attachment size for a provider — its native ceiling when it has a large-file - * path, else the inline base64 threshold. Used for UI limits and validation, never as the - * base64 hydration cap (which stays at {@link INLINE_ATTACHMENT_THRESHOLD_BYTES}). + * path, else the inline base64 threshold. Used for UI limits and validation. It is not the + * base64 hydration cap: that is chosen per request, because it depends on whether an upload + * path is actually reachable — see the agent handler's `inlineMaxBytes`. */ export function getProviderAttachmentMaxBytes(providerId: ProviderId | string): number { return getProviderFileAttachment(providerId).maxBytes } +const MEBIBYTE = 1024 * 1024 + +/** + * Renders a size and the ceiling it violated, both in one unit derived from the ceiling. + * + * Ceilings are authored in whichever unit the vendor publishes — decimal MB for OpenAI, binary + * MiB for everyone else — so a single fixed divisor is wrong for one group or the other: 1024² + * reports OpenAI's 50 MB as "48MB", and 10⁶ reports Anthropic's 50 MiB as "52MB". Either way the + * user is told a limit that does not exist. Taking the unit from the ceiling keeps the number + * they see equal to the number the vendor documents, and keeps both figures in the same sentence + * directly comparable. + */ +export function formatAttachmentSizes( + bytes: number, + limitBytes: number +): { size: string; limit: string } { + const divisor = limitBytes % MEBIBYTE === 0 ? MEBIBYTE : 1_000_000 + /** + * The size rounds up and the ceiling rounds down, so an over-limit file can never render as + * the same number as the limit it broke. Rounding both to the nearest hundredth instead let a + * file one byte over a 20 MiB cap print as "20.00MB exceeds the 20MB limit" — a sentence that + * tells the user to shrink to a size they are already under. + */ + const render = (value: number, round: (n: number) => number) => { + const scaled = round((value / divisor) * 100) / 100 + return Number.isInteger(scaled) ? String(scaled) : scaled.toFixed(2) + } + return { size: render(bytes, Math.ceil), limit: render(limitBytes, Math.floor) } +} + export function inferAttachmentMimeType(file: UserFile): string { const explicitType = file.type?.trim().toLowerCase() return resolveFileType({ @@ -389,8 +437,7 @@ export function prepareProviderAttachments( const maxBytes = getProviderAttachmentMaxBytes(providerId) if (Number.isFinite(file.size) && file.size > maxBytes) { - const sizeMB = (file.size / (1024 * 1024)).toFixed(2) - const maxMB = (maxBytes / (1024 * 1024)).toFixed(0) + const { size: sizeMB, limit: maxMB } = formatAttachmentSizes(file.size, maxBytes) throw new Error( `File "${file.name}" (${sizeMB}MB) exceeds the ${maxMB}MB agent attachment limit for provider "${providerId}"` ) diff --git a/apps/sim/providers/azure-openai/index.test.ts b/apps/sim/providers/azure-openai/index.test.ts index 44310504746..f1da6cec347 100644 --- a/apps/sim/providers/azure-openai/index.test.ts +++ b/apps/sim/providers/azure-openai/index.test.ts @@ -76,6 +76,11 @@ vi.mock('@/providers/trace-enrichment', () => ({ enrichLastModelSegmentFromChatCompletions: vi.fn(), })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn(() => ({ input: 0, output: 0, total: 0 })), prepareToolExecution: vi.fn((_tool, args) => ({ toolParams: args, executionParams: args })), prepareToolsWithUsageControl: mockPrepareTools, diff --git a/apps/sim/providers/azure-openai/index.ts b/apps/sim/providers/azure-openai/index.ts index bd979cc3d9f..9ecb86396a3 100644 --- a/apps/sim/providers/azure-openai/index.ts +++ b/apps/sim/providers/azure-openai/index.ts @@ -5,6 +5,7 @@ import { AzureOpenAI } from 'openai' import type { ChatCompletion, ChatCompletionContentPart, + ChatCompletionCreateParams, ChatCompletionCreateParamsBase, ChatCompletionCreateParamsStreaming, ChatCompletionMessageParam, @@ -44,11 +45,15 @@ import type { import { ProviderError } from '@/providers/types' import { calculateCost, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, } from '@/providers/utils' +/** `verbosity` narrowed from `string` to a literal union in openai v5. */ +type ChatCompletionVerbosity = NonNullable + const logger = createLogger('AzureOpenAIProvider') /** @@ -138,7 +143,7 @@ async function executeChatCompletionsRequest( if (request.reasoningEffort !== undefined && request.reasoningEffort !== 'auto') payload.reasoning_effort = request.reasoningEffort as ReasoningEffort if (request.verbosity !== undefined && request.verbosity !== 'auto') - payload.verbosity = request.verbosity + payload.verbosity = request.verbosity as ChatCompletionVerbosity if (request.responseFormat) { payload.response_format = { @@ -270,7 +275,7 @@ async function executeChatCompletionsRequest( enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'azure_openai' } ) @@ -289,7 +294,8 @@ async function executeChatCompletionsRequest( content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) if (!toolCallsInResponse || toolCallsInResponse.length === 0) { break } @@ -474,7 +480,7 @@ async function executeChatCompletionsRequest( enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'azure_openai' } ) @@ -495,7 +501,7 @@ async function executeChatCompletionsRequest( if ( iterationCount === MAX_TOOL_ITERATIONS && - currentResponse.choices[0]?.message?.tool_calls?.length + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall)?.length ) { /** * The capped turn still requests tools, so make one tool-disabled call to @@ -531,7 +537,7 @@ async function executeChatCompletionsRequest( enrichLastModelSegmentFromChatCompletions( timeSegments, synthesisResponse, - synthesisResponse.choices[0]?.message?.tool_calls, + synthesisResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'azure_openai' } ) } diff --git a/apps/sim/providers/baseten/index.test.ts b/apps/sim/providers/baseten/index.test.ts index d9e450ece2f..d0a4ed0308c 100644 --- a/apps/sim/providers/baseten/index.test.ts +++ b/apps/sim/providers/baseten/index.test.ts @@ -52,6 +52,11 @@ vi.mock('@/providers/trace-enrichment', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn().mockReturnValue({ input: 0, output: 0, total: 0 }), generateSchemaInstructions: vi.fn(() => 'SCHEMA_INSTRUCTIONS'), prepareToolExecution: vi.fn(() => ({ toolParams: { x: 1 }, executionParams: { x: 1 } })), diff --git a/apps/sim/providers/baseten/index.ts b/apps/sim/providers/baseten/index.ts index d33d4d7ec8a..e8893ad02d9 100644 --- a/apps/sim/providers/baseten/index.ts +++ b/apps/sim/providers/baseten/index.ts @@ -31,6 +31,7 @@ import { ProviderError } from '@/providers/types' import { calculateCost, generateSchemaInstructions, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -247,7 +248,8 @@ export const basetenProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -436,7 +438,8 @@ export const basetenProvider: ProviderConfig = { } if (iterationCount === MAX_TOOL_ITERATIONS) { - const pendingToolCalls = currentResponse.choices[0]?.message?.tool_calls + const pendingToolCalls = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions(timeSegments, currentResponse, pendingToolCalls, { model: request.model, provider: 'baseten', @@ -487,7 +490,7 @@ export const basetenProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, finalResponse, - finalResponse.choices[0]?.message?.tool_calls, + finalResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'baseten' } ) } @@ -541,7 +544,7 @@ export const basetenProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, finalResponse, - finalResponse.choices[0]?.message?.tool_calls, + finalResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'baseten' } ) } diff --git a/apps/sim/providers/bedrock/index.test.ts b/apps/sim/providers/bedrock/index.test.ts index 17d593d048c..bf4b2fdecdd 100644 --- a/apps/sim/providers/bedrock/index.test.ts +++ b/apps/sim/providers/bedrock/index.test.ts @@ -38,6 +38,11 @@ vi.mock('@/providers/models', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn().mockReturnValue({ input: 0, output: 0, total: 0, pricing: null }), prepareToolExecution: vi.fn((_tool, args) => ({ toolParams: args, diff --git a/apps/sim/providers/bedrock/streaming-tool-loop.test.ts b/apps/sim/providers/bedrock/streaming-tool-loop.test.ts index cf18f0e334c..ebb828e1dea 100644 --- a/apps/sim/providers/bedrock/streaming-tool-loop.test.ts +++ b/apps/sim/providers/bedrock/streaming-tool-loop.test.ts @@ -27,6 +27,11 @@ vi.mock('@/tools', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, prepareToolExecution: vi.fn(() => ({ toolParams: { url: 'https://example.com' }, executionParams: { url: 'https://example.com' }, diff --git a/apps/sim/providers/cerebras/index.ts b/apps/sim/providers/cerebras/index.ts index adbd329b118..0e56187af2b 100644 --- a/apps/sim/providers/cerebras/index.ts +++ b/apps/sim/providers/cerebras/index.ts @@ -24,6 +24,7 @@ import type { import { ProviderError } from '@/providers/types' import { calculateCost, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -197,7 +198,8 @@ export const cerebrasProvider: ProviderConfig = { const toolCallSignatures = new Set() try { while (iterationCount < MAX_TOOL_ITERATIONS) { - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -353,7 +355,7 @@ export const cerebrasProvider: ProviderConfig = { let usedForcedTools: string[] = [] if (typeof originalToolChoice === 'object' && forcedTools.length > 0) { const toolTracking = trackForcedToolUsage( - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), originalToolChoice, logger, 'openai', @@ -408,7 +410,7 @@ export const cerebrasProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'cerebras' } ) @@ -450,7 +452,8 @@ export const cerebrasProvider: ProviderConfig = { } } - const cappedToolCalls = currentResponse.choices[0]?.message?.tool_calls + const cappedToolCalls = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) if (iterationCount === MAX_TOOL_ITERATIONS && cappedToolCalls?.length) { enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -492,7 +495,7 @@ export const cerebrasProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'cerebras' } ) iterationCount++ diff --git a/apps/sim/providers/deepseek/index.test.ts b/apps/sim/providers/deepseek/index.test.ts index ff5cf829824..174123c7ffa 100644 --- a/apps/sim/providers/deepseek/index.test.ts +++ b/apps/sim/providers/deepseek/index.test.ts @@ -47,6 +47,11 @@ vi.mock('@/providers/trace-enrichment', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn(() => ({ input: 0, output: 0, total: 0 })), prepareToolExecution: vi.fn((_tool, args) => ({ toolParams: args, executionParams: args })), prepareToolsWithUsageControl: mockPrepareToolsWithUsageControl, diff --git a/apps/sim/providers/deepseek/index.ts b/apps/sim/providers/deepseek/index.ts index aaa27ffe123..9595656d68c 100644 --- a/apps/sim/providers/deepseek/index.ts +++ b/apps/sim/providers/deepseek/index.ts @@ -22,6 +22,7 @@ import type { import { ProviderError } from '@/providers/types' import { calculateCost, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, trackForcedToolUsage, @@ -305,11 +306,9 @@ export const deepseekProvider: ProviderConfig = { }, ] - if ( - typeof originalToolChoice === 'object' && - currentResponse.choices[0]?.message?.tool_calls - ) { - const toolCallsResponse = currentResponse.choices[0].message.tool_calls + const toolCallsResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) + if (typeof originalToolChoice === 'object' && toolCallsResponse?.length) { const result = trackForcedToolUsage( toolCallsResponse, originalToolChoice, @@ -328,7 +327,8 @@ export const deepseekProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -522,11 +522,9 @@ export const deepseekProvider: ProviderConfig = { request.abortSignal ? { signal: request.abortSignal } : undefined ) - if ( - typeof nextPayload.tool_choice === 'object' && - currentResponse.choices[0]?.message?.tool_calls - ) { - const toolCallsResponse = currentResponse.choices[0].message.tool_calls + const toolCallsResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) + if (typeof nextPayload.tool_choice === 'object' && toolCallsResponse?.length) { const result = trackForcedToolUsage( toolCallsResponse, nextPayload.tool_choice, @@ -571,7 +569,7 @@ export const deepseekProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'deepseek' } ) } diff --git a/apps/sim/providers/file-attachments.server.test.ts b/apps/sim/providers/file-attachments.server.test.ts new file mode 100644 index 00000000000..ca05e9692e5 --- /dev/null +++ b/apps/sim/providers/file-attachments.server.test.ts @@ -0,0 +1,169 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + buildOpenAIMessageContent, + INLINE_ATTACHMENT_THRESHOLD_BYTES, + LARGE_FILE_PATH_THRESHOLD_BYTES, +} from '@/providers/attachments' +import { + attachLargeFileRemoteUrls, + getInlineHydrationMaxBytes, + uploadLargeFilesToProvider, +} from '@/providers/file-attachments.server' +import type { ProviderRequest } from '@/providers/types' + +const { + mockDownloadServableFileFromStorage, + mockGeneratePresignedDownloadUrl, + mockHasCloudStorage, + mockVerifyFileAccess, +} = vi.hoisted(() => ({ + mockDownloadServableFileFromStorage: vi.fn(), + mockGeneratePresignedDownloadUrl: vi.fn(), + mockHasCloudStorage: vi.fn(), + mockVerifyFileAccess: vi.fn(), +})) + +vi.mock('@google/genai', () => ({ + FileState: { PROCESSING: 'PROCESSING', FAILED: 'FAILED' }, + GoogleGenAI: class {}, +})) + +vi.mock('@/lib/uploads', () => ({ + StorageService: { + hasCloudStorage: mockHasCloudStorage, + generatePresignedDownloadUrl: mockGeneratePresignedDownloadUrl, + }, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mockDownloadServableFileFromStorage, +})) + +vi.mock('@/app/api/files/authorization', () => ({ + verifyFileAccess: mockVerifyFileAccess, +})) + +/** The exact file from the reported failure: 9,591,617 bytes — over 6 MiB, under 50 MB. */ +const CSV_BYTES = 9_591_617 + +function makeRequest(size: number): ProviderRequest { + return { + model: 'gpt-4.1', + apiKey: 'sk-test', + userId: 'user-1', + workflowId: 'workflow-1', + messages: [ + { + role: 'user', + content: 'what does this say', + files: [ + { + id: 'file-1', + name: 'data_10mb.csv', + key: 'workspace/2f1d8c3e-5b6a-4c7d-8e9f-0a1b2c3d4e5f/data_10mb.csv', + url: '', + size, + type: 'text/csv', + context: 'workspace', + }, + ], + }, + ], + } as unknown as ProviderRequest +} + +describe('OpenAI large-file attachment lifecycle', () => { + beforeEach(() => { + vi.clearAllMocks() + mockHasCloudStorage.mockReturnValue(true) + mockVerifyFileAccess.mockResolvedValue(true) + mockGeneratePresignedDownloadUrl.mockResolvedValue('https://storage.example.com/signed') + mockDownloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.alloc(CSV_BYTES, 0x61), + contentType: 'text/csv', + }) + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(JSON.stringify({ id: 'file-abc' }), { status: 200 })) + ) + }) + + it('uploads to the Files API and references the file by id instead of inlining it', async () => { + const request = makeRequest(CSV_BYTES) + + await attachLargeFileRemoteUrls(request, 'openai') + await uploadLargeFilesToProvider(request, 'openai') + + const [url, init] = (fetch as unknown as ReturnType).mock.calls[0] + expect(url).toBe('https://api.openai.com/v1/files') + expect(init.method).toBe('POST') + expect(init.headers.Authorization).toBe('Bearer sk-test') + + const form = init.body as FormData + expect(form.get('purpose')).toBe('user_data') + expect(form.get('expires_after[anchor]')).toBe('created_at') + expect(form.get('expires_after[seconds]')).toBe('3600') + expect((form.get('file') as File).size).toBe(CSV_BYTES) + + const file = request.messages?.[0].files?.[0] + expect(file?.providerFileId).toBe('file-abc') + + const content = buildOpenAIMessageContent( + 'what does this say', + request.messages?.[0].files, + 'openai' + ) + expect(content).toEqual([ + { type: 'input_text', text: 'what does this say' }, + { type: 'input_file', file_id: 'file-abc' }, + ]) + }) + + /** Exactly at the crossover — `shouldUseLargeFilePath` uses `>`, so this must stay inline. */ + it('leaves a file exactly at the upload crossover on the base64 path', async () => { + const request = makeRequest(LARGE_FILE_PATH_THRESHOLD_BYTES) + + await attachLargeFileRemoteUrls(request, 'openai') + await uploadLargeFilesToProvider(request, 'openai') + + expect(fetch).not.toHaveBeenCalled() + expect(request.messages?.[0].files?.[0].providerFileId).toBeUndefined() + expect(request.messages?.[0].files?.[0].remoteUrl).toBeUndefined() + }) + + /** + * The hydration cap has to track `shouldUseLargeFilePath`'s crossover exactly. Stopping short + * of it leaves a band with neither base64 nor a handle — the defect this function was added to + * remove — and `remote-url` deliberately crosses over later than `files-api`. + */ + it('caps base64 hydration exactly where each strategy hands off to an upload', () => { + mockHasCloudStorage.mockReturnValue(true) + expect(getInlineHydrationMaxBytes('openai')).toBe(LARGE_FILE_PATH_THRESHOLD_BYTES) + expect(getInlineHydrationMaxBytes('anthropic')).toBe(INLINE_ATTACHMENT_THRESHOLD_BYTES) + expect(getInlineHydrationMaxBytes('bedrock')).toBe(INLINE_ATTACHMENT_THRESHOLD_BYTES) + + mockHasCloudStorage.mockReturnValue(false) + expect(getInlineHydrationMaxBytes('openai')).toBe(INLINE_ATTACHMENT_THRESHOLD_BYTES) + expect(getInlineHydrationMaxBytes('anthropic')).toBe(INLINE_ATTACHMENT_THRESHOLD_BYTES) + }) + + /** + * Local and disk-backed deployments have no cloud storage, so the upload path cannot read the + * bytes back. These files inline as base64 today and must keep doing so rather than hard-fail. + */ + it('leaves the file for the inline path when cloud storage is unavailable', async () => { + mockHasCloudStorage.mockReturnValue(false) + const request = makeRequest(CSV_BYTES) + + await attachLargeFileRemoteUrls(request, 'openai') + await uploadLargeFilesToProvider(request, 'openai') + + expect(fetch).not.toHaveBeenCalled() + const file = request.messages?.[0].files?.[0] + expect(file?.remoteUrl).toBeUndefined() + expect(file?.providerFileId).toBeUndefined() + }) +}) diff --git a/apps/sim/providers/file-attachments.server.ts b/apps/sim/providers/file-attachments.server.ts index 4ca03ef07aa..39fcbf9267e 100644 --- a/apps/sim/providers/file-attachments.server.ts +++ b/apps/sim/providers/file-attachments.server.ts @@ -8,9 +8,12 @@ import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils. import { verifyFileAccess } from '@/app/api/files/authorization' import type { UserFile } from '@/executor/types' import { + formatAttachmentSizes, getProviderAttachmentMaxBytes, getProviderFileStrategy, + INLINE_ATTACHMENT_THRESHOLD_BYTES, inferAttachmentMimeType, + LARGE_FILE_PATH_THRESHOLD_BYTES, shouldUseLargeFilePath, } from '@/providers/attachments' import type { Message, ProviderId, ProviderRequest } from '@/providers/types' @@ -32,12 +35,36 @@ function* iterateRequestFiles(messages: Message[] | undefined): Generator maxBytes) { - const sizeMB = (file.size / (1024 * 1024)).toFixed(2) - const maxMB = (maxBytes / (1024 * 1024)).toFixed(0) + const { size: sizeMB, limit: maxMB } = formatAttachmentSizes(file.size, maxBytes) throw new Error( `File "${file.name}" (${sizeMB}MB) exceeds the ${maxMB}MB agent attachment limit for provider "${providerId}"` ) @@ -71,11 +97,9 @@ export async function attachLargeFileRemoteUrls( if (!StorageService.hasCloudStorage()) { logger.warn( - `[${requestId}] "${file.name}" exceeds the inline limit for "${providerId}" but cloud storage is unavailable` - ) - throw new Error( - `File "${file.name}" exceeds the inline attachment limit and requires cloud file storage, which is not configured` + `[${requestId}] Sending "${file.name}" inline for "${providerId}": the large-file path needs cloud storage, which is not configured` ) + continue } if (!request.userId) { diff --git a/apps/sim/providers/fireworks/index.test.ts b/apps/sim/providers/fireworks/index.test.ts index 563117f4b27..9dcd6d9cfe5 100644 --- a/apps/sim/providers/fireworks/index.test.ts +++ b/apps/sim/providers/fireworks/index.test.ts @@ -55,6 +55,11 @@ vi.mock('@/providers/trace-enrichment', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn().mockReturnValue({ input: 0, output: 0, total: 0 }), generateSchemaInstructions: vi.fn(() => 'SCHEMA_INSTRUCTIONS'), prepareToolExecution: vi.fn(() => ({ toolParams: { x: 1 }, executionParams: { x: 1 } })), diff --git a/apps/sim/providers/fireworks/index.ts b/apps/sim/providers/fireworks/index.ts index 12c94ddf48c..8147bb0741f 100644 --- a/apps/sim/providers/fireworks/index.ts +++ b/apps/sim/providers/fireworks/index.ts @@ -32,6 +32,7 @@ import { ProviderError } from '@/providers/types' import { calculateCost, generateSchemaInstructions, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -253,7 +254,8 @@ export const fireworksProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -442,7 +444,8 @@ export const fireworksProvider: ProviderConfig = { } if (iterationCount === MAX_TOOL_ITERATIONS) { - const pendingToolCalls = currentResponse.choices[0]?.message?.tool_calls + const pendingToolCalls = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions(timeSegments, currentResponse, pendingToolCalls, { model: request.model, provider: 'fireworks', @@ -493,7 +496,7 @@ export const fireworksProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, finalResponse, - finalResponse.choices[0]?.message?.tool_calls, + finalResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'fireworks' } ) } @@ -547,7 +550,7 @@ export const fireworksProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, finalResponse, - finalResponse.choices[0]?.message?.tool_calls, + finalResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'fireworks' } ) } diff --git a/apps/sim/providers/gemini/core.ts b/apps/sim/providers/gemini/core.ts index 65cc21c5cb7..1a81099d521 100644 --- a/apps/sim/providers/gemini/core.ts +++ b/apps/sim/providers/gemini/core.ts @@ -476,18 +476,20 @@ function collapseMessagesToInput(request: ProviderRequest): { } /** - * Extracts text content from a completed interaction's outputs array. - * The outputs array can contain text, thought, google_search_result, and other types. - * We concatenate all text outputs to get the full research report. + * Extracts the report text from a completed interaction's step timeline. + * + * The v2 Interactions schema replaced the flat `outputs` array with `steps`, a + * type-discriminated timeline: the model's prose lives in `model_output` steps as text + * content, alongside thought, tool-call, and tool-result steps we deliberately skip. */ -function extractTextFromInteractionOutputs(outputs: Interactions.Interaction['outputs']): string { - if (!outputs || outputs.length === 0) return '' +function extractTextFromInteractionSteps(steps: Interactions.Interaction['steps']): string { + if (!steps || steps.length === 0) return '' const textParts: string[] = [] - for (const output of outputs) { - if (output.type === 'text') { - const text = (output as Interactions.TextContent).text - if (text) textParts.push(text) + for (const step of steps) { + if (step.type !== 'model_output') continue + for (const content of step.content ?? []) { + if (content.type === 'text' && content.text) textParts.push(content.text) } } @@ -506,10 +508,7 @@ interface DeepResearchUsage { /** * Extracts token usage from an Interaction's Usage object. * The Interactions API provides total_input_tokens, total_output_tokens, total_tokens, - * total_cached_tokens, and total_reasoning_tokens (for thinking models). - * - * Also handles the raw API field name total_thought_tokens which the SDK may - * map to total_reasoning_tokens. + * total_cached_tokens, and total_thought_tokens (for thinking models). * * The Interactions API supports implicit caching, and `total_cached_tokens` is a * subset of `total_input_tokens` there just as `cachedContentTokenCount` is of @@ -525,10 +524,7 @@ function extractInteractionUsage(usage: Interactions.Usage | undefined): DeepRes const inputTokens = usage.total_input_tokens ?? 0 const outputTokens = usage.total_output_tokens ?? 0 - const reasoningTokens = - usage.total_reasoning_tokens ?? - ((usage as Record).total_thought_tokens as number) ?? - 0 + const reasoningTokens = usage.total_thought_tokens ?? 0 const cachedTokens = usage.total_cached_tokens ?? 0 const totalTokens = usage.total_tokens ?? inputTokens + outputTokens @@ -586,13 +582,13 @@ function buildDeepResearchResponse( * Creates a ReadableStream from a deep research streaming interaction. * * Deep research streaming returns InteractionSSEEvent chunks including: - * - interaction.start: initial interaction with ID - * - content.delta: incremental text and thought_summary updates - * - content.start / content.stop: output boundaries - * - interaction.complete: final event (outputs is undefined in streaming; must reconstruct) + * - interaction.created: initial interaction with ID + * - step.delta: incremental text updates + * - step.start / step.stop: step boundaries + * - interaction.completed: final event (steps is undefined in streaming; must reconstruct) * - error: error events * - * We stream text deltas to the client and track usage from the interaction.complete event. + * We stream text deltas to the client and track usage from the interaction.completed event. */ function createDeepResearchStream( stream: AsyncIterable, @@ -613,20 +609,20 @@ function createDeepResearchStream( async start(controller) { try { for await (const event of stream) { - if (event.event_type === 'content.delta') { - const delta = (event as Interactions.ContentDelta).delta - if (delta?.type === 'text' && 'text' in delta && delta.text) { + if (event.event_type === 'step.delta') { + const { delta } = event + if (delta?.type === 'text' && delta.text) { fullContent += delta.text controller.enqueue(new TextEncoder().encode(delta.text)) } - } else if (event.event_type === 'interaction.complete') { - const interaction = (event as Interactions.InteractionEvent).interaction + } else if (event.event_type === 'interaction.completed') { + const { interaction } = event if (interaction?.usage) { completionUsage = extractInteractionUsage(interaction.usage) } completedInteractionId = interaction?.id - } else if (event.event_type === 'interaction.start') { - const interaction = (event as Interactions.InteractionEvent).interaction + } else if (event.event_type === 'interaction.created') { + const { interaction } = event if (interaction?.id) { completedInteractionId = interaction.id } @@ -722,10 +718,17 @@ export async function executeDeepResearchRequest( // Streaming mode: create a streaming interaction and return a StreamingExecution if (request.stream) { - const streamParams: Interactions.CreateAgentInteractionParamsStreaming = { + /** + * `satisfies`, not an annotation: as of @google/genai 2.13.0 the namespace alias resolves + * to `CreateAgentInteraction`, whose `stream` is a plain `boolean`, so annotating erases + * the literal that discriminates `interactions.create`'s overloads and the call falls + * through to the union-returning signature. `satisfies` keeps the literal while still + * rejecting a misspelled or unknown field. + */ + const streamParams = { ...baseParams, - stream: true, - } + stream: true as const, + } satisfies Interactions.CreateAgentInteractionParamsStreaming const streamResponse = await ai.interactions.create( streamParams, @@ -805,10 +808,11 @@ export async function executeDeepResearchRequest( } // Non-streaming mode: create and poll - const createParams: Interactions.CreateAgentInteractionParamsNonStreaming = { + /** `satisfies` for the same overload-discrimination reason as `streamParams` above. */ + const createParams = { ...baseParams, - stream: false, - } + stream: false as const, + } satisfies Interactions.CreateAgentInteractionParamsNonStreaming const interaction = await ai.interactions.create( createParams, @@ -835,6 +839,15 @@ export async function executeDeepResearchRequest( throw new Error(`Deep research interaction was cancelled: ${interactionId}`) } + /** + * Interactions v2 added terminal statuses beyond failed/cancelled. Without this they are + * polled until the hour-long ceiling and then reported as a Sim timeout, hiding a cause + * the caller can act on — `budget_exceeded` most of all. + */ + if (result.status === 'budget_exceeded' || result.status === 'incomplete') { + throw new Error(`Deep research interaction ended as "${result.status}": ${interactionId}`) + } + logger.info('Deep research in progress, polling...', { interactionId, status: result.status, @@ -855,7 +868,7 @@ export async function executeDeepResearchRequest( ) } - const content = extractTextFromInteractionOutputs(result.outputs) + const content = extractTextFromInteractionSteps(result.steps) const usage = extractInteractionUsage(result.usage) logger.info('Deep research completed', { diff --git a/apps/sim/providers/gemini/streaming-tool-loop.test.ts b/apps/sim/providers/gemini/streaming-tool-loop.test.ts index e9b4a020f18..3d8db0ecc6e 100644 --- a/apps/sim/providers/gemini/streaming-tool-loop.test.ts +++ b/apps/sim/providers/gemini/streaming-tool-loop.test.ts @@ -28,6 +28,11 @@ vi.mock('@/tools', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, prepareToolExecution: vi.fn(() => ({ toolParams: { url: 'https://httpbin.org/get' }, executionParams: { url: 'https://httpbin.org/get' }, diff --git a/apps/sim/providers/groq/index.test.ts b/apps/sim/providers/groq/index.test.ts index 7cf12b31de0..a10fdce4454 100644 --- a/apps/sim/providers/groq/index.test.ts +++ b/apps/sim/providers/groq/index.test.ts @@ -47,6 +47,11 @@ vi.mock('@/providers/trace-enrichment', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn(() => ({ input: 0, output: 0, total: 0 })), prepareToolExecution: vi.fn((_tool, args) => ({ toolParams: args, executionParams: args })), prepareToolsWithUsageControl: mockPrepareToolsWithUsageControl, diff --git a/apps/sim/providers/groq/index.ts b/apps/sim/providers/groq/index.ts index 06e5f431be5..04861fc4d00 100644 --- a/apps/sim/providers/groq/index.ts +++ b/apps/sim/providers/groq/index.ts @@ -27,6 +27,7 @@ import type { import { ProviderError } from '@/providers/types' import { calculateCost, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, trackForcedToolUsage, @@ -302,7 +303,8 @@ export const groqProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -450,7 +452,7 @@ export const groqProvider: ProviderConfig = { let usedForcedTools: string[] = [] if (typeof originalToolChoice === 'object' && forcedTools.length > 0) { const toolTracking = trackForcedToolUsage( - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), originalToolChoice, logger, 'openai', @@ -508,7 +510,7 @@ export const groqProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'groq' } ) } diff --git a/apps/sim/providers/kimi/index.ts b/apps/sim/providers/kimi/index.ts index ead35a5ddfa..9adadada5d4 100644 --- a/apps/sim/providers/kimi/index.ts +++ b/apps/sim/providers/kimi/index.ts @@ -29,6 +29,7 @@ import { ProviderError } from '@/providers/types' import { calculateCost, enforceStrictSchema, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -273,11 +274,9 @@ export const kimiProvider: ProviderConfig = { }, ] - if ( - typeof originalToolChoice === 'object' && - currentResponse.choices[0]?.message?.tool_calls - ) { - const toolCallsResponse = currentResponse.choices[0].message.tool_calls + const toolCallsResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) + if (typeof originalToolChoice === 'object' && toolCallsResponse?.length) { const result = trackForcedToolUsage( toolCallsResponse, originalToolChoice, @@ -296,7 +295,8 @@ export const kimiProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -463,11 +463,9 @@ export const kimiProvider: ProviderConfig = { request.abortSignal ? { signal: request.abortSignal } : undefined ) - if ( - typeof nextPayload.tool_choice === 'object' && - currentResponse.choices[0]?.message?.tool_calls - ) { - const toolCallsResponse = currentResponse.choices[0].message.tool_calls + const toolCallsResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) + if (typeof nextPayload.tool_choice === 'object' && toolCallsResponse?.length) { const result = trackForcedToolUsage( toolCallsResponse, nextPayload.tool_choice, @@ -507,7 +505,8 @@ export const kimiProvider: ProviderConfig = { } if (iterationCount === MAX_TOOL_ITERATIONS) { - const cappedToolCalls = currentResponse.choices[0]?.message?.tool_calls + const cappedToolCalls = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, @@ -552,7 +551,7 @@ export const kimiProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'kimi' } ) iterationCount++ diff --git a/apps/sim/providers/litellm/index.test.ts b/apps/sim/providers/litellm/index.test.ts index 50563bf21db..09525507656 100644 --- a/apps/sim/providers/litellm/index.test.ts +++ b/apps/sim/providers/litellm/index.test.ts @@ -55,6 +55,11 @@ vi.mock('@/providers/litellm/utils', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn(() => ({ input: 0, output: 0, total: 0 })), sumToolCosts: vi.fn(() => 0), prepareToolExecution: vi.fn((_tool, toolArgs) => ({ diff --git a/apps/sim/providers/litellm/index.ts b/apps/sim/providers/litellm/index.ts index a77e423b367..8000cad1c6e 100644 --- a/apps/sim/providers/litellm/index.ts +++ b/apps/sim/providers/litellm/index.ts @@ -27,6 +27,7 @@ import { ProviderError } from '@/providers/types' import { calculateCost, enforceStrictSchema, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -267,8 +268,11 @@ export const litellmProvider: ProviderConfig = { response: any, toolChoice: string | { type: string; function?: { name: string }; name?: string; any?: any } ) => { - if (typeof toolChoice === 'object' && response.choices[0]?.message?.tool_calls) { - const toolCallsResponse = response.choices[0].message.tool_calls + const toolCallsResponse = + typeof toolChoice === 'object' + ? response.choices?.[0]?.message?.tool_calls?.filter(isFunctionToolCall) + : undefined + if (toolCallsResponse?.length) { const result = trackForcedToolUsage( toolCallsResponse, toolChoice, @@ -329,7 +333,8 @@ export const litellmProvider: ProviderConfig = { } } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -533,7 +538,7 @@ export const litellmProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'litellm' } ) } @@ -582,12 +587,12 @@ export const litellmProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'litellm' } ) } else if ( iterationCount === MAX_TOOL_ITERATIONS && - currentResponse.choices[0]?.message?.tool_calls?.length + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall)?.length ) { /** * The capped turn still requests tools, so make one tool-disabled call @@ -623,7 +628,7 @@ export const litellmProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, synthesisResponse, - synthesisResponse.choices[0]?.message?.tool_calls, + synthesisResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'litellm' } ) } diff --git a/apps/sim/providers/meta/index.ts b/apps/sim/providers/meta/index.ts index fc76172a23e..971f1ede2c5 100644 --- a/apps/sim/providers/meta/index.ts +++ b/apps/sim/providers/meta/index.ts @@ -23,6 +23,7 @@ import type { import { ProviderError } from '@/providers/types' import { calculateCost, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -244,7 +245,8 @@ export const metaProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -425,7 +427,8 @@ export const metaProvider: ProviderConfig = { } if (iterationCount === MAX_TOOL_ITERATIONS) { - const cappedToolCalls = currentResponse.choices[0]?.message?.tool_calls + const cappedToolCalls = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, @@ -470,7 +473,7 @@ export const metaProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'meta' } ) iterationCount++ @@ -524,7 +527,7 @@ export const metaProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'meta' } ) } diff --git a/apps/sim/providers/mistral/index.test.ts b/apps/sim/providers/mistral/index.test.ts index ec5871ad47a..403759143bc 100644 --- a/apps/sim/providers/mistral/index.test.ts +++ b/apps/sim/providers/mistral/index.test.ts @@ -36,6 +36,11 @@ vi.mock('@/providers/trace-enrichment', () => ({ enrichLastModelSegmentFromChatCompletions: vi.fn(), })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn(() => ({ input: 0, output: 0, total: 0 })), prepareToolExecution: vi.fn((_tool, args) => ({ toolParams: args, executionParams: args })), prepareToolsWithUsageControl: vi.fn((tools) => ({ diff --git a/apps/sim/providers/mistral/index.ts b/apps/sim/providers/mistral/index.ts index 52453e72487..c16731da64c 100644 --- a/apps/sim/providers/mistral/index.ts +++ b/apps/sim/providers/mistral/index.ts @@ -23,6 +23,7 @@ import type { import { ProviderError } from '@/providers/types' import { calculateCost, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -202,8 +203,11 @@ export const mistralProvider: ProviderConfig = { response: any, toolChoice: string | { type: string; function?: { name: string }; name?: string; any?: any } ) => { - if (typeof toolChoice === 'object' && response.choices[0]?.message?.tool_calls) { - const toolCallsResponse = response.choices[0].message.tool_calls + const toolCallsResponse = + typeof toolChoice === 'object' + ? response.choices?.[0]?.message?.tool_calls?.filter(isFunctionToolCall) + : undefined + if (toolCallsResponse?.length) { const result = trackForcedToolUsage( toolCallsResponse, toolChoice, @@ -256,7 +260,8 @@ export const mistralProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -457,11 +462,11 @@ export const mistralProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'mistral' } ) - if (currentResponse.choices[0]?.message?.tool_calls?.length) { + if (currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall)?.length) { /** * The capped turn still requests tools, so make one tool-disabled call * to synthesize an answer from the tool results already gathered. @@ -496,7 +501,7 @@ export const mistralProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, synthesisResponse, - synthesisResponse.choices[0]?.message?.tool_calls, + synthesisResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'mistral' } ) } diff --git a/apps/sim/providers/models.ts b/apps/sim/providers/models.ts index 98be92cd78d..cb46b3ec0dc 100644 --- a/apps/sim/providers/models.ts +++ b/apps/sim/providers/models.ts @@ -33,6 +33,7 @@ import { xAIIcon, ZaiIcon, } from '@/components/icons' +import { LARGE_VALUE_THRESHOLD_BYTES } from '@/lib/execution/payloads/large-value-ref' import type { ModelPricing, ProviderId } from '@/providers/types' /** How a model's thinking appears on the agent-events stream. */ @@ -131,7 +132,7 @@ export interface ProviderDefinition { export type ProviderFileAttachmentStrategy = 'inline' | 'files-api' | 'remote-url' export interface ProviderFileAttachment { - /** Maximum attachment size the provider accepts, in bytes. */ + /** Maximum size of a single attachment the provider accepts, in bytes. */ maxBytes: number strategy: ProviderFileAttachmentStrategy } @@ -139,6 +140,18 @@ export interface ProviderFileAttachment { /** Inline base64 attachment cap, also the fallback limit for providers without a large-file path. */ export const INLINE_ATTACHMENT_MAX_BYTES = 10 * 1024 * 1024 +/** + * Size above which an attachment should prefer the provider's Files API over base64, when the + * deployment can reach one. + * + * Set by the execution payload store, not by any provider. Base64 inflates bytes by 4/3 and a + * single stored value may not exceed {@link LARGE_VALUE_THRESHOLD_BYTES}, so past three quarters + * of that ceiling the encoded copy no longer fits the cache. Inlining still succeeds above this + * point — the cache write is skipped, not fatal — but it carries a needlessly large encoded + * payload, so an upload is preferred wherever one is available. + */ +export const LARGE_FILE_PATH_THRESHOLD_BYTES = Math.floor(LARGE_VALUE_THRESHOLD_BYTES / 4) * 3 + const DEFAULT_FILE_ATTACHMENT: ProviderFileAttachment = { maxBytes: INLINE_ATTACHMENT_MAX_BYTES, strategy: 'inline', @@ -301,7 +314,8 @@ export const PROVIDER_DEFINITIONS: Record = { }, openai: { id: 'openai', - fileAttachment: { maxBytes: 50 * 1024 * 1024, strategy: 'files-api' }, + /** "each file must be under 50 MB" — decimal MB; OpenAI writes no MiB anywhere on that page. */ + fileAttachment: { maxBytes: 50_000_000, strategy: 'files-api' }, name: 'OpenAI', description: "OpenAI's models", defaultModel: 'gpt-4.1', diff --git a/apps/sim/providers/nvidia/index.ts b/apps/sim/providers/nvidia/index.ts index 885f6049e81..d9a6335026f 100644 --- a/apps/sim/providers/nvidia/index.ts +++ b/apps/sim/providers/nvidia/index.ts @@ -24,6 +24,7 @@ import type { import { ProviderError } from '@/providers/types' import { calculateCost, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -224,11 +225,9 @@ export const nvidiaProvider: ProviderConfig = { }, ] - if ( - typeof originalToolChoice === 'object' && - currentResponse.choices[0]?.message?.tool_calls - ) { - const toolCallsResponse = currentResponse.choices[0].message.tool_calls + const toolCallsResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) + if (typeof originalToolChoice === 'object' && toolCallsResponse?.length) { const result = trackForcedToolUsage( toolCallsResponse, originalToolChoice, @@ -247,7 +246,8 @@ export const nvidiaProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -414,11 +414,9 @@ export const nvidiaProvider: ProviderConfig = { request.abortSignal ? { signal: request.abortSignal } : undefined ) - if ( - typeof nextPayload.tool_choice === 'object' && - currentResponse.choices[0]?.message?.tool_calls - ) { - const toolCallsResponse = currentResponse.choices[0].message.tool_calls + const toolCallsResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) + if (typeof nextPayload.tool_choice === 'object' && toolCallsResponse?.length) { const result = trackForcedToolUsage( toolCallsResponse, nextPayload.tool_choice, @@ -458,7 +456,8 @@ export const nvidiaProvider: ProviderConfig = { } if (iterationCount === MAX_TOOL_ITERATIONS) { - const cappedToolCalls = currentResponse.choices[0]?.message?.tool_calls + const cappedToolCalls = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, @@ -507,7 +506,7 @@ export const nvidiaProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'nvidia' } ) iterationCount++ @@ -559,7 +558,7 @@ export const nvidiaProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'nvidia' } ) } diff --git a/apps/sim/providers/ollama-cloud/index.test.ts b/apps/sim/providers/ollama-cloud/index.test.ts index e199c2ebbf3..fd1d56b804b 100644 --- a/apps/sim/providers/ollama-cloud/index.test.ts +++ b/apps/sim/providers/ollama-cloud/index.test.ts @@ -72,6 +72,11 @@ vi.mock('@/providers/ollama-cloud/utils', () => ({ }, })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: () => ({ input: 0, output: 0, total: 0, pricing: null }), generateSchemaInstructions: () => 'SCHEMA_INSTRUCTIONS', prepareToolExecution: (_tool: unknown, args: Record) => ({ diff --git a/apps/sim/providers/ollama/core.ts b/apps/sim/providers/ollama/core.ts index 064caf79350..ff62ca8957f 100644 --- a/apps/sim/providers/ollama/core.ts +++ b/apps/sim/providers/ollama/core.ts @@ -23,6 +23,7 @@ import { ProviderError } from '@/providers/types' import { calculateCost, generateSchemaInstructions, + isFunctionToolCall, prepareToolExecution, sumToolCosts, } from '@/providers/utils' @@ -265,7 +266,8 @@ export async function executeOllamaProviderRequest( } } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -454,7 +456,7 @@ export async function executeOllamaProviderRequest( enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: providerId } ) } @@ -501,12 +503,12 @@ export async function executeOllamaProviderRequest( enrichLastModelSegmentFromChatCompletions( timeSegments, finalResponse, - finalResponse.choices[0]?.message?.tool_calls, + finalResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: providerId } ) } else if ( iterationCount === MAX_TOOL_ITERATIONS && - currentResponse.choices[0]?.message?.tool_calls?.length + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall)?.length ) { /** * The capped turn still requests tools, so make one tool-disabled call to @@ -542,7 +544,7 @@ export async function executeOllamaProviderRequest( enrichLastModelSegmentFromChatCompletions( timeSegments, synthesisResponse, - synthesisResponse.choices[0]?.message?.tool_calls, + synthesisResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: providerId } ) } diff --git a/apps/sim/providers/ollama/index.test.ts b/apps/sim/providers/ollama/index.test.ts index a0c358d7559..99465b457c8 100644 --- a/apps/sim/providers/ollama/index.test.ts +++ b/apps/sim/providers/ollama/index.test.ts @@ -59,6 +59,11 @@ vi.mock('@/providers/ollama/utils', () => ({ }, })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: () => ({ input: 0, output: 0, total: 0, pricing: null }), generateSchemaInstructions: () => 'SCHEMA_INSTRUCTIONS', prepareToolExecution: (_tool: unknown, args: Record) => ({ diff --git a/apps/sim/providers/openai-compat/streaming-tool-loop.test.ts b/apps/sim/providers/openai-compat/streaming-tool-loop.test.ts index b08b4050cba..651f968ab9c 100644 --- a/apps/sim/providers/openai-compat/streaming-tool-loop.test.ts +++ b/apps/sim/providers/openai-compat/streaming-tool-loop.test.ts @@ -24,6 +24,11 @@ vi.mock('@/tools', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, prepareToolExecution: mockPrepareToolExecution, calculateCost: () => ({ input: 0.01, output: 0.02, total: 0.03 }), sumToolCosts: () => 0, diff --git a/apps/sim/providers/openai/core.cache-key.test.ts b/apps/sim/providers/openai/core.cache-key.test.ts index c12bfa34928..8e0c9375a58 100644 --- a/apps/sim/providers/openai/core.cache-key.test.ts +++ b/apps/sim/providers/openai/core.cache-key.test.ts @@ -13,6 +13,11 @@ import type { ProviderRequest } from '@/providers/types' vi.mock('@/providers', () => ({ MAX_TOOL_ITERATIONS: 5 })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: () => ({ input: 0, output: 0, total: 0 }), sumToolCosts: () => 0, enforceStrictSchema: (schema: unknown) => schema, diff --git a/apps/sim/providers/openai/core.reasoning.test.ts b/apps/sim/providers/openai/core.reasoning.test.ts index a9b588951b0..6c32af673ae 100644 --- a/apps/sim/providers/openai/core.reasoning.test.ts +++ b/apps/sim/providers/openai/core.reasoning.test.ts @@ -15,6 +15,11 @@ import { executeTool } from '@/tools' vi.mock('@/providers', () => ({ MAX_TOOL_ITERATIONS: 5 })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: () => ({ input: 0, output: 0, total: 0 }), sumToolCosts: () => 0, enforceStrictSchema: (schema: unknown) => schema, diff --git a/apps/sim/providers/openai/streaming-tool-loop.ts b/apps/sim/providers/openai/streaming-tool-loop.ts index 1715891cc63..591f8bb3d37 100644 --- a/apps/sim/providers/openai/streaming-tool-loop.ts +++ b/apps/sim/providers/openai/streaming-tool-loop.ts @@ -11,6 +11,7 @@ import { createOpenAIUsageAccumulator, } from '@/providers/openai/usage' import { + convertResponseOutputToInputItems, extractResponseText, extractResponseToolCalls, isMaxOutputTokensIncompleteResponse, @@ -464,7 +465,7 @@ export function createOpenAIResponsesStreamingToolLoopStream( break } - currentInput.push(...turn.response.output) + currentInput.push(...convertResponseOutputToInputItems(turn.response.output)) if (typeof currentToolChoice === 'object') { for (const toolCall of executableTools) { diff --git a/apps/sim/providers/openai/utils.ts b/apps/sim/providers/openai/utils.ts index fa6e295f071..93d05f6b9c8 100644 --- a/apps/sim/providers/openai/utils.ts +++ b/apps/sim/providers/openai/utils.ts @@ -332,11 +332,20 @@ export function extractResponseReasoning(output: OpenAI.Responses.ResponseOutput /** * Converts Responses API output items into input items for subsequent calls. + * + * Echoing output items straight back as input is exactly what the Responses API asks for in a + * tool loop, but the SDK models `ResponseOutputItem` and `ResponseInputItem` as separate unions + * that diverge on members Sim never produces — computer-use call outputs (whose `status` admits + * `failed`, which the input shape rejects) and the `AdditionalTools` escape hatch. Narrowing + * member by member would have to be redone on every SDK bump, so the conversion is asserted + * once, here, and every caller goes through it rather than pushing raw output items. */ export function convertResponseOutputToInputItems( output: OpenAI.Responses.ResponseOutputItem[] ): ResponsesInputItem[] { - return Array.isArray(output) ? output : [] + if (!Array.isArray(output)) return [] + // double-cast-allowed: the SDK's output and input item unions diverge only on members Sim never emits + return output as unknown as ResponsesInputItem[] } /** diff --git a/apps/sim/providers/openrouter/index.test.ts b/apps/sim/providers/openrouter/index.test.ts index c611a5ee7f8..c51f2151af7 100644 --- a/apps/sim/providers/openrouter/index.test.ts +++ b/apps/sim/providers/openrouter/index.test.ts @@ -60,6 +60,11 @@ vi.mock('@/providers/trace-enrichment', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn(() => ({ input: 0, output: 0, total: 0 })), prepareToolsWithUsageControl: mockPrepareTools, prepareToolExecution: vi.fn((_tool: unknown, toolArgs: Record) => ({ diff --git a/apps/sim/providers/openrouter/index.ts b/apps/sim/providers/openrouter/index.ts index e2949fd9dfb..85f4ce7a602 100644 --- a/apps/sim/providers/openrouter/index.ts +++ b/apps/sim/providers/openrouter/index.ts @@ -38,6 +38,7 @@ import { ProviderError } from '@/providers/types' import { calculateCost, generateSchemaInstructions, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -262,7 +263,8 @@ export const openRouterProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -457,7 +459,8 @@ export const openRouterProvider: ProviderConfig = { } if (iterationCount === MAX_TOOL_ITERATIONS) { - const pendingToolCalls = currentResponse.choices[0]?.message?.tool_calls + const pendingToolCalls = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions(timeSegments, currentResponse, pendingToolCalls, { model: request.model, provider: 'openrouter', @@ -508,7 +511,7 @@ export const openRouterProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, finalResponse, - finalResponse.choices[0]?.message?.tool_calls, + finalResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'openrouter' } ) } @@ -562,7 +565,7 @@ export const openRouterProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, finalResponse, - finalResponse.choices[0]?.message?.tool_calls, + finalResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'openrouter' } ) } diff --git a/apps/sim/providers/sakana/index.ts b/apps/sim/providers/sakana/index.ts index d005e4d0fef..d7273c28401 100644 --- a/apps/sim/providers/sakana/index.ts +++ b/apps/sim/providers/sakana/index.ts @@ -23,6 +23,7 @@ import type { import { ProviderError } from '@/providers/types' import { calculateCost, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -221,11 +222,9 @@ export const sakanaProvider: ProviderConfig = { }, ] - if ( - typeof originalToolChoice === 'object' && - currentResponse.choices[0]?.message?.tool_calls - ) { - const toolCallsResponse = currentResponse.choices[0].message.tool_calls + const toolCallsResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) + if (typeof originalToolChoice === 'object' && toolCallsResponse?.length) { const result = trackForcedToolUsage( toolCallsResponse, originalToolChoice, @@ -244,7 +243,8 @@ export const sakanaProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -417,11 +417,9 @@ export const sakanaProvider: ProviderConfig = { request.abortSignal ? { signal: request.abortSignal } : undefined ) - if ( - typeof nextPayload.tool_choice === 'object' && - currentResponse.choices[0]?.message?.tool_calls - ) { - const toolCallsResponse = currentResponse.choices[0].message.tool_calls + const toolCallsResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) + if (typeof nextPayload.tool_choice === 'object' && toolCallsResponse?.length) { const result = trackForcedToolUsage( toolCallsResponse, nextPayload.tool_choice, @@ -461,7 +459,8 @@ export const sakanaProvider: ProviderConfig = { } if (iterationCount === MAX_TOOL_ITERATIONS) { - const cappedToolCalls = currentResponse.choices[0]?.message?.tool_calls + const cappedToolCalls = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, @@ -510,7 +509,7 @@ export const sakanaProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'sakana' } ) iterationCount++ @@ -564,7 +563,7 @@ export const sakanaProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'sakana' } ) } diff --git a/apps/sim/providers/settled-tool-streams.test.ts b/apps/sim/providers/settled-tool-streams.test.ts index 0cba631abe4..44f4d4622ab 100644 --- a/apps/sim/providers/settled-tool-streams.test.ts +++ b/apps/sim/providers/settled-tool-streams.test.ts @@ -69,6 +69,11 @@ vi.mock('@/providers/trace-enrichment', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn(() => ({ input: 1, output: 2, total: 3 })), enforceStrictSchema: vi.fn((schema) => schema), generateSchemaInstructions: vi.fn(() => 'SCHEMA_INSTRUCTIONS'), diff --git a/apps/sim/providers/together/index.test.ts b/apps/sim/providers/together/index.test.ts index 40624dad957..c9c75846cb9 100644 --- a/apps/sim/providers/together/index.test.ts +++ b/apps/sim/providers/together/index.test.ts @@ -52,6 +52,11 @@ vi.mock('@/providers/trace-enrichment', () => ({ })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn().mockReturnValue({ input: 0, output: 0, total: 0 }), generateSchemaInstructions: vi.fn(() => 'SCHEMA_INSTRUCTIONS'), prepareToolExecution: vi.fn(() => ({ toolParams: { x: 1 }, executionParams: { x: 1 } })), diff --git a/apps/sim/providers/together/index.ts b/apps/sim/providers/together/index.ts index 615a66f967e..5d791eb12a6 100644 --- a/apps/sim/providers/together/index.ts +++ b/apps/sim/providers/together/index.ts @@ -31,6 +31,7 @@ import { ProviderError } from '@/providers/types' import { calculateCost, generateSchemaInstructions, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -247,7 +248,8 @@ export const togetherProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -436,7 +438,8 @@ export const togetherProvider: ProviderConfig = { } if (iterationCount === MAX_TOOL_ITERATIONS) { - const pendingToolCalls = currentResponse.choices[0]?.message?.tool_calls + const pendingToolCalls = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions(timeSegments, currentResponse, pendingToolCalls, { model: request.model, provider: 'together', @@ -487,7 +490,7 @@ export const togetherProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, finalResponse, - finalResponse.choices[0]?.message?.tool_calls, + finalResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'together' } ) } @@ -541,7 +544,7 @@ export const togetherProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, finalResponse, - finalResponse.choices[0]?.message?.tool_calls, + finalResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'together' } ) } diff --git a/apps/sim/providers/trace-enrichment.ts b/apps/sim/providers/trace-enrichment.ts index caa82abf61e..342fd38b570 100644 --- a/apps/sim/providers/trace-enrichment.ts +++ b/apps/sim/providers/trace-enrichment.ts @@ -16,7 +16,9 @@ interface ChatCompletionLike { choices: Array<{ message?: { content?: string | null - tool_calls?: Array | null + /** Loose on purpose — the raw SDK response is passed here; only the separate + * `toolCallsInResponse` argument is required to be narrowed. */ + tool_calls?: Array<{ id: string; function?: { name: string; arguments: string } }> | null reasoning_content?: string | null reasoning?: string | null reasoning_details?: OpenRouterReasoningDetail[] | null @@ -34,6 +36,13 @@ interface ChatCompletionLike { } | null } +/** + * `function` stays required on purpose. The SDK's `ChatCompletionMessageToolCall` union gained a + * `custom` variant with no `function` in v5, and callers narrow that away with + * `isFunctionToolCall` before enriching. Making this optional to accept the raw union would let + * the custom shape satisfy this interface structurally, and every enrich call site would then + * type-check whether or not it narrowed — silently turning the guard into unenforced convention. + */ interface ChatCompletionToolCallLike { id: string function: { name: string; arguments: string } @@ -111,6 +120,12 @@ export function enrichLastModelSegment( * returns the raw string if it is not valid JSON. */ function parseToolCallArguments(rawArguments: string): Record | string { + /** + * `isFunctionToolCall` only proves `function` is present, not that it is well formed — a + * gateway can send `function: {}`. Without this the JSON parse below receives `undefined` and + * this returns it, breaking the declared return type. + */ + if (typeof rawArguments !== 'string') return '' try { const parsed = JSON.parse(rawArguments) if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { @@ -180,7 +195,7 @@ export function enrichLastModelSegmentFromChatCompletions( const toolCalls: IterationToolCall[] = (toolCallsInResponse ?? []).map((tc) => ({ id: tc.id, - name: tc.function.name, + name: tc.function.name ?? '', arguments: parseToolCallArguments(tc.function.arguments), })) diff --git a/apps/sim/providers/utils.ts b/apps/sim/providers/utils.ts index 12b7c0cc966..2ad2bb18cd1 100644 --- a/apps/sim/providers/utils.ts +++ b/apps/sim/providers/utils.ts @@ -1235,6 +1235,30 @@ export function prepareToolsWithUsageControl( } } +/** + * Narrows the SDK's `ChatCompletionMessageToolCall` union to its function variant. + * + * v5 of the `openai` SDK widened that union with a `custom` tool call carrying no `function` + * field, so every `.function` access needs narrowing first. Sim only ever declares function + * tools, so a custom call should not arrive. + * + * Deliberately tests for the `function` payload rather than `type === 'function'`: many + * OpenAI-compatible vendors omit `type` on tool calls entirely, and discriminating on it would + * silently drop every tool call those providers return. Total by construction, because these + * same gateways are the ones that emit a malformed `tool_calls` entry, and this now runs on + * every tool-bearing response. + */ +export function isFunctionToolCall( + toolCall: OpenAI.Chat.Completions.ChatCompletionMessageToolCall +): toolCall is OpenAI.Chat.Completions.ChatCompletionMessageFunctionToolCall { + return ( + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + toolCall.function != null + ) +} + /** * Checks if a forced tool has been used in a response and manages the tool_choice accordingly * @@ -1562,8 +1586,11 @@ export function checkForForcedToolUsageOpenAI( let hasUsedForcedTool = false let updatedUsedForcedTools = [...usedForcedTools] - if (typeof toolChoice === 'object' && response.choices[0]?.message?.tool_calls) { - const toolCallsResponse = response.choices[0].message.tool_calls + const toolCallsResponse = + typeof toolChoice === 'object' + ? response.choices?.[0]?.message?.tool_calls?.filter(isFunctionToolCall) + : undefined + if (toolCallsResponse?.length) { const result = trackForcedToolUsage( toolCallsResponse, toolChoice, diff --git a/apps/sim/providers/vllm/index.test.ts b/apps/sim/providers/vllm/index.test.ts index 342067d8022..85ac6e80f1c 100644 --- a/apps/sim/providers/vllm/index.test.ts +++ b/apps/sim/providers/vllm/index.test.ts @@ -60,6 +60,11 @@ vi.mock('@/providers/trace-enrichment', () => ({ enrichLastModelSegmentFromChatCompletions: vi.fn(), })) vi.mock('@/providers/utils', () => ({ + isFunctionToolCall: (toolCall: unknown) => + typeof toolCall === 'object' && + toolCall !== null && + 'function' in toolCall && + (toolCall as { function?: unknown }).function != null, calculateCost: vi.fn(() => ({ input: 0, output: 0, total: 0 })), prepareToolExecution: vi.fn((_tool, args) => ({ toolParams: args, executionParams: args })), prepareToolsWithUsageControl: mockPrepareTools, diff --git a/apps/sim/providers/vllm/index.ts b/apps/sim/providers/vllm/index.ts index 4d2928266f8..d704d31444f 100644 --- a/apps/sim/providers/vllm/index.ts +++ b/apps/sim/providers/vllm/index.ts @@ -27,6 +27,7 @@ import type { import { ProviderError } from '@/providers/types' import { calculateCost, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -339,7 +340,8 @@ export const vllmProvider: ProviderConfig = { } } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -551,11 +553,11 @@ export const vllmProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'vllm' } ) - if (currentResponse.choices[0]?.message?.tool_calls?.length) { + if (currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall)?.length) { /** * The capped turn still requests tools, so make one tool-disabled call * to synthesize an answer from the tool results already gathered. @@ -593,7 +595,7 @@ export const vllmProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, synthesisResponse, - synthesisResponse.choices[0]?.message?.tool_calls, + synthesisResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'vllm' } ) } diff --git a/apps/sim/providers/xai/index.ts b/apps/sim/providers/xai/index.ts index 292f6111c4a..ebe05b2dc67 100644 --- a/apps/sim/providers/xai/index.ts +++ b/apps/sim/providers/xai/index.ts @@ -24,6 +24,7 @@ import type { import { ProviderError } from '@/providers/types' import { calculateCost, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -230,7 +231,8 @@ export const xAIProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -468,7 +470,8 @@ export const xAIProvider: ProviderConfig = { } if (iterationCount === MAX_TOOL_ITERATIONS) { - const pendingToolCalls = currentResponse.choices[0]?.message?.tool_calls + const pendingToolCalls = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, @@ -519,7 +522,7 @@ export const xAIProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, finalResponse, - finalResponse.choices[0]?.message?.tool_calls, + finalResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'xai' } ) } diff --git a/apps/sim/providers/zai/index.ts b/apps/sim/providers/zai/index.ts index acc5bb4ec36..d9da39de22f 100644 --- a/apps/sim/providers/zai/index.ts +++ b/apps/sim/providers/zai/index.ts @@ -23,6 +23,7 @@ import type { import { ProviderError } from '@/providers/types' import { calculateCost, + isFunctionToolCall, prepareToolExecution, prepareToolsWithUsageControl, sumToolCosts, @@ -256,7 +257,8 @@ export const zaiProvider: ProviderConfig = { content = currentResponse.choices[0].message.content } - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + const toolCallsInResponse = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, @@ -431,7 +433,8 @@ export const zaiProvider: ProviderConfig = { } if (iterationCount === MAX_TOOL_ITERATIONS) { - const cappedToolCalls = currentResponse.choices[0]?.message?.tool_calls + const cappedToolCalls = + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall) enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, @@ -484,7 +487,7 @@ export const zaiProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'zai' } ) iterationCount++ @@ -539,7 +542,7 @@ export const zaiProvider: ProviderConfig = { enrichLastModelSegmentFromChatCompletions( timeSegments, currentResponse, - currentResponse.choices[0]?.message?.tool_calls, + currentResponse.choices[0]?.message?.tool_calls?.filter(isFunctionToolCall), { model: request.model, provider: 'zai' } ) } diff --git a/bun.lock b/bun.lock index 43993ccde48..0873de33ce0 100644 --- a/bun.lock +++ b/bun.lock @@ -144,7 +144,7 @@ "dependencies": { "@1password/sdk": "0.3.1", "@a2a-js/sdk": "1.0.0-alpha.0", - "@anthropic-ai/sdk": "0.114.0", + "@anthropic-ai/sdk": "0.115.0", "@aws-sdk/client-appconfig": "3.1032.0", "@aws-sdk/client-appconfigdata": "3.1032.0", "@aws-sdk/client-athena": "3.1032.0", @@ -180,7 +180,7 @@ "@earendil-works/pi-coding-agent": "0.80.10", "@floating-ui/dom": "1.7.6", "@google-cloud/storage": "7.21.0", - "@google/genai": "1.34.0", + "@google/genai": "2.13.0", "@hookform/resolvers": "5.2.2", "@linear/sdk": "40.0.0", "@marsidev/react-turnstile": "1.4.2", @@ -307,7 +307,7 @@ "nodemailer": "9.0.1", "nuqs": "2.8.9", "officeparser": "^5.2.0", - "openai": "^4.91.1", + "openai": "7.0.0", "pdf-lib": "1.17.1", "pdfjs-dist": "5.4.296", "postgres": "^3.4.5", @@ -760,7 +760,7 @@ "@antfu/install-pkg": ["@antfu/install-pkg@1.1.0", "", { "dependencies": { "package-manager-detector": "^1.3.0", "tinyexec": "^1.0.1" } }, "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ=="], - "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.114.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1", "standardwebhooks": "^1.0.0" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-zRFGTVMFEm77gt70Q0B+CDRKa9AcguydCcp6bD/dXWv8UkfsVFqCbaqU8+4B/pob3+Vy434LgtrBmwSvxfKr3g=="], + "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.115.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1", "standardwebhooks": "^1.0.0" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-BJrFIVyjNuU8lfDyIJTvlRYzgQg+zEl78BxE7fq8esULsGz9IRQvGtW5spq3tydmtjQb/GFdooKGdGsetpx+lQ=="], "@apidevtools/json-schema-ref-parser": ["@apidevtools/json-schema-ref-parser@11.9.3", "", { "dependencies": { "@jsdevtools/ono": "^7.1.3", "@types/json-schema": "^7.0.15", "js-yaml": "^4.1.0" } }, "sha512-60vepv88RwcJtSHrD6MjIL6Ta3SOYbgfnkHb+ppAVK+o9mXprRtulx7VlRl3lN3bbvysAfCS7WMVfhUYemB0IQ=="], @@ -1190,7 +1190,7 @@ "@google-cloud/storage": ["@google-cloud/storage@7.21.0", "", { "dependencies": { "@google-cloud/paginator": "^5.0.0", "@google-cloud/projectify": "^4.0.0", "@google-cloud/promisify": "<4.1.0", "abort-controller": "^3.0.0", "async-retry": "^1.3.3", "duplexify": "^4.1.3", "fast-xml-parser": "^5.3.4", "gaxios": "^6.0.2", "google-auth-library": "^9.6.3", "html-entities": "^2.5.2", "mime": "^3.0.0", "p-limit": "^3.0.1", "retry-request": "^7.0.0", "teeny-request": "^9.0.0" } }, "sha512-l+IFTkd+6Y5LoAuXyYCKNAKtw/Ci+rAMqgdTB1jv4iZiLhw0rtq+0qjIRbBizXkNzEFmXiXUW0H7sZQQvk1ffA=="], - "@google/genai": ["@google/genai@1.34.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.24.0" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-vu53UMPvjmb7PGzlYu6Tzxso8Dfhn+a7eQFaS2uNemVtDZKwzSpJ5+ikqBbXplF7RGB1STcVDqCkPvquiwb2sw=="], + "@google/genai": ["@google/genai@2.13.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-GM7C8Kaomvjz05x5JEO6+l3d/pciL9LxAG9dUjJLD7nTPZ9X0Cfsf2Z7eET6UjgWyUmxXCHtYnQoQ77F9+ZIOQ=="], "@graphql-typed-document-node/core": ["@graphql-typed-document-node/core@3.2.0", "", { "peerDependencies": { "graphql": "^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" } }, "sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ=="], @@ -3716,7 +3716,7 @@ "oniguruma-to-es": ["oniguruma-to-es@4.3.6", "", { "dependencies": { "oniguruma-parser": "^0.12.2", "regex": "^6.1.0", "regex-recursion": "^6.0.2" } }, "sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA=="], - "openai": ["openai@4.104.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" }, "peerDependencies": { "ws": "^8.18.0", "zod": "^3.23.8" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA=="], + "openai": ["openai@7.0.0", "", { "peerDependencies": { "@aws-sdk/credential-provider-node": ">=3.972.0 <4", "@smithy/hash-node": ">=4.3.0 <5", "@smithy/signature-v4": ">=5.4.0 <6", "ws": "^8.18.0", "zod": "^3.25 || ^4.0" }, "optionalPeers": ["@aws-sdk/credential-provider-node", "@smithy/hash-node", "@smithy/signature-v4", "ws", "zod"] }, "sha512-NaAHnDTmut8tl1Is/T2bmRGQm67fEb9VrxdlVradQoh6WnHNPToQeOZiVjLa7LV9uq0Ha4C5twgOsp4+0UWwYg=="], "openapi-fetch": ["openapi-fetch@0.14.1", "", { "dependencies": { "openapi-typescript-helpers": "^0.0.15" } }, "sha512-l7RarRHxlEZYjMLd/PR0slfMVse2/vvIAGm75/F7J6MlQ8/b9uUQmUF2kCPrQhJqMXSxmYWObVgeYXbFYzZR+A=="], @@ -4638,6 +4638,10 @@ "@browserbasehq/stagehand/@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.39.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg=="], + "@browserbasehq/stagehand/@google/genai": ["@google/genai@1.34.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.24.0" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-vu53UMPvjmb7PGzlYu6Tzxso8Dfhn+a7eQFaS2uNemVtDZKwzSpJ5+ikqBbXplF7RGB1STcVDqCkPvquiwb2sw=="], + + "@browserbasehq/stagehand/openai": ["openai@4.104.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" }, "peerDependencies": { "ws": "^8.18.0", "zod": "^3.23.8" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA=="], + "@cerebras/cerebras_cloud_sdk/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="], "@earendil-works/pi-ai/@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.91.1", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw=="], @@ -5186,8 +5190,6 @@ "nuqs/@standard-schema/spec": ["@standard-schema/spec@1.0.0", "", {}, "sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA=="], - "openai/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="], - "p-retry/retry": ["retry@0.13.1", "", {}, "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="], "parse-entities/@types/unist": ["@types/unist@2.0.11", "", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="], @@ -5334,6 +5336,8 @@ "@browserbasehq/stagehand/@anthropic-ai/sdk/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="], + "@browserbasehq/stagehand/openai/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="], + "@cerebras/cerebras_cloud_sdk/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], "@earendil-works/pi-ai/@aws-sdk/client-bedrock-runtime/@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1048.0", "", { "dependencies": { "@aws-sdk/core": "^3.974.11", "@aws-sdk/nested-clients": "^3.997.9", "@aws-sdk/types": "^3.973.8", "@smithy/core": "^3.24.2", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-k0y/GcuesuSfWyUM0WamrGyeZmltRYaPbHO82UDA6mZ/doB+FOHKutikPAtSXMn/hDz970cF+iRuuiYO9VEbAA=="], @@ -5662,8 +5666,6 @@ "node-gyp/which/isexe": ["isexe@4.0.0", "", {}, "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw=="], - "openai/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], - "posthog-js/@opentelemetry/exporter-logs-otlp-http/@opentelemetry/core": ["@opentelemetry/core@2.2.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw=="], "posthog-js/@opentelemetry/exporter-logs-otlp-http/@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.208.0", "", { "dependencies": { "@opentelemetry/core": "2.2.0", "@opentelemetry/otlp-transformer": "0.208.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-gMd39gIfVb2OgxldxUtOwGJYSH8P1kVFFlJLuut32L6KgUC4gl1dMhn+YC2mGn0bDOiQYSk/uHOdSjuKp58vvA=="], @@ -5706,6 +5708,8 @@ "@browserbasehq/stagehand/@anthropic-ai/sdk/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], + "@browserbasehq/stagehand/openai/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], + "@google-cloud/storage/google-auth-library/gcp-metadata/google-logging-utils": ["google-logging-utils@0.0.2", "", {}, "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ=="], "@trigger.dev/core/socket.io-client/engine.io-client/ws": ["ws@8.17.1", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ=="],